From 00ee0a294aaf6431aafb698ddefb28e5c0321095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Mon, 7 Jun 2021 00:18:59 +0800 Subject: [PATCH] support mysql invisible index --- .../druid/sql/ast/SQLIndexOptions.java | 9 + .../statement/SQLAlterTableAlterIndex.java | 227 ++++++++++++++++++ .../mysql/parser/MySqlStatementParser.java | 13 +- .../druid/sql/parser/SQLExprParser.java | 4 + .../sql/visitor/SQLASTOutputVisitor.java | 19 +- .../druid/sql/visitor/SQLASTVisitor.java | 8 + .../sql/mysql/visitor/MySqlResourceTest.java | 16 ++ src/test/resources/bvt/parser/mysql-28.txt | 13 + src/test/resources/bvt/parser/mysql-29.txt | 3 + src/test/resources/bvt/parser/mysql-30.txt | 4 + src/test/resources/bvt/parser/mysql-31.txt | 4 + src/test/resources/bvt/parser/odps-4.txt | 12 + src/test/resources/bvt/parser/odps-5.txt | 47 ++++ src/test/resources/bvt/parser/odps-6.txt | 1 + src/test/resources/bvt/parser/odps-7.txt | 118 +++++++++ 15 files changed, 494 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableAlterIndex.java create mode 100644 src/test/resources/bvt/parser/mysql-28.txt create mode 100644 src/test/resources/bvt/parser/mysql-29.txt create mode 100644 src/test/resources/bvt/parser/mysql-30.txt create mode 100644 src/test/resources/bvt/parser/mysql-31.txt create mode 100644 src/test/resources/bvt/parser/odps-4.txt create mode 100644 src/test/resources/bvt/parser/odps-5.txt create mode 100644 src/test/resources/bvt/parser/odps-6.txt create mode 100644 src/test/resources/bvt/parser/odps-7.txt diff --git a/src/main/java/com/alibaba/druid/sql/ast/SQLIndexOptions.java b/src/main/java/com/alibaba/druid/sql/ast/SQLIndexOptions.java index 0e843e941a..71363e84cf 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/SQLIndexOptions.java +++ b/src/main/java/com/alibaba/druid/sql/ast/SQLIndexOptions.java @@ -21,6 +21,7 @@ public class SQLIndexOptions extends SQLObjectImpl { private String algorithm; private String lock; private boolean invisible; + private boolean visible; private List otherOptions = new ArrayList(); public SQLIndexOptions() { @@ -133,6 +134,14 @@ public class SQLIndexOptions extends SQLObjectImpl { this.invisible = invisible; } + public boolean isVisible() { + return visible; + } + + public void setVisible(boolean visible) { + this.visible = visible; + } + public List getOtherOptions() { return otherOptions; } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableAlterIndex.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableAlterIndex.java new file mode 100644 index 0000000000..0d047d0b1b --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableAlterIndex.java @@ -0,0 +1,227 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.ast.statement; + +import com.alibaba.druid.sql.ast.*; +import com.alibaba.druid.sql.dialect.mysql.ast.MySqlKey; +import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlTableIndex; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; + +import java.util.List; + +public class SQLAlterTableAlterIndex extends SQLObjectImpl implements SQLAlterTableItem, SQLIndex { + + private SQLIndexDefinition indexDefinition = new SQLIndexDefinition(); + + public SQLAlterTableAlterIndex() { + indexDefinition.setParent(this); + } + + public SQLIndexDefinition getIndexDefinition() { + return indexDefinition; + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + if (visitor.visit(this)) { + if (indexDefinition.getName() != null) { + indexDefinition.getName().accept(visitor); + } + + for (int i = 0; i < getColumns().size(); i++) { + final SQLSelectOrderByItem item = getColumns().get(i); + if (item != null) { + item.accept(visitor); + } + } + for (int i = 0; i < getCovering().size(); i++) { + final SQLName item = getCovering().get(i); + if (item != null) { + item.accept(visitor); + } + } + } + visitor.endVisit(this); + } + + public boolean isUnique() { + return indexDefinition.getType() != null && indexDefinition.getType().equalsIgnoreCase("UNIQUE"); + } + + public void setUnique(boolean unique) { + indexDefinition.setType("UNIQUE"); + } + + public List getItems() { + return indexDefinition.getColumns(); + } + + public void addItem(SQLSelectOrderByItem item) { + if (item != null) { + item.setParent(this); + } + indexDefinition.getColumns().add(item); + } + + public SQLName getName() { + return indexDefinition.getName(); + } + + public void setName(SQLName name) { + indexDefinition.setName(name); + } + + public String getType() { + return indexDefinition.getType(); + } + + public void setType(String type) { + indexDefinition.setType(type); + } + + public String getUsing() { + if (indexDefinition.hasOptions()) { + return indexDefinition.getOptions().getIndexType(); + } + return null; + } + + public void setUsing(String using) { + indexDefinition.getOptions().setIndexType(using); + } + + public boolean isKey() { + return indexDefinition.isKey(); + } + + public void setKey(boolean key) { + indexDefinition.setKey(key); + } + + public void cloneTo(MySqlTableIndex x) { + indexDefinition.cloneTo(x.getIndexDefinition()); + } + + public void cloneTo(MySqlKey x) { + indexDefinition.cloneTo(x.getIndexDefinition()); + } + + public SQLExpr getComment() { + if (indexDefinition.hasOptions()) { + return indexDefinition.getOptions().getComment(); + } + return null; + } + + public void setComment(SQLExpr comment) { + if (comment != null) { + comment.setParent(this); + } + indexDefinition.getOptions().setComment(comment); + } + + public SQLExpr getKeyBlockSize() { + if (indexDefinition.hasOptions()) { + return indexDefinition.getOptions().getKeyBlockSize(); + } + return null; + } + + public void setKeyBlockSize(SQLExpr keyBlockSize) { + indexDefinition.getOptions().setKeyBlockSize(keyBlockSize); + } + + public String getParserName() { + if (indexDefinition.hasOptions()) { + return indexDefinition.getOptions().getParserName(); + } + return null; + } + + public void setParserName(String parserName) { + indexDefinition.getOptions().setParserName(parserName); + } + + public boolean isHashMapType() { + return indexDefinition.isHashMapType(); + } + + public void setHashMapType(boolean hashMapType) { + indexDefinition.setHashMapType(hashMapType); + } + + protected SQLExpr getOption(long hash64) { + return indexDefinition.getOption(hash64); + } + + public String getDistanceMeasure() { + return indexDefinition.getDistanceMeasure(); + } + + public String getAlgorithm() { + return indexDefinition.getAlgorithm(); + } + + public void addOption(String name, SQLExpr value) { + indexDefinition.addOption(name, value); + } + + public List getOptions() { + return indexDefinition.getCompatibleOptions(); + } + + public boolean isGlobal() { + return indexDefinition.isGlobal(); + } + + public void setGlobal(boolean global) { + indexDefinition.setGlobal(global); + } + + public SQLExpr getDbPartitionBy() { + return indexDefinition.getDbPartitionBy(); + } + + public void setDbPartitionBy(SQLExpr x) { + indexDefinition.setDbPartitionBy(x); + } + + public SQLExpr getTablePartitionBy() { + return indexDefinition.getTbPartitionBy(); + } + + public void setTablePartitionBy(SQLExpr x) { + indexDefinition.setTbPartitionBy(x); + } + + public SQLExpr getTablePartitions() { + return indexDefinition.getTbPartitions(); + } + + public void setTablePartitions(SQLExpr x) { + indexDefinition.setTbPartitions(x); + } + + @Override + public List getCovering() { + return indexDefinition.getCovering(); + } + + @Override + public List getColumns() { + return indexDefinition.getColumns(); + } +} diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java index 814678269f..f7189f5901 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java @@ -6531,8 +6531,19 @@ public class MySqlStatementParser extends SQLStatementParser { if (lexer.token() == Token.INDEX) { // Caution: Not in MySql documents. lexer.nextToken(); - MySqlAlterTableAlterFullTextIndex alterIndex = new MySqlAlterTableAlterFullTextIndex(); + SQLName indexName = this.exprParser.name(); + + if (lexer.identifierEquals("VISIBLE")) { + SQLAlterTableAlterIndex alterIndex = new SQLAlterTableAlterIndex(); + alterIndex.setName(indexName); + lexer.nextToken(); + alterIndex.getIndexDefinition().getOptions().setVisible(true); + stmt.addItem(alterIndex); + break; + } + + MySqlAlterTableAlterFullTextIndex alterIndex = new MySqlAlterTableAlterFullTextIndex(); alterIndex.setIndexName(indexName); accept(Token.SET); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index 680f85efe9..25881e565d 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -4780,6 +4780,10 @@ public class SQLExprParser extends SQLParser { lexer.identifierEquals("INVISIBLE")) { lexer.nextToken(); indexDefinition.getOptions().setInvisible(true); + } else if ((DbType.mysql == dbType) && + lexer.identifierEquals("VISIBLE")) { + lexer.nextToken(); + indexDefinition.getOptions().setVisible(true); } else if ((DbType.mysql == dbType) && lexer.token == Token.HINT && lexer.stringVal().trim().equals("!80000 INVISIBLE")) { lexer.nextToken(); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 7d5200eac4..bae9c05a52 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -6035,6 +6035,8 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet if (x.isInvisible()) { print0(ucase ? " INVISIBLE" : " invisible"); + } else if (x.isVisible()) { + print0(ucase ? " VISIBLE" : " visible"); } SQLExpr keyBlockSize = x.getKeyBlockSize(); @@ -6128,9 +6130,11 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet print0(ucase ? "HASHMAP " : "hashmap "); } - print('('); - printAndAccept(x.getColumns(), ", "); - print(')'); + if (x.getColumns().size() > 0) { + print('('); + printAndAccept(x.getColumns(), ", "); + print(')'); + } if (x.getAnalyzerName() != null) { print0(ucase ? " WITH ANALYZER " : " with analyzer "); @@ -6193,6 +6197,15 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return false; } + @Override + public boolean visit(SQLAlterTableAlterIndex x) { + print0(ucase ? "ALTER " : "alter "); + + visit(x.getIndexDefinition()); + + return false; + } + @Override public boolean visit(SQLAlterTableAddConstraint x) { if (x.isWithNoCheck()) { diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index 85b8c0cc1a..d85614728a 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -881,6 +881,14 @@ public interface SQLASTVisitor { return true; } + default void endVisit(SQLAlterTableAlterIndex x) { + + } + + default boolean visit(SQLAlterTableAlterIndex x) { + return true; + } + default void endVisit(SQLAlterTableAddConstraint x) { } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/visitor/MySqlResourceTest.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/visitor/MySqlResourceTest.java index 90835353e0..7c85c2b0b4 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/visitor/MySqlResourceTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/visitor/MySqlResourceTest.java @@ -103,6 +103,22 @@ public class MySqlResourceTest extends TestCase { exec_test("bvt/parser/mysql-27.txt"); } + public void test_28() throws Exception { + exec_test("bvt/parser/mysql-28.txt"); + } + + public void test_29() throws Exception { + exec_test("bvt/parser/mysql-29.txt"); + } + + public void test_30() throws Exception { + exec_test("bvt/parser/mysql-30.txt"); + } + + public void test_31() throws Exception { + exec_test("bvt/parser/mysql-31.txt"); + } + public void exec_test(String resource) throws Exception { // System.out.println(resource); InputStream is = null; diff --git a/src/test/resources/bvt/parser/mysql-28.txt b/src/test/resources/bvt/parser/mysql-28.txt new file mode 100644 index 0000000000..4803731ad7 --- /dev/null +++ b/src/test/resources/bvt/parser/mysql-28.txt @@ -0,0 +1,13 @@ +CREATE TABLE t1 ( + i INT, + j INT, + k INT, + INDEX i_idx (i) INVISIBLE +) ENGINE = InnoDB; +--------------------------- +CREATE TABLE t1 ( + i INT, + j INT, + k INT, + INDEX i_idx(i) INVISIBLE +) ENGINE = InnoDB; \ No newline at end of file diff --git a/src/test/resources/bvt/parser/mysql-29.txt b/src/test/resources/bvt/parser/mysql-29.txt new file mode 100644 index 0000000000..710f590205 --- /dev/null +++ b/src/test/resources/bvt/parser/mysql-29.txt @@ -0,0 +1,3 @@ +CREATE INDEX j_idx ON t1 (j) INVISIBLE; +--------------------------- +CREATE INDEX j_idx ON t1 (j) INVISIBLE; \ No newline at end of file diff --git a/src/test/resources/bvt/parser/mysql-30.txt b/src/test/resources/bvt/parser/mysql-30.txt new file mode 100644 index 0000000000..9ee5ba05f7 --- /dev/null +++ b/src/test/resources/bvt/parser/mysql-30.txt @@ -0,0 +1,4 @@ +ALTER TABLE t1 ADD INDEX k_idx (k) INVISIBLE; +--------------------------- +ALTER TABLE t1 + ADD INDEX k_idx (k) INVISIBLE; \ No newline at end of file diff --git a/src/test/resources/bvt/parser/mysql-31.txt b/src/test/resources/bvt/parser/mysql-31.txt new file mode 100644 index 0000000000..294046d08b --- /dev/null +++ b/src/test/resources/bvt/parser/mysql-31.txt @@ -0,0 +1,4 @@ +ALTER TABLE t1 ALTER INDEX i_idx VISIBLE; +--------------------------- +ALTER TABLE t1 + ALTER i_idx VISIBLE; \ No newline at end of file diff --git a/src/test/resources/bvt/parser/odps-4.txt b/src/test/resources/bvt/parser/odps-4.txt new file mode 100644 index 0000000000..dbf1ab45fc --- /dev/null +++ b/src/test/resources/bvt/parser/odps-4.txt @@ -0,0 +1,12 @@ +create table lt_tmp_adm_ddm_algo_user_cat2_interval_buy_history_days_20201114_dd lifecycle 7 as +select user_id,cat2_id,interval,gmt_create_dd + ,ROW_NUMBER() OVER(PARTITION BY user_id,cat2_id,interval ORDER BY gmt_create_dd) AS row_num +from( + select distinct trd_biz_consumer_id as user_id + ,coalesce(trd_cate_2_id,sh_cate_2_id) as cat2_id + ,mapHour2RangeBySlices(split_part(trd_gmt_create, ' ', 2), '6.0,10.5,14.0,16.0,21.0') interval + ,datetrunc(trd_gmt_create,'DD') gmt_create_dd + from kbalgo.adm_ddm_algo_dwd_evt_ot_trd_trade_di + where dt<='20201114' and dt > '20201015' + and length(trd_biz_consumer_id)>3 and length(coalesce(trd_cate_2_id,sh_cate_2_id))>3 +)t; \ No newline at end of file diff --git a/src/test/resources/bvt/parser/odps-5.txt b/src/test/resources/bvt/parser/odps-5.txt new file mode 100644 index 0000000000..6fb594abc3 --- /dev/null +++ b/src/test/resources/bvt/parser/odps-5.txt @@ -0,0 +1,47 @@ +create table if not exists crm_hp_marketing_group_meta_cron ( + + + brand_id string comment '人群包适用的crm brand id', + track_id string comment 'md5(group_id + brand_id + dt + hh)', + algo_name string comment + '算法名称, 主要供效果看板模块追踪算法级别的转化情况使用', + + + group_type string comment + '人群包组成分类, 这个信息涉及到人群包投放链路的匹配，分类如下: \n' + 'MEMBER_ONLY: 只有会员，适用于CRM平台会员营销链路或者代运营投放链路\n' + 'CUSTOMER_ONLY: 只有顾客，适用于CRM平台顾客营销链路或者代运营投放链路\n' + 'POTENTIAL_ONLY: 潜客，适用于CRM平台潜客营销链路或者代运营投放链路\n' + 'MEMBER_AND_CUSTOMER' + 'CUSTOMER_AND_POTENTIAL' + 'ALL: 可能包含全部三类人群，只适用于代运营投放链路', + scene string comment + '人群包适用场景, MEMBER_RECRUIT-会员招募, TRADE_PROMOTION-促成交易', + platform string comment + '人群投放平台, CRM-crm人群管理, OP-代运营平台', + channel string comment + '投放的端是 koubei-口碑 eleme-饿了么 wanjiadenghuo-支付宝万家灯火', + + + name string comment + '人群包id对应的名称，一般用于展示在相应前端系统里, 便于使用者理解人群包的内容', + description string comment '对人群包的详细描述', + + + create_time string comment '创建时间', + creator string comment '创建人', + num bigint comment '人群包规模, 不同oneid的数量', + customerNum bigint comment '人群包的顾客数量', + memberNum bigint comment '人群包的会员数量', + proCustomerNum bigint comment '人群包的潜客数量', + ext_info string comment '以json字典表示的额外信息' +) comment 'CRM精准营销人群信息定义表，一次性' +partitioned by ( + + dt string comment '日期', + hh string comment '小时, 00-23', + + + + group_id string comment '人群包id = algo_name_id = algo_name + "_" + creator' +) lifecycle 9999; \ No newline at end of file diff --git a/src/test/resources/bvt/parser/odps-6.txt b/src/test/resources/bvt/parser/odps-6.txt new file mode 100644 index 0000000000..c35e917fe9 --- /dev/null +++ b/src/test/resources/bvt/parser/odps-6.txt @@ -0,0 +1 @@ +update t set (a, b, c) = (select x, y, z from e where t.d = e.d) where t.d = e.d; \ No newline at end of file diff --git a/src/test/resources/bvt/parser/odps-7.txt b/src/test/resources/bvt/parser/odps-7.txt new file mode 100644 index 0000000000..d71e2ac120 --- /dev/null +++ b/src/test/resources/bvt/parser/odps-7.txt @@ -0,0 +1,118 @@ +--odps sql +--********************************************************************-- +--author:tianzhao +--create time:2020-04-07 16:50:53 +--job info: 用户维度表（公司用户+外部用户） +--********************************************************************-- + + + +INSERT OVERWRITE TABLE dim_user +PARTITION( ds='${ds}' ) + +select +a.user_id -- xxxxx this is comment +,partner_id +,app_code +,user_level +,trim(user_name) user_name +,real_name +,user_phone +,user_catalog +,user_type +,user_password +,user_status +,user_info_status +,user_remark +,promoter_name +,promoter_id +,user_role +,if(a.deleted_time = 0 or a.deleted_time is null, 0, 1) is_del +,created_at create_gmt +,updated_at update_gmt +,audit_time audit_gmt +,audit_user_id +,a.tags +,employee_id +,nickname +,email +,department_id +,base_point +,point +,add_up_point +,if(type = 0, 0, 1) is_real_user +,account_id +,IF(a.deleted_time = 0, null, FROM_UNIXTIME(CAST(a.deleted_time/1000 as BIGINT))) del_gmt +-- +,role_id +,set_role_gmt +,role_name +,domain_id +,role_remark +-- +,c.manager_shop_id +,c.manager_shop_name +,c.manager_shop_create_gmt +from ( + select * + from ods_rose_rose_user_df + where ds = '${ds}' + and id is not null + and user_id is not null +) a +left join ( + select aa.user_id + ,COLLECT_SET(aa.role_id) role_id + ,COLLECT_SET(aa.create_role_gmt) set_role_gmt + ,COLLECT_SET(bb.role_name) role_name + ,COLLECT_SET(bb.domain_id) domain_id + ,COLLECT_SET(bb.role_remark) role_remark + from ( + select user_id, role_id, created_at create_role_gmt + from ods_rose_rose_user_role_df + where ds = '${ds}' + and user_id is not null + and role_id is not null + ) aa + left join ( + select role_id + ,max(if(is_deleted = 1, NULL, role_name)) role_name + ,max(if(is_deleted = 1, NULL, domain_id)) domain_id + ,max(if(is_deleted = 1, NULL, role_remark)) role_remark + from ods_rose_rose_role_df + where ds = '${ds}' + and role_id is not null + group by role_id + ) bb + on aa.role_id = bb.role_id + group by aa.user_id +) b +on a.user_id = b.user_id +left join( + select + tmp3.user_id + ,collect_list(tmp3.shop_id) manager_shop_id + ,collect_list(tmp3.shop_name) manager_shop_name + ,collect_list(tmp3.created_at) manager_shop_create_gmt + from + ( + select tmp3.user_id, tmp1.shop_id, tmp1.created_at,tmp2.shop_name + from + ( + select user_id,shop_id,created_at + from adbb5.ods_rose_shop_user_df + where ds = '${ds}' + and shop_id is not null + ) tmp1 join + ( + select shop_id,shop_name + from ods_rose_shop_df + where ds = '${ds}' + and is_deleted = 0 + ) tmp2 + on tmp3.shop_id = tmp2.shop_id + ) tmp3 + group by tmp3.user_id +) c +on a.user_id = c.user_id +; \ No newline at end of file