From 58abdd2f22c5e16587f62fca8aef9a55eb3aeb3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=AB=98=E9=93=81?= <shaojin.wensj@alibaba-inc.com>
Date: Sat, 17 Apr 2021 22:15:25 +0800
Subject: [PATCH] improved sql parser, improved clickhouse support & mysql
 repository support

---
 .../java/com/alibaba/druid/sql/SQLUtils.java  |  3 +
 .../druid/sql/ast/SQLJSONValueExpr.java       | 28 +++++++
 .../ast/ClickhouseCreateTableStatement.java   | 80 ++++++++++++++++++
 .../parser/ClickhouseCreateTableParser.java   | 69 +++++++++++++++
 .../parser/ClickhouseStatementParser.java     |  9 +-
 .../visitor/ClickSchemaStatVisitor.java       | 19 +++++
 .../visitor/ClickhouseOutputVisitor.java      | 43 +++++++++-
 .../clickhouse/visitor/ClickhouseVisitor.java |  8 +-
 .../mysql/parser/MySqlCreateTableParser.java  | 15 ++++
 .../dialect/mysql/parser/MySqlExprParser.java |  8 +-
 .../com/alibaba/druid/sql/parser/Lexer.java   | 23 ++++-
 .../sql/parser/SQLCreateTableParser.java      | 16 ++--
 .../druid/sql/parser/SQLExprParser.java       | 20 ++++-
 .../druid/sql/parser/SQLStatementParser.java  |  4 +-
 .../com/alibaba/druid/sql/parser/SQLType.java |  6 ++
 .../sql/repository/SchemaRepository.java      |  4 +
 .../support/opds/udf/MetaSqlExtract.java      |  8 +-
 .../java/com/alibaba/druid/util/FnvHash.java  |  4 +
 .../bvt/sql/canal/CanalSQLSchemaTest.java     | 83 +++++++++++++++++++
 .../clickhouse/ClickHouseResourceTest.java    | 20 +++++
 .../resources/bvt/parser/clickhouse-5.txt     |  4 +
 .../resources/bvt/parser/clickhouse-6.txt     | 14 ++++
 .../resources/bvt/parser/clickhouse-7.txt     | 18 ++++
 .../resources/bvt/parser/clickhouse-8.txt     |  3 +
 24 files changed, 481 insertions(+), 28 deletions(-)
 create mode 100644 src/main/java/com/alibaba/druid/sql/ast/SQLJSONValueExpr.java
 create mode 100644 src/main/java/com/alibaba/druid/sql/dialect/clickhouse/ast/ClickhouseCreateTableStatement.java
 create mode 100644 src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java
 create mode 100644 src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickSchemaStatVisitor.java
 create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/canal/CanalSQLSchemaTest.java
 create mode 100644 src/test/resources/bvt/parser/clickhouse-5.txt
 create mode 100644 src/test/resources/bvt/parser/clickhouse-6.txt
 create mode 100644 src/test/resources/bvt/parser/clickhouse-7.txt
 create mode 100644 src/test/resources/bvt/parser/clickhouse-8.txt

diff --git a/src/main/java/com/alibaba/druid/sql/SQLUtils.java b/src/main/java/com/alibaba/druid/sql/SQLUtils.java
index 30e57ba6e6..adc6df0ffe 100644
--- a/src/main/java/com/alibaba/druid/sql/SQLUtils.java
+++ b/src/main/java/com/alibaba/druid/sql/SQLUtils.java
@@ -23,6 +23,7 @@ import com.alibaba.druid.sql.dialect.ads.visitor.AdsOutputVisitor;
 import com.alibaba.druid.sql.dialect.antspark.visitor.AntsparkOutputVisitor;
 import com.alibaba.druid.sql.dialect.antspark.visitor.AntsparkSchemaStatVisitor;
 import com.alibaba.druid.sql.dialect.blink.vsitor.BlinkOutputVisitor;
+import com.alibaba.druid.sql.dialect.clickhouse.visitor.ClickSchemaStatVisitor;
 import com.alibaba.druid.sql.dialect.clickhouse.visitor.ClickhouseOutputVisitor;
 import com.alibaba.druid.sql.dialect.db2.visitor.DB2OutputVisitor;
 import com.alibaba.druid.sql.dialect.db2.visitor.DB2SchemaStatVisitor;
@@ -547,6 +548,8 @@ public class SQLUtils {
                 return new HiveSchemaStatVisitor(repository);
             case antspark:
                 return new AntsparkSchemaStatVisitor(repository);
+            case clickhouse:
+                return new ClickSchemaStatVisitor(repository);
             default:
                 return new SchemaStatVisitor(repository);
         }
diff --git a/src/main/java/com/alibaba/druid/sql/ast/SQLJSONValueExpr.java b/src/main/java/com/alibaba/druid/sql/ast/SQLJSONValueExpr.java
new file mode 100644
index 0000000000..c01bbbfd4d
--- /dev/null
+++ b/src/main/java/com/alibaba/druid/sql/ast/SQLJSONValueExpr.java
@@ -0,0 +1,28 @@
+package com.alibaba.druid.sql.ast;
+
+import com.alibaba.druid.sql.visitor.SQLASTVisitor;
+
+public class SQLJSONValueExpr extends SQLExprImpl {
+    private SQLExpr json;
+    private SQLExpr path;
+
+    @Override
+    public boolean equals(Object o) {
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return 0;
+    }
+
+    @Override
+    protected void accept0(SQLASTVisitor v) {
+
+    }
+
+    @Override
+    public SQLExpr clone() {
+        return null;
+    }
+}
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/ast/ClickhouseCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/ast/ClickhouseCreateTableStatement.java
new file mode 100644
index 0000000000..d79b4c4eb0
--- /dev/null
+++ b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/ast/ClickhouseCreateTableStatement.java
@@ -0,0 +1,80 @@
+package com.alibaba.druid.sql.dialect.clickhouse.ast;
+
+import com.alibaba.druid.DbType;
+import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.sql.ast.SQLOrderBy;
+import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
+import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
+import com.alibaba.druid.sql.dialect.clickhouse.visitor.ClickhouseVisitor;
+import com.alibaba.druid.sql.visitor.SQLASTVisitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ClickhouseCreateTableStatement extends SQLCreateTableStatement {
+    protected final List<SQLAssignItem> settings    = new ArrayList<SQLAssignItem>();
+    private SQLOrderBy orderBy;
+    private SQLExpr partitionBy;
+    private SQLExpr sampleBy;
+
+    public ClickhouseCreateTableStatement() {
+        super(DbType.clickhouse);
+    }
+
+    public SQLOrderBy getOrderBy() {
+        return orderBy;
+    }
+
+    public void setOrderBy(SQLOrderBy x) {
+        if (x != null) {
+            x.setParent(this);
+        }
+
+        this.orderBy = x;
+    }
+
+    public SQLExpr getPartitionBy() {
+        return partitionBy;
+    }
+
+    public void setPartitionBy(SQLExpr x) {
+        if (x != null) {
+            x.setParent(this);
+        }
+
+        this.partitionBy = x;
+    }
+
+    public SQLExpr getSampleBy() {
+        return sampleBy;
+    }
+
+    public void setSampleBy(SQLExpr x) {
+        if (x != null) {
+            x.setParent(this);
+        }
+
+        this.sampleBy = x;
+    }
+
+    public List<SQLAssignItem> getSettings() {
+        return settings;
+    }
+
+    @Override
+    protected void accept0(SQLASTVisitor v) {
+        if (v instanceof ClickhouseVisitor) {
+            ClickhouseVisitor vv = (ClickhouseVisitor) v;
+            if (vv.visit(this)) {
+                acceptChild(vv);
+            }
+            vv.endVisit(this);
+            return;
+        }
+
+        if (v.visit(this)) {
+            acceptChild(v);
+        }
+        v.endVisit(this);
+    }
+}
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java
new file mode 100644
index 0000000000..5cc452f094
--- /dev/null
+++ b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java
@@ -0,0 +1,69 @@
+package com.alibaba.druid.sql.dialect.clickhouse.parser;
+
+import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.sql.ast.SQLOrderBy;
+import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
+import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
+import com.alibaba.druid.sql.dialect.clickhouse.ast.ClickhouseCreateTableStatement;
+import com.alibaba.druid.sql.parser.SQLCreateTableParser;
+import com.alibaba.druid.sql.parser.SQLExprParser;
+import com.alibaba.druid.sql.parser.Token;
+import com.alibaba.druid.util.FnvHash;
+
+public class ClickhouseCreateTableParser extends SQLCreateTableParser {
+    public ClickhouseCreateTableParser(SQLExprParser exprParser) {
+        super(exprParser);
+    }
+
+    protected SQLCreateTableStatement newCreateStatement() {
+        return new ClickhouseCreateTableStatement();
+    }
+
+    protected void parseCreateTableRest(SQLCreateTableStatement stmt) {
+        ClickhouseCreateTableStatement ckStmt = (ClickhouseCreateTableStatement) stmt;
+        if (lexer.identifierEquals(FnvHash.Constants.ENGINE)) {
+            lexer.nextToken();
+            if (lexer.token() == Token.EQ) {
+                lexer.nextToken();
+            }
+            stmt.setEngine(
+                    this.exprParser.expr()
+            );
+        }
+
+        if (lexer.identifierEquals("PARTITION")) {
+            lexer.nextToken();
+            accept(Token.BY);
+            SQLExpr expr = this.exprParser.expr();
+            ckStmt.setPartitionBy(expr);
+        }
+
+        if (lexer.token() == Token.ORDER) {
+            SQLOrderBy orderBy = this.exprParser.parseOrderBy();
+            ckStmt.setOrderBy(orderBy);
+        }
+
+        if (lexer.identifierEquals("SAMPLE")) {
+            lexer.nextToken();
+            accept(Token.BY);
+            SQLExpr expr = this.exprParser.expr();
+            ckStmt.setSampleBy(expr);
+        }
+
+        if (lexer.identifierEquals("SETTINGS")) {
+            lexer.nextToken();
+            for (;;) {
+                SQLAssignItem item = this.exprParser.parseAssignItem();
+                item.setParent(ckStmt);
+                ckStmt.getSettings().add(item);
+
+                if (lexer.token() == Token.COMMA) {
+                    lexer.nextToken();
+                    continue;
+                }
+
+                break;
+            }
+        }
+    }
+}
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java
index a9dca133c7..2ac7365579 100644
--- a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java
+++ b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java
@@ -1,10 +1,7 @@
 package com.alibaba.druid.sql.dialect.clickhouse.parser;
 
 import com.alibaba.druid.sql.ast.statement.SQLWithSubqueryClause;
-import com.alibaba.druid.sql.parser.Lexer;
-import com.alibaba.druid.sql.parser.SQLParserFeature;
-import com.alibaba.druid.sql.parser.SQLStatementParser;
-import com.alibaba.druid.sql.parser.Token;
+import com.alibaba.druid.sql.parser.*;
 
 public class ClickhouseStatementParser extends SQLStatementParser {
     public ClickhouseStatementParser(String sql) {
@@ -69,4 +66,8 @@ public class ClickhouseStatementParser extends SQLStatementParser {
 
         return withQueryClause;
     }
+
+    public SQLCreateTableParser getSQLCreateTableParser() {
+        return new ClickhouseCreateTableParser(this.exprParser);
+    }
 }
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickSchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickSchemaStatVisitor.java
new file mode 100644
index 0000000000..679f942832
--- /dev/null
+++ b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickSchemaStatVisitor.java
@@ -0,0 +1,19 @@
+package com.alibaba.druid.sql.dialect.clickhouse.visitor;
+
+import com.alibaba.druid.DbType;
+import com.alibaba.druid.sql.repository.SchemaRepository;
+import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+
+public class ClickSchemaStatVisitor extends SchemaStatVisitor implements ClickhouseVisitor {
+        {
+            dbType = DbType.antspark;
+        }
+
+    public ClickSchemaStatVisitor() {
+            super(DbType.antspark);
+        }
+
+    public ClickSchemaStatVisitor(SchemaRepository repository) {
+            super(repository);
+        }
+}
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseOutputVisitor.java
index 7d068912a6..50e278ae6b 100644
--- a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseOutputVisitor.java
+++ b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseOutputVisitor.java
@@ -1,14 +1,16 @@
 package com.alibaba.druid.sql.dialect.clickhouse.visitor;
 
 import com.alibaba.druid.DbType;
-import com.alibaba.druid.sql.ast.SQLDataType;
-import com.alibaba.druid.sql.ast.SQLName;
-import com.alibaba.druid.sql.ast.SQLRowDataType;
-import com.alibaba.druid.sql.ast.SQLStructDataType;
+import com.alibaba.druid.sql.ast.*;
+import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
+import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
 import com.alibaba.druid.sql.ast.statement.SQLSelect;
 import com.alibaba.druid.sql.ast.statement.SQLWithSubqueryClause;
+import com.alibaba.druid.sql.dialect.clickhouse.ast.ClickhouseCreateTableStatement;
 import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
 
+import java.util.List;
+
 public class ClickhouseOutputVisitor extends SQLASTOutputVisitor implements ClickhouseVisitor {
     public ClickhouseOutputVisitor(Appendable appender) {
         super(appender);
@@ -71,4 +73,37 @@ public class ClickhouseOutputVisitor extends SQLASTOutputVisitor implements Clic
 
         return false;
     }
+
+    @Override
+    public boolean visit(ClickhouseCreateTableStatement x) {
+        super.visit((SQLCreateTableStatement) x);
+
+        SQLExpr partitionBy = x.getPartitionBy();
+        if (partitionBy != null) {
+            println();
+            print0(ucase ? "PARTITION BY " : "partition by ");
+            partitionBy.accept(this);
+        }
+
+        SQLOrderBy orderBy = x.getOrderBy();
+        if (orderBy != null) {
+            println();
+            orderBy.accept(this);
+        }
+
+        SQLExpr sampleBy = x.getSampleBy();
+        if (sampleBy != null) {
+            println();
+            print0(ucase ? "SAMPLE BY " : "sample by ");
+            sampleBy.accept(this);
+        }
+
+        List<SQLAssignItem> settings = x.getSettings();
+        if (!settings.isEmpty()) {
+            println();
+            print0(ucase ? "SETTINGS " : "settings ");
+            printAndAccept(settings, ", ");
+        }
+        return false;
+    }
 }
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseVisitor.java
index 942278a022..64f13311fe 100644
--- a/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseVisitor.java
+++ b/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/visitor/ClickhouseVisitor.java
@@ -1,8 +1,14 @@
 package com.alibaba.druid.sql.dialect.clickhouse.visitor;
 
-import com.alibaba.druid.sql.ast.statement.SQLWithSubqueryClause;
+import com.alibaba.druid.sql.ast.statement.SQLNotNullConstraint;
+import com.alibaba.druid.sql.dialect.clickhouse.ast.ClickhouseCreateTableStatement;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 
 public interface ClickhouseVisitor extends SQLASTVisitor {
+    default boolean visit(ClickhouseCreateTableStatement x) {
+        return true;
+    }
 
+    default void endVisit(ClickhouseCreateTableStatement x) {
+    }
 }
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java
index e1646b9338..cb80b5ce08 100644
--- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java
+++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java
@@ -359,6 +359,10 @@ public class MySqlCreateTableParser extends SQLCreateTableParser {
                 }
             }
 
+            if (lexer.token() == Token.HINT) {
+                lexer.nextToken();
+            }
+
             accept(Token.RPAREN);
 
             if (lexer.token() == Token.HINT && lexer.stringVal().charAt(0) == '!') {
@@ -405,6 +409,17 @@ public class MySqlCreateTableParser extends SQLCreateTableParser {
                 continue;
             }
 
+            if (lexer.identifierEquals("BLOCK_FORMAT")) {
+                lexer.nextToken();
+                if (lexer.token() == Token.EQ) {
+                    lexer.nextToken();
+                }
+
+                SQLExpr expr = this.exprParser.primary();
+                stmt.addOption("BLOCK_FORMAT", expr);
+                continue;
+            }
+
             if (lexer.identifierEquals(FnvHash.Constants.REPLICA_NUM)) {
                 lexer.nextToken();
                 if (lexer.token() == Token.EQ) {
diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java
index afbe3d291c..de747d12a5 100644
--- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java
+++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java
@@ -1137,7 +1137,7 @@ public class MySqlExprParser extends SQLExprParser {
             accept(Token.RPAREN);
         }
 
-        if (lexer.identifierEquals(FnvHash.Constants.STORED)) {
+        if (lexer.identifierEquals(FnvHash.Constants.STORED) || lexer.identifierEquals("PERSISTENT")) {
             lexer.nextToken();
             column.setStored(true);
         }
@@ -1153,6 +1153,7 @@ public class MySqlExprParser extends SQLExprParser {
             column.setDelimiter(expr);
             return parseColumnRest(column);
         }
+
         if (lexer.identifierEquals("delimiter_tokenizer")) {
             lexer.nextToken();
             SQLExpr expr = this.expr();
@@ -1238,6 +1239,11 @@ public class MySqlExprParser extends SQLExprParser {
             }
         }
 
+        if (lexer.identifierEquals(FnvHash.Constants.ARRAY)) {
+            lexer.nextToken();
+            dataType = new SQLArrayDataType(dataType);
+        }
+
         return dataType;
     }
 
diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java
index 5cea6a1498..c87585aa63 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java
@@ -797,12 +797,21 @@ public class Lexer {
             nextToken();
             if (token == Token.USER || identifierEquals(FnvHash.Constants.USER)) {
                 return SQLType.ADD_USER;
+            } else if (token == TABLE) {
+                return SQLType.ADD_TABLE;
             }
         } else if (hashCode == FnvHash.Constants.REMOVE) {
             nextToken();
             if (token == Token.USER || identifierEquals(FnvHash.Constants.USER)) {
                 return SQLType.REMOVE_USER;
             }
+        } else if (hashCode == FnvHash.Constants.TUNNEL) {
+            nextToken();
+            if (identifierEquals(FnvHash.Constants.DOWNLOAD)) {
+                return SQLType.TUNNEL_DOWNLOAD;
+            }
+        } else if (hashCode == FnvHash.Constants.UPLOAD) {
+            return SQLType.UPLOAD;
         }
 
         return SQLType.UNKNOWN;
@@ -811,7 +820,6 @@ public class Lexer {
     public final SQLType scanSQLTypeV2() {
 
         SQLType sqlType = scanSQLType();
-
         if (sqlType == SQLType.CREATE) {
             nextToken();
             if (token == Token.USER || identifierEquals(FnvHash.Constants.USER)) {
@@ -869,6 +877,19 @@ public class Lexer {
             } else if (token == TABLE) {
                 return SQLType.ALTER_TABLE;
             }
+        } else if (sqlType == SQLType.INSERT) {
+            for (int i = 0; i < 1000;++i) {
+                nextToken();
+
+                if (token == SELECT) {
+                    return SQLType.INSERT_SELECT;
+                } else if (token == VALUES) {
+                    return SQLType.INSERT_VALUES;
+                } else if (token == ERROR || token == EOF) {
+                    break;
+                }
+            }
+            return sqlType;
         }
 
         return sqlType;
diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLCreateTableParser.java
index 44a484fdc4..a5bbbcf793 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/SQLCreateTableParser.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/SQLCreateTableParser.java
@@ -88,7 +88,7 @@ public class SQLCreateTableParser extends SQLDDLParser {
 
         accept(Token.TABLE);
 
-        if (lexer.token() == Token.IF) {
+        if (lexer.token() == Token.IF || lexer.identifierEquals("IF")) {
             lexer.nextToken();
             accept(Token.NOT);
             accept(Token.EXISTS);
@@ -182,19 +182,15 @@ public class SQLCreateTableParser extends SQLDDLParser {
             createTable.setPartitioning(partitionClause);
         }
 
-        if (dbType == DbType.clickhouse) {
-            if (lexer.identifierEquals(FnvHash.Constants.ENGINE)) {
-                lexer.nextToken();
-                accept(Token.EQ);
-                createTable.setEngine(
-                        this.exprParser.expr()
-                );
-            }
-        }
+        parseCreateTableRest(createTable);
 
         return createTable;
     }
 
+    protected void parseCreateTableRest(SQLCreateTableStatement stmt) {
+
+    }
+
     public SQLPartitionBy parsePartitionBy() {
         return null;
     }
diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
index f01a40f1aa..823cc0b2dd 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
@@ -594,7 +594,11 @@ public class SQLExprParser extends SQLParser {
             }
             break;
             case DEFAULT:
-                sqlExpr = new SQLDefaultExpr();
+                if (dbType == DbType.clickhouse) {
+                    sqlExpr = new SQLIdentifierExpr(lexer.stringVal());
+                } else {
+                    sqlExpr = new SQLDefaultExpr();
+                }
                 lexer.nextToken();
                 break;
             case DUAL:
@@ -798,7 +802,8 @@ public class SQLExprParser extends SQLParser {
                     cast.setExpr(
                             expr());
                     accept(Token.AS);
-                    cast.setDataType(parseDataType(false));
+                    cast.setDataType(
+                            parseDataType(false));
                     accept(Token.RPAREN);
 
                     sqlExpr = cast;
@@ -1758,6 +1763,9 @@ public class SQLExprParser extends SQLParser {
             } else if (lexer.token == Token.VARIANT && lexer.stringVal().startsWith("$")) {
                 name = lexer.stringVal();
                 lexer.nextToken();
+            } else if (lexer.token == Token.LITERAL_INT && dbType == DbType.hive) {
+                name = lexer.numberString();
+                lexer.nextToken();
             } else {
                 throw new ParserException("error : " + lexer.info());
             }
@@ -3826,6 +3834,10 @@ public class SQLExprParser extends SQLParser {
                 charType.setHints(hints);
             }
 
+            if (lexer.identifierEquals(FnvHash.Constants.ARRAY)) {
+                return parseDataTypeRest(charType);
+            }
+
             return charType;
         }
 
@@ -3852,6 +3864,10 @@ public class SQLExprParser extends SQLParser {
                 charType.setHints(hints);
             }
 
+            if (lexer.identifierEquals(FnvHash.Constants.ARRAY)) {
+                return parseDataTypeRest(charType);
+            }
+
             return charType;
         }
 
diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
index 0e138ca018..29f17f4659 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
@@ -3947,9 +3947,7 @@ public class SQLStatementParser extends SQLParser {
             accept(Token.DATABASE);
         }
 
-
-
-        if (lexer.token == Token.IF) {
+        if (lexer.token == Token.IF || lexer.identifierEquals("IF")) {
             lexer.nextToken();
             accept(Token.NOT);
             accept(Token.EXISTS);
diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLType.java b/src/main/java/com/alibaba/druid/sql/parser/SQLType.java
index 54a3ff65b4..96a6fb7d56 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/SQLType.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/SQLType.java
@@ -3,6 +3,8 @@ package com.alibaba.druid.sql.parser;
 public enum SQLType {
     SELECT,
     UPDATE,
+    INSERT_SELECT,
+    INSERT_VALUES,
     INSERT,
     DELETE,
     MERGE,
@@ -50,5 +52,9 @@ public enum SQLType {
     ALTER_TABLE,
     READ,
 
+    ADD_TABLE,
+    TUNNEL_DOWNLOAD,
+    UPLOAD,
+
     UNKNOWN
 }
diff --git a/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java b/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java
index 8352a70545..2f27f0dd87 100644
--- a/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java
+++ b/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java
@@ -912,6 +912,10 @@ public class SchemaRepository {
         String name = x1.computeName();
         SchemaObject table = schema.findTableOrView(name);
         if (table != null) {
+            if (x1.isIfNotExists()) {
+                return table;
+            }
+
             LOG.info("replaced table '" + name + "'");
         }
 
diff --git a/src/main/java/com/alibaba/druid/support/opds/udf/MetaSqlExtract.java b/src/main/java/com/alibaba/druid/support/opds/udf/MetaSqlExtract.java
index b857838588..60690a0b52 100644
--- a/src/main/java/com/alibaba/druid/support/opds/udf/MetaSqlExtract.java
+++ b/src/main/java/com/alibaba/druid/support/opds/udf/MetaSqlExtract.java
@@ -20,8 +20,12 @@ public class MetaSqlExtract  extends UDF {
             return null;
         }
 
-        String part = xml.substring(p0, p1);
-        return StringEscapeUtils.unescapeXml(part);
+        String sql = xml.substring(p0, p1);
+        if (sql.startsWith("<![CDATA[")) {
+            sql = sql.substring("<![CDATA[".length(), sql.length() - 3);
+        }
+
+        return StringEscapeUtils.unescapeXml(sql);
     }
 
 }
diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java
index 55bd9ae9f8..03258b6311 100644
--- a/src/main/java/com/alibaba/druid/util/FnvHash.java
+++ b/src/main/java/com/alibaba/druid/util/FnvHash.java
@@ -1119,5 +1119,9 @@ public final class FnvHash {
         long ASOF = fnv1a_64_lower("ASOF");
         long JSON_SET = fnv1a_64_lower("JSON_SET");
         long JSONB_SET = fnv1a_64_lower("JSONB_SET");
+
+        long TUNNEL = fnv1a_64_lower("TUNNEL");
+        long DOWNLOAD = fnv1a_64_lower("DOWNLOAD");
+        long UPLOAD = fnv1a_64_lower("UPLOAD");
     }
 }
diff --git a/src/test/java/com/alibaba/druid/bvt/sql/canal/CanalSQLSchemaTest.java b/src/test/java/com/alibaba/druid/bvt/sql/canal/CanalSQLSchemaTest.java
new file mode 100644
index 0000000000..15514e03ac
--- /dev/null
+++ b/src/test/java/com/alibaba/druid/bvt/sql/canal/CanalSQLSchemaTest.java
@@ -0,0 +1,83 @@
+package com.alibaba.druid.bvt.sql.canal;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.druid.sql.repository.SchemaObject;
+import com.alibaba.druid.sql.repository.SchemaRepository;
+import com.alibaba.druid.util.JdbcConstants;
+
+public class CanalSQLSchemaTest {
+
+    @Test
+    public void testSimple() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql1 = "CREATE TABLE `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, "
+                      + "`key1` longtext NOT NULL COMMENT 'key1', `value1` longtext NOT NULL COMMENT 'value1', PRIMARY KEY (`id`) )"
+                      + "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4";
+        String sql2 = " CREATE TABLE IF NOT EXISTS `table_x1` ( `id` bigint(20) NOT NULL AUTO_INCREMENT,"
+                      + "`key1` longtext NOT NULL COMMENT 'key1',PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4";
+        repository.console(sql1);
+        repository.console(sql2);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("table_x1");
+        Assert.assertTrue(
+                table.findColumn("value1") != null);
+    }
+
+    @Test
+    public void test_block_format() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " CREATE TABLE `parent` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,"
+                     + "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`)"
+                     + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("parent");
+        Assert.assertTrue(table.findColumn("id") != null);
+    }
+
+    @Test
+    public void test_json_index() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " CREATE TABLE `articles` ( \n" +
+                " \t`article_id` bigint NOT NULL AUTO_INCREMENT, \n" +
+                " \t`tags` json DEFAULT NULL, PRIMARY KEY (`article_id`), \n" +
+                " \tKEY `articles_tags` (\n" +
+                " \t\t(\n" +
+                " \t\t\tcast(json_extract(`tags`,_utf8mb4'$[*]') \n" +
+                " \t\t\t\tas char(40) array)\n" +
+                " \t\t)\n" +
+                " \t)\n" +
+                " ) ENGINE=InnoDB AUTO_INCREMENT=1054 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("articles");
+        Assert.assertTrue(table.findColumn("article_id") != null);
+    }
+
+    @Test
+    public void test_invisible() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " CREATE TABLE `proposal_order_info` (`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,"
+                     + "`created_at` timestamp NULL DEFAULT NULL, " + "PRIMARY KEY (`id`) , "
+                     + "KEY `idx_create_time` (`created_at`) /*!80000 INVISIBLE */"
+                     + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 BLOCK_FORMAT=ENCRYPTED";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("proposal_order_info");
+        Assert.assertTrue(table.findColumn("id") != null);
+    }
+
+    @Test
+    public void test_persistent() throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        String sql = " create table example_vc_tbl( c1 int not null auto_increment primary key,"
+                     + "c2 varchar(70), vc1 int as (length(c2)) virtual,"
+                     + "DIM_SUM varchar(128) AS (MD5(UPPER(CONCAT(c2, c1)))) PERSISTENT)";
+        repository.console(sql);
+        repository.setDefaultSchema("test");
+        SchemaObject table = repository.findTable("example_vc_tbl");
+        Assert.assertTrue(table.findColumn("DIM_SUM") != null);
+    }
+}
diff --git a/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java b/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java
index 46f395e3b7..0a45fb815d 100644
--- a/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java
+++ b/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java
@@ -51,6 +51,26 @@ public class ClickHouseResourceTest extends OracleTest {
     public void test_3() throws Exception {
         exec_test("bvt/parser/clickhouse-3.txt");
     }
+//
+//    public void test_4() throws Exception {
+//        exec_test("bvt/parser/clickhouse-4.txt");
+//    }
+
+    public void test_5() throws Exception {
+        exec_test("bvt/parser/clickhouse-5.txt");
+    }
+
+    public void test_6() throws Exception {
+        exec_test("bvt/parser/clickhouse-6.txt");
+    }
+
+    public void test_7() throws Exception {
+        exec_test("bvt/parser/clickhouse-7.txt");
+    }
+
+    public void test_8() throws Exception {
+        exec_test("bvt/parser/clickhouse-8.txt");
+    }
 
     public void exec_test(String resource) throws Exception {
         System.out.println(resource);
diff --git a/src/test/resources/bvt/parser/clickhouse-5.txt b/src/test/resources/bvt/parser/clickhouse-5.txt
new file mode 100644
index 0000000000..27b7beedea
--- /dev/null
+++ b/src/test/resources/bvt/parser/clickhouse-5.txt
@@ -0,0 +1,4 @@
+select * from default.first_table
+---------------------------
+SELECT *
+FROM default.first_table
\ No newline at end of file
diff --git a/src/test/resources/bvt/parser/clickhouse-6.txt b/src/test/resources/bvt/parser/clickhouse-6.txt
new file mode 100644
index 0000000000..f23ba1ca04
--- /dev/null
+++ b/src/test/resources/bvt/parser/clickhouse-6.txt
@@ -0,0 +1,14 @@
+CREATE TABLE if not EXISTS  wengzi_test_db.first_table  (
+    `product_code` String,
+    `package_name` String
+)
+ENGINE = MergeTree
+ORDER BY package_name
+SETTINGS index_granularity = 8192;
+---------------------------
+CREATE TABLE IF NOT EXISTS wengzi_test_db.first_table (
+	`product_code` String,
+	`package_name` String
+) ENGINE = MergeTree
+ORDER BY package_name
+SETTINGS index_granularity = 8192;
\ No newline at end of file
diff --git a/src/test/resources/bvt/parser/clickhouse-7.txt b/src/test/resources/bvt/parser/clickhouse-7.txt
new file mode 100644
index 0000000000..befc626feb
--- /dev/null
+++ b/src/test/resources/bvt/parser/clickhouse-7.txt
@@ -0,0 +1,18 @@
+CREATE TABLE if not EXISTS  wengzi_test_db.first_table  (
+    `product_code` String,
+    `package_name` String
+)
+ENGINE MergeTree()
+    PARTITION BY toYYYYMM(EventDate)
+    ORDER BY (CounterID, EventDate, intHash32(UserID))
+    SAMPLE BY intHash32(UserID)
+    SETTINGS index_granularity=8192
+---------------------------
+CREATE TABLE IF NOT EXISTS wengzi_test_db.first_table (
+	`product_code` String,
+	`package_name` String
+) ENGINE = MergeTree()
+PARTITION BY toYYYYMM(EventDate)
+ORDER BY (CounterID, EventDate, intHash32(UserID))
+SAMPLE BY intHash32(UserID)
+SETTINGS index_granularity = 8192
\ No newline at end of file
diff --git a/src/test/resources/bvt/parser/clickhouse-8.txt b/src/test/resources/bvt/parser/clickhouse-8.txt
new file mode 100644
index 0000000000..d2fc3fd227
--- /dev/null
+++ b/src/test/resources/bvt/parser/clickhouse-8.txt
@@ -0,0 +1,3 @@
+CREATE DATABASE IF NOT EXISTS wengzi_test_db;
+---------------------------
+CREATE DATABASE IF NOT EXISTS wengzi_test_db;
\ No newline at end of file