From f10ba8537c10e386f50604e4dd8a906897a0c96e Mon Sep 17 00:00:00 2001 From: "shaojin.wensj" Date: Mon, 12 Dec 2022 19:28:37 +0800 Subject: [PATCH] improved sql parser --- .../main/java/com/alibaba/druid/VERSION.java | 2 +- .../ast/statement/SQLAlterTableStatement.java | 14 ++++ .../parser/ClickhouseCreateTableParser.java | 2 +- .../clickhouse/parser/ClickhouseLexer.java | 1 + .../parser/ClickhouseSelectParser.java | 71 +++++++++++++++++++ .../parser/ClickhouseStatementParser.java | 4 ++ .../sql/dialect/mysql/parser/MySqlLexer.java | 6 +- .../odps/visitor/OdpsOutputVisitor.java | 12 ++-- .../com/alibaba/druid/sql/parser/Lexer.java | 33 +++++++++ .../druid/sql/parser/SQLParserFeature.java | 5 +- .../druid/sql/parser/SQLParserUtils.java | 36 +++++----- .../druid/sql/parser/SQLStatementParser.java | 8 ++- .../SchemaResolveVisitorFactory.java | 7 +- .../sql/visitor/SQLASTOutputVisitor.java | 7 ++ .../druid/sql/visitor/SchemaStatVisitor.java | 11 ++- .../clickhouse/ClickHouseResourceTest.java | 8 ++- .../sql/mysql/select/MySqlSelectTest_310.java | 13 +++- .../resources/bvt/parser/clickhouse-15.txt | 12 ++++ .../resources/bvt/parser/clickhouse-16.txt | 13 ++++ 19 files changed, 231 insertions(+), 34 deletions(-) create mode 100644 core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseSelectParser.java create mode 100644 core/src/test/resources/bvt/parser/clickhouse-15.txt create mode 100644 core/src/test/resources/bvt/parser/clickhouse-16.txt diff --git a/core/src/main/java/com/alibaba/druid/VERSION.java b/core/src/main/java/com/alibaba/druid/VERSION.java index 62140fc309..cf39ab75c2 100644 --- a/core/src/main/java/com/alibaba/druid/VERSION.java +++ b/core/src/main/java/com/alibaba/druid/VERSION.java @@ -18,7 +18,7 @@ package com.alibaba.druid; public final class VERSION { public static final int MajorVersion = 1; public static final int MinorVersion = 2; - public static final int RevisionVersion = 15; + public static final int RevisionVersion = 16; public static String getVersionNumber() { return VERSION.MajorVersion + "." + VERSION.MinorVersion + "." + VERSION.RevisionVersion; diff --git a/core/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java b/core/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java index bea10e39e6..456802d48d 100644 --- a/core/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java +++ b/core/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java @@ -52,6 +52,9 @@ public class SQLAlterTableStatement extends SQLStatementImpl implements SQLDDLSt private boolean ifExists; private boolean notClustered; + // clickhouse + private SQLName on; + public SQLAlterTableStatement() { } @@ -288,4 +291,15 @@ public class SQLAlterTableStatement extends SQLStatementImpl implements SQLDDLSt public DDLObjectType getDDLObjectType() { return DDLObjectType.TABLE; } + + public SQLName getOn() { + return on; + } + + public void setOn(SQLName x) { + if (x != null) { + x.setParent(this); + } + this.on = x; + } } diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java index 1a942a79c9..4945bacdfa 100644 --- a/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseCreateTableParser.java @@ -31,7 +31,7 @@ public class ClickhouseCreateTableParser extends SQLCreateTableParser { ); } - if (lexer.identifierEquals("PARTITION")) { + if (lexer.token() == Token.PARTITION) { lexer.nextToken(); accept(Token.BY); SQLExpr expr = this.exprParser.expr(); diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseLexer.java b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseLexer.java index 882c4a6cd7..2c3530623e 100644 --- a/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseLexer.java +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseLexer.java @@ -29,6 +29,7 @@ public class ClickhouseLexer extends Lexer { map.put("SHOW", Token.SHOW); map.put("ALL", Token.ALL); map.put("GLOBAL", Token.GLOBAL); + map.put("PARTITION", Token.PARTITION); DEFAULT_KEYWORDS = new Keywords(map); } diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseSelectParser.java b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseSelectParser.java new file mode 100644 index 0000000000..d17b7bf265 --- /dev/null +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseSelectParser.java @@ -0,0 +1,71 @@ +package com.alibaba.druid.sql.dialect.clickhouse.parser; + +import com.alibaba.druid.sql.ast.statement.SQLWithSubqueryClause; +import com.alibaba.druid.sql.parser.*; +import com.alibaba.druid.util.FnvHash; + +public class ClickhouseSelectParser + extends SQLSelectParser { + public ClickhouseSelectParser(Lexer lexer) { + super(lexer); + } + + public ClickhouseSelectParser(SQLExprParser exprParser, SQLSelectListCache selectListCache) { + super(exprParser, selectListCache); + } + + public SQLWithSubqueryClause parseWith() { + SQLWithSubqueryClause withQueryClause = new SQLWithSubqueryClause(); + if (lexer.hasComment() && lexer.isKeepComments()) { + withQueryClause.addBeforeComment(lexer.readAndResetComments()); + } + + accept(Token.WITH); + + if (lexer.token() == Token.RECURSIVE || lexer.identifierEquals(FnvHash.Constants.RECURSIVE)) { + lexer.nextToken(); + withQueryClause.setRecursive(true); + } + + for (; ; ) { + SQLWithSubqueryClause.Entry entry = new SQLWithSubqueryClause.Entry(); + entry.setParent(withQueryClause); + + if (lexer.token() == Token.LPAREN) { + lexer.nextToken(); + + switch (lexer.token()) { + case SELECT: + case LPAREN: + case WITH: + case FROM: + entry.setSubQuery(select()); + break; + default: + break; + } + + accept(Token.RPAREN); + } else { + throw new ParserException("TODO"); + } + + accept(Token.AS); + + String alias = this.lexer.stringVal(); + lexer.nextToken(); + entry.setAlias(alias); + + withQueryClause.addEntry(entry); + + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + + break; + } + + return withQueryClause; + } +} diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java index 1677b15e95..c57c65fee8 100644 --- a/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/clickhouse/parser/ClickhouseStatementParser.java @@ -16,6 +16,10 @@ public class ClickhouseStatementParser extends SQLStatementParser { super(new ClickhouseExprParser(lexer)); } + public SQLSelectParser createSQLSelectParser() { + return new ClickhouseSelectParser(this.exprParser, selectListCache); + } + @Override public SQLWithSubqueryClause parseWithQuery() { SQLWithSubqueryClause withQueryClause = new SQLWithSubqueryClause(); diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java b/core/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java index 2b4b56c96e..12a7bbc51c 100644 --- a/core/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java @@ -548,6 +548,7 @@ public class MySqlLexer extends Lexer { Token lastToken = this.token; if (ch == '-') { + boolean supportStandardComment = (features & SQLParserFeature.MySQLSupportStandardComment.mask) != 0; /* * just for tddl test case; * test case : MySqlSelectTest_plus_sub_comment.java @@ -558,7 +559,10 @@ public class MySqlLexer extends Lexer { scanChar(); token = Token.SUB; return; - } else if ((before_1 == ' ' || (before_1 != '-' && before_1 != '+')) && (next_2 == ' ' || next_2 == EOI || next_2 == '\n')) { + } else if (supportStandardComment + || ((before_1 == ' ' || (before_1 != '-' && before_1 != '+')) + && (next_2 == ' ' || next_2 == EOI || next_2 == '\n')) + ) { // it is comments } else if ((before_1 == '-' || before_1 == '+') && next_2 == ' ') { throw new ParserException("illegal state. " + info()); diff --git a/core/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/core/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index ae7c859ef9..a3ea8818c7 100644 --- a/core/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -188,12 +188,6 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit print0(ucase ? " SHARDS" : " shards"); } - if (x.getLifecycle() != null) { - println(); - print0(ucase ? "LIFECYCLE " : "lifecycle "); - x.getLifecycle().accept(this); - } - SQLSelect select = x.getSelect(); if (select != null) { println(); @@ -233,6 +227,12 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit this.printTblProperties(x); + if (x.getLifecycle() != null) { + println(); + print0(ucase ? "LIFECYCLE " : "lifecycle "); + x.getLifecycle().accept(this); + } + SQLExpr using = x.getUsing(); if (using != null) { println(); diff --git a/core/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/core/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index f38ce8bba9..247669ad0a 100644 --- a/core/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/core/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -2573,6 +2573,39 @@ public class Lexer { } } + public List scanLineArgument() { + List args = new ArrayList<>(); + while (ch == ' ') { + scanChar(); + } + + int start = pos; + for (;;) { + if (ch == ' ' || ch == '\r' || ch == '\n' || ch == EOI) { + String arg = text.substring(start, pos); + arg = arg.trim(); + if (arg.length() > 0) { + args.add(arg); + } + scanChar(); + start = pos; + if (ch == '\r' || ch == '\n' || ch == EOI) { + break; + } else { + continue; + } + } + + scanChar(); + } + if (ch == EOI) { + token = EOF; + } else { + nextToken(); + } + return args; + } + private void scanMultiLineComment() { Token lastToken = this.token; int depth = 1; diff --git a/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java b/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java index 620efef08e..27c5b69660 100644 --- a/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java +++ b/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java @@ -51,9 +51,10 @@ public enum SQLParserFeature { EnableMultiUnion, Spark, - Presto; + Presto, + MySQLSupportStandardComment; - private SQLParserFeature() { + SQLParserFeature() { mask = (1 << ordinal()); } diff --git a/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java b/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java index 58cee5dfc2..6c77d3761a 100644 --- a/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java +++ b/core/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java @@ -603,21 +603,17 @@ public class SQLParserUtils { lexer.nextTokenForSet(); token = lexer.token; continue; - } else if (dbType == DbType.odps && (lexer.identifierEquals("pai") || lexer.identifierEquals("jar"))) { - if (lexer.startPos - start > 0) { - int semiIndex = sql.indexOf(';', lexer.startPos); - if (semiIndex != -1) { - lexer.pos = semiIndex - 1; - lexer.nextToken(); - token = lexer.token; - continue; - } - String str = sql.substring(start, lexer.startPos).trim(); - if (str.isEmpty()) { - lexer.startPos = sql.length(); - paiOrJar = true; - break; - } + } else if (dbType == DbType.odps + && lexer.ch != '.' + && (lexer.identifierEquals("pai") || lexer.identifierEquals("jar"))) { + lexer.scanLineArgument(); + paiOrJar = true; + } + + if (lexer.identifierEquals("USING")) { + lexer.nextToken(); + if (lexer.identifierEquals("jar")) { + lexer.nextToken(); } } @@ -637,7 +633,11 @@ public class SQLParserUtils { } if (start != sql.length() && token != Token.SEMI) { - String splitSql = sql.substring(start, lexer.startPos).trim(); + int end = lexer.startPos; + if (end > sql.length()) { + end = sql.length(); + } + String splitSql = sql.substring(start, end).trim(); if (!paiOrJar) { splitSql = removeComment(splitSql, dbType).trim(); } else { @@ -715,6 +715,10 @@ public class SQLParserUtils { return sql; } + if ((sql.startsWith("pai") || sql.startsWith("PAI")) && sql.indexOf(';') == -1) { + return sql; + } + boolean containsComment = false; { Lexer lexer = createLexer(sql, dbType); diff --git a/core/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/core/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index 37d40c152d..b20a748d96 100644 --- a/core/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/core/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -1868,6 +1868,12 @@ public class SQLStatementParser extends SQLParser { stmt.setName(this.exprParser.name()); + if (lexer.token == ON && dbType == DbType.clickhouse) { + lexer.nextToken(); + acceptIdentifier("CLUSTER"); + stmt.setOn(this.exprParser.name()); + } + for (; ; ) { if (lexer.token == Token.DROP) { parseAlterDrop(stmt); @@ -3198,7 +3204,7 @@ public class SQLStatementParser extends SQLParser { break; } } - } else if (acceptSubQuery && (lexer.token == Token.SELECT || lexer.token == Token.LPAREN)) { + } else if (acceptSubQuery && (lexer.token == WITH || lexer.token == Token.SELECT || lexer.token == Token.LPAREN)) { SQLSelect select = this.createSQLSelectParser().select(); insertStatement.setQuery(select); } else if (lexer.identifierEquals(FnvHash.Constants.VALUE)) { diff --git a/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java b/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java index 284d0faa13..3b4396e833 100644 --- a/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java +++ b/core/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java @@ -1843,8 +1843,13 @@ class SchemaResolveVisitorFactory { if (context != null && alias_hash != 0) { context.addTableSource(alias_hash, entry); } + } else if (entry.getExpr() != null) { + entry.getExpr().accept(visitor); } else { - entry.getReturningStatement().accept(visitor); + SQLStatement returningStatement = entry.getReturningStatement(); + if (returningStatement != null) { + returningStatement.accept(visitor); + } } } } diff --git a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 98d23b773d..a6324e343f 100644 --- a/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -5561,6 +5561,13 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } printTableSourceExpr(x.getName()); + + SQLName on = x.getOn(); + if (on != null) { + print(ucase ? " ON CLUSTER " : " on cluster "); + printExpr(on); + } + this.indentCount++; for (int i = 0; i < x.getItems().size(); ++i) { SQLAlterTableItem item = x.getItems().get(i); diff --git a/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java b/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java index 9524b59624..879223986e 100644 --- a/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java +++ b/core/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java @@ -1832,19 +1832,26 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter { String alias = x.getAlias(); SQLWithSubqueryClause with = (SQLWithSubqueryClause) x.getParent(); + SQLStatement returningStatement = x.getReturningStatement(); + SQLExpr expr = x.getExpr(); + if (Boolean.TRUE == with.getRecursive()) { SQLSelect select = x.getSubQuery(); if (select != null) { select.accept(this); } else { - x.getReturningStatement().accept(this); + returningStatement.accept(this); } } else { SQLSelect select = x.getSubQuery(); if (select != null) { select.accept(this); + } else if (expr != null) { + expr.accept(this); } else { - x.getReturningStatement().accept(this); + if (returningStatement != null) { + returningStatement.accept(this); + } } } diff --git a/core/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java b/core/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java index 5bb3b94d5d..84a37d779b 100644 --- a/core/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java +++ b/core/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouseResourceTest.java @@ -39,8 +39,12 @@ public class ClickHouseResourceTest extends OracleTest { exec_test("bvt/parser/clickhouse-0.txt"); } - public void test_1() throws Exception { - exec_test("bvt/parser/clickhouse-1.txt"); + public void test_15() throws Exception { + exec_test("bvt/parser/clickhouse-15.txt"); + } + + public void test_16() throws Exception { + exec_test("bvt/parser/clickhouse-16.txt"); } public void test_2() throws Exception { diff --git a/core/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_310.java b/core/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_310.java index 0dd37cf186..be2567ba3e 100644 --- a/core/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_310.java +++ b/core/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_310.java @@ -19,6 +19,7 @@ import com.alibaba.druid.DbType; import com.alibaba.druid.sql.MysqlTest; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.parser.SQLParserFeature; public class MySqlSelectTest_310 extends MysqlTest { @@ -40,7 +41,7 @@ public class MySqlSelectTest_310 assertEquals("SELECT orders.extras -> '$.case_no' > 1", stmt.toString()); } - public void test_() throws Exception { + public void test_2() throws Exception { String sql = "SELECT orders.extras -> '$.case_no' > 1"; SQLStatement stmt = SQLUtils @@ -48,4 +49,14 @@ public class MySqlSelectTest_310 assertEquals("SELECT orders.extras -> '$.case_no' > 1", stmt.toString()); } + + public void test_3() throws Exception { + String sql = "--abc\nSELECT orders.extras -> '$.case_no' > 1"; + + SQLStatement stmt = SQLUtils + .parseSingleStatement(sql, DbType.mysql, SQLParserFeature.MySQLSupportStandardComment); + + assertEquals("-- abc\n" + + "SELECT orders.extras -> '$.case_no' > 1", stmt.toString()); + } } \ No newline at end of file diff --git a/core/src/test/resources/bvt/parser/clickhouse-15.txt b/core/src/test/resources/bvt/parser/clickhouse-15.txt new file mode 100644 index 0000000000..fd4bb3ba9b --- /dev/null +++ b/core/src/test/resources/bvt/parser/clickhouse-15.txt @@ -0,0 +1,12 @@ +WITH '2019-08-01 15:23:00' as ts_upper_bound +SELECT * +FROM hits +WHERE + EventDate = toDate(ts_upper_bound) AND + EventTime <= ts_upper_bound; +--------------------------- +WITH '2019-08-01 15:23:00' AS ts_upper_bound +SELECT * +FROM hits +WHERE EventDate = toDate(ts_upper_bound) + AND EventTime <= ts_upper_bound; \ No newline at end of file diff --git a/core/src/test/resources/bvt/parser/clickhouse-16.txt b/core/src/test/resources/bvt/parser/clickhouse-16.txt new file mode 100644 index 0000000000..2d84c179a8 --- /dev/null +++ b/core/src/test/resources/bvt/parser/clickhouse-16.txt @@ -0,0 +1,13 @@ +WITH sum(bytes) as s +SELECT + formatReadableSize(s), + table +FROM system.parts +GROUP BY table +ORDER BY s; +--------------------------- +WITH sum(bytes) AS s +SELECT formatReadableSize(s), table +FROM system.parts +GROUP BY table +ORDER BY s; \ No newline at end of file