improved sql parser

2023-03-04 19:17:16 +00:00 · 2022-12-12 19:28:37 +08:00
parent 9366077912
commit f10ba8537c
19 changed files with 231 additions and 34 deletions
@@ -18,7 +18,7 @@ package com.alibaba.druid;
 public final class VERSION {
    public static final int MajorVersion = 1;
    public static final int MinorVersion = 2;
-    public static final int RevisionVersion = 15;
+    public static final int RevisionVersion = 16;

    public static String getVersionNumber() {
        return VERSION.MajorVersion + "." + VERSION.MinorVersion + "." + VERSION.RevisionVersion;
@@ -52,6 +52,9 @@ public class SQLAlterTableStatement extends SQLStatementImpl implements SQLDDLSt
    private boolean ifExists;
    private boolean notClustered;

+    // clickhouse
+    private SQLName on;
+
    public SQLAlterTableStatement() {
    }

@@ -288,4 +291,15 @@ public class SQLAlterTableStatement extends SQLStatementImpl implements SQLDDLSt
    public DDLObjectType getDDLObjectType() {
        return DDLObjectType.TABLE;
    }
+
+    public SQLName getOn() {
+        return on;
+    }
+
+    public void setOn(SQLName x) {
+        if (x != null) {
+            x.setParent(this);
+        }
+        this.on = x;
+    }
 }
@@ -31,7 +31,7 @@ public class ClickhouseCreateTableParser extends SQLCreateTableParser {
            );
        }

-        if (lexer.identifierEquals("PARTITION")) {
+        if (lexer.token() == Token.PARTITION) {
            lexer.nextToken();
            accept(Token.BY);
            SQLExpr expr = this.exprParser.expr();
@@ -29,6 +29,7 @@ public class ClickhouseLexer extends Lexer {
        map.put("SHOW", Token.SHOW);
        map.put("ALL", Token.ALL);
        map.put("GLOBAL", Token.GLOBAL);
+        map.put("PARTITION", Token.PARTITION);

        DEFAULT_KEYWORDS = new Keywords(map);
    }
@@ -0,0 +1,71 @@
+package com.alibaba.druid.sql.dialect.clickhouse.parser;
+
+import com.alibaba.druid.sql.ast.statement.SQLWithSubqueryClause;
+import com.alibaba.druid.sql.parser.*;
+import com.alibaba.druid.util.FnvHash;
+
+public class ClickhouseSelectParser
+        extends SQLSelectParser {
+    public ClickhouseSelectParser(Lexer lexer) {
+        super(lexer);
+    }
+
+    public ClickhouseSelectParser(SQLExprParser exprParser, SQLSelectListCache selectListCache) {
+        super(exprParser, selectListCache);
+    }
+
+    public SQLWithSubqueryClause parseWith() {
+        SQLWithSubqueryClause withQueryClause = new SQLWithSubqueryClause();
+        if (lexer.hasComment() && lexer.isKeepComments()) {
+            withQueryClause.addBeforeComment(lexer.readAndResetComments());
+        }
+
+        accept(Token.WITH);
+
+        if (lexer.token() == Token.RECURSIVE || lexer.identifierEquals(FnvHash.Constants.RECURSIVE)) {
+            lexer.nextToken();
+            withQueryClause.setRecursive(true);
+        }
+
+        for (; ; ) {
+            SQLWithSubqueryClause.Entry entry = new SQLWithSubqueryClause.Entry();
+            entry.setParent(withQueryClause);
+
+            if (lexer.token() == Token.LPAREN) {
+                lexer.nextToken();
+
+                switch (lexer.token()) {
+                    case SELECT:
+                    case LPAREN:
+                    case WITH:
+                    case FROM:
+                        entry.setSubQuery(select());
+                        break;
+                    default:
+                        break;
+                }
+
+                accept(Token.RPAREN);
+            } else {
+                throw new ParserException("TODO");
+            }
+
+            accept(Token.AS);
+
+            String alias = this.lexer.stringVal();
+            lexer.nextToken();
+            entry.setAlias(alias);
+
+            withQueryClause.addEntry(entry);
+
+            if (lexer.token() == Token.COMMA) {
+                lexer.nextToken();
+                continue;
+            }
+
+            break;
+        }
+
+        return withQueryClause;
+    }
+}
@@ -16,6 +16,10 @@ public class ClickhouseStatementParser extends SQLStatementParser {
        super(new ClickhouseExprParser(lexer));
    }

+    public SQLSelectParser createSQLSelectParser() {
+        return new ClickhouseSelectParser(this.exprParser, selectListCache);
+    }
+
    @Override
    public SQLWithSubqueryClause parseWithQuery() {
        SQLWithSubqueryClause withQueryClause = new SQLWithSubqueryClause();
@@ -548,6 +548,7 @@ public class MySqlLexer extends Lexer {
        Token lastToken = this.token;

        if (ch == '-') {
+            boolean supportStandardComment = (features & SQLParserFeature.MySQLSupportStandardComment.mask) != 0;
            /*
             * just for tddl test case;
             * test case : MySqlSelectTest_plus_sub_comment.java
@@ -558,7 +559,10 @@ public class MySqlLexer extends Lexer {
                scanChar();
                token = Token.SUB;
                return;
-            } else if ((before_1 == ' ' || (before_1 != '-' && before_1 != '+')) && (next_2 == ' ' || next_2 == EOI || next_2 == '\n')) {
+            } else if (supportStandardComment
+                    || ((before_1 == ' ' || (before_1 != '-' && before_1 != '+'))
+                    && (next_2 == ' ' || next_2 == EOI || next_2 == '\n'))
+            ) {
                // it is comments
            } else if ((before_1 == '-' || before_1 == '+') && next_2 == ' ') {
                throw new ParserException("illegal state. " + info());
@@ -188,12 +188,6 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit
            print0(ucase ? " SHARDS" : " shards");
        }

-        if (x.getLifecycle() != null) {
-            println();
-            print0(ucase ? "LIFECYCLE " : "lifecycle ");
-            x.getLifecycle().accept(this);
-        }
-
        SQLSelect select = x.getSelect();
        if (select != null) {
            println();
@@ -233,6 +227,12 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit

        this.printTblProperties(x);

+        if (x.getLifecycle() != null) {
+            println();
+            print0(ucase ? "LIFECYCLE " : "lifecycle ");
+            x.getLifecycle().accept(this);
+        }
+
        SQLExpr using = x.getUsing();
        if (using != null) {
            println();
@@ -2573,6 +2573,39 @@ public class Lexer {
        }
    }

+    public List<String> scanLineArgument() {
+        List<String> args = new ArrayList<>();
+        while (ch == ' ') {
+            scanChar();
+        }
+
+        int start = pos;
+        for (;;) {
+            if (ch == ' ' || ch == '\r' || ch == '\n' || ch == EOI) {
+                String arg = text.substring(start, pos);
+                arg = arg.trim();
+                if (arg.length() > 0) {
+                    args.add(arg);
+                }
+                scanChar();
+                start = pos;
+                if (ch == '\r' || ch == '\n' || ch == EOI) {
+                    break;
+                } else {
+                    continue;
+                }
+            }
+
+            scanChar();
+        }
+        if (ch == EOI) {
+            token = EOF;
+        } else {
+            nextToken();
+        }
+        return args;
+    }
+
    private void scanMultiLineComment() {
        Token lastToken = this.token;
        int depth = 1;
@@ -51,9 +51,10 @@ public enum SQLParserFeature {
    EnableMultiUnion,

    Spark,
-    Presto;
+    Presto,
+    MySQLSupportStandardComment;

-    private SQLParserFeature() {
+    SQLParserFeature() {
        mask = (1 << ordinal());
    }

@@ -603,21 +603,17 @@ public class SQLParserUtils {
                lexer.nextTokenForSet();
                token = lexer.token;
                continue;
-            } else if (dbType == DbType.odps && (lexer.identifierEquals("pai") || lexer.identifierEquals("jar"))) {
-                if (lexer.startPos - start > 0) {
-                    int semiIndex = sql.indexOf(';', lexer.startPos);
-                    if (semiIndex != -1) {
-                        lexer.pos = semiIndex - 1;
-                        lexer.nextToken();
-                        token = lexer.token;
-                        continue;
-                    }
-                    String str = sql.substring(start, lexer.startPos).trim();
-                    if (str.isEmpty()) {
-                        lexer.startPos = sql.length();
-                        paiOrJar = true;
-                        break;
-                    }
+            } else if (dbType == DbType.odps
+                    && lexer.ch != '.'
+                    && (lexer.identifierEquals("pai") || lexer.identifierEquals("jar"))) {
+                lexer.scanLineArgument();
+                paiOrJar = true;
+            }
+
+            if (lexer.identifierEquals("USING")) {
+                lexer.nextToken();
+                if (lexer.identifierEquals("jar")) {
+                    lexer.nextToken();
                }
            }

@@ -637,7 +633,11 @@ public class SQLParserUtils {
        }

        if (start != sql.length() && token != Token.SEMI) {
-            String splitSql = sql.substring(start, lexer.startPos).trim();
+            int end = lexer.startPos;
+            if (end > sql.length()) {
+                end = sql.length();
+            }
+            String splitSql = sql.substring(start, end).trim();
            if (!paiOrJar) {
                splitSql = removeComment(splitSql, dbType).trim();
            } else {
@@ -715,6 +715,10 @@ public class SQLParserUtils {
            return sql;
        }

+        if ((sql.startsWith("pai") || sql.startsWith("PAI")) && sql.indexOf(';') == -1) {
+            return sql;
+        }
+
        boolean containsComment = false;
        {
            Lexer lexer = createLexer(sql, dbType);
@@ -1868,6 +1868,12 @@ public class SQLStatementParser extends SQLParser {

            stmt.setName(this.exprParser.name());

+            if (lexer.token == ON && dbType == DbType.clickhouse) {
+                lexer.nextToken();
+                acceptIdentifier("CLUSTER");
+                stmt.setOn(this.exprParser.name());
+            }
+
            for (; ; ) {
                if (lexer.token == Token.DROP) {
                    parseAlterDrop(stmt);
@@ -3198,7 +3204,7 @@ public class SQLStatementParser extends SQLParser {
                    break;
                }
            }
-        } else if (acceptSubQuery && (lexer.token == Token.SELECT || lexer.token == Token.LPAREN)) {
+        } else if (acceptSubQuery && (lexer.token == WITH || lexer.token == Token.SELECT || lexer.token == Token.LPAREN)) {
            SQLSelect select = this.createSQLSelectParser().select();
            insertStatement.setQuery(select);
        } else if (lexer.identifierEquals(FnvHash.Constants.VALUE)) {
@@ -1843,8 +1843,13 @@ class SchemaResolveVisitorFactory {
                if (context != null && alias_hash != 0) {
                    context.addTableSource(alias_hash, entry);
                }
+            } else if (entry.getExpr() != null) {
+                entry.getExpr().accept(visitor);
            } else {
-                entry.getReturningStatement().accept(visitor);
+                SQLStatement returningStatement = entry.getReturningStatement();
+                if (returningStatement != null) {
+                    returningStatement.accept(visitor);
+                }
            }
        }
    }
@@ -5561,6 +5561,13 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
        }

        printTableSourceExpr(x.getName());
+
+        SQLName on = x.getOn();
+        if (on != null) {
+            print(ucase ? " ON CLUSTER " : " on cluster ");
+            printExpr(on);
+        }
+
        this.indentCount++;
        for (int i = 0; i < x.getItems().size(); ++i) {
            SQLAlterTableItem item = x.getItems().get(i);
@@ -1832,19 +1832,26 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter {
        String alias = x.getAlias();
        SQLWithSubqueryClause with = (SQLWithSubqueryClause) x.getParent();

+        SQLStatement returningStatement = x.getReturningStatement();
+        SQLExpr expr = x.getExpr();
+
        if (Boolean.TRUE == with.getRecursive()) {
            SQLSelect select = x.getSubQuery();
            if (select != null) {
                select.accept(this);
            } else {
-                x.getReturningStatement().accept(this);
+                returningStatement.accept(this);
            }
        } else {
            SQLSelect select = x.getSubQuery();
            if (select != null) {
                select.accept(this);
+            } else if (expr != null) {
+                expr.accept(this);
            } else {
-                x.getReturningStatement().accept(this);
+                if (returningStatement != null) {
+                    returningStatement.accept(this);
+                }
            }
        }

@@ -39,8 +39,12 @@ public class ClickHouseResourceTest extends OracleTest {
        exec_test("bvt/parser/clickhouse-0.txt");
    }

-    public void test_1() throws Exception {
-        exec_test("bvt/parser/clickhouse-1.txt");
+    public void test_15() throws Exception {
+        exec_test("bvt/parser/clickhouse-15.txt");
+    }
+
+    public void test_16() throws Exception {
+        exec_test("bvt/parser/clickhouse-16.txt");
    }

    public void test_2() throws Exception {
@@ -19,6 +19,7 @@ import com.alibaba.druid.DbType;
 import com.alibaba.druid.sql.MysqlTest;
 import com.alibaba.druid.sql.SQLUtils;
 import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.parser.SQLParserFeature;

 public class MySqlSelectTest_310
        extends MysqlTest {
@@ -40,7 +41,7 @@ public class MySqlSelectTest_310
        assertEquals("SELECT orders.extras -> '$.case_no' > 1", stmt.toString());
    }

-    public void test_() throws Exception {
+    public void test_2() throws Exception {
        String sql = "SELECT orders.extras -> '$.case_no' > 1";

        SQLStatement stmt = SQLUtils
@@ -48,4 +49,14 @@ public class MySqlSelectTest_310

        assertEquals("SELECT orders.extras -> '$.case_no' > 1", stmt.toString());
    }
+
+    public void test_3() throws Exception {
+        String sql = "--abc\nSELECT orders.extras -> '$.case_no' > 1";
+
+        SQLStatement stmt = SQLUtils
+                .parseSingleStatement(sql, DbType.mysql, SQLParserFeature.MySQLSupportStandardComment);
+
+        assertEquals("-- abc\n" +
+                "SELECT orders.extras -> '$.case_no' > 1", stmt.toString());
+    }
 }
@@ -0,0 +1,12 @@
+WITH '2019-08-01 15:23:00' as ts_upper_bound
+SELECT *
+FROM hits
+WHERE
+    EventDate = toDate(ts_upper_bound) AND
+    EventTime <= ts_upper_bound;
+---------------------------
+WITH '2019-08-01 15:23:00' AS ts_upper_bound
+SELECT *
+FROM hits
+WHERE EventDate = toDate(ts_upper_bound)
+	AND EventTime <= ts_upper_bound;
@@ -0,0 +1,13 @@
+WITH sum(bytes) as s
+SELECT
+    formatReadableSize(s),
+    table
+FROM system.parts
+GROUP BY table
+ORDER BY s;
+---------------------------
+WITH sum(bytes) AS s
+SELECT formatReadableSize(s), table
+FROM system.parts
+GROUP BY table
+ORDER BY s;