From 00cc7dba7c4def11dd8cf3f71082d54a6a57fe4f Mon Sep 17 00:00:00 2001 From: "shaojin.wensj" Date: Sun, 12 Jun 2022 05:30:36 +0800 Subject: [PATCH] improve sql parser --- src/main/java/com/alibaba/druid/VERSION.java | 2 +- .../statement/MySqlCreateTableStatement.java | 14 ++++ .../mysql/parser/MySqlCreateTableParser.java | 22 ++++++ .../dialect/mysql/parser/MySqlExprParser.java | 10 ++- .../druid/sql/parser/SQLParserUtils.java | 79 +++++++++++++++++-- .../sql/postgresql/select/PGSelectTest86.java | 21 +++++ 6 files changed, 140 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/alibaba/druid/VERSION.java b/src/main/java/com/alibaba/druid/VERSION.java index 90120b66d0..06292631b1 100644 --- a/src/main/java/com/alibaba/druid/VERSION.java +++ b/src/main/java/com/alibaba/druid/VERSION.java @@ -19,7 +19,7 @@ public final class VERSION { public final static int MajorVersion = 1; public final static int MinorVersion = 2; - public final static int RevisionVersion = 9; + public final static int RevisionVersion = 10; public static String getVersionNumber() { return VERSION.MajorVersion + "." + VERSION.MinorVersion + "." + VERSION.RevisionVersion; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlCreateTableStatement.java index 4152d9ff20..8c77d4b007 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlCreateTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlCreateTableStatement.java @@ -665,4 +665,18 @@ public class MySqlCreateTableStatement extends SQLCreateTableStatement implement } addOption("ENGINE", x); } + + public void setPageChecksum(SQLExpr x) { + if (x != null) { + x.setParent(this); + } + addOption("PAGE_CHECKSUM", x); + } + + public void setTransactional(SQLExpr x) { + if (x != null) { + x.setParent(this); + } + addOption("TRANSACTIONAL", x); + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java index 515e4f6f5e..ab7875a3ed 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java @@ -396,6 +396,28 @@ public class MySqlCreateTableParser extends SQLCreateTableParser { continue; } + if (lexer.identifierEquals("PAGE_CHECKSUM")) { + lexer.nextToken(); + if (lexer.token() == Token.EQ) { + lexer.nextToken(); + } + + SQLExpr expr = this.exprParser.expr(); + stmt.setPageChecksum(expr); + continue; + } + + if (lexer.identifierEquals("TRANSACTIONAL")) { + lexer.nextToken(); + if (lexer.token() == Token.EQ) { + lexer.nextToken(); + } + + SQLExpr expr = this.exprParser.expr(); + stmt.setTransactional(expr); + continue; + } + if (lexer.identifierEquals(FnvHash.Constants.BLOCK_SIZE)) { lexer.nextToken(); if (lexer.token() == Token.EQ) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java index 7e611581bd..299c51262e 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java @@ -1686,7 +1686,15 @@ public class MySqlExprParser extends SQLExprParser { SQLPartition partitionDef = new SQLPartition(); - partitionDef.setName(this.name()); + SQLName name; + if (lexer.token() == Token.LITERAL_INT) { + Number number = lexer.integerValue(); + name = new SQLIdentifierExpr(number.toString()); + lexer.nextToken(); + } else { + name = this.name(); + } + partitionDef.setName(name); SQLPartitionValue values = this.parsePartitionValues(); if (values != null) { diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java index b8811a7819..6cb84f4709 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java @@ -527,7 +527,7 @@ public class SQLParserUtils { lexer.config(SQLParserFeature.SkipComments, false); lexer.config(SQLParserFeature.KeepComments, true); - boolean set = false; + boolean set = false, paiOrJar = false; int start = 0; Token token = lexer.token; for (;lexer.token != Token.EOF;) { @@ -535,7 +535,9 @@ public class SQLParserUtils { int len = lexer.startPos - start; if (len > 0) { String lineSql = sql.substring(start, lexer.startPos); - String splitSql = set ? lineSql.trim() : removeComment(lineSql, dbType + String splitSql = set + ? removeLeftComment(lineSql, dbType) + : removeComment(lineSql, dbType ).trim(); if (!splitSql.isEmpty()) { list.add(splitSql); @@ -569,6 +571,15 @@ public class SQLParserUtils { lexer.nextTokenForSet(); token = lexer.token; continue; + } else if (lexer.identifierEquals("pai") || lexer.identifierEquals("jar")) { + if (lexer.startPos - start > 0) { + String str = sql.substring(start, lexer.startPos).trim(); + if (str.isEmpty()) { + lexer.startPos = sql.length(); + paiOrJar = true; + break; + } + } } if (lexer.token == Token.SET) { @@ -580,10 +591,14 @@ public class SQLParserUtils { } if (start != sql.length() && token != Token.SEMI) { - String splitSql = removeComment( - sql.substring(start, lexer.startPos) - , dbType - ).trim(); + String splitSql = sql.substring(start, lexer.startPos).trim(); + if (!paiOrJar) { + splitSql = removeComment(splitSql, dbType).trim(); + } else { + if (splitSql.endsWith(";")) { + splitSql = splitSql.substring(0, splitSql.length() - 1).trim(); + } + } if (!splitSql.isEmpty()) { list.add(splitSql); } @@ -592,6 +607,58 @@ public class SQLParserUtils { return list; } + public static String removeLeftComment(String sql, DbType dbType) { + if (dbType == null) { + dbType = DbType.other; + } + + sql = sql.trim(); + if (sql.startsWith("jar")) { + return sql; + } + + boolean containsComment = false; + { + Lexer lexer = createLexer(sql, dbType); + lexer.config(SQLParserFeature.SkipComments, false); + lexer.config(SQLParserFeature.KeepComments, true); + + while (lexer.token != Token.EOF) { + if (lexer.token == Token.LINE_COMMENT || lexer.token == Token.MULTI_LINE_COMMENT) { + containsComment = true; + break; + } + lexer.nextToken(); + } + + if (!containsComment) { + return sql; + } + } + + StringBuilder sb = new StringBuilder(); + + Lexer lexer = createLexer(sql, dbType); + lexer.config(SQLParserFeature.SkipComments, false); + lexer.config(SQLParserFeature.KeepComments, true); + lexer.nextToken(); + + int start = 0; + for (; lexer.token != Token.EOF; lexer.nextToken()) { + if (lexer.token == Token.LINE_COMMENT || lexer.token == Token.MULTI_LINE_COMMENT) { + continue; + } + start = lexer.startPos; + break; + } + + if (start != sql.length()) { + sb.append(sql.substring(start, sql.length())); + } + + return sb.toString(); + } + public static String removeComment(String sql, DbType dbType) { if (dbType == null) { dbType = DbType.other; diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest86.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest86.java index 02146ba150..dcf516c8cb 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest86.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest86.java @@ -25,4 +25,25 @@ public class PGSelectTest86 "from xxx_point point\n" + "\tinner join xxx_cc cc on point.id = cc.point_id", stmt.toLowerCaseString()); } + + public void test_1() throws Exception { + String sql = "select\n" + + " COUNT(1) res\n" + + " from TB_EXPERIENCE exp\n" + + " where exp.creatdate>DATE_SUB(CURDATE(), interval 1 MONTH)\n" + + " and exp.cstatus = 1"; + + final List statements = SQLUtils.parseStatements(sql, JdbcConstants.POSTGRESQL); + assertEquals(1, statements.size()); + final SQLStatement stmt = statements.get(0); + + assertEquals("SELECT *\n" + + "FROM xxx_point point\n" + + "\tINNER JOIN xxx_cc cc ON point.id = cc.point_id" + , stmt.toString()); + + assertEquals("select *\n" + + "from xxx_point point\n" + + "\tinner join xxx_cc cc on point.id = cc.point_id", stmt.toLowerCaseString()); + } }