From 7467aa9d7930d7152d3a47722bfbea417ff36beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Sun, 4 Jul 2021 20:29:10 +0800 Subject: [PATCH] improved odps parser --- druid-spring-boot-starter/pom.xml | 5 +- pom.xml | 14 ++- .../sql/ast/statement/SQLUnionOperator.java | 1 + .../odps/ast/OdpsAddTableStatement.java | 21 ++++- .../odps/parser/OdpsCreateTableParser.java | 8 ++ .../dialect/odps/parser/OdpsExprParser.java | 3 +- .../dialect/odps/parser/OdpsSelectParser.java | 2 +- .../odps/parser/OdpsStatementParser.java | 33 ++++++- .../dialect/odps/visitor/OdpsASTVisitor.java | 8 ++ .../odps/visitor/OdpsOutputVisitor.java | 25 +++++- .../alibaba/druid/sql/parser/CharTypes.java | 3 + .../com/alibaba/druid/sql/parser/Lexer.java | 6 ++ .../druid/sql/parser/SQLExprParser.java | 86 ++++++++++++++++++- .../alibaba/druid/sql/parser/SQLParser.java | 22 +++++ .../druid/sql/parser/SQLSelectParser.java | 25 +++++- .../druid/sql/parser/SQLStatementParser.java | 3 +- 16 files changed, 250 insertions(+), 15 deletions(-) diff --git a/druid-spring-boot-starter/pom.xml b/druid-spring-boot-starter/pom.xml index d2b3daffd3..854b72e8bf 100644 --- a/druid-spring-boot-starter/pom.xml +++ b/druid-spring-boot-starter/pom.xml @@ -4,14 +4,14 @@ 4.0.0 com.alibaba druid-spring-boot-starter - 1.2.5 + 1.2.6 jar druid-spring-boot-starter Spring Boot with Druid support,help you simplify Druid config in Spring Boot. https://github.com/alibaba/druid - 1.2.5 + 1.2.6 1.5.12.RELEASE @@ -23,7 +23,6 @@ - diff --git a/pom.xml b/pom.xml index a2ec6d6295..4e700b8e3e 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.alibaba druid - 1.2.5 + 1.2.7_preview_04 jar druid A JDBC datasource implementation. @@ -15,7 +15,7 @@ 4.13.1 1.19 - false + true true UTF-8 @@ -464,6 +464,13 @@ provided + + io.trino + trino-jdbc + 352 + provided + + org.quartz-scheduler quartz @@ -521,7 +528,7 @@ com.google.guava guava - 29.0-jre + 30.1-jre provided @@ -715,5 +722,6 @@ 4.2.0 test + diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionOperator.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionOperator.java index 85ccb7d143..f6f78d5838 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionOperator.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionOperator.java @@ -20,6 +20,7 @@ public enum SQLUnionOperator { UNION_ALL("UNION ALL"), MINUS("MINUS"), MINUS_DISTINCT("MINUS DISTINCT"), + MINUS_ALL("MINUS ALL"), EXCEPT("EXCEPT"), EXCEPT_ALL("EXCEPT ALL"), EXCEPT_DISTINCT("EXCEPT DISTINCT"), diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsAddTableStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsAddTableStatement.java index eab2f8f2bf..99656a4f1a 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsAddTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsAddTableStatement.java @@ -22,6 +22,7 @@ import com.alibaba.druid.sql.ast.SQLPartitionValue; import com.alibaba.druid.sql.ast.statement.SQLAlterStatement; import com.alibaba.druid.sql.ast.statement.SQLAssignItem; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; +import com.alibaba.druid.sql.ast.statement.SQLPrivilegeItem; import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor; import java.util.ArrayList; @@ -30,11 +31,13 @@ import java.util.List; public class OdpsAddTableStatement extends OdpsStatementImpl implements SQLAlterStatement { private SQLExprTableSource table; - private final List partitoins = new ArrayList(); protected SQLExpr comment; protected boolean force; + protected final List privileges = new ArrayList(); + protected SQLName toPackage; + public OdpsAddTableStatement() { super.dbType = DbType.odps; } @@ -85,4 +88,20 @@ public class OdpsAddTableStatement extends OdpsStatementImpl implements SQLAlter public List getPartitoins() { return partitoins; } + + public SQLName getToPackage() { + return toPackage; + } + + public void setToPackage(SQLName x) { + if (x != null) { + x.setParent(this); + } + + this.toPackage = x; + } + + public List getPrivileges() { + return privileges; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java index cbb2a804b7..6ad2f6c8f7 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java @@ -143,6 +143,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { case CLOSE: case IN: case OUT: + case INOUT: case LIMIT: case FULL: case MINUS: @@ -150,6 +151,13 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { case TRIGGER: case USE: case LIKE: + case DISTRIBUTE: + case DELETE: + case UPDATE: + case IS: + case LEFT: + case RIGHT: + case REPEAT: column = this.exprParser.parseColumn(stmt); break; default: diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java index 610d6f14df..33c4ea0aa1 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java @@ -45,7 +45,8 @@ public class OdpsExprParser extends SQLExprParser { "STDDEV", // "SUM", // "ROW_NUMBER", - "WM_CONCAT"// + "WM_CONCAT", + "COLLECT_LIST"// }; AGGREGATE_FUNCTIONS_CODES = FnvHash.fnv1a_64_lower(strings, true); AGGREGATE_FUNCTIONS = new String[AGGREGATE_FUNCTIONS_CODES.length]; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java index dc132e8d54..517a2a10bd 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java @@ -115,7 +115,7 @@ public class OdpsSelectParser extends SQLSelectParser { } else if (lexer.token() == Token.ALL) { Lexer.SavePoint mark = lexer.mark(); lexer.nextToken(); - if (lexer.token() == Token.DOT) { + if (lexer.token() == Token.DOT || lexer.token() == Token.COMMA) { lexer.reset(mark); } else { queryBlock.setDistionOption(SQLSetQuantifier.ALL); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java index 66e4b5930d..8820361dfe 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java @@ -129,6 +129,20 @@ public class OdpsStatementParser extends SQLStatementParser { acceptIdentifier("f"); stmt.setForce(true); } + + if (lexer.token() == Token.TO) { + lexer.nextToken(); + acceptIdentifier("PACKAGE"); + SQLName packageName = this.exprParser.name(); + stmt.setToPackage(packageName); + + if (lexer.token() == Token.WITH) { + lexer.nextToken(); + acceptIdentifier("PRIVILEGES"); + parsePrivileages(stmt.getPrivileges(), stmt); + } + } + statementList.add(stmt); return true; } @@ -290,6 +304,17 @@ public class OdpsStatementParser extends SQLStatementParser { return true; } + if (lexer.identifierEquals("INSTALL")) { + lexer.nextToken(); + acceptIdentifier("PACKAGE"); + OdpsInstallPackageStatement stmt = new OdpsInstallPackageStatement(); + stmt.setPackageName( + this.exprParser.name() + ); + statementList.add(stmt); + return true; + } + if (lexer.identifierEquals(FnvHash.Constants.KILL)) { SQLStatement stmt = parseKill(); statementList.add(stmt); @@ -387,7 +412,13 @@ public class OdpsStatementParser extends SQLStatementParser { HiveMultiInsertStatement stmt = new HiveMultiInsertStatement(); if (lexer.token() == Token.IDENTIFIER) { - SQLName tableName = this.exprParser.name(); + Lexer.SavePoint mark = lexer.mark(); + SQLExpr tableName = this.exprParser.name(); + if (lexer.token() == Token.LPAREN) { + lexer.reset(mark); + tableName = this.exprParser.primary(); + } + SQLTableSource from = new SQLExprTableSource(tableName); if (lexer.token() == Token.IDENTIFIER) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java index e808f35fdd..c5d19c06a9 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java @@ -240,4 +240,12 @@ public interface OdpsASTVisitor extends HiveASTVisitor { } + default boolean visit(OdpsInstallPackageStatement x) { + return true; + } + + default void endVisit(OdpsInstallPackageStatement x) { + + } + } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index 1845ad9a07..b6761cbdd6 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -668,7 +668,12 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit public boolean visit(SQLDataType x) { String dataTypeName = x.getName(); - print0(ucase ? dataTypeName.toUpperCase() : dataTypeName.toLowerCase()); + if (dataTypeName.indexOf('<') != -1 || dataTypeName.equals("Object")) { + print0(dataTypeName); + } else { + print0(ucase ? dataTypeName.toUpperCase() : dataTypeName.toLowerCase()); + } + if (x.getArguments().size() > 0) { print('('); printAndAccept(x.getArguments(), ", "); @@ -850,6 +855,18 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit print0(" -f"); } + SQLName toPackage = x.getToPackage(); + if (toPackage != null) { + print0(ucase ? " TO PACKAGE " : " to package "); + printExpr(toPackage); + + List privileges = x.getPrivileges(); + if (!privileges.isEmpty()) { + print0(ucase ? " WITH PRIVILEGES " : " with privileges "); + printAndAccept(privileges, ", "); + } + } + return false; } @@ -1052,4 +1069,10 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit super.visit((SQLMethodInvokeExpr) x); return false; } + + public boolean visit(OdpsInstallPackageStatement x) { + print0(ucase ? "INSTALL PACKAGE " : "install package "); + printExpr(x.getPackageName()); + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java b/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java index 2a2fd0e992..fd104aa71f 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java +++ b/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java @@ -112,6 +112,9 @@ public class CharTypes { } whitespaceFlags[160] = true; // 特别处理 + whitespaceFlags[223] = true; // 特别处理, odps + whitespaceFlags[229] = true; // 特别处理, odps + whitespaceFlags[231] = true; // 特别处理, odps } /** diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index e7f392232e..235cdb3ba7 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -568,6 +568,12 @@ public class Lexer { return; } + if (ch == '(') { + scanChar(); + token = Token.LPAREN; + return; + } + if (ch == '$' && isVaraintChar(charAt(pos + 1))) { scanVariable(); return; diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index f0eb5eed83..9d414f20b7 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -97,7 +97,7 @@ public class SQLExprParser extends SQLParser { Token token = lexer.token; if (token == Token.COMMA) { return expr; - } else if (token == Token.EQ) { + } else if (token == Token.EQ || token == Token.EQEQ) { expr = relationalRest(expr); expr = andRest(expr); expr = xorRest(expr); @@ -701,6 +701,7 @@ public class SQLExprParser extends SQLParser { case TOP: case SHOW: case INOUT: + case OUTER: sqlExpr = new SQLIdentifierExpr(lexer.stringVal()); lexer.nextToken(); break; @@ -873,6 +874,7 @@ public class SQLExprParser extends SQLParser { case INTERVAL: case LBRACE: case IF: + case CHECK: sqlExpr = primary(); while (lexer.token == Token.HINT) { @@ -1099,12 +1101,30 @@ public class SQLExprParser extends SQLParser { case DIV: case DISTRIBUTE: case UNIQUE: + case PROCEDURE: if (dbType == DbType.odps || dbType == DbType.hive) { sqlExpr = new SQLIdentifierExpr(lexer.stringVal()); lexer.nextToken(); break; } throw new ParserException("ERROR. " + lexer.info()); + case AS: { + if (dbType == DbType.odps) { + Lexer.SavePoint mark = lexer.mark(); + String str = lexer.stringVal(); + lexer.nextToken(); + switch (lexer.token) { + case COMMA: + sqlExpr = new SQLIdentifierExpr(str); + lexer.nextToken(); + break; + default: + lexer.reset(mark); + break; + } + } + break; + } case DISTINCT: if (dbType == DbType.elastic_search || dbType == DbType.mysql) { Lexer.SavePoint mark = lexer.mark(); @@ -1120,7 +1140,22 @@ public class SQLExprParser extends SQLParser { throw new ParserException("ERROR. " + lexer.info()); case IN: if (dbType == DbType.odps) { + String str = lexer.stringVal(); lexer.nextToken(); + switch (lexer.token) { + case DOT: + case LT: + case EQ: + case GT: + sqlExpr = new SQLIdentifierExpr(str); + break; + default: + break; + } + if (sqlExpr != null) { + break; + } + accept(Token.LPAREN); SQLInListExpr in = new SQLInListExpr(); in.setExpr( @@ -1189,7 +1224,15 @@ public class SQLExprParser extends SQLParser { case PLUS: case SUB: case RPAREN: + case WHERE: + case GROUP: + case SEMI: return new SQLIdentifierExpr(str); + case IDENTIFIER: + if (dbType == DbType.odps) { + return new SQLIdentifierExpr(str); + } + break; default: break; } @@ -1289,8 +1332,21 @@ public class SQLExprParser extends SQLParser { break; } + Lexer.SavePoint mark = lexer.mark(); + SQLExpr value = expr(); + switch (lexer.token) { + case COMMA: + case RPAREN: + case WHERE: + case FROM: + lexer.reset(mark); + return new SQLIdentifierExpr(str); + default: + break; + } + if (lexer.token() != Token.IDENTIFIER) { throw new ParserException("Syntax error. " + lexer.info()); } @@ -2065,6 +2121,8 @@ public class SQLExprParser extends SQLParser { case SHOW: case FOR: case LEAVE: + case REPEAT: + case IS: identName = lexer.stringVal(); lexer.nextToken(); break; @@ -2077,6 +2135,13 @@ public class SQLExprParser extends SQLParser { case TRIGGER: case USE: case LIKE: + case DISTRIBUTE: + case DELETE: + case UPDATE: + case PROCEDURE: + case LEFT: + case RIGHT: + case TABLE: if (dbType == DbType.odps) { identName = lexer.stringVal(); lexer.nextToken(); @@ -3474,6 +3539,7 @@ public class SQLExprParser extends SQLParser { case BETWEEN: case IS: case EQ: + case EQEQ: case IN: case CONTAINS: case BANG_TILDE_STAR: @@ -3928,6 +3994,24 @@ public class SQLExprParser extends SQLParser { lexer.nextToken(); } + if (lexer.token == Token.LT && dbType == DbType.odps) { + lexer.nextToken(); + typeName += '<'; + for (;;) { + SQLDataType itemType = this.parseDataType(); + typeName += itemType.toString(); + if (lexer.token == Token.COMMA) { + lexer.nextToken(); + typeName += ", "; + continue; + } else { + break; + } + } + accept(Token.GT); + typeName += '>'; + } + SQLDataType dataType = new SQLDataTypeImpl(typeName); dataType.setDbType(dbType); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java index 2121a6e0b2..7a962d7c80 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java @@ -201,12 +201,17 @@ public class SQLParser { return strVal; } } + case OUTER: case IN: { Lexer.SavePoint mark = lexer.mark(); String strVal = lexer.stringVal(); lexer.nextToken(); switch (lexer.token) { case WHERE: + case GROUP: + case LEFT: + case RIGHT: + case FULL: case RPAREN: { return strVal; } @@ -221,6 +226,9 @@ public class SQLParser { case CHECK: case LEAVE: case TRIGGER: + case CREATE: + case ASC: + case DESC: if (dbType == DbType.odps || dbType == DbType.hive) { String strVal = lexer.stringVal(); lexer.nextToken(); @@ -250,6 +258,17 @@ public class SQLParser { lexer.nextToken(); return strVal; } + case DISTRIBUTE: { + String strVal = lexer.stringVal(); + Lexer.SavePoint mark = lexer.mark(); + lexer.nextToken(); + if (lexer.token == Token.BY) { + lexer.reset(mark); + } else { + return strVal; + } + break; + } default: break; } @@ -334,6 +353,8 @@ public class SQLParser { case KILL: case COMMENT: case TABLESPACE: + case REPEAT: + case PRIMARY: alias = lexer.stringVal(); lexer.nextToken(); break; @@ -529,6 +550,7 @@ public class SQLParser { case TABLESPACE: case CREATE: case DELETE: + case PRIMARY: alias = lexer.stringVal(); lexer.nextToken(); return alias; diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java index b05c122e24..ccc4d8a8d1 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java @@ -339,6 +339,9 @@ public class SQLSelectParser extends SQLParser { if (lexer.token == Token.DISTINCT) { union.setOperator(SQLUnionOperator.MINUS_DISTINCT); lexer.nextToken(); + } else if (lexer.token == Token.ALL) { + union.setOperator(SQLUnionOperator.MINUS_ALL); + lexer.nextToken(); } SQLSelectQuery right = this.query(union, false); @@ -806,9 +809,14 @@ public class SQLSelectParser extends SQLParser { } if (lexer.token == Token.ALL) { + Lexer.SavePoint mark = lexer.mark(); lexer.nextToken(); if (!lexer.identifierEquals(FnvHash.Constants.GROUPING)) { - throw new ParserException("group by all syntax error. " + lexer.info()); + if (dbType == DbType.odps) { + lexer.reset(mark); + } else { + throw new ParserException("group by all syntax error. " + lexer.info()); + } } } else if (lexer.token == Token.DISTINCT) { lexer.nextToken(); @@ -1027,7 +1035,8 @@ public class SQLSelectParser extends SQLParser { } else if (lexer.token == Token.LPAREN) { tableSource = parseTableSource(); - while (lexer.token == Token.UNION && tableSource instanceof SQLUnionQueryTableSource) { + while ((lexer.token == Token.UNION || lexer.token == Token.EXCEPT || lexer.token == Token.INTERSECT) + && tableSource instanceof SQLUnionQueryTableSource) { SQLUnionQueryTableSource unionQueryTableSource = (SQLUnionQueryTableSource) tableSource; SQLUnionQuery union = unionQueryTableSource.getUnion(); unionQueryTableSource.setUnion( @@ -1225,6 +1234,18 @@ public class SQLSelectParser extends SQLParser { asof = true; } + if (lexer.token == Token.OUTER) { + Lexer.SavePoint mark = lexer.mark(); + String str = lexer.stringVal(); + lexer.nextToken(); + if (tableSource.getAlias() == null && + !lexer.identifierEquals(FnvHash.Constants.APPLY)) { + tableSource.setAlias(str); + } else { + lexer.reset(mark); + } + } + switch (lexer.token) { case LEFT: lexer.nextToken(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index 64d0446dad..120a633b3f 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -4342,10 +4342,11 @@ public class SQLStatementParser extends SQLParser { lexer.nextToken(); for (;;) { - if (lexer.token == Token.CONSTRAINT) { SQLTableConstraint constraint = (SQLTableConstraint) this.exprParser.parseConstaint(); createView.addColumn(constraint); + } else if (lexer.token == Token.RPAREN) { + break; } else { SQLColumnDefinition column = new SQLColumnDefinition(); column.setDbType(dbType);