From 094f0921da497cd40346cc7fbd7225d3dcb24549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Thu, 2 Sep 2021 22:01:51 +0800 Subject: [PATCH] improved odps support --- .../java/com/alibaba/druid/sql/SQLUtils.java | 4 - .../SQLAlterTableMergePartition.java | 9 + .../ast/statement/SQLAlterTableStatement.java | 9 + .../ast/statement/SQLCreateViewStatement.java | 13 ++ .../statement/SQLExternalRecordFormat.java | 10 +- .../sql/ast/statement/SQLJoinTableSource.java | 5 + .../ast/statement/SQLShowStatisticStmt.java | 6 + .../statement/SQLUnionQueryTableSource.java | 18 +- .../sql/ast/statement/SQLUpdateStatement.java | 10 + .../stmt/HiveCreateFunctionStatement.java | 14 +- .../hive/visitor/HiveOutputVisitor.java | 35 ++- .../hive/visitor/HiveSchemaStatVisitor.java | 2 +- .../dialect/odps/ast/OdpsCountStatement.java | 7 +- .../sql/dialect/odps/ast/OdpsNewExpr.java | 9 + .../odps/parser/OdpsCreateTableParser.java | 17 ++ .../dialect/odps/parser/OdpsExprParser.java | 22 +- .../sql/dialect/odps/parser/OdpsLexer.java | 13 +- .../dialect/odps/parser/OdpsSelectParser.java | 8 +- .../odps/parser/OdpsStatementParser.java | 217 +++++++++++++++++- .../dialect/odps/visitor/OdpsASTVisitor.java | 16 ++ .../odps/visitor/OdpsOutputVisitor.java | 5 +- .../com/alibaba/druid/sql/parser/Lexer.java | 27 ++- .../druid/sql/parser/SQLExprParser.java | 103 ++++++++- .../alibaba/druid/sql/parser/SQLParser.java | 116 +++++++++- .../druid/sql/parser/SQLSelectParser.java | 76 +++++- .../druid/sql/parser/SQLStatementParser.java | 95 +++++++- .../sql/visitor/SQLASTOutputVisitor.java | 34 ++- .../druid/sql/visitor/SQLASTVisitor.java | 25 ++ .../druid/sql/visitor/SchemaStatVisitor.java | 10 +- .../java/com/alibaba/druid/util/FnvHash.java | 1 + 30 files changed, 853 insertions(+), 83 deletions(-) diff --git a/src/main/java/com/alibaba/druid/sql/SQLUtils.java b/src/main/java/com/alibaba/druid/sql/SQLUtils.java index 8868f4bd8f..cf6939c624 100644 --- a/src/main/java/com/alibaba/druid/sql/SQLUtils.java +++ b/src/main/java/com/alibaba/druid/sql/SQLUtils.java @@ -267,10 +267,6 @@ public class SQLUtils { } public static SQLExpr toSQLExpr(String sql, DbType dbType) { - if (sql.indexOf("${") != -1 && dbType == DbType.odps) { - return new SQLIdentifierExpr(sql); - } - SQLExprParser parser = SQLParserUtils.createExprParser(sql, dbType); SQLExpr expr = parser.expr(); diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableMergePartition.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableMergePartition.java index 180356de4e..9901cc6bf7 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableMergePartition.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableMergePartition.java @@ -27,6 +27,7 @@ public class SQLAlterTableMergePartition extends SQLObjectImpl implements SQLAlt private final List partitions = new ArrayList(4); private boolean ifExists = false; private SQLPartitionSpec overwritePartition; + private boolean purge; public List getPartitions() { return partitions; @@ -65,4 +66,12 @@ public class SQLAlterTableMergePartition extends SQLObjectImpl implements SQLAlt } this.overwritePartition = x; } + + public boolean isPurge() { + return purge; + } + + public void setPurge(boolean purge) { + this.purge = purge; + } } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java index f206cba167..967e08fbe0 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableStatement.java @@ -54,6 +54,7 @@ public class SQLAlterTableStatement extends SQLStatementImpl implements SQLDDLSt protected int shards; private boolean ifExists = false; + private boolean notClustered = false; public SQLAlterTableStatement(){ @@ -267,4 +268,12 @@ public class SQLAlterTableStatement extends SQLStatementImpl implements SQLDDLSt public void setShards(int shards) { this.shards = shards; } + + public boolean isNotClustered() { + return notClustered; + } + + public void setNotClustered(boolean notClustered) { + this.notClustered = notClustered; + } } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateViewStatement.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateViewStatement.java index 7ed709ca83..7631c1641a 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateViewStatement.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateViewStatement.java @@ -54,6 +54,8 @@ public class SQLCreateViewStatement extends SQLStatementImpl implements SQLCreat protected boolean onCluster; private SQLName to; + private SQLBlockStatement script; + public SQLCreateViewStatement(){ } @@ -354,6 +356,17 @@ public class SQLCreateViewStatement extends SQLStatementImpl implements SQLCreat this.returnsDataType = x; } + public SQLBlockStatement getScript() { + return script; + } + + public void setScript(SQLBlockStatement x) { + if (x != null) { + x.setParent(this); + } + this.script = x; + } + public SQLCreateViewStatement clone() { SQLCreateViewStatement x = new SQLCreateViewStatement(); diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLExternalRecordFormat.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLExternalRecordFormat.java index 815b348f5b..ae7ebdbe6c 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLExternalRecordFormat.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLExternalRecordFormat.java @@ -2,10 +2,11 @@ package com.alibaba.druid.sql.ast.statement; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObjectImpl; -import com.alibaba.druid.sql.dialect.oracle.ast.OracleSQLObjectImpl; -import com.alibaba.druid.sql.dialect.oracle.visitor.OracleASTVisitor; import com.alibaba.druid.sql.visitor.SQLASTVisitor; +import java.util.ArrayList; +import java.util.List; + public class SQLExternalRecordFormat extends SQLObjectImpl { private SQLExpr delimitedBy; private SQLExpr terminatedBy; @@ -15,6 +16,7 @@ public class SQLExternalRecordFormat extends SQLObjectImpl { private SQLExpr linesTerminatedBy; private SQLExpr nullDefinedAs; private SQLExpr serde; + protected final List serdeProperties = new ArrayList(); private Boolean logfile; private Boolean badfile; @@ -165,6 +167,10 @@ public class SQLExternalRecordFormat extends SQLObjectImpl { this.missingFieldValuesAreNull = missingFieldValuesAreNull; } + public List getSerdeProperties() { + return serdeProperties; + } + public SQLExternalRecordFormat clone() { SQLExternalRecordFormat x = new SQLExternalRecordFormat(); diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLJoinTableSource.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLJoinTableSource.java index e9828b4507..9b3eb37f42 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLJoinTableSource.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLJoinTableSource.java @@ -712,6 +712,7 @@ public class SQLJoinTableSource extends SQLTableSourceImpl implements SQLReplace protected final List arguments = new ArrayList(); protected String alias; protected final List columns = new ArrayList(); + protected List sortBy = new ArrayList(); public UDJ() { @@ -726,6 +727,10 @@ public class SQLJoinTableSource extends SQLTableSourceImpl implements SQLReplace v.endVisit(this); } + public List getSortBy() { + return sortBy; + } + public UDJ clone() { UDJ x = new UDJ(); x.function = function; diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLShowStatisticStmt.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLShowStatisticStmt.java index cfca213e1c..7ef7a86693 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLShowStatisticStmt.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLShowStatisticStmt.java @@ -16,6 +16,7 @@ package com.alibaba.druid.sql.ast.statement; import com.alibaba.druid.DbType; +import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.SQLStatementImpl; import com.alibaba.druid.sql.visitor.SQLASTVisitor; @@ -29,6 +30,7 @@ public class SQLShowStatisticStmt extends SQLStatementImpl implements SQLShowSta private SQLExprTableSource tableSource; private boolean full; private final List partitions = new ArrayList(4); + private final List columns = new ArrayList<>(); public SQLShowStatisticStmt() { super (DbType.odps); @@ -73,4 +75,8 @@ public class SQLShowStatisticStmt extends SQLStatementImpl implements SQLShowSta } this.partitions.add(partition); } + + public List getColumns() { + return columns; + } } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionQueryTableSource.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionQueryTableSource.java index bfc31344c5..c80e546326 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionQueryTableSource.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionQueryTableSource.java @@ -17,13 +17,17 @@ package com.alibaba.druid.sql.ast.statement; import com.alibaba.druid.FastsqlException; import com.alibaba.druid.sql.ast.SQLHint; +import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.visitor.SQLASTVisitor; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class SQLUnionQueryTableSource extends SQLTableSourceImpl { private SQLUnionQuery union; + protected List columns = new ArrayList(); public SQLUnionQueryTableSource() { @@ -93,19 +97,28 @@ public class SQLUnionQueryTableSource extends SQLTableSourceImpl { return x; } + public List getColumns() { + return columns; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; SQLUnionQueryTableSource that = (SQLUnionQueryTableSource) o; - return union != null ? union.equals(that.union) : that.union == null; + if (union != null ? !union.equals(that.union) : that.union != null) return false; + return columns != null ? columns.equals(that.columns) : that.columns == null; } @Override public int hashCode() { - return union != null ? union.hashCode() : 0; + int result = super.hashCode(); + result = 31 * result + (union != null ? union.hashCode() : 0); + result = 31 * result + (columns != null ? columns.hashCode() : 0); + return result; } public SQLTableSource findTableSourceWithColumn(long columnNameHash, String columnName, int option) { @@ -122,4 +135,5 @@ public class SQLUnionQueryTableSource extends SQLTableSourceImpl { } return null; } + } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateStatement.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateStatement.java index 3c1ef03eb9..a8d3cf81ac 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateStatement.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateStatement.java @@ -37,6 +37,8 @@ public class SQLUpdateStatement extends SQLStatementImpl implements SQLReplaceab protected SQLTableSource tableSource; protected List returning; + protected List partitions; + // for mysql protected SQLOrderBy orderBy; @@ -121,6 +123,14 @@ public class SQLUpdateStatement extends SQLStatementImpl implements SQLReplaceab return null; } + public List getPartitions() { + return partitions; + } + + public void setPartitions(List partitions) { + this.partitions = partitions; + } + public SQLExpr getWhere() { return where; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateFunctionStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateFunctionStatement.java index 486ed27d12..7c64433ea9 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateFunctionStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateFunctionStatement.java @@ -1,8 +1,6 @@ package com.alibaba.druid.sql.dialect.hive.stmt; import com.alibaba.druid.sql.ast.SQLExpr; -import com.alibaba.druid.sql.ast.SQLName; -import com.alibaba.druid.sql.ast.SQLStatementImpl; import com.alibaba.druid.sql.ast.statement.SQLCreateFunctionStatement; import com.alibaba.druid.sql.ast.statement.SQLCreateStatement; import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor; @@ -11,7 +9,7 @@ import com.alibaba.druid.sql.visitor.SQLASTVisitor; public class HiveCreateFunctionStatement extends SQLCreateFunctionStatement implements SQLCreateStatement { protected boolean declare = false; protected SQLExpr className; - protected SQLExpr locationn; + protected SQLExpr location; protected SQLExpr symbol; protected ResourceType resourceType; protected String code; @@ -28,7 +26,7 @@ public class HiveCreateFunctionStatement extends SQLCreateFunctionStatement impl if (visitor.visit(this)) { this.acceptChild(visitor, name); this.acceptChild(visitor, className); - this.acceptChild(visitor, locationn); + this.acceptChild(visitor, location); this.acceptChild(visitor, symbol); } visitor.endVisit(this); @@ -45,15 +43,15 @@ public class HiveCreateFunctionStatement extends SQLCreateFunctionStatement impl this.className = x; } - public SQLExpr getLocationn() { - return locationn; + public SQLExpr getLocation() { + return location; } - public void setLocationn(SQLExpr x) { + public void setLocation(SQLExpr x) { if (x != null) { x.setParent(this); } - this.locationn = x; + this.location = x; } public SQLExpr getSymbol() { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java index 0933def0ae..3eada42abe 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java @@ -20,12 +20,10 @@ import com.alibaba.druid.sql.ast.*; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.ast.statement.*; -import com.alibaba.druid.sql.dialect.hive.ast.HiveInputOutputFormat; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert; import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement; import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement; import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateFunctionStatement; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.hive.stmt.HiveLoadDataStatement; import com.alibaba.druid.sql.dialect.hive.stmt.HiveMsckRepairStatement; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; @@ -55,12 +53,17 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis if (x.hasBeforeComment()) { printlnComments(x.getBeforeCommentsDirect()); } - if (x.isOverwrite()) { - print0(ucase ? "INSERT OVERWRITE TABLE " : "insert overwrite table "); - } else { - print0(ucase ? "INSERT INTO TABLE " : "insert into table "); + SQLExprTableSource tableSource = x.getTableSource(); + + if (tableSource != null) { + if (x.isOverwrite()) { + print0(ucase ? "INSERT OVERWRITE TABLE " : "insert overwrite table "); + } else { + print0(ucase ? "INSERT INTO TABLE " : "insert into table "); + } + + tableSource.accept(this); } - x.getTableSource().accept(this); List partitions = x.getPartitions(); if (partitions != null) { @@ -121,7 +124,10 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis this.indentCount--; println(); print0(") "); - print0(x.getFrom().getAlias()); + String alias = x.getFrom().getAlias(); + if (alias != null) { + print0(alias); + } } else { print0(ucase ? "FROM " : "from "); from.accept(this); @@ -253,12 +259,12 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis } indentCount++; - SQLExpr location = x.getLocationn(); + SQLExpr location = x.getLocation(); + HiveCreateFunctionStatement.ResourceType resourceType = x.getResourceType(); + if (location != null) { println(); - HiveCreateFunctionStatement.ResourceType resourceType = x.getResourceType(); - if (resourceType != null) { print0(ucase ? "USING " : "using "); print0(resourceType.name()); @@ -269,6 +275,13 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis location.accept(this); } + String code = x.getCode(); + if (code != null) { + println(); + print0(ucase ? "USING" : "using"); + print0(code); + } + SQLExpr symbol = x.getSymbol(); if (symbol != null) { println(); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveSchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveSchemaStatVisitor.java index 112b354fdd..db2a931f96 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveSchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveSchemaStatVisitor.java @@ -56,7 +56,7 @@ public class HiveSchemaStatVisitor extends SchemaStatVisitor implements HiveASTV setMode(x, TableStat.Mode.Insert); SQLExprTableSource tableSource = x.getTableSource(); - SQLExpr tableName = tableSource.getExpr(); + SQLExpr tableName = tableSource != null ? tableSource.getExpr() : null; if (tableName instanceof SQLName) { TableStat stat = getTableStat((SQLName) tableName); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCountStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCountStatement.java index 9edaa6a926..a16f33907d 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCountStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCountStatement.java @@ -16,7 +16,6 @@ package com.alibaba.druid.sql.dialect.odps.ast; import com.alibaba.druid.DbType; -import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.statement.SQLAlterStatement; import com.alibaba.druid.sql.ast.statement.SQLAssignItem; @@ -31,7 +30,7 @@ public class OdpsCountStatement private SQLExprTableSource table; - private final List partitoins = new ArrayList(); + private final List partitions = new ArrayList(); public OdpsCountStatement() { @@ -62,7 +61,7 @@ public class OdpsCountStatement this.setTable(new SQLExprTableSource(table)); } - public List getPartitoins() { - return partitoins; + public List getPartitions() { + return partitions; } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsNewExpr.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsNewExpr.java index fc1dfa662b..77f07028a0 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsNewExpr.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsNewExpr.java @@ -9,6 +9,7 @@ import com.alibaba.druid.sql.visitor.SQLASTVisitor; import java.io.IOException; public class OdpsNewExpr extends SQLMethodInvokeExpr implements OdpsObject { + private boolean array = false; @Override public OdpsNewExpr clone() { OdpsNewExpr x = new OdpsNewExpr(); @@ -59,4 +60,12 @@ public class OdpsNewExpr extends SQLMethodInvokeExpr implements OdpsObject { } super.output(buf); } + + public boolean isArray() { + return array; + } + + public void setArray(boolean array) { + this.array = array; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java index 6749d0b1af..58fac1b746 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java @@ -67,6 +67,10 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { stmt.setComment(this.exprParser.primary()); } + if (lexer.token() == Token.SEMI || lexer.token() == Token.EOF) { + return stmt; + } + for (;;) { if (lexer.identifierEquals(FnvHash.Constants.TBLPROPERTIES)) { parseTblProperties(stmt); @@ -134,6 +138,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { case GROUP: case INDEX: case ENABLE: + case DISABLE: case DESC: case ALL: case INTERVAL: @@ -173,9 +178,20 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { case TABLE: case DO: case WHILE: + case LOOP: case FOR: case RLIKE: case PROCEDURE: + case GRANT: + case EXCEPT: + case CREATE: + case PARTITIONED: + case UNION: + case PRIMARY: + case INNER: + case TO: + case DECLARE: + case REFERENCES: column = this.exprParser.parseColumn(stmt); break; default: @@ -222,6 +238,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { case GROUP: case INTERVAL: case LOOP: + case USER: break; default: throw new ParserException("expect identifier. " + lexer.info()); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java index 8c5e003d31..ccd08c5f43 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java @@ -100,6 +100,16 @@ public class OdpsExprParser extends SQLExprParser { SQLDateTimeExpr ts = new SQLDateTimeExpr(literal); expr = ts; + } else if (FnvHash.Constants.DATE == hash_lower + && lexer.stringVal().charAt(0) != '`' + && (lexer.token() == Token.LITERAL_CHARS + || lexer.token() == Token.LITERAL_ALIAS) + ) { + String literal = lexer.stringVal(); + lexer.nextToken(); + + SQLDateExpr d = new SQLDateExpr(literal); + expr = d; } else { expr = new SQLIdentifierExpr(stringVal); if (lexer.token() != Token.COMMA) { @@ -256,9 +266,15 @@ public class OdpsExprParser extends SQLExprParser { lexer.nextToken(); String methodName = lexer.stringVal(); newExpr.setMethodName(methodName); - accept(Token.LPAREN); - this.exprList(newExpr.getArguments(), newExpr); - accept(Token.RPAREN); + if (lexer.token() == Token.LBRACKET) { + lexer.nextToken(); + this.exprList(newExpr.getArguments(), newExpr); + accept(Token.RBRACKET); + } else { + accept(Token.LPAREN); + this.exprList(newExpr.getArguments(), newExpr); + accept(Token.RPAREN); + } expr = newExpr; } else if (lexer.identifierEquals("java") || lexer.identifierEquals("com")) { SQLName name = this.name(); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java index 2c79a96eca..31b5d98b9a 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java @@ -142,7 +142,8 @@ public class OdpsLexer extends Lexer { for (;;) { ch = charAt(++pos); - if (!isIdentifierChar(ch)) { + if (ch != 'ó' && ch != 'å' && ch != 'é' && ch != 'í' + && !isIdentifierChar(ch)) { if (ch == '{' && charAt(pos - 1) == '$') { int endIndex = this.text.indexOf('}', pos); if (endIndex != -1) { @@ -181,6 +182,16 @@ public class OdpsLexer extends Lexer { } this.ch = charAt(pos); + // bufPos + { + final int LEN = "USING#CODE".length(); + if (bufPos == LEN && text.regionMatches(mark, "USING#CODE", 0, LEN)) { + bufPos = "USING".length(); + pos -= 5; + this.ch = charAt(pos); + } + } + stringVal = addSymbol(); Token tok = keywords.getKeyword(stringVal); if (tok != null) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java index 3561491194..6e1442b1da 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java @@ -122,7 +122,7 @@ public class OdpsSelectParser extends SQLSelectParser { } else if (lexer.token() == Token.UNIQUE) { Lexer.SavePoint mark = lexer.mark(); lexer.nextToken(); - if (lexer.token() == Token.DOT) { + if (lexer.token() == Token.DOT || lexer.token() == Token.COMMA) { lexer.reset(mark); } else { queryBlock.setDistionOption(SQLSetQuantifier.UNIQUE); @@ -230,6 +230,12 @@ public class OdpsSelectParser extends SQLSelectParser { ); lexer.nextToken(); + if (tableSource instanceof SQLLateralViewTableSource) { + if (lexer.token() == Token.AS) { + parseLateralViewAs((SQLLateralViewTableSource) tableSource); + } + } + tableSource = parseTableSourceRest(tableSource); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java index 297e88fa15..0d23474ef5 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java @@ -30,6 +30,7 @@ import com.alibaba.druid.util.FnvHash; import java.util.List; +import static com.alibaba.druid.sql.parser.Token.IDENTIFIER; import static com.alibaba.druid.sql.parser.Token.OVERWRITE; public class OdpsStatementParser extends SQLStatementParser { @@ -82,8 +83,6 @@ public class OdpsStatementParser extends SQLStatementParser { return true; } - - if (lexer.identifierEquals("ADD")) { lexer.nextToken(); @@ -104,6 +103,14 @@ public class OdpsStatementParser extends SQLStatementParser { return true; } + if (lexer.identifierEquals("ACCOUNTPROVIDER")) { + lexer.nextToken(); + OdpsAddAccountProviderStatement stmt = new OdpsAddAccountProviderStatement(); + stmt.setProvider(this.exprParser.name()); + statementList.add(stmt); + return true; + } + if (lexer.token() == Token.TABLE) { lexer.nextToken(); OdpsAddTableStatement stmt = new OdpsAddTableStatement(); @@ -255,6 +262,12 @@ public class OdpsStatementParser extends SQLStatementParser { lexer.nextToken(); stmt.setObject(this.exprParser.expr()); + if (lexer.identifierEquals("ROLES") + && stmt.getObject() instanceof SQLIdentifierExpr && ((SQLIdentifierExpr) stmt.getObject()).nameEquals("TENANT")) { + lexer.nextToken(); + stmt.setObject(new SQLIdentifierExpr("TENANT ROLES")); + } + statementList.add(stmt); return true; @@ -280,7 +293,7 @@ public class OdpsStatementParser extends SQLStatementParser { if (lexer.token() == Token.PARTITION) { lexer.nextToken(); - this.exprParser.parseAssignItem(stmt.getPartitoins(), stmt); + this.exprParser.parseAssignItem(stmt.getPartitions(), stmt); } statementList.add(stmt); return true; @@ -329,6 +342,24 @@ public class OdpsStatementParser extends SQLStatementParser { return true; } + if (lexer.identifierEquals(FnvHash.Constants.MERGE)) { + SQLStatement stmt = parseMerge(); + statementList.add(stmt); + return true; + } + + if (lexer.identifierEquals(FnvHash.Constants.CLONE)) { + SQLStatement stmt = parseClone(); + statementList.add(stmt); + return true; + } + + if (lexer.identifierEquals(FnvHash.Constants.UNLOAD)) { + SQLStatement stmt = parseUnload(); + statementList.add(stmt); + return true; + } + if (lexer.token() == Token.FUNCTION) { HiveCreateFunctionStatement stmt = (HiveCreateFunctionStatement) parseHiveCreateFunction(); stmt.setDeclare(true); @@ -348,7 +379,13 @@ public class OdpsStatementParser extends SQLStatementParser { case LITERAL_FLOAT: case LITERAL_CHARS: case IDENTIFIER: - case CASE: { + case CASE: + case CAST: + case IF: + case VARIANT: + case REPLACE: + case NEW: + { SQLExpr expr = this.exprParser.expr(); SQLExprStatement stmt = new SQLExprStatement( new SQLAssignItem(new SQLIdentifierExpr(variant), expr) @@ -373,7 +410,8 @@ public class OdpsStatementParser extends SQLStatementParser { ); } - if (lexer.token() == Token.EQ) { + if (lexer.token() == Token.EQ || lexer.token() == Token.COLONEQ) { + lexer.nextToken(); stmt.setInitValue( this.exprParser.expr() ); @@ -452,6 +490,103 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } + public SQLStatement parseUnload() { + acceptIdentifier("UNLOAD"); + OdpsUnloadStatement stmt = new OdpsUnloadStatement(); + + accept(Token.FROM); + if (lexer.token() == Token.LPAREN || lexer.token() == Token.SELECT) { + stmt.setFrom( + this.createSQLSelectParser().parseTableSource() + ); + } else { + stmt.setFrom( + this.exprParser.name() + ); + } + + if (lexer.token() == Token.PARTITION) { + lexer.nextToken(); + this.exprParser.parseAssignItem(stmt.getPartitions(), stmt); + } + + accept(Token.INTO); + + if (lexer.identifierEquals("LOCATION")) { + lexer.nextToken(); + stmt.setLocation(this.exprParser.primary()); + } + + if (lexer.identifierEquals("ROW")) { + SQLExternalRecordFormat format = this.exprParser.parseRowFormat(); + stmt.setRowFormat(format); + } + + if (lexer.identifierEquals(FnvHash.Constants.STORED)) { + lexer.nextToken(); + if (lexer.token() == Token.BY) { + lexer.nextToken(); + } else { + accept(Token.AS); + } + stmt.setStoredAs( + this.exprParser.name()); + } + + if (lexer.token() == Token.WITH) { + lexer.nextToken(); + acceptIdentifier("SERDEPROPERTIES"); + this.exprParser.parseAssignItem(stmt.getSerdeProperties(), stmt); + } + + if (identifierEquals("PROPERTIES")) { + lexer.nextToken(); + this.exprParser.parseAssignItem(stmt.getProperties(), stmt); + } + + return stmt; + } + + public SQLStatement parseClone() { + acceptIdentifier("CLONE"); + accept(Token.TABLE); + SQLCloneTableStatement stmt = new SQLCloneTableStatement(); + + stmt.setFrom( + this.exprParser.name()); + + if (lexer.token() == Token.PARTITION) { + lexer.nextToken(); + this.exprParser.parseAssignItem(stmt.getPartitions(), stmt); + } + + accept(Token.TO); + + stmt.setTo( + this.exprParser.name()); + + if (lexer.token() == Token.IF) { + lexer.nextToken(); + accept(Token.EXISTS); + accept(OVERWRITE); + stmt.setOverwriteIfExists(true); + } + + return stmt; + } + + public SQLStatement parseBlock() { + SQLBlockStatement block = new SQLBlockStatement(); + if (lexer.identifierEquals(FnvHash.Constants.BEGIN)) { + lexer.nextToken(); + } else { + accept(Token.BEGIN); + } + this.parseStatementList(block.getStatementList(), -1, block); + accept(Token.END); + return block; + } + protected OdpsStatisticClause parseStaticClause() { if (lexer.identifierEquals("TABLE_COUNT")) { lexer.nextToken(); @@ -497,7 +632,7 @@ public class OdpsStatementParser extends SQLStatementParser { HiveMultiInsertStatement stmt = new HiveMultiInsertStatement(); - if (lexer.token() == Token.IDENTIFIER) { + if (lexer.token() == Token.IDENTIFIER || lexer.token() == Token.VARIANT) { Lexer.SavePoint mark = lexer.mark(); SQLExpr tableName = this.exprParser.name(); if (lexer.token() == Token.LPAREN) { @@ -532,6 +667,9 @@ public class OdpsStatementParser extends SQLStatementParser { if (lexer.token() == Token.INSERT) { alias = null; + } else if (lexer.token() == Token.SELECT) { + // skip + alias = null; } else { if (lexer.token() == Token.AS) { lexer.nextToken(); @@ -553,6 +691,16 @@ public class OdpsStatementParser extends SQLStatementParser { stmt.setFrom(tableSource); } + if(lexer.token() == Token.SELECT) { + SQLSelectParser selectParser = createSQLSelectParser(); + SQLSelect query = selectParser.select(); + + HiveInsert insert = new HiveInsert(); + insert.setQuery(query); + stmt.addItem(insert); + return stmt; + } + for (;;) { HiveInsert insert = parseHiveInsert(); stmt.addItem(insert); @@ -616,6 +764,16 @@ public class OdpsStatementParser extends SQLStatementParser { accept(Token.RPAREN); } + if (identifierEquals("COLUMNS")) { + lexer.nextToken(); + + if (lexer.token() != Token.SEMI) { + accept(Token.LPAREN); + this.exprParser.names(stmt.getColumns(), stmt); + accept(Token.RPAREN); + } + } + return stmt; } @@ -646,11 +804,16 @@ public class OdpsStatementParser extends SQLStatementParser { if (lexer.token() == Token.FROM || lexer.token() == Token.IN) { lexer.nextToken(); stmt.setDatabase(this.exprParser.name()); + } else if (lexer.token() == IDENTIFIER) { + SQLName database = exprParser.name(); + stmt.setDatabase(database); } if (lexer.token() == Token.LIKE) { lexer.nextToken(); stmt.setLike(this.exprParser.expr()); + } else if (lexer.token() == Token.LITERAL_CHARS) { + stmt.setLike(this.exprParser.expr()); } return stmt; @@ -714,6 +877,48 @@ public class OdpsStatementParser extends SQLStatementParser { return parseShowCreateTable(); } + if (lexer.identifierEquals(FnvHash.Constants.FUNCTIONS)) { + lexer.nextToken(); + + SQLShowFunctionsStatement stmt = new SQLShowFunctionsStatement(); + if (lexer.token() == Token.LIKE) { + lexer.nextToken(); + SQLExpr like = this.exprParser.expr(); + stmt.setLike(like); + } + + return stmt; + } + + if (lexer.identifierEquals(FnvHash.Constants.ROLES)) { + lexer.nextToken(); + + SQLShowRolesStatement stmt = new SQLShowRolesStatement(); + return stmt; + } + + if (lexer.identifierEquals("HISTORY")) { + lexer.nextToken(); + SQLShowHistoryStatement stmt = new SQLShowHistoryStatement(); + + if (lexer.token() == Token.FOR) { + lexer.nextToken(); + if (lexer.identifierEquals(FnvHash.Constants.TABLES)) { + lexer.nextToken(); + stmt.setTables(true); + } else if (lexer.token() == Token.TABLE) { + lexer.nextToken(); + stmt.setTable( + new SQLExprTableSource( + this.exprParser.name() + ) + ); + } + } + + return stmt; + } + throw new ParserException("TODO " + lexer.info()); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java index 4c655415de..3b91632dd3 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java @@ -256,4 +256,20 @@ public interface OdpsASTVisitor extends HiveASTVisitor { } + default boolean visit(OdpsAddAccountProviderStatement x) { + return true; + } + + default void endVisit(OdpsAddAccountProviderStatement x) { + + } + + default boolean visit(OdpsUnloadStatement x) { + return true; + } + + default void endVisit(OdpsUnloadStatement x) { + + } + } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index 4494628dc2..0d86e4f4d9 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -21,7 +21,6 @@ import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; -import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.dialect.hive.stmt.HiveLoadDataStatement; import com.alibaba.druid.sql.dialect.hive.visitor.HiveOutputVisitor; import com.alibaba.druid.sql.dialect.odps.ast.*; @@ -30,8 +29,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import static com.alibaba.druid.sql.dialect.odps.ast.OdpsAddFileStatement.FileType.JAR; - public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisitor { private Set builtInFunctions = new HashSet(); @@ -953,7 +950,7 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit print0(ucase ? "COUNT " : "count "); x.getTable().accept(this); - List partitoins = x.getPartitoins(); + List partitoins = x.getPartitions(); if (partitoins.size() > 0) { print0(ucase ? " PARTITION (" : " partition ("); printAndAccept(partitoins, ", "); diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index c8f21840ae..53d2447618 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -2217,7 +2217,7 @@ public class Lexer { this.ch = charAt(pos); if (dbType == DbType.odps) { - while (isDigit(this.ch)) { + while (isIdentifierChar(this.ch)) { ++pos; bufPos++; this.ch = charAt(pos); @@ -2733,8 +2733,14 @@ public class Lexer { } if (numberSale > 0 || numberExp) { + if (text.charAt(mark) == '.' && isIdentifierChar(ch)) { + pos = mark + 1; + ch = charAt(pos); + token = Token.DOT; + return; + } token = Token.LITERAL_FLOAT; - } else { + } else if (ch != '`') { if (isFirstIdentifierChar(ch) && ch != ')' && !(ch == 'b' && bufPos == 1 && charAt(pos - 1) == '0' && dbType != DbType.odps) @@ -2767,6 +2773,7 @@ public class Lexer { public void scanHexaDecimal() { mark = pos; + bufPos = 0; if (ch == '-') { bufPos++; @@ -2782,6 +2789,22 @@ public class Lexer { ch = charAt(++pos); } + if (isIdentifierChar(ch)) { + for (;;) { + bufPos++; + ch = charAt(++pos); + if (!isIdentifierChar(ch)) { + break; + } + } + mark -= 2; + bufPos += 2; + stringVal = addSymbol(); + hash_lower = FnvHash.hashCode64(stringVal); + token = Token.IDENTIFIER; + return; + } + token = Token.LITERAL_HEX; } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index d49c026a43..239f9aee96 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -253,7 +253,7 @@ public class SQLExprParser extends SQLParser { Lexer.SavePoint savePoint = lexer.mark(); lexer.nextToken(); - if (lexer.token == Token.COMMA || lexer.token == Token.EOF) { + if (lexer.token == Token.COMMA || lexer.token == Token.EOF || lexer.token == Token.ON || lexer.token == Token.RPAREN) { lexer.reset(savePoint); return expr; } @@ -1131,6 +1131,10 @@ public class SQLExprParser extends SQLParser { case PROCEDURE: case REFERENCES: case IS: + case REVOKE: + case DECLARE: + case DROP: + case RLIKE: if (dbType == DbType.odps || dbType == DbType.hive) { sqlExpr = new SQLIdentifierExpr(lexer.stringVal()); lexer.nextToken(); @@ -1178,6 +1182,7 @@ public class SQLExprParser extends SQLParser { case EQ: case GT: case RPAREN: + case IS: sqlExpr = new SQLIdentifierExpr(str); break; default: @@ -1250,6 +1255,11 @@ public class SQLExprParser extends SQLParser { switch (lexer.token) { case DOT: case SLASH: + case EQ: + case GT: + case GTEQ: + case LT: + case LTEQ: return primaryRest(new SQLIdentifierExpr(str)); case COMMA: case PLUS: @@ -1357,6 +1367,10 @@ public class SQLExprParser extends SQLParser { case THEN: case END: case AS: + case UNION: + case SEMI: + case ASC: + case DESC: return new SQLIdentifierExpr(str); case PLUS: case SUB: { @@ -1368,6 +1382,8 @@ public class SQLExprParser extends SQLParser { lexer.reset(mark); break; } + } else { + lexer.reset(mark); } return new SQLIdentifierExpr(str); @@ -1385,6 +1401,8 @@ public class SQLExprParser extends SQLParser { case RPAREN: case WHERE: case FROM: + case AS: + case ORDER: lexer.reset(mark); return new SQLIdentifierExpr(str); default: @@ -2009,7 +2027,14 @@ public class SQLExprParser extends SQLParser { while (lexer.token == Token.COMMA) { lexer.nextToken(); - name = name(); + if (parent instanceof SQLLateralViewTableSource && lexer.token == Token.NULL) { + name = new SQLIdentifierExpr(lexer.stringVal()); + lexer.nextToken(); + } else { + name = name(); + } + + name.setParent(parent); exprCol.add(name); } @@ -2179,6 +2204,7 @@ public class SQLExprParser extends SQLParser { case LOOP: case IS: case LOCK: + case REFERENCES: identName = lexer.stringVal(); lexer.nextToken(); break; @@ -2199,6 +2225,13 @@ public class SQLExprParser extends SQLParser { case RIGHT: case TABLE: case RLIKE: + case CREATE: + case PARTITIONED: + case UNION: + case PRIMARY: + case TO: + case DECLARE: + case AS: if (dbType == DbType.odps) { identName = lexer.stringVal(); lexer.nextToken(); @@ -2365,9 +2398,25 @@ public class SQLExprParser extends SQLParser { exprList(over.getPartitionBy(), over); accept(Token.RPAREN); - if (over.getPartitionBy().size() == 1 && lexer.token == Token.COMMA) { - lexer.nextToken(); - exprList(over.getPartitionBy(), over); + if (over.getPartitionBy().size() == 1) { + switch (lexer.token) { + case SLASH: + case DIV: + case STAR: + case PLUS: + case SUB: + SQLExpr first = this.exprRest(over.getPartitionBy().get(0)); + first.setParent(over); + over.getPartitionBy().set(0, first); + break; + default: + break; + } + + if (lexer.token == Token.COMMA) { + lexer.nextToken(); + exprList(over.getPartitionBy(), over); + } } } else if (lexer.token == Token.ALL) { SQLName name = this.name(); @@ -4801,10 +4850,29 @@ public class SQLExprParser extends SQLParser { name += lexer.stringVal(); lexer.nextToken(); } + propertyExpr.setName(name); + var = this.primaryRest(propertyExpr); } + if (var instanceof SQLPropertyExpr && lexer.token == Token.IDENTIFIER && dbType == DbType.odps) { + SQLPropertyExpr propertyExpr = (SQLPropertyExpr) var; + + if (identifierEquals("DATEADD")) { + String func = lexer.stringVal(); + lexer.nextToken(); + if (lexer.token == Token.LPAREN) { + lexer.nextToken(); + accept(Token.RPAREN); + func += "()"; + } + + String name = propertyExpr.getName() + func; + propertyExpr.setName(name); + } + } + if (variant && var instanceof SQLIdentifierExpr) { SQLIdentifierExpr ident = (SQLIdentifierExpr) var; if (lexer.identifierEquals(FnvHash.Constants.CLUSTER) @@ -4820,6 +4888,7 @@ public class SQLExprParser extends SQLParser { } item.setTarget(var); + if (lexer.token == Token.COLONEQ) { lexer.nextToken(); } else if (lexer.token == Token.TRUE || lexer.identifierEquals(FnvHash.Constants.TRUE)) { @@ -4862,6 +4931,24 @@ public class SQLExprParser extends SQLParser { } else { SQLExpr expr = expr(); + if (dbType == DbType.odps) { + while (lexer.token == Token.LITERAL_FLOAT && lexer.numberString().startsWith(".")) { + if (expr instanceof SQLNumberExpr) { + String numStr = ((SQLNumberExpr) expr).getLiteral(); + numStr += lexer.numberString(); + expr = new SQLIdentifierExpr(numStr); + lexer.nextToken(); + } else if (expr instanceof SQLIdentifierExpr) { + String ident = ((SQLIdentifierExpr) expr).getName(); + ident += lexer.numberString(); + expr = new SQLIdentifierExpr(ident); + lexer.nextToken(); + } else { + break; + } + } + } + if (lexer.token == Token.COLON) { if (dbType == DbType.hive || dbType == DbType.odps) { Lexer.SavePoint mark = lexer.mark(); @@ -6252,6 +6339,12 @@ public class SQLExprParser extends SQLParser { if (lexer.identifierEquals(FnvHash.Constants.SERDE)) { lexer.nextToken(); format.setSerde(this.expr()); + + if (lexer.token == Token.WITH) { + lexer.nextToken(); + acceptIdentifier("serdeproperties"); + this.parseAssignItem(format.getSerdeProperties(), format); + } } return format; diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java index ea29cf7470..3bb13a49a6 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java @@ -121,6 +121,8 @@ public class SQLParser { case RIGHT: case FULL: case ON: + case GROUP: + case ORDER: return ident; case JOIN: if (hash != FnvHash.Constants.NATURAL @@ -202,7 +204,8 @@ public class SQLParser { } } case OUTER: - case IN: { + case IN: + { Lexer.SavePoint mark = lexer.mark(); String strVal = lexer.stringVal(); lexer.nextToken(); @@ -212,7 +215,9 @@ public class SQLParser { case LEFT: case RIGHT: case FULL: - case RPAREN: { + case RPAREN: + case ON: + { return strVal; } default: @@ -232,12 +237,26 @@ public class SQLParser { case DESC: case SCHEMA: case IS: + case DECLARE: + case DROP: if (dbType == DbType.odps || dbType == DbType.hive) { String strVal = lexer.stringVal(); lexer.nextToken(); return strVal; } break; + case PARTITION: + if (dbType == DbType.odps || dbType == DbType.hive) { + Lexer.SavePoint mark = lexer.mark(); + String strVal = lexer.stringVal(); + lexer.nextToken(); + if (lexer.token == Token.LPAREN) { + lexer.reset(mark); + return null; + } + return strVal; + } + break; case TABLE: if (dbType == DbType.odps){ Lexer.SavePoint mark = lexer.mark(); @@ -257,6 +276,7 @@ public class SQLParser { case REFERENCES: case REPEAT: case USE: + case MOD: case OUT: { String strVal = lexer.stringVal(); lexer.nextToken(); @@ -268,16 +288,77 @@ public class SQLParser { lexer.nextToken(); if (lexer.token == Token.BY) { lexer.reset(mark); + return null; } else { return strVal; } - break; } + case MINUS: + case EXCEPT: + if (dbType == DbType.odps){ + Lexer.SavePoint mark = lexer.mark(); + String strVal = lexer.stringVal(); + lexer.nextToken(); + switch (lexer.token) { + case SEMI: + case EOF: + case GROUP: + return strVal; + default: + lexer.reset(mark); + break; + } + } + break; default: break; } } + if (must) { + if (dbType == DbType.odps) { + switch (lexer.token) { + case GROUP: + case ORDER: { + Lexer.SavePoint mark = lexer.mark(); + String strVal = lexer.stringVal(); + lexer.nextToken(); + if (lexer.token == Token.BY) { + lexer.reset(mark); + return null; + } else { + return strVal; + } + } + case UNION: { + Lexer.SavePoint mark = lexer.mark(); + String strVal = lexer.stringVal(); + lexer.nextToken(); + if (lexer.token == Token.ALL) { + lexer.reset(mark); + return null; + } else { + return strVal; + } + } + case LIMIT: { + Lexer.SavePoint mark = lexer.mark(); + String strVal = lexer.stringVal(); + lexer.nextToken(); + if (lexer.token == Token.LITERAL_INT) { + lexer.reset(mark); + return null; + } else { + return strVal; + } + } + default: + break; + } + } + return this.alias(); + } + return this.as(); } @@ -363,6 +444,7 @@ public class SQLParser { case LEAVE: case ENABLE: case DISABLE: + case DISTRIBUTE: alias = lexer.stringVal(); lexer.nextToken(); break; @@ -370,7 +452,15 @@ public class SQLParser { case EXCEPT: case DESC: case INOUT: - case MINUS: { + case MINUS: + case UPDATE: + case DELETE: + case TABLE: + case UNION: + case EXPLAIN: + case CREATE: + case LIMIT: + { alias = lexer.stringVal(); Lexer.SavePoint mark = lexer.mark(); @@ -571,12 +661,30 @@ public class SQLParser { case REFERENCES: case INTO: case USE: + case LEAVE: + case DISTRIBUTE: + case AS: + case FOR: + case PARTITIONED: + case REPLACE: alias = lexer.stringVal(); lexer.nextToken(); return alias; case QUES: alias = "?"; lexer.nextToken(); + return alias; + case UNION: { + Lexer.SavePoint mark = lexer.mark(); + String strVal = lexer.stringVal(); + lexer.nextToken(); + if (lexer.token == Token.ALL) { + lexer.reset(mark); + return null; + } else { + return strVal; + } + } default: break; } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java index a48eecea56..1ca2e74bca 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java @@ -155,6 +155,10 @@ public class SQLSelectParser extends SQLParser { Lexer.SavePoint mark = lexer.mark(); lexer.nextToken(); + if (lexer.token == Token.UNION && dbType == DbType.odps) { + continue; // skip + } + if (lexer.token == Token.ALL) { if (union.getOperator() == SQLUnionOperator.UNION_ALL) { lexer.nextToken(); @@ -1280,26 +1284,45 @@ public class SQLSelectParser extends SQLParser { && ((hash = lexer.hash_lower()) == FnvHash.Constants.STRAIGHT_JOIN || hash == FnvHash.Constants.CROSS))) { - String alias = tableAlias(false); + boolean must = false; + if (lexer.token == Token.AS) { + lexer.nextToken(); + must = true; + } + String alias = tableAlias(must); if (alias != null) { if (isEnabled(SQLParserFeature.IgnoreNameQuotes) && alias.length() > 1) { alias = StringUtils.removeNameQuotes(alias); } tableSource.setAlias(alias); - if (tableSource instanceof SQLValuesTableSource + if ((tableSource instanceof SQLValuesTableSource) && ((SQLValuesTableSource) tableSource).getColumns().size() == 0) { SQLValuesTableSource values = (SQLValuesTableSource) tableSource; accept(Token.LPAREN); this.exprParser.names(values.getColumns(), values); accept(Token.RPAREN); - } else if (tableSource instanceof SQLSubqueryTableSource) { - SQLSubqueryTableSource values = (SQLSubqueryTableSource) tableSource; + } else if (tableSource instanceof SQLSubqueryTableSource) { + SQLSubqueryTableSource subQuery = (SQLSubqueryTableSource) tableSource; if (lexer.token == Token.LPAREN) { lexer.nextToken(); - this.exprParser.names(values.getColumns(), values); + this.exprParser.names(subQuery.getColumns(), subQuery); accept(Token.RPAREN); } + } else if (tableSource instanceof SQLUnionQueryTableSource) { + SQLUnionQueryTableSource union = (SQLUnionQueryTableSource) tableSource; + if (lexer.token == Token.LPAREN) { + lexer.nextToken(); + this.exprParser.names(union.getColumns(), union); + accept(Token.RPAREN); + } + } else if (lexer.token == Token.LPAREN + && tableSource instanceof SQLExprTableSource + && ((SQLExprTableSource) tableSource).getExpr() instanceof SQLVariantRefExpr) { + lexer.nextToken(); + SQLExprTableSource exprTableSource = (SQLExprTableSource) tableSource; + this.exprParser.names(exprTableSource.getColumns(), exprTableSource); + accept(Token.RPAREN); } if (lexer.token == Token.WHERE) { @@ -1469,6 +1492,9 @@ public class SQLSelectParser extends SQLParser { && rightTableSource.getAlias() == null && ((SQLValuesTableSource) rightTableSource).getColumns().size() == 0 ) { + if (lexer.token == Token.AS) { + lexer.nextToken(); + } rightTableSource.setAlias(tableAlias(true)); if (lexer.token == Token.LPAREN) { @@ -1659,6 +1685,12 @@ public class SQLSelectParser extends SQLParser { this.exprParser.names(udj.getColumns(), udj); accept(Token.RPAREN); + if (lexer.identifierEquals(FnvHash.Constants.SORT)) { + lexer.nextToken(); + accept(Token.BY); + this.exprParser.orderBy(udj.getSortBy(), udj); + } + join.setUdj(udj); } } else if (lexer.token == Token.USING @@ -1820,16 +1852,44 @@ public class SQLSelectParser extends SQLParser { lateralViewTabSrc.setMethod(udtf); String alias = as(); - lateralViewTabSrc.setAlias(alias); + if (alias != null) { + lateralViewTabSrc.setAlias(alias); + } if (lexer.token == Token.AS) { - lexer.nextToken(); - this.exprParser.names(lateralViewTabSrc.getColumns()); + parseLateralViewAs(lateralViewTabSrc); } return parseTableSourceRest(lateralViewTabSrc); } + public void parseLateralViewAs(SQLLateralViewTableSource lateralViewTabSrc) { + accept(Token.AS); + + Lexer.SavePoint mark = null; + for (;;) { + SQLName name; + if (lexer.token == Token.NULL) { + name = new SQLIdentifierExpr(lexer.stringVal()); + lexer.nextToken(); + } else { + name = this.exprParser.name(); + if (name instanceof SQLPropertyExpr) { + lexer.reset(mark); + break; + } + } + name.setParent(lateralViewTabSrc); + lateralViewTabSrc.getColumns().add(name); + if (lexer.token == Token.COMMA) { + mark = lexer.mark(); + lexer.nextToken(); + continue; + } + break; + } + } + public SQLValuesTableSource parseValues() { accept(Token.VALUES); SQLValuesTableSource tableSource = new SQLValuesTableSource(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index c1c49a14a7..2a824a4dd7 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -2275,6 +2275,11 @@ public class SQLStatementParser extends SQLParser { item.setIfExists(true); } + if (lexer.identifierEquals(Constants.PURGE)) { + lexer.nextToken(); + item.setPurge(true); + } + stmt.addItem(item); } else { acceptIdentifier("SMALLFILES"); @@ -2336,6 +2341,10 @@ public class SQLStatementParser extends SQLParser { lexer.nextToken(); acceptIdentifier("PARTITIONS"); stmt.addItem(new SQLAlterTableRecoverPartitions()); + } else if (dbType == DbType.odps && lexer.token == Token.NOT) { + lexer.nextToken(); + acceptIdentifier("CLUSTERED"); + stmt.setNotClustered(true); } else { break; } @@ -3456,6 +3465,9 @@ public class SQLStatementParser extends SQLParser { } else if (lexer.identifierEquals(FnvHash.Constants.TABLEGROUP)) { lexer.reset(markBp, markChar, Token.CREATE); return parseCreateTableGroup(); + } else if (lexer.identifierEquals(FnvHash.Constants.SQL) && dbType == DbType.odps) { + lexer.reset(markBp, markChar, Token.CREATE); + return parseCreateFunction(); } else if (lexer.identifierEquals(FnvHash.Constants.DIMENSION)) { lexer.reset(markBp, markChar, Token.CREATE); return parseCreateTable(); @@ -4255,6 +4267,12 @@ public class SQLStatementParser extends SQLParser { udpateStatement.setTableSource(tableSource); } + if (dbType == DbType.odps && lexer.token == PARTITION) { + lexer.nextToken(); + udpateStatement.setPartitions(new ArrayList<>()); + this.exprParser.parseAssignItem(udpateStatement.getPartitions(), udpateStatement); + } + parseUpdateSet(udpateStatement); if (lexer.token == (Token.WHERE)) { @@ -4460,6 +4478,12 @@ public class SQLStatementParser extends SQLParser { this.accept(Token.AS); + if (lexer.identifierEquals(FnvHash.Constants.BEGIN)) { + SQLBlockStatement block = (SQLBlockStatement) this.parseBlock(); + createView.setScript(block); + return createView; + } + SQLSelectParser selectParser = this.createSQLSelectParser(); createView.setSubQuery(selectParser.select()); @@ -4513,9 +4537,18 @@ public class SQLStatementParser extends SQLParser { protected SQLAlterTableAddColumn parseAlterTableAddColumn() { boolean odps = DbType.odps == dbType || DbType.hive == dbType; + boolean columns = false; if (odps) { - acceptIdentifier("COLUMNS"); - accept(Token.LPAREN); + if (lexer.identifierEquals("COLUMNS")) { + lexer.nextToken(); + if (lexer.token == LPAREN) { + lexer.nextToken(); + columns = true; + } + } else if (lexer.token == LPAREN) { + lexer.nextToken(); + columns = true; + } } SQLAlterTableAddColumn item = new SQLAlterTableAddColumn(); @@ -4546,7 +4579,7 @@ public class SQLStatementParser extends SQLParser { break; } - if (odps) { + if (odps && columns) { accept(Token.RPAREN); } @@ -4975,7 +5008,11 @@ public class SQLStatementParser extends SQLParser { } public SQLStatement parseMerge() { - accept(Token.MERGE); + if (lexer.identifierEquals(Constants.MERGE)) { + lexer.nextToken(); + } else { + accept(Token.MERGE); + } SQLMergeStatement stmt = new SQLMergeStatement(); stmt.setDbType(dbType); @@ -4996,7 +5033,11 @@ public class SQLStatementParser extends SQLParser { stmt.getInto().setAlias(tableAlias()); - accept(Token.USING); + if (lexer.identifierEquals(Constants.USING)) { + lexer.nextToken(); + } else { + accept(Token.USING); + } SQLTableSource using = this.createSQLSelectParser().parseTableSource(); stmt.setUsing(using); @@ -5008,7 +5049,7 @@ public class SQLStatementParser extends SQLParser { boolean insertFlag = false; if (lexer.token == Token.WHEN) { lexer.nextToken(); - if (lexer.token == Token.MATCHED) { + if (lexer.token == Token.MATCHED || lexer.identifierEquals(Constants.MATCHED)) { SQLMergeStatement.MergeUpdateClause updateClause = new SQLMergeStatement.MergeUpdateClause(); lexer.nextToken(); @@ -5101,7 +5142,11 @@ public class SQLStatementParser extends SQLParser { if (insertFlag) { SQLMergeStatement.MergeInsertClause insertClause = new SQLMergeStatement.MergeInsertClause(); - accept(Token.MATCHED); + if (lexer.identifierEquals(Constants.MATCHED)) { + lexer.nextToken(); + } else { + accept(Token.MATCHED); + } accept(Token.THEN); accept(Token.INSERT); @@ -6391,6 +6436,12 @@ public class SQLStatementParser extends SQLParser { accept(Token.RPAREN); } + if (lexer.token == LPAREN) { + lexer.nextToken(); + this.exprParser.exprList(insert.getColumns(), insert); + accept(RPAREN); + } + if (lexer.token == Token.VALUES) { lexer.nextToken(); @@ -6751,9 +6802,9 @@ public class SQLStatementParser extends SQLParser { protected SQLStatement parseMsck() { lexer.nextToken(); - lexer.identifierEquals("REPAIR"); - - lexer.nextToken(); + if (lexer.identifierEquals("REPAIR")) { + lexer.nextToken(); + } HiveMsckRepairStatement stmt = new HiveMsckRepairStatement(); @@ -6986,6 +7037,13 @@ public class SQLStatementParser extends SQLParser { lexer.nextToken(); stmt.setTemporary(true); } + + boolean sql = false; // TODO for odps + if (lexer.identifierEquals(FnvHash.Constants.SQL)) { + lexer.nextToken(); + sql = true; + } + accept(Token.FUNCTION); SQLName name = this.exprParser.name(); @@ -7006,6 +7064,15 @@ public class SQLStatementParser extends SQLParser { accept(RPAREN); } + if (lexer.identifierEquals(Constants.RETURNS)) { + lexer.nextToken(); + if (lexer.token == VARIANT) { + lexer.nextToken(); // TODO + } + SQLDataType returnDataType = this.exprParser.parseDataType(); + stmt.setReturnDataType(returnDataType); + } + if (lexer.token() == Token.AS) { lexer.setToken(Token.IDENTIFIER); lexer.nextToken(); @@ -7016,7 +7083,7 @@ public class SQLStatementParser extends SQLParser { if (lexer.identifierEquals(FnvHash.Constants.LOCATION)) { lexer.nextToken(); SQLExpr location = this.exprParser.primary(); - stmt.setLocationn(location); + stmt.setLocation(location); } if (lexer.identifierEquals(FnvHash.Constants.SYMBOL)) { @@ -7046,7 +7113,7 @@ public class SQLStatementParser extends SQLParser { } SQLExpr location = this.exprParser.primary(); - stmt.setLocationn(location); + stmt.setLocation(location); } return stmt; @@ -7073,4 +7140,8 @@ public class SQLStatementParser extends SQLParser { } return stmt; } + + public SQLStatement parseClone() { + throw new ParserException("TODO " + lexer.info()); + } } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index f6878c9e53..522982f156 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -5015,7 +5015,15 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet print0(ucase ? "AS" : "as"); println(); - x.getSubQuery().accept(this); + SQLSelect subQuery = x.getSubQuery(); + if (subQuery != null) { + subQuery.accept(this); + } + + SQLBlockStatement script = x.getScript(); + if (script != null) { + script.accept(this); + } if (x.isWithCheckOption()) { println(); @@ -5535,6 +5543,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet print0(ucase ? " SHARDS" : " shards"); } + if (x.isNotClustered()) { + print0(ucase ? " NOT CLUSTERED" : " not clustered"); + } + return false; } @@ -6337,9 +6349,15 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet println(); print(')'); - if (x.getAlias() != null) { - print(' '); - print0(x.getAlias()); + final List columns = x.getColumns(); + final String alias = x.getAlias(); + if (alias != null) { + if (columns.size() > 0) { + print0(" AS "); + } else { + print(' '); + } + print0(alias); } return false; @@ -11107,6 +11125,14 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return false; } + public boolean visit(SQLCloneTableStatement x) { + print0(ucase ? "CLONE TABLE " : "clone table "); + x.getFrom().accept(this); + print0(ucase ? " TO " : " to "); + x.getTo().accept(this); + return false; + } + public char getNameQuote() { return quote; } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index 7a8be80904..0a342e11d5 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -2667,4 +2667,29 @@ public interface SQLASTVisitor { default boolean visit(SQLTableDataType x) { return true; } + + default void endVisit(SQLCloneTableStatement x) { + + } + + default boolean visit(SQLCloneTableStatement x) { + return true; + } + + default void endVisit(SQLShowHistoryStatement x) { + + } + + default boolean visit(SQLShowHistoryStatement x) { + return true; + } + + default void endVisit(SQLShowRolesStatement x) { + + } + + default boolean visit(SQLShowRolesStatement x) { + return true; + } + } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java index e6b6dab634..35d07cf301 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java @@ -2272,7 +2272,15 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter { repository.resolve(x); } - x.getSubQuery().accept(this); + SQLSelect subQuery = x.getSubQuery(); + if (subQuery != null) { + subQuery.accept(this); + } + + SQLBlockStatement script = x.getScript(); + if (script != null) { + script.accept(this); + } return false; } diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index f6c96c8bfb..99a7072736 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -374,6 +374,7 @@ public final class FnvHash { long REGEXP = fnv1a_64_lower("REGEXP"); long RLIKE = fnv1a_64_lower("RLIKE"); long USING = fnv1a_64_lower("USING"); + long MATCHED = fnv1a_64_lower("MATCHED"); long IGNORE = fnv1a_64_lower("IGNORE"); long FORCE = fnv1a_64_lower("FORCE"); long CROSS = fnv1a_64_lower("CROSS");