From a40b4ecb2f40fa97ec49d011803ed3b7f7b63f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Mon, 5 Jul 2021 14:47:10 +0800 Subject: [PATCH] improved odps parser --- .../dialect/hive/parser/HiveSelectParser.java | 79 +---------------- .../odps/parser/OdpsCreateTableParser.java | 50 ++++++----- .../dialect/odps/parser/OdpsExprParser.java | 7 +- .../dialect/odps/parser/OdpsSelectParser.java | 4 + .../odps/parser/OdpsStatementParser.java | 6 ++ .../druid/sql/parser/SQLExprParser.java | 6 ++ .../alibaba/druid/sql/parser/SQLParser.java | 4 + .../druid/sql/parser/SQLSelectParser.java | 84 +++++++++++++++++++ .../druid/sql/parser/SQLStatementParser.java | 10 +++ .../sql/visitor/SQLASTOutputVisitor.java | 6 ++ .../druid/sql/visitor/SQLASTVisitor.java | 8 ++ 11 files changed, 165 insertions(+), 99 deletions(-) diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java index 70eace377d..8a9d209ec1 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java @@ -42,82 +42,7 @@ public class HiveSelectParser extends SQLSelectParser { return new HiveExprParser(lexer); } - public SQLTableSource parseTableSourceRest(SQLTableSource tableSource) { - if (lexer.identifierEquals(FnvHash.Constants.TABLESAMPLE) && tableSource instanceof SQLExprTableSource) { - Lexer.SavePoint mark = lexer.mark(); - lexer.nextToken(); - if (lexer.token() == Token.LPAREN) { - lexer.nextToken(); - - SQLTableSampling sampling = new SQLTableSampling(); - - if (lexer.identifierEquals(FnvHash.Constants.BUCKET)) { - lexer.nextToken(); - SQLExpr bucket = this.exprParser.primary(); - sampling.setBucket(bucket); - - if (lexer.token() == Token.OUT) { - lexer.nextToken(); - accept(Token.OF); - SQLExpr outOf = this.exprParser.primary(); - sampling.setOutOf(outOf); - } - - if (lexer.token() == Token.ON) { - lexer.nextToken(); - SQLExpr on = this.exprParser.expr(); - sampling.setOn(on); - } - } - - if (lexer.token() == Token.LITERAL_INT || lexer.token() == Token.LITERAL_FLOAT) { - SQLExpr val = this.exprParser.primary(); - - if (lexer.identifierEquals(FnvHash.Constants.ROWS)) { - lexer.nextToken(); - sampling.setRows(val); - } else { - acceptIdentifier("PERCENT"); - sampling.setPercent(val); - } - } - - if (lexer.token() == Token.IDENTIFIER) { - String strVal = lexer.stringVal(); - char first = strVal.charAt(0); - char last = strVal.charAt(strVal.length() - 1); - if (last >= 'a' && last <= 'z') { - last -= 32; // to upper - } - - boolean match = false; - if ((first == '.' || (first >= '0' && first <= '9'))) { - switch (last) { - case 'B': - case 'K': - case 'M': - case 'G': - case 'T': - case 'P': - match = true; - break; - default: - break; - } - } - SQLSizeExpr size = new SQLSizeExpr(strVal.substring(0, strVal.length() - 2), last); - sampling.setByteLength(size); - lexer.nextToken(); - } - - final SQLExprTableSource table = (SQLExprTableSource) tableSource; - table.setSampling(sampling); - - accept(Token.RPAREN); - } else { - lexer.reset(mark); - } - } - return super.parseTableSourceRest(tableSource); + public void parseTableSourceSample(SQLTableSource tableSource) { + parseTableSourceSampleHive(tableSource); } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java index 6ad2f6c8f7..42a5d40e9b 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java @@ -158,6 +158,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { case LEFT: case RIGHT: case REPEAT: + case COMPUTE: column = this.exprParser.parseColumn(stmt); break; default: @@ -196,6 +197,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { for (;;) { switch (lexer.token()) { + case INDEX: case KEY: case IDENTIFIER: case GROUP: @@ -335,26 +337,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { } if (lexer.identifierEquals(FnvHash.Constants.TBLPROPERTIES)) { - lexer.nextToken(); - accept(Token.LPAREN); - - for (;;) { - String name = lexer.stringVal(); - lexer.nextToken(); - accept(Token.EQ); - SQLExpr value = this.exprParser.primary(); - stmt.addTblProperty(name, value); - if (lexer.token() == Token.COMMA) { - lexer.nextToken(); - if (lexer.token() == Token.RPAREN) { - break; - } - continue; - } - break; - } - - accept(Token.RPAREN); + parseTblProperties(stmt); } if (lexer.identifierEquals(FnvHash.Constants.LOCATION)) { @@ -363,6 +346,10 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { stmt.setLocation(location); } + if (lexer.identifierEquals(FnvHash.Constants.TBLPROPERTIES)) { + parseTblProperties(stmt); + } + if (lexer.identifierEquals(FnvHash.Constants.USING)) { lexer.nextToken(); SQLExpr using = this.exprParser.expr(); @@ -376,4 +363,27 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { return stmt; } + + private void parseTblProperties(OdpsCreateTableStatement stmt) { + acceptIdentifier("TBLPROPERTIES"); + accept(Token.LPAREN); + + for (;;) { + String name = lexer.stringVal(); + lexer.nextToken(); + accept(Token.EQ); + SQLExpr value = this.exprParser.primary(); + stmt.addTblProperty(name, value); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + if (lexer.token() == Token.RPAREN) { + break; + } + continue; + } + break; + } + + accept(Token.RPAREN); + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java index 33c4ea0aa1..a2254441ba 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java @@ -240,9 +240,12 @@ public class OdpsExprParser extends SQLExprParser { OdpsNewExpr newExpr = new OdpsNewExpr(); if (lexer.identifierEquals(FnvHash.Constants.GSON) - || lexer.identifierEquals(GSONBUILDER)) { + || lexer.identifierEquals(GSONBUILDER) + || lexer.identifierEquals("STRING") + ) { lexer.nextToken(); - newExpr.setMethodName(lexer.stringVal()); + String methodName = lexer.stringVal(); + newExpr.setMethodName(methodName); accept(Token.LPAREN); this.exprList(newExpr.getArguments(), newExpr); accept(Token.RPAREN); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java index 517a2a10bd..8619c885bb 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java @@ -233,4 +233,8 @@ public class OdpsSelectParser extends SQLSelectParser { } return tableSource; } + + public void parseTableSourceSample(SQLTableSource tableSource) { + parseTableSourceSampleHive(tableSource); + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java index 8820361dfe..4107d80931 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java @@ -618,6 +618,12 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } + if (lexer.identifierEquals("RECYCLEBIN")) { + lexer.nextToken(); + SQLShowRecylebinStatement stmt = new SQLShowRecylebinStatement(); + return stmt; + } + if (lexer.token() == Token.CREATE) { return parseShowCreateTable(); } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index 9d414f20b7..80eb77e370 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -1102,6 +1102,7 @@ public class SQLExprParser extends SQLParser { case DISTRIBUTE: case UNIQUE: case PROCEDURE: + case REFERENCES: if (dbType == DbType.odps || dbType == DbType.hive) { sqlExpr = new SQLIdentifierExpr(lexer.stringVal()); lexer.nextToken(); @@ -1144,9 +1145,11 @@ public class SQLExprParser extends SQLParser { lexer.nextToken(); switch (lexer.token) { case DOT: + case COMMA: case LT: case EQ: case GT: + case RPAREN: sqlExpr = new SQLIdentifierExpr(str); break; default: @@ -1227,6 +1230,7 @@ public class SQLExprParser extends SQLParser { case WHERE: case GROUP: case SEMI: + case AS: return new SQLIdentifierExpr(str); case IDENTIFIER: if (dbType == DbType.odps) { @@ -4734,6 +4738,8 @@ public class SQLExprParser extends SQLParser { || lexer.identifierEquals("utf8mb4") ) { // skip + } else if (lexer.token == Token.EQEQ && dbType == DbType.odps) { + lexer.nextToken(); } else { accept(Token.EQ); } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java index 7a962d7c80..23ab83d00c 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java @@ -251,6 +251,7 @@ public class SQLParser { } break; case SHOW: + case REFERENCES: case REPEAT: case USE: case OUT: { @@ -382,6 +383,7 @@ public class SQLParser { case SHOW: case SEQUENCE: case TO: + case REFERENCES: if (dbType == DbType.odps || dbType == DbType.hive) { alias = lexer.stringVal(); lexer.nextToken(); @@ -551,6 +553,8 @@ public class SQLParser { case CREATE: case DELETE: case PRIMARY: + case FOREIGN: + case REFERENCES: alias = lexer.stringVal(); lexer.nextToken(); return alias; diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java index ccc4d8a8d1..122b2ba0dd 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java @@ -1152,7 +1152,91 @@ public class SQLSelectParser extends SQLParser { return tableSource; } + public void parseTableSourceSample(SQLTableSource tableSource) { + + } + + public void parseTableSourceSampleHive(SQLTableSource tableSource) { + if (lexer.identifierEquals(FnvHash.Constants.TABLESAMPLE) && tableSource instanceof SQLExprTableSource) { + Lexer.SavePoint mark = lexer.mark(); + lexer.nextToken(); + if (lexer.token() == Token.LPAREN) { + lexer.nextToken(); + + SQLTableSampling sampling = new SQLTableSampling(); + + if (lexer.identifierEquals(FnvHash.Constants.BUCKET)) { + lexer.nextToken(); + SQLExpr bucket = this.exprParser.primary(); + sampling.setBucket(bucket); + + if (lexer.token() == Token.OUT) { + lexer.nextToken(); + accept(Token.OF); + SQLExpr outOf = this.exprParser.primary(); + sampling.setOutOf(outOf); + } + + if (lexer.token() == Token.ON) { + lexer.nextToken(); + SQLExpr on = this.exprParser.expr(); + sampling.setOn(on); + } + } + + if (lexer.token() == Token.LITERAL_INT || lexer.token() == Token.LITERAL_FLOAT) { + SQLExpr val = this.exprParser.primary(); + + if (lexer.identifierEquals(FnvHash.Constants.ROWS)) { + lexer.nextToken(); + sampling.setRows(val); + } else { + acceptIdentifier("PERCENT"); + sampling.setPercent(val); + } + } + + if (lexer.token() == Token.IDENTIFIER) { + String strVal = lexer.stringVal(); + char first = strVal.charAt(0); + char last = strVal.charAt(strVal.length() - 1); + if (last >= 'a' && last <= 'z') { + last -= 32; // to upper + } + + boolean match = false; + if ((first == '.' || (first >= '0' && first <= '9'))) { + switch (last) { + case 'B': + case 'K': + case 'M': + case 'G': + case 'T': + case 'P': + match = true; + break; + default: + break; + } + } + SQLSizeExpr size = new SQLSizeExpr(strVal.substring(0, strVal.length() - 2), last); + sampling.setByteLength(size); + lexer.nextToken(); + } + + final SQLExprTableSource table = (SQLExprTableSource) tableSource; + table.setSampling(sampling); + + accept(Token.RPAREN); + } else { + lexer.reset(mark); + } + } + } + public SQLTableSource parseTableSourceRest(SQLTableSource tableSource) { + parseTableSourceSample(tableSource); + if (lexer.hasComment() && lexer.isKeepComments() && !(tableSource instanceof SQLSubqueryTableSource)) { diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index 120a633b3f..c243be3e06 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -1133,6 +1133,16 @@ public class SQLStatementParser extends SQLParser { return stmt; } + + if (lexer.identifierEquals("TEMPORARY")) { + lexer.nextToken(); + acceptIdentifier("OUTPUT"); + SQLPurgeTemporaryOutputStatement stmt = new SQLPurgeTemporaryOutputStatement(); + stmt.setName( + this.exprParser.name()); + return stmt; + } + SQLPurgeLogsStatement stmt = new SQLPurgeLogsStatement(); if (lexer.token == Token.BINARY) { diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 699f556921..827614c2b9 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -11080,6 +11080,12 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return super.visit((SQLMethodInvokeExpr) x); } + public boolean visit(SQLPurgeTemporaryOutputStatement x) { + print0(ucase ? "PURGE TEMPORARY OUTPUT " : "purge temporary output "); + printExpr(x.getName()); + return false; + } + public char getNameQuote() { return quote; } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index 4ace1e6084..44583a8167 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -2212,6 +2212,14 @@ public interface SQLASTVisitor { } + default boolean visit(SQLPurgeTemporaryOutputStatement x) { + return true; + } + + default void endVisit(SQLPurgeTemporaryOutputStatement x) { + + } + default boolean visit(SQLPurgeLogsStatement x) { return true; }