From 2dcd0b87008e23316a9525b40b8414248fc6428c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Fri, 25 Jun 2021 11:00:34 +0800 Subject: [PATCH] improved odps sql parser support --- .../odps/ast/OdpsSelectQueryBlock.java | 20 +++++-- .../dialect/odps/ast/OdpsTransformExpr.java | 9 ++-- .../odps/parser/OdpsCreateTableParser.java | 3 +- .../dialect/odps/parser/OdpsExprParser.java | 18 ++++++- .../dialect/odps/parser/OdpsSelectParser.java | 3 +- .../odps/visitor/OdpsOutputVisitor.java | 25 ++++++++- .../com/alibaba/druid/sql/parser/Lexer.java | 53 +++++++++++++++---- .../druid/sql/parser/SQLExprParser.java | 26 +++++++++ .../alibaba/druid/sql/parser/SQLParser.java | 16 ++++++ .../druid/sql/parser/SQLSelectParser.java | 8 +++ .../druid/sql/parser/SQLStatementParser.java | 6 +++ .../druid/sql/visitor/SQLASTVisitor.java | 8 +++ 12 files changed, 173 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java index ce4b960100..3a1bcf5da5 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java @@ -17,10 +17,7 @@ package com.alibaba.druid.sql.dialect.odps.ast; import com.alibaba.druid.DbType; import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLExpr; -import com.alibaba.druid.sql.ast.SQLHint; -import com.alibaba.druid.sql.ast.SQLLimit; -import com.alibaba.druid.sql.ast.SQLOrderBy; +import com.alibaba.druid.sql.ast.*; import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; @@ -30,7 +27,7 @@ import com.alibaba.druid.sql.visitor.SQLASTVisitor; import java.util.ArrayList; public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { - + private SQLZOrderBy zOrderBy; public OdpsSelectQueryBlock(){ dbType = DbType.odps; @@ -64,6 +61,7 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { acceptChild(visitor, this.where); acceptChild(visitor, this.groupBy); acceptChild(visitor, this.orderBy); + acceptChild(visitor, this.zOrderBy); acceptChild(visitor, this.clusterBy); acceptChild(visitor, this.distributeBy); acceptChild(visitor, this.sortBy); @@ -85,4 +83,16 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { setLimit(new SQLLimit(new SQLIntegerExpr(rowCount))); } + + public SQLZOrderBy getZOrderBy() { + return zOrderBy; + } + + public void setZOrderBy(SQLZOrderBy x) { + if (x != null) { + x.setParent(this); + } + + this.zOrderBy = x; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsTransformExpr.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsTransformExpr.java index 8b078b33d4..d5cf66188c 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsTransformExpr.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsTransformExpr.java @@ -2,6 +2,7 @@ package com.alibaba.druid.sql.dialect.odps.ast; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLExprImpl; +import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition; import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat; import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor; import com.alibaba.druid.sql.visitor.SQLASTVisitor; @@ -11,10 +12,10 @@ import java.util.List; public class OdpsTransformExpr extends SQLExprImpl implements OdpsObject { private SQLExternalRecordFormat inputRowFormat; - private final List inputColumns = new ArrayList(); - private final List outputColumns = new ArrayList(); + private final List inputColumns = new ArrayList<>(); + private final List outputColumns = new ArrayList<>(); private SQLExpr using; - private final List resources = new ArrayList(); + private final List resources = new ArrayList<>(); private SQLExternalRecordFormat outputRowFormat; @Override @@ -58,7 +59,7 @@ public class OdpsTransformExpr extends SQLExprImpl implements OdpsObject { return inputColumns; } - public List getOutputColumns() { + public List getOutputColumns() { return outputColumns; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java index e46f146b92..34560a1662 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java @@ -256,7 +256,8 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { accept(Token.RPAREN); } - if (stmt.getClusteredBy().size() > 0 || stmt.getSortedBy().size() > 0) { + if (stmt.getClusteringType() != ClusteringType.Range && + (stmt.getClusteredBy().size() > 0 || stmt.getSortedBy().size() > 0)) { accept(Token.INTO); if (lexer.token() == Token.LITERAL_INT) { stmt.setBuckets(lexer.integerValue().intValue()); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java index 437b361380..4d5d4a4335 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java @@ -18,6 +18,7 @@ package com.alibaba.druid.sql.dialect.odps.parser; import com.alibaba.druid.DbType; import com.alibaba.druid.sql.ast.*; import com.alibaba.druid.sql.ast.expr.*; +import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition; import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.dialect.odps.ast.OdpsNewExpr; @@ -27,6 +28,7 @@ import com.alibaba.druid.sql.parser.*; import com.alibaba.druid.util.FnvHash; import java.util.Arrays; +import java.util.List; public class OdpsExprParser extends SQLExprParser { public final static String[] AGGREGATE_FUNCTIONS; @@ -202,7 +204,21 @@ public class OdpsExprParser extends SQLExprParser { if (lexer.token() == Token.AS) { lexer.nextToken(); - this.exprList(transformExpr.getOutputColumns(), transformExpr); + List outputColumns = transformExpr.getOutputColumns(); + + if (lexer.token() == Token.LPAREN) { + lexer.nextToken(); + for (; ; ) { + SQLColumnDefinition column = this.parseColumn(); + outputColumns.add(column); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } + accept(Token.RPAREN); + } } if (lexer.identifierEquals(FnvHash.Constants.ROW)) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java index f9e964a794..dc132e8d54 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java @@ -143,7 +143,8 @@ public class OdpsSelectParser extends SQLSelectParser { parseGroupBy(queryBlock); queryBlock.setOrderBy(this.exprParser.parseOrderBy()); - + queryBlock.setZOrderBy(this.exprParser.parseZOrderBy()); + if (lexer.token() == Token.DISTRIBUTE) { lexer.nextToken(); accept(Token.BY); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index 2c2b998ec4..6c3de2008d 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -471,6 +471,12 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit x.getOrderBy().accept(this); } + SQLZOrderBy zorderBy = x.getZOrderBy(); + if (zorderBy != null) { + println(); + zorderBy.accept(this); + } + final List distributeBy = x.getDistributeByDirect(); if (distributeBy.size() > 0) { println(); @@ -517,6 +523,23 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit return false; } + public boolean visit(SQLZOrderBy x) { + int itemSize = x.getItems().size(); + if (itemSize > 0) { + print0(ucase ? "ZORDER BY " : "zorder by "); + this.indentCount++; + for (int i = 0; i < itemSize; ++i) { + if (i != 0) { + println(", "); + } + x.getItems().get(i).accept(this); + } + this.indentCount--; + } + + return false; + } + @Override public boolean visit(OdpsAddStatisticStatement x) { print0(ucase ? "ADD STATISTIC " : "add statistic "); @@ -946,7 +969,7 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit printAndAccept(resources, ", "); } - List outputColumns = x.getOutputColumns(); + List outputColumns = x.getOutputColumns(); if (!outputColumns.isEmpty()) { println(); print0(ucase ? "AS (" : "as ("); diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index c87585aa63..e7f392232e 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -675,17 +675,33 @@ public class Lexer { ch = charAt(++pos); } - if (ch == '/' && pos + 1 < text.length() && text.charAt(pos + 1) == '*') { - int index = text.indexOf("*/", pos + 2); - if (index == -1) { - return SQLType.UNKNOWN; - } + if (ch == '/') { + if (pos + 1 < text.length() && text.charAt(pos + 1) == '*') { + int index = text.indexOf("*/", pos + 2); + if (index == -1) { + return SQLType.UNKNOWN; + } - pos = index + 2; - ch = charAt(pos); - continue; + pos = index + 2; + ch = charAt(pos); + continue; + } else if (pos + 2 < text.length() + && text.charAt(pos + 1) == ' ' + && text.charAt(pos + 2) == '*' + && dbType == DbType.odps + ) { + int index = text.indexOf("* /", pos + 3); + if (index == -1) { + return SQLType.UNKNOWN; + } + + pos = index + 3; + ch = charAt(pos); + continue; + } } + if (ch == '-' && pos + 1 < text.length() && text.charAt(pos + 1) == '-') { int index = text.indexOf('\n', pos + 2); if (index == -1) { @@ -1181,6 +1197,12 @@ public class Lexer { bufPos = 0; continue; } + } else if (nextChar == ' ' && charAt(pos + 2) == ' ' && charAt(pos + 3) == '*') { + scanComment(); + if ((token == Token.LINE_COMMENT || token == Token.MULTI_LINE_COMMENT) && skipComment) { + bufPos = 0; + continue; + } } else { token = Token.SLASH; scanChar(); @@ -2091,6 +2113,12 @@ public class Lexer { mark = pos; bufPos = 0; scanChar(); + + if (dbType == DbType.odps && ch == ' ') { + mark = pos; + bufPos = 0; + scanChar(); + } } // /*+ */ @@ -2114,13 +2142,20 @@ public class Lexer { for (;;) { char c2; if (ch == '*' - && ((c2 = charAt(pos + 1)) == '/' || (c2 == ' ' && charAt(pos + 2) == '/')) + && ((c2 = charAt(pos + 1)) == '/' + || (c2 == ' ' && charAt(pos + 2) == '/') + || (c2 == ' ' && charAt(pos + 2) == ' ' && charAt(pos + 3) == '/')) ) { if (c2 == ' ') { bufPos ++; scanChar(); } + if (charAt(pos + 1) == ' ') { + bufPos ++; + scanChar(); + } + bufPos += 2; scanChar(); scanChar(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index 7ad6882aa3..52576f9174 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -1271,6 +1271,7 @@ public class SQLExprParser extends SQLParser { case ORDER: case THEN: case END: + case AS: return new SQLIdentifierExpr(str); default: break; @@ -2398,6 +2399,31 @@ public class SQLExprParser extends SQLParser { return null; } + public SQLZOrderBy parseZOrderBy() { + if (lexer.identifierEquals("ZORDER")) { + SQLZOrderBy orderBy = new SQLZOrderBy(); + + lexer.nextToken(); + + if (lexer.identifierEquals(FnvHash.Constants.SIBLINGS)) { + lexer.nextToken(); + orderBy.setSibings(true); + } + + accept(Token.BY); + + orderBy(orderBy.getItems(), orderBy); + + if (lexer.token == Token.ORDER) { + throw new ParserException(lexer.info()); // dual order by + } + + return orderBy; + } + + return null; + } + public SQLOrderBy parseDistributeBy() { if (lexer.token == Token.DISTRIBUTE || lexer.identifierEquals("DISTRIBUTE")) { SQLOrderBy orderBy = new SQLOrderBy(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java index 7c34606299..aa2df62e4b 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java @@ -83,6 +83,10 @@ public class SQLParser { || token == Token.SELECT || token == Token.FROM || token == Token.WHERE) { + if (token == Token.WHERE && dbType == DbType.odps) { + return null; + } + if (must) { throw new ParserException("illegal alias. " + lexer.info()); } @@ -113,8 +117,18 @@ public class SQLParser { case COMMA: case WHERE: case INNER: + case LEFT: + case RIGHT: + case FULL: case ON: return ident; + case JOIN: + if (hash != FnvHash.Constants.NATURAL + && hash != FnvHash.Constants.CROSS) { + return ident; + } + lexer.reset(mark); + break; default: lexer.reset(mark); break; @@ -509,6 +523,8 @@ public class SQLParser { case BY: case EXCEPT: case TABLESPACE: + case CREATE: + case DELETE: alias = lexer.stringVal(); lexer.nextToken(); return alias; diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java index 14e4669a18..e85db2da1b 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java @@ -1022,6 +1022,14 @@ public class SQLSelectParser extends SQLParser { } } else if (lexer.token == Token.LPAREN) { tableSource = parseTableSource(); + + while (lexer.token == Token.UNION && tableSource instanceof SQLUnionQueryTableSource) { + SQLUnionQueryTableSource unionQueryTableSource = (SQLUnionQueryTableSource) tableSource; + SQLUnionQuery union = unionQueryTableSource.getUnion(); + unionQueryTableSource.setUnion( + (SQLUnionQuery) queryRest(union) + ); + } accept(Token.RPAREN); } else { tableSource = parseTableSource(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index 197fd31bfc..64d0446dad 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -911,6 +911,12 @@ public class SQLStatementParser extends SQLParser { accept(Token.VIEW); + if (lexer.token == Token.IF) { + lexer.nextToken(); + accept(Token.EXISTS); + stmt.setIfExists(true); + } + stmt.setName(this.exprParser.name()); return stmt; } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index d85614728a..4ace1e6084 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -278,6 +278,14 @@ public interface SQLASTVisitor { } + default boolean visit(SQLZOrderBy x) { + return true; + } + + default void endVisit(SQLZOrderBy x) { + + } + default boolean visit(SQLSelectOrderByItem x) { return true; }