Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improved odps sql parser support

Browse Source
This commit is contained in:
高铁
2021-06-28 20:58:45 +08:00
parent cb457de0c5
commit 2dcd0b8700
12 changed files with 173 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java
+15 -5
View File
@@ -17,10 +17,7 @@ package com.alibaba.druid.sql.dialect.odps.ast;
import com.alibaba.druid.DbType;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLHint;
import com.alibaba.druid.sql.ast.SQLLimit;
import com.alibaba.druid.sql.ast.SQLOrderBy;
import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
@@ -30,7 +27,7 @@ import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import java.util.ArrayList;
public class OdpsSelectQueryBlock extends SQLSelectQueryBlock {
private SQLZOrderBy zOrderBy;
public OdpsSelectQueryBlock(){
dbType = DbType.odps;
@@ -64,6 +61,7 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock {
acceptChild(visitor, this.where);
acceptChild(visitor, this.groupBy);
acceptChild(visitor, this.orderBy);
acceptChild(visitor, this.zOrderBy);
acceptChild(visitor, this.clusterBy);
acceptChild(visitor, this.distributeBy);
acceptChild(visitor, this.sortBy);
@@ -85,4 +83,16 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock {
setLimit(new SQLLimit(new SQLIntegerExpr(rowCount)));
}
public SQLZOrderBy getZOrderBy() {
return zOrderBy;
}
public void setZOrderBy(SQLZOrderBy x) {
if (x != null) {
x.setParent(this);
}
this.zOrderBy = x;
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsTransformExpr.java
+5 -4
View File
@@ -2,6 +2,7 @@ package com.alibaba.druid.sql.dialect.odps.ast;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLExprImpl;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
@@ -11,10 +12,10 @@ import java.util.List;
public class OdpsTransformExpr extends SQLExprImpl implements OdpsObject {
private SQLExternalRecordFormat inputRowFormat;
private final List<SQLExpr> inputColumns = new ArrayList<SQLExpr>();
private final List<SQLExpr> outputColumns = new ArrayList<SQLExpr>();
private final List<SQLExpr> inputColumns = new ArrayList<>();
private final List<SQLColumnDefinition> outputColumns = new ArrayList<>();
private SQLExpr using;
private final List<SQLExpr> resources = new ArrayList<SQLExpr>();
private final List<SQLExpr> resources = new ArrayList<>();
private SQLExternalRecordFormat outputRowFormat;
@Override
@@ -58,7 +59,7 @@ public class OdpsTransformExpr extends SQLExprImpl implements OdpsObject {
return inputColumns;
}
public List<SQLExpr> getOutputColumns() {
public List<SQLColumnDefinition> getOutputColumns() {
return outputColumns;
}
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java
+2 -1
View File
@@ -256,7 +256,8 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
accept(Token.RPAREN);
}
if (stmt.getClusteredBy().size() > 0 || stmt.getSortedBy().size() > 0) {
if (stmt.getClusteringType() != ClusteringType.Range &&
(stmt.getClusteredBy().size() > 0 || stmt.getSortedBy().size() > 0)) {
accept(Token.INTO);
if (lexer.token() == Token.LITERAL_INT) {
stmt.setBuckets(lexer.integerValue().intValue());
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java
+17 -1
View File
@@ -18,6 +18,7 @@ package com.alibaba.druid.sql.dialect.odps.parser;
import com.alibaba.druid.DbType;
import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsNewExpr;
@@ -27,6 +28,7 @@ import com.alibaba.druid.sql.parser.*;
import com.alibaba.druid.util.FnvHash;
import java.util.Arrays;
import java.util.List;
public class OdpsExprParser extends SQLExprParser {
public final static String[] AGGREGATE_FUNCTIONS;
@@ -202,7 +204,21 @@ public class OdpsExprParser extends SQLExprParser {
if (lexer.token() == Token.AS) {
lexer.nextToken();
this.exprList(transformExpr.getOutputColumns(), transformExpr);
List<SQLColumnDefinition> outputColumns = transformExpr.getOutputColumns();
if (lexer.token() == Token.LPAREN) {
lexer.nextToken();
for (; ; ) {
SQLColumnDefinition column = this.parseColumn();
outputColumns.add(column);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
}
break;
}
accept(Token.RPAREN);
}
}
if (lexer.identifierEquals(FnvHash.Constants.ROW)) {
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java
+2 -1
View File
@@ -143,7 +143,8 @@ public class OdpsSelectParser extends SQLSelectParser {
parseGroupBy(queryBlock);
queryBlock.setOrderBy(this.exprParser.parseOrderBy());
queryBlock.setZOrderBy(this.exprParser.parseZOrderBy());
if (lexer.token() == Token.DISTRIBUTE) {
lexer.nextToken();
accept(Token.BY);
src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java
+24 -1
View File
@@ -471,6 +471,12 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit
x.getOrderBy().accept(this);
}
SQLZOrderBy zorderBy = x.getZOrderBy();
if (zorderBy != null) {
println();
zorderBy.accept(this);
}
final List<SQLSelectOrderByItem> distributeBy = x.getDistributeByDirect();
if (distributeBy.size() > 0) {
println();
@@ -517,6 +523,23 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit
return false;
}
public boolean visit(SQLZOrderBy x) {
int itemSize = x.getItems().size();
if (itemSize > 0) {
print0(ucase ? "ZORDER BY " : "zorder by ");
this.indentCount++;
for (int i = 0; i < itemSize; ++i) {
if (i != 0) {
println(", ");
}
x.getItems().get(i).accept(this);
}
this.indentCount--;
}
return false;
}
@Override
public boolean visit(OdpsAddStatisticStatement x) {
print0(ucase ? "ADD STATISTIC " : "add statistic ");
@@ -946,7 +969,7 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit
printAndAccept(resources, ", ");
}
List<SQLExpr> outputColumns = x.getOutputColumns();
List<SQLColumnDefinition> outputColumns = x.getOutputColumns();
if (!outputColumns.isEmpty()) {
println();
print0(ucase ? "AS (" : "as (");
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+44 -9
View File
@@ -675,17 +675,33 @@ public class Lexer {
ch = charAt(++pos);
}
if (ch == '/' && pos + 1 < text.length() && text.charAt(pos + 1) == '*') {
int index = text.indexOf("*/", pos + 2);
if (index == -1) {
return SQLType.UNKNOWN;
}
if (ch == '/') {
if (pos + 1 < text.length() && text.charAt(pos + 1) == '*') {
int index = text.indexOf("*/", pos + 2);
if (index == -1) {
return SQLType.UNKNOWN;
}
pos = index + 2;
ch = charAt(pos);
continue;
pos = index + 2;
ch = charAt(pos);
continue;
} else if (pos + 2 < text.length()
&& text.charAt(pos + 1) == ' '
&& text.charAt(pos + 2) == '*'
&& dbType == DbType.odps
) {
int index = text.indexOf("* /", pos + 3);
if (index == -1) {
return SQLType.UNKNOWN;
}
pos = index + 3;
ch = charAt(pos);
continue;
}
}
if (ch == '-' && pos + 1 < text.length() && text.charAt(pos + 1) == '-') {
int index = text.indexOf('\n', pos + 2);
if (index == -1) {
@@ -1181,6 +1197,12 @@ public class Lexer {
bufPos = 0;
continue;
}
} else if (nextChar == ' ' && charAt(pos + 2) == ' ' && charAt(pos + 3) == '*') {
scanComment();
if ((token == Token.LINE_COMMENT || token == Token.MULTI_LINE_COMMENT) && skipComment) {
bufPos = 0;
continue;
}
} else {
token = Token.SLASH;
scanChar();
@@ -2091,6 +2113,12 @@ public class Lexer {
mark = pos;
bufPos = 0;
scanChar();
if (dbType == DbType.odps && ch == ' ') {
mark = pos;
bufPos = 0;
scanChar();
}
}
// /*+ */
@@ -2114,13 +2142,20 @@ public class Lexer {
for (;;) {
char c2;
if (ch == '*'
&& ((c2 = charAt(pos + 1)) == '/' || (c2 == ' ' && charAt(pos + 2) == '/'))
&& ((c2 = charAt(pos + 1)) == '/'
|| (c2 == ' ' && charAt(pos + 2) == '/')
|| (c2 == ' ' && charAt(pos + 2) == ' ' && charAt(pos + 3) == '/'))
) {
if (c2 == ' ') {
bufPos ++;
scanChar();
}
if (charAt(pos + 1) == ' ') {
bufPos ++;
scanChar();
}
bufPos += 2;
scanChar();
scanChar();
src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+26
View File
@@ -1271,6 +1271,7 @@ public class SQLExprParser extends SQLParser {
case ORDER:
case THEN:
case END:
case AS:
return new SQLIdentifierExpr(str);
default:
break;
@@ -2398,6 +2399,31 @@ public class SQLExprParser extends SQLParser {
return null;
}
public SQLZOrderBy parseZOrderBy() {
if (lexer.identifierEquals("ZORDER")) {
SQLZOrderBy orderBy = new SQLZOrderBy();
lexer.nextToken();
if (lexer.identifierEquals(FnvHash.Constants.SIBLINGS)) {
lexer.nextToken();
orderBy.setSibings(true);
}
accept(Token.BY);
orderBy(orderBy.getItems(), orderBy);
if (lexer.token == Token.ORDER) {
throw new ParserException(lexer.info()); // dual order by
}
return orderBy;
}
return null;
}
public SQLOrderBy parseDistributeBy() {
if (lexer.token == Token.DISTRIBUTE || lexer.identifierEquals("DISTRIBUTE")) {
SQLOrderBy orderBy = new SQLOrderBy();
src/main/java/com/alibaba/druid/sql/parser/SQLParser.java
+16
View File
@@ -83,6 +83,10 @@ public class SQLParser {
|| token == Token.SELECT
|| token == Token.FROM
|| token == Token.WHERE) {
if (token == Token.WHERE && dbType == DbType.odps) {
return null;
}
if (must) {
throw new ParserException("illegal alias. " + lexer.info());
}
@@ -113,8 +117,18 @@ public class SQLParser {
case COMMA:
case WHERE:
case INNER:
case LEFT:
case RIGHT:
case FULL:
case ON:
return ident;
case JOIN:
if (hash != FnvHash.Constants.NATURAL
&& hash != FnvHash.Constants.CROSS) {
return ident;
}
lexer.reset(mark);
break;
default:
lexer.reset(mark);
break;
@@ -509,6 +523,8 @@ public class SQLParser {
case BY:
case EXCEPT:
case TABLESPACE:
case CREATE:
case DELETE:
alias = lexer.stringVal();
lexer.nextToken();
return alias;
src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java
+8
View File
@@ -1022,6 +1022,14 @@ public class SQLSelectParser extends SQLParser {
}
} else if (lexer.token == Token.LPAREN) {
tableSource = parseTableSource();
while (lexer.token == Token.UNION && tableSource instanceof SQLUnionQueryTableSource) {
SQLUnionQueryTableSource unionQueryTableSource = (SQLUnionQueryTableSource) tableSource;
SQLUnionQuery union = unionQueryTableSource.getUnion();
unionQueryTableSource.setUnion(
(SQLUnionQuery) queryRest(union)
);
}
accept(Token.RPAREN);
} else {
tableSource = parseTableSource();
src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
+6
View File
@@ -911,6 +911,12 @@ public class SQLStatementParser extends SQLParser {
accept(Token.VIEW);
if (lexer.token == Token.IF) {
lexer.nextToken();
accept(Token.EXISTS);
stmt.setIfExists(true);
}
stmt.setName(this.exprParser.name());
return stmt;
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java
+8
View File
@@ -278,6 +278,14 @@ public interface SQLASTVisitor {
}
default boolean visit(SQLZOrderBy x) {
return true;
}
default void endVisit(SQLZOrderBy x) {
}
default boolean visit(SQLSelectOrderByItem x) {
return true;
}