Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improved sql parser

Browse Source
This commit is contained in:
高铁
2021-01-17 22:56:30 +08:00
parent 7a056ed7ae
commit c8bbfae175
21 changed files with 748 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/ast/SQLPartitionSpec.java
+81
View File
@@ -0,0 +1,81 @@
package com.alibaba.druid.sql.ast;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class SQLPartitionSpec extends SQLObjectImpl implements Cloneable {
private List<Item> items = new ArrayList<Item>();
@Override
protected void accept0(SQLASTVisitor v) {
if (v.visit(this)) {
acceptChild(v, items);
}
v.endVisit(this);
}
public void addItem(Item item) {
item.setParent(this);
items.add(item);
}
public List<Item> getItems() {
return items;
}
public SQLPartitionSpec clone() {
SQLPartitionSpec x = new SQLPartitionSpec();
for (Item item : items) {
x.addItem(item.clone());
}
return x;
}
public static class Item extends SQLObjectImpl implements Cloneable {
private SQLName column;
private SQLExpr value;
@Override
protected void accept0(SQLASTVisitor v) {
if (v.visit(this)) {
acceptChild(v, column);
acceptChild(v, value);
}
v.endVisit(this);
}
public Item clone() {
Item x = new Item();
if (column != null) {
x.setColumn(column.clone());
}
if (value != null) {
x.setValue(value.clone());
}
return x;
}
public SQLName getColumn() {
return column;
}
public void setColumn(SQLName column) {
this.column = column;
}
public SQLExpr getValue() {
return value;
}
public void setValue(SQLExpr value) {
this.value = value;
}
}
}
src/main/java/com/alibaba/druid/sql/ast/expr/SQLMethodInvokeExpr.java
+12 -10
View File
@@ -33,14 +33,14 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,
private static final long serialVersionUID = 1L;
protected String methodName;
protected long methodNameHashCode64;
protected SQLExpr owner;
protected final List<SQLExpr> arguments = new ArrayList<SQLExpr>();
private SQLExpr from;
private SQLExpr using;
private SQLExpr _for;
private String trimOption;
protected final List<SQLExpr> arguments = new ArrayList<SQLExpr>();
protected String methodName;
protected long methodNameHashCode64;
protected SQLExpr owner;
protected SQLExpr from;
protected SQLExpr using;
protected SQLExpr _for;
protected String trimOption;
protected transient SQLDataType resolvedReturnDataType;
public SQLMethodInvokeExpr(){
@@ -273,7 +273,11 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,
public SQLMethodInvokeExpr clone() {
SQLMethodInvokeExpr x = new SQLMethodInvokeExpr();
cloneTo(x);
return x;
}
public void cloneTo(SQLMethodInvokeExpr x) {
x.methodName = methodName;
if (owner != null) {
@@ -306,8 +310,6 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,
x.putAttribute(key, value);
}
}
return x;
}
@Override
src/main/java/com/alibaba/druid/sql/ast/statement/SQLAlterTableMergePartition.java
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLObjectImpl;
import com.alibaba.druid.sql.ast.SQLPartitionSpec;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import java.util.ArrayList;
import java.util.List;
public class SQLAlterTableMergePartition extends SQLObjectImpl implements SQLAlterTableItem {
private final List<SQLPartitionSpec> partitions = new ArrayList<SQLPartitionSpec>(4);
private boolean ifExists = false;
private SQLPartitionSpec overwritePartition;
public List<SQLPartitionSpec> getPartitions() {
return partitions;
}
public void addPartition(SQLPartitionSpec x) {
if (x != null) {
x.setParent(this);
}
this.partitions.add(x);
}
public boolean isIfExists() {
return ifExists;
}
public void setIfExists(boolean ifExists) {
this.ifExists = ifExists;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, partitions);
}
visitor.endVisit(this);
}
public SQLPartitionSpec getOverwritePartition() {
return overwritePartition;
}
public void setOverwritePartition(SQLPartitionSpec x) {
if (x != null) {
x.setParent(this);
}
this.overwritePartition = x;
}
}
src/main/java/com/alibaba/druid/sql/ast/statement/SQLUnionOperator.java
+2
View File
@@ -21,7 +21,9 @@ public enum SQLUnionOperator {
MINUS("MINUS"),
EXCEPT("EXCEPT"),
EXCEPT_ALL("EXCEPT ALL"),
EXCEPT_DISTINCT("EXCEPT DISTINCT"),
INTERSECT("INTERSECT"),
INTERSECT_ALL("INTERSECT ALL"),
INTERSECT_DISTINCT("INTERSECT DISTINCT"),
DISTINCT("UNION DISTINCT");
src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveMsckRepairStatement.java
+8
View File
@@ -13,6 +13,7 @@ import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class HiveMsckRepairStatement extends SQLStatementImpl implements SQLAlterStatement {
private SQLExprTableSource table;
private SQLName database;
private boolean addPartitions;
public HiveMsckRepairStatement() {
super(DbType.hive);
@@ -34,6 +35,13 @@ public class HiveMsckRepairStatement extends SQLStatementImpl implements SQLAlte
visitor.endVisit(this);
}
public boolean isAddPartitions() {
return addPartitions;
}
public void setAddPartitions(boolean addPartitions) {
this.addPartitions = addPartitions;
}
public SQLExprTableSource getTable() {
return table;
src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java
+4
View File
@@ -328,6 +328,10 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
print0(ucase ? " TABLE " : " table ");
table.accept(this);
}
if (x.isAddPartitions()) {
print0(ucase ? " ADD PARTITIONS" : " add partitions");
}
return false;
}
src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsNewExpr.java
+62
View File
@@ -0,0 +1,62 @@
package com.alibaba.druid.sql.dialect.odps.ast;
import com.alibaba.druid.FastsqlException;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import java.io.IOException;
public class OdpsNewExpr extends SQLMethodInvokeExpr implements OdpsObject {
@Override
public OdpsNewExpr clone() {
OdpsNewExpr x = new OdpsNewExpr();
cloneTo(x);
return x;
}
@Override
public void accept0(SQLASTVisitor v) {
accept0((OdpsASTVisitor) v);
}
@Override
public void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
if (this.owner != null) {
this.owner.accept(visitor);
}
for (SQLExpr arg : this.arguments) {
if (arg != null) {
arg.accept(visitor);
}
}
if (this.from != null) {
this.from.accept(visitor);
}
if (this.using != null) {
this.using.accept(visitor);
}
if (this._for != null) {
this._for.accept(visitor);
}
visitor.endVisit(this);
}
visitor.endVisit(this);
}
public void output(Appendable buf) {
try {
buf.append("new ");
} catch (IOException ex) {
throw new FastsqlException("output error", ex);
}
super.output(buf);
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java
+10 -1
View File
@@ -142,7 +142,14 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
case CLOSE:
case IN:
case OUT:
column = this.exprParser.parseColumn();
case LIMIT:
case FULL:
case MINUS:
case VALUES:
case TRIGGER:
case USE:
case LIKE:
column = this.exprParser.parseColumn(stmt);
break;
default:
throw new ParserException("expect identifier. " + lexer.info());
@@ -182,6 +189,8 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
switch (lexer.token()) {
case KEY:
case IDENTIFIER:
case GROUP:
case INTERVAL:
break;
default:
throw new ParserException("expect identifier. " + lexer.info());
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java
+36 -13
View File
@@ -20,6 +20,7 @@ import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsNewExpr;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsTransformExpr;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsUDTFSQLSelectItem;
import com.alibaba.druid.sql.parser.*;
@@ -75,16 +76,31 @@ public class OdpsExprParser extends SQLExprParser {
return new SQLCharExpr(chars);
}
final static long GSONBUILDER = FnvHash.fnv1a_64_lower("GSONBUILDER");
@Override
public SQLSelectItem parseSelectItem() {
SQLExpr expr;
if (lexer.token() == Token.IDENTIFIER) {
expr = new SQLIdentifierExpr(lexer.stringVal());
String stringVal = lexer.stringVal();
long hash_lower = lexer.hash_lower();
lexer.nextTokenComma();
if (lexer.token() != Token.COMMA) {
expr = this.primaryRest(expr);
expr = this.exprRest(expr);
if (FnvHash.Constants.DATETIME == hash_lower
&& lexer.stringVal().charAt(0) != '`'
&& lexer.token() == Token.LITERAL_CHARS) {
String literal = lexer.stringVal();
lexer.nextToken();
SQLDateTimeExpr ts = new SQLDateTimeExpr(literal);
expr = ts;
} else {
expr = new SQLIdentifierExpr(stringVal);
if (lexer.token() != Token.COMMA) {
expr = this.primaryRest(expr);
expr = this.exprRest(expr);
}
}
} else {
expr = expr();
@@ -123,23 +139,23 @@ public class OdpsExprParser extends SQLExprParser {
} else {
alias = as();
}
SQLSelectItem item = new SQLSelectItem(expr, alias);
if (lexer.hasComment() && lexer.isKeepComments()) {
item.addAfterComment(lexer.readAndResetComments());
}
return item;
}
public SQLExpr primaryRest(SQLExpr expr) {
if(lexer.token() == Token.COLON) {
lexer.nextToken();
expr = dotRest(expr);
return expr;
}
if (lexer.token() == Token.LBRACKET) {
SQLArrayExpr array = new SQLArrayExpr();
array.setExpr(expr);
@@ -197,13 +213,20 @@ public class OdpsExprParser extends SQLExprParser {
return transformExpr;
}
if (expr instanceof SQLIdentifierExpr && ((SQLIdentifierExpr) expr).nameHashCode64() == FnvHash.Constants.NEW) {
if (expr instanceof SQLIdentifierExpr
&& ((SQLIdentifierExpr) expr).nameHashCode64() == FnvHash.Constants.NEW) {
SQLIdentifierExpr ident = (SQLIdentifierExpr) expr;
if (lexer.identifierEquals(FnvHash.Constants.GSON)) {
ident.setName(ident.getName() + ' ' + lexer.stringVal());
OdpsNewExpr newExpr = new OdpsNewExpr();
if (lexer.identifierEquals(FnvHash.Constants.GSON)
|| lexer.identifierEquals(GSONBUILDER)) {
lexer.nextToken();
} else if (lexer.identifierEquals("java")) {
newExpr.setMethodName(lexer.stringVal());
accept(Token.LPAREN);
this.exprList(newExpr.getArguments(), newExpr);
accept(Token.RPAREN);
expr = newExpr;
} else if (lexer.identifierEquals("java") || lexer.identifierEquals("com")) {
SQLName name = this.name();
String strName = ident.getName() + ' ' + name.toString();
if (lexer.token() == Token.LT) {
@@ -221,7 +244,7 @@ public class OdpsExprParser extends SQLExprParser {
}
}
return super.primaryRest(expr);
}
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java
+51 -14
View File
@@ -55,6 +55,7 @@ public class OdpsSelectParser extends SQLSelectParser {
if (lexer.token() == Token.FROM) {
parseFrom(queryBlock);
parseWhere(queryBlock);
parseGroupBy(queryBlock);
if (lexer.token() == Token.SELECT) {
lexer.nextToken();
@@ -84,6 +85,10 @@ public class OdpsSelectParser extends SQLSelectParser {
parseSelectList(queryBlock);
}
if (queryBlock.getWhere() == null && lexer.token() == Token.WHERE) {
parseWhere(queryBlock);
}
} else {
accept(Token.SELECT);
@@ -92,22 +97,29 @@ public class OdpsSelectParser extends SQLSelectParser {
}
if (lexer.token() == Token.COMMENT) {
String tokenStr = lexer.stringVal();
lexer.nextToken();
if (lexer.token() == Token.COMMA) {
queryBlock.addSelectItem(new SQLIdentifierExpr(tokenStr));
lexer.nextToken();
}
}
if (lexer.token() == Token.DISTINCT) {
queryBlock.setDistionOption(SQLSetQuantifier.DISTINCT);
lexer.nextToken();
} else if (lexer.token() == Token.UNIQUE) {
queryBlock.setDistionOption(SQLSetQuantifier.UNIQUE);
lexer.nextToken();
} else if (lexer.token() == Token.ALL) {
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
if (lexer.token() == Token.DOT) {
lexer.reset(mark);
} else {
queryBlock.setDistionOption(SQLSetQuantifier.ALL);
if (queryBlock.getSelectList().isEmpty()) {
if (lexer.token() == Token.DISTINCT) {
queryBlock.setDistionOption(SQLSetQuantifier.DISTINCT);
lexer.nextToken();
} else if (lexer.token() == Token.UNIQUE) {
queryBlock.setDistionOption(SQLSetQuantifier.UNIQUE);
lexer.nextToken();
} else if (lexer.token() == Token.ALL) {
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
if (lexer.token() == Token.DOT) {
lexer.reset(mark);
} else {
queryBlock.setDistionOption(SQLSetQuantifier.ALL);
}
}
}
@@ -121,6 +133,7 @@ public class OdpsSelectParser extends SQLSelectParser {
}
parseWhere(queryBlock);
parseGroupBy(queryBlock);
}
if (lexer.identifierEquals(FnvHash.Constants.WINDOW)) {
@@ -193,6 +206,30 @@ public class OdpsSelectParser extends SQLSelectParser {
lexer.nextToken();
return new SQLExprTableSource(new SQLIdentifierExpr(str));
}
return super.parseTableSource();
SQLTableSource tableSource = super.parseTableSource();
if (lexer.token() == Token.TABLE && tableSource.getAlias() == null) {
tableSource.setAlias(
lexer.stringVal()
);
lexer.nextToken();
tableSource = parseTableSourceRest(tableSource);
}
return tableSource;
}
protected SQLTableSource primaryTableSourceRest(SQLTableSource tableSource) {
if (lexer.identifierEquals(FnvHash.Constants.LATERAL) || lexer.token() == Token.LATERAL) {
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
if (lexer.token() == Token.VIEW) {
tableSource = parseLateralView(tableSource);
} else {
lexer.reset(mark);
}
}
return tableSource;
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java
+6 -6
View File
@@ -49,12 +49,12 @@ public class OdpsStatementParser extends SQLStatementParser {
SQLSelect select = new OdpsSelectParser(this.exprParser)
.select();
if (select.getWithSubQuery() == null && select.getQuery() instanceof SQLSelectQueryBlock) {
SQLSelectQueryBlock queryBlock = (SQLSelectQueryBlock) select.getQuery();
if (queryBlock.getFrom() == null && queryBlock.getWhere() != null) {
throw new ParserException("none from query not support where clause.");
}
}
// if (select.getWithSubQuery() == null && select.getQuery() instanceof SQLSelectQueryBlock) {
// SQLSelectQueryBlock queryBlock = (SQLSelectQueryBlock) select.getQuery();
// if (queryBlock.getFrom() == null && queryBlock.getWhere() != null) {
// throw new ParserException("none from query not support where clause.");
// }
// }
return new SQLSelectStatement(select, DbType.odps);
}
src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java
+8
View File
@@ -232,4 +232,12 @@ public interface OdpsASTVisitor extends HiveASTVisitor {
}
default boolean visit(OdpsNewExpr x) {
return true;
}
default void endVisit(OdpsNewExpr x) {
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java
+13 -1
View File
@@ -18,6 +18,7 @@ package com.alibaba.druid.sql.dialect.odps.visitor;
import com.alibaba.druid.DbType;
import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
@@ -593,7 +594,11 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit
protected void printMethodOwner(SQLExpr owner) {
owner.accept(this);
print(':');
if (owner instanceof SQLMethodInvokeExpr) {
print('.');
} else {
print(':');
}
}
protected void printJoinType(JoinType joinType) {
@@ -956,4 +961,11 @@ public class OdpsOutputVisitor extends HiveOutputVisitor implements OdpsASTVisit
return false;
}
@Override
public boolean visit(OdpsNewExpr x) {
print0("new ");
super.visit((SQLMethodInvokeExpr) x);
return false;
}
}
src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+72 -17
View File
@@ -779,6 +779,7 @@ public class SQLExprParser extends SQLParser {
parseQueryPlanHint(sqlExpr);
}
break;
case FROM:
case SELECT:
SQLQueryExpr queryExpr = new SQLQueryExpr(
createSelectParser()
@@ -866,6 +867,7 @@ public class SQLExprParser extends SQLParser {
case NULL:
case INTERVAL:
case LBRACE:
case IF:
sqlExpr = primary();
while (lexer.token == Token.HINT) {
@@ -1079,6 +1081,10 @@ public class SQLExprParser extends SQLParser {
case DELETE:
case BY:
case UPDATE:
case LOOP:
case LIKE:
case UNION:
case CREATE:
if (dbType == DbType.odps) {
sqlExpr = new SQLIdentifierExpr(lexer.stringVal());
lexer.nextToken();
@@ -1135,6 +1141,15 @@ public class SQLExprParser extends SQLParser {
}
}
throw new ParserException("ERROR. " + lexer.info());
case COLONCOLON:
if (dbType == DbType.odps) {
lexer.nextToken();
SQLMethodInvokeExpr method = (SQLMethodInvokeExpr) this.primary();
method.setOwner(new SQLIdentifierExpr(""));
sqlExpr = method;
break;
}
throw new ParserException("ERROR. " + lexer.info());
default:
throw new ParserException("ERROR. " + lexer.info());
}
@@ -1152,7 +1167,7 @@ public class SQLExprParser extends SQLParser {
SQLExpr sqlExpr;
String str = lexer.stringVal();
lexer.nextToken();
if (lexer.token == Token.DOT) {
if (lexer.token == Token.DOT || lexer.token == Token.SLASH) {
return primaryRest(new SQLIdentifierExpr(str));
} else if (lexer.token == Token.COMMA) {
return new SQLIdentifierExpr(str);
@@ -1244,6 +1259,9 @@ public class SQLExprParser extends SQLParser {
case SLASH:
case DOT:
case FROM:
case ORDER:
case THEN:
case END:
return new SQLIdentifierExpr(str);
default:
break;
@@ -1357,7 +1375,7 @@ public class SQLExprParser extends SQLParser {
if (lexer.token == Token.LPAREN &&
!(expr instanceof SQLIntegerExpr) && !(expr instanceof SQLHexExpr)) {
SQLExpr method = methodRest(expr, true);
if (lexer.token == Token.LBRACKET) {
if (lexer.token == Token.LBRACKET || lexer.token == Token.DOT) {
method = primaryRest(method);
}
return method;
@@ -2023,6 +2041,10 @@ public class SQLExprParser extends SQLParser {
case VALUES:
case IN:
case OUT:
case LIMIT:
case TRIGGER:
case USE:
case LIKE:
if (dbType == DbType.odps) {
identName = lexer.stringVal();
lexer.nextToken();
@@ -2227,7 +2249,7 @@ public class SQLExprParser extends SQLParser {
|| lexer.token == Token.LITERAL_FLOAT
|| lexer.token == Token.LITERAL_CHARS
) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenBegin(betweenBegin);
} else if (lexer.token == Token.IDENTIFIER) {
long hash = lexer.hash_lower();
@@ -2252,7 +2274,7 @@ public class SQLExprParser extends SQLParser {
|| lexer.token == Token.LITERAL_FLOAT
|| lexer.token == Token.LITERAL_CHARS
) {
SQLExpr betweenEnd = this.primary();
SQLExpr betweenEnd = this.additive();
over.setWindowingBetweenEnd(betweenEnd);
} else if (lexer.token == Token.IDENTIFIER) {
long hash = lexer.hash_lower();
@@ -2261,7 +2283,7 @@ public class SQLExprParser extends SQLParser {
&& hash != FnvHash.Constants.FOLLOWING
&& hash != FnvHash.Constants.CURRENT
&& hash != FnvHash.Constants.UNBOUNDED) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenEnd(betweenBegin);
}
}
@@ -2275,7 +2297,7 @@ public class SQLExprParser extends SQLParser {
|| lexer.token == Token.LITERAL_FLOAT
|| lexer.token == Token.LITERAL_CHARS
) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenBegin(betweenBegin);
} else if (lexer.token == Token.IDENTIFIER) {
long hash = lexer.hash_lower();
@@ -2284,7 +2306,7 @@ public class SQLExprParser extends SQLParser {
&& hash != FnvHash.Constants.FOLLOWING
&& hash != FnvHash.Constants.CURRENT
&& hash != FnvHash.Constants.UNBOUNDED) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenBegin(betweenBegin);
}
}
@@ -4261,17 +4283,21 @@ public class SQLExprParser extends SQLParser {
lexer.nextToken();
if (dbType == DbType.odps) {
if (lexer.token == Token.LITERAL_ALIAS) {
String tmp = lexer.stringVal();
if (tmp.length() > 2 && tmp.charAt(0) == '"' && tmp.charAt(tmp.length() - 1) == '"') {
tmp = tmp.substring(1, tmp.length() - 1);
}
for (;;) {
if (lexer.token == Token.LITERAL_ALIAS) {
String tmp = lexer.stringVal();
if (tmp.length() > 2 && tmp.charAt(0) == '"' && tmp.charAt(tmp.length() - 1) == '"') {
tmp = tmp.substring(1, tmp.length() - 1);
}
stringVal += tmp;
lexer.nextToken();
} else if (lexer.token == Token.LITERAL_CHARS) {
stringVal += lexer.stringVal();
lexer.nextToken();
stringVal += tmp;
lexer.nextToken();
} else if (lexer.token == Token.LITERAL_CHARS) {
stringVal += lexer.stringVal();
lexer.nextToken();
} else {
break;
}
}
}
@@ -5612,6 +5638,35 @@ public class SQLExprParser extends SQLParser {
throw new ParserException("TODO");
}
public SQLPartitionSpec parsePartitionSpec() {
SQLPartitionSpec spec = new SQLPartitionSpec();
accept(Token.PARTITION);
accept(Token.LPAREN);
for (;;) {
SQLPartitionSpec.Item item = new SQLPartitionSpec.Item();
item.setColumn(
this.name());
accept(Token.EQ);
item.setValue(
this.expr());
spec.addItem(item);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
} else {
break;
}
}
accept(Token.RPAREN);
return spec;
}
protected SQLPartitionBy parsePartitionBy() {
lexer.nextToken();
accept(Token.BY);
src/main/java/com/alibaba/druid/sql/parser/SQLParser.java
+7
View File
@@ -297,6 +297,7 @@ public class SQLParser {
case DEFERRED:
case OUTER:
case DO:
case LOOP:
case STORE:
case MOD:
case ANY:
@@ -337,6 +338,9 @@ public class SQLParser {
case INNER:
case CLOSE:
case VALUES:
case SHOW:
case SEQUENCE:
case TO:
if (dbType == DbType.odps) {
alias = lexer.stringVal();
lexer.nextToken();
@@ -345,6 +349,7 @@ public class SQLParser {
break;
case GROUP:
case ORDER:
case DEFAULT:
if (dbType == DbType.odps) {
Lexer.SavePoint mark = lexer.mark();
alias = lexer.stringVal();
@@ -486,6 +491,7 @@ public class SQLParser {
case PARTITION:
case UPDATE:
case DO:
case LOOP:
case REPEAT:
case DEFAULT:
case LIKE:
@@ -499,6 +505,7 @@ public class SQLParser {
case IN:
case OUT:
case BY:
case EXCEPT:
alias = lexer.stringVal();
lexer.nextToken();
return alias;
src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java
+101 -92
View File
@@ -232,6 +232,9 @@ public class SQLSelectParser extends SQLParser {
if (lexer.token == Token.ALL) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.EXCEPT_ALL);
} else if (lexer.token == Token.DISTINCT) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.EXCEPT_DISTINCT);
} else {
union.setOperator(SQLUnionOperator.EXCEPT);
}
@@ -283,6 +286,9 @@ public class SQLSelectParser extends SQLParser {
if (lexer.token() == Token.DISTINCT) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.INTERSECT_DISTINCT);
} else if (lexer.token == Token.ALL) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.INTERSECT_ALL);
} else {
union.setOperator(SQLUnionOperator.INTERSECT);
}
@@ -596,111 +602,114 @@ public class SQLSelectParser extends SQLParser {
}
public void parseWhere(SQLSelectQueryBlock queryBlock) {
if (lexer.token == Token.WHERE) {
lexer.nextTokenIdent();
if (lexer.token != Token.WHERE) {
return;
}
List<String> beforeComments = null;
if (lexer.hasComment() && lexer.isKeepComments()) {
beforeComments = lexer.readAndResetComments();
}
lexer.nextTokenIdent();
SQLExpr where;
List<String> beforeComments = null;
if (lexer.hasComment() && lexer.isKeepComments()) {
beforeComments = lexer.readAndResetComments();
}
if (lexer.token == Token.IDENTIFIER) {
String ident = lexer.stringVal();
long hash_lower = lexer.hash_lower();
lexer.nextTokenEq();
SQLExpr where;
SQLExpr identExpr;
if (lexer.token == Token.LITERAL_CHARS) {
String literal = lexer.stringVal;
if (hash_lower == FnvHash.Constants.TIMESTAMP) {
identExpr = new SQLTimestampExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.DATE) {
identExpr = new SQLDateExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.REAL) {
identExpr = new SQLRealExpr(Float.parseFloat(literal));
lexer.nextToken();
} else {
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
if (lexer.token == Token.IDENTIFIER) {
String ident = lexer.stringVal();
long hash_lower = lexer.hash_lower();
lexer.nextTokenEq();
SQLExpr identExpr;
if (lexer.token == Token.LITERAL_CHARS) {
String literal = lexer.stringVal;
if (hash_lower == FnvHash.Constants.TIMESTAMP) {
identExpr = new SQLTimestampExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.DATE) {
identExpr = new SQLDateExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.REAL) {
identExpr = new SQLRealExpr(Float.parseFloat(literal));
lexer.nextToken();
} else {
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
if (lexer.token == Token.DOT) {
identExpr = this.exprParser.primaryRest(identExpr);
}
if (lexer.token == Token.EQ) {
SQLExpr rightExp;
lexer.nextToken();
try {
rightExp = this.exprParser.bitOr();
} catch (EOFParserException e) {
throw new ParserException("EOF, " + ident + "=", e);
}
where = new SQLBinaryOpExpr(identExpr, SQLBinaryOperator.Equality, rightExp, dbType);
switch (lexer.token) {
case BETWEEN:
case IS:
case EQ:
case IN:
case CONTAINS:
case BANG_TILDE_STAR:
case TILDE_EQ:
case LT:
case LTEQ:
case LTEQGT:
case GT:
case GTEQ:
case LTGT:
case BANGEQ:
case LIKE:
case NOT:
where = this.exprParser.relationalRest(where);
break;
default:
break;
}
where = this.exprParser.andRest(where);
where = this.exprParser.xorRest(where);
where = this.exprParser.orRest(where);
} else {
identExpr = this.exprParser.primaryRest(identExpr);
where = this.exprParser.exprRest(identExpr);
}
} else {
while (lexer.token == Token.HINT) {
lexer.nextToken();
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
if (lexer.token == Token.DOT) {
identExpr = this.exprParser.primaryRest(identExpr);
}
if (lexer.token == Token.EQ) {
SQLExpr rightExp;
lexer.nextToken();
try {
rightExp = this.exprParser.bitOr();
} catch (EOFParserException e) {
throw new ParserException("EOF, " + ident + "=", e);
}
where = this.exprParser.expr();
while (lexer.token == Token.HINT) {
lexer.nextToken();
where = new SQLBinaryOpExpr(identExpr, SQLBinaryOperator.Equality, rightExp, dbType);
switch (lexer.token) {
case BETWEEN:
case IS:
case EQ:
case IN:
case CONTAINS:
case BANG_TILDE_STAR:
case TILDE_EQ:
case LT:
case LTEQ:
case LTEQGT:
case GT:
case GTEQ:
case LTGT:
case BANGEQ:
case LIKE:
case NOT:
where = this.exprParser.relationalRest(where);
break;
default:
break;
}
where = this.exprParser.andRest(where);
where = this.exprParser.xorRest(where);
where = this.exprParser.orRest(where);
} else {
identExpr = this.exprParser.primaryRest(identExpr);
where = this.exprParser.exprRest(identExpr);
}
// where = this.exprParser.expr();
if (beforeComments != null) {
where.addBeforeComment(beforeComments);
} else {
while (lexer.token == Token.HINT) {
lexer.nextToken();
}
if (lexer.hasComment() && lexer.isKeepComments() //
&& lexer.token != Token.INSERT // odps multi-insert
) {
where.addAfterComment(lexer.readAndResetComments());
where = this.exprParser.expr();
while (lexer.token == Token.HINT) {
lexer.nextToken();
}
queryBlock.setWhere(where);
}
// where = this.exprParser.expr();
if (beforeComments != null) {
where.addBeforeComment(beforeComments);
}
if (lexer.hasComment() && lexer.isKeepComments() //
&& lexer.token != Token.INSERT // odps multi-insert
) {
where.addAfterComment(lexer.readAndResetComments());
}
queryBlock.setWhere(where);
}
protected void parseSortBy(SQLSelectQueryBlock queryBlock) {
@@ -1391,7 +1400,7 @@ public class SQLSelectParser extends SQLParser {
rightTableSource = exprTableSource;
}
primaryTableSourceRest(rightTableSource);
rightTableSource = primaryTableSourceRest(rightTableSource);
}
if (lexer.token == Token.USING
@@ -1438,7 +1447,7 @@ public class SQLSelectParser extends SQLParser {
}
}
primaryTableSourceRest(rightTableSource);
rightTableSource = primaryTableSourceRest(rightTableSource);
}
if (lexer.token == Token.WITH) {
src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
+42 -2
View File
@@ -2183,8 +2183,41 @@ public class SQLStatementParser extends SQLParser {
stmt.addItem(item);
} else if (DbType.odps == dbType && lexer.identifierEquals("MERGE")) {
lexer.nextToken();
acceptIdentifier("SMALLFILES");
stmt.setMergeSmallFiles(true);
boolean ifExists = false;
if (lexer.token == Token.IF) {
lexer.nextToken();
accept(Token.EXISTS);
ifExists = true;
}
if (lexer.token == PARTITION) {
SQLAlterTableMergePartition item = new SQLAlterTableMergePartition();
for (;;) {
item.addPartition(
this.getExprParser().parsePartitionSpec()
);
if (lexer.token == COMMA) {
lexer.nextToken();
} else {
break;
}
}
accept(OVERWRITE);
item.setOverwritePartition(
this.getExprParser().parsePartitionSpec()
);
if (ifExists) {
item.setIfExists(true);
}
stmt.addItem(item);
} else {
acceptIdentifier("SMALLFILES");
stmt.setMergeSmallFiles(true);
}
} else if (DbType.odps == dbType && lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) {
lexer.nextToken();
accept(Token.BY);
@@ -6627,6 +6660,13 @@ public class SQLStatementParser extends SQLParser {
SQLExpr tableExpr = this.exprParser.expr();
stmt.setTable(tableExpr);
}
if (lexer.identifierEquals(FnvHash.Constants.ADD)) {
lexer.nextToken();
acceptIdentifier("PARTITIONS");
stmt.setAddPartitions(true);
}
return stmt;
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java
+34
View File
@@ -30,6 +30,7 @@ import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlPrimaryKey;
import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.*;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsNewExpr;
import com.alibaba.druid.sql.dialect.oracle.ast.OracleSegmentAttributes;
import com.alibaba.druid.sql.dialect.oracle.ast.expr.OracleCursorExpr;
import com.alibaba.druid.sql.dialect.oracle.ast.expr.OracleDatetimeExpr;
@@ -9038,6 +9039,34 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
return false;
}
public boolean visit(SQLAlterTableMergePartition x) {
print0(ucase ? "MERGE " : "merge ");
if (x.isIfExists()) {
print0(ucase ? "IF EXISTS " : "if exists ");
}
println();
printlnAndAccept(x.getPartitions(), ", ");
println();
print0(ucase ? "OVERWRITE" : "overwrite");
println();
x.getOverwritePartition().accept(this);
return false;
}
public boolean visit(SQLPartitionSpec x) {
print0(ucase ? "PARTITION (" : "partition (");
printAndAccept(x.getItems(), ", ");
print(')');
return false;
}
public boolean visit(SQLPartitionSpec.Item x) {
x.getColumn().accept(this);
print0(" = ");
x.getValue().accept(this);
return false;
}
public boolean visit(SQLAlterTableSubpartitionAvailablePartitionNum x) {
print0(ucase ? "SUBPARTITION_AVAILABLE_PARTITION_NUM = " : "subpartition_available_partition_num = ");
x.getNumber().accept(this);
@@ -10960,6 +10989,11 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
return false;
}
public boolean visit(OdpsNewExpr x) {
print0(ucase ? "NEW " : "new ");
return super.visit((SQLMethodInvokeExpr) x);
}
public char getNameQuote() {
return quote;
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java
+25
View File
@@ -2595,4 +2595,29 @@ public interface SQLASTVisitor {
default boolean visit(SQLListResourceGroupStatement x) {
return true;
}
default void endVisit(SQLAlterTableMergePartition x) {
}
default boolean visit(SQLAlterTableMergePartition x) {
return true;
}
default void endVisit(SQLPartitionSpec x) {
}
default boolean visit(SQLPartitionSpec x) {
return true;
}
default void endVisit(SQLPartitionSpec.Item x) {
}
default boolean visit(SQLPartitionSpec.Item x) {
return true;
}
}
src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest21_issue3980.java
+59
View File
@@ -0,0 +1,59 @@
package com.alibaba.druid.bvt.sql.schemaStat;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.repository.SchemaRepository;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
public class SchemaStatTest21_issue3980 extends TestCase {
public void test_schemaStat() throws Exception {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = "select n.user_name,\n" +
"n.user_passwd,\n" +
"n.user_project,\n" +
"n.start_date,\n" +
"n.end_date\n" +
"from (\n" +
"select t.name as user_name,\n" +
"t.passwd as user_passwd,\n" +
"cast(from_unixtime(t.from_time, \"yyyyMMdd\") as int) as start_date,\n" +
"cast(from_unixtime(t.to_time, \"yyyyMMdd\") as int) as end_date\n" +
"from tableA as t\n" +
"where t.user_id = 1\n" +
"union all\n" +
"select p.project as user_project\n" +
"from tableB as p\n" +
"where p.project_id = 10\n" +
") as n;";
SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, JdbcConstants.MYSQL);
SQLStatement stmt = parser.parseStatementList().get(0);
SchemaStatVisitor statVisitor = SQLUtils.createSchemaStatVisitor(repository);
stmt.accept(statVisitor);
System.out.println("Tables : " + statVisitor.getTables());
System.out.println("columns : " + statVisitor.getColumns());
System.out.println(statVisitor.getGroupByColumns()); // group by
System.out.println("relationships : " + statVisitor.getRelationships()); // group by
System.out.println("conditions : " + statVisitor.getConditions());
System.out.println("functions : " + statVisitor.getFunctions());
assertEquals(7, statVisitor.getColumns().size());
assertEquals(2, statVisitor.getConditions().size());
assertEquals(2, statVisitor.getFunctions().size());
SQLPropertyExpr expr = (SQLPropertyExpr) statVisitor.getFunctions().get(0).getArguments().get(0);
SQLIdentifierExpr tableAlias = (SQLIdentifierExpr) expr.getOwner();
tableAlias.getResolvedTableSource();
}
}
src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest22_yiran.java
+47
View File
@@ -0,0 +1,47 @@
package com.alibaba.druid.bvt.sql.schemaStat;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.repository.SchemaRepository;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
public class SchemaStatTest22_yiran extends TestCase {
public void test_schemaStat() throws Exception {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
repository.console("CREATE TABLE DEPT (depno INT4, dname ENC_TEXT)");
String sql = "SELECT * FROM ( SELECT dname FROM DEPT) X WHERE X.dname='cs';";
// sql = "SELECT * FROM ( SELECT * FROM DEPT) X WHERE X.dname='cs'";
SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, JdbcConstants.MYSQL);
SQLStatement stmt = parser.parseStatementList().get(0);
SchemaStatVisitor statVisitor = SQLUtils.createSchemaStatVisitor(repository);
stmt.accept(statVisitor);
SQLSelectQueryBlock q = ((SQLSelectStatement) stmt).getSelect().getFirstQueryBlock();
SQLPropertyExpr whereLeft = (SQLPropertyExpr) ((SQLBinaryOpExpr) q.getWhere()).getLeft();
SQLColumnDefinition resolvedColumn = whereLeft.getResolvedColumn();
assertNotNull(resolvedColumn);
System.out.println("Tables : " + statVisitor.getTables());
System.out.println("columns : " + statVisitor.getColumns());
// System.out.println(statVisitor.getGroupByColumns()); // group by
System.out.println("relationships : " + statVisitor.getRelationships()); // group by
System.out.println("conditions : " + statVisitor.getConditions());
assertEquals(1, statVisitor.getColumns().size());
assertEquals(1, statVisitor.getConditions().size());
assertEquals(0, statVisitor.getFunctions().size());
}
}