improved sql parser

This commit is contained in:
高铁
2021-01-17 22:56:30 +08:00
parent 7a056ed7ae
commit c8bbfae175
21 changed files with 748 additions and 156 deletions
@@ -779,6 +779,7 @@ public class SQLExprParser extends SQLParser {
parseQueryPlanHint(sqlExpr);
}
break;
case FROM:
case SELECT:
SQLQueryExpr queryExpr = new SQLQueryExpr(
createSelectParser()
@@ -866,6 +867,7 @@ public class SQLExprParser extends SQLParser {
case NULL:
case INTERVAL:
case LBRACE:
case IF:
sqlExpr = primary();
while (lexer.token == Token.HINT) {
@@ -1079,6 +1081,10 @@ public class SQLExprParser extends SQLParser {
case DELETE:
case BY:
case UPDATE:
case LOOP:
case LIKE:
case UNION:
case CREATE:
if (dbType == DbType.odps) {
sqlExpr = new SQLIdentifierExpr(lexer.stringVal());
lexer.nextToken();
@@ -1135,6 +1141,15 @@ public class SQLExprParser extends SQLParser {
}
}
throw new ParserException("ERROR. " + lexer.info());
case COLONCOLON:
if (dbType == DbType.odps) {
lexer.nextToken();
SQLMethodInvokeExpr method = (SQLMethodInvokeExpr) this.primary();
method.setOwner(new SQLIdentifierExpr(""));
sqlExpr = method;
break;
}
throw new ParserException("ERROR. " + lexer.info());
default:
throw new ParserException("ERROR. " + lexer.info());
}
@@ -1152,7 +1167,7 @@ public class SQLExprParser extends SQLParser {
SQLExpr sqlExpr;
String str = lexer.stringVal();
lexer.nextToken();
if (lexer.token == Token.DOT) {
if (lexer.token == Token.DOT || lexer.token == Token.SLASH) {
return primaryRest(new SQLIdentifierExpr(str));
} else if (lexer.token == Token.COMMA) {
return new SQLIdentifierExpr(str);
@@ -1244,6 +1259,9 @@ public class SQLExprParser extends SQLParser {
case SLASH:
case DOT:
case FROM:
case ORDER:
case THEN:
case END:
return new SQLIdentifierExpr(str);
default:
break;
@@ -1357,7 +1375,7 @@ public class SQLExprParser extends SQLParser {
if (lexer.token == Token.LPAREN &&
!(expr instanceof SQLIntegerExpr) && !(expr instanceof SQLHexExpr)) {
SQLExpr method = methodRest(expr, true);
if (lexer.token == Token.LBRACKET) {
if (lexer.token == Token.LBRACKET || lexer.token == Token.DOT) {
method = primaryRest(method);
}
return method;
@@ -2023,6 +2041,10 @@ public class SQLExprParser extends SQLParser {
case VALUES:
case IN:
case OUT:
case LIMIT:
case TRIGGER:
case USE:
case LIKE:
if (dbType == DbType.odps) {
identName = lexer.stringVal();
lexer.nextToken();
@@ -2227,7 +2249,7 @@ public class SQLExprParser extends SQLParser {
|| lexer.token == Token.LITERAL_FLOAT
|| lexer.token == Token.LITERAL_CHARS
) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenBegin(betweenBegin);
} else if (lexer.token == Token.IDENTIFIER) {
long hash = lexer.hash_lower();
@@ -2252,7 +2274,7 @@ public class SQLExprParser extends SQLParser {
|| lexer.token == Token.LITERAL_FLOAT
|| lexer.token == Token.LITERAL_CHARS
) {
SQLExpr betweenEnd = this.primary();
SQLExpr betweenEnd = this.additive();
over.setWindowingBetweenEnd(betweenEnd);
} else if (lexer.token == Token.IDENTIFIER) {
long hash = lexer.hash_lower();
@@ -2261,7 +2283,7 @@ public class SQLExprParser extends SQLParser {
&& hash != FnvHash.Constants.FOLLOWING
&& hash != FnvHash.Constants.CURRENT
&& hash != FnvHash.Constants.UNBOUNDED) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenEnd(betweenBegin);
}
}
@@ -2275,7 +2297,7 @@ public class SQLExprParser extends SQLParser {
|| lexer.token == Token.LITERAL_FLOAT
|| lexer.token == Token.LITERAL_CHARS
) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenBegin(betweenBegin);
} else if (lexer.token == Token.IDENTIFIER) {
long hash = lexer.hash_lower();
@@ -2284,7 +2306,7 @@ public class SQLExprParser extends SQLParser {
&& hash != FnvHash.Constants.FOLLOWING
&& hash != FnvHash.Constants.CURRENT
&& hash != FnvHash.Constants.UNBOUNDED) {
SQLExpr betweenBegin = this.primary();
SQLExpr betweenBegin = this.additive();
over.setWindowingBetweenBegin(betweenBegin);
}
}
@@ -4261,17 +4283,21 @@ public class SQLExprParser extends SQLParser {
lexer.nextToken();
if (dbType == DbType.odps) {
if (lexer.token == Token.LITERAL_ALIAS) {
String tmp = lexer.stringVal();
if (tmp.length() > 2 && tmp.charAt(0) == '"' && tmp.charAt(tmp.length() - 1) == '"') {
tmp = tmp.substring(1, tmp.length() - 1);
}
for (;;) {
if (lexer.token == Token.LITERAL_ALIAS) {
String tmp = lexer.stringVal();
if (tmp.length() > 2 && tmp.charAt(0) == '"' && tmp.charAt(tmp.length() - 1) == '"') {
tmp = tmp.substring(1, tmp.length() - 1);
}
stringVal += tmp;
lexer.nextToken();
} else if (lexer.token == Token.LITERAL_CHARS) {
stringVal += lexer.stringVal();
lexer.nextToken();
stringVal += tmp;
lexer.nextToken();
} else if (lexer.token == Token.LITERAL_CHARS) {
stringVal += lexer.stringVal();
lexer.nextToken();
} else {
break;
}
}
}
@@ -5612,6 +5638,35 @@ public class SQLExprParser extends SQLParser {
throw new ParserException("TODO");
}
public SQLPartitionSpec parsePartitionSpec() {
SQLPartitionSpec spec = new SQLPartitionSpec();
accept(Token.PARTITION);
accept(Token.LPAREN);
for (;;) {
SQLPartitionSpec.Item item = new SQLPartitionSpec.Item();
item.setColumn(
this.name());
accept(Token.EQ);
item.setValue(
this.expr());
spec.addItem(item);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
} else {
break;
}
}
accept(Token.RPAREN);
return spec;
}
protected SQLPartitionBy parsePartitionBy() {
lexer.nextToken();
accept(Token.BY);
@@ -297,6 +297,7 @@ public class SQLParser {
case DEFERRED:
case OUTER:
case DO:
case LOOP:
case STORE:
case MOD:
case ANY:
@@ -337,6 +338,9 @@ public class SQLParser {
case INNER:
case CLOSE:
case VALUES:
case SHOW:
case SEQUENCE:
case TO:
if (dbType == DbType.odps) {
alias = lexer.stringVal();
lexer.nextToken();
@@ -345,6 +349,7 @@ public class SQLParser {
break;
case GROUP:
case ORDER:
case DEFAULT:
if (dbType == DbType.odps) {
Lexer.SavePoint mark = lexer.mark();
alias = lexer.stringVal();
@@ -486,6 +491,7 @@ public class SQLParser {
case PARTITION:
case UPDATE:
case DO:
case LOOP:
case REPEAT:
case DEFAULT:
case LIKE:
@@ -499,6 +505,7 @@ public class SQLParser {
case IN:
case OUT:
case BY:
case EXCEPT:
alias = lexer.stringVal();
lexer.nextToken();
return alias;
@@ -232,6 +232,9 @@ public class SQLSelectParser extends SQLParser {
if (lexer.token == Token.ALL) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.EXCEPT_ALL);
} else if (lexer.token == Token.DISTINCT) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.EXCEPT_DISTINCT);
} else {
union.setOperator(SQLUnionOperator.EXCEPT);
}
@@ -283,6 +286,9 @@ public class SQLSelectParser extends SQLParser {
if (lexer.token() == Token.DISTINCT) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.INTERSECT_DISTINCT);
} else if (lexer.token == Token.ALL) {
lexer.nextToken();
union.setOperator(SQLUnionOperator.INTERSECT_ALL);
} else {
union.setOperator(SQLUnionOperator.INTERSECT);
}
@@ -596,111 +602,114 @@ public class SQLSelectParser extends SQLParser {
}
public void parseWhere(SQLSelectQueryBlock queryBlock) {
if (lexer.token == Token.WHERE) {
lexer.nextTokenIdent();
if (lexer.token != Token.WHERE) {
return;
}
List<String> beforeComments = null;
if (lexer.hasComment() && lexer.isKeepComments()) {
beforeComments = lexer.readAndResetComments();
}
lexer.nextTokenIdent();
SQLExpr where;
List<String> beforeComments = null;
if (lexer.hasComment() && lexer.isKeepComments()) {
beforeComments = lexer.readAndResetComments();
}
if (lexer.token == Token.IDENTIFIER) {
String ident = lexer.stringVal();
long hash_lower = lexer.hash_lower();
lexer.nextTokenEq();
SQLExpr where;
SQLExpr identExpr;
if (lexer.token == Token.LITERAL_CHARS) {
String literal = lexer.stringVal;
if (hash_lower == FnvHash.Constants.TIMESTAMP) {
identExpr = new SQLTimestampExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.DATE) {
identExpr = new SQLDateExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.REAL) {
identExpr = new SQLRealExpr(Float.parseFloat(literal));
lexer.nextToken();
} else {
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
if (lexer.token == Token.IDENTIFIER) {
String ident = lexer.stringVal();
long hash_lower = lexer.hash_lower();
lexer.nextTokenEq();
SQLExpr identExpr;
if (lexer.token == Token.LITERAL_CHARS) {
String literal = lexer.stringVal;
if (hash_lower == FnvHash.Constants.TIMESTAMP) {
identExpr = new SQLTimestampExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.DATE) {
identExpr = new SQLDateExpr(literal);
lexer.nextToken();
} else if (hash_lower == FnvHash.Constants.REAL) {
identExpr = new SQLRealExpr(Float.parseFloat(literal));
lexer.nextToken();
} else {
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
if (lexer.token == Token.DOT) {
identExpr = this.exprParser.primaryRest(identExpr);
}
if (lexer.token == Token.EQ) {
SQLExpr rightExp;
lexer.nextToken();
try {
rightExp = this.exprParser.bitOr();
} catch (EOFParserException e) {
throw new ParserException("EOF, " + ident + "=", e);
}
where = new SQLBinaryOpExpr(identExpr, SQLBinaryOperator.Equality, rightExp, dbType);
switch (lexer.token) {
case BETWEEN:
case IS:
case EQ:
case IN:
case CONTAINS:
case BANG_TILDE_STAR:
case TILDE_EQ:
case LT:
case LTEQ:
case LTEQGT:
case GT:
case GTEQ:
case LTGT:
case BANGEQ:
case LIKE:
case NOT:
where = this.exprParser.relationalRest(where);
break;
default:
break;
}
where = this.exprParser.andRest(where);
where = this.exprParser.xorRest(where);
where = this.exprParser.orRest(where);
} else {
identExpr = this.exprParser.primaryRest(identExpr);
where = this.exprParser.exprRest(identExpr);
}
} else {
while (lexer.token == Token.HINT) {
lexer.nextToken();
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
if (lexer.token == Token.DOT) {
identExpr = this.exprParser.primaryRest(identExpr);
}
if (lexer.token == Token.EQ) {
SQLExpr rightExp;
lexer.nextToken();
try {
rightExp = this.exprParser.bitOr();
} catch (EOFParserException e) {
throw new ParserException("EOF, " + ident + "=", e);
}
where = this.exprParser.expr();
while (lexer.token == Token.HINT) {
lexer.nextToken();
where = new SQLBinaryOpExpr(identExpr, SQLBinaryOperator.Equality, rightExp, dbType);
switch (lexer.token) {
case BETWEEN:
case IS:
case EQ:
case IN:
case CONTAINS:
case BANG_TILDE_STAR:
case TILDE_EQ:
case LT:
case LTEQ:
case LTEQGT:
case GT:
case GTEQ:
case LTGT:
case BANGEQ:
case LIKE:
case NOT:
where = this.exprParser.relationalRest(where);
break;
default:
break;
}
where = this.exprParser.andRest(where);
where = this.exprParser.xorRest(where);
where = this.exprParser.orRest(where);
} else {
identExpr = this.exprParser.primaryRest(identExpr);
where = this.exprParser.exprRest(identExpr);
}
// where = this.exprParser.expr();
if (beforeComments != null) {
where.addBeforeComment(beforeComments);
} else {
while (lexer.token == Token.HINT) {
lexer.nextToken();
}
if (lexer.hasComment() && lexer.isKeepComments() //
&& lexer.token != Token.INSERT // odps multi-insert
) {
where.addAfterComment(lexer.readAndResetComments());
where = this.exprParser.expr();
while (lexer.token == Token.HINT) {
lexer.nextToken();
}
queryBlock.setWhere(where);
}
// where = this.exprParser.expr();
if (beforeComments != null) {
where.addBeforeComment(beforeComments);
}
if (lexer.hasComment() && lexer.isKeepComments() //
&& lexer.token != Token.INSERT // odps multi-insert
) {
where.addAfterComment(lexer.readAndResetComments());
}
queryBlock.setWhere(where);
}
protected void parseSortBy(SQLSelectQueryBlock queryBlock) {
@@ -1391,7 +1400,7 @@ public class SQLSelectParser extends SQLParser {
rightTableSource = exprTableSource;
}
primaryTableSourceRest(rightTableSource);
rightTableSource = primaryTableSourceRest(rightTableSource);
}
if (lexer.token == Token.USING
@@ -1438,7 +1447,7 @@ public class SQLSelectParser extends SQLParser {
}
}
primaryTableSourceRest(rightTableSource);
rightTableSource = primaryTableSourceRest(rightTableSource);
}
if (lexer.token == Token.WITH) {
@@ -2183,8 +2183,41 @@ public class SQLStatementParser extends SQLParser {
stmt.addItem(item);
} else if (DbType.odps == dbType && lexer.identifierEquals("MERGE")) {
lexer.nextToken();
acceptIdentifier("SMALLFILES");
stmt.setMergeSmallFiles(true);
boolean ifExists = false;
if (lexer.token == Token.IF) {
lexer.nextToken();
accept(Token.EXISTS);
ifExists = true;
}
if (lexer.token == PARTITION) {
SQLAlterTableMergePartition item = new SQLAlterTableMergePartition();
for (;;) {
item.addPartition(
this.getExprParser().parsePartitionSpec()
);
if (lexer.token == COMMA) {
lexer.nextToken();
} else {
break;
}
}
accept(OVERWRITE);
item.setOverwritePartition(
this.getExprParser().parsePartitionSpec()
);
if (ifExists) {
item.setIfExists(true);
}
stmt.addItem(item);
} else {
acceptIdentifier("SMALLFILES");
stmt.setMergeSmallFiles(true);
}
} else if (DbType.odps == dbType && lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) {
lexer.nextToken();
accept(Token.BY);
@@ -6627,6 +6660,13 @@ public class SQLStatementParser extends SQLParser {
SQLExpr tableExpr = this.exprParser.expr();
stmt.setTable(tableExpr);
}
if (lexer.identifierEquals(FnvHash.Constants.ADD)) {
lexer.nextToken();
acceptIdentifier("PARTITIONS");
stmt.setAddPartitions(true);
}
return stmt;
}