improved parser performance.

This commit is contained in:
wenshao
2017-08-26 22:53:51 +08:00
parent 76c441949f
commit a983b2685b
75 changed files with 2998 additions and 2053 deletions
@@ -44,7 +44,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
accept(Token.TABLE);
if (lexer.token() == Token.IF || identifierEquals("IF")) {
if (lexer.token() == Token.IF || lexer.identifierEquals("IF")) {
lexer.nextToken();
accept(Token.NOT);
accept(Token.EXISTS);
@@ -54,7 +54,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
stmt.setName(this.exprParser.name());
if (identifierEquals("LIFECYCLE")) {
if (lexer.identifierEquals("LIFECYCLE")) {
lexer.nextToken();
stmt.setLifecycle(this.exprParser.expr());
}
@@ -137,7 +137,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
accept(Token.RPAREN);
}
if (identifierEquals("CLUSTERED")) {
if (lexer.identifierEquals("CLUSTERED")) {
lexer.nextToken();
accept(Token.BY);
accept(Token.LPAREN);
@@ -145,7 +145,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
accept(Token.RPAREN);
}
if (identifierEquals("SORTED")) {
if (lexer.identifierEquals("SORTED")) {
lexer.nextToken();
accept(Token.BY);
accept(Token.LPAREN);
@@ -164,7 +164,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
acceptIdentifier("BUCKETS");
}
if (identifierEquals("LIFECYCLE")) {
if (lexer.identifierEquals("LIFECYCLE")) {
lexer.nextToken();
stmt.setLifecycle(this.exprParser.expr());
}
@@ -24,26 +24,42 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsUDTFSQLSelectItem;
import com.alibaba.druid.sql.parser.*;
import com.alibaba.druid.util.FNVUtils;
import com.alibaba.druid.util.JdbcConstants;
public class OdpsExprParser extends SQLExprParser {
import java.util.Arrays;
public final static String[] AGGREGATE_FUNCTIONS = { "AVG", //
"COUNT", //
"LAG",
"LEAD",
"MAX", //
"MIN", //
"STDDEV", //
"SUM", //
"ROW_NUMBER",
"WM_CONCAT"//
};
public class OdpsExprParser extends SQLExprParser {
public final static String[] AGGREGATE_FUNCTIONS;
public final static long[] AGGREGATE_FUNCTIONS_CODES;
static {
String[] strings = { "AVG", //
"COUNT", //
"LAG",
"LEAD",
"MAX", //
"MIN", //
"STDDEV", //
"SUM", //
"ROW_NUMBER",
"WM_CONCAT"//
};
AGGREGATE_FUNCTIONS_CODES = FNVUtils.fnv_64_lower(strings, true);
AGGREGATE_FUNCTIONS = new String[AGGREGATE_FUNCTIONS_CODES.length];
for (String str : strings) {
long hash = FNVUtils.fnv_64_lower(str);
int index = Arrays.binarySearch(AGGREGATE_FUNCTIONS_CODES, hash);
AGGREGATE_FUNCTIONS[index] = str;
}
}
public OdpsExprParser(Lexer lexer){
super(lexer, JdbcConstants.ODPS);
this.aggregateFunctions = AGGREGATE_FUNCTIONS;
this.aggregateFunctionHashCodes = AGGREGATE_FUNCTIONS_CODES;
}
public OdpsExprParser(String sql){
@@ -157,7 +173,7 @@ public class OdpsExprParser extends SQLExprParser {
}
public SQLExpr relationalRest(SQLExpr expr) {
if (identifierEquals("REGEXP")) {
if (lexer.identifierEquals("REGEXP")) {
lexer.nextToken();
SQLExpr rightExp = equality();
@@ -182,6 +182,9 @@ public class OdpsLexer extends Lexer {
}
public void scanIdentifier() {
hash_lower = 0;
hash = 0;
final char first = ch;
if (first == '`') {
@@ -362,8 +365,4 @@ public class OdpsLexer extends Lexer {
protected final void scanString() {
scanString2();
}
protected final void scanAlias() {
scanAlias2();
}
}
@@ -90,7 +90,7 @@ public class OdpsSelectParser extends SQLSelectParser {
accept(Token.BY);
this.exprParser.exprList(queryBlock.getDistributeBy(), queryBlock);
if (identifierEquals("SORT")) {
if (lexer.identifierEquals("SORT")) {
lexer.nextToken();
accept(Token.BY);
@@ -168,7 +168,7 @@ public class OdpsSelectParser extends SQLSelectParser {
return parseLateralView(tableSource);
}
if (identifierEquals("LATERAL")) {
if (lexer.identifierEquals("LATERAL")) {
lexer.nextToken();
return parseLateralView(tableSource);
}
@@ -86,7 +86,7 @@ public class OdpsStatementParser extends SQLStatementParser {
return true;
}
if (identifierEquals("ANALYZE")) {
if (lexer.identifierEquals("ANALYZE")) {
lexer.nextToken();
accept(Token.TABLE);
@@ -110,10 +110,10 @@ public class OdpsStatementParser extends SQLStatementParser {
return true;
}
if (identifierEquals("ADD")) {
if (lexer.identifierEquals("ADD")) {
lexer.nextToken();
if (identifierEquals("STATISTIC")) {
if (lexer.identifierEquals("STATISTIC")) {
lexer.nextToken();
OdpsAddStatisticStatement stmt = new OdpsAddStatisticStatement();
stmt.setTable(this.exprParser.name());
@@ -125,10 +125,10 @@ public class OdpsStatementParser extends SQLStatementParser {
throw new ParserException("TODO " + lexer.info());
}
if (identifierEquals("REMOVE")) {
if (lexer.identifierEquals("REMOVE")) {
lexer.nextToken();
if (identifierEquals("STATISTIC")) {
if (lexer.identifierEquals("STATISTIC")) {
lexer.nextToken();
OdpsRemoveStatisticStatement stmt = new OdpsRemoveStatisticStatement();
stmt.setTable(this.exprParser.name());
@@ -140,7 +140,7 @@ public class OdpsStatementParser extends SQLStatementParser {
throw new ParserException("TODO " + lexer.info());
}
if (identifierEquals("READ")) {
if (lexer.identifierEquals("READ")) {
OdpsReadStatement stmt = new OdpsReadStatement();
if (lexer.hasComment() && lexer.isKeepComments()) {
@@ -172,7 +172,7 @@ public class OdpsStatementParser extends SQLStatementParser {
return true;
}
if (identifierEquals("LIST")) {
if (lexer.identifierEquals("LIST")) {
OdpsListStmt stmt = new OdpsListStmt();
lexer.nextToken();
@@ -183,7 +183,7 @@ public class OdpsStatementParser extends SQLStatementParser {
return true;
}
if (lexer.token() == Token.DESC || identifierEquals("DESCRIBE")) {
if (lexer.token() == Token.DESC || lexer.identifierEquals("DESCRIBE")) {
SQLStatement stmt = parseDescribe();
statementList.add(stmt);
return true;
@@ -193,30 +193,30 @@ public class OdpsStatementParser extends SQLStatementParser {
}
protected OdpsStatisticClause parseStaticClause() {
if (identifierEquals("TABLE_COUNT")) {
if (lexer.identifierEquals("TABLE_COUNT")) {
lexer.nextToken();
return new OdpsStatisticClause.TableCount();
} else if (identifierEquals("NULL_VALUE")) {
} else if (lexer.identifierEquals("NULL_VALUE")) {
lexer.nextToken();
OdpsStatisticClause.NullValue null_value = new OdpsStatisticClause.NullValue();
null_value.setColumn(this.exprParser.name());
return null_value;
} else if (identifierEquals("COLUMN_SUM")) {
} else if (lexer.identifierEquals("COLUMN_SUM")) {
lexer.nextToken();
OdpsStatisticClause.ColumnSum column_sum = new OdpsStatisticClause.ColumnSum();
column_sum.setColumn(this.exprParser.name());
return column_sum;
} else if (identifierEquals("COLUMN_MAX")) {
} else if (lexer.identifierEquals("COLUMN_MAX")) {
lexer.nextToken();
OdpsStatisticClause.ColumnMax column_max = new OdpsStatisticClause.ColumnMax();
column_max.setColumn(this.exprParser.name());
return column_max;
} else if (identifierEquals("COLUMN_MIN")) {
} else if (lexer.identifierEquals("COLUMN_MIN")) {
lexer.nextToken();
OdpsStatisticClause.ColumnMin column_min = new OdpsStatisticClause.ColumnMin();
column_min.setColumn(this.exprParser.name());
return column_min;
} else if (identifierEquals("EXPRESSION_CONDITION")) {
} else if (lexer.identifierEquals("EXPRESSION_CONDITION")) {
lexer.nextToken();
OdpsStatisticClause.ExpressionCondition expr_condition = new OdpsStatisticClause.ExpressionCondition();
expr_condition.setExpr(this.exprParser.expr());
@@ -320,7 +320,7 @@ public class OdpsStatementParser extends SQLStatementParser {
public SQLStatement parseShow() {
accept(Token.SHOW);
if (identifierEquals("PARTITIONS")) {
if (lexer.identifierEquals("PARTITIONS")) {
lexer.nextToken();
OdpsShowPartitionsStmt stmt = new OdpsShowPartitionsStmt();
@@ -331,7 +331,7 @@ public class OdpsStatementParser extends SQLStatementParser {
return stmt;
}
if (identifierEquals("STATISTIC")) {
if (lexer.identifierEquals("STATISTIC")) {
lexer.nextToken();
OdpsShowStatisticStmt stmt = new OdpsShowStatisticStmt();
@@ -342,7 +342,7 @@ public class OdpsStatementParser extends SQLStatementParser {
return stmt;
}
if (identifierEquals("TABLES")) {
if (lexer.identifierEquals("TABLES")) {
lexer.nextToken();
SQLShowTablesStatement stmt = new SQLShowTablesStatement();
@@ -360,7 +360,7 @@ public class OdpsStatementParser extends SQLStatementParser {
return stmt;
}
if (identifierEquals("GRANTS")) {
if (lexer.identifierEquals("GRANTS")) {
lexer.nextToken();
OdpsShowGrantsStmt stmt = new OdpsShowGrantsStmt();
@@ -389,7 +389,7 @@ public class OdpsStatementParser extends SQLStatementParser {
accept(Token.SET);
if (identifierEquals("LABEL")) {
if (lexer.identifierEquals("LABEL")) {
OdpsSetLabelStatement stmt = new OdpsSetLabelStatement();
if (comments != null) {
@@ -436,12 +436,12 @@ public class OdpsStatementParser extends SQLStatementParser {
accept(Token.GRANT);
OdpsGrantStmt stmt = new OdpsGrantStmt();
if (identifierEquals("LABEL")) {
if (lexer.identifierEquals("LABEL")) {
stmt.setLabel(true);
lexer.nextToken();
stmt.setLabel(this.exprParser.expr());
} else {
if (identifierEquals("SUPER")) {
if (lexer.identifierEquals("SUPER")) {
stmt.setSuper(true);
lexer.nextToken();
}
@@ -452,10 +452,10 @@ public class OdpsStatementParser extends SQLStatementParser {
if (lexer.token() == Token.ON) {
lexer.nextToken();
if (identifierEquals("PROJECT")) {
if (lexer.identifierEquals("PROJECT")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.PROJECT);
} else if (identifierEquals("PACKAGE")) {
} else if (lexer.identifierEquals("PACKAGE")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.PACKAGE);
} else if (lexer.token() == Token.FUNCTION) {
@@ -469,22 +469,22 @@ public class OdpsStatementParser extends SQLStatementParser {
this.exprParser.names(stmt.getColumns(), stmt);
accept(Token.RPAREN);
}
} else if (identifierEquals("RESOURCE")) {
} else if (lexer.identifierEquals("RESOURCE")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.RESOURCE);
} else if (identifierEquals("INSTANCE")) {
} else if (lexer.identifierEquals("INSTANCE")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.INSTANCE);
} else if (identifierEquals("JOB")) {
} else if (lexer.identifierEquals("JOB")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.JOB);
} else if (identifierEquals("VOLUME")) {
} else if (lexer.identifierEquals("VOLUME")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.VOLUME);
} else if (identifierEquals("OfflineModel")) {
} else if (lexer.identifierEquals("OfflineModel")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.OfflineModel);
} else if (identifierEquals("XFLOW")) {
} else if (lexer.identifierEquals("XFLOW")) {
lexer.nextToken();
stmt.setObjectType(SQLObjectType.XFLOW);
}
@@ -497,7 +497,7 @@ public class OdpsStatementParser extends SQLStatementParser {
if (lexer.token() == Token.USER) {
lexer.nextToken();
stmt.setSubjectType(SQLObjectType.USER);
} else if (identifierEquals("ROLE")) {
} else if (lexer.identifierEquals("ROLE")) {
lexer.nextToken();
stmt.setSubjectType(SQLObjectType.ROLE);
}
@@ -537,43 +537,43 @@ public class OdpsStatementParser extends SQLStatementParser {
} else if (lexer.token() == Token.ALTER) {
lexer.nextToken();
privilege = "ALTER";
} else if (identifierEquals("DESCRIBE")) {
} else if (lexer.identifierEquals("DESCRIBE")) {
privilege = "DESCRIBE";
lexer.nextToken();
} else if (identifierEquals("READ")) {
} else if (lexer.identifierEquals("READ")) {
privilege = "READ";
lexer.nextToken();
} else if (identifierEquals("WRITE")) {
} else if (lexer.identifierEquals("WRITE")) {
privilege = "WRITE";
lexer.nextToken();
} else if (identifierEquals("EXECUTE")) {
} else if (lexer.identifierEquals("EXECUTE")) {
lexer.nextToken();
privilege = "EXECUTE";
} else if (identifierEquals("LIST")) {
} else if (lexer.identifierEquals("LIST")) {
lexer.nextToken();
privilege = "LIST";
} else if (identifierEquals("CreateTable")) {
} else if (lexer.identifierEquals("CreateTable")) {
lexer.nextToken();
privilege = "CreateTable";
} else if (identifierEquals("CreateInstance")) {
} else if (lexer.identifierEquals("CreateInstance")) {
lexer.nextToken();
privilege = "CreateInstance";
} else if (identifierEquals("CreateFunction")) {
} else if (lexer.identifierEquals("CreateFunction")) {
lexer.nextToken();
privilege = "CreateFunction";
} else if (identifierEquals("CreateResource")) {
} else if (lexer.identifierEquals("CreateResource")) {
lexer.nextToken();
privilege = "CreateResource";
} else if (identifierEquals("CreateJob")) {
} else if (lexer.identifierEquals("CreateJob")) {
lexer.nextToken();
privilege = "CreateJob";
} else if (identifierEquals("CreateVolume")) {
} else if (lexer.identifierEquals("CreateVolume")) {
lexer.nextToken();
privilege = "CreateVolume";
} else if (identifierEquals("CreateOfflineModel")) {
} else if (lexer.identifierEquals("CreateOfflineModel")) {
lexer.nextToken();
privilege = "CreateOfflineModel";
} else if (identifierEquals("CreateXflow")) {
} else if (lexer.identifierEquals("CreateXflow")) {
lexer.nextToken();
privilege = "CreateXflow";
}