Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improved sql parser.

Browse Source
This commit is contained in:
wenshao
2017-09-09 07:41:39 +08:00
parent 6545e23629
commit 3330643479
50 changed files with 1898 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+93 -22
View File
@@ -19,19 +19,9 @@ import static com.alibaba.druid.sql.parser.CharTypes.isFirstIdentifierChar;
import static com.alibaba.druid.sql.parser.CharTypes.isIdentifierChar;
import static com.alibaba.druid.sql.parser.CharTypes.isWhitespace;
import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI;
import static com.alibaba.druid.sql.parser.Token.COLONCOLON;
import static com.alibaba.druid.sql.parser.Token.COLONEQ;
import static com.alibaba.druid.sql.parser.Token.COMMA;
import static com.alibaba.druid.sql.parser.Token.EOF;
import static com.alibaba.druid.sql.parser.Token.ERROR;
import static com.alibaba.druid.sql.parser.Token.LBRACE;
import static com.alibaba.druid.sql.parser.Token.LBRACKET;
import static com.alibaba.druid.sql.parser.Token.LITERAL_ALIAS;
import static com.alibaba.druid.sql.parser.Token.LITERAL_CHARS;
import static com.alibaba.druid.sql.parser.Token.LPAREN;
import static com.alibaba.druid.sql.parser.Token.RBRACE;
import static com.alibaba.druid.sql.parser.Token.RBRACKET;
import static com.alibaba.druid.sql.parser.Token.RPAREN;
import static com.alibaba.druid.sql.parser.SQLParserFeature.KeepComments;
import static com.alibaba.druid.sql.parser.SQLParserFeature.OptimizedForParameterized;
import static com.alibaba.druid.sql.parser.Token.*;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -49,6 +39,7 @@ import com.alibaba.druid.util.StringUtils;
* @author wenshao [szujobs@hotmail.com]
*/
public class Lexer {
protected static SymbolTable symbols_l2 = new SymbolTable(512);
protected int features = 0; //SQLParserFeature.of(SQLParserFeature.EnableSQLBinaryOpExprGroup);
protected final String text;
@@ -69,29 +60,24 @@ public class Lexer {
protected long hash;
protected int commentCount = 0;
protected List<String> comments = new ArrayList<String>(2);
protected boolean skipComment = true;
private SavePoint savePoint = null;
/*
* anti sql injection
*/
private boolean allowComment = true;
private int varIndex = -1;
protected CommentHandler commentHandler;
protected boolean endOfComment = false;
protected boolean keepComments = false;
protected int line = 0;
protected int lines = 0;
protected String dbType;
protected boolean optimizedForParameterized = false;
private int startPos;
private int posLine;
private int posColumn;
@@ -316,6 +302,12 @@ public class Lexer {
return;
}
if (ch == '.') {
scanChar();
token = DOT;
return;
}
if (ch == 'a' || ch == 'A') {
char ch_next = charAt(pos + 1);
if (ch_next == 's' || ch_next == 'S') {
@@ -837,6 +829,7 @@ public class Lexer {
protected void scanString() {
mark = pos;
boolean hasSpecial = false;
Token preToken = this.token;
for (;;) {
if (isEOF()) {
@@ -875,7 +868,11 @@ public class Lexer {
}
if (!hasSpecial) {
stringVal = subString(mark + 1, bufPos);
if (preToken == Token.AS) {
stringVal = subString(mark, bufPos + 2);
} else {
stringVal = subString(mark + 1, bufPos);
}
} else {
stringVal = new String(buf, 0, bufPos);
}
@@ -902,7 +899,12 @@ public class Lexer {
throw new ParserException("unclosed str. " + info());
}
String stringVal = subString(startIndex, endIndex - startIndex);
String stringVal;
if (token == Token.AS) {
stringVal = subString(pos, endIndex + 1 - pos);
} else {
stringVal = subString(startIndex, endIndex - startIndex);
}
// hasSpecial = stringVal.indexOf('\\') != -1;
if (!hasSpecial) {
@@ -1537,6 +1539,15 @@ public class Lexer {
return stringVal;
}
private final void stringVal(StringBuffer out) {
if (stringVal != null) {
out.append(stringVal);
return;
}
out.append(text, mark, mark + bufPos);
}
public final boolean identifierEquals(String text) {
if (token != Token.IDENTIFIER) {
return false;
@@ -1799,9 +1810,69 @@ public class Lexer {
public void config(SQLParserFeature feature, boolean state) {
features = SQLParserFeature.config(features, feature, state);
if (feature == OptimizedForParameterized) {
optimizedForParameterized = state;
} else if (feature == KeepComments) {
this.keepComments = state;
}
}
public final boolean isEnabled(SQLParserFeature feature) {
return SQLParserFeature.isEnabled(this.features, feature);
}
public static String parameterize(String sql, String dbType) {
Lexer lexer = SQLParserUtils.createLexer(sql, dbType);
lexer.optimizedForParameterized = true; // optimized
lexer.nextToken();
StringBuffer buf = new StringBuffer();
for_:
for (;;) {
Token token = lexer.token;
switch (token) {
case LITERAL_ALIAS:
case LITERAL_FLOAT:
case LITERAL_CHARS:
case LITERAL_INT:
case LITERAL_NCHARS:
case LITERAL_HEX:
case VARIANT:
if (buf.length() != 0) {
buf.append(' ');
}
buf.append('?');
break;
case COMMA:
buf.append(',');
break;
case EQ:
buf.append('=');
break;
case EOF:
break for_;
case ERROR:
return sql;
case SELECT:
buf.append("SELECT");
break;
case UPDATE:
buf.append("UPDATE");
break;
default:
if (buf.length() != 0) {
buf.append(' ');
}
lexer.stringVal(buf);
break;
}
lexer.nextToken();
}
return buf.toString();
}
}
src/main/java/com/alibaba/druid/sql/parser/SQLCreateTableParser.java
+3 -3
View File
@@ -31,13 +31,13 @@ public class SQLCreateTableParser extends SQLDDLParser {
super(exprParser);
}
public SQLCreateTableStatement parseCrateTable() {
public SQLCreateTableStatement parseCreateTable() {
List<String> comments = null;
if (lexer.isKeepComments() && lexer.hasComment()) {
comments = lexer.readAndResetComments();
}
SQLCreateTableStatement stmt = parseCrateTable(true);
SQLCreateTableStatement stmt = parseCreateTable(true);
if (comments != null) {
stmt.addBeforeComment(comments);
}
@@ -45,7 +45,7 @@ public class SQLCreateTableParser extends SQLDDLParser {
return stmt;
}
public SQLCreateTableStatement parseCrateTable(boolean acceptCreate) {
public SQLCreateTableStatement parseCreateTable(boolean acceptCreate) {
SQLCreateTableStatement createTable = newCreateStatement();
if (acceptCreate) {
src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+88 -40
View File
@@ -237,7 +237,14 @@ public class SQLExprParser extends SQLParser {
expr = new SQLBinaryOpExpr(expr, SQLBinaryOperator.DIV, rightExp, getDbType());
expr = multiplicativeRest(expr);
} else if (lexer.identifierEquals(FnvHash.Constants.MOD)) {
Lexer.SavePoint savePoint = lexer.mark();
lexer.nextToken();
if (lexer.token == Token.COMMA) {
lexer.reset(savePoint);
return expr;
}
SQLExpr rightExp = primary();
rightExp = relationalRest(rightExp);
@@ -308,10 +315,11 @@ public class SQLExprParser extends SQLParser {
long hash_lower = lexer.hash_lower;
lexer.nextToken();
if ("DATE".equalsIgnoreCase(ident)
if (hash_lower == FnvHash.Constants.DATE
&& lexer.token == Token.LITERAL_CHARS
&& (JdbcConstants.ORACLE.equals(getDbType())
|| JdbcConstants.POSTGRESQL.equals(getDbType()))) {
&& (JdbcConstants.ORACLE.equals(dbType)
|| JdbcConstants.POSTGRESQL.equals(dbType)
|| JdbcConstants.MYSQL.equals(dbType))) {
String literal = lexer.stringVal();
lexer.nextToken();
@@ -450,6 +458,7 @@ public class SQLExprParser extends SQLParser {
case DESC:
case DO:
case GROUP:
case MOD:
sqlExpr = new SQLIdentifierExpr(lexer.stringVal());
lexer.nextToken();
break;
@@ -1021,42 +1030,7 @@ public class SQLExprParser extends SQLParser {
}
if (lexer.token == Token.LPAREN) {
lexer.nextToken();
if (lexer.token == Token.DISTINCT) {
lexer.nextToken();
String aggreateMethodName = expr.toString() + "." + name;
SQLAggregateExpr aggregateExpr = new SQLAggregateExpr(aggreateMethodName, SQLAggregateOption.DISTINCT);
if (lexer.token == Token.RPAREN) {
lexer.nextToken();
} else {
if (lexer.token == Token.PLUS) {
aggregateExpr.getArguments().add(new SQLIdentifierExpr("+"));
lexer.nextToken();
} else {
exprList(aggregateExpr.getArguments(), aggregateExpr);
}
accept(Token.RPAREN);
}
expr = aggregateExpr;
} else {
SQLMethodInvokeExpr methodInvokeExpr = new SQLMethodInvokeExpr(name);
methodInvokeExpr.setOwner(expr);
if (lexer.token == Token.RPAREN) {
lexer.nextToken();
} else {
if (lexer.token == Token.PLUS) {
methodInvokeExpr.addParameter(new SQLIdentifierExpr("+"));
lexer.nextToken();
} else {
exprList(methodInvokeExpr.getParameters(), methodInvokeExpr);
}
accept(Token.RPAREN);
}
expr = methodInvokeExpr;
}
expr = methodRest(expr, name);
} else {
expr = new SQLPropertyExpr(expr, name, hash_lower);
}
@@ -1066,6 +1040,46 @@ public class SQLExprParser extends SQLParser {
return expr;
}
private SQLExpr methodRest(SQLExpr expr, String name) {
lexer.nextToken();
if (lexer.token == Token.DISTINCT) {
lexer.nextToken();
String aggreateMethodName = expr.toString() + "." + name;
SQLAggregateExpr aggregateExpr = new SQLAggregateExpr(aggreateMethodName, SQLAggregateOption.DISTINCT);
if (lexer.token == Token.RPAREN) {
lexer.nextToken();
} else {
if (lexer.token == Token.PLUS) {
aggregateExpr.getArguments().add(new SQLIdentifierExpr("+"));
lexer.nextToken();
} else {
exprList(aggregateExpr.getArguments(), aggregateExpr);
}
accept(Token.RPAREN);
}
expr = aggregateExpr;
} else {
SQLMethodInvokeExpr methodInvokeExpr = new SQLMethodInvokeExpr(name);
methodInvokeExpr.setOwner(expr);
if (lexer.token == Token.RPAREN) {
lexer.nextToken();
} else {
if (lexer.token == Token.PLUS) {
methodInvokeExpr.addParameter(new SQLIdentifierExpr("+"));
lexer.nextToken();
} else {
exprList(methodInvokeExpr.getParameters(), methodInvokeExpr);
}
accept(Token.RPAREN);
}
expr = methodInvokeExpr;
}
return expr;
}
public final SQLExpr groupComparisionRest(SQLExpr expr) {
return expr;
}
@@ -2089,7 +2103,7 @@ public class SQLExprParser extends SQLParser {
}
SQLDataType dataType = new SQLDataTypeImpl(typeName);
dataType.setDbType(dbType);
return parseDataTypeRest(dataType);
}
@@ -2576,6 +2590,39 @@ public class SQLExprParser extends SQLParser {
}
token = lexer.token;
if (token == Token.DOT) {
lexer.nextToken();
String name;
long name_hash_lower;
if (lexer.token == Token.STAR) {
name = "*";
name_hash_lower = FnvHash.Constants.STAR;
} else {
name = lexer.stringVal();
name_hash_lower = lexer.hash_lower();
}
lexer.nextTokenComma();
token = lexer.token;
if (token == Token.LPAREN) {
expr = methodRest(expr, name);
token = lexer.token;
} else {
if (name_hash_lower == FnvHash.Constants.NEXTVAL) {
expr = new SQLSequenceExpr((SQLIdentifierExpr) expr, SQLSequenceExpr.Function.NextVal);
} else if (name_hash_lower == FnvHash.Constants.CURRVAL) {
expr = new SQLSequenceExpr((SQLIdentifierExpr) expr, SQLSequenceExpr.Function.CurrVal);
} else if (name_hash_lower == FnvHash.Constants.PREVVAL) {
expr = new SQLSequenceExpr((SQLIdentifierExpr) expr, SQLSequenceExpr.Function.PrevVal);
} else {
expr = new SQLPropertyExpr(expr, name, name_hash_lower);
}
}
}
if (token == Token.COMMA) {
return new SQLSelectItem(expr, null, connectByRoot);
}
@@ -2583,6 +2630,7 @@ public class SQLExprParser extends SQLParser {
if (token == Token.AS) {
lexer.nextToken();
String as = lexer.stringVal();
lexer.nextTokenComma();
return new SQLSelectItem(expr, as, connectByRoot);
}
src/main/java/com/alibaba/druid/sql/parser/SQLParser.java
+3 -2
View File
@@ -103,8 +103,8 @@ public class SQLParser {
if (token == Token.AS) {
lexer.nextToken();
alias = alias();
alias = lexer.stringVal();
lexer.nextToken();
if (alias != null) {
while (lexer.token == Token.DOT) {
@@ -142,6 +142,7 @@ public class SQLParser {
case OUTER:
case DO:
case STORE:
case MOD:
alias = lexer.stringVal();
lexer.nextToken();
break;
src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java
+3 -1
View File
@@ -17,7 +17,9 @@ package com.alibaba.druid.sql.parser;
public enum SQLParserFeature {
KeepInsertValueClauseOriginalString,
EnableSQLBinaryOpExprGroup
EnableSQLBinaryOpExprGroup,
OptimizedForParameterized,
KeepComments
;
private SQLParserFeature(){
src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java
+17 -6
View File
@@ -42,26 +42,37 @@ import com.alibaba.druid.util.JdbcUtils;
public class SQLParserUtils {
public static SQLStatementParser createSQLStatementParser(String sql, String dbType) {
boolean keepComments;
SQLParserFeature[] features;
if (JdbcConstants.ODPS.equals(dbType) || JdbcConstants.MYSQL.equals(dbType)) {
keepComments = true;
features = new SQLParserFeature[] {SQLParserFeature.KeepComments};
} else {
keepComments = false;
features = new SQLParserFeature[] {};
}
return createSQLStatementParser(sql, dbType, keepComments);
return createSQLStatementParser(sql, dbType, features);
}
public static SQLStatementParser createSQLStatementParser(String sql, String dbType, boolean keepComments) {
SQLParserFeature[] features;
if (keepComments) {
features = new SQLParserFeature[] {SQLParserFeature.KeepComments};
} else {
features = new SQLParserFeature[] {};
}
return createSQLStatementParser(sql, dbType, features);
}
public static SQLStatementParser createSQLStatementParser(String sql, String dbType, SQLParserFeature... features) {
if (JdbcUtils.ORACLE.equals(dbType) || JdbcUtils.ALI_ORACLE.equals(dbType)) {
return new OracleStatementParser(sql);
}
if (JdbcUtils.MYSQL.equals(dbType)) {
return new MySqlStatementParser(sql, keepComments);
return new MySqlStatementParser(sql, features);
}
if (JdbcUtils.MARIADB.equals(dbType)) {
return new MySqlStatementParser(sql, keepComments);
return new MySqlStatementParser(sql, features);
}
if (JdbcUtils.POSTGRESQL.equals(dbType)
src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
+2 -3
View File
@@ -24,7 +24,6 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.ast.statement.SQLCreateTriggerStatement.TriggerEvent;
import com.alibaba.druid.sql.ast.statement.SQLCreateTriggerStatement.TriggerType;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleExprParser;
import com.alibaba.druid.util.FnvHash;
@@ -1814,7 +1813,7 @@ public class SQLStatementParser extends SQLParser {
if (token == Token.TABLE || lexer.identifierEquals("GLOBAL")) {
SQLCreateTableParser createTableParser = getSQLCreateTableParser();
SQLCreateTableStatement stmt = createTableParser.parseCrateTable(false);
SQLCreateTableStatement stmt = createTableParser.parseCreateTable(false);
if (comments != null) {
stmt.addBeforeComment(comments);
@@ -2230,7 +2229,7 @@ public class SQLStatementParser extends SQLParser {
public SQLCreateTableStatement parseCreateTable() {
// SQLCreateTableParser parser = new SQLCreateTableParser(this.lexer);
// return parser.parseCrateTable();
// return parser.parseCreateTable();
throw new ParserException("TODO");
}
src/main/java/com/alibaba/druid/sql/parser/SymbolTable.java
-2
View File
@@ -27,8 +27,6 @@ public class SymbolTable {
public SymbolTable(int tableSize){
this.indexMask = tableSize - 1;
this.entries = new Entry[tableSize];
// this.addSymbol("id", 0, 2, "id".hashCode());
}