Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improved sql parser.

Browse Source
This commit is contained in:
wenshao
2017-10-18 15:55:24 +08:00
parent 29a5b5637c
commit 377d9346b8
18 changed files with 705 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/ast/SQLRecordDataType.java
+56
View File
@@ -0,0 +1,56 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.ast;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import java.util.ArrayList;
import java.util.List;
public class SQLRecordDataType extends SQLDataTypeImpl implements SQLDataType {
private final List<SQLColumnDefinition> columns = new ArrayList<SQLColumnDefinition>();
public List<SQLColumnDefinition> getColumns() {
return columns;
}
public void addColumn(SQLColumnDefinition column) {
column.setParent(this);
this.columns.add(column);
}
public SQLRecordDataType clone() {
SQLRecordDataType x = new SQLRecordDataType();
cloneTo(x);
for (SQLColumnDefinition c : columns) {
SQLColumnDefinition c2 = c.clone();
c2.setParent(x);
x.columns.add(c2);
}
return x;
}
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, this.columns);
}
visitor.endVisit(this);
}
}
src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java
+12 -6
View File
@@ -24,12 +24,10 @@ import com.alibaba.druid.sql.ast.SQLObjectImpl;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause;
public abstract class SQLInsertInto extends SQLObjectImpl {
protected SQLExprTableSource tableSource;
protected final List<SQLExpr> columns = new ArrayList<SQLExpr>();
protected SQLSelect query;
protected SQLExprTableSource tableSource;
protected final List<SQLExpr> columns = new ArrayList<SQLExpr>();
protected transient String columnsString;
protected SQLSelect query;
protected final List<ValuesClause> valuesList = new ArrayList<ValuesClause>();
public SQLInsertInto(){
@@ -139,4 +137,12 @@ public abstract class SQLInsertInto extends SQLObjectImpl {
}
valuesList.add(valueClause);
}
public String getColumnsString() {
return columnsString;
}
public void setColumnsString(String columnsString) {
this.columnsString = columnsString;
}
}
src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java
+106 -65
View File
@@ -2299,8 +2299,9 @@ public class MySqlStatementParser extends SQLStatementParser {
}
public SQLInsertStatement parseInsert() {
MySqlInsertStatement insertStatement = new MySqlInsertStatement();
MySqlInsertStatement stmt = new MySqlInsertStatement();
SQLName tableName = null;
if (lexer.token() == Token.INSERT) {
lexer.nextToken();
@@ -2309,31 +2310,31 @@ public class MySqlStatementParser extends SQLStatementParser {
long hash = lexer.hash_lower();
if (hash == FnvHash.Constants.LOW_PRIORITY) {
insertStatement.setLowPriority(true);
stmt.setLowPriority(true);
lexer.nextToken();
continue;
}
if (hash == FnvHash.Constants.DELAYED) {
insertStatement.setDelayed(true);
stmt.setDelayed(true);
lexer.nextToken();
continue;
}
if (hash == FnvHash.Constants.HIGH_PRIORITY) {
insertStatement.setHighPriority(true);
stmt.setHighPriority(true);
lexer.nextToken();
continue;
}
if (hash == FnvHash.Constants.IGNORE) {
insertStatement.setIgnore(true);
stmt.setIgnore(true);
lexer.nextToken();
continue;
}
if (hash == FnvHash.Constants.ROLLBACK_ON_FAIL) {
insertStatement.setRollbackOnFail(true);
stmt.setRollbackOnFail(true);
lexer.nextToken();
continue;
}
@@ -2347,18 +2348,18 @@ public class MySqlStatementParser extends SQLStatementParser {
lexer.nextToken();
}
SQLName tableName = this.exprParser.name();
insertStatement.setTableName(tableName);
tableName = this.exprParser.name();
stmt.setTableName(tableName);
if (lexer.token() == Token.HINT) {
String comment = "/*" + lexer.stringVal() + "*/";
lexer.nextToken();
insertStatement.getTableSource().addAfterComment(comment);
stmt.getTableSource().addAfterComment(comment);
}
if (lexer.token() == Token.IDENTIFIER
&& !lexer.identifierEquals(FnvHash.Constants.VALUE)) {
insertStatement.setAlias(lexer.stringVal());
stmt.setAlias(lexer.stringVal());
lexer.nextToken();
}
@@ -2366,69 +2367,109 @@ public class MySqlStatementParser extends SQLStatementParser {
int columnSize = 0;
if (lexer.token() == Token.LPAREN) {
lexer.nextToken();
if (lexer.token() == Token.SELECT) {
SQLSelect select = this.exprParser.createSelectParser().select();
select.setParent(insertStatement);
insertStatement.setQuery(select);
} else {
List<SQLExpr> columns = insertStatement.getColumns();
//this.exprParser.exprList(insertStatement.getColumns(), insertStatement);
boolean useInsertColumnsCache = lexer.isEnabled(SQLParserFeature.UseInsertColumnsCache);
InsertColumnsCache insertColumnsCache = null;
if (lexer.token() != Token.RPAREN) {
for (; ; ) {
String identName;
long hash;
InsertColumnsCache.Entry cachedColumns = null;
if (useInsertColumnsCache) {
insertColumnsCache = this.insertColumnsCache;
if (insertColumnsCache == null) {
insertColumnsCache = InsertColumnsCache.global;
}
Token token = lexer.token();
if (token == Token.IDENTIFIER) {
identName = lexer.stringVal();
hash = lexer.hash_lower();
} else if (token == Token.LITERAL_CHARS) {
identName = '\'' + lexer.stringVal() + '\'';
hash = 0;
} else {
identName = lexer.stringVal();
hash = 0;
}
lexer.nextTokenComma();
SQLExpr expr = new SQLIdentifierExpr(identName, hash);
while (lexer.token() == Token.DOT) {
lexer.nextToken();
String propertyName = lexer.stringVal();
lexer.nextToken();
expr = new SQLPropertyExpr(expr, propertyName);
}
expr.setParent(insertStatement);
columns.add(expr);
columnSize++;
if (lexer.token() == Token.COMMA) {
lexer.nextTokenIdent();
continue;
}
break;
}
columnSize = insertStatement.getColumns().size();
if (tableName != null) {
cachedColumns = insertColumnsCache.get(tableName.hashCode64());
}
}
accept(Token.RPAREN);
int pos = lexer.pos();
if (cachedColumns != null
&& lexer.text.startsWith(cachedColumns.columnsString, pos)) {
List<SQLExpr> columns = stmt.getColumns();
List<SQLExpr> cachedColumns2 = cachedColumns.columns;
for (int i = 0, size = cachedColumns2.size(); i < size; i++) {
columns.add(cachedColumns2.get(i).clone());
}
stmt.setColumnsString(cachedColumns.columnsString);
int p2 = pos + cachedColumns.columnsString.length();
lexer.reset(p2);
lexer.nextToken();
} else {
lexer.nextToken();
if (lexer.token() == Token.SELECT) {
SQLSelect select = this.exprParser.createSelectParser().select();
select.setParent(stmt);
stmt.setQuery(select);
} else {
List<SQLExpr> columns = stmt.getColumns();
if (lexer.token() != Token.RPAREN) {
for (; ; ) {
String identName;
long hash;
Token token = lexer.token();
if (token == Token.IDENTIFIER) {
identName = lexer.stringVal();
hash = lexer.hash_lower();
} else if (token == Token.LITERAL_CHARS) {
identName = '\'' + lexer.stringVal() + '\'';
hash = 0;
} else {
identName = lexer.stringVal();
hash = 0;
}
lexer.nextTokenComma();
SQLExpr expr = new SQLIdentifierExpr(identName, hash);
while (lexer.token() == Token.DOT) {
lexer.nextToken();
String propertyName = lexer.stringVal();
lexer.nextToken();
expr = new SQLPropertyExpr(expr, propertyName);
}
expr.setParent(stmt);
columns.add(expr);
columnSize++;
if (lexer.token() == Token.COMMA) {
lexer.nextTokenIdent();
continue;
}
break;
}
columnSize = stmt.getColumns().size();
if (insertColumnsCache != null && tableName != null) {
String columnsString = lexer.subString(pos, lexer.pos() - pos);
List<SQLExpr> clonedColumns = new ArrayList<SQLExpr>(columnSize);
for (int i = 0; i < columns.size(); i++) {
clonedColumns.add(columns.get(i).clone());
}
insertColumnsCache.put(tableName.hashCode64(), columnsString, clonedColumns);
stmt.setColumnsString(columnsString);
}
}
}
accept(Token.RPAREN);
}
}
if (lexer.token() == Token.VALUES || lexer.identifierEquals(FnvHash.Constants.VALUE)) {
lexer.nextTokenLParen();
parseValueClause(insertStatement.getValuesList(), columnSize, insertStatement);
parseValueClause(stmt.getValuesList(), columnSize, stmt);
} else if (lexer.token() == Token.SET) {
lexer.nextToken();
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause();
insertStatement.addValueCause(values);
stmt.addValueCause(values);
for (; ; ) {
SQLName name = this.exprParser.name();
insertStatement.addColumn(name);
stmt.addColumn(name);
if (lexer.token() == Token.EQ) {
lexer.nextToken();
} else {
@@ -2446,13 +2487,13 @@ public class MySqlStatementParser extends SQLStatementParser {
} else if (lexer.token() == (Token.SELECT)) {
SQLSelect select = this.exprParser.createSelectParser().select();
select.setParent(insertStatement);
insertStatement.setQuery(select);
select.setParent(stmt);
stmt.setQuery(select);
} else if (lexer.token() == (Token.LPAREN)) {
lexer.nextToken();
SQLSelect select = this.exprParser.createSelectParser().select();
select.setParent(insertStatement);
insertStatement.setQuery(select);
select.setParent(stmt);
stmt.setQuery(select);
accept(Token.RPAREN);
}
@@ -2462,7 +2503,7 @@ public class MySqlStatementParser extends SQLStatementParser {
accept(Token.KEY);
accept(Token.UPDATE);
List<SQLExpr> duplicateKeyUpdate = insertStatement.getDuplicateKeyUpdate();
List<SQLExpr> duplicateKeyUpdate = stmt.getDuplicateKeyUpdate();
for (;;) {
SQLName name = this.exprParser.name();
accept(Token.EQ);
@@ -2474,7 +2515,7 @@ public class MySqlStatementParser extends SQLStatementParser {
}
SQLBinaryOpExpr assignment = new SQLBinaryOpExpr(name, SQLBinaryOperator.Equality, value);
assignment.setParent(insertStatement);
assignment.setParent(stmt);
duplicateKeyUpdate.add(assignment);
if (lexer.token() == Token.COMMA) {
@@ -2485,7 +2526,7 @@ public class MySqlStatementParser extends SQLStatementParser {
}
}
return insertStatement;
return stmt;
}
public MySqlSelectParser createSQLSelectParser() {
src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleStatementParser.java
+28 -3
View File
@@ -2656,14 +2656,39 @@ public class OracleStatementParser extends SQLStatementParser {
if (type) {
accept(Token.IS);
acceptIdentifier("REF");
accept(Token.CURSOR);
varItem.setDataType(new SQLDataTypeImpl("REF CURSOR"));
if (lexer.identifierEquals(FnvHash.Constants.RECORD)) {
lexer.nextToken();
SQLRecordDataType recordDataType = new SQLRecordDataType();
accept(Token.LPAREN);
for (;;) {
SQLColumnDefinition column = this.exprParser.parseColumn();
recordDataType.addColumn(column);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
}
break;
}
accept(Token.RPAREN);
varItem.setDataType(recordDataType);
} else {
acceptIdentifier("REF");
accept(Token.CURSOR);
varItem.setDataType(new SQLDataTypeImpl("REF CURSOR"));
}
} else {
varItem.setDataType(this.exprParser.parseDataType(false));
}
varItem.setParent(varDecl);
if (lexer.token() == Token.COLONEQ) {
lexer.nextToken();
SQLExpr defaultVal = this.exprParser.expr();
varItem.setValue(defaultVal);
}
varDecl.getItems().add(varItem);
accept(Token.SEMI);
src/main/java/com/alibaba/druid/sql/parser/InsertColumnsCache.java
+80
View File
@@ -0,0 +1,80 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.parser;
import com.alibaba.druid.sql.ast.SQLExpr;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class InsertColumnsCache {
public static InsertColumnsCache global;
static {
global = new InsertColumnsCache(8192);
}
public ConcurrentMap<Long, Entry> cache = new ConcurrentHashMap<Long, Entry>();
private final Entry[] buckets;
private final int indexMask;
public InsertColumnsCache(int tableSize){
this.indexMask = tableSize - 1;
this.buckets = new Entry[tableSize];
}
public final Entry get(long hashCode64) {
final int bucket = ((int) hashCode64) & indexMask;
for (Entry entry = buckets[bucket]; entry != null; entry = entry.next) {
if (hashCode64 == entry.hashCode64) {
return entry;
}
}
return null;
}
public boolean put(long hashCode64, String columnsString, List<SQLExpr> columns) {
final int bucket = ((int) hashCode64) & indexMask;
for (Entry entry = buckets[bucket]; entry != null; entry = entry.next) {
if (hashCode64 == entry.hashCode64) {
return true;
}
}
Entry entry = new Entry(hashCode64, columnsString, columns, buckets[bucket]);
buckets[bucket] = entry; // 并发是处理时会可能导致缓存丢失，但不影响正确性
return false;
}
public final static class Entry {
public final long hashCode64;
public final String columnsString;
public final List<SQLExpr> columns;
public final Entry next;
public Entry(long hashCode64, String columnsString, List<SQLExpr> columns, Entry next) {
this.hashCode64 = hashCode64;
this.columnsString = columnsString;
this.columns = columns;
this.next = next;
}
}
}
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+6 -1
View File
@@ -43,7 +43,7 @@ public class Lexer {
protected static SymbolTable symbols_l2 = new SymbolTable(512);
protected int features = 0; //SQLParserFeature.of(SQLParserFeature.EnableSQLBinaryOpExprGroup);
protected final String text;
public final String text;
protected int pos;
protected int mark;
@@ -211,6 +211,11 @@ public class Lexer {
this.reset(this.savePoint);
}
public void reset(int pos) {
this.pos = pos;
this.ch = charAt(pos);
}
public Lexer(String input, boolean skipComment){
this.skipComment = skipComment;
src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+6 -1
View File
@@ -1244,12 +1244,17 @@ public class SQLExprParser extends SQLParser {
public SQLName name() {
String identName;
long hash = 0;
if (lexer.token == Token.LITERAL_ALIAS) {
identName = lexer.stringVal();
lexer.nextToken();
} else if (lexer.token == Token.IDENTIFIER) {
identName = lexer.stringVal();
char c0 = identName.charAt(0);
if (c0 != '[') {
hash = lexer.hash_lower();
}
lexer.nextToken();
} else if (lexer.token == Token.LITERAL_CHARS) {
identName = '\'' + lexer.stringVal() + '\'';
@@ -1312,7 +1317,7 @@ public class SQLExprParser extends SQLParser {
}
}
SQLName name = new SQLIdentifierExpr(identName);
SQLName name = new SQLIdentifierExpr(identName, hash);
name = nameRest(name);
src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java
+1
View File
@@ -17,6 +17,7 @@ package com.alibaba.druid.sql.parser;
public enum SQLParserFeature {
KeepInsertValueClauseOriginalString,
UseInsertColumnsCache,
EnableSQLBinaryOpExprGroup,
OptimizedForParameterized,
KeepComments,
src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
+1
View File
@@ -34,6 +34,7 @@ public class SQLStatementParser extends SQLParser {
protected boolean parseCompleteValues = true;
protected int parseValuesSize = 3;
protected SQLSelectListCache selectListCache = null;
protected InsertColumnsCache insertColumnsCache = null;
public SQLStatementParser(String sql){
this(sql, null);
src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java
+69 -3
View File
@@ -643,6 +643,38 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
SQLExpr left = x.getLeft();
SQLExpr right = x.getRight();
if (inputParameters != null
&& operator != SQLBinaryOperator.Equality) {
int varIndex = -1;
if (right instanceof SQLVariantRefExpr) {
varIndex = ((SQLVariantRefExpr) right).getIndex();
}
Object param = null;
if (varIndex >= 0 && varIndex < inputParameters.size()) {
param = inputParameters.get(varIndex);
}
if (param instanceof Collection) {
Collection values = (Collection) param;
if (values.size() > 0) {
print('(');
int valIndex = 0;
for (Object value : values) {
if (valIndex++ != 0) {
print0(ucase ? " OR " : " or ");
}
printExpr(left);
printOperator(operator);
printParameter(value);
}
print(')');
return false;
}
}
}
if (operator.isRelational()
&& left instanceof SQLIntegerExpr
&& right instanceof SQLIntegerExpr) {
@@ -4604,8 +4636,15 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
@Override
public boolean visit(SQLDeclareItem x) {
SQLDataType dataType = x.getDataType();
if (dataType instanceof SQLRecordDataType) {
print0(ucase ? "TYPE " : "type ");
}
x.getName().accept(this);
if (x.getType() == SQLDeclareItem.Type.TABLE) {
print0(ucase ? " TABLE" : " table");
int size = x.getTableElementList().size();
@@ -4628,9 +4667,14 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
} else if (x.getType() == SQLDeclareItem.Type.CURSOR) {
print0(ucase ? " CURSOR" : " cursor");
} else {
if (x.getDataType() != null) {
print(' ');
x.getDataType().accept(this);
if (dataType != null) {
if (dataType instanceof SQLRecordDataType) {
print0(ucase ? " IS " : " is ");
} else {
print(' ');
}
dataType.accept(this);
}
if (x.getValue() != null) {
if (JdbcConstants.MYSQL.equals(getDbType())) {
@@ -5720,6 +5764,28 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
return false;
}
@Override
public boolean visit(SQLRecordDataType x) {
print0(ucase ? "RECORD (" : "record (");
indentCount++;
println();
List<SQLColumnDefinition> columns = x.getColumns();
for (int i = 0; i < columns.size(); i++) {
if (i != 0) {
println();
}
columns.get(i).accept(this);
if (i != columns.size() - 1) {
print0(", ");
}
}
indentCount--;
println();
print(')');
return false;
}
@Override
public boolean visit(SQLExprStatement x) {
x.getExpr().accept(this);
src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java
+3
View File
@@ -793,4 +793,7 @@ public interface SQLASTVisitor {
boolean visit(SQLDropSynonymStatement x);
void endVisit(SQLDropSynonymStatement x);
boolean visit(SQLRecordDataType x);
void endVisit(SQLRecordDataType x);
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java
+10
View File
@@ -1846,6 +1846,16 @@ public class SQLASTVisitorAdapter implements SQLASTVisitor {
}
@Override
public boolean visit(SQLRecordDataType x) {
return true;
}
@Override
public void endVisit(SQLRecordDataType x) {
}
public final boolean isEnabled(VisitorFeature feature) {
return VisitorFeature.isEnabled(this.features, feature);
}
src/main/java/com/alibaba/druid/util/FnvHash.java
+1
View File
@@ -521,6 +521,7 @@ public final class FnvHash {
long TRUNC = fnv1a_64_lower("TRUNC");
long SYSTIMESTAMP = fnv1a_64_lower("SYSTIMESTAMP");
long TYPE = fnv1a_64_lower("TYPE");
long RECORD = fnv1a_64_lower("RECORD");
long MAP = fnv1a_64_lower("MAP");
long ONLY = fnv1a_64_lower("ONLY");
long MEMBER = fnv1a_64_lower("MEMBER");