improved sql parser. for issue #2080

This commit is contained in:
wenshao
2017-10-31 22:59:25 +08:00
parent 46e8847c22
commit 1cfe3c639a
13 changed files with 316 additions and 80 deletions
@@ -15,6 +15,7 @@
*/
package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLObject;
@@ -24,5 +25,8 @@ public interface SQLConstraint extends SQLObject {
void setName(SQLName value);
SQLExpr getComment();
void setComment(SQLExpr x);
void simplify();
}
@@ -17,6 +17,7 @@ package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLCommentHint;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLObjectImpl;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
@@ -24,12 +25,12 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import java.util.List;
public abstract class SQLConstraintImpl extends SQLObjectImpl implements SQLConstraint {
protected String dbType;
protected String dbType;
protected SQLName name;
private Boolean enable;
private Boolean validate;
private Boolean rely;
protected Boolean enable;
protected Boolean validate;
protected Boolean rely;
protected SQLExpr comment;
public List<SQLCommentHint> hints;
@@ -106,6 +107,17 @@ public abstract class SQLConstraintImpl extends SQLObjectImpl implements SQLCons
this.dbType = dbType;
}
public SQLExpr getComment() {
return comment;
}
public void setComment(SQLExpr x) {
if (x != null) {
x.setParent(this);
}
this.comment = x;
}
public void simplify() {
if (this.name instanceof SQLIdentifierExpr) {
SQLIdentifierExpr identExpr = (SQLIdentifierExpr) this.name;
@@ -92,9 +92,9 @@ public class SQLInsertStatement extends SQLInsertInto implements SQLStatement {
public static class ValuesClause extends SQLObjectImpl {
private final List<SQLExpr> values;
private transient String originalString;
private final List<SQLExpr> values;
private transient String originalString;
private transient int replaceCount;
public ValuesClause(){
this(new ArrayList<SQLExpr>());
@@ -151,6 +151,14 @@ public class SQLInsertStatement extends SQLInsertInto implements SQLStatement {
public void setOriginalString(String originalString) {
this.originalString = originalString;
}
public int getReplaceCount() {
return replaceCount;
}
public void incrementReplaceCount() {
this.replaceCount++;
}
}
@Override
@@ -29,14 +29,7 @@ import com.alibaba.druid.sql.ast.SQLSubPartitionByHash;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
import com.alibaba.druid.sql.ast.expr.SQLNumberExpr;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLCheck;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLForeignKeyConstraint;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLTableConstraint;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlKey;
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlPrimaryKey;
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlUnique;
@@ -784,6 +777,8 @@ public class MySqlCreateTableParser extends SQLCreateTableParser {
name = this.exprParser.name();
}
SQLTableConstraint constraint = null;
if (lexer.token() == (Token.KEY)) {
lexer.nextToken();
@@ -840,41 +835,44 @@ public class MySqlCreateTableParser extends SQLCreateTableParser {
lexer.nextToken();
}
return key;
}
if (lexer.token() == Token.PRIMARY) {
constraint = key;
} else if (lexer.token() == Token.PRIMARY) {
MySqlPrimaryKey pk = this.getExprParser().parsePrimaryKey();
if (name != null) {
pk.setName(name);
}
pk.setHasConstaint(hasConstaint);
return pk;
}
if (lexer.token() == Token.UNIQUE) {
constraint = pk;
} else if (lexer.token() == Token.UNIQUE) {
MySqlUnique uk = this.getExprParser().parseUnique();
if (name != null) {
uk.setName(name);
}
uk.setHasConstaint(hasConstaint);
return uk;
}
if (lexer.token() == Token.FOREIGN) {
constraint = uk;
} else if (lexer.token() == Token.FOREIGN) {
MysqlForeignKey fk = this.getExprParser().parseForeignKey();
fk.setName(name);
fk.setHasConstraint(hasConstaint);
return fk;
}
if (lexer.token() == Token.CHECK) {
constraint = fk;
} else if (lexer.token() == Token.CHECK) {
lexer.nextToken();
SQLCheck check = new SQLCheck();
check.setName(name);
SQLExpr expr = this.exprParser.expr();
check.setExpr(expr);
return check;
constraint = check;
}
if (constraint != null) {
if (lexer.token() == Token.COMMENT) {
lexer.nextToken();
SQLExpr comment = this.exprParser.primary();
constraint.setComment(comment);
}
return constraint;
}
throw new ParserException("TODO. " + lexer.info());
@@ -15,6 +15,7 @@
*/
package com.alibaba.druid.sql.dialect.mysql.parser;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.SQLParameter.ParameterType;
import com.alibaba.druid.sql.ast.expr.*;
@@ -36,6 +37,7 @@ import com.alibaba.druid.sql.dialect.mysql.ast.clause.MySqlSelectIntoStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.*;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlLockTableStatement.LockType;
import com.alibaba.druid.sql.parser.*;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.FnvHash;
import com.alibaba.druid.util.JdbcConstants;
@@ -2385,12 +2387,14 @@ public class MySqlStatementParser extends SQLStatementParser {
int pos = lexer.pos();
if (cachedColumns != null
&& lexer.text.startsWith(cachedColumns.columnsString, pos)) {
List<SQLExpr> columns = stmt.getColumns();
List<SQLExpr> cachedColumns2 = cachedColumns.columns;
for (int i = 0, size = cachedColumns2.size(); i < size; i++) {
columns.add(cachedColumns2.get(i).clone());
if (!lexer.isEnabled(SQLParserFeature.OptimizedForParameterized)) {
List<SQLExpr> columns = stmt.getColumns();
List<SQLExpr> cachedColumns2 = cachedColumns.columns;
for (int i = 0, size = cachedColumns2.size(); i < size; i++) {
columns.add(cachedColumns2.get(i).clone());
}
}
stmt.setColumnsString(cachedColumns.columnsString);
stmt.setColumnsString(cachedColumns.columnsFormattedString);
int p2 = pos + cachedColumns.columnsString.length();
lexer.reset(p2);
lexer.nextToken();
@@ -2449,8 +2453,14 @@ public class MySqlStatementParser extends SQLStatementParser {
clonedColumns.add(columns.get(i).clone());
}
insertColumnsCache.put(tableName.hashCode64(), columnsString, clonedColumns);
stmt.setColumnsString(columnsString);
StringBuilder buf = new StringBuilder();
SQLASTOutputVisitor outputVisitor = SQLUtils.createOutputVisitor(buf, dbType);
outputVisitor.printInsertColumns(columns);
String formattedColumnsString = buf.toString();
insertColumnsCache.put(tableName.hashCode64(), columnsString, formattedColumnsString, clonedColumns);
stmt.setColumnsString(formattedColumnsString);
}
}
}
@@ -859,27 +859,32 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV
tableSource.accept(this);
}
List<SQLExpr> columns = x.getColumns();
if (columns.size() > 0) {
this.indentCount++;
print0(" (");
for (int i = 0, size = columns.size(); i < size; ++i) {
if (i != 0) {
if (i % 5 == 0) {
println();
String columnsString = x.getColumnsString();
if (columnsString != null) {
print0(columnsString);
} else {
List<SQLExpr> columns = x.getColumns();
if (columns.size() > 0) {
this.indentCount++;
print0(" (");
for (int i = 0, size = columns.size(); i < size; ++i) {
if (i != 0) {
if (i % 5 == 0) {
println();
}
print0(", ");
}
print0(", ");
}
SQLExpr column = columns.get(i);
if (column instanceof SQLIdentifierExpr) {
print0(((SQLIdentifierExpr) column).getName());
} else {
printExpr(column);
SQLExpr column = columns.get(i);
if (column instanceof SQLIdentifierExpr) {
print0(((SQLIdentifierExpr) column).getName());
} else {
printExpr(column);
}
}
print(')');
this.indentCount--;
}
print(')');
this.indentCount--;
}
List<SQLInsertStatement.ValuesClause> valuesList = x.getValuesList();
@@ -2682,6 +2687,12 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV
printAndAccept(x.getColumns(), ", ");
print(')');
SQLExpr comment = x.getComment();
if (comment != null) {
print0(" COMMENT ");
comment.accept(this);
}
return false;
}
@@ -4025,4 +4036,40 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV
x.accept(this);
}
}
public void printInsertColumns(List<SQLExpr> columns) {
final int size = columns.size();
if (size > 0) {
if (size > 5) {
this.indentCount++;
print(' ');
}
print('(');
for (int i = 0; i < size; ++i) {
if (i != 0) {
if (i % 5 == 0) {
println();
}
print0(", ");
}
SQLExpr column = columns.get(i);
if (column instanceof SQLIdentifierExpr) {
visit((SQLIdentifierExpr) column);
} else {
printExpr(column);
}
String dataType = (String) column.getAttribute("dataType");
if (dataType != null) {
print(' ');
print(dataType);
}
}
print(')');
if (size > 5) {
this.indentCount--;
}
}
}
} //
@@ -49,7 +49,7 @@ public class InsertColumnsCache {
return null;
}
public boolean put(long hashCode64, String columnsString, List<SQLExpr> columns) {
public boolean put(long hashCode64, String columnsString, String columnsFormattedString, List<SQLExpr> columns) {
final int bucket = ((int) hashCode64) & indexMask;
for (Entry entry = buckets[bucket]; entry != null; entry = entry.next) {
@@ -58,7 +58,7 @@ public class InsertColumnsCache {
}
}
Entry entry = new Entry(hashCode64, columnsString, columns, buckets[bucket]);
Entry entry = new Entry(hashCode64, columnsString, columnsFormattedString, columns, buckets[bucket]);
buckets[bucket] = entry; // 并发是处理时会可能导致缓存丢失,但不影响正确性
return false;
@@ -67,12 +67,14 @@ public class InsertColumnsCache {
public final static class Entry {
public final long hashCode64;
public final String columnsString;
public final String columnsFormattedString;
public final List<SQLExpr> columns;
public final Entry next;
public Entry(long hashCode64, String columnsString, List<SQLExpr> columns, Entry next) {
public Entry(long hashCode64, String columnsString, String columnsFormattedString, List<SQLExpr> columns, Entry next) {
this.hashCode64 = hashCode64;
this.columnsString = columnsString;
this.columnsFormattedString = columnsFormattedString;
this.columns = columns;
this.next = next;
}
@@ -15,25 +15,24 @@
*/
package com.alibaba.druid.sql.parser;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.support.logging.Log;
import com.alibaba.druid.support.logging.LogFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
public class SQLSelectListCache {
private final String dbType;
private final List<String> sqlCache = new ArrayList<String>();
private final List<SQLSelectQueryBlock> queryBlockCache = new ArrayList<SQLSelectQueryBlock>();
private final List<String> printSqlList = new ArrayList<String>();
private final static Log LOG = LogFactory.getLog(SQLSelectListCache.class);
private final String dbType;
private final List<Entry> entries = new CopyOnWriteArrayList<Entry>();
public SQLSelectListCache(String dbType) {
this.dbType = dbType;
}
public synchronized void add(String select) {
public void add(String select) {
if (select == null || select.length() == 0) {
return;
}
@@ -48,9 +47,23 @@ public class SQLSelectListCache {
selectParser.accept(Token.FROM);
selectParser.accept(Token.EOF);
queryBlockCache.add(queryBlock);
printSqlList.add(queryBlock.toString());
sqlCache.add(select.substring(6));
entries.add(
new Entry(select.substring(6)
, queryBlock
, queryBlock.toString())
);
if (entries.size() > 5) {
LOG.warn("SelectListCache is too large.");
}
}
public int getSize() {
return entries.size();
}
public void clear() {
entries.clear();
}
public boolean match(Lexer lexer, SQLSelectQueryBlock queryBlock) {
@@ -61,12 +74,13 @@ public class SQLSelectListCache {
int pos = lexer.pos;
String text = lexer.text;
for (int i = 0; i < sqlCache.size(); i++) {
String block = sqlCache.get(i);
for (int i = 0; i < entries.size(); i++) {
Entry entry = entries.get(i);
String block = entry.sql;
if (text.startsWith(block, pos)) {
//SQLSelectQueryBlock queryBlockCached = queryBlockCache.get(i);
// queryBlockCached.cloneSelectListTo(queryBlock);
String printSql = printSqlList.get(i);
String printSql = entry.printSql;
queryBlock.setCachedSelectList(printSql);
int len = pos + block.length();
@@ -76,4 +90,16 @@ public class SQLSelectListCache {
}
return false;
}
private static class Entry {
public final String sql;
public final SQLSelectQueryBlock queryBlock;
public final String printSql;
public Entry(String sql, SQLSelectQueryBlock queryBlock, String printSql) {
this.sql = sql;
this.queryBlock = queryBlock;
this.printSql = printSql;
}
}
}
@@ -3134,6 +3134,8 @@ public class SQLStatementParser extends SQLParser {
}
protected void parseValueClause(List<SQLInsertStatement.ValuesClause> valueClauseList, int columnSize, SQLObject parent) {
final boolean optimizedForParameterized = lexer.isEnabled(SQLParserFeature.OptimizedForParameterized);
for (int i = 0; ; ++i) {
int startPos = lexer.pos() - 1;
@@ -3149,17 +3151,34 @@ public class SQLStatementParser extends SQLParser {
} else {
valueExprList = new ArrayList<SQLExpr>();
}
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause(valueExprList);
values.setParent(parent);
for (; ; ) {
SQLExpr expr;
if (lexer.token() == Token.LITERAL_INT) {
expr = new SQLIntegerExpr(lexer.integerValue());
if (optimizedForParameterized) {
expr = new SQLVariantRefExpr("?");
values.incrementReplaceCount();
} else {
expr = new SQLIntegerExpr(lexer.integerValue());
}
lexer.nextTokenComma();
} else if (lexer.token() == Token.LITERAL_CHARS) {
expr = new SQLCharExpr(lexer.stringVal());
if (optimizedForParameterized) {
expr = new SQLVariantRefExpr("?");
values.incrementReplaceCount();
} else {
expr = new SQLCharExpr(lexer.stringVal());
}
lexer.nextTokenComma();
} else if (lexer.token() == Token.LITERAL_NCHARS) {
expr = new SQLNCharExpr(lexer.stringVal());
if (optimizedForParameterized) {
expr = new SQLVariantRefExpr("?");
values.incrementReplaceCount();
} else {
expr = new SQLNCharExpr(lexer.stringVal());
}
lexer.nextTokenComma();
} else {
expr = exprParser.expr();
@@ -3167,10 +3186,12 @@ public class SQLStatementParser extends SQLParser {
if (lexer.token() == Token.COMMA) {
valueExprList.add(expr);
expr.setParent(values);
lexer.nextTokenValue();
continue;
} else if (lexer.token() == Token.RPAREN) {
valueExprList.add(expr);
expr.setParent(values);
break;
} else {
expr = this.exprParser.primaryRest(expr);
@@ -3188,9 +3209,6 @@ public class SQLStatementParser extends SQLParser {
}
}
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause(valueExprList);
values.setParent(parent);
if (lexer.isEnabled(SQLParserFeature.KeepInsertValueClauseOriginalString)) {
int endPos = lexer.pos();
String orginalString = lexer.subString(startPos, endPos - startPos);
@@ -34,6 +34,7 @@ import com.alibaba.druid.util.JdbcUtils;
public class ParameterizedOutputVisitorUtils {
private final static SQLParserFeature[] features = {
SQLParserFeature.EnableSQLBinaryOpExprGroup,
SQLParserFeature.UseInsertColumnsCache,
SQLParserFeature.OptimizedForParameterized
};
@@ -2392,7 +2392,12 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
x.getTableSource().accept(this);
printInsertColumns(x.getColumns());
String columnsString = x.getColumnsString();
if (columnsString != null) {
print0(columnsString);
} else {
printInsertColumns(x.getColumns());
}
if (!x.getValuesList().isEmpty()) {
println();
@@ -2408,7 +2413,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
return false;
}
protected void printInsertColumns(List<SQLExpr> columns) {
public void printInsertColumns(List<SQLExpr> columns) {
final int size = columns.size();
if (size > 0) {
if (size > 5) {
@@ -2849,6 +2854,8 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
print('(');
this.indentCount++;
this.replaceCount += x.getReplaceCount();
final List<SQLExpr> values = x.getValues();
for (int i = 0, size = values.size(); i < size; ++i) {
if (i != 0) {