diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java index 9c4ecdc442..2c25b6b3e6 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java @@ -27,6 +27,7 @@ public abstract class SQLInsertInto extends SQLObjectImpl { protected SQLExprTableSource tableSource; protected final List columns = new ArrayList(); protected transient String columnsString; + protected transient long columnsStringHash; protected SQLSelect query; protected final List valuesList = new ArrayList(); @@ -142,7 +143,12 @@ public abstract class SQLInsertInto extends SQLObjectImpl { return columnsString; } - public void setColumnsString(String columnsString) { + public long getColumnsStringHash() { + return columnsStringHash; + } + + public void setColumnsString(String columnsString, long columnsStringHash) { this.columnsString = columnsString; + this.columnsStringHash = columnsStringHash; } } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java index e185120e8d..c902e2bd9d 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java @@ -54,6 +54,7 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery protected List sortBy; protected String cachedSelectList; // optimized for SelectListCache + protected long cachedSelectListHash; // optimized for SelectListCache protected String dbType; @@ -666,8 +667,13 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery return cachedSelectList; } - public void setCachedSelectList(String cachedSelectList) { + public void setCachedSelectList(String cachedSelectList, long cachedSelectListHash) { this.cachedSelectList = cachedSelectList; + this.cachedSelectListHash = cachedSelectListHash; + } + + public long getCachedSelectListHash() { + return cachedSelectListHash; } public String getDbType() { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java index a51409c905..deada0d47b 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java @@ -2403,7 +2403,7 @@ public class MySqlStatementParser extends SQLStatementParser { columns.add(cachedColumns2.get(i).clone()); } } - stmt.setColumnsString(cachedColumns.columnsFormattedString); + stmt.setColumnsString(cachedColumns.columnsFormattedString, cachedColumns.columnsFormattedStringHash); int p2 = pos + cachedColumns.columnsString.length(); lexer.reset(p2); lexer.nextToken(); @@ -2467,9 +2467,10 @@ public class MySqlStatementParser extends SQLStatementParser { outputVisitor.printInsertColumns(columns); String formattedColumnsString = buf.toString(); + long columnsFormattedStringHash = FnvHash.fnv1a_64_lower(formattedColumnsString); insertColumnsCache.put(tableName.hashCode64(), columnsString, formattedColumnsString, clonedColumns); - stmt.setColumnsString(formattedColumnsString); + stmt.setColumnsString(formattedColumnsString, columnsFormattedStringHash); } } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java index b3ef2d0938..b72f02812d 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java @@ -118,7 +118,9 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV String cachedSelectList = x.getCachedSelectList(); if (cachedSelectList != null) { - print0(cachedSelectList); + if (!isEnabled(VisitorFeature.OutputSkipSelectListCacheString)) { + print0(cachedSelectList); + } } else { print0(ucase ? "SELECT " : "select "); @@ -861,7 +863,9 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV String columnsString = x.getColumnsString(); if (columnsString != null) { - print0(columnsString); + if (!isEnabled(VisitorFeature.OutputSkipInsertColumnsString)) { + print0(columnsString); + } } else { List columns = x.getColumns(); if (columns.size() > 0) { diff --git a/src/main/java/com/alibaba/druid/sql/parser/InsertColumnsCache.java b/src/main/java/com/alibaba/druid/sql/parser/InsertColumnsCache.java index e168a68ae1..dcda5d6f6e 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/InsertColumnsCache.java +++ b/src/main/java/com/alibaba/druid/sql/parser/InsertColumnsCache.java @@ -16,6 +16,7 @@ package com.alibaba.druid.sql.parser; import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.util.FnvHash; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -68,6 +69,7 @@ public class InsertColumnsCache { public final long hashCode64; public final String columnsString; public final String columnsFormattedString; + public final long columnsFormattedStringHash; public final List columns; public final Entry next; @@ -75,6 +77,7 @@ public class InsertColumnsCache { this.hashCode64 = hashCode64; this.columnsString = columnsString; this.columnsFormattedString = columnsFormattedString; + this.columnsFormattedStringHash = FnvHash.fnv1a_64_lower(columnsFormattedString); this.columns = columns; this.next = next; } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java index b84e22d325..df7bbfed6a 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java @@ -21,6 +21,7 @@ public enum SQLParserFeature { UseInsertColumnsCache, EnableSQLBinaryOpExprGroup, OptimizedForParameterized, + OptimizedForForParameterizedSkipValue, KeepComments, SkipComments, StrictForWall diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectListCache.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectListCache.java index fc821fe7d3..0e44e9eebe 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectListCache.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectListCache.java @@ -18,6 +18,7 @@ package com.alibaba.druid.sql.parser; import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; import com.alibaba.druid.support.logging.Log; import com.alibaba.druid.support.logging.LogFactory; +import com.alibaba.druid.util.FnvHash; import java.util.ArrayList; import java.util.List; @@ -47,10 +48,13 @@ public class SQLSelectListCache { selectParser.accept(Token.FROM); selectParser.accept(Token.EOF); + String printSql = queryBlock.toString(); + long printSqlHash = FnvHash.fnv1a_64_lower(printSql); entries.add( new Entry(select.substring(6) , queryBlock - , queryBlock.toString()) + , printSql + , printSqlHash) ); if (entries.size() > 5) { @@ -80,8 +84,7 @@ public class SQLSelectListCache { if (text.startsWith(block, pos)) { //SQLSelectQueryBlock queryBlockCached = queryBlockCache.get(i); // queryBlockCached.cloneSelectListTo(queryBlock); - String printSql = entry.printSql; - queryBlock.setCachedSelectList(printSql); + queryBlock.setCachedSelectList(entry.printSql, entry.printSqlHash); int len = pos + block.length(); lexer.reset(len, text.charAt(len), Token.FROM); @@ -92,14 +95,16 @@ public class SQLSelectListCache { } private static class Entry { - public final String sql; + public final String sql; public final SQLSelectQueryBlock queryBlock; - public final String printSql; + public final String printSql; + public final long printSqlHash; - public Entry(String sql, SQLSelectQueryBlock queryBlock, String printSql) { + public Entry(String sql, SQLSelectQueryBlock queryBlock, String printSql, long printSqlHash) { this.sql = sql; this.queryBlock = queryBlock; this.printSql = printSql; + this.printSqlHash = printSqlHash; } } } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index 15f36e72c3..b32204f859 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -3145,7 +3145,7 @@ public class SQLStatementParser extends SQLParser { } protected void parseValueClause(List valueClauseList, int columnSize, SQLObject parent) { - final boolean optimizedForParameterized = lexer.isEnabled(SQLParserFeature.OptimizedForParameterized); + final boolean optimizedForParameterized = lexer.isEnabled(SQLParserFeature.OptimizedForForParameterizedSkipValue); for (int i = 0; ; ++i) { int startPos = lexer.pos() - 1; diff --git a/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java b/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java index b661f82720..734b63e05d 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java @@ -18,26 +18,33 @@ package com.alibaba.druid.sql.visitor; import java.util.List; import com.alibaba.druid.sql.ast.*; -import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement; -import com.alibaba.druid.sql.ast.statement.SQLInsertStatement; -import com.alibaba.druid.sql.ast.statement.SQLSelect; -import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.db2.visitor.DB2OutputVisitor; +import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement; +import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitor; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor; import com.alibaba.druid.sql.dialect.oracle.visitor.OracleParameterizedOutputVisitor; import com.alibaba.druid.sql.dialect.phoenix.visitor.PhoenixOutputVisitor; import com.alibaba.druid.sql.dialect.postgresql.visitor.PGOutputVisitor; import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerOutputVisitor; import com.alibaba.druid.sql.parser.*; +import com.alibaba.druid.util.FnvHash; import com.alibaba.druid.util.JdbcUtils; public class ParameterizedOutputVisitorUtils { - private final static SQLParserFeature[] features = { + private final static SQLParserFeature[] defaultFeatures = { SQLParserFeature.EnableSQLBinaryOpExprGroup, SQLParserFeature.UseInsertColumnsCache, SQLParserFeature.OptimizedForParameterized }; + private final static SQLParserFeature[] defaultFeatures2 = { + SQLParserFeature.EnableSQLBinaryOpExprGroup, + SQLParserFeature.UseInsertColumnsCache, + SQLParserFeature.OptimizedForParameterized, + SQLParserFeature.OptimizedForForParameterizedSkipValue, + }; + public static String parameterize(String sql, String dbType) { return parameterize(sql, dbType, null, null); } @@ -57,6 +64,11 @@ public class ParameterizedOutputVisitorUtils { public static String parameterize(String sql , String dbType , SQLSelectListCache selectListCache, List outParameters) { + + final SQLParserFeature[] features = outParameters == null + ? defaultFeatures2 + : defaultFeatures; + SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType, features); if (selectListCache != null) { @@ -102,6 +114,99 @@ public class ParameterizedOutputVisitorUtils { return out.toString(); } + public static long parameterizeHash(String sql + , String dbType + , List outParameters) { + return parameterizeHash(sql, dbType, null, outParameters); + } + + public static long parameterizeHash(String sql + , String dbType + , SQLSelectListCache selectListCache + , List outParameters) { + + final SQLParserFeature[] features = outParameters == null + ? defaultFeatures2 + : defaultFeatures; + + SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType, features); + + if (selectListCache != null) { + parser.setSelectListCache(selectListCache); + } + + List statementList = parser.parseStatementList(); + final int stmtSize = statementList.size(); + if (stmtSize == 0) { + return 0L; + } + + StringBuilder out = new StringBuilder(sql.length()); + ParameterizedVisitor visitor = createParameterizedOutputVisitor(out, dbType); + if (outParameters != null) { + visitor.setOutputParameters(outParameters); + } + + if (stmtSize == 1) { + SQLStatement stmt = statementList.get(0); + if (stmt.getClass() == SQLSelectStatement.class) { + SQLSelectStatement selectStmt = (SQLSelectStatement) stmt; + + if (selectListCache != null) { + SQLSelectQueryBlock queryBlock = selectStmt.getSelect().getQueryBlock(); + if (queryBlock != null) { + String cachedSelectList = queryBlock.getCachedSelectList(); + long cachedSelectListHash = queryBlock.getCachedSelectListHash(); + if (cachedSelectList != null) { + visitor.config(VisitorFeature.OutputSkipSelectListCacheString, true); + } + + visitor.visit(selectStmt); + return FnvHash.fnv1a_64_lower(cachedSelectListHash, out); + } + } + + visitor.visit(selectStmt); + } else if (stmt.getClass() == MySqlInsertStatement.class) { + MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt; + String columnsString = insertStmt.getColumnsString(); + if (columnsString != null) { + long columnsStringHash = insertStmt.getColumnsStringHash(); + visitor.config(VisitorFeature.OutputSkipInsertColumnsString, true); + + ((MySqlASTVisitor) visitor).visit(insertStmt); + return FnvHash.fnv1a_64_lower(columnsStringHash, out); + } + } else { + stmt.accept(visitor); + } + + return FnvHash.fnv1a_64_lower(out); + } + + for (int i = 0; i < statementList.size(); i++) { + if (i > 0) { + out.append(";\n"); + } + SQLStatement stmt = statementList.get(i); + + if (stmt.hasBeforeComment()) { + stmt.getBeforeCommentsDirect().clear(); + } + + Class stmtClass = stmt.getClass(); + if (stmtClass == SQLSelectStatement.class) { // only for performance + SQLSelectStatement selectStatement = (SQLSelectStatement) stmt; + visitor.visit(selectStatement); + visitor.postVisit(selectStatement); + } else { + stmt.accept(visitor); + } + } + + return FnvHash.fnv1a_64_lower(out); + } + public static String parameterize(List statementList, String dbType) { StringBuilder out = new StringBuilder(); ParameterizedVisitor visitor = createParameterizedOutputVisitor(out, dbType); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedVisitor.java index 0246bef57d..e3f23cd2b9 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedVisitor.java @@ -26,4 +26,6 @@ public interface ParameterizedVisitor extends PrintableVisitor { String getDbType(); void setOutputParameters(List parameters); + + void config(VisitorFeature feature, boolean state); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/VisitorFeature.java b/src/main/java/com/alibaba/druid/sql/visitor/VisitorFeature.java index 07dd21cf15..8bdba1b6b1 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/VisitorFeature.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/VisitorFeature.java @@ -20,8 +20,9 @@ public enum VisitorFeature { OutputPrettyFormat, OutputParameterized, OutputDesensitize, - OutputUseInsertValueClauseOriginalString - + OutputUseInsertValueClauseOriginalString, + OutputSkipSelectListCacheString, + OutputSkipInsertColumnsString, ; private VisitorFeature(){ diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index 6331e06170..15e401dbaa 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -134,6 +134,38 @@ public final class FnvHash { return hashCode; } + public static long fnv1a_64_lower(StringBuilder key) { + long hashCode = BASIC; + for (int i = 0; i < key.length(); ++i) { + char ch = key.charAt(i); + + if (ch >= 'A' && ch <= 'Z') { + ch = (char) (ch + 32); + } + + hashCode ^= ch; + hashCode *= PRIME; + } + + return hashCode; + } + + public static long fnv1a_64_lower(long basic, StringBuilder key) { + long hashCode = basic; + for (int i = 0; i < key.length(); ++i) { + char ch = key.charAt(i); + + if (ch >= 'A' && ch <= 'Z') { + ch = (char) (ch + 32); + } + + hashCode ^= ch; + hashCode *= PRIME; + } + + return hashCode; + } + public static long hashCode64(String key, int offset, int end) { long hashCode = BASIC; for (int i = offset; i < end; ++i) { diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParamTest.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParamTest.java new file mode 100644 index 0000000000..3ec04e74e1 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParamTest.java @@ -0,0 +1,24 @@ +package com.alibaba.druid.bvt.sql.mysql.param; + +import com.alibaba.druid.sql.parser.SQLSelectListCache; +import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; +import com.alibaba.druid.util.FnvHash; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +public class MySqlParamTest extends TestCase { + public final static String dbType = JdbcConstants.MYSQL; + + public void test_for_mysql_param() throws Exception { + String sql = "/* 0b853c4a26094480140194289e3d24/0.1.1.2.1//2e3b9cf7/ */select `miller_cart`.`CART_ID`,`miller_cart`.`SKU_ID`,`miller_cart`.`ITEM_ID`,`miller_cart`.`QUANTITY`,`miller_cart`.`USER_ID`,`miller_cart`.`SELLER_ID`,`miller_cart`.`STATUS`,`miller_cart`.`EXT_STATUS`,`miller_cart`.`TYPE`,`miller_cart`.`SUB_TYPE`,`miller_cart`.`GMT_CREATE`,`miller_cart`.`GMT_MODIFIED`,`miller_cart`.`ATTRIBUTE`,`miller_cart`.`ATTRIBUTE_CC`,`miller_cart`.`EX2` from `miller_cart_0304` `miller_cart` where ((`miller_cart`.`USER_ID` = 2732851504) AND ((`miller_cart`.`STATUS` = 1) AND (`miller_cart`.`TYPE` IN (0,5,10)))) limit 0,200"; + long hash1 = ParameterizedOutputVisitorUtils.parameterizeHash(sql, dbType, null, null); + long hash2 = FnvHash.fnv1a_64_lower(ParameterizedOutputVisitorUtils.parameterize(sql, dbType)); + assertEquals(hash1, hash2); + + SQLSelectListCache cache = new SQLSelectListCache(dbType); + cache.add("select `miller_cart`.`CART_ID`,`miller_cart`.`SKU_ID`,`miller_cart`.`ITEM_ID`,`miller_cart`.`QUANTITY`,`miller_cart`.`USER_ID`,`miller_cart`.`SELLER_ID`,`miller_cart`.`STATUS`,`miller_cart`.`EXT_STATUS`,`miller_cart`.`TYPE`,`miller_cart`.`SUB_TYPE`,`miller_cart`.`GMT_CREATE`,`miller_cart`.`GMT_MODIFIED`,`miller_cart`.`ATTRIBUTE`,`miller_cart`.`ATTRIBUTE_CC`,`miller_cart`.`EX2` from"); + + long hash3 = ParameterizedOutputVisitorUtils.parameterizeHash(sql, dbType, cache, null); + assertEquals(hash1, hash3); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_58.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_58.java new file mode 100644 index 0000000000..25696678c0 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_58.java @@ -0,0 +1,45 @@ +package com.alibaba.druid.bvt.sql.mysql.param; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.parser.SQLParserFeature; +import com.alibaba.druid.sql.parser.SQLParserUtils; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; +import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by wenshao on 16/8/23. + */ +public class MySqlParameterizedOutputVisitorTest_58 extends TestCase { + final String dbType = JdbcConstants.MYSQL; + public void test_for_parameterize() throws Exception { + + String sql = "select * from t where id = 101"; + + List params = new ArrayList(); + String psql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType, params); + assertEquals("SELECT *\n" + + "FROM t\n" + + "WHERE id = ?", psql); + assertEquals(1, params.size()); + assertEquals(101, params.get(0)); + } + + public void test_for_parameterize_insert() throws Exception { + String sql = "insert into mytab(fid,fname)values(1001,'wenshao');"; + + List params = new ArrayList(); + String psql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType, params); + assertEquals("INSERT INTO mytab(fid, fname)\n" + + "VALUES (?, ?);", psql); + assertEquals(2, params.size()); + assertEquals(1001, params.get(0)); + assertEquals("wenshao", params.get(1)); + } +}