improved sql parser.

src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java

@@ -27,6 +27,7 @@ public abstract class SQLInsertInto extends SQLObjectImpl {
     protected SQLExprTableSource        tableSource;
     protected final List<SQLExpr>       columns = new ArrayList<SQLExpr>();
     protected transient String          columnsString;
+    protected transient long            columnsStringHash;
     protected SQLSelect                 query;
     protected final List<ValuesClause>  valuesList = new ArrayList<ValuesClause>();
 
@@ -142,7 +143,12 @@ public abstract class SQLInsertInto extends SQLObjectImpl {
         return columnsString;
     }
 
-    public void setColumnsString(String columnsString) {
+    public long getColumnsStringHash() {
+        return columnsStringHash;
+    }
+
+    public void setColumnsString(String columnsString, long columnsStringHash) {
         this.columnsString = columnsString;
+        this.columnsStringHash = columnsStringHash;
     }
 }

src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java

@@ -54,6 +54,7 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery
     protected List<SQLSelectOrderByItem> sortBy;
 
     protected String                     cachedSelectList; // optimized for SelectListCache
+    protected long                       cachedSelectListHash; // optimized for SelectListCache
 
     protected String                     dbType;
 
@@ -666,8 +667,13 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery
         return cachedSelectList;
     }
 
-    public void setCachedSelectList(String cachedSelectList) {
+    public void setCachedSelectList(String cachedSelectList, long cachedSelectListHash) {
         this.cachedSelectList = cachedSelectList;
+        this.cachedSelectListHash = cachedSelectListHash;
+    }
+
+    public long getCachedSelectListHash() {
+        return cachedSelectListHash;
     }
 
     public String getDbType() {

src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java

@@ -2403,7 +2403,7 @@ public class MySqlStatementParser extends SQLStatementParser {
                         columns.add(cachedColumns2.get(i).clone());
                     }
                 }
-                stmt.setColumnsString(cachedColumns.columnsFormattedString);
+                stmt.setColumnsString(cachedColumns.columnsFormattedString, cachedColumns.columnsFormattedStringHash);
                 int p2 = pos + cachedColumns.columnsString.length();
                 lexer.reset(p2);
                 lexer.nextToken();
@@ -2467,9 +2467,10 @@ public class MySqlStatementParser extends SQLStatementParser {
                             outputVisitor.printInsertColumns(columns);
 
                             String formattedColumnsString = buf.toString();
+                            long columnsFormattedStringHash = FnvHash.fnv1a_64_lower(formattedColumnsString);
 
                             insertColumnsCache.put(tableName.hashCode64(), columnsString, formattedColumnsString, clonedColumns);
-                            stmt.setColumnsString(formattedColumnsString);
+                            stmt.setColumnsString(formattedColumnsString, columnsFormattedStringHash);
                         }
                     }
                 }

src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java

@@ -118,7 +118,9 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV
         String cachedSelectList = x.getCachedSelectList();
 
         if (cachedSelectList != null) {
-            print0(cachedSelectList);
+            if (!isEnabled(VisitorFeature.OutputSkipSelectListCacheString)) {
+                print0(cachedSelectList);
+            }
         } else {
             print0(ucase ? "SELECT " : "select ");
 
@@ -861,7 +863,9 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV
 
         String columnsString = x.getColumnsString();
         if (columnsString != null) {
-            print0(columnsString);
+            if (!isEnabled(VisitorFeature.OutputSkipInsertColumnsString)) {
+                print0(columnsString);
+            }
         } else {
             List<SQLExpr> columns = x.getColumns();
             if (columns.size() > 0) {

src/main/java/com/alibaba/druid/sql/parser/InsertColumnsCache.java

@@ -16,6 +16,7 @@
 package com.alibaba.druid.sql.parser;
 
 import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.util.FnvHash;
 
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
@@ -68,6 +69,7 @@ public class InsertColumnsCache {
         public final long hashCode64;
         public final String columnsString;
         public final String columnsFormattedString;
+        public final long columnsFormattedStringHash;
         public final List<SQLExpr> columns;
         public final Entry next;
 
@@ -75,6 +77,7 @@ public class InsertColumnsCache {
             this.hashCode64 = hashCode64;
             this.columnsString = columnsString;
             this.columnsFormattedString = columnsFormattedString;
+            this.columnsFormattedStringHash = FnvHash.fnv1a_64_lower(columnsFormattedString);
             this.columns = columns;
             this.next = next;
         }

src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java

@@ -21,6 +21,7 @@ public enum SQLParserFeature {
     UseInsertColumnsCache,
     EnableSQLBinaryOpExprGroup,
     OptimizedForParameterized,
+    OptimizedForForParameterizedSkipValue,
     KeepComments,
     SkipComments,
     StrictForWall

src/main/java/com/alibaba/druid/sql/parser/SQLSelectListCache.java

@@ -18,6 +18,7 @@ package com.alibaba.druid.sql.parser;
 import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
 import com.alibaba.druid.support.logging.Log;
 import com.alibaba.druid.support.logging.LogFactory;
+import com.alibaba.druid.util.FnvHash;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,10 +48,13 @@ public class SQLSelectListCache {
         selectParser.accept(Token.FROM);
         selectParser.accept(Token.EOF);
 
+        String printSql = queryBlock.toString();
+        long printSqlHash = FnvHash.fnv1a_64_lower(printSql);
         entries.add(
                 new Entry(select.substring(6)
                         , queryBlock
-                        , queryBlock.toString())
+                        , printSql
+                        , printSqlHash)
         );
 
         if (entries.size() > 5) {
@@ -80,8 +84,7 @@ public class SQLSelectListCache {
             if (text.startsWith(block, pos)) {
                 //SQLSelectQueryBlock queryBlockCached = queryBlockCache.get(i);
                 // queryBlockCached.cloneSelectListTo(queryBlock);
-                String printSql = entry.printSql;
-                queryBlock.setCachedSelectList(printSql);
+                queryBlock.setCachedSelectList(entry.printSql, entry.printSqlHash);
 
                 int len = pos + block.length();
                 lexer.reset(len, text.charAt(len), Token.FROM);
@@ -92,14 +95,16 @@ public class SQLSelectListCache {
     }
 
     private static class Entry {
-        public final String sql;
+        public final String              sql;
         public final SQLSelectQueryBlock queryBlock;
-        public final String printSql;
+        public final String              printSql;
+        public final long                printSqlHash;
 
-        public Entry(String sql, SQLSelectQueryBlock queryBlock, String printSql) {
+        public Entry(String sql, SQLSelectQueryBlock queryBlock, String printSql, long printSqlHash) {
             this.sql = sql;
             this.queryBlock = queryBlock;
             this.printSql = printSql;
+            this.printSqlHash = printSqlHash;
         }
     }
 }

src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java

@@ -3145,7 +3145,7 @@ public class SQLStatementParser extends SQLParser {
     }
 
     protected void parseValueClause(List<SQLInsertStatement.ValuesClause> valueClauseList, int columnSize, SQLObject parent) {
-        final boolean optimizedForParameterized = lexer.isEnabled(SQLParserFeature.OptimizedForParameterized);
+        final boolean optimizedForParameterized = lexer.isEnabled(SQLParserFeature.OptimizedForForParameterizedSkipValue);
 
         for (int i = 0; ; ++i) {
             int startPos = lexer.pos() - 1;

src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java

@@ -18,26 +18,33 @@ package com.alibaba.druid.sql.visitor;
 import java.util.List;
 
 import com.alibaba.druid.sql.ast.*;
-import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement;
-import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
-import com.alibaba.druid.sql.ast.statement.SQLSelect;
-import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
+import com.alibaba.druid.sql.ast.statement.*;
 import com.alibaba.druid.sql.dialect.db2.visitor.DB2OutputVisitor;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement;
+import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitor;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
 import com.alibaba.druid.sql.dialect.oracle.visitor.OracleParameterizedOutputVisitor;
 import com.alibaba.druid.sql.dialect.phoenix.visitor.PhoenixOutputVisitor;
 import com.alibaba.druid.sql.dialect.postgresql.visitor.PGOutputVisitor;
 import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerOutputVisitor;
 import com.alibaba.druid.sql.parser.*;
+import com.alibaba.druid.util.FnvHash;
 import com.alibaba.druid.util.JdbcUtils;
 
 public class ParameterizedOutputVisitorUtils {
-    private final static SQLParserFeature[] features = {
+    private final static SQLParserFeature[] defaultFeatures = {
             SQLParserFeature.EnableSQLBinaryOpExprGroup,
             SQLParserFeature.UseInsertColumnsCache,
             SQLParserFeature.OptimizedForParameterized
     };
 
+    private final static SQLParserFeature[] defaultFeatures2 = {
+            SQLParserFeature.EnableSQLBinaryOpExprGroup,
+            SQLParserFeature.UseInsertColumnsCache,
+            SQLParserFeature.OptimizedForParameterized,
+            SQLParserFeature.OptimizedForForParameterizedSkipValue,
+    };
+
     public static String parameterize(String sql, String dbType) {
         return parameterize(sql, dbType, null, null);
     }
@@ -57,6 +64,11 @@ public class ParameterizedOutputVisitorUtils {
     public static String parameterize(String sql
             , String dbType
             , SQLSelectListCache selectListCache, List<Object> outParameters) {
+
+        final SQLParserFeature[] features = outParameters == null
+                ? defaultFeatures2
+                : defaultFeatures;
+
         SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType, features);
 
         if (selectListCache != null) {
@@ -102,6 +114,99 @@ public class ParameterizedOutputVisitorUtils {
         return out.toString();
     }
 
+    public static long parameterizeHash(String sql
+            , String dbType
+            , List<Object> outParameters) {
+        return parameterizeHash(sql, dbType, null, outParameters);
+    }
+
+    public static long parameterizeHash(String sql
+            , String dbType
+            , SQLSelectListCache selectListCache
+            , List<Object> outParameters) {
+
+        final SQLParserFeature[] features = outParameters == null
+                ? defaultFeatures2
+                : defaultFeatures;
+
+        SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType, features);
+
+        if (selectListCache != null) {
+            parser.setSelectListCache(selectListCache);
+        }
+
+        List<SQLStatement> statementList = parser.parseStatementList();
+        final int stmtSize = statementList.size();
+        if (stmtSize == 0) {
+            return 0L;
+        }
+
+        StringBuilder out = new StringBuilder(sql.length());
+        ParameterizedVisitor visitor = createParameterizedOutputVisitor(out, dbType);
+        if (outParameters != null) {
+            visitor.setOutputParameters(outParameters);
+        }
+
+        if (stmtSize == 1) {
+            SQLStatement stmt = statementList.get(0);
+            if (stmt.getClass() == SQLSelectStatement.class) {
+                SQLSelectStatement selectStmt = (SQLSelectStatement) stmt;
+
+                if (selectListCache != null) {
+                    SQLSelectQueryBlock queryBlock = selectStmt.getSelect().getQueryBlock();
+                    if (queryBlock != null) {
+                        String cachedSelectList = queryBlock.getCachedSelectList();
+                        long cachedSelectListHash = queryBlock.getCachedSelectListHash();
+                        if (cachedSelectList != null) {
+                            visitor.config(VisitorFeature.OutputSkipSelectListCacheString, true);
+                        }
+
+                        visitor.visit(selectStmt);
+                        return FnvHash.fnv1a_64_lower(cachedSelectListHash, out);
+                    }
+                }
+
+                visitor.visit(selectStmt);
+            } else if (stmt.getClass() == MySqlInsertStatement.class) {
+                MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt;
+                String columnsString = insertStmt.getColumnsString();
+                if (columnsString != null) {
+                    long columnsStringHash = insertStmt.getColumnsStringHash();
+                    visitor.config(VisitorFeature.OutputSkipInsertColumnsString, true);
+
+                    ((MySqlASTVisitor) visitor).visit(insertStmt);
+                    return FnvHash.fnv1a_64_lower(columnsStringHash, out);
+                }
+            } else {
+                stmt.accept(visitor);
+            }
+
+            return FnvHash.fnv1a_64_lower(out);
+        }
+
+        for (int i = 0; i < statementList.size(); i++) {
+            if (i > 0) {
+                out.append(";\n");
+            }
+            SQLStatement stmt = statementList.get(i);
+
+            if (stmt.hasBeforeComment()) {
+                stmt.getBeforeCommentsDirect().clear();
+            }
+
+            Class<?> stmtClass = stmt.getClass();
+            if (stmtClass == SQLSelectStatement.class) { // only for performance
+                SQLSelectStatement selectStatement = (SQLSelectStatement) stmt;
+                visitor.visit(selectStatement);
+                visitor.postVisit(selectStatement);
+            } else {
+                stmt.accept(visitor);
+            }
+        }
+
+        return FnvHash.fnv1a_64_lower(out);
+    }
+
     public static String parameterize(List<SQLStatement> statementList, String dbType) {
         StringBuilder out = new StringBuilder();
         ParameterizedVisitor visitor = createParameterizedOutputVisitor(out, dbType);

src/main/java/com/alibaba/druid/sql/visitor/ParameterizedVisitor.java

@@ -26,4 +26,6 @@ public interface ParameterizedVisitor extends PrintableVisitor {
     String getDbType();
 
     void setOutputParameters(List<Object> parameters);
+
+    void config(VisitorFeature feature, boolean state);
 }

src/main/java/com/alibaba/druid/sql/visitor/VisitorFeature.java

@@ -20,8 +20,9 @@ public enum  VisitorFeature {
     OutputPrettyFormat,
     OutputParameterized,
     OutputDesensitize,
-    OutputUseInsertValueClauseOriginalString
-
+    OutputUseInsertValueClauseOriginalString,
+    OutputSkipSelectListCacheString,
+    OutputSkipInsertColumnsString,
     ;
 
     private VisitorFeature(){

src/main/java/com/alibaba/druid/util/FnvHash.java

@@ -134,6 +134,38 @@ public final class FnvHash {
         return hashCode;
     }
 
+    public static long fnv1a_64_lower(StringBuilder key) {
+        long hashCode = BASIC;
+        for (int i = 0; i < key.length(); ++i) {
+            char ch = key.charAt(i);
+
+            if (ch >= 'A' && ch <= 'Z') {
+                ch = (char) (ch + 32);
+            }
+
+            hashCode ^= ch;
+            hashCode *= PRIME;
+        }
+
+        return hashCode;
+    }
+
+    public static long fnv1a_64_lower(long basic, StringBuilder key) {
+        long hashCode = basic;
+        for (int i = 0; i < key.length(); ++i) {
+            char ch = key.charAt(i);
+
+            if (ch >= 'A' && ch <= 'Z') {
+                ch = (char) (ch + 32);
+            }
+
+            hashCode ^= ch;
+            hashCode *= PRIME;
+        }
+
+        return hashCode;
+    }
+
     public static long hashCode64(String key, int offset, int end) {
         long hashCode = BASIC;
         for (int i = offset; i < end; ++i) {

src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParamTest.java

@@ -0,0 +1,24 @@
+package com.alibaba.druid.bvt.sql.mysql.param;
+
+import com.alibaba.druid.sql.parser.SQLSelectListCache;
+import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils;
+import com.alibaba.druid.util.FnvHash;
+import com.alibaba.druid.util.JdbcConstants;
+import junit.framework.TestCase;
+
+public class MySqlParamTest extends TestCase {
+    public final static String dbType = JdbcConstants.MYSQL;
+
+    public void test_for_mysql_param() throws Exception {
+        String sql = "/* 0b853c4a26094480140194289e3d24/0.1.1.2.1//2e3b9cf7/ */select `miller_cart`.`CART_ID`,`miller_cart`.`SKU_ID`,`miller_cart`.`ITEM_ID`,`miller_cart`.`QUANTITY`,`miller_cart`.`USER_ID`,`miller_cart`.`SELLER_ID`,`miller_cart`.`STATUS`,`miller_cart`.`EXT_STATUS`,`miller_cart`.`TYPE`,`miller_cart`.`SUB_TYPE`,`miller_cart`.`GMT_CREATE`,`miller_cart`.`GMT_MODIFIED`,`miller_cart`.`ATTRIBUTE`,`miller_cart`.`ATTRIBUTE_CC`,`miller_cart`.`EX2` from `miller_cart_0304` `miller_cart` where ((`miller_cart`.`USER_ID` = 2732851504) AND ((`miller_cart`.`STATUS` = 1) AND (`miller_cart`.`TYPE` IN (0,5,10)))) limit 0,200";
+        long hash1 = ParameterizedOutputVisitorUtils.parameterizeHash(sql, dbType, null, null);
+        long hash2 = FnvHash.fnv1a_64_lower(ParameterizedOutputVisitorUtils.parameterize(sql, dbType));
+        assertEquals(hash1, hash2);
+
+        SQLSelectListCache cache = new SQLSelectListCache(dbType);
+        cache.add("select `miller_cart`.`CART_ID`,`miller_cart`.`SKU_ID`,`miller_cart`.`ITEM_ID`,`miller_cart`.`QUANTITY`,`miller_cart`.`USER_ID`,`miller_cart`.`SELLER_ID`,`miller_cart`.`STATUS`,`miller_cart`.`EXT_STATUS`,`miller_cart`.`TYPE`,`miller_cart`.`SUB_TYPE`,`miller_cart`.`GMT_CREATE`,`miller_cart`.`GMT_MODIFIED`,`miller_cart`.`ATTRIBUTE`,`miller_cart`.`ATTRIBUTE_CC`,`miller_cart`.`EX2` from");
+
+        long hash3 = ParameterizedOutputVisitorUtils.parameterizeHash(sql, dbType, cache, null);
+        assertEquals(hash1, hash3);
+    }
+}

src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_58.java

@@ -0,0 +1,45 @@
+package com.alibaba.druid.bvt.sql.mysql.param;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.parser.SQLParserFeature;
+import com.alibaba.druid.sql.parser.SQLParserUtils;
+import com.alibaba.druid.sql.parser.SQLStatementParser;
+import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils;
+import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
+import com.alibaba.druid.util.JdbcConstants;
+import junit.framework.TestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by wenshao on 16/8/23.
+ */
+public class MySqlParameterizedOutputVisitorTest_58 extends TestCase {
+    final String dbType = JdbcConstants.MYSQL;
+    public void test_for_parameterize() throws Exception {
+
+        String sql = "select * from t where id = 101";
+
+        List<Object> params = new ArrayList<Object>();
+        String psql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType, params);
+        assertEquals("SELECT *\n" +
+                "FROM t\n" +
+                "WHERE id = ?", psql);
+        assertEquals(1, params.size());
+        assertEquals(101, params.get(0));
+    }
+
+    public void test_for_parameterize_insert() throws Exception {
+        String sql = "insert into mytab(fid,fname)values(1001,'wenshao');";
+
+        List<Object> params = new ArrayList<Object>();
+        String psql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType, params);
+        assertEquals("INSERT INTO mytab(fid, fname)\n" +
+                "VALUES (?, ?);", psql);
+        assertEquals(2, params.size());
+        assertEquals(1001, params.get(0));
+        assertEquals("wenshao", params.get(1));
+    }
+}