improved sql parser.

src/main/java/com/alibaba/druid/sql/ast/SQLObjectImpl.java

@@ -190,7 +190,12 @@ public abstract class SQLObjectImpl implements SQLObject {
     }
     
     public boolean hasBeforeComment() {
-        List<String> comments = getBeforeCommentsDirect();
+        if (attributes == null) {
+            return false;
+        }
+
+        List<String> comments = (List<String>) attributes.get("format.before_comment");
+
         if (comments == null) {
             return false;
         }

src/main/java/com/alibaba/druid/sql/ast/expr/SQLAggregateExpr.java

@@ -22,7 +22,6 @@ import java.util.List;
 import com.alibaba.druid.sql.SQLUtils;
 import com.alibaba.druid.sql.ast.*;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
-import com.alibaba.druid.util.FnvHash.Constants;
 import com.alibaba.druid.util.FnvHash;
 
 public class SQLAggregateExpr extends SQLExprImpl implements Serializable, SQLReplaceable {

src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java

@@ -52,7 +52,6 @@ import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleUnique;
 import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleUsingIndexClause;
 import com.alibaba.druid.sql.parser.*;
 import com.alibaba.druid.util.FnvHash;
-import com.alibaba.druid.util.FnvHash.Constants;
 import com.alibaba.druid.util.JdbcConstants;
 
 public class OracleExprParser extends SQLExprParser {

src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleToMySqlOutputVisitor.java

@@ -28,7 +28,6 @@ import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
 import com.alibaba.druid.sql.ast.statement.SQLTableSource;
 import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleSelectQueryBlock;
 import com.alibaba.druid.util.FnvHash;
-import com.alibaba.druid.util.FnvHash.Constants;
 
 public class OracleToMySqlOutputVisitor extends OracleOutputVisitor {
 

src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java

@@ -45,7 +45,6 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
 import com.alibaba.druid.sql.ast.statement.SQLUnique;
 import com.alibaba.druid.sql.ast.statement.SQLUpdateSetItem;
 import com.alibaba.druid.util.FnvHash;
-import com.alibaba.druid.util.FnvHash.Constants;
 import com.alibaba.druid.util.JdbcConstants;
 
 public class SQLExprParser extends SQLParser {

src/main/java/com/alibaba/druid/util/FnvHash.java

@@ -36,7 +36,19 @@ public final class FnvHash {
         return hash;
     }
 
+    public static long fnv1a_64(char[] chars) {
+        if (chars == null) {
+            return 0;
+        }
+        long hash = BASIC;
+        for (int i = 0; i < chars.length; ++i) {
+            char c = chars[i];
+            hash ^= c;
+            hash *= PRIME;
+        }
 
+        return hash;
+    }
 
     /**
      * lower & normalized & fnv_1a_64

src/test/java/com/alibaba/druid/benckmark/FnvHashTest.java

@@ -0,0 +1,38 @@
+package com.alibaba.druid.benckmark;
+
+import com.alibaba.druid.util.FnvHash;
+import junit.framework.TestCase;
+
+public class FnvHashTest extends TestCase {
+    static String sql = "SELECT id, item_id, rule_id, tag_id, ext , gmt_create, gmt_modified FROM wukong_preview_item_tag WHERE item_id = ? AND rule_id = ?";
+    static char[] chars = sql.toCharArray();
+
+    public void test_perf_fnv() throws Exception {
+        for (int i = 0; i < 5; ++i) {
+//            perf_hashCode64(sql); // 168
+            perf_hashCode64(chars); // 169
+        }
+    }
+
+    public long perf_hashCode64(String sql) {
+        long val = 0;
+        long startMillis = System.currentTimeMillis();
+        for (int i = 0; i < 1000 * 1000; ++i) {
+            val = FnvHash.fnv1a_64(sql);
+        }
+        long millis = System.currentTimeMillis() - startMillis;
+        System.out.println("millis : " + millis);
+        return val;
+    }
+
+    public long perf_hashCode64(char[] sql) {
+        long val = 0;
+        long startMillis = System.currentTimeMillis();
+        for (int i = 0; i < 1000 * 1000; ++i) {
+            val = FnvHash.fnv1a_64(sql);
+        }
+        long millis = System.currentTimeMillis() - startMillis;
+        System.out.println("millis : " + millis);
+        return val;
+    }
+}

src/test/java/com/alibaba/druid/benckmark/sql/MySqlParameterizedBenchmark.java

@@ -2,13 +2,16 @@ package com.alibaba.druid.benckmark.sql;
 
 import com.alibaba.druid.sql.SQLUtils;
 import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.dialect.mysql.parser.MySqlLexer;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
+import com.alibaba.druid.sql.parser.Token;
 import com.alibaba.druid.sql.repository.SchemaRepository;
 import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+import com.alibaba.druid.util.FnvHash;
 import com.alibaba.druid.util.JdbcConstants;
 import com.alibaba.druid.wall.WallUtils;
 import com.alibaba.druid.wall.WallVisitor;
@@ -29,17 +32,24 @@ public class MySqlParameterizedBenchmark extends TestCase {
         String sql4 = "UPDATE ROLLBACK_ON_FAIL TARGET_AFFECT_ROW 1 "
                 + "`table_3966` AS `table_3966_11` SET `version` = `version` + 3, `gmt_modified` = NOW(), `optype` = ?, `feature` = ? "
                 + "WHERE `sub_biz_order_id` = ? AND `biz_order_type` = ? AND `id` = ? AND `ti_id` = ? AND `optype` = ? AND `root_id` = ?";
+        String sql5= "SELECT id, item_id, rule_id, tag_id, ext , gmt_create, gmt_modified FROM wukong_preview_item_tag WHERE item_id = ? AND rule_id = ?";
 
         for (int i = 0; i < 5; ++i) {
-            perf(sql); // 6740 6201 4752 4514 4391 4218 4127 4124
+//            perf(sql); // 6740 6201 4752 4514 4391 4218 4127 4124
 //            perf(sql2); // 2948 2928 2869 2780 2502
 //            perf(sql3); // 15093 10392 10416 10154 10007 9126 8907
 //            perf(sql4); // 4429 4190 4023 3747
+//            perf(sql5); // 1917
 
 //            perf_parse(sql); // 4643 4377 4345 3801 3627 3228 2961 2959
 //            perf_parse(sql2); // 1918 1779 1666 1646
 //            perf_parse(sql3); // 9174 5875 5805 5536 5717
 //            perf_parse(sql4); // 2953 2502
+            perf_parse(sql5); // 1339
+
+//            perf_lexer(sql5); // 813
+
+//            perf_hashCode64(sql5); // 181
 
 //            perf_stat(sql); // 15214 11793 13628 13561 13259 9946 7637 7444 7389 7326 7176 6687 5973 5660
 
@@ -101,4 +111,37 @@ public class MySqlParameterizedBenchmark extends TestCase {
         long millis = System.currentTimeMillis() - startMillis;
         System.out.println("millis : " + millis);
     }
+
+    public void perf_lexer(String sql) {
+        long startMillis = System.currentTimeMillis();
+        for (int i = 0; i < 1000 * 1000; ++i) {
+            int literal_count = 0;
+
+            MySqlLexer lexer = new MySqlLexer(sql);
+
+            for (; ; ) {
+                lexer.nextToken();
+                Token token = lexer.token();
+
+                if (token == Token.LITERAL_INT || token == Token.LITERAL_CHARS) {
+                    literal_count++;
+                    break;
+                } else if (token == Token.EOF || token == Token.ERROR) {
+                    break;
+                }
+            }
+
+        }
+        long millis = System.currentTimeMillis() - startMillis;
+        System.out.println("millis : " + millis);
+    }
+
+    public void perf_hashCode64(String sql) {
+        long startMillis = System.currentTimeMillis();
+        for (int i = 0; i < 1000 * 1000; ++i) {
+            FnvHash.fnv1a_64(sql);
+        }
+        long millis = System.currentTimeMillis() - startMillis;
+        System.out.println("millis : " + millis);
+    }
 }

src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest56.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 1999-2017 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.postgresql.select;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+import com.alibaba.druid.util.JdbcConstants;
+import junit.framework.TestCase;
+
+import java.util.List;
+
+public class PGSelectTest56 extends TestCase {
+    private final String dbType = JdbcConstants.POSTGRESQL;
+
+    public void test_0() throws Exception {
+        String sql = "SELECT to_char((CreateDate || ' ' || CAST (HourArgment AS VARCHAR) || ':00:00') :: TIMESTAMP,'YYYY-MM-DD HH24') \"intoTime \" FROM analyzedata.StatTime AS cs WHERE 1 = 1 AND cs.CreateDate >= to_date( '2017-08-31 00:00:00', 'YYYY-MM-DD HH24:MI:SS' ) AND cs.CreateDate <= to_date( '2017-08-31 23:00:00', 'YYYY-MM-DD HH24:MI:SS' ) GROUP BY to_char(( CreateDate || ' ' || CAST (HourArgment AS VARCHAR) || ':00:00') :: TIMESTAMP, 'YYYY-MM-DD HH24')";
+
+        List<SQLStatement> stmtList = SQLUtils.parseStatements(sql, dbType);
+        SQLStatement stmt = stmtList.get(0);
+
+        assertEquals("SELECT to_char(CreateDate || ' ' || CAST(HourArgment AS VARCHAR) || ':00:00'::TIMESTAMP, 'YYYY-MM-DD HH24') AS \"intoTime \"\n" +
+                "FROM analyzedata.StatTime cs\n" +
+                "WHERE 1 = 1\n" +
+                "\tAND cs.CreateDate >= to_date('2017-08-31 00:00:00', 'YYYY-MM-DD HH24:MI:SS')\n" +
+                "\tAND cs.CreateDate <= to_date('2017-08-31 23:00:00', 'YYYY-MM-DD HH24:MI:SS')\n" +
+                "GROUP BY to_char(CreateDate || ' ' || CAST(HourArgment AS VARCHAR) || ':00:00'::TIMESTAMP, 'YYYY-MM-DD HH24')", SQLUtils.toPGString(stmt));
+        
+        assertEquals("select b.*, st_astext(p.pos) as pos, st_astext(p.polygon) as polygon\n" +
+                "from ts_biz b\n" +
+                "\tjoin (\n" +
+                "\t\tselect *\n" +
+                "\t\tfrom ts_polygon\n" +
+                "\t\twhere type in (?)\n" +
+                "\t\t\tand st_intersects(st_transform(ST_GeomFromText(?, 4326), 26986), st_transform(polygon, 26986))\n" +
+                "\t) p\n" +
+                "\ton b.objid = p.objid\n" +
+                "\t\tand b.type = p.type\n" +
+                "\t\tand (b.cp_code = p.cp_code\n" +
+                "\t\t\tor (b.cp_code is null\n" +
+                "\t\t\t\tand p.cp_code is null))\n" +
+                "where b.cp_code = ?\n" +
+                "\tor b.cp_code is null", SQLUtils.toPGString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
+
+        assertEquals(1, stmtList.size());
+
+        SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType);
+        stmt.accept(visitor);
+
+//        System.out.println("Tables : " + visitor.getTables());
+//        System.out.println("fields : " + visitor.getColumns());
+//        System.out.println("coditions : " + visitor.getConditions());
+
+        assertEquals(7, visitor.getColumns().size());
+        assertEquals(2, visitor.getTables().size());
+    }
+}