From 5badd35b7fe3b482deaaae7c87f9c09eeb6bf1d3 Mon Sep 17 00:00:00 2001 From: wenshao Date: Mon, 4 Sep 2017 03:10:45 +0800 Subject: [PATCH] improved sql parser. --- .../alibaba/druid/sql/ast/SQLObjectImpl.java | 7 +- .../druid/sql/ast/expr/SQLAggregateExpr.java | 1 - .../oracle/parser/OracleExprParser.java | 1 - .../visitor/OracleToMySqlOutputVisitor.java | 1 - .../druid/sql/parser/SQLExprParser.java | 1 - .../java/com/alibaba/druid/util/FnvHash.java | 12 ++++ .../alibaba/druid/benckmark/FnvHashTest.java | 38 ++++++++++ .../sql/MySqlParameterizedBenchmark.java | 45 +++++++++++- .../sql/postgresql/select/PGSelectTest56.java | 70 +++++++++++++++++++ 9 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/benckmark/FnvHashTest.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest56.java diff --git a/src/main/java/com/alibaba/druid/sql/ast/SQLObjectImpl.java b/src/main/java/com/alibaba/druid/sql/ast/SQLObjectImpl.java index cfc588970e..52f4209040 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/SQLObjectImpl.java +++ b/src/main/java/com/alibaba/druid/sql/ast/SQLObjectImpl.java @@ -190,7 +190,12 @@ public abstract class SQLObjectImpl implements SQLObject { } public boolean hasBeforeComment() { - List comments = getBeforeCommentsDirect(); + if (attributes == null) { + return false; + } + + List comments = (List) attributes.get("format.before_comment"); + if (comments == null) { return false; } diff --git a/src/main/java/com/alibaba/druid/sql/ast/expr/SQLAggregateExpr.java b/src/main/java/com/alibaba/druid/sql/ast/expr/SQLAggregateExpr.java index decf1df11c..a85706cf74 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/expr/SQLAggregateExpr.java +++ b/src/main/java/com/alibaba/druid/sql/ast/expr/SQLAggregateExpr.java @@ -22,7 +22,6 @@ import java.util.List; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.*; import com.alibaba.druid.sql.visitor.SQLASTVisitor; -import com.alibaba.druid.util.FnvHash.Constants; import com.alibaba.druid.util.FnvHash; public class SQLAggregateExpr extends SQLExprImpl implements Serializable, SQLReplaceable { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java index d7705cf78f..8768adc522 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java @@ -52,7 +52,6 @@ import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleUnique; import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleUsingIndexClause; import com.alibaba.druid.sql.parser.*; import com.alibaba.druid.util.FnvHash; -import com.alibaba.druid.util.FnvHash.Constants; import com.alibaba.druid.util.JdbcConstants; public class OracleExprParser extends SQLExprParser { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleToMySqlOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleToMySqlOutputVisitor.java index 2ffd4fd141..3dc5c73598 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleToMySqlOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleToMySqlOutputVisitor.java @@ -28,7 +28,6 @@ import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource; import com.alibaba.druid.sql.ast.statement.SQLTableSource; import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleSelectQueryBlock; import com.alibaba.druid.util.FnvHash; -import com.alibaba.druid.util.FnvHash.Constants; public class OracleToMySqlOutputVisitor extends OracleOutputVisitor { diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index ed08057ddc..a107db2747 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -45,7 +45,6 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; import com.alibaba.druid.sql.ast.statement.SQLUnique; import com.alibaba.druid.sql.ast.statement.SQLUpdateSetItem; import com.alibaba.druid.util.FnvHash; -import com.alibaba.druid.util.FnvHash.Constants; import com.alibaba.druid.util.JdbcConstants; public class SQLExprParser extends SQLParser { diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index d3a54c6b96..4e25e52df2 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -36,7 +36,19 @@ public final class FnvHash { return hash; } + public static long fnv1a_64(char[] chars) { + if (chars == null) { + return 0; + } + long hash = BASIC; + for (int i = 0; i < chars.length; ++i) { + char c = chars[i]; + hash ^= c; + hash *= PRIME; + } + return hash; + } /** * lower & normalized & fnv_1a_64 diff --git a/src/test/java/com/alibaba/druid/benckmark/FnvHashTest.java b/src/test/java/com/alibaba/druid/benckmark/FnvHashTest.java new file mode 100644 index 0000000000..63ed8a9d2c --- /dev/null +++ b/src/test/java/com/alibaba/druid/benckmark/FnvHashTest.java @@ -0,0 +1,38 @@ +package com.alibaba.druid.benckmark; + +import com.alibaba.druid.util.FnvHash; +import junit.framework.TestCase; + +public class FnvHashTest extends TestCase { + static String sql = "SELECT id, item_id, rule_id, tag_id, ext , gmt_create, gmt_modified FROM wukong_preview_item_tag WHERE item_id = ? AND rule_id = ?"; + static char[] chars = sql.toCharArray(); + + public void test_perf_fnv() throws Exception { + for (int i = 0; i < 5; ++i) { +// perf_hashCode64(sql); // 168 + perf_hashCode64(chars); // 169 + } + } + + public long perf_hashCode64(String sql) { + long val = 0; + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + val = FnvHash.fnv1a_64(sql); + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + return val; + } + + public long perf_hashCode64(char[] sql) { + long val = 0; + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + val = FnvHash.fnv1a_64(sql); + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + return val; + } +} diff --git a/src/test/java/com/alibaba/druid/benckmark/sql/MySqlParameterizedBenchmark.java b/src/test/java/com/alibaba/druid/benckmark/sql/MySqlParameterizedBenchmark.java index 3a51b0e097..5d71b9cacf 100644 --- a/src/test/java/com/alibaba/druid/benckmark/sql/MySqlParameterizedBenchmark.java +++ b/src/test/java/com/alibaba/druid/benckmark/sql/MySqlParameterizedBenchmark.java @@ -2,13 +2,16 @@ package com.alibaba.druid.benckmark.sql; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlLexer; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor; +import com.alibaba.druid.sql.parser.Token; import com.alibaba.druid.sql.repository.SchemaRepository; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; import com.alibaba.druid.sql.visitor.SQLASTVisitor; import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.FnvHash; import com.alibaba.druid.util.JdbcConstants; import com.alibaba.druid.wall.WallUtils; import com.alibaba.druid.wall.WallVisitor; @@ -29,17 +32,24 @@ public class MySqlParameterizedBenchmark extends TestCase { String sql4 = "UPDATE ROLLBACK_ON_FAIL TARGET_AFFECT_ROW 1 " + "`table_3966` AS `table_3966_11` SET `version` = `version` + 3, `gmt_modified` = NOW(), `optype` = ?, `feature` = ? " + "WHERE `sub_biz_order_id` = ? AND `biz_order_type` = ? AND `id` = ? AND `ti_id` = ? AND `optype` = ? AND `root_id` = ?"; + String sql5= "SELECT id, item_id, rule_id, tag_id, ext , gmt_create, gmt_modified FROM wukong_preview_item_tag WHERE item_id = ? AND rule_id = ?"; for (int i = 0; i < 5; ++i) { - perf(sql); // 6740 6201 4752 4514 4391 4218 4127 4124 +// perf(sql); // 6740 6201 4752 4514 4391 4218 4127 4124 // perf(sql2); // 2948 2928 2869 2780 2502 // perf(sql3); // 15093 10392 10416 10154 10007 9126 8907 // perf(sql4); // 4429 4190 4023 3747 +// perf(sql5); // 1917 // perf_parse(sql); // 4643 4377 4345 3801 3627 3228 2961 2959 // perf_parse(sql2); // 1918 1779 1666 1646 // perf_parse(sql3); // 9174 5875 5805 5536 5717 // perf_parse(sql4); // 2953 2502 + perf_parse(sql5); // 1339 + +// perf_lexer(sql5); // 813 + +// perf_hashCode64(sql5); // 181 // perf_stat(sql); // 15214 11793 13628 13561 13259 9946 7637 7444 7389 7326 7176 6687 5973 5660 @@ -101,4 +111,37 @@ public class MySqlParameterizedBenchmark extends TestCase { long millis = System.currentTimeMillis() - startMillis; System.out.println("millis : " + millis); } + + public void perf_lexer(String sql) { + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + int literal_count = 0; + + MySqlLexer lexer = new MySqlLexer(sql); + + for (; ; ) { + lexer.nextToken(); + Token token = lexer.token(); + + if (token == Token.LITERAL_INT || token == Token.LITERAL_CHARS) { + literal_count++; + break; + } else if (token == Token.EOF || token == Token.ERROR) { + break; + } + } + + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + } + + public void perf_hashCode64(String sql) { + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + FnvHash.fnv1a_64(sql); + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest56.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest56.java new file mode 100644 index 0000000000..4919c86dfe --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest56.java @@ -0,0 +1,70 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.postgresql.select; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class PGSelectTest56 extends TestCase { + private final String dbType = JdbcConstants.POSTGRESQL; + + public void test_0() throws Exception { + String sql = "SELECT to_char((CreateDate || ' ' || CAST (HourArgment AS VARCHAR) || ':00:00') :: TIMESTAMP,'YYYY-MM-DD HH24') \"intoTime \" FROM analyzedata.StatTime AS cs WHERE 1 = 1 AND cs.CreateDate >= to_date( '2017-08-31 00:00:00', 'YYYY-MM-DD HH24:MI:SS' ) AND cs.CreateDate <= to_date( '2017-08-31 23:00:00', 'YYYY-MM-DD HH24:MI:SS' ) GROUP BY to_char(( CreateDate || ' ' || CAST (HourArgment AS VARCHAR) || ':00:00') :: TIMESTAMP, 'YYYY-MM-DD HH24')"; + + List stmtList = SQLUtils.parseStatements(sql, dbType); + SQLStatement stmt = stmtList.get(0); + + assertEquals("SELECT to_char(CreateDate || ' ' || CAST(HourArgment AS VARCHAR) || ':00:00'::TIMESTAMP, 'YYYY-MM-DD HH24') AS \"intoTime \"\n" + + "FROM analyzedata.StatTime cs\n" + + "WHERE 1 = 1\n" + + "\tAND cs.CreateDate >= to_date('2017-08-31 00:00:00', 'YYYY-MM-DD HH24:MI:SS')\n" + + "\tAND cs.CreateDate <= to_date('2017-08-31 23:00:00', 'YYYY-MM-DD HH24:MI:SS')\n" + + "GROUP BY to_char(CreateDate || ' ' || CAST(HourArgment AS VARCHAR) || ':00:00'::TIMESTAMP, 'YYYY-MM-DD HH24')", SQLUtils.toPGString(stmt)); + + assertEquals("select b.*, st_astext(p.pos) as pos, st_astext(p.polygon) as polygon\n" + + "from ts_biz b\n" + + "\tjoin (\n" + + "\t\tselect *\n" + + "\t\tfrom ts_polygon\n" + + "\t\twhere type in (?)\n" + + "\t\t\tand st_intersects(st_transform(ST_GeomFromText(?, 4326), 26986), st_transform(polygon, 26986))\n" + + "\t) p\n" + + "\ton b.objid = p.objid\n" + + "\t\tand b.type = p.type\n" + + "\t\tand (b.cp_code = p.cp_code\n" + + "\t\t\tor (b.cp_code is null\n" + + "\t\t\t\tand p.cp_code is null))\n" + + "where b.cp_code = ?\n" + + "\tor b.cp_code is null", SQLUtils.toPGString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + assertEquals(1, stmtList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); + + assertEquals(7, visitor.getColumns().size()); + assertEquals(2, visitor.getTables().size()); + } +}