Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improved sql parser.

Browse Source
This commit is contained in:
wenshao
2017-09-11 01:28:15 +08:00
parent 19501d678d
commit 20c3f9304e
11 changed files with 374 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/dialect/h2/parser/H2Lexer.java
+1
View File
@@ -23,6 +23,7 @@ public class H2Lexer extends Lexer {
map.put("USING", Token.USING);
map.put("ROW", Token.ROW);
map.put("LIMIT", Token.LIMIT);
DEFAULT_H2_KEYWORDS = new Keywords(map);
}
src/main/java/com/alibaba/druid/sql/dialect/h2/parser/H2StatementParser.java
+2
View File
@@ -9,6 +9,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.dialect.h2.ast.H2MergeStatement;
import com.alibaba.druid.sql.parser.*;
import com.alibaba.druid.util.FnvHash;
import com.alibaba.druid.util.JdbcConstants;
public class H2StatementParser extends SQLStatementParser {
public H2StatementParser(String sql) {
@@ -32,6 +33,7 @@ public class H2StatementParser extends SQLStatementParser {
accept(Token.INTO);
SQLReplaceStatement stmt = new SQLReplaceStatement();
stmt.setDbType(JdbcConstants.H2);
SQLName tableName = exprParser.name();
stmt.setTableName(tableName);
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+4
View File
@@ -1960,4 +1960,8 @@ public class Lexer {
return buf.toString();
}
public String getSource() {
return text;
}
}
src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java
+1 -1
View File
@@ -310,7 +310,7 @@ public class SQLSelectParser extends SQLParser {
return withQueryClause;
}
protected void parseWhere(SQLSelectQueryBlock queryBlock) {
public void parseWhere(SQLSelectQueryBlock queryBlock) {
if (lexer.token == Token.WHERE) {
lexer.nextToken();
src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
+1
View File
@@ -22,6 +22,7 @@ import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.ast.statement.SQLCreateTriggerStatement.TriggerType;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleExprParser;
import com.alibaba.druid.util.FnvHash;
import com.alibaba.druid.util.JdbcConstants;
src/test/java/com/alibaba/druid/benckmark/sql/MySqlParameterizedBenchmark.java
+4 -1
View File
@@ -54,9 +54,10 @@ public class MySqlParameterizedBenchmark extends TestCase {
String sql10 = "update seller_spu_site_1976 set gmt_modified = NOW(), spu_id = 0, site_id = null, site_en = null, features = null, outer_id = null, auction_status = -2 where ((sellerid = 2027808696) AND (item_id = 552891941272))";
String sql11 = "select seller_item_sku.item_id,seller_item_sku.gmt_modified,seller_item_sku.gmt_create,seller_item_sku.sku_id,seller_item_sku.seller_id,seller_item_sku.outer_id,seller_item_sku.status,seller_item_sku.sync_version,seller_item_sku.sku_feature from seller_item_sku_2047 seller_item_sku where ((seller_item_sku.item_id = 558182960563) AND (seller_item_sku.seller_id = 673619967) AND (seller_item_sku.sku_id = 3634115286014))";
String sql12 = "/* 0ba9776a15048393743937000d1936/0//7d21f1f2/ */select `auction_relation`.`id`,`auction_relation`.`item_id`,`auction_relation`.`sku_id`,`auction_relation`.`user_id`,`auction_relation`.`target_id`,`auction_relation`.`extra_id`,`auction_relation`.`type`,`auction_relation`.`target_type`,`auction_relation`.`type_attr`,`auction_relation`.`status`,`auction_relation`.`target_user_id`,`auction_relation`.`options`,`auction_relation`.`features`,`auction_relation`.`version`,`auction_relation`.`sub_type`,`auction_relation`.`gmt_create`,`auction_relation`.`gmt_modified` from `auction_relation_1080` `auction_relation` where ((`auction_relation`.`item_id` = 556887226360) AND (`auction_relation`.`target_Type` IN (1,2,3,4)) AND (`auction_relation`.`status` = 0) AND (`auction_relation`.`type` IN (16,17,6,7,8,9,10,11,12,13,14,15)))";
String sql13 = "/* 0b802d4e15048393680475983ea57a/0.1.5.24.1.1113363181//f7972bef/ */SELECT id, dispute_id, buyer_id, seller_id, total_fee, refund_fee, max_apply_goods_fee, apply_goods_fee, apply_carriage_fee, refund_goods_fee, refund_carriage_fee, refund_point, refund_coupon, refund_return_point, refund_cash, real_deduct_refund_point, real_refund_return_point, refund_return_commission, gmt_create, gmt_modified, attributes, attributes_cc FROM dispute_funds_0502 AS dispute_funds WHERE dispute_id = 3079439578090614";
for (int i = 0; i < 5; ++i) {
perf(sql); // 6740 6201 4752 4514 4391 4218 4127 4124
// perf(sql); // 6740 6201 4752 4514 4391 4218 4127 4124
// perf(sql2); // 2948 2928 2869 2780 2502
// perf(sql3); // 15093 10392 10416 10154 10007 9126 8907
// perf(sql4); // 4429 4190 4023 3747
@@ -65,6 +66,7 @@ public class MySqlParameterizedBenchmark extends TestCase {
// perf(sql8); // 601 585 314
// perf(sql9); // 2403 2392 2188 2169
// perf(sql10); // 3163 2866
perf(sql13); //
// perf_parse(sql); // 4643 4377 4345 3801 3627 3228 2961 2959
// perf_parse(sql2); // 1918 1779 1666 1646
@@ -77,6 +79,7 @@ public class MySqlParameterizedBenchmark extends TestCase {
// perf_parse(sql10); // 2291 2025 2024 2010
// perf_parse(sql11); // 3209
// perf_parse(sql12); // 4873 4767
// perf_parse(sql13); //
// perf_lexer(sql4); // 2051 1802
// perf_lexer(sql5); // 1125 1054
src/test/java/com/alibaba/druid/benckmark/sql/SqlHolder.java
+88 -14
View File
@@ -1,12 +1,15 @@
package com.alibaba.druid.benckmark.sql;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlSelectParser;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.druid.sql.parser.LayoutCharacters;
@@ -63,21 +66,21 @@ public class SqlHolder {
return;
}
if (text.equals("select @@session.tx_read_only")) {
SQLSelect select = new SQLSelect();
MySqlSelectQueryBlock queryBlock = new MySqlSelectQueryBlock();
queryBlock.addSelectItem(new SQLPropertyExpr(new SQLVariantRefExpr("@@session"), "tx_read_only"));
select.setQuery(queryBlock);
ast = new SQLSelectStatement(select);
parsed = true;
return;
}
// if (text.equals("select @@session.tx_read_only")) {
// SQLSelect select = new SQLSelect();
// MySqlSelectQueryBlock queryBlock = new MySqlSelectQueryBlock();
// queryBlock.addSelectItem(new SQLPropertyExpr(new SQLVariantRefExpr("@@session"), "tx_read_only"));
// select.setQuery(queryBlock);
//
// ast = new SQLSelectStatement(select);
// parsed = true;
// return;
// }
// ast = SQLUtils.parseStatements(text, dialect).get(0);
SQLParserFeature[] features = {SQLParserFeature.EnableSQLBinaryOpExprGroup, SQLParserFeature.OptimizedForParameterized};
try {
ast = new MySqlStatementParser(text, features).parseStatement();
ast = new SqlHolderParser(text, features).parseStatement();
} catch (ParserException e) {
throw new UnsupportedOperationException(e);
}
@@ -118,9 +121,9 @@ public class SqlHolder {
public String parameterize(Set<String> physicalNames, List<Object> params) {
ensureParsed();
if (text.equals("select @@session.tx_read_only")) {
return text;
}
// if (text.equals("select @@session.tx_read_only")) {
// return text;
// }
return Templates.parameterize(ast, physicalNames, params);
}
@@ -235,4 +238,75 @@ public class SqlHolder {
}
return tableName;
}
public static class SqlHolderParser extends MySqlStatementParser {
public SqlHolderParser(String sql, SQLParserFeature... features) {
super(sql, features);
}
public void parseStatementList(List<SQLStatement> statementList, int max, SQLObject parent) {
// special optimized
if (lexer.token() == Token.SELECT) {
String source = lexer.getSource();
if ("select @@session.tx_read_only".equals(source)) {
SQLSelect select = new SQLSelect();
MySqlSelectQueryBlock queryBlock = new MySqlSelectQueryBlock();
queryBlock.addSelectItem(new SQLPropertyExpr(new SQLVariantRefExpr("@@session"), "tx_read_only"));
select.setQuery(queryBlock);
SQLSelectStatement stmt = new SQLSelectStatement(select);
statementList.add(stmt);
lexer.reset(29, '\u001A', Token.EOF);
return;
}
int pos = lexer.pos();
if (source.startsWith(" id, dispute_id, buyer_id, seller_id, total_fee, refund_fee, max_apply_goods_fee, apply_goods_fee, apply_carriage_fee, refund_goods_fee, refund_carriage_fee, refund_point, refund_coupon, refund_return_point, refund_cash, real_deduct_refund_point, real_refund_return_point, refund_return_commission, gmt_create, gmt_modified, attributes, attributes_cc FROM", pos)) {
MySqlSelectQueryBlock queryBlock = new MySqlSelectQueryBlock();
queryBlock.addSelectItem(new SQLIdentifierExpr("id"));
queryBlock.addSelectItem(new SQLIdentifierExpr("dispute_id"));
queryBlock.addSelectItem(new SQLIdentifierExpr("buyer_id"));
queryBlock.addSelectItem(new SQLIdentifierExpr("seller_id"));
queryBlock.addSelectItem(new SQLIdentifierExpr("total_fee"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_fee"));
queryBlock.addSelectItem(new SQLIdentifierExpr("max_apply_goods_fee"));
queryBlock.addSelectItem(new SQLIdentifierExpr("apply_goods_fee"));
queryBlock.addSelectItem(new SQLIdentifierExpr("apply_carriage_fee"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_goods_fee"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_carriage_fee"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_point"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_coupon"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_return_point"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_cash"));
queryBlock.addSelectItem(new SQLIdentifierExpr("real_deduct_refund_point"));
queryBlock.addSelectItem(new SQLIdentifierExpr("real_refund_return_point"));
queryBlock.addSelectItem(new SQLIdentifierExpr("refund_return_commission"));
queryBlock.addSelectItem(new SQLIdentifierExpr("gmt_create"));
queryBlock.addSelectItem(new SQLIdentifierExpr("gmt_modified"));
queryBlock.addSelectItem(new SQLIdentifierExpr("attributes"));
queryBlock.addSelectItem(new SQLIdentifierExpr("attributes_cc"));
lexer.reset(430, ' ', Token.FROM);
MySqlSelectParser selectParser = new MySqlSelectParser(this.exprParser);
selectParser.parseFrom(queryBlock);
selectParser.parseWhere(queryBlock);
SQLSelect select = new SQLSelect(queryBlock);
SQLSelectStatement stmt = new SQLSelectStatement(select);
stmt.setDbType(JdbcConstants.MYSQL);
statementList.add(stmt);
return;
}
}
super.parseStatementList(statementList, max, parent);
}
}
}
src/test/java/com/alibaba/druid/bvt/sql/h2/H2_Explain_0.java
+67
View File
@@ -0,0 +1,67 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.h2;
import com.alibaba.druid.sql.OracleTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import org.junit.Assert;
import java.util.List;
public class H2_Explain_0 extends OracleTest {
public void test_0() throws Exception {
String sql = //
"EXPLAIN SELECT * FROM TEST WHERE ID=1"; //
// System.out.println(sql);
List<SQLStatement> stmtList = SQLUtils.toStatementList(sql, JdbcConstants.H2);
SQLStatement stmt = stmtList.get(0);
Assert.assertEquals(1, stmtList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.H2);
stmt.accept(visitor);
{
String text = SQLUtils.toSQLString(stmt, JdbcConstants.H2);
assertEquals("EXPLAIN\n" +
"SELECT *\n" +
"FROM TEST\n" +
"WHERE ID = 1", text);
}
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(2, visitor.getColumns().size());
assertEquals(1, visitor.getConditions().size());
assertEquals(0, visitor.getRelationships().size());
assertEquals(0, visitor.getOrderByColumns().size());
Assert.assertTrue(visitor.containsTable("TEST"));
}
}
src/test/java/com/alibaba/druid/bvt/sql/h2/H2_MergeTest_1.java
+65
View File
@@ -0,0 +1,65 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.h2;
import com.alibaba.druid.sql.OracleTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import org.junit.Assert;
import java.util.List;
public class H2_MergeTest_1 extends OracleTest {
public void test_0() throws Exception {
String sql = //
"MERGE INTO CUSTOMER KEY (ID) VALUES (8, 'Loki', 32, 'Hyderabad', 3000);"; //
// System.out.println(sql);
List<SQLStatement> stmtList = SQLUtils.toStatementList(sql, JdbcConstants.H2);
SQLStatement stmt = stmtList.get(0);
Assert.assertEquals(1, stmtList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.H2);
stmt.accept(visitor);
{
String text = SQLUtils.toSQLString(stmt, JdbcConstants.H2);
assertEquals("MERGE INTO CUSTOMER KEY (ID)\n" +
"VALUES (8, 'Loki', 32, 'Hyderabad', 3000);", text);
}
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(1, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertEquals(0, visitor.getRelationships().size());
assertEquals(0, visitor.getOrderByColumns().size());
Assert.assertTrue(visitor.containsTable("CUSTOMER"));
}
}
src/test/java/com/alibaba/druid/bvt/sql/h2/H2_Select_0_limit.java
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.h2;
import com.alibaba.druid.sql.OracleTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import org.junit.Assert;
import java.util.List;
public class H2_Select_0_limit extends OracleTest {
public void test_0() throws Exception {
String sql = //
"SELECT * FROM CUSTOMER LIMIT 1000;"; //
// System.out.println(sql);
List<SQLStatement> stmtList = SQLUtils.toStatementList(sql, JdbcConstants.H2);
SQLStatement stmt = stmtList.get(0);
Assert.assertEquals(1, stmtList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.H2);
stmt.accept(visitor);
{
String text = SQLUtils.toSQLString(stmt, JdbcConstants.H2);
assertEquals("SELECT *\n" +
"FROM CUSTOMER\n" +
"LIMIT 1000;", text);
}
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(1, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertEquals(0, visitor.getRelationships().size());
assertEquals(0, visitor.getOrderByColumns().size());
Assert.assertTrue(visitor.containsTable("CUSTOMER"));
}
}
src/test/java/com/alibaba/druid/bvt/sql/h2/H2_Select_1_nulls_first.java
+75
View File
@@ -0,0 +1,75 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.h2;
import com.alibaba.druid.sql.OracleTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import org.junit.Assert;
import java.util.List;
public class H2_Select_1_nulls_first extends OracleTest {
public void test_0() throws Exception {
String sql = //
"SELECT * FROM (SELECT ID, COUNT(*) FROM TEST\n" +
" GROUP BY ID UNION SELECT NULL, COUNT(*) FROM TEST)\n" +
" ORDER BY 1 NULLS LAST;"; //
// System.out.println(sql);
List<SQLStatement> stmtList = SQLUtils.toStatementList(sql, JdbcConstants.H2);
SQLStatement stmt = stmtList.get(0);
Assert.assertEquals(1, stmtList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.H2);
stmt.accept(visitor);
{
String text = SQLUtils.toSQLString(stmt, JdbcConstants.H2);
assertEquals("SELECT *\n" +
"FROM (\n" +
"\tSELECT ID, COUNT(*)\n" +
"\tFROM TEST\n" +
"\tGROUP BY ID\n" +
"\tUNION\n" +
"\tSELECT NULL, COUNT(*)\n" +
"\tFROM TEST\n" +
")\n" +
"ORDER BY 1 NULLS LAST;", text);
}
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(2, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertEquals(0, visitor.getRelationships().size());
assertEquals(0, visitor.getOrderByColumns().size());
Assert.assertTrue(visitor.containsTable("TEST"));
}
}