From d1b21e20c103dc174dc0b596b02f71c22991a679 Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 19 Oct 2017 11:17:10 +0800 Subject: [PATCH] imporved sql parser. --- .../ast/statement/SQLColumnPrimaryKey.java | 3 + .../oracle/parser/OracleStatementParser.java | 24 ++++- .../sql/visitor/SQLASTOutputVisitor.java | 8 +- ...ameterizedOutputVisitorTest_restore_0.java | 8 +- ...ameterizedOutputVisitorTest_restore_1.java | 44 +++++++++ .../druid/bvt/sql/mysql/param/ParseUtil.java | 13 ++- .../create/OracleCreatePackageTest5.java | 96 +++++++++++++++++++ .../create/OracleCreateTableTest79.java | 69 +++++++++++++ 8 files changed, 253 insertions(+), 12 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_1.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreatePackageTest5.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest79.java diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLColumnPrimaryKey.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLColumnPrimaryKey.java index 1373598c4e..ec030d4afa 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLColumnPrimaryKey.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLColumnPrimaryKey.java @@ -18,6 +18,9 @@ package com.alibaba.druid.sql.ast.statement; import com.alibaba.druid.sql.visitor.SQLASTVisitor; public class SQLColumnPrimaryKey extends SQLConstraintImpl implements SQLColumnConstraint { + public SQLColumnPrimaryKey() { + + } @Override protected void accept0(SQLASTVisitor visitor) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleStatementParser.java index 10e6398380..a5086b3921 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleStatementParser.java @@ -873,8 +873,8 @@ public class OracleStatementParser extends SQLStatementParser { accept(Token.FUNCTION); - SQLName procedureName = this.exprParser.name(); - stmt.setName(procedureName); + SQLName functionName = this.exprParser.name(); + stmt.setName(functionName); if (lexer.token() == Token.LPAREN) { lexer.nextToken(); @@ -963,13 +963,13 @@ public class OracleStatementParser extends SQLStatementParser { stmt.setBlock(block); - if (lexer.identifierEquals(procedureName.getSimpleName())) { + if (lexer.identifierEquals(functionName.getSimpleName())) { lexer.nextToken(); } // return stmt; - if (lexer.identifierEquals(procedureName.getSimpleName())) { + if (lexer.identifierEquals(functionName.getSimpleName())) { lexer.nextToken(); } @@ -2705,6 +2705,22 @@ public class OracleStatementParser extends SQLStatementParser { stmt.getStatements().add(proc); } else if (lexer.token() == Token.END) { break; + + } else if (lexer.token() == Token.BEGIN) { + lexer.nextToken(); + SQLBlockStatement block = new SQLBlockStatement(); + parseStatementList(block.getStatementList(), -1, block); + accept(Token.END); + block.setParent(stmt); + stmt.getStatements().add(block); + + if (lexer.identifierEquals(pkgName.getSimpleName())) { + lexer.nextToken(); + accept(Token.SEMI); + return stmt; + } + + break; } else { throw new ParserException("TODO : " + lexer.info()); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 8d3907b51e..6cfa9460e8 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -666,7 +666,13 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet print0(ucase ? " OR " : " or "); } printExpr(left); - printOperator(operator); + print(' '); + if (operator == SQLBinaryOperator.Is) { + print('='); + } else { + printOperator(operator); + } + print(' '); printParameter(value); } print(')'); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_0.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_0.java index 0cd5041b26..ca89d8ea06 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_0.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_0.java @@ -16,16 +16,16 @@ import java.util.List; */ public class MySqlParameterizedOutputVisitorTest_restore_0 extends TestCase { public void test_for_parameterize() throws Exception { - String sqlTemplate = "SELECT id, name, x, y, city_code FROM `geo_poi_raw_data` `geo_poi_raw_data` WHERE 1 = 1 AND `id` > ? ORDER BY `id`"; + String sqlTemplate = "SELECT id, name, x, y, city_code FROM `gpo_abi_raw_data` `gpo_abi_raw_data` WHERE 1 = 1 AND `id` > ? ORDER BY `id`"; String params = "[[1200000,1250000]]"; params = params.replaceAll("''", "'"); sqlTemplate = SQLUtils.formatMySql(sqlTemplate); - String table = "[\"`geo_poi_raw_data`\"]"; + String table = "[\"`gpo_abi_raw_data`\"]"; String formattedSql = ParseUtil.restore(sqlTemplate, table, params); assertEquals("SELECT id, name, x, y, city_code\n" + - "FROM `geo_poi_raw_data` `geo_poi_raw_data`\n" + + "FROM `gpo_abi_raw_data` `gpo_abi_raw_data`\n" + "WHERE 1 = 1\n" + - "\tAND (`id`>1200000 OR `id`>1250000)\n" + + "\tAND (`id` > 1200000 OR `id` > 1250000)\n" + "ORDER BY `id`", formattedSql); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_1.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_1.java new file mode 100644 index 0000000000..1393bcd056 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_restore_1.java @@ -0,0 +1,44 @@ +package com.alibaba.druid.bvt.sql.mysql.param; + +import com.alibaba.druid.sql.SQLUtils; +import junit.framework.TestCase; + +/** + * Created by wenshao on 16/8/23. + */ +public class MySqlParameterizedOutputVisitorTest_restore_1 extends TestCase { + public void test_for_parameterize() throws Exception { + String sqlTemplate = "SELECT `buyer_resource`.`RESOURCE_ID`, `buyer_resource`.`RESOURCE_PROVIDER`, `buyer_resource`.`BUYER_ID`, `buyer_resource`.`RESOURCE_TYPE`, `buyer_resource`.`SUB_RESOURCE_TYPE`\n" + + "\t, `buyer_resource`.`STATUS`, `buyer_resource`.`START_TIME`, `buyer_resource`.`END_TIME`, `buyer_resource`.`FEATURE`, `buyer_resource`.`GMT_CREATED`\n" + + "\t, `buyer_resource`.`GMT_MODIFIED`, `buyer_resource`.`source`, `buyer_resource`.`seller_id`, `buyer_resource`.`original_Resource_Id`, `buyer_resource`.`business_unit`\n" + + "\t, `buyer_resource`.`resource_code`, `buyer_resource`.`OPTIONS`, `buyer_resource`.`AVAILABLE_COUNT`, `buyer_resource`.`TOTAL_COUNT`, `buyer_resource`.`OUT_INSTANCE_ID`\n" + + "\t, `buyer_resource`.`CONSUME_ID`, `buyer_resource`.`GROUP_ID`, `buyer_resource`.`BUSINESS_ID`, `buyer_resource`.`rule`, `buyer_resource`.`market_place`\n" + + "\t, `buyer_resource`.`VERSION`\n" + + "FROM buyer_resource `buyer_resource`\n" + + "WHERE `buyer_resource`.`BUYER_ID` = ?\n" + + "\tAND `buyer_resource`.`STATUS` = ?\n" + + "\tAND `buyer_resource`.`START_TIME` <= ?\n" + + "\tAND `buyer_resource`.`END_TIME` >= ?\n" + + "\tAND `buyer_resource`.`seller_id` = ?\n" + + "\tAND (`buyer_resource`.`AVAILABLE_COUNT` IS ?)\n" + + "LIMIT ?, ?"; + String params = "[1957025290,1,\"2017-10-16 23:34:28.519\",\"2017-10-16 23:34:28.519\",2933220011L,[0,-1],0,20]"; + params = params.replaceAll("''", "'"); + sqlTemplate = SQLUtils.formatMySql(sqlTemplate); + String formattedSql = ParseUtil.restore(sqlTemplate, null, params); + assertEquals("SELECT `buyer_resource`.`RESOURCE_ID`, `buyer_resource`.`RESOURCE_PROVIDER`, `buyer_resource`.`BUYER_ID`, `buyer_resource`.`RESOURCE_TYPE`, `buyer_resource`.`SUB_RESOURCE_TYPE`\n" + + "\t, `buyer_resource`.`STATUS`, `buyer_resource`.`START_TIME`, `buyer_resource`.`END_TIME`, `buyer_resource`.`FEATURE`, `buyer_resource`.`GMT_CREATED`\n" + + "\t, `buyer_resource`.`GMT_MODIFIED`, `buyer_resource`.`source`, `buyer_resource`.`seller_id`, `buyer_resource`.`original_Resource_Id`, `buyer_resource`.`business_unit`\n" + + "\t, `buyer_resource`.`resource_code`, `buyer_resource`.`OPTIONS`, `buyer_resource`.`AVAILABLE_COUNT`, `buyer_resource`.`TOTAL_COUNT`, `buyer_resource`.`OUT_INSTANCE_ID`\n" + + "\t, `buyer_resource`.`CONSUME_ID`, `buyer_resource`.`GROUP_ID`, `buyer_resource`.`BUSINESS_ID`, `buyer_resource`.`rule`, `buyer_resource`.`market_place`\n" + + "\t, `buyer_resource`.`VERSION`\n" + + "FROM buyer_resource `buyer_resource`\n" + + "WHERE `buyer_resource`.`BUYER_ID` = 1957025290\n" + + "\tAND `buyer_resource`.`STATUS` = 1\n" + + "\tAND `buyer_resource`.`START_TIME` <= '2017-10-16 23:34:28.519'\n" + + "\tAND `buyer_resource`.`END_TIME` >= '2017-10-16 23:34:28.519'\n" + + "\tAND `buyer_resource`.`seller_id` = 2933220011\n" + + "\tAND (`buyer_resource`.`AVAILABLE_COUNT` = 0 OR `buyer_resource`.`AVAILABLE_COUNT` = -1)\n" + + "LIMIT 0, 20", formattedSql); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/ParseUtil.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/ParseUtil.java index 7215d0781f..2e5d8edbf6 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/ParseUtil.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/param/ParseUtil.java @@ -120,7 +120,12 @@ public class ParseUtil { } public static String restore(String sql, String table, String params/*JSONArray paramsArray, JSONArray destArray*/) { - JSONArray destArray = JSON.parseArray(table.replaceAll("''", "'")); + JSONArray destArray = null; + + if (table != null) { + destArray = JSON.parseArray(table.replaceAll("''", "'")); + } + JSONArray paramsArray = JSON.parseArray(params.replaceAll("''", "'")); String dbType = JdbcConstants.MYSQL; List stmtList = SQLUtils.parseStatements(sql, dbType); @@ -141,8 +146,10 @@ public class ParseUtil { srcArray.add(entry.getKey().getName()); }*/ - for (int i = 0; i < srcArray.size(); i++) { - visitor.addTableMapping(srcArray.getString(i), destArray.getString(i)); + if (destArray != null) { + for (int i = 0; i < srcArray.size(); i++) { + visitor.addTableMapping(srcArray.getString(i), destArray.getString(i)); + } } stmt.accept(visitor); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreatePackageTest5.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreatePackageTest5.java new file mode 100644 index 0000000000..ef2e763f40 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreatePackageTest5.java @@ -0,0 +1,96 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR XONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.oracle.create; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; + +import java.util.List; + +public class OracleCreatePackageTest5 extends OracleTest { + + public void test_types() throws Exception { + String sql = // + "CREATE OR REPLACE package body cms_con_attributes_pkg is\n" + + " function get_hc_org_id(in_hc_con_id number) return number is\n" + + " hc_org_id number default null;\n" + + " sqlStr varchar2(500);\n" + + " begin\n" + + " sqlStr :='select h.org_id as hc_org_id\n" + + " from ecc_cpr.ecc_cpr_hc_con_heahers h\n" + + " where h.hc_con_id =' || to_char(in_hc_con_id);\n" + + " execute immediate sqlStr into hc_org_id;\n" + + " return hc_org_id;\n" + + " end get_hc_org_id;\n" + + "begin\n" + + " return;\n" + + "end cms_con_attributes_pkg;"; + + System.out.println(sql); + + OracleStatementParser parser = new OracleStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + print(statementList); + + assertEquals(1, statementList.size()); + + assertEquals("CREATE OR REPLACE PACKAGE BODY cms_con_attributes_pkg\n" + + "BEGIN\n" + + "\tFUNCTION get_hc_org_id (\n" + + "\t\tin_hc_con_id number\n" + + "\t)\n" + + "\tRETURN number\n" + + "\tIS\n" + + "\thc_org_id number := NULL;\n" + + "\t\tsqlStr varchar2(500);\n" + + "\tBEGIN\n" + + "\t\tsqlStr := 'select h.org_id as hc_org_id\n" + + " from ecc_cpr.ecc_cpr_hc_con_heahers h\n" + + " where h.hc_con_id =' || to_char(in_hc_con_id);\n" + + "\t\tEXECUTE IMMEDIATE sqlStr INTO hc_org_id;\n" + + "\t\tRETURN hc_org_id;\n" + + "\tEND;\n" + + "\tBEGIN\n" + + "\t\tRETURN;\n" + + "\tEND;\n" + + "END cms_con_attributes_pkg;",// + SQLUtils.toSQLString(stmt, JdbcConstants.ORACLE)); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + +// Assert.assertEquals(2, visitor.getTables().size()); +// +// Assert.assertEquals(5, visitor.getColumns().size()); +// +// Assert.assertTrue(visitor.containsColumn("employees", "employee_id")); +// Assert.assertTrue(visitor.containsColumn("employees", "*")); +// Assert.assertTrue(visitor.containsColumn("departments", "department_id")); +// Assert.assertTrue(visitor.containsColumn("employees", "salary")); +// Assert.assertTrue(visitor.containsColumn("employees", "commission_pct")); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest79.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest79.java new file mode 100644 index 0000000000..2a97f98df6 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest79.java @@ -0,0 +1,69 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.oracle.create; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.util.JdbcConstants; + +import java.util.List; + +public class OracleCreateTableTest79 extends OracleTest { + + public void test_types() throws Exception { + String sql = // + "CREATE TABLE rules (id NUMBER(38) NOT NULL PRIMARY KEY, name varchar(200), plugin_rule_key varchar(200) NOT NULL, plugin_config_key varchar(200), plugin_name varchar(255) NOT NULL, description source, priority NUMBER(38), cardinality varchar(10), parent_id NUMBER(38), status varchar(40), language varchar(20), created_at TIMESTAMP, updated_at TIMESTAMP) "; + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ORACLE); + SQLStatement stmt = statementList.get(0); + print(statementList); + + assertEquals(1, statementList.size()); +// + assertEquals("CREATE TABLE rules (\n" + + "\tid NUMBER(38) NOT NULL PRIMARY KEY,\n" + + "\tname varchar(200),\n" + + "\tplugin_rule_key varchar(200) NOT NULL,\n" + + "\tplugin_config_key varchar(200),\n" + + "\tplugin_name varchar(255) NOT NULL,\n" + + "\tdescription source,\n" + + "\tpriority NUMBER(38),\n" + + "\tcardinality varchar(10),\n" + + "\tparent_id NUMBER(38),\n" + + "\tstatus varchar(40),\n" + + "\tlanguage varchar(20),\n" + + "\tcreated_at TIMESTAMP,\n" + + "\tupdated_at TIMESTAMP\n" + + ")",// + SQLUtils.toSQLString(stmt, JdbcConstants.ORACLE)); +// +// SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ORACLE); +// stmt.accept(visitor); +// +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("relationships : " + visitor.getRelationships()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); +// +// assertEquals(1, visitor.getTables().size()); +// +// assertEquals(3, visitor.getColumns().size()); +// +// assertTrue(visitor.getColumns().contains(new TableStat.Column("JWGZPT.A", "XM"))); + } +}