From 1c900720d832e93e73fdda4a36b200ed76f09f85 Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 10 Aug 2017 14:29:21 +0800 Subject: [PATCH] improved sql parser. --- .../mysql/parser/MySqlCreateTableParser.java | 4 + .../druid/sql/parser/SQLSelectParser.java | 9 ++ .../sql/repository/SchemaRepository.java | 25 ++++- .../sql/visitor/SQLASTOutputVisitor.java | 12 ++- ...eateTable_showColumns_repository_test.java | 14 +++ .../sql/oracle/OracleFlashbackQueryTest3.java | 2 +- .../sql/oracle/select/OracleSelectTest71.java | 98 +++++++++++++++++++ .../sql/oracle/select/OracleSelectTest72.java | 64 ++++++++++++ ...Oracle_getCreateTableScriptTest_sonar.java | 36 +++++++ 9 files changed, 258 insertions(+), 6 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest71.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest72.java create mode 100644 src/test/java/com/alibaba/druid/oracle/Oracle_getCreateTableScriptTest_sonar.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java index f1e15ab832..1cf137db1f 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java @@ -186,6 +186,10 @@ public class MySqlCreateTableParser extends SQLCreateTableParser { } for (;;) { + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + } + if (identifierEquals("ENGINE")) { lexer.nextToken(); if (lexer.token() == Token.EQ) { diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java index 11f5e7565a..253fbfc901 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java @@ -279,6 +279,12 @@ public class SQLSelectParser extends SQLParser { accept(Token.BY); SQLSelectGroupByClause groupBy = new SQLSelectGroupByClause(); + if (identifierEquals("ROLLUP")) { + lexer.nextToken(); + accept(Token.LPAREN); + groupBy.setWithRollUp(true); + } + for (;;) { SQLExpr item = parseGroupByItem(); @@ -291,6 +297,9 @@ public class SQLSelectParser extends SQLParser { lexer.nextToken(); } + if (groupBy.isWithRollUp()) { + accept(Token.RPAREN); + } if (lexer.token() == (Token.HAVING)) { lexer.nextToken(); diff --git a/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java b/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java index b402e6ca2b..b73a7558f0 100644 --- a/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java +++ b/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java @@ -275,10 +275,27 @@ public class SchemaRepository { for (SQLStatement stmt : stmtList) { if (stmt instanceof MySqlShowColumnsStatement) { MySqlShowColumnsStatement showColumns = ((MySqlShowColumnsStatement) stmt); - SQLName table = showColumns.getTable(); - SchemaObject schemaObject = findTable(table); - MySqlCreateTableStatement createTableStmt = (MySqlCreateTableStatement) schemaObject.getStatement(); - createTableStmt.showCoumns(buf); + SQLName db = showColumns.getDatabase(); + Schema schema; + if (db == null) { + schema = getDefaultSchema(); + } else { + schema = findSchema(db.getSimpleName()); + } + + SQLName table = null; + SchemaObject schemaObject = null; + if (schema != null) { + table = showColumns.getTable(); + schemaObject = schema.findTable(table.getSimpleName()); + } + + if (schemaObject == null) { + buf.append("ERROR 1146 (42S02): Table '" + table + "' doesn't exist\n"); + } else { + MySqlCreateTableStatement createTableStmt = (MySqlCreateTableStatement) schemaObject.getStatement(); + createTableStmt.showCoumns(buf); + } } else if (stmt instanceof MySqlShowCreateTableStatement) { MySqlShowCreateTableStatement showCreateTableStmt = (MySqlShowCreateTableStatement) stmt; SQLName table = showCreateTableStmt.getName(); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index dee4f6713f..44cf60d714 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -1307,9 +1307,16 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } public boolean visit(SQLSelectGroupByClause x) { + + boolean oracle = JdbcConstants.ORACLE.equals(dbType); + boolean rollup = x.isWithRollUp(); + int itemSize = x.getItems().size(); if (itemSize > 0) { print0(ucase ? "GROUP BY " : "group by "); + if (oracle && rollup) { + print0(ucase ? "ROLLUP (" : "rollup ("); + } incrementIndent(); for (int i = 0; i < itemSize; ++i) { if (i != 0) { @@ -1321,6 +1328,9 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } x.getItems().get(i).accept(this); } + if (oracle && rollup) { + print(')'); + } decrementIndent(); } @@ -1330,7 +1340,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet x.getHaving().accept(this); } - if (x.isWithRollUp()) { + if (x.isWithRollUp() && !oracle) { print0(ucase ? " WITH ROLLUP" : " with rollup"); } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlCreateTable_showColumns_repository_test.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlCreateTable_showColumns_repository_test.java index f615cd0c92..a84bc01537 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlCreateTable_showColumns_repository_test.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlCreateTable_showColumns_repository_test.java @@ -916,6 +916,10 @@ public class MySqlCreateTable_showColumns_repository_test extends MysqlTest { , repository.console("show create table test1")); + assertEquals("ERROR 1146 (42S02): Table 'test1' doesn't exist\n" + + , repository.console("show columns from test1")); + assertEquals("CREATE TABLE test1_xx (\n" + "\tc_decimal_1 INT(11) NOT NULL DEFAULT NULL,\n" + "\t`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id',\n" + @@ -968,5 +972,15 @@ public class MySqlCreateTable_showColumns_repository_test extends MysqlTest { ")" , repository.console("show create table test5")); + + + repository.console("CREATE TABLE tddl5_00.ljh_test1(be_id VARCHAR(20) NOT NULL DEFAULT '' COMMENT '企业标识')ENGINE = InnoDB, CHARACTER SET = utf8, COMMENT = '应用层-汇总销售及销售毛利表'"); + + assertEquals("+-------+-------------+------+-----+---------+-------+\n" + + "| Field | Type | Null | Key | Default | Extra |\n" + + "+-------+-------------+------+-----+---------+-------+\n" + + "| be_id | VARCHAR(20) | NO | | '' | |\n" + + "+-------+-------------+------+-----+---------+-------+\n" + , repository.console("show columns from tddl5_00.ljh_test1")); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleFlashbackQueryTest3.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleFlashbackQueryTest3.java index d8fc831a78..38633d521b 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleFlashbackQueryTest3.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleFlashbackQueryTest3.java @@ -36,7 +36,7 @@ public class OracleFlashbackQueryTest3 extends TestCase { "\t, AVG(salary) * 12 AS \"Average Sal\"\n" + "FROM employees e, departments d\n" + "WHERE d.department_id = e.department_id\n" + - "GROUP BY ROLLUP(department_name, job_id);"; + "GROUP BY ROLLUP (department_name, job_id);"; OracleStatementParser parser = new OracleStatementParser(sql); SQLSelectStatement stmt = (SQLSelectStatement) parser.parseStatementList().get(0); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest71.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest71.java new file mode 100644 index 0000000000..27638869bb --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest71.java @@ -0,0 +1,98 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.oracle.select; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import org.junit.Assert; + +import java.util.List; + +public class OracleSelectTest71 extends OracleTest { + + public void test_0() throws Exception { + String sql = // + "select * from (\n" + + " select a.*,rownum rn from (SELECT\n" + + " A.ID,A.SIM_NO,A.ICCID,A.IMSI,A.IMEI,A.OPERATOR,\n" + + " A.CHARGES_PLAN_ID,A.USED_TRAFFIC,\n" + + " A.STATUS,A.MEMO,A.TYPE,A.TER_ID,B.ID DEVICE_ID,C.TER_CODE,\n" + + " D.ID VECL_ID,D.VIN VECL_VIN,D.LPN VECL_LPN,D.LPC VECL_LPC,E.APP_KEY\n" + + " FROM\n" + + " BAS_SIM A LEFT JOIN BAS_DEVICE B ON A.TER_ID=B.TER_ID\n" + + " LEFT JOIN BAS_TERMINAL C ON A.TER_ID=C.ID\n" + + " LEFT JOIN BAS_VEHICLE D ON B.VECL_ID=D.ID\n" + + " LEFT JOIN BAS_APPLICATION E ON A.APP_ID=E.ID\n" + + " WHERE 1=1\n" + + " \n" + + " \n" + + " AND A.SIM_NO LIKE '%'||?||'%' \n" + + " order by A.SIM_NO ASC ) a\n" + + "\n" + + " ) where rn > ?*(?-1) and rn <= ?*?"; // + + OracleStatementParser parser = new OracleStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + print(statementList); + + Assert.assertEquals(1, statementList.size()); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(5, visitor.getTables().size()); + + assertEquals(26, visitor.getColumns().size()); + + { + String text = SQLUtils.toOracleString(stmt); + + assertEquals("SELECT *\n" + + "FROM (\n" + + "\tSELECT a.*, rownum AS rn\n" + + "\tFROM (\n" + + "\t\tSELECT A.ID, A.SIM_NO, A.ICCID, A.IMSI, A.IMEI\n" + + "\t\t\t, A.OPERATOR, A.CHARGES_PLAN_ID, A.USED_TRAFFIC, A.STATUS, A.MEMO\n" + + "\t\t\t, A.TYPE, A.TER_ID, B.ID AS DEVICE_ID, C.TER_CODE, D.ID AS VECL_ID\n" + + "\t\t\t, D.VIN AS VECL_VIN, D.LPN AS VECL_LPN, D.LPC AS VECL_LPC, E.APP_KEY\n" + + "\t\tFROM BAS_SIM A\n" + + "\t\tLEFT JOIN BAS_DEVICE B ON A.TER_ID = B.TER_ID \n" + + "\t\tLEFT JOIN BAS_TERMINAL C ON A.TER_ID = C.ID \n" + + "\t\tLEFT JOIN BAS_VEHICLE D ON B.VECL_ID = D.ID \n" + + "\t\t\tLEFT JOIN BAS_APPLICATION E ON A.APP_ID = E.ID \n" + + "\t\tWHERE 1 = 1\n" + + "\t\t\tAND A.SIM_NO LIKE ('%' || ? || '%')\n" + + "\t\tORDER BY A.SIM_NO ASC\n" + + "\t) a\n" + + ")\n" + + "WHERE rn > ? * (? - 1)\n" + + "\tAND rn <= ? * ?", text); + } + // Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("acduser.vw_acd_info", "xzqh"))); + + // Assert.assertTrue(visitor.getOrderByColumns().contains(new TableStat.Column("employees", "last_name"))); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest72.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest72.java new file mode 100644 index 0000000000..e48ee60cd3 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest72.java @@ -0,0 +1,64 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.oracle.select; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import org.junit.Assert; + +import java.util.List; + +public class OracleSelectTest72 extends OracleTest { + + public void test_0() throws Exception { + String sql = // + "Select A,B,C,sum(E) from test group by rollup(A,B,C)"; // + + OracleStatementParser parser = new OracleStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + print(statementList); + + Assert.assertEquals(1, statementList.size()); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + + assertEquals(4, visitor.getColumns().size()); + + { + String text = SQLUtils.toOracleString(stmt); + + assertEquals("SELECT A, B, C, SUM(E)\n" + + "FROM test\n" + + "GROUP BY ROLLUP (A, B, C)", text); + } + // Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("acduser.vw_acd_info", "xzqh"))); + + // Assert.assertTrue(visitor.getOrderByColumns().contains(new TableStat.Column("employees", "last_name"))); + } +} diff --git a/src/test/java/com/alibaba/druid/oracle/Oracle_getCreateTableScriptTest_sonar.java b/src/test/java/com/alibaba/druid/oracle/Oracle_getCreateTableScriptTest_sonar.java new file mode 100644 index 0000000000..82b0256e49 --- /dev/null +++ b/src/test/java/com/alibaba/druid/oracle/Oracle_getCreateTableScriptTest_sonar.java @@ -0,0 +1,36 @@ +package com.alibaba.druid.oracle; + +import com.alibaba.druid.DbTestCase; +import com.alibaba.druid.util.JdbcConstants; +import com.alibaba.druid.util.JdbcUtils; + +import java.sql.Connection; +import java.util.Collections; +import java.util.List; + +/** + * Created by wenshao on 23/07/2017. + */ +public class Oracle_getCreateTableScriptTest_sonar extends DbTestCase { + public Oracle_getCreateTableScriptTest_sonar() { + super("pool_config/oracle_db_sonar.properties"); + } + + public void test_oracle() throws Exception { + Connection conn = getConnection(); + + // 从Oracle通过DBMS_METADATA.GET_DDL获取CreateTable语句列表 + //String createTableScript = JdbcUtils.getCreateTableScript(conn, JdbcConstants.ORACLE); + //System.out.println(createTableScript); + + List tables = JdbcUtils.showTables(conn, JdbcConstants.ORACLE); + for (String table : tables) { + Object cnt = JdbcUtils.executeQuery(conn, "select count(*) CNT from " + table, Collections.emptyList()) + .get(0) + .get("CNT"); + System.out.println(table + " : " + cnt); + } + + conn.close(); + } +}