From 84541241f7f4153d2aa00bc55a800bc95eaae7a7 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 27 May 2018 16:38:55 +0800 Subject: [PATCH] bug fixed for sql parser. for issue #2488 --- .../mysql/parser/MySqlStatementParser.java | 8 ++ .../druid/sql/parser/SQLExprParser.java | 4 +- .../sql/mysql/insert/MySqlInsertTest_36.java | 74 ++++++++++ .../oracle/select/OracleSelectTest128.java | 128 ++++++++++++++++++ .../sql/postgresql/select/PGSelectTest72.java | 54 ++++++++ 5 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_36.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest128.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest72.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java index 42df97a23e..0b6b0663fb 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java @@ -2421,6 +2421,10 @@ public class MySqlStatementParser extends SQLStatementParser { lexer.nextToken(); } + if (lexer.token() == Token.LINE_COMMENT) { + lexer.nextToken(); + } + tableName = this.exprParser.name(); stmt.setTableName(tableName); @@ -2540,6 +2544,10 @@ public class MySqlStatementParser extends SQLStatementParser { } } + if (lexer.token() == Token.LINE_COMMENT) { + lexer.nextToken(); + } + if (lexer.token() == Token.VALUES || lexer.identifierEquals(FnvHash.Constants.VALUE)) { lexer.nextTokenLParen(); parseValueClause(stmt.getValuesList(), columnSize, stmt); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index 725182c519..942816d35b 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -1492,7 +1492,7 @@ public class SQLExprParser extends SQLParser { } SQLOver.WindowingType windowingType = null; - if (lexer.identifierEquals(FnvHash.Constants.ROWS)) { + if (lexer.identifierEquals(FnvHash.Constants.ROWS) || lexer.token == Token.ROWS) { windowingType = SQLOver.WindowingType.ROWS; } else if (lexer.identifierEquals(FnvHash.Constants.RANGE)) { @@ -1519,7 +1519,7 @@ public class SQLExprParser extends SQLParser { accept(Token.AND); SQLExpr betweenEnd; - if (lexer.identifierEquals(FnvHash.Constants.CURRENT)) { + if (lexer.identifierEquals(FnvHash.Constants.CURRENT) || lexer.token == Token.CURRENT) { lexer.nextToken(); if (lexer.identifierEquals(FnvHash.Constants.ROW)) { lexer.nextToken(); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_36.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_36.java new file mode 100644 index 0000000000..c4acea1925 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_36.java @@ -0,0 +1,74 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.mysql.insert; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement; +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor; +import com.alibaba.druid.sql.parser.SQLParserFeature; +import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; +import com.alibaba.druid.sql.visitor.VisitorFeature; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class MySqlInsertTest_36 extends TestCase { + + public void test_insert_0() throws Exception { + String sql = "insert into -- @@@\n" + + " tablex(id, value) -- @@@\n" + + " values (?, ?)"; + + MySqlStatementParser parser = new MySqlStatementParser(sql, false, true); + parser.config(SQLParserFeature.KeepInsertValueClauseOriginalString, true); + + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + + MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt; + + assertEquals(1, insertStmt.getValuesList().size()); + assertEquals(2, insertStmt.getValues().getValues().size()); + assertEquals(2, insertStmt.getColumns().size()); + assertEquals(1, statementList.size()); + + MySqlSchemaStatVisitor v = new MySqlSchemaStatVisitor(); + stmt.accept(v); + + String formatSql = "INSERT INTO tablex (id, value)\n" + + "VALUES (?, ?)"; + assertEquals(formatSql, SQLUtils.toMySqlString(insertStmt)); + + String psql = ParameterizedOutputVisitorUtils.parameterize(sql, JdbcConstants.MYSQL); + assertEquals("INSERT INTO tablex(id, value)\n" + + "VALUES (?, ?)", psql); + + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, v.getTables().size()); +// assertEquals(1, visitor.getColumns().size()); +// assertEquals(0, visitor.getConditions().size()); +// assertEquals(0, visitor.getOrderByColumns().size()); + + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest128.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest128.java new file mode 100644 index 0000000000..e427d5bef6 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest128.java @@ -0,0 +1,128 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.oracle.select; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + + +public class OracleSelectTest128 extends TestCase { + public void test_0() throws Exception { + String sql = "SELECT id,\n" + + " MAX(myRank) area\n" + + "FROM\n" + + " (SELECT t2.id ,\n" + + " wmsys.wm_concat(t2.name) over (partition BY t2.id order by to_number(t2.cc) ASC ) myRank\n" + + " FROM\n" + + " (SELECT adr.name,\n" + + " t1.*\n" + + " FROM\n" + + " (SELECT t.id,\n" + + " t.addresscode,\n" + + " regexp_substr(t.addresscode, '[^_]+', 1, x.n) cc\n" + + " FROM\n" + + " ( SELECT id,addresscode FROM srm1.CONSIGNEE_ADDRESS ca\n" + + " ) t,\n" + + " (SELECT ROWNUM n FROM dual CONNECT BY ROWNUM <= 5\n" + + " ) x\n" + + " ORDER BY 1\n" + + " ) t1\n" + + " LEFT JOIN srm1.address adr\n" + + " ON adr.id = t1.cc\n" + + " WHERE t1.cc IS NOT NULL\n" + + " ) t2\n" + + " )\n" + + "GROUP BY id"; + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ORACLE); + + assertEquals(1, statementList.size()); + + SQLSelectStatement stmt = (SQLSelectStatement) statementList.get(0); + + assertEquals("SELECT id, MAX(myRank) AS area\n" + + "FROM (\n" + + "\tSELECT t2.id, wmsys.wm_concat(t2.name) OVER (PARTITION BY t2.id ORDER BY to_number(t2.cc) ASC) AS myRank\n" + + "\tFROM (\n" + + "\t\tSELECT adr.name, t1.*\n" + + "\t\tFROM (\n" + + "\t\t\tSELECT t.id, t.addresscode\n" + + "\t\t\t\t, regexp_substr(t.addresscode, '[^_]+', 1, x.n) AS cc\n" + + "\t\t\tFROM (\n" + + "\t\t\t\tSELECT id, addresscode\n" + + "\t\t\t\tFROM srm1.CONSIGNEE_ADDRESS ca\n" + + "\t\t\t) t, (\n" + + "\t\t\t\tSELECT ROWNUM AS n\n" + + "\t\t\t\tFROM dual\n" + + "\t\t\t\tCONNECT BY ROWNUM <= 5\n" + + "\t\t\t) x\n" + + "\t\t\tORDER BY 1\n" + + "\t\t) t1\n" + + "\t\t\tLEFT JOIN srm1.address adr ON adr.id = t1.cc \n" + + "\t\tWHERE t1.cc IS NOT NULL\n" + + "\t) t2\n" + + ")\n" + + "GROUP BY id", stmt.toString()); + + assertEquals("select id, max(myRank) as area\n" + + "from (\n" + + "\tselect t2.id, wmsys.wm_concat(t2.name) over (partition by t2.id order by to_number(t2.cc) asc) as myRank\n" + + "\tfrom (\n" + + "\t\tselect adr.name, t1.*\n" + + "\t\tfrom (\n" + + "\t\t\tselect t.id, t.addresscode\n" + + "\t\t\t\t, regexp_substr(t.addresscode, '[^_]+', 1, x.n) as cc\n" + + "\t\t\tfrom (\n" + + "\t\t\t\tselect id, addresscode\n" + + "\t\t\t\tfrom srm1.CONSIGNEE_ADDRESS ca\n" + + "\t\t\t) t, (\n" + + "\t\t\t\tselect ROWNUM as n\n" + + "\t\t\t\tfrom dual\n" + + "\t\t\t\tconnect by ROWNUM <= 5\n" + + "\t\t\t) x\n" + + "\t\t\torder by 1\n" + + "\t\t) t1\n" + + "\t\t\tleft join srm1.address adr on adr.id = t1.cc \n" + + "\t\twhere t1.cc is not null\n" + + "\t) t2\n" + + ")\n" + + "group by id", stmt.toLowerCaseString()); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(2, visitor.getTables().size()); + assertEquals(4, visitor.getColumns().size()); + assertEquals(2, visitor.getConditions().size()); + assertEquals(0, visitor.getRelationships().size()); + assertEquals(1, visitor.getOrderByColumns().size()); + + assertTrue(visitor.containsColumn("srm1.CONSIGNEE_ADDRESS", "id")); + } + +} \ No newline at end of file diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest72.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest72.java new file mode 100644 index 0000000000..0edb9d4bb1 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest72.java @@ -0,0 +1,54 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.postgresql.select; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class PGSelectTest72 extends TestCase { + private final String dbType = JdbcConstants.POSTGRESQL; + + public void test_0() throws Exception { + String sql = "select a.*, stddev_samp(a.idx_yield_day) over(order by a.action_date rows between 60 preceding and current row) as idx_volatility from idx_a a"; + System.out.println(sql); + + List stmtList = SQLUtils.parseStatements(sql, dbType); + SQLStatement stmt = stmtList.get(0); + + assertEquals("SELECT a.*, stddev_samp(a.idx_yield_day) OVER (ORDER BY a.action_date ROWS BETWEEN 60 PRECEDING AND CURRENT ROW) AS idx_volatility\n" + + "FROM idx_a a", SQLUtils.toPGString(stmt)); + + assertEquals("select a.*, stddev_samp(a.idx_yield_day) over (order by a.action_date rows between 60 preceding and current row) as idx_volatility\n" + + "from idx_a a", SQLUtils.toPGString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + assertEquals(1, stmtList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); + + assertEquals(3, visitor.getColumns().size()); + assertEquals(1, visitor.getTables().size()); + } +}