From 6220b9a65bcae1b892f21d7d7788ccd9ecc8ea64 Mon Sep 17 00:00:00 2001 From: wenshao Date: Tue, 31 Oct 2017 15:38:39 +0800 Subject: [PATCH] improved sql parser. --- .../druid/sql/ast/SQLDataTypeImpl.java | 5 ++ .../oracle/visitor/OracleOutputVisitor.java | 2 +- .../postgresql/visitor/PGOutputVisitor.java | 20 ++++- .../druid/sql/parser/SQLExprParser.java | 14 ++++ .../druid/sql/parser/SQLParserFeature.java | 1 + .../create/OracleCreateTableTest41.java | 2 +- .../create/OracleCreateTableTest42.java | 2 +- .../druid/bvt/sql/oracle/pl/Oracle_pl_4.java | 77 +++++++++++++++++++ .../bvt/sql/postgresql/PGInsertTest10.java | 55 +++++++++++++ .../bvt/sql/postgresql/PGInsertTest11.java | 55 +++++++++++++ .../select/PGSelectTest61_type_string.java | 51 ++++++++++++ 11 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/oracle/pl/Oracle_pl_4.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest10.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest11.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest61_type_string.java diff --git a/src/main/java/com/alibaba/druid/sql/ast/SQLDataTypeImpl.java b/src/main/java/com/alibaba/druid/sql/ast/SQLDataTypeImpl.java index 59e36cc897..8934376e79 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/SQLDataTypeImpl.java +++ b/src/main/java/com/alibaba/druid/sql/ast/SQLDataTypeImpl.java @@ -48,6 +48,11 @@ public class SQLDataTypeImpl extends SQLObjectImpl implements SQLDataType { addArgument(new SQLIntegerExpr(precision)); } + public SQLDataTypeImpl(String name, SQLExpr arg) { + this(name); + addArgument(arg); + } + public SQLDataTypeImpl(String name, int precision, int scale) { this(name); addArgument(new SQLIntegerExpr(precision)); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java index a4889a63d6..cbb901aae4 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java @@ -2922,7 +2922,7 @@ public class OracleOutputVisitor extends SQLASTOutputVisitor implements OracleAS if (count != 0) { print0(", "); } - print0(ucase ? "FOREIGHN KEY" : "foreighn key"); + print0(ucase ? "FOREIGN KEY" : "foreign key"); count++; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java index 0705b8384a..62e7f1d1d0 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java @@ -41,6 +41,7 @@ import com.alibaba.druid.sql.dialect.postgresql.ast.stmt.PGSelectQueryBlock.Fetc import com.alibaba.druid.sql.dialect.postgresql.ast.stmt.PGSelectQueryBlock.ForClause; import com.alibaba.druid.sql.dialect.postgresql.ast.stmt.PGSelectQueryBlock.WindowClause; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; +import com.alibaba.druid.util.FnvHash; import com.alibaba.druid.util.JdbcConstants; import java.util.LinkedHashSet; @@ -453,17 +454,32 @@ public class PGOutputVisitor extends SQLASTOutputVisitor implements PGASTVisitor @Override public boolean visit(PGTypeCastExpr x) { SQLExpr expr = x.getExpr(); + SQLDataType dataType = x.getDataType(); + + if (dataType.nameHashCode64() == FnvHash.Constants.VARBIT) { + dataType.accept(this); + print(' '); + printExpr(expr); + return false; + } + if (expr != null) { if (expr instanceof SQLBinaryOpExpr) { print('('); expr.accept(this); print(')'); + } else if (expr instanceof PGTypeCastExpr && dataType.getArguments().size() == 0) { + dataType.accept(this); + print('('); + visit((PGTypeCastExpr) expr); + print(')'); + return false; } else { expr.accept(this); } } print0("::"); - x.getDataType().accept(this); + dataType.accept(this); return false; } @@ -1775,7 +1791,7 @@ public class PGOutputVisitor extends SQLASTOutputVisitor implements PGASTVisitor if (count != 0) { print0(", "); } - print0(ucase ? "FOREIGHN KEY" : "foreighn key"); + print0(ucase ? "FOREIGN KEY" : "foreign key"); count++; } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index ffa1adf5f5..b63f12c968 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -27,6 +27,7 @@ import com.alibaba.druid.sql.ast.*; import com.alibaba.druid.sql.ast.expr.*; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.oracle.ast.expr.OracleArgumentExpr; +import com.alibaba.druid.sql.dialect.postgresql.ast.expr.PGTypeCastExpr; import com.alibaba.druid.util.FnvHash; import com.alibaba.druid.util.JdbcConstants; @@ -1012,6 +1013,19 @@ public class SQLExprParser extends SQLParser { } else if (hash_lower == FnvHash.Constants.POSITION && JdbcConstants.MYSQL.equals(dbType)) { return parsePosition(); + } else if (hash_lower == FnvHash.Constants.INT4 && JdbcConstants.POSTGRESQL.equals(dbType)) { + PGTypeCastExpr castExpr = new PGTypeCastExpr(); + castExpr.setExpr(this.expr()); + castExpr.setDataType(new SQLDataTypeImpl(methodName)); + accept(Token.RPAREN); + return castExpr; + } else if (hash_lower == FnvHash.Constants.VARBIT && JdbcConstants.POSTGRESQL.equals(dbType)) { + PGTypeCastExpr castExpr = new PGTypeCastExpr(); + SQLExpr len = this.primary(); + castExpr.setDataType(new SQLDataTypeImpl(methodName, len)); + accept(Token.RPAREN); + castExpr.setExpr(this.expr()); + return castExpr; } aggMethodName = getAggreateFunction(hash_lower); } else if (expr instanceof SQLPropertyExpr) { diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java index 52434819ce..b84e22d325 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParserFeature.java @@ -17,6 +17,7 @@ package com.alibaba.druid.sql.parser; public enum SQLParserFeature { KeepInsertValueClauseOriginalString, + KeepSelectListOriginalString, // for improved sql parameterized performance UseInsertColumnsCache, EnableSQLBinaryOpExprGroup, OptimizedForParameterized, diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest41.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest41.java index 6dd1344383..2f8e6b10d2 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest41.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest41.java @@ -99,7 +99,7 @@ public class OracleCreateTableTest41 extends OracleTest { "\tSUPPLEMENTAL LOG GROUP \"GGS_16346\" (\"NUM\") ALWAYS,\n" + "\tSUPPLEMENTAL LOG DATA (PRIMARY KEY) COLUMNS,\n" + "\tSUPPLEMENTAL LOG DATA (UNIQUE INDEX) COLUMNS,\n" + - "\tSUPPLEMENTAL LOG DATA (FOREIGHN KEY) COLUMNS,\n" + + "\tSUPPLEMENTAL LOG DATA (FOREIGN KEY) COLUMNS,\n" + "\tSUPPLEMENTAL LOG DATA (ALL) COLUMNS,\n" + "\tSUPPLEMENTAL LOG GROUP \"ALLINS_POLNO\" (\"NUM\") ALWAYS\n" + ")\n" + diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest42.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest42.java index 439c03d978..776d7f0449 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest42.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/create/OracleCreateTableTest42.java @@ -73,7 +73,7 @@ public class OracleCreateTableTest42 extends OracleTest { "\tSUPPLEMENTAL LOG GROUP \"GGS_15754\" (\"GROUP_NAME\", \"GROUP_KEY\", \"LOG_CMPLT_CSN\", \"LOG_CMPLT_XIDS_SEQ\") ALWAYS,\n" + "\tSUPPLEMENTAL LOG DATA (PRIMARY KEY) COLUMNS,\n" + "\tSUPPLEMENTAL LOG DATA (UNIQUE INDEX) COLUMNS,\n" + - "\tSUPPLEMENTAL LOG DATA (FOREIGHN KEY) COLUMNS\n" + + "\tSUPPLEMENTAL LOG DATA (FOREIGN KEY) COLUMNS\n" + ")\n" + "PCTFREE 10\n" + "PCTUSED 40\n" + diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/pl/Oracle_pl_4.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/pl/Oracle_pl_4.java new file mode 100644 index 0000000000..f9be94f991 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/pl/Oracle_pl_4.java @@ -0,0 +1,77 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.oracle.pl; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; + +import java.util.List; + +public class Oracle_pl_4 extends OracleTest { + + public void test_0() throws Exception { + String sql = "begin buf_b := HEXTORAW('636865642f4c'); \n" + + " dbms_lob.write(loc_b, 2048, 28493, buf_b);\n" + + "END;\n" + + "\n "; // + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ORACLE); + assertEquals(1, statementList.size()); + SQLStatement stmt = statementList.get(0); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ORACLE); + for (SQLStatement statement : statementList) { + statement.accept(visitor); + } + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("relationships : " + visitor.getRelationships()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(0, visitor.getTables().size()); + +// Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("employees"))); +// Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("emp_name"))); + +// Assert.assertEquals(7, visitor.getColumns().size()); +// Assert.assertEquals(3, visitor.getConditions().size()); +// Assert.assertEquals(1, visitor.getRelationships().size()); + + // Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("employees", "salary"))); + + { + String output = SQLUtils.toOracleString(stmt); + assertEquals("BEGIN\n" + + "\tbuf_b := HEXTORAW('636865642f4c');\n" + + "\tdbms_lob.write(loc_b, 2048, 28493, buf_b);\n" + + "END;", // + output); + } + { + String output = SQLUtils.toOracleString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION); + assertEquals("begin\n" + + "\tbuf_b := HEXTORAW('636865642f4c');\n" + + "\tdbms_lob.write(loc_b, 2048, 28493, buf_b);\n" + + "end;", // + output); + } + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest10.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest10.java new file mode 100644 index 0000000000..a7c45841e3 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest10.java @@ -0,0 +1,55 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.postgresql; + +import com.alibaba.druid.sql.PGTest; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; +import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor; +import com.alibaba.druid.stat.TableStat; + +import java.util.List; + +public class PGInsertTest10 extends PGTest { + + public void test_0() throws Exception { + String sql = "INSERT INTO test VALUES (B'101', B'00');"; + + PGSQLStatementParser parser = new PGSQLStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); +// print(statementList); + + assertEquals(1, statementList.size()); + + PGSchemaStatVisitor visitor = new PGSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); + + assertTrue(visitor.getTables().containsKey(new TableStat.Name("test"))); + assertEquals(0, visitor.getColumns().size()); +// +// assertTrue(visitor.getColumns().contains(new TableStat.Column("distributors", "did"))); +// assertTrue(visitor.getColumns().contains(new TableStat.Column("distributors", "dname"))); + + assertEquals("INSERT INTO test\n" + + "VALUES (B'101', B'00');", stmt.toString()); + } + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest11.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest11.java new file mode 100644 index 0000000000..e24316af8d --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGInsertTest11.java @@ -0,0 +1,55 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.postgresql; + +import com.alibaba.druid.sql.PGTest; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; +import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor; +import com.alibaba.druid.stat.TableStat; + +import java.util.List; + +public class PGInsertTest11 extends PGTest { + + public void test_0() throws Exception { + String sql = "INSERT INTO test VALUES (B'10'::bit(3), B'101');"; + + PGSQLStatementParser parser = new PGSQLStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); +// print(statementList); + + assertEquals(1, statementList.size()); + + PGSchemaStatVisitor visitor = new PGSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); + + assertTrue(visitor.getTables().containsKey(new TableStat.Name("test"))); + assertEquals(0, visitor.getColumns().size()); +// +// assertTrue(visitor.getColumns().contains(new TableStat.Column("distributors", "did"))); +// assertTrue(visitor.getColumns().contains(new TableStat.Column("distributors", "dname"))); + + assertEquals("INSERT INTO test\n" + + "VALUES (B'10'::bit(3), B'101');", stmt.toString()); + } + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest61_type_string.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest61_type_string.java new file mode 100644 index 0000000000..a63032f56f --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest61_type_string.java @@ -0,0 +1,51 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.postgresql.select; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class PGSelectTest61_type_string extends TestCase { + private final String dbType = JdbcConstants.POSTGRESQL; + + public void test_0() throws Exception { + String sql = "select int4(varbit(4) '1010');"; + + List stmtList = SQLUtils.parseStatements(sql, dbType); + SQLStatement stmt = stmtList.get(0); + + assertEquals("SELECT int4(varbit(4) '1010');", SQLUtils.toPGString(stmt)); + + assertEquals("select int4(varbit(4) '1010');", SQLUtils.toPGString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + assertEquals(1, stmtList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); + + assertEquals(0, visitor.getColumns().size()); + assertEquals(0, visitor.getTables().size()); + } +}