From 103ee3e1a3c446590d0bd61e8e29f1d0d643b7e4 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 21 Aug 2016 13:58:38 +0800 Subject: [PATCH] improved odps sql parser. bug fixed for issue #1375 --- .../sql/dialect/odps/parser/OdpsLexer.java | 4 + .../odps/visitor/OdpsOutputVisitor.java | 29 ++++- .../com/alibaba/druid/sql/parser/Lexer.java | 117 +++++++++++++++++- .../sql/visitor/SQLASTOutputVisitor.java | 7 +- .../druid/bvt/sql/odps/OdpsSelectTest14.java | 58 +++++++++ .../druid/bvt/sql/odps/OdpsSelectTest15.java | 58 +++++++++ .../druid/bvt/sql/odps/OdpsSelectTest16.java | 58 +++++++++ .../bvt/sql/oracle/OracleSelectTest56.java | 78 +++++++++--- 8 files changed, 381 insertions(+), 28 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest14.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest15.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest16.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java index bf472fb457..936a975b66 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java @@ -352,4 +352,8 @@ public class OdpsLexer extends Lexer { protected final void scanString() { scanString2(); } + + protected final void scanAlias() { + scanAlias2(); + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index f46b3a636f..30b78d854e 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -881,10 +881,31 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis if (text == null) { print0(ucase ? "NULL" : "null"); } else { - print('\''); - String text2 = text.replaceAll("\\'", "\\\\'"); - print0(text2); - print('\''); + StringBuilder buf = new StringBuilder(text.length() + 2); + buf.append('\''); + for (int i = 0; i < text.length(); ++i) { + char ch = text.charAt(i); + switch (ch) { + case '\\': + buf.append("\\\\"); + break; + case '\'': + buf.append("\\'"); + break; + case '\0': + buf.append("\\0"); + break; + case '\n': + buf.append("\\n"); + break; + default: + buf.append(ch); + break; + } + } + buf.append('\''); + + print0(buf.toString()); } return false; diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index e166d2c99f..cb2273670f 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -809,7 +809,7 @@ public class Lexer { } } - private final void scanAlias() { + protected void scanAlias() { mark = pos; if (buf == null) { @@ -856,6 +856,121 @@ public class Lexer { //stringVal = subString(mark + 1, bufPos); } + protected final void scanAlias2() { + { + boolean hasSpecial = false; + int startIndex = pos + 1; + int endIndex = -1; // text.indexOf('\'', startIndex); + for (int i = startIndex; i < text.length(); ++i) { + final char ch = text.charAt(i); + if (ch == '\\') { + hasSpecial = true; + continue; + } + if (ch == '"') { + endIndex = i; + break; + } + } + + if (endIndex == -1) { + throw new ParserException("unclosed str"); + } + + String stringVal = subString(startIndex, endIndex - startIndex); + // hasSpecial = stringVal.indexOf('\\') != -1; + + if (!hasSpecial) { + this.stringVal = stringVal; + int pos = endIndex + 1; + char ch = charAt(pos); + if (ch != '\'') { + this.pos = pos; + this.ch = ch; + token = LITERAL_CHARS; + return; + } + } + } + + mark = pos; + boolean hasSpecial = false; + for (;;) { + if (isEOF()) { + lexError("unclosed.str.lit"); + return; + } + + ch = charAt(++pos); + + if (ch == '\\') { + scanChar(); + if (!hasSpecial) { + initBuff(bufPos); + arraycopy(mark + 1, buf, 0, bufPos); + hasSpecial = true; + } + + switch (ch) { + case '0': + putChar('\0'); + break; + case '\'': + putChar('\''); + break; + case '"': + putChar('"'); + break; + case 'b': + putChar('\b'); + break; + case 'n': + putChar('\n'); + break; + case 'r': + putChar('\r'); + break; + case 't': + putChar('\t'); + break; + case '\\': + putChar('\\'); + break; + case 'Z': + putChar((char) 0x1A); // ctrl + Z + break; + default: + putChar(ch); + break; + } + + continue; + } + if (ch == '\"') { + scanChar(); + token = LITERAL_CHARS; + break; + } + + if (!hasSpecial) { + bufPos++; + continue; + } + + if (bufPos == buf.length) { + putChar(ch); + } else { + buf[bufPos++] = ch; + } + } + + if (!hasSpecial) { + stringVal = subString(mark + 1, bufPos); + } else { + stringVal = new String(buf, 0, bufPos); + } + } + public void scanSharp() { scanVariable(); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 78fe42d7ff..3c063d5b4f 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -757,11 +757,12 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Printab x.getFrom().accept(this); } - if (x.getWhere() != null) { + SQLExpr where = x.getWhere(); + if (where != null) { println(); print0(ucase ? "WHERE " : "where "); - x.getWhere().setParent(x); - x.getWhere().accept(this); + where.setParent(x); + where.accept(this); } if (x.getGroupBy() != null) { diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest14.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest14.java new file mode 100644 index 0000000000..3f9072fbc0 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest14.java @@ -0,0 +1,58 @@ +/* + * Copyright 1999-2101 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.odps; + +import java.util.List; + +import org.junit.Assert; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; + +import junit.framework.TestCase; + +public class OdpsSelectTest14 extends TestCase { + + public void test_select() throws Exception { + String sql = "SELECT split_part(content, '\\001')[1] FROM dual;";// + Assert.assertEquals("SELECT split_part(content, '\\001')[1]\n" + + "FROM dual;", SQLUtils.formatOdps(sql)); + Assert.assertEquals("select split_part(content, '\\001')[1]\n" + + "from dual;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS); + SQLStatement stmt = statementList.get(0); + + Assert.assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + Assert.assertEquals(1, visitor.getTables().size()); + Assert.assertEquals(1, visitor.getColumns().size()); + Assert.assertEquals(0, visitor.getConditions().size()); + +// Assert.assertTrue(visitor.getColumns().contains(new Column("abc", "name"))); + } + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest15.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest15.java new file mode 100644 index 0000000000..ea2dfc79da --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest15.java @@ -0,0 +1,58 @@ +/* + * Copyright 1999-2101 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.odps; + +import java.util.List; + +import org.junit.Assert; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; + +import junit.framework.TestCase; + +public class OdpsSelectTest15 extends TestCase { + + public void test_select() throws Exception { + String sql = "SELECT '\\n' FROM dual;";// + Assert.assertEquals("SELECT '\\n'" + + "\nFROM dual;", SQLUtils.formatOdps(sql)); + Assert.assertEquals("select '\\n'" + + "\nfrom dual;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS); + SQLStatement stmt = statementList.get(0); + + Assert.assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + Assert.assertEquals(1, visitor.getTables().size()); + Assert.assertEquals(0, visitor.getColumns().size()); + Assert.assertEquals(0, visitor.getConditions().size()); + +// Assert.assertTrue(visitor.getColumns().contains(new Column("abc", "name"))); + } + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest16.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest16.java new file mode 100644 index 0000000000..9405e368f0 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest16.java @@ -0,0 +1,58 @@ +/* + * Copyright 1999-2101 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.odps; + +import java.util.List; + +import org.junit.Assert; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; + +import junit.framework.TestCase; + +public class OdpsSelectTest16 extends TestCase { + + public void test_select() throws Exception { + String sql = "SELECT \"\\n\" FROM dual;";// + Assert.assertEquals("SELECT '\\n'" + + "\nFROM dual;", SQLUtils.formatOdps(sql)); + Assert.assertEquals("select '\\n'" + + "\nfrom dual;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS); + SQLStatement stmt = statementList.get(0); + + Assert.assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + Assert.assertEquals(1, visitor.getTables().size()); + Assert.assertEquals(0, visitor.getColumns().size()); + Assert.assertEquals(0, visitor.getConditions().size()); + +// Assert.assertTrue(visitor.getColumns().contains(new Column("abc", "name"))); + } + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest56.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest56.java index 0194dff6ba..1a71418e06 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest56.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest56.java @@ -29,47 +29,85 @@ public class OracleSelectTest56 extends OracleTest { public void test_0() throws Exception { String sql = // - "SELECT TRIM(BOTH FROM EUCD) AS \"value\",NTLANG1 AS \"text\" " // - + " FROM T_HT_WREM_ENUMLANG_D" - + " WHERE TYPE=?" - + " ORDER BY \"value\" ASC"; // + "select " + + "\nAA.ID," + + "\nAA.CODE," + + "\nAA.TYPE," + + "\nAA.STATUS," + + "\nAA.EMPLOYEENAME," + + "\nAA.CREATORNAME," + + "\nAA.OPERATIONTYPE," + + "\nAA.CREATEDATE," + + "\nAA.REMARK," + + "\nW.NAME," + + "\nDD.DESC" + + "\nfrom " + + "\na AA," + + "\nw W," + + "\nd DD" + + "\nwhere " + + "\nAA.employeeNo IN (" + + "\nSELECT employeeno FROM employeeauditor ea WHERE auditorno = 1 GROUP BY employeeno" + + "\nUNION ALL" + + "\nSELECT 1 FROM dual " + + "\n) and" + + "\nAA.WNO = W.WNO(+) and" + + "\nAA.DEPTNO = DD.DEPTNO(+)"; // OracleStatementParser parser = new OracleStatementParser(sql); List statementList = parser.parseStatementList(); SQLStatement stmt = statementList.get(0); - print(statementList); +// print(statementList); Assert.assertEquals(1, statementList.size()); OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); stmt.accept(visitor); - System.out.println("Tables : " + visitor.getTables()); - System.out.println("fields : " + visitor.getColumns()); - System.out.println("coditions : " + visitor.getConditions()); - System.out.println("relationships : " + visitor.getRelationships()); - System.out.println("orderBy : " + visitor.getOrderByColumns()); +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("relationships : " + visitor.getRelationships()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); - Assert.assertEquals(1, visitor.getTables().size()); + Assert.assertEquals(4, visitor.getTables().size()); - Assert.assertEquals(4, visitor.getColumns().size()); + Assert.assertEquals(18, visitor.getColumns().size()); { String text = SQLUtils.toOracleString(stmt); - Assert.assertEquals("SELECT TRIM(EUCD) AS \"value\", NTLANG1 AS \"text\"" - + "\nFROM T_HT_WREM_ENUMLANG_D" - + "\nWHERE TYPE = ?" - + "\nORDER BY \"value\" ASC", text); + Assert.assertEquals("SELECT AA.ID, AA.CODE, AA.TYPE, AA.STATUS, AA.EMPLOYEENAME\n" + + "\t, AA.CREATORNAME, AA.OPERATIONTYPE, AA.CREATEDATE, AA.REMARK, W.NAME\n" + + "\t, DD.DESC\n" + + "FROM a AA, w W, d DD\n" + + "WHERE AA.employeeNo IN (SELECT employeeno\n" + + "\t\tFROM employeeauditor ea\n" + + "\t\tWHERE auditorno = 1\n" + + "\t\tGROUP BY employeeno\n" + + "\t\tUNION ALL\n" + + "\t\tSELECT 1\n" + + "\t\tFROM dual)\n" + + "\tAND AA.WNO = W.WNO(+)\n" + + "\tAND AA.DEPTNO = DD.DEPTNO(+)", text); } { String text = SQLUtils.toOracleString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION); - Assert.assertEquals("select TRIM(EUCD) as \"value\", NTLANG1 as \"text\"" - + "\nfrom T_HT_WREM_ENUMLANG_D" - + "\nwhere TYPE = ?" - + "\norder by \"value\" asc", text); + Assert.assertEquals("select AA.ID, AA.CODE, AA.TYPE, AA.STATUS, AA.EMPLOYEENAME\n" + + "\t, AA.CREATORNAME, AA.OPERATIONTYPE, AA.CREATEDATE, AA.REMARK, W.NAME\n" + + "\t, DD.DESC\n" + + "from a AA, w W, d DD\n" + + "where AA.employeeNo in (select employeeno\n" + + "\t\tfrom employeeauditor ea\n" + + "\t\twhere auditorno = 1\n" + + "\t\tgroup by employeeno\n" + + "\t\tunion all\n" + + "\t\tselect 1\n" + + "\t\tfrom dual)\n" + + "\tand AA.WNO = W.WNO(+)\n" + + "\tand AA.DEPTNO = DD.DEPTNO(+)", text); } // Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("acduser.vw_acd_info", "xzqh")));