Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improved odps sql parser. bug fixed for issue #1375

Browse Source
This commit is contained in:
wenshao
2016-08-21 13:58:38 +08:00
parent 0565f9c1a9
commit 103ee3e1a3
8 changed files with 381 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java
+4
View File
@@ -352,4 +352,8 @@ public class OdpsLexer extends Lexer {
protected final void scanString() {
scanString2();
}
protected final void scanAlias() {
scanAlias2();
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java
+25 -4
View File
@@ -881,10 +881,31 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
if (text == null) {
print0(ucase ? "NULL" : "null");
} else {
print('\'');
String text2 = text.replaceAll("\\'", "\\\\'");
print0(text2);
print('\'');
StringBuilder buf = new StringBuilder(text.length() + 2);
buf.append('\'');
for (int i = 0; i < text.length(); ++i) {
char ch = text.charAt(i);
switch (ch) {
case '\\':
buf.append("\\\\");
break;
case '\'':
buf.append("\\'");
break;
case '\0':
buf.append("\\0");
break;
case '\n':
buf.append("\\n");
break;
default:
buf.append(ch);
break;
}
}
buf.append('\'');
print0(buf.toString());
}
return false;
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+116 -1
View File
@@ -809,7 +809,7 @@ public class Lexer {
}
}
private final void scanAlias() {
protected void scanAlias() {
mark = pos;
if (buf == null) {
@@ -856,6 +856,121 @@ public class Lexer {
//stringVal = subString(mark + 1, bufPos);
}
protected final void scanAlias2() {
{
boolean hasSpecial = false;
int startIndex = pos + 1;
int endIndex = -1; // text.indexOf('\'', startIndex);
for (int i = startIndex; i < text.length(); ++i) {
final char ch = text.charAt(i);
if (ch == '\\') {
hasSpecial = true;
continue;
}
if (ch == '"') {
endIndex = i;
break;
}
}
if (endIndex == -1) {
throw new ParserException("unclosed str");
}
String stringVal = subString(startIndex, endIndex - startIndex);
// hasSpecial = stringVal.indexOf('\\') != -1;
if (!hasSpecial) {
this.stringVal = stringVal;
int pos = endIndex + 1;
char ch = charAt(pos);
if (ch != '\'') {
this.pos = pos;
this.ch = ch;
token = LITERAL_CHARS;
return;
}
}
}
mark = pos;
boolean hasSpecial = false;
for (;;) {
if (isEOF()) {
lexError("unclosed.str.lit");
return;
}
ch = charAt(++pos);
if (ch == '\\') {
scanChar();
if (!hasSpecial) {
initBuff(bufPos);
arraycopy(mark + 1, buf, 0, bufPos);
hasSpecial = true;
}
switch (ch) {
case '0':
putChar('\0');
break;
case '\'':
putChar('\'');
break;
case '"':
putChar('"');
break;
case 'b':
putChar('\b');
break;
case 'n':
putChar('\n');
break;
case 'r':
putChar('\r');
break;
case 't':
putChar('\t');
break;
case '\\':
putChar('\\');
break;
case 'Z':
putChar((char) 0x1A); // ctrl + Z
break;
default:
putChar(ch);
break;
}
continue;
}
if (ch == '\"') {
scanChar();
token = LITERAL_CHARS;
break;
}
if (!hasSpecial) {
bufPos++;
continue;
}
if (bufPos == buf.length) {
putChar(ch);
} else {
buf[bufPos++] = ch;
}
}
if (!hasSpecial) {
stringVal = subString(mark + 1, bufPos);
} else {
stringVal = new String(buf, 0, bufPos);
}
}
public void scanSharp() {
scanVariable();
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java
+4 -3
View File
@@ -757,11 +757,12 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Printab
x.getFrom().accept(this);
}
if (x.getWhere() != null) {
SQLExpr where = x.getWhere();
if (where != null) {
println();
print0(ucase ? "WHERE " : "where ");
x.getWhere().setParent(x);
x.getWhere().accept(this);
where.setParent(x);
where.accept(this);
}
if (x.getGroupBy() != null) {
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest14.java
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 1999-2101 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.odps;
import java.util.List;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
public class OdpsSelectTest14 extends TestCase {
public void test_select() throws Exception {
String sql = "SELECT split_part(content, '\\001')[1] FROM dual;";//
Assert.assertEquals("SELECT split_part(content, '\\001')[1]\n" +
"FROM dual;", SQLUtils.formatOdps(sql));
Assert.assertEquals("select split_part(content, '\\001')[1]\n" +
"from dual;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS);
SQLStatement stmt = statementList.get(0);
Assert.assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
// System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
Assert.assertEquals(1, visitor.getTables().size());
Assert.assertEquals(1, visitor.getColumns().size());
Assert.assertEquals(0, visitor.getConditions().size());
// Assert.assertTrue(visitor.getColumns().contains(new Column("abc", "name")));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest15.java
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 1999-2101 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.odps;
import java.util.List;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
public class OdpsSelectTest15 extends TestCase {
public void test_select() throws Exception {
String sql = "SELECT '\\n' FROM dual;";//
Assert.assertEquals("SELECT '\\n'" +
"\nFROM dual;", SQLUtils.formatOdps(sql));
Assert.assertEquals("select '\\n'" +
"\nfrom dual;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS);
SQLStatement stmt = statementList.get(0);
Assert.assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
// System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
Assert.assertEquals(1, visitor.getTables().size());
Assert.assertEquals(0, visitor.getColumns().size());
Assert.assertEquals(0, visitor.getConditions().size());
// Assert.assertTrue(visitor.getColumns().contains(new Column("abc", "name")));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest16.java
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 1999-2101 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.odps;
import java.util.List;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
public class OdpsSelectTest16 extends TestCase {
public void test_select() throws Exception {
String sql = "SELECT \"\\n\" FROM dual;";//
Assert.assertEquals("SELECT '\\n'" +
"\nFROM dual;", SQLUtils.formatOdps(sql));
Assert.assertEquals("select '\\n'" +
"\nfrom dual;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS);
SQLStatement stmt = statementList.get(0);
Assert.assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
// System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
Assert.assertEquals(1, visitor.getTables().size());
Assert.assertEquals(0, visitor.getColumns().size());
Assert.assertEquals(0, visitor.getConditions().size());
// Assert.assertTrue(visitor.getColumns().contains(new Column("abc", "name")));
}
}
src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest56.java
+58 -20
View File
@@ -29,47 +29,85 @@ public class OracleSelectTest56 extends OracleTest {
public void test_0() throws Exception {
String sql = //
"SELECT TRIM(BOTH FROM EUCD) AS \"value\",NTLANG1 AS \"text\" " //
+ " FROM T_HT_WREM_ENUMLANG_D"
+ " WHERE TYPE=?"
+ " ORDER BY \"value\" ASC"; //
"select "
+ "\nAA.ID,"
+ "\nAA.CODE,"
+ "\nAA.TYPE,"
+ "\nAA.STATUS,"
+ "\nAA.EMPLOYEENAME,"
+ "\nAA.CREATORNAME,"
+ "\nAA.OPERATIONTYPE,"
+ "\nAA.CREATEDATE,"
+ "\nAA.REMARK,"
+ "\nW.NAME,"
+ "\nDD.DESC"
+ "\nfrom "
+ "\na AA,"
+ "\nw W,"
+ "\nd DD"
+ "\nwhere "
+ "\nAA.employeeNo IN ("
+ "\nSELECT employeeno FROM employeeauditor ea WHERE auditorno = 1 GROUP BY employeeno"
+ "\nUNION ALL"
+ "\nSELECT 1 FROM dual "
+ "\n) and"
+ "\nAA.WNO = W.WNO(+) and"
+ "\nAA.DEPTNO = DD.DEPTNO(+)"; //
OracleStatementParser parser = new OracleStatementParser(sql);
List<SQLStatement> statementList = parser.parseStatementList();
SQLStatement stmt = statementList.get(0);
print(statementList);
// print(statementList);
Assert.assertEquals(1, statementList.size());
OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor();
stmt.accept(visitor);
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println("orderBy : " + visitor.getOrderByColumns());
// System.out.println("Tables : " + visitor.getTables());
// System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("relationships : " + visitor.getRelationships());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
Assert.assertEquals(1, visitor.getTables().size());
Assert.assertEquals(4, visitor.getTables().size());
Assert.assertEquals(4, visitor.getColumns().size());
Assert.assertEquals(18, visitor.getColumns().size());
{
String text = SQLUtils.toOracleString(stmt);
Assert.assertEquals("SELECT TRIM(EUCD) AS \"value\", NTLANG1 AS \"text\""
+ "\nFROM T_HT_WREM_ENUMLANG_D"
+ "\nWHERE TYPE = ?"
+ "\nORDER BY \"value\" ASC", text);
Assert.assertEquals("SELECT AA.ID, AA.CODE, AA.TYPE, AA.STATUS, AA.EMPLOYEENAME\n" +
"\t, AA.CREATORNAME, AA.OPERATIONTYPE, AA.CREATEDATE, AA.REMARK, W.NAME\n" +
"\t, DD.DESC\n" +
"FROM a AA, w W, d DD\n" +
"WHERE AA.employeeNo IN (SELECT employeeno\n" +
"\t\tFROM employeeauditor ea\n" +
"\t\tWHERE auditorno = 1\n" +
"\t\tGROUP BY employeeno\n" +
"\t\tUNION ALL\n" +
"\t\tSELECT 1\n" +
"\t\tFROM dual)\n" +
"\tAND AA.WNO = W.WNO(+)\n" +
"\tAND AA.DEPTNO = DD.DEPTNO(+)", text);
}
{
String text = SQLUtils.toOracleString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION);
Assert.assertEquals("select TRIM(EUCD) as \"value\", NTLANG1 as \"text\""
+ "\nfrom T_HT_WREM_ENUMLANG_D"
+ "\nwhere TYPE = ?"
+ "\norder by \"value\" asc", text);
Assert.assertEquals("select AA.ID, AA.CODE, AA.TYPE, AA.STATUS, AA.EMPLOYEENAME\n" +
"\t, AA.CREATORNAME, AA.OPERATIONTYPE, AA.CREATEDATE, AA.REMARK, W.NAME\n" +
"\t, DD.DESC\n" +
"from a AA, w W, d DD\n" +
"where AA.employeeNo in (select employeeno\n" +
"\t\tfrom employeeauditor ea\n" +
"\t\twhere auditorno = 1\n" +
"\t\tgroup by employeeno\n" +
"\t\tunion all\n" +
"\t\tselect 1\n" +
"\t\tfrom dual)\n" +
"\tand AA.WNO = W.WNO(+)\n" +
"\tand AA.DEPTNO = DD.DEPTNO(+)", text);
}
// Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("acduser.vw_acd_info", "xzqh")));