improved odps sql parser, support ARRAY/MAP/STRUCT dataType, for issue #2145

This commit is contained in:
wenshao
2017-12-02 12:50:06 +08:00
parent fd2ba2a290
commit 569c50977a
10 changed files with 701 additions and 1 deletions
@@ -0,0 +1,109 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.ast;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import com.alibaba.druid.util.FnvHash;
import java.util.Collections;
import java.util.List;
public class SQLArrayDataType extends SQLObjectImpl implements SQLDataType {
private String dbType;
private SQLDataType componentType;
public SQLArrayDataType(SQLDataType componentType) {
setComponentType(componentType);
}
public SQLArrayDataType(SQLDataType componentType, String dbType) {
this.dbType = dbType;
setComponentType(componentType);
}
@Override
public String getName() {
return "ARRAY";
}
@Override
public long nameHashCode64() {
return FnvHash.Constants.ARRAY;
}
@Override
public void setName(String name) {
throw new UnsupportedOperationException();
}
@Override
public List<SQLExpr> getArguments() {
return Collections.emptyList();
}
@Override
public Boolean getWithTimeZone() {
return null;
}
@Override
public void setWithTimeZone(Boolean value) {
throw new UnsupportedOperationException();
}
@Override
public boolean isWithLocalTimeZone() {
return false;
}
@Override
public void setWithLocalTimeZone(boolean value) {
throw new UnsupportedOperationException();
}
@Override
public void setDbType(String dbType) {
dbType = dbType;
}
@Override
public String getDbType() {
return dbType;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, componentType);
}
visitor.endVisit(this);
}
public SQLArrayDataType clone() {
return null;
}
public SQLDataType getComponentType() {
return componentType;
}
public void setComponentType(SQLDataType x) {
if (x != null) {
x.setParent(this);
}
this.componentType = x;
}
}
@@ -0,0 +1,139 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.ast;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import com.alibaba.druid.util.FnvHash;
import java.util.Collections;
import java.util.List;
public class SQLMapDataType extends SQLObjectImpl implements SQLDataType {
private String dbType;
private SQLDataType keyType;
private SQLDataType valueType;
public SQLMapDataType() {
}
public SQLMapDataType(SQLDataType keyType, SQLDataType valueType) {
this.setKeyType(keyType);
this.setValueType(valueType);
}
public SQLMapDataType(SQLDataType keyType, SQLDataType valueType, String dbType) {
this.setKeyType(keyType);
this.setValueType(valueType);
this.dbType = dbType;
}
@Override
public String getName() {
return "MAP";
}
@Override
public long nameHashCode64() {
return FnvHash.Constants.MAP;
}
@Override
public void setName(String name) {
throw new UnsupportedOperationException();
}
@Override
public List<SQLExpr> getArguments() {
return Collections.emptyList();
}
@Override
public Boolean getWithTimeZone() {
return null;
}
@Override
public void setWithTimeZone(Boolean value) {
throw new UnsupportedOperationException();
}
@Override
public boolean isWithLocalTimeZone() {
return false;
}
@Override
public void setWithLocalTimeZone(boolean value) {
throw new UnsupportedOperationException();
}
@Override
public void setDbType(String dbType) {
dbType = dbType;
}
@Override
public String getDbType() {
return dbType;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, keyType);
acceptChild(visitor, valueType);
}
visitor.endVisit(this);
}
public SQLMapDataType clone() {
SQLMapDataType x = new SQLMapDataType();
x.dbType = dbType;
if (keyType != null) {
x.setKeyType(keyType.clone());
}
if (valueType != null) {
x.setValueType(valueType.clone());
}
return x;
}
public SQLDataType getKeyType() {
return keyType;
}
public void setKeyType(SQLDataType x) {
if (x != null) {
x.setParent(this);
}
this.keyType = x;
}
public SQLDataType getValueType() {
return valueType;
}
public void setValueType(SQLDataType x) {
if (x != null) {
x.setParent(this);
}
this.valueType = x;
}
}
@@ -0,0 +1,155 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.ast;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import com.alibaba.druid.util.FnvHash;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class SQLStructDataType extends SQLObjectImpl implements SQLDataType {
private String dbType;
private List<Field> fields = new ArrayList<Field>();
public SQLStructDataType() {
}
public SQLStructDataType(String dbType) {
this.dbType = dbType;
}
@Override
public String getName() {
return "STRUCT";
}
@Override
public long nameHashCode64() {
return FnvHash.Constants.STRUCT;
}
@Override
public void setName(String name) {
throw new UnsupportedOperationException();
}
@Override
public List<SQLExpr> getArguments() {
return Collections.emptyList();
}
@Override
public Boolean getWithTimeZone() {
return null;
}
@Override
public void setWithTimeZone(Boolean value) {
throw new UnsupportedOperationException();
}
@Override
public boolean isWithLocalTimeZone() {
return false;
}
@Override
public void setWithLocalTimeZone(boolean value) {
throw new UnsupportedOperationException();
}
@Override
public void setDbType(String dbType) {
dbType = dbType;
}
@Override
public String getDbType() {
return dbType;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, fields);
}
visitor.endVisit(this);
}
public SQLStructDataType clone() {
SQLStructDataType x = new SQLStructDataType(dbType);
for (Field field : fields) {
x.addField(field.name, field.dataType.clone());
}
return x;
}
public List<Field> getFields() {
return fields;
}
public void addField(SQLName name, SQLDataType dataType) {
Field field = new Field(name, dataType);
field.setParent(this);
fields.add(field);
}
public static class Field extends SQLObjectImpl {
private SQLName name;
private SQLDataType dataType;
public Field(SQLName name, SQLDataType dataType) {
setName(name);
setDataType(dataType);
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, name);
acceptChild(visitor, dataType);
}
visitor.endVisit(this);
}
public SQLName getName() {
return name;
}
public void setName(SQLName x) {
if (x != null) {
x.setParent(this);
}
this.name = x;
}
public SQLDataType getDataType() {
return dataType;
}
public void setDataType(SQLDataType x) {
if (x != null) {
x.setParent(this);
}
this.dataType = x;
}
}
}
@@ -15,7 +15,7 @@
*/
package com.alibaba.druid.sql.dialect.odps.parser;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.SQLArrayExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
@@ -167,6 +167,55 @@ public class OdpsExprParser extends SQLExprParser {
return super.relationalRest(expr);
}
public SQLDataType parseDataType() {
if (lexer.identifierEquals(FnvHash.Constants.ARRAY)) {
lexer.nextToken();
accept(Token.LT);
SQLDataType itemType = parseDataType();
accept(Token.GT);
return new SQLArrayDataType(itemType, dbType);
}
if (lexer.identifierEquals(FnvHash.Constants.MAP)) {
lexer.nextToken();
accept(Token.LT);
SQLDataType keyType = parseDataType();
accept(Token.COMMA);
SQLDataType valueType = parseDataType();
accept(Token.GT);
return new SQLMapDataType(keyType, valueType, dbType);
}
if (lexer.identifierEquals(FnvHash.Constants.STRUCT)) {
lexer.nextToken();
SQLStructDataType struct = new SQLStructDataType(dbType);
accept(Token.LT);
for (;;) {
SQLName name = this.name();
accept(Token.COLON);
SQLDataType dataType = this.parseDataType();
struct.addField(name, dataType);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
}
break;
}
accept(Token.GT);
return struct;
// throw new ParserException("TODO : " + lexer.info());
}
return super.parseDataType();
}
@Override
public OdpsSelectParser createSelectParser() {
return new OdpsSelectParser(this);
@@ -6041,4 +6041,39 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
return false;
}
@Override
public boolean visit(SQLArrayDataType x) {
print0(ucase ? "ARRAY<" : "array<");
x.getComponentType().accept(this);
print('>');
return false;
}
@Override
public boolean visit(SQLMapDataType x) {
print0(ucase ? "MAP<" : "map<");
x.getKeyType().accept(this);
print0(", ");
x.getValueType().accept(this);
print('>');
return false;
}
@Override
public boolean visit(SQLStructDataType x) {
print0(ucase ? "STRUCT<" : "struct<");
printAndAccept(x.getFields(), ", ");
print('>');
return false;
}
@Override
public boolean visit(SQLStructDataType.Field x) {
x.getName().accept(this);
print(':');
x.getDataType().accept(this);
print('>');
return false;
}
}
@@ -802,4 +802,15 @@ public interface SQLASTVisitor {
boolean visit(SQLExternalRecordFormat x);
void endVisit(SQLExternalRecordFormat x);
boolean visit(SQLArrayDataType x);
void endVisit(SQLArrayDataType x);
boolean visit(SQLMapDataType x);
void endVisit(SQLMapDataType x);
boolean visit(SQLStructDataType x);
void endVisit(SQLStructDataType x);
boolean visit(SQLStructDataType.Field x);
void endVisit(SQLStructDataType.Field x);
}
@@ -1874,6 +1874,46 @@ public class SQLASTVisitorAdapter implements SQLASTVisitor {
}
@Override
public boolean visit(SQLArrayDataType x) {
return true;
}
@Override
public void endVisit(SQLArrayDataType x) {
}
@Override
public boolean visit(SQLMapDataType x) {
return true;
}
@Override
public void endVisit(SQLMapDataType x) {
}
@Override
public boolean visit(SQLStructDataType x) {
return true;
}
@Override
public void endVisit(SQLStructDataType x) {
}
@Override
public boolean visit(SQLStructDataType.Field x) {
return true;
}
@Override
public void endVisit(SQLStructDataType.Field x) {
}
public final boolean isEnabled(VisitorFeature feature) {
return VisitorFeature.isEnabled(this.features, feature);
}
@@ -615,6 +615,8 @@ public final class FnvHash {
long SORTED = fnv1a_64_lower("SORTED");
long LIFECYCLE = fnv1a_64_lower("LIFECYCLE");
long PARTITIONS = fnv1a_64_lower("PARTITIONS");
long ARRAY = fnv1a_64_lower("ARRAY");
long STRUCT = fnv1a_64_lower("STRUCT");
long ROLLBACK = fnv1a_64_lower("ROLLBACK");
long SAVEPOINT = fnv1a_64_lower("SAVEPOINT");
@@ -0,0 +1,91 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.odps;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class OdpsCreateTableTest6_array extends TestCase {
public void test_select() throws Exception {
// 1095288847322
String sql = "CREATE TABLE src_thrift\n" +
"(\n" +
" aint bigint,\n" +
" astring string,\n" +
" lint ARRAY<bigint>,\n" +
" lstring ARRAY<STRING>,\n" +
" --lintString ARRAY<INTSTRING>\n" +
" mStringString MAP<STRING, STRING>\n" +
" --attribute Map<String,Map<String,Map<String,PropValueUnion>>>,\n" +
" --unionField1 PropValueUnion,\n" +
" --unionField2 PropValueUnion,\n" +
" -- unionField3 PropValueUnion\n" +
");";//
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS);
SQLStatement stmt = statementList.get(0);
assertEquals("CREATE TABLE src_thrift (\n" +
"\taint BIGINT,\n" +
"\tastring STRING,\n" +
"\tlint ARRAY<BIGINT>,\n" +
"\tlstring ARRAY<STRING>, -- lintString ARRAY<INTSTRING>\n" +
"\tmStringString MAP<STRING, STRING> -- attribute Map<String,Map<String,Map<String,PropValueUnion>>>,\n" +
"\t-- unionField1 PropValueUnion,\n" +
"\t-- unionField2 PropValueUnion,\n" +
"\t-- unionField3 PropValueUnion\n" +
");", stmt.toString());
assertEquals("create table src_thrift (\n" +
"\taint bigint,\n" +
"\tastring string,\n" +
"\tlint array<bigint>,\n" +
"\tlstring array<string>, -- lintString ARRAY<INTSTRING>\n" +
"\tmStringString map<string, string> -- attribute Map<String,Map<String,Map<String,PropValueUnion>>>,\n" +
"\t-- unionField1 PropValueUnion,\n" +
"\t-- unionField2 PropValueUnion,\n" +
"\t-- unionField3 PropValueUnion\n" +
");", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(5, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
// System.out.println(SQLUtils.formatOdps(sql));
// assertTrue(visitor.getColumns().contains(new Column("abc", "name")));
}
}
@@ -0,0 +1,69 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.odps;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class OdpsCreateTableTest7_struct extends TestCase {
public void test_select() throws Exception {
// 1095288847322
String sql = "CREATE TABLE DEST3(k STRUCT<col1:STRING, col2:STRING>, v BIGINT) STORED AS TEXTFILE;";//
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS);
SQLStatement stmt = statementList.get(0);
assertEquals("CREATE TABLE DEST3 (\n" +
"\tk STRUCT<col1:STRING>, col2:STRING>>,\n" +
"\tv BIGINT\n" +
")\n" +
"STORED AS TEXTFILE;", stmt.toString());
assertEquals("create table DEST3 (\n" +
"\tk struct<col1:string>, col2:string>>,\n" +
"\tv bigint\n" +
")\n" +
"stored as TEXTFILE;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(2, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
// System.out.println(SQLUtils.formatOdps(sql));
// assertTrue(visitor.getColumns().contains(new Column("abc", "name")));
}
}