From 569c50977a40475db70f9a9f73dd8510ca8ebe4e Mon Sep 17 00:00:00 2001 From: wenshao Date: Sat, 2 Dec 2017 12:50:06 +0800 Subject: [PATCH] improved odps sql parser, support ARRAY/MAP/STRUCT dataType, for issue #2145 --- .../druid/sql/ast/SQLArrayDataType.java | 109 ++++++++++++ .../alibaba/druid/sql/ast/SQLMapDataType.java | 139 ++++++++++++++++ .../druid/sql/ast/SQLStructDataType.java | 155 ++++++++++++++++++ .../dialect/odps/parser/OdpsExprParser.java | 51 +++++- .../sql/visitor/SQLASTOutputVisitor.java | 35 ++++ .../druid/sql/visitor/SQLASTVisitor.java | 11 ++ .../sql/visitor/SQLASTVisitorAdapter.java | 40 +++++ .../java/com/alibaba/druid/util/FnvHash.java | 2 + .../sql/odps/OdpsCreateTableTest6_array.java | 91 ++++++++++ .../sql/odps/OdpsCreateTableTest7_struct.java | 69 ++++++++ 10 files changed, 701 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/alibaba/druid/sql/ast/SQLArrayDataType.java create mode 100644 src/main/java/com/alibaba/druid/sql/ast/SQLMapDataType.java create mode 100644 src/main/java/com/alibaba/druid/sql/ast/SQLStructDataType.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest6_array.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest7_struct.java diff --git a/src/main/java/com/alibaba/druid/sql/ast/SQLArrayDataType.java b/src/main/java/com/alibaba/druid/sql/ast/SQLArrayDataType.java new file mode 100644 index 0000000000..afc251252b --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/ast/SQLArrayDataType.java @@ -0,0 +1,109 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.ast; + +import com.alibaba.druid.sql.visitor.SQLASTVisitor; +import com.alibaba.druid.util.FnvHash; + +import java.util.Collections; +import java.util.List; + +public class SQLArrayDataType extends SQLObjectImpl implements SQLDataType { + private String dbType; + private SQLDataType componentType; + + public SQLArrayDataType(SQLDataType componentType) { + setComponentType(componentType); + } + + public SQLArrayDataType(SQLDataType componentType, String dbType) { + this.dbType = dbType; + setComponentType(componentType); + } + + @Override + public String getName() { + return "ARRAY"; + } + + @Override + public long nameHashCode64() { + return FnvHash.Constants.ARRAY; + } + + @Override + public void setName(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public List getArguments() { + return Collections.emptyList(); + } + + @Override + public Boolean getWithTimeZone() { + return null; + } + + @Override + public void setWithTimeZone(Boolean value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isWithLocalTimeZone() { + return false; + } + + @Override + public void setWithLocalTimeZone(boolean value) { + throw new UnsupportedOperationException(); + } + + @Override + public void setDbType(String dbType) { + dbType = dbType; + } + + @Override + public String getDbType() { + return dbType; + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + if (visitor.visit(this)) { + acceptChild(visitor, componentType); + } + visitor.endVisit(this); + } + + public SQLArrayDataType clone() { + return null; + } + + public SQLDataType getComponentType() { + return componentType; + } + + public void setComponentType(SQLDataType x) { + if (x != null) { + x.setParent(this); + } + this.componentType = x; + } +} diff --git a/src/main/java/com/alibaba/druid/sql/ast/SQLMapDataType.java b/src/main/java/com/alibaba/druid/sql/ast/SQLMapDataType.java new file mode 100644 index 0000000000..f4fa11bdfd --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/ast/SQLMapDataType.java @@ -0,0 +1,139 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.ast; + +import com.alibaba.druid.sql.visitor.SQLASTVisitor; +import com.alibaba.druid.util.FnvHash; + +import java.util.Collections; +import java.util.List; + +public class SQLMapDataType extends SQLObjectImpl implements SQLDataType { + private String dbType; + private SQLDataType keyType; + private SQLDataType valueType; + + public SQLMapDataType() { + + } + + public SQLMapDataType(SQLDataType keyType, SQLDataType valueType) { + this.setKeyType(keyType); + this.setValueType(valueType); + } + + public SQLMapDataType(SQLDataType keyType, SQLDataType valueType, String dbType) { + this.setKeyType(keyType); + this.setValueType(valueType); + this.dbType = dbType; + } + + @Override + public String getName() { + return "MAP"; + } + + @Override + public long nameHashCode64() { + return FnvHash.Constants.MAP; + } + + @Override + public void setName(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public List getArguments() { + return Collections.emptyList(); + } + + @Override + public Boolean getWithTimeZone() { + return null; + } + + @Override + public void setWithTimeZone(Boolean value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isWithLocalTimeZone() { + return false; + } + + @Override + public void setWithLocalTimeZone(boolean value) { + throw new UnsupportedOperationException(); + } + + @Override + public void setDbType(String dbType) { + dbType = dbType; + } + + @Override + public String getDbType() { + return dbType; + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + if (visitor.visit(this)) { + acceptChild(visitor, keyType); + acceptChild(visitor, valueType); + } + visitor.endVisit(this); + } + + public SQLMapDataType clone() { + SQLMapDataType x = new SQLMapDataType(); + x.dbType = dbType; + + if (keyType != null) { + x.setKeyType(keyType.clone()); + } + + if (valueType != null) { + x.setValueType(valueType.clone()); + } + + return x; + } + + public SQLDataType getKeyType() { + return keyType; + } + + public void setKeyType(SQLDataType x) { + if (x != null) { + x.setParent(this); + } + this.keyType = x; + } + + public SQLDataType getValueType() { + return valueType; + } + + public void setValueType(SQLDataType x) { + if (x != null) { + x.setParent(this); + } + this.valueType = x; + } +} diff --git a/src/main/java/com/alibaba/druid/sql/ast/SQLStructDataType.java b/src/main/java/com/alibaba/druid/sql/ast/SQLStructDataType.java new file mode 100644 index 0000000000..7d00877161 --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/ast/SQLStructDataType.java @@ -0,0 +1,155 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.ast; + +import com.alibaba.druid.sql.visitor.SQLASTVisitor; +import com.alibaba.druid.util.FnvHash; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class SQLStructDataType extends SQLObjectImpl implements SQLDataType { + private String dbType; + private List fields = new ArrayList(); + + public SQLStructDataType() { + + } + + public SQLStructDataType(String dbType) { + this.dbType = dbType; + } + + @Override + public String getName() { + return "STRUCT"; + } + + @Override + public long nameHashCode64() { + return FnvHash.Constants.STRUCT; + } + + @Override + public void setName(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public List getArguments() { + return Collections.emptyList(); + } + + @Override + public Boolean getWithTimeZone() { + return null; + } + + @Override + public void setWithTimeZone(Boolean value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isWithLocalTimeZone() { + return false; + } + + @Override + public void setWithLocalTimeZone(boolean value) { + throw new UnsupportedOperationException(); + } + + @Override + public void setDbType(String dbType) { + dbType = dbType; + } + + @Override + public String getDbType() { + return dbType; + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + if (visitor.visit(this)) { + acceptChild(visitor, fields); + } + visitor.endVisit(this); + } + + public SQLStructDataType clone() { + SQLStructDataType x = new SQLStructDataType(dbType); + + for (Field field : fields) { + x.addField(field.name, field.dataType.clone()); + } + + return x; + } + + public List getFields() { + return fields; + } + + public void addField(SQLName name, SQLDataType dataType) { + Field field = new Field(name, dataType); + field.setParent(this); + fields.add(field); + } + + public static class Field extends SQLObjectImpl { + private SQLName name; + private SQLDataType dataType; + + public Field(SQLName name, SQLDataType dataType) { + setName(name); + setDataType(dataType); + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + if (visitor.visit(this)) { + acceptChild(visitor, name); + acceptChild(visitor, dataType); + } + visitor.endVisit(this); + } + + public SQLName getName() { + return name; + } + + public void setName(SQLName x) { + if (x != null) { + x.setParent(this); + } + this.name = x; + } + + public SQLDataType getDataType() { + return dataType; + } + + public void setDataType(SQLDataType x) { + if (x != null) { + x.setParent(this); + } + this.dataType = x; + } + } +} diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java index c8404c76a2..c80a47bf00 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java @@ -15,7 +15,7 @@ */ package com.alibaba.druid.sql.dialect.odps.parser; -import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.*; import com.alibaba.druid.sql.ast.expr.SQLArrayExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; @@ -167,6 +167,55 @@ public class OdpsExprParser extends SQLExprParser { return super.relationalRest(expr); } + public SQLDataType parseDataType() { + if (lexer.identifierEquals(FnvHash.Constants.ARRAY)) { + lexer.nextToken(); + accept(Token.LT); + SQLDataType itemType = parseDataType(); + accept(Token.GT); + + return new SQLArrayDataType(itemType, dbType); + } + + if (lexer.identifierEquals(FnvHash.Constants.MAP)) { + lexer.nextToken(); + accept(Token.LT); + + SQLDataType keyType = parseDataType(); + accept(Token.COMMA); + SQLDataType valueType = parseDataType(); + accept(Token.GT); + + return new SQLMapDataType(keyType, valueType, dbType); + } + + if (lexer.identifierEquals(FnvHash.Constants.STRUCT)) { + lexer.nextToken(); + + SQLStructDataType struct = new SQLStructDataType(dbType); + accept(Token.LT); + for (;;) { + SQLName name = this.name(); + accept(Token.COLON); + SQLDataType dataType = this.parseDataType(); + struct.addField(name, dataType); + + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } + accept(Token.GT); + return struct; +// throw new ParserException("TODO : " + lexer.info()); + } + + + return super.parseDataType(); + } + + @Override public OdpsSelectParser createSelectParser() { return new OdpsSelectParser(this); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index c214b8602b..2931da8ad2 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -6041,4 +6041,39 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return false; } + + @Override + public boolean visit(SQLArrayDataType x) { + print0(ucase ? "ARRAY<" : "array<"); + x.getComponentType().accept(this); + print('>'); + return false; + } + + @Override + public boolean visit(SQLMapDataType x) { + print0(ucase ? "MAP<" : "map<"); + x.getKeyType().accept(this); + print0(", "); + x.getValueType().accept(this); + print('>'); + return false; + } + + @Override + public boolean visit(SQLStructDataType x) { + print0(ucase ? "STRUCT<" : "struct<"); + printAndAccept(x.getFields(), ", "); + print('>'); + return false; + } + + @Override + public boolean visit(SQLStructDataType.Field x) { + x.getName().accept(this); + print(':'); + x.getDataType().accept(this); + print('>'); + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index 2dc3a7a7c3..36753d245c 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -802,4 +802,15 @@ public interface SQLASTVisitor { boolean visit(SQLExternalRecordFormat x); void endVisit(SQLExternalRecordFormat x); + + boolean visit(SQLArrayDataType x); + void endVisit(SQLArrayDataType x); + + boolean visit(SQLMapDataType x); + void endVisit(SQLMapDataType x); + + boolean visit(SQLStructDataType x); + void endVisit(SQLStructDataType x); + boolean visit(SQLStructDataType.Field x); + void endVisit(SQLStructDataType.Field x); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java index 651cfc5af5..226ecc5b04 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java @@ -1874,6 +1874,46 @@ public class SQLASTVisitorAdapter implements SQLASTVisitor { } + @Override + public boolean visit(SQLArrayDataType x) { + return true; + } + + @Override + public void endVisit(SQLArrayDataType x) { + + } + + @Override + public boolean visit(SQLMapDataType x) { + return true; + } + + @Override + public void endVisit(SQLMapDataType x) { + + } + + @Override + public boolean visit(SQLStructDataType x) { + return true; + } + + @Override + public void endVisit(SQLStructDataType x) { + + } + + @Override + public boolean visit(SQLStructDataType.Field x) { + return true; + } + + @Override + public void endVisit(SQLStructDataType.Field x) { + + } + public final boolean isEnabled(VisitorFeature feature) { return VisitorFeature.isEnabled(this.features, feature); } diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index 1e9d2e10e6..cd5430db98 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -615,6 +615,8 @@ public final class FnvHash { long SORTED = fnv1a_64_lower("SORTED"); long LIFECYCLE = fnv1a_64_lower("LIFECYCLE"); long PARTITIONS = fnv1a_64_lower("PARTITIONS"); + long ARRAY = fnv1a_64_lower("ARRAY"); + long STRUCT = fnv1a_64_lower("STRUCT"); long ROLLBACK = fnv1a_64_lower("ROLLBACK"); long SAVEPOINT = fnv1a_64_lower("SAVEPOINT"); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest6_array.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest6_array.java new file mode 100644 index 0000000000..f8119fabd1 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest6_array.java @@ -0,0 +1,91 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.odps; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class OdpsCreateTableTest6_array extends TestCase { + + public void test_select() throws Exception { + // 1095288847322 + String sql = "CREATE TABLE src_thrift\n" + + "(\n" + + " aint bigint,\n" + + " astring string,\n" + + " lint ARRAY,\n" + + " lstring ARRAY,\n" + + " --lintString ARRAY\n" + + " mStringString MAP\n" + + " --attribute Map>>,\n" + + " --unionField1 PropValueUnion,\n" + + " --unionField2 PropValueUnion,\n" + + " -- unionField3 PropValueUnion\n" + + ");";// + + + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS); + SQLStatement stmt = statementList.get(0); + + assertEquals("CREATE TABLE src_thrift (\n" + + "\taint BIGINT,\n" + + "\tastring STRING,\n" + + "\tlint ARRAY,\n" + + "\tlstring ARRAY, -- lintString ARRAY\n" + + "\tmStringString MAP -- attribute Map>>,\n" + + "\t-- unionField1 PropValueUnion,\n" + + "\t-- unionField2 PropValueUnion,\n" + + "\t-- unionField3 PropValueUnion\n" + + ");", stmt.toString()); + + assertEquals("create table src_thrift (\n" + + "\taint bigint,\n" + + "\tastring string,\n" + + "\tlint array,\n" + + "\tlstring array, -- lintString ARRAY\n" + + "\tmStringString map -- attribute Map>>,\n" + + "\t-- unionField1 PropValueUnion,\n" + + "\t-- unionField2 PropValueUnion,\n" + + "\t-- unionField3 PropValueUnion\n" + + ");", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(5, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + +// System.out.println(SQLUtils.formatOdps(sql)); + +// assertTrue(visitor.getColumns().contains(new Column("abc", "name"))); + } + + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest7_struct.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest7_struct.java new file mode 100644 index 0000000000..1af4b2562a --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsCreateTableTest7_struct.java @@ -0,0 +1,69 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.odps; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class OdpsCreateTableTest7_struct extends TestCase { + + public void test_select() throws Exception { + // 1095288847322 + String sql = "CREATE TABLE DEST3(k STRUCT, v BIGINT) STORED AS TEXTFILE;";// + + + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS); + SQLStatement stmt = statementList.get(0); + + assertEquals("CREATE TABLE DEST3 (\n" + + "\tk STRUCT, col2:STRING>>,\n" + + "\tv BIGINT\n" + + ")\n" + + "STORED AS TEXTFILE;", stmt.toString()); + + assertEquals("create table DEST3 (\n" + + "\tk struct, col2:string>>,\n" + + "\tv bigint\n" + + ")\n" + + "stored as TEXTFILE;", SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(2, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + +// System.out.println(SQLUtils.formatOdps(sql)); + +// assertTrue(visitor.getColumns().contains(new Column("abc", "name"))); + } + + +}