From 361472d9f3b9d7ea9c7d52697f536c37fa74a472 Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 4 Jan 2018 15:50:01 +0800 Subject: [PATCH] improved hive parser support. --- .../hive/parser/HiveCreateTableParser.java | 20 +++++ .../hive/visitor/HiveOutputVisitor.java | 16 ++++ .../druid/sql/visitor/SchemaStatVisitor.java | 4 + .../java/com/alibaba/druid/util/FnvHash.java | 2 + .../bvt/sql/hive/HiveCreateDatabase_0.java | 63 +++++++++++++++ .../bvt/sql/hive/HiveCreateTableTest_4.java | 80 +++++++++++++++++++ 6 files changed, 185 insertions(+) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateDatabase_0.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_4.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java index e944ecd94c..a188e0cfc1 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java @@ -216,6 +216,26 @@ public class HiveCreateTableParser extends SQLCreateTableParser { stmt.setStoredAs(name); } + if (lexer.identifierEquals(FnvHash.Constants.TBLPROPERTIES)) { + lexer.nextToken(); + accept(Token.LPAREN); + + for (;;) { + String name = lexer.stringVal(); + lexer.nextToken(); + accept(Token.EQ); + SQLExpr value = this.exprParser.primary(); + stmt.getTableOptions().put(name, value); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } + + accept(Token.RPAREN); + } + if (lexer.token() == Token.AS) { lexer.nextToken(); SQLSelect select = this.createSQLSelectParser().select(); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java index 5d2ee3e63f..a9b68ae2c6 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java @@ -17,11 +17,13 @@ package com.alibaba.druid.sql.dialect.hive.visitor; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; +import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; import java.util.List; +import java.util.Map; public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVisitor { public HiveOutputVisitor(Appendable appender) { @@ -142,6 +144,20 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis printExpr(storedAs); } + Map tableOptions = x.getTableOptions(); + if (tableOptions.size() > 0) { + println(); + print0(ucase ? "TBLPROPERTIES (" : "tblproperties ("); + int i = 0; + for (Map.Entry option : tableOptions.entrySet()) { + print0(option.getKey()); + print0(" = "); + option.getValue().accept(this); + ++i; + } + print(')'); + } + SQLSelect select = x.getSelect(); if (printSelect && select != null) { println(); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java index 6c5d4a60b0..606843e32f 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java @@ -2437,4 +2437,8 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter { public boolean visit(SQLExternalRecordFormat x) { return false; } + + public boolean visit(SQLCreateDatabaseStatement x) { + return false; + } } diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index 7733b8322e..ca15da036f 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -630,5 +630,7 @@ public final class FnvHash { long NOCACHE = fnv1a_64_lower("NOCACHE"); long NOPARALLEL = fnv1a_64_lower("NOPARALLEL"); long EXIST = fnv1a_64_lower("EXIST"); + + long TBLPROPERTIES = fnv1a_64_lower("TBLPROPERTIES"); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateDatabase_0.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateDatabase_0.java new file mode 100644 index 0000000000..fe15f3ecaa --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateDatabase_0.java @@ -0,0 +1,63 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import org.junit.Assert; + +import java.util.List; + +public class HiveCreateDatabase_0 extends OracleTest { + + public void test_0() throws Exception { + String sql = // + "CREATE DATABASE merge_data;"; // + + List statementList = SQLUtils.toStatementList(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + System.out.println(stmt.toString()); + + Assert.assertEquals(1, statementList.size()); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + { + String text = SQLUtils.toSQLString(stmt, JdbcConstants.HIVE); + + assertEquals("CREATE DATABASE merge_data;", text); + } + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(0, visitor.getTables().size()); + assertEquals(0, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + assertEquals(0, visitor.getRelationships().size()); + assertEquals(0, visitor.getOrderByColumns().size()); + +// Assert.assertTrue(visitor.containsTable("merge_data.transactions")); + + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_4.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_4.java new file mode 100644 index 0000000000..d5a668dcd3 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_4.java @@ -0,0 +1,80 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import org.junit.Assert; + +import java.util.List; + +public class HiveCreateTableTest_4 extends OracleTest { + + public void test_0() throws Exception { + String sql = // + "CREATE TABLE merge_data.transactions(\n" + + "\tID int,\n" + + "\tTranValue string,\n" + + "\tlast_update_user string)\n" + + "PARTITIONED BY (tran_date string)\n" + + "CLUSTERED BY (ID) into 5 buckets \n" + + "STORED AS ORC TBLPROPERTIES ('transactional'='true');\n"; // + + List statementList = SQLUtils.toStatementList(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + System.out.println(stmt.toString()); + + Assert.assertEquals(1, statementList.size()); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + { + String text = SQLUtils.toSQLString(stmt, JdbcConstants.HIVE); + + assertEquals("CREATE TABLE merge_data.transactions (\n" + + "\tID int,\n" + + "\tTranValue string,\n" + + "\tlast_update_user string\n" + + ")\n" + + "PARTITIONED BY (\n" + + "\ttran_date string\n" + + ")\n" + + "CLUSTERED BY (ID)\n" + + "INTO 5 BUCKETS\n" + + "STORE AS ORC\n" + + "TBLPROPERTIES (transactional = 'true');", text); + } + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(3, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + assertEquals(0, visitor.getRelationships().size()); + assertEquals(0, visitor.getOrderByColumns().size()); + + Assert.assertTrue(visitor.containsTable("merge_data.transactions")); + + } +}