From 316f59e3d2db806cbf9ca1bf2f6c2524a9b56140 Mon Sep 17 00:00:00 2001 From: wenshao Date: Sat, 2 Dec 2017 13:10:02 +0800 Subject: [PATCH] improved odps & hive sql parser support. --- .../odps/ast/OdpsCreateTableStatement.java | 13 +++ .../odps/parser/OdpsCreateTableParser.java | 14 ++++ .../odps/visitor/OdpsOutputVisitor.java | 14 ++++ .../bvt/sql/hive/HiveCreateTableTest_3.java | 82 +++++++++++++++++++ 4 files changed, 123 insertions(+) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_3.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCreateTableStatement.java index a33624693e..933a779a7e 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCreateTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsCreateTableStatement.java @@ -31,6 +31,7 @@ public class OdpsCreateTableStatement extends SQLCreateTableStatement { private SQLExprTableSource like; + protected SQLExpr storedBy; protected SQLExpr lifecycle; public OdpsCreateTableStatement(){ @@ -69,9 +70,21 @@ public class OdpsCreateTableStatement extends SQLCreateTableStatement { this.acceptChild(visitor, partitionColumns); this.acceptChild(visitor, clusteredBy); this.acceptChild(visitor, sortedBy); + this.acceptChild(visitor, storedBy); this.acceptChild(visitor, lifecycle); this.acceptChild(visitor, select); } visitor.endVisit(this); } + + public SQLExpr getStoredBy() { + return storedBy; + } + + public void setStoredBy(SQLExpr x) { + if (x != null) { + x.setParent(this); + } + this.storedBy = x; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java index 9382d1f851..30aaba85d0 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java @@ -15,6 +15,7 @@ */ package com.alibaba.druid.sql.dialect.odps.parser; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition; import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; @@ -178,6 +179,19 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { lexer.nextToken(); stmt.setLifecycle(this.exprParser.expr()); } + + while (lexer.identifierEquals(FnvHash.Constants.STORED)) { + lexer.nextToken(); + if (lexer.token() == Token.AS) { + lexer.nextToken(); + SQLName storedAs = this.exprParser.name(); + stmt.setStoredAs(storedAs); + } else { + accept(Token.BY); + SQLExpr storedBy = this.exprParser.expr(); + stmt.setStoredBy(storedBy); + } + } return stmt; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index afda1ccfaa..92a8134764 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -161,6 +161,20 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis x.getLifecycle().accept(this); } + SQLExpr storedBy = x.getStoredBy(); + if (storedBy != null) { + println(); + print0(ucase ? "STORED BY " : "stored by "); + storedBy.accept(this); + } + + SQLExpr storedAs = x.getStoredAs(); + if (storedAs != null) { + println(); + print0(ucase ? "STORED AS " : "stored as "); + storedAs.accept(this); + } + if (x.getSelect() != null) { println(); print0(ucase ? "AS" : "as"); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_3.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_3.java new file mode 100644 index 0000000000..27a5160324 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_3.java @@ -0,0 +1,82 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import org.junit.Assert; + +import java.util.List; + +public class HiveCreateTableTest_3 extends OracleTest { + + public void test_0() throws Exception { + String sql = // + "CREATE TABLE pub_ae_sess_type_dimt0_text\n" + + "( \n" + + " SESSION_TYPE_ID BIGINT COMMENT '会话类型',\n" + + " DW_INS_DATE STRING COMMENT '数据仓库插入时间' \n" + + ")\n" + + "COMMENT 'session来源类型维表'\n" + + "PARTITIONED BY(date STRING COMMENT '日期', pos STRING COMMENT '位置')\n" + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','\n" + + "STORED AS SEQUENCEFILE"; // + + List statementList = SQLUtils.toStatementList(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + System.out.println(stmt.toString()); + + Assert.assertEquals(1, statementList.size()); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + { + String text = SQLUtils.toSQLString(stmt, JdbcConstants.HIVE); + + assertEquals("CREATE TABLE pub_ae_sess_type_dimt0_text (\n" + + "\tSESSION_TYPE_ID BIGINT COMMENT '会话类型',\n" + + "\tDW_INS_DATE STRING COMMENT '数据仓库插入时间'\n" + + ")\n" + + "COMMENT 'session来源类型维表'\n" + + "PARTITIONED BY (\n" + + "\tdate STRING COMMENT '日期',\n" + + "\tpos STRING COMMENT '位置'\n" + + ")\n" + + "ROW FORMAT DELIMITED \n" + + "FIELDS TERMINATED BY ','\n" + + "STORE AS SEQUENCEFILE", text); + } + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(2, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + assertEquals(0, visitor.getRelationships().size()); + assertEquals(0, visitor.getOrderByColumns().size()); + + Assert.assertTrue(visitor.containsTable("pub_ae_sess_type_dimt0_text")); + + } +}