diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateSetItem.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateSetItem.java index b5853a6051..608fdf9d39 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateSetItem.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateSetItem.java @@ -36,8 +36,11 @@ public class SQLUpdateSetItem extends SQLObjectImpl implements SQLReplaceable { return column; } - public void setColumn(SQLExpr column) { - this.column = column; + public void setColumn(SQLExpr x) { + if (x != null) { + x.setParent(this); + } + this.column = x; } public SQLExpr getValue() { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java index 873da9b785..1e2a105e53 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java @@ -120,6 +120,41 @@ public class HiveCreateTableParser extends SQLCreateTableParser { } } + if (lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) { + lexer.nextToken(); + accept(Token.BY); + accept(Token.LPAREN); + this.exprParser.names(stmt.getClusteredBy()); + accept(Token.RPAREN); + } + + if (lexer.identifierEquals(FnvHash.Constants.SORTED)) { + lexer.nextToken(); + accept(Token.BY); + accept(Token.LPAREN); + for (; ; ) { + SQLSelectOrderByItem item = this.exprParser.parseSelectOrderByItem(); + stmt.addSortedByItem(item); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } + accept(Token.RPAREN); + } + + if (stmt.getClusteredBy().size() > 0 || stmt.getSortedBy().size() > 0) { + accept(Token.INTO); + if (lexer.token() == Token.LITERAL_INT) { + stmt.setBuckets(lexer.integerValue().intValue()); + lexer.nextToken(); + } else { + throw new ParserException("into buckets must be integer. " + lexer.info()); + } + acceptIdentifier("BUCKETS"); + } + if (lexer.identifierEquals(FnvHash.Constants.STORED)) { lexer.nextToken(); accept(Token.AS); @@ -132,7 +167,6 @@ public class HiveCreateTableParser extends SQLCreateTableParser { SQLSelect select = this.createSQLSelectParser().select(); stmt.setSelect(select); } - return stmt; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveExprParser.java index 19e1161412..3fef164517 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveExprParser.java @@ -15,9 +15,12 @@ */ package com.alibaba.druid.sql.dialect.hive.parser; +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; import com.alibaba.druid.sql.parser.Lexer; import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.SQLParserFeature; +import com.alibaba.druid.sql.parser.Token; import com.alibaba.druid.util.FnvHash; import java.util.Arrays; @@ -54,4 +57,14 @@ public class HiveExprParser extends SQLExprParser { this.aggregateFunctions = AGGREGATE_FUNCTIONS; this.aggregateFunctionHashCodes = AGGREGATE_FUNCTIONS_CODES; } + + public SQLExpr primaryRest(SQLExpr expr) { + if (lexer.token() == Token.COLONCOLON) { + lexer.nextToken(); + String propertyName = lexer.stringVal(); + lexer.nextToken(); + expr = new SQLPropertyExpr(expr, propertyName); + } + return super.primaryRest(expr); + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java index f97cd8608b..26600471b2 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java @@ -15,9 +15,11 @@ */ package com.alibaba.druid.sql.dialect.hive.parser; +import com.alibaba.druid.sql.ast.statement.SQLTableSource; import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.SQLSelectListCache; import com.alibaba.druid.sql.parser.SQLSelectParser; +import com.alibaba.druid.sql.parser.Token; public class HiveSelectParser extends SQLSelectParser { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateTableStatement.java index 306bc46382..ea0ff92397 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateTableStatement.java @@ -15,11 +15,65 @@ */ package com.alibaba.druid.sql.dialect.hive.stmt; +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; +import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; +import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; import com.alibaba.druid.util.JdbcConstants; +import java.util.ArrayList; +import java.util.List; + public class HiveCreateTableStatement extends SQLCreateTableStatement { + protected final List clusteredBy = new ArrayList(); + protected final List sortedBy = new ArrayList(); + protected int buckets; + public HiveCreateTableStatement() { this.dbType = JdbcConstants.HIVE; } + + public List getClusteredBy() { + return clusteredBy; + } + + public List getSortedBy() { + return sortedBy; + } + + public void addSortedByItem(SQLSelectOrderByItem item) { + item.setParent(this); + this.sortedBy.add(item); + } + + public int getBuckets() { + return buckets; + } + + public void setBuckets(int buckets) { + this.buckets = buckets; + } + + @Override + protected void accept0(SQLASTVisitor visitor) { + if (visitor instanceof HiveASTVisitor) { + accept0((HiveASTVisitor) visitor); + } else { + super.accept0(visitor); + } + } + + protected void accept0(HiveASTVisitor visitor) { + if (visitor.visit(this)) { + this.acceptChild(visitor, tableSource); + this.acceptChild(visitor, tableElementList); + this.acceptChild(visitor, inherits); + this.acceptChild(visitor, clusteredBy); + this.acceptChild(visitor, sortedBy); + this.acceptChild(visitor, select); + } + visitor.endVisit(this); + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitor.java index 4dc4be3b71..69df3f61ff 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitor.java @@ -15,7 +15,10 @@ */ package com.alibaba.druid.sql.dialect.hive.visitor; +import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.visitor.SQLASTVisitor; public interface HiveASTVisitor extends SQLASTVisitor { + boolean visit(HiveCreateTableStatement x); + void endVisit(HiveCreateTableStatement x); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitorAdapter.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitorAdapter.java index 619393d6cb..3ceb440c12 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitorAdapter.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitorAdapter.java @@ -15,7 +15,17 @@ */ package com.alibaba.druid.sql.dialect.hive.visitor; +import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter; public class HiveASTVisitorAdapter extends SQLASTVisitorAdapter implements HiveASTVisitor { + @Override + public boolean visit(HiveCreateTableStatement x) { + return true; + } + + @Override + public void endVisit(HiveCreateTableStatement x) { + + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java index 27e0a1d5c4..0b2b2a83d6 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java @@ -15,9 +15,12 @@ */ package com.alibaba.druid.sql.dialect.hive.visitor; +import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.ast.statement.SQLSelect; +import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; +import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; import java.util.List; @@ -34,4 +37,84 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis public HiveOutputVisitor(Appendable appender, boolean parameterized) { super(appender, parameterized); } + + + @Override + public boolean visit(HiveCreateTableStatement x) { + printCreateTable(x, true); + + return false; + } + + protected void printCreateTable(HiveCreateTableStatement x, boolean printSelect) { + print0(ucase ? "CREATE " : "create "); + + final SQLCreateTableStatement.Type tableType = x.getType(); + if (SQLCreateTableStatement.Type.GLOBAL_TEMPORARY.equals(tableType)) { + print0(ucase ? "GLOBAL TEMPORARY " : "global temporary "); + } else if (SQLCreateTableStatement.Type.LOCAL_TEMPORARY.equals(tableType)) { + print0(ucase ? "LOCAL TEMPORARY " : "local temporary "); + } + print0(ucase ? "TABLE " : "table "); + + if (x.isIfNotExiists()) { + print0(ucase ? "IF NOT EXISTS " : "if not exists "); + } + + printTableSourceExpr(x.getName()); + + printTableElements(x.getTableElementList()); + + SQLExprTableSource inherits = x.getInherits(); + if (inherits != null) { + print0(ucase ? " INHERITS (" : " inherits ("); + inherits.accept(this); + print(')'); + } + + List clusteredBy = x.getClusteredBy(); + if (clusteredBy.size() > 0) { + println(); + print0(ucase ? "CLUSTERED BY (" : "clustered by ("); + printAndAccept(clusteredBy, ","); + print(')'); + } + + List sortedBy = x.getSortedBy(); + if (sortedBy.size() > 0) { + println(); + print0(ucase ? "SORTED BY (" : "sorted by ("); + printAndAccept(sortedBy, ", "); + print(')'); + } + + int buckets = x.getBuckets(); + if (buckets > 0) { + println(); + print0(ucase ? "INTO " : "into "); + print(buckets); + print0(ucase ? " BUCKETS" : " buckets"); + } + + SQLName storedAs = x.getStoredAs(); + if (storedAs != null) { + println(); + print0(ucase ? "STORE AS " : "store as "); + printExpr(storedAs); + } + + SQLSelect select = x.getSelect(); + if (printSelect && select != null) { + println(); + print0(ucase ? "AS" : "as"); + + println(); + visit(select); + } + } + + @Override + public void endVisit(HiveCreateTableStatement x) { + + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java index 19ad187d02..a246edb231 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java @@ -24,6 +24,7 @@ import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.sql.parser.SQLCreateTableParser; import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.Token; +import com.alibaba.druid.util.FnvHash; public class OdpsCreateTableParser extends SQLCreateTableParser { @@ -137,7 +138,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { accept(Token.RPAREN); } - if (lexer.identifierEquals("CLUSTERED")) { + if (lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) { lexer.nextToken(); accept(Token.BY); accept(Token.LPAREN); @@ -145,7 +146,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { accept(Token.RPAREN); } - if (lexer.identifierEquals("SORTED")) { + if (lexer.identifierEquals(FnvHash.Constants.SORTED)) { lexer.nextToken(); accept(Token.BY); accept(Token.LPAREN); @@ -164,7 +165,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { acceptIdentifier("BUCKETS"); } - if (lexer.identifierEquals("LIFECYCLE")) { + if (lexer.identifierEquals(FnvHash.Constants.LIFECYCLE)) { lexer.nextToken(); stmt.setLifecycle(this.exprParser.expr()); } diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index 15e401dbaa..12689694a8 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -611,5 +611,8 @@ public final class FnvHash { long INT4 = fnv1a_64_lower("INT4"); long VARBIT = fnv1a_64_lower("VARBIT"); + long CLUSTERED = fnv1a_64_lower("CLUSTERED"); + long SORTED = fnv1a_64_lower("SORTED"); + long LIFECYCLE = fnv1a_64_lower("LIFECYCLE"); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_0.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_0.java new file mode 100644 index 0000000000..9404409537 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_0.java @@ -0,0 +1,53 @@ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class HiveCreateTableTest_0 extends TestCase { + public void test_select() throws Exception { + String sql = "CREATE TABLE students (name VARCHAR(64), age INT, gpa DECIMAL(3, 2))\n" + + " CLUSTERED BY (age) INTO 2 BUCKETS STORED AS ORC;";// + assertEquals("CREATE TABLE students (\n" + + "\tname VARCHAR(64),\n" + + "\tage INT,\n" + + "\tgpa DECIMAL(3, 2)\n" + + ")\n" + + "CLUSTERED BY (age)\n" + + "INTO 2 BUCKETS\n" + + "STORE AS ORC;", SQLUtils.formatHive(sql)); + assertEquals("create table students (\n" + + "\tname VARCHAR(64),\n" + + "\tage INT,\n" + + "\tgpa DECIMAL(3, 2)\n" + + ")\n" + + "clustered by (age)\n" + + "into 2 buckets\n" + + "store as ORC;", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + + assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(3, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + + assertTrue(visitor.containsColumn("students", "name")); + assertTrue(visitor.containsColumn("students", "age")); + assertTrue(visitor.containsColumn("students", "gpa")); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/h2/H2_CreateTableTest_0.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_1.java similarity index 91% rename from src/test/java/com/alibaba/druid/bvt/sql/h2/H2_CreateTableTest_0.java rename to src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_1.java index 88cecaee3a..8f37fd565c 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/h2/H2_CreateTableTest_0.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_1.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.druid.bvt.sql.h2; +package com.alibaba.druid.bvt.sql.hive; import com.alibaba.druid.sql.OracleTest; import com.alibaba.druid.sql.SQLUtils; @@ -24,7 +24,7 @@ import org.junit.Assert; import java.util.List; -public class H2_CreateTableTest_0 extends OracleTest { +public class HiveCreateTableTest_1 extends OracleTest { public void test_0() throws Exception { String sql = // @@ -45,7 +45,8 @@ public class H2_CreateTableTest_0 extends OracleTest { { String text = SQLUtils.toSQLString(stmt, JdbcConstants.HIVE); - assertEquals("CREATE TABLE IF NOT EXISTS sc001.test STORE AS parquet\n" + + assertEquals("CREATE TABLE IF NOT EXISTS sc001.test\n" + + "STORE AS parquet\n" + "AS\n" + "SELECT *\n" + "FROM sc002.test;", text);