improved hive sql parser support.

src/main/java/com/alibaba/druid/sql/ast/statement/SQLUpdateSetItem.java

@@ -36,8 +36,11 @@ public class SQLUpdateSetItem extends SQLObjectImpl implements SQLReplaceable {
         return column;
     }
 
-    public void setColumn(SQLExpr column) {
-        this.column = column;
+    public void setColumn(SQLExpr x) {
+        if (x != null) {
+            x.setParent(this);
+        }
+        this.column = x;
     }
 
     public SQLExpr getValue() {

src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java

@@ -120,6 +120,41 @@ public class HiveCreateTableParser extends SQLCreateTableParser {
             }
         }
 
+        if (lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) {
+            lexer.nextToken();
+            accept(Token.BY);
+            accept(Token.LPAREN);
+            this.exprParser.names(stmt.getClusteredBy());
+            accept(Token.RPAREN);
+        }
+
+        if (lexer.identifierEquals(FnvHash.Constants.SORTED)) {
+            lexer.nextToken();
+            accept(Token.BY);
+            accept(Token.LPAREN);
+            for (; ; ) {
+                SQLSelectOrderByItem item = this.exprParser.parseSelectOrderByItem();
+                stmt.addSortedByItem(item);
+                if (lexer.token() == Token.COMMA) {
+                    lexer.nextToken();
+                    continue;
+                }
+                break;
+            }
+            accept(Token.RPAREN);
+        }
+
+        if (stmt.getClusteredBy().size() > 0 || stmt.getSortedBy().size() > 0) {
+            accept(Token.INTO);
+            if (lexer.token() == Token.LITERAL_INT) {
+                stmt.setBuckets(lexer.integerValue().intValue());
+                lexer.nextToken();
+            } else {
+                throw new ParserException("into buckets must be integer. " + lexer.info());
+            }
+            acceptIdentifier("BUCKETS");
+        }
+
         if (lexer.identifierEquals(FnvHash.Constants.STORED)) {
             lexer.nextToken();
             accept(Token.AS);
@@ -132,7 +167,6 @@ public class HiveCreateTableParser extends SQLCreateTableParser {
             SQLSelect select = this.createSQLSelectParser().select();
             stmt.setSelect(select);
         }
-
         return stmt;
     }
 

src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveExprParser.java

@@ -15,9 +15,12 @@
  */
 package com.alibaba.druid.sql.dialect.hive.parser;
 
+import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
 import com.alibaba.druid.sql.parser.Lexer;
 import com.alibaba.druid.sql.parser.SQLExprParser;
 import com.alibaba.druid.sql.parser.SQLParserFeature;
+import com.alibaba.druid.sql.parser.Token;
 import com.alibaba.druid.util.FnvHash;
 
 import java.util.Arrays;
@@ -54,4 +57,14 @@ public class HiveExprParser extends SQLExprParser {
         this.aggregateFunctions = AGGREGATE_FUNCTIONS;
         this.aggregateFunctionHashCodes = AGGREGATE_FUNCTIONS_CODES;
     }
+
+    public SQLExpr primaryRest(SQLExpr expr) {
+        if (lexer.token() == Token.COLONCOLON) {
+            lexer.nextToken();
+            String propertyName = lexer.stringVal();
+            lexer.nextToken();
+            expr = new SQLPropertyExpr(expr, propertyName);
+        }
+        return super.primaryRest(expr);
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java

@@ -15,9 +15,11 @@
  */
 package com.alibaba.druid.sql.dialect.hive.parser;
 
+import com.alibaba.druid.sql.ast.statement.SQLTableSource;
 import com.alibaba.druid.sql.parser.SQLExprParser;
 import com.alibaba.druid.sql.parser.SQLSelectListCache;
 import com.alibaba.druid.sql.parser.SQLSelectParser;
+import com.alibaba.druid.sql.parser.Token;
 
 public class HiveSelectParser extends SQLSelectParser {
 

src/main/java/com/alibaba/druid/sql/dialect/hive/stmt/HiveCreateTableStatement.java

@@ -15,11 +15,65 @@
  */
 package com.alibaba.druid.sql.dialect.hive.stmt;
 
+import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.sql.ast.SQLName;
 import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
+import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
+import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor;
+import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 import com.alibaba.druid.util.JdbcConstants;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class HiveCreateTableStatement extends SQLCreateTableStatement {
+    protected final List<SQLName> clusteredBy = new ArrayList<SQLName>();
+    protected final List<SQLSelectOrderByItem> sortedBy = new ArrayList<SQLSelectOrderByItem>();
+    protected int buckets;
+
     public HiveCreateTableStatement() {
         this.dbType = JdbcConstants.HIVE;
     }
+
+    public List<SQLName> getClusteredBy() {
+        return clusteredBy;
+    }
+
+    public List<SQLSelectOrderByItem> getSortedBy() {
+        return sortedBy;
+    }
+
+    public void addSortedByItem(SQLSelectOrderByItem item) {
+        item.setParent(this);
+        this.sortedBy.add(item);
+    }
+
+    public int getBuckets() {
+        return buckets;
+    }
+
+    public void setBuckets(int buckets) {
+        this.buckets = buckets;
+    }
+
+    @Override
+    protected void accept0(SQLASTVisitor visitor) {
+        if (visitor instanceof HiveASTVisitor) {
+            accept0((HiveASTVisitor) visitor);
+        } else {
+            super.accept0(visitor);
+        }
+    }
+
+    protected void accept0(HiveASTVisitor visitor) {
+        if (visitor.visit(this)) {
+            this.acceptChild(visitor, tableSource);
+            this.acceptChild(visitor, tableElementList);
+            this.acceptChild(visitor, inherits);
+            this.acceptChild(visitor, clusteredBy);
+            this.acceptChild(visitor, sortedBy);
+            this.acceptChild(visitor, select);
+        }
+        visitor.endVisit(this);
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitor.java

@@ -15,7 +15,10 @@
  */
 package com.alibaba.druid.sql.dialect.hive.visitor;
 
+import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 
 public interface HiveASTVisitor extends SQLASTVisitor {
+    boolean visit(HiveCreateTableStatement x);
+    void endVisit(HiveCreateTableStatement x);
 }

src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveASTVisitorAdapter.java

@@ -15,7 +15,17 @@
  */
 package com.alibaba.druid.sql.dialect.hive.visitor;
 
+import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
 import com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter;
 
 public class HiveASTVisitorAdapter extends SQLASTVisitorAdapter implements HiveASTVisitor {
+    @Override
+    public boolean visit(HiveCreateTableStatement x) {
+        return true;
+    }
+
+    @Override
+    public void endVisit(HiveCreateTableStatement x) {
+
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java

@@ -15,9 +15,12 @@
  */
 package com.alibaba.druid.sql.dialect.hive.visitor;
 
+import com.alibaba.druid.sql.ast.SQLName;
 import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
 import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
 import com.alibaba.druid.sql.ast.statement.SQLSelect;
+import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
+import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
 import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
 
 import java.util.List;
@@ -34,4 +37,84 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
     public HiveOutputVisitor(Appendable appender, boolean parameterized) {
         super(appender, parameterized);
     }
+
+
+    @Override
+    public boolean visit(HiveCreateTableStatement x) {
+        printCreateTable(x, true);
+
+        return false;
+    }
+
+    protected void printCreateTable(HiveCreateTableStatement x, boolean printSelect) {
+        print0(ucase ? "CREATE " : "create ");
+
+        final SQLCreateTableStatement.Type tableType = x.getType();
+        if (SQLCreateTableStatement.Type.GLOBAL_TEMPORARY.equals(tableType)) {
+            print0(ucase ? "GLOBAL TEMPORARY " : "global temporary ");
+        } else if (SQLCreateTableStatement.Type.LOCAL_TEMPORARY.equals(tableType)) {
+            print0(ucase ? "LOCAL TEMPORARY " : "local temporary ");
+        }
+        print0(ucase ? "TABLE " : "table ");
+
+        if (x.isIfNotExiists()) {
+            print0(ucase ? "IF NOT EXISTS " : "if not exists ");
+        }
+
+        printTableSourceExpr(x.getName());
+
+        printTableElements(x.getTableElementList());
+
+        SQLExprTableSource inherits = x.getInherits();
+        if (inherits != null) {
+            print0(ucase ? " INHERITS (" : " inherits (");
+            inherits.accept(this);
+            print(')');
+        }
+
+        List<SQLName> clusteredBy = x.getClusteredBy();
+        if (clusteredBy.size() > 0) {
+            println();
+            print0(ucase ? "CLUSTERED BY (" : "clustered by (");
+            printAndAccept(clusteredBy, ",");
+            print(')');
+        }
+
+        List<SQLSelectOrderByItem> sortedBy = x.getSortedBy();
+        if (sortedBy.size() > 0) {
+            println();
+            print0(ucase ? "SORTED BY (" : "sorted by (");
+            printAndAccept(sortedBy, ", ");
+            print(')');
+        }
+
+        int buckets = x.getBuckets();
+        if (buckets > 0) {
+            println();
+            print0(ucase ? "INTO " : "into ");
+            print(buckets);
+            print0(ucase ? " BUCKETS" : " buckets");
+        }
+
+        SQLName storedAs = x.getStoredAs();
+        if (storedAs != null) {
+            println();
+            print0(ucase ? "STORE AS " : "store as ");
+            printExpr(storedAs);
+        }
+
+        SQLSelect select = x.getSelect();
+        if (printSelect && select != null) {
+            println();
+            print0(ucase ? "AS" : "as");
+
+            println();
+            visit(select);
+        }
+    }
+
+    @Override
+    public void endVisit(HiveCreateTableStatement x) {
+
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java

@@ -24,6 +24,7 @@ import com.alibaba.druid.sql.parser.ParserException;
 import com.alibaba.druid.sql.parser.SQLCreateTableParser;
 import com.alibaba.druid.sql.parser.SQLExprParser;
 import com.alibaba.druid.sql.parser.Token;
+import com.alibaba.druid.util.FnvHash;
 
 public class OdpsCreateTableParser extends SQLCreateTableParser {
 
@@ -137,7 +138,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
             accept(Token.RPAREN);
         }
 
-        if (lexer.identifierEquals("CLUSTERED")) {
+        if (lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) {
             lexer.nextToken();
             accept(Token.BY);
             accept(Token.LPAREN);
@@ -145,7 +146,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
             accept(Token.RPAREN);
         }
 
-        if (lexer.identifierEquals("SORTED")) {
+        if (lexer.identifierEquals(FnvHash.Constants.SORTED)) {
             lexer.nextToken();
             accept(Token.BY);
             accept(Token.LPAREN);
@@ -164,7 +165,7 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
             acceptIdentifier("BUCKETS");
         }
         
-        if (lexer.identifierEquals("LIFECYCLE")) {
+        if (lexer.identifierEquals(FnvHash.Constants.LIFECYCLE)) {
             lexer.nextToken();
             stmt.setLifecycle(this.exprParser.expr());
         }

src/main/java/com/alibaba/druid/util/FnvHash.java

@@ -611,5 +611,8 @@ public final class FnvHash {
 
         long INT4 = fnv1a_64_lower("INT4");
         long VARBIT = fnv1a_64_lower("VARBIT");
+        long CLUSTERED = fnv1a_64_lower("CLUSTERED");
+        long SORTED = fnv1a_64_lower("SORTED");
+        long LIFECYCLE = fnv1a_64_lower("LIFECYCLE");
     }
 }

src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_0.java

@@ -0,0 +1,53 @@
+package com.alibaba.druid.bvt.sql.hive;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+import com.alibaba.druid.util.JdbcConstants;
+import junit.framework.TestCase;
+
+import java.util.List;
+
+public class HiveCreateTableTest_0 extends TestCase {
+    public void test_select() throws Exception {
+        String sql = "CREATE TABLE students (name VARCHAR(64), age INT, gpa DECIMAL(3, 2))\n" +
+                "  CLUSTERED BY (age) INTO 2 BUCKETS STORED AS ORC;";//
+        assertEquals("CREATE TABLE students (\n" +
+                "\tname VARCHAR(64),\n" +
+                "\tage INT,\n" +
+                "\tgpa DECIMAL(3, 2)\n" +
+                ")\n" +
+                "CLUSTERED BY (age)\n" +
+                "INTO 2 BUCKETS\n" +
+                "STORE AS ORC;", SQLUtils.formatHive(sql));
+        assertEquals("create table students (\n" +
+                "\tname VARCHAR(64),\n" +
+                "\tage INT,\n" +
+                "\tgpa DECIMAL(3, 2)\n" +
+                ")\n" +
+                "clustered by (age)\n" +
+                "into 2 buckets\n" +
+                "store as ORC;", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
+
+        List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
+        SQLStatement stmt = statementList.get(0);
+
+        assertEquals(1, statementList.size());
+
+        SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
+        stmt.accept(visitor);
+
+        System.out.println("Tables : " + visitor.getTables());
+      System.out.println("fields : " + visitor.getColumns());
+//      System.out.println("coditions : " + visitor.getConditions());
+//      System.out.println("orderBy : " + visitor.getOrderByColumns());
+
+        assertEquals(1, visitor.getTables().size());
+        assertEquals(3, visitor.getColumns().size());
+        assertEquals(0, visitor.getConditions().size());
+
+        assertTrue(visitor.containsColumn("students", "name"));
+        assertTrue(visitor.containsColumn("students", "age"));
+        assertTrue(visitor.containsColumn("students", "gpa"));
+    }
+}

src/test/java/com/alibaba/druid/bvt/sql/hive/HiveCreateTableTest_1.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.alibaba.druid.bvt.sql.h2;
+package com.alibaba.druid.bvt.sql.hive;
 
 import com.alibaba.druid.sql.OracleTest;
 import com.alibaba.druid.sql.SQLUtils;
@@ -24,7 +24,7 @@ import org.junit.Assert;
 
 import java.util.List;
 
-public class H2_CreateTableTest_0 extends OracleTest {
+public class HiveCreateTableTest_1 extends OracleTest {
 
     public void test_0() throws Exception {
         String sql = //
@@ -45,7 +45,8 @@ public class H2_CreateTableTest_0 extends OracleTest {
         {
             String text = SQLUtils.toSQLString(stmt, JdbcConstants.HIVE);
 
-            assertEquals("CREATE TABLE IF NOT EXISTS sc001.test STORE AS parquet\n" +
+            assertEquals("CREATE TABLE IF NOT EXISTS sc001.test\n" +
+                    "STORE AS parquet\n" +
                     "AS\n" +
                     "SELECT *\n" +
                     "FROM sc002.test;", text);