improved hive sql parser support.

This commit is contained in:
wenshao
2017-11-13 21:11:19 +08:00
parent 8f214d54f7
commit 7a2609826a
5 changed files with 149 additions and 4 deletions
@@ -120,6 +120,37 @@ public class HiveCreateTableParser extends SQLCreateTableParser {
}
}
if (lexer.token() == Token.PARTITIONED) {
lexer.nextToken();
accept(Token.BY);
accept(Token.LPAREN);
for (;;) {
if (lexer.token() != Token.IDENTIFIER) {
throw new ParserException("expect identifier. " + lexer.info());
}
SQLColumnDefinition column = this.exprParser.parseColumn();
stmt.addPartitionColumn(column);
if (lexer.isKeepComments() && lexer.hasComment()) {
column.addAfterComment(lexer.readAndResetComments());
}
if (lexer.token() != Token.COMMA) {
break;
} else {
lexer.nextToken();
if (lexer.isKeepComments() && lexer.hasComment()) {
column.addAfterComment(lexer.readAndResetComments());
}
}
}
accept(Token.RPAREN);
}
if (lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) {
lexer.nextToken();
accept(Token.BY);
@@ -39,6 +39,7 @@ public class HiveLexer extends Lexer {
map.put("ROW", Token.ROW);
map.put("LIMIT", Token.LIMIT);
map.put("PARTITIONED", Token.PARTITIONED);
DEFAULT_HIVE_KEYWORDS = new Keywords(map);
}
@@ -17,6 +17,7 @@ package com.alibaba.druid.sql.dialect.hive.stmt;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor;
@@ -27,6 +28,7 @@ import java.util.ArrayList;
import java.util.List;
public class HiveCreateTableStatement extends SQLCreateTableStatement {
protected List<SQLColumnDefinition> partitionColumns = new ArrayList<SQLColumnDefinition>(2);
protected final List<SQLName> clusteredBy = new ArrayList<SQLName>();
protected final List<SQLSelectOrderByItem> sortedBy = new ArrayList<SQLSelectOrderByItem>();
protected int buckets;
@@ -65,6 +67,17 @@ public class HiveCreateTableStatement extends SQLCreateTableStatement {
}
}
public List<SQLColumnDefinition> getPartitionColumns() {
return partitionColumns;
}
public void addPartitionColumn(SQLColumnDefinition column) {
if (column != null) {
column.setParent(this);
}
this.partitionColumns.add(column);
}
protected void accept0(HiveASTVisitor visitor) {
if (visitor.visit(this)) {
this.acceptChild(visitor, tableSource);
@@ -16,10 +16,7 @@
package com.alibaba.druid.sql.dialect.hive.visitor;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
@@ -72,6 +69,33 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
print(')');
}
int partitionSize = x.getPartitionColumns().size();
if (partitionSize > 0) {
println();
print0(ucase ? "PARTITIONED BY (" : "partitioned by (");
this.indentCount++;
println();
for (int i = 0; i < partitionSize; ++i) {
SQLColumnDefinition column = x.getPartitionColumns().get(i);
column.accept(this);
if (i != partitionSize - 1) {
print(',');
}
if (this.isPrettyFormat() && column.hasAfterComment()) {
print(' ');
printlnComment(column.getAfterCommentsDirect());
}
if (i != partitionSize - 1) {
println();
}
}
this.indentCount--;
println();
print(')');
}
List<SQLName> clusteredBy = x.getClusteredBy();
if (clusteredBy.size() > 0) {
println();
@@ -0,0 +1,76 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.OracleTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import org.junit.Assert;
import java.util.List;
public class HiveCreateTableTest_2 extends OracleTest {
public void test_0() throws Exception {
String sql = //
"CREATE TABLE pageviews (userid VARCHAR(64), link STRING, came_from STRING)\n" +
" PARTITIONED BY (datestamp STRING) CLUSTERED BY (userid) INTO 256 BUCKETS STORED AS ORC;"; //
List<SQLStatement> statementList = SQLUtils.toStatementList(sql, JdbcConstants.HIVE);
SQLStatement stmt = statementList.get(0);
print(statementList);
Assert.assertEquals(1, statementList.size());
OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor();
stmt.accept(visitor);
{
String text = SQLUtils.toSQLString(stmt, JdbcConstants.HIVE);
assertEquals("CREATE TABLE pageviews (\n" +
"\tuserid VARCHAR(64),\n" +
"\tlink STRING,\n" +
"\tcame_from STRING\n" +
")\n" +
"PARTITIONED BY (\n" +
"\tdatestamp STRING\n" +
")\n" +
"CLUSTERED BY (userid)\n" +
"INTO 256 BUCKETS\n" +
"STORE AS ORC;", text);
}
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(3, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertEquals(0, visitor.getRelationships().size());
assertEquals(0, visitor.getOrderByColumns().size());
Assert.assertTrue(visitor.containsTable("pageviews"));
}
}