mirror of
https://github.com/alibaba/druid.git
synced 2023-03-04 19:17:16 +00:00
improved hive sql parser support.
This commit is contained in:
@@ -120,6 +120,37 @@ public class HiveCreateTableParser extends SQLCreateTableParser {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (lexer.token() == Token.PARTITIONED) {
|
||||
lexer.nextToken();
|
||||
accept(Token.BY);
|
||||
accept(Token.LPAREN);
|
||||
|
||||
for (;;) {
|
||||
if (lexer.token() != Token.IDENTIFIER) {
|
||||
throw new ParserException("expect identifier. " + lexer.info());
|
||||
}
|
||||
|
||||
SQLColumnDefinition column = this.exprParser.parseColumn();
|
||||
stmt.addPartitionColumn(column);
|
||||
|
||||
if (lexer.isKeepComments() && lexer.hasComment()) {
|
||||
column.addAfterComment(lexer.readAndResetComments());
|
||||
}
|
||||
|
||||
if (lexer.token() != Token.COMMA) {
|
||||
break;
|
||||
} else {
|
||||
lexer.nextToken();
|
||||
if (lexer.isKeepComments() && lexer.hasComment()) {
|
||||
column.addAfterComment(lexer.readAndResetComments());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
accept(Token.RPAREN);
|
||||
}
|
||||
|
||||
if (lexer.identifierEquals(FnvHash.Constants.CLUSTERED)) {
|
||||
lexer.nextToken();
|
||||
accept(Token.BY);
|
||||
|
||||
@@ -39,6 +39,7 @@ public class HiveLexer extends Lexer {
|
||||
|
||||
map.put("ROW", Token.ROW);
|
||||
map.put("LIMIT", Token.LIMIT);
|
||||
map.put("PARTITIONED", Token.PARTITIONED);
|
||||
|
||||
DEFAULT_HIVE_KEYWORDS = new Keywords(map);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ package com.alibaba.druid.sql.dialect.hive.stmt;
|
||||
|
||||
import com.alibaba.druid.sql.ast.SQLExpr;
|
||||
import com.alibaba.druid.sql.ast.SQLName;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
|
||||
import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor;
|
||||
@@ -27,6 +28,7 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class HiveCreateTableStatement extends SQLCreateTableStatement {
|
||||
protected List<SQLColumnDefinition> partitionColumns = new ArrayList<SQLColumnDefinition>(2);
|
||||
protected final List<SQLName> clusteredBy = new ArrayList<SQLName>();
|
||||
protected final List<SQLSelectOrderByItem> sortedBy = new ArrayList<SQLSelectOrderByItem>();
|
||||
protected int buckets;
|
||||
@@ -65,6 +67,17 @@ public class HiveCreateTableStatement extends SQLCreateTableStatement {
|
||||
}
|
||||
}
|
||||
|
||||
public List<SQLColumnDefinition> getPartitionColumns() {
|
||||
return partitionColumns;
|
||||
}
|
||||
|
||||
public void addPartitionColumn(SQLColumnDefinition column) {
|
||||
if (column != null) {
|
||||
column.setParent(this);
|
||||
}
|
||||
this.partitionColumns.add(column);
|
||||
}
|
||||
|
||||
protected void accept0(HiveASTVisitor visitor) {
|
||||
if (visitor.visit(this)) {
|
||||
this.acceptChild(visitor, tableSource);
|
||||
|
||||
@@ -16,10 +16,7 @@
|
||||
package com.alibaba.druid.sql.dialect.hive.visitor;
|
||||
|
||||
import com.alibaba.druid.sql.ast.SQLName;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLSelect;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
|
||||
import com.alibaba.druid.sql.ast.statement.*;
|
||||
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
|
||||
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
|
||||
|
||||
@@ -72,6 +69,33 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
|
||||
print(')');
|
||||
}
|
||||
|
||||
int partitionSize = x.getPartitionColumns().size();
|
||||
if (partitionSize > 0) {
|
||||
println();
|
||||
print0(ucase ? "PARTITIONED BY (" : "partitioned by (");
|
||||
this.indentCount++;
|
||||
println();
|
||||
for (int i = 0; i < partitionSize; ++i) {
|
||||
SQLColumnDefinition column = x.getPartitionColumns().get(i);
|
||||
column.accept(this);
|
||||
|
||||
if (i != partitionSize - 1) {
|
||||
print(',');
|
||||
}
|
||||
if (this.isPrettyFormat() && column.hasAfterComment()) {
|
||||
print(' ');
|
||||
printlnComment(column.getAfterCommentsDirect());
|
||||
}
|
||||
|
||||
if (i != partitionSize - 1) {
|
||||
println();
|
||||
}
|
||||
}
|
||||
this.indentCount--;
|
||||
println();
|
||||
print(')');
|
||||
}
|
||||
|
||||
List<SQLName> clusteredBy = x.getClusteredBy();
|
||||
if (clusteredBy.size() > 0) {
|
||||
println();
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright 1999-2017 Alibaba Group Holding Ltd.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.alibaba.druid.bvt.sql.hive;
|
||||
|
||||
import com.alibaba.druid.sql.OracleTest;
|
||||
import com.alibaba.druid.sql.SQLUtils;
|
||||
import com.alibaba.druid.sql.ast.SQLStatement;
|
||||
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor;
|
||||
import com.alibaba.druid.util.JdbcConstants;
|
||||
import org.junit.Assert;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class HiveCreateTableTest_2 extends OracleTest {
|
||||
|
||||
public void test_0() throws Exception {
|
||||
String sql = //
|
||||
"CREATE TABLE pageviews (userid VARCHAR(64), link STRING, came_from STRING)\n" +
|
||||
" PARTITIONED BY (datestamp STRING) CLUSTERED BY (userid) INTO 256 BUCKETS STORED AS ORC;"; //
|
||||
|
||||
List<SQLStatement> statementList = SQLUtils.toStatementList(sql, JdbcConstants.HIVE);
|
||||
SQLStatement stmt = statementList.get(0);
|
||||
print(statementList);
|
||||
|
||||
|
||||
|
||||
Assert.assertEquals(1, statementList.size());
|
||||
|
||||
OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor();
|
||||
stmt.accept(visitor);
|
||||
|
||||
{
|
||||
String text = SQLUtils.toSQLString(stmt, JdbcConstants.HIVE);
|
||||
|
||||
assertEquals("CREATE TABLE pageviews (\n" +
|
||||
"\tuserid VARCHAR(64),\n" +
|
||||
"\tlink STRING,\n" +
|
||||
"\tcame_from STRING\n" +
|
||||
")\n" +
|
||||
"PARTITIONED BY (\n" +
|
||||
"\tdatestamp STRING\n" +
|
||||
")\n" +
|
||||
"CLUSTERED BY (userid)\n" +
|
||||
"INTO 256 BUCKETS\n" +
|
||||
"STORE AS ORC;", text);
|
||||
}
|
||||
|
||||
System.out.println("Tables : " + visitor.getTables());
|
||||
System.out.println("fields : " + visitor.getColumns());
|
||||
System.out.println("coditions : " + visitor.getConditions());
|
||||
System.out.println("relationships : " + visitor.getRelationships());
|
||||
System.out.println("orderBy : " + visitor.getOrderByColumns());
|
||||
|
||||
assertEquals(1, visitor.getTables().size());
|
||||
assertEquals(3, visitor.getColumns().size());
|
||||
assertEquals(0, visitor.getConditions().size());
|
||||
assertEquals(0, visitor.getRelationships().size());
|
||||
assertEquals(0, visitor.getOrderByColumns().size());
|
||||
|
||||
Assert.assertTrue(visitor.containsTable("pageviews"));
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user