improved hive sql parser. for issue #2180

This commit is contained in:
wenshao
2018-01-30 20:08:36 +08:00
parent e3ededf08e
commit 5a74b65d73
24 changed files with 1032 additions and 80 deletions
@@ -0,0 +1,129 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.dialect.hive.ast;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLInsertInto;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class HiveInsert extends SQLInsertInto {
private boolean overwrite = false;
private List<SQLAssignItem> partitions = new ArrayList<SQLAssignItem>();
public HiveInsert() {
}
public boolean isOverwrite() {
return overwrite;
}
public void setOverwrite(boolean overwrite) {
this.overwrite = overwrite;
}
public List<SQLAssignItem> getPartitions() {
return partitions;
}
public void addPartition(SQLAssignItem partition) {
if (partition != null) {
partition.setParent(this);
}
this.partitions.add(partition);
}
public void setPartitions(List<SQLAssignItem> partitions) {
this.partitions = partitions;
}
public void cloneTo(HiveInsert x) {
cloneTo(x);
x.overwrite = overwrite;
for (SQLAssignItem item : partitions) {
x.addPartition(item.clone());
}
}
@Override
public SQLInsertInto clone() {
HiveInsert x = new HiveInsert();
cloneTo(x);
return x;
}
public SQLExprTableSource getTableSource() {
return tableSource;
}
public void setTableSource(SQLExprTableSource tableSource) {
if (tableSource != null) {
tableSource.setParent(this);
}
this.tableSource = tableSource;
}
public void setTableSource(SQLName tableName) {
this.setTableSource(new SQLExprTableSource(tableName));
}
public SQLSelect getQuery() {
return query;
}
public void setQuery(SQLSelect query) {
if (query != null) {
query.setParent(this);
}
this.query = query;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor instanceof HiveASTVisitor) {
accept0((HiveASTVisitor) visitor);
} else {
accept0((OdpsASTVisitor) visitor);
}
}
protected void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, tableSource);
acceptChild(visitor, partitions);
acceptChild(visitor, query);
}
visitor.endVisit(this);
}
protected void accept0(HiveASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, tableSource);
acceptChild(visitor, partitions);
acceptChild(visitor, query);
}
visitor.endVisit(this);
}
}
@@ -0,0 +1,97 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.dialect.hive.ast;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import com.alibaba.druid.util.JdbcConstants;
import java.util.ArrayList;
import java.util.List;
public class HiveInsertStatement extends SQLInsertStatement implements SQLStatement {
private boolean overwrite = false;
private List<SQLAssignItem> partitions = new ArrayList<SQLAssignItem>();
public HiveInsertStatement() {
dbType = JdbcConstants.HIVE;
}
@Override
public String getDbType() {
return dbType;
}
public boolean isOverwrite() {
return overwrite;
}
public void setOverwrite(boolean overwrite) {
this.overwrite = overwrite;
}
public List<SQLAssignItem> getPartitions() {
return partitions;
}
public HiveInsertStatement clone() {
HiveInsertStatement x = new HiveInsertStatement();
super.cloneTo(x);
return x;
}
public void addPartition(SQLAssignItem partition) {
if (partition != null) {
partition.setParent(this);
}
this.partitions.add(partition);
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor instanceof OdpsASTVisitor) {
accept0((OdpsASTVisitor) visitor);
} else if (visitor instanceof HiveASTVisitor) {
accept0((HiveASTVisitor) visitor);
} else {
super.accept0(visitor);
}
}
protected void accept0(HiveASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, tableSource);
acceptChild(visitor, partitions);
acceptChild(visitor, valuesList);
acceptChild(visitor, query);
}
visitor.endVisit(this);
}
protected void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, tableSource);
acceptChild(visitor, partitions);
acceptChild(visitor, valuesList);
acceptChild(visitor, query);
}
visitor.endVisit(this);
}
}
@@ -0,0 +1,73 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.dialect.hive.ast;
import com.alibaba.druid.sql.ast.SQLStatementImpl;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.hive.visitor.HiveASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import com.alibaba.druid.util.JdbcConstants;
import java.util.ArrayList;
import java.util.List;
public class HiveMultiInsertStatement extends SQLStatementImpl {
private SQLTableSource from;
private List<HiveInsert> items = new ArrayList<HiveInsert>();
public HiveMultiInsertStatement() {
dbType = JdbcConstants.HIVE;
}
public void setFrom(SQLTableSource from) {
this.from = from;
}
public SQLTableSource getFrom() {
return from;
}
public List<HiveInsert> getItems() {
return items;
}
public void addItem(HiveInsert item) {
if (item != null) {
item.setParent(this);
}
this.items.add(item);
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor instanceof HiveASTVisitor) {
accept0((HiveASTVisitor) visitor);
} else {
acceptChild(visitor, from);
acceptChild(visitor, items);
}
}
public void accept0(HiveASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, from);
acceptChild(visitor, items);
}
visitor.endVisit(this);
}
}
@@ -35,13 +35,18 @@ public class HiveLexer extends Lexer {
map.put("CONCAT", Token.CONCAT);
map.put("CONTINUE", Token.CONTINUE);
map.put("MERGE", Token.MERGE);
map.put("MATCHED", Token.MATCHED);
map.put("USING", Token.USING);
map.put("TRUE", Token.TRUE);
map.put("FALSE", Token.FALSE);
map.put("ROW", Token.ROW);
map.put("LIMIT", Token.LIMIT);
map.put("PARTITIONED", Token.PARTITIONED);
map.put("PARTITION", Token.PARTITION);
map.put("OVERWRITE", Token.OVERWRITE);
map.put("SORT", Token.SORT);
map.put("IF", Token.IF);
map.put("TRUE", Token.TRUE);
map.put("FALSE", Token.FALSE);
DEFAULT_HIVE_KEYWORDS = new Keywords(map);
}
@@ -18,11 +18,17 @@ package com.alibaba.druid.sql.dialect.hive.parser;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLReplaceStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.parser.*;
import com.alibaba.druid.util.JdbcConstants;
import java.util.List;
public class HiveStatementParser extends SQLStatementParser {
public HiveStatementParser(String sql) {
super (new HiveExprParser(sql));
@@ -76,4 +82,60 @@ public class HiveStatementParser extends SQLStatementParser {
public SQLCreateTableParser getSQLCreateTableParser() {
return new HiveCreateTableParser(this.exprParser);
}
public SQLStatement parseInsert() {
if (lexer.token() == Token.FROM) {
lexer.nextToken();
HiveMultiInsertStatement stmt = new HiveMultiInsertStatement();
if (lexer.token() == Token.IDENTIFIER) {
SQLName tableName = this.exprParser.name();
SQLExprTableSource from = new SQLExprTableSource(tableName);
stmt.setFrom(from);
if (lexer.token() == Token.IDENTIFIER) {
from.setAlias(lexer.stringVal());
lexer.nextToken();
}
} else {
accept(Token.LPAREN);
SQLSelectParser selectParser = createSQLSelectParser();
SQLSelect select = selectParser.select();
accept(Token.RPAREN);
String alias = lexer.stringVal();
accept(Token.IDENTIFIER);
SQLSubqueryTableSource from = new SQLSubqueryTableSource(select, alias);
stmt.setFrom(from);
}
for (;;) {
HiveInsert insert = parseHiveInsert();
stmt.addItem(insert);
if (lexer.token() != Token.INSERT) {
break;
}
}
return stmt;
}
return parseHiveInsertStmt();
}
public boolean parseStatementListDialect(List<SQLStatement> statementList) {
if (lexer.token() == Token.FROM) {
SQLStatement stmt = this.parseInsert();
statementList.add(stmt);
return true;
}
return false;
}
}
@@ -15,10 +15,22 @@
*/
package com.alibaba.druid.sql.dialect.hive.visitor;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public interface HiveASTVisitor extends SQLASTVisitor {
boolean visit(HiveCreateTableStatement x);
void endVisit(HiveCreateTableStatement x);
boolean visit(HiveMultiInsertStatement x);
void endVisit(HiveMultiInsertStatement x);
boolean visit(HiveInsertStatement x);
void endVisit(HiveInsertStatement x);
boolean visit(HiveInsert x);
void endVisit(HiveInsert x);
}
@@ -15,6 +15,9 @@
*/
package com.alibaba.druid.sql.dialect.hive.visitor;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter;
@@ -28,4 +31,34 @@ public class HiveASTVisitorAdapter extends SQLASTVisitorAdapter implements HiveA
public void endVisit(HiveCreateTableStatement x) {
}
@Override
public boolean visit(HiveMultiInsertStatement x) {
return true;
}
@Override
public void endVisit(HiveMultiInsertStatement x) {
}
@Override
public boolean visit(HiveInsertStatement x) {
return true;
}
@Override
public void endVisit(HiveInsertStatement x) {
}
@Override
public boolean visit(HiveInsert x) {
return true;
}
@Override
public void endVisit(HiveInsert x) {
}
}
@@ -19,6 +19,9 @@ import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
@@ -188,4 +191,140 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
return false;
}
@Override
public void endVisit(HiveMultiInsertStatement x) {
}
@Override
public boolean visit(HiveMultiInsertStatement x) {
SQLTableSource from = x.getFrom();
if (x.getFrom() != null) {
if (from instanceof SQLSubqueryTableSource) {
SQLSelect select = ((SQLSubqueryTableSource) from).getSelect();
print0(ucase ? "FROM (" : "from (");
this.indentCount++;
println();
select.accept(this);
this.indentCount--;
println();
print0(") ");
print0(x.getFrom().getAlias());
} else {
print0(ucase ? "FROM " : "from ");
from.accept(this);
}
println();
}
for (int i = 0; i < x.getItems().size(); ++i) {
HiveInsert insert = x.getItems().get(i);
if (i != 0) {
println();
}
insert.accept(this);
}
return false;
}
@Override
public void endVisit(HiveInsertStatement x) {
}
public boolean visit(HiveInsertStatement x) {
if (x.hasBeforeComment()) {
printlnComments(x.getBeforeCommentsDirect());
}
if (x.isOverwrite()) {
print0(ucase ? "INSERT OVERWRITE TABLE " : "insert overwrite table ");
} else {
print0(ucase ? "INSERT INTO TABLE " : "insert into table ");
}
x.getTableSource().accept(this);
int partitions = x.getPartitions().size();
if (partitions > 0) {
print0(ucase ? " PARTITION (" : " partition (");
for (int i = 0; i < partitions; ++i) {
if (i != 0) {
print0(", ");
}
SQLAssignItem assign = x.getPartitions().get(i);
assign.getTarget().accept(this);
if (assign.getValue() != null) {
print('=');
assign.getValue().accept(this);
}
}
print(')');
}
println();
SQLSelect select = x.getQuery();
List<SQLInsertStatement.ValuesClause> valuesList = x.getValuesList();
if (select != null) {
select.accept(this);
} else if (!valuesList.isEmpty()) {
print0(ucase ? "VALUES " : "values ");
printAndAccept(valuesList, ", ");
}
return false;
}
@Override
public boolean visit(HiveInsert x) {
if (x.hasBeforeComment()) {
printlnComments(x.getBeforeCommentsDirect());
}
if (x.isOverwrite()) {
print0(ucase ? "INSERT OVERWRITE TABLE " : "insert overwrite table ");
} else {
print0(ucase ? "INSERT INTO TABLE " : "insert into table ");
}
x.getTableSource().accept(this);
int partitions = x.getPartitions().size();
if (partitions > 0) {
print0(ucase ? " PARTITION (" : " partition (");
for (int i = 0; i < partitions; ++i) {
if (i != 0) {
print0(", ");
}
SQLAssignItem assign = x.getPartitions().get(i);
assign.getTarget().accept(this);
if (assign.getValue() != null) {
print('=');
assign.getValue().accept(this);
}
}
print(')');
}
println();
SQLSelect select = x.getQuery();
List<SQLInsertStatement.ValuesClause> valuesList = x.getValuesList();
if (select != null) {
select.accept(this);
} else if (!valuesList.isEmpty()) {
print0(ucase ? "VALUES " : "values ");
printAndAccept(valuesList, ", ");
}
return false;
}
@Override
public void endVisit(HiveInsert x) {
}
}
@@ -15,7 +15,107 @@
*/
package com.alibaba.druid.sql.dialect.hive.visitor;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.stat.TableStat;
import com.alibaba.druid.util.JdbcConstants;
public class HiveSchemaStatVisitor extends SchemaStatVisitor implements HiveASTVisitor {
public HiveSchemaStatVisitor() {
super (JdbcConstants.HIVE);
}
@Override
public boolean visit(HiveCreateTableStatement x) {
return super.visit((SQLCreateTableStatement) x);
}
@Override
public void endVisit(HiveCreateTableStatement x) {
}
@Override
public boolean visit(HiveInsert x) {
setMode(x, TableStat.Mode.Insert);
SQLExprTableSource tableSource = x.getTableSource();
SQLExpr tableName = tableSource.getExpr();
if (tableName instanceof SQLName) {
TableStat stat = getTableStat((SQLName) tableName);
stat.incrementInsertCount();
}
for (SQLAssignItem partition : x.getPartitions()) {
partition.accept(this);
}
accept(x.getQuery());
return false;
}
@Override
public void endVisit(HiveInsert x) {
}
@Override
public void endVisit(HiveMultiInsertStatement x) {
}
@Override
public boolean visit(HiveMultiInsertStatement x) {
if (repository != null
&& x.getParent() == null) {
repository.resolve(x);
}
return true;
}
@Override
public void endVisit(HiveInsertStatement x) {
}
@Override
public boolean visit(HiveInsertStatement x) {
if (repository != null
&& x.getParent() == null) {
repository.resolve(x);
}
setMode(x, TableStat.Mode.Insert);
SQLExprTableSource tableSource = x.getTableSource();
SQLExpr tableName = tableSource.getExpr();
if (tableName instanceof SQLName) {
TableStat stat = getTableStat((SQLName) tableName);
stat.incrementInsertCount();
}
for (SQLAssignItem partition : x.getPartitions()) {
partition.accept(this);
}
accept(x.getQuery());
return false;
}
public class HiveSchemaStatVisitor extends SchemaStatVisitor {
}