improved hive sql parser. for issue #2180

This commit is contained in:
wenshao
2018-01-30 20:08:36 +08:00
parent e3ededf08e
commit 5a74b65d73
24 changed files with 1032 additions and 80 deletions
@@ -1,100 +0,0 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.dialect.odps.ast;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLObjectImpl;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class OdpsInsert extends SQLObjectImpl {
private boolean overwrite = false;
private List<SQLAssignItem> partitions = new ArrayList<SQLAssignItem>();
protected SQLExprTableSource tableSource;
private SQLSelect query;
public boolean isOverwrite() {
return overwrite;
}
public void setOverwrite(boolean overwrite) {
this.overwrite = overwrite;
}
public List<SQLAssignItem> getPartitions() {
return partitions;
}
public void addPartition(SQLAssignItem partition) {
if (partition != null) {
partition.setParent(this);
}
this.partitions.add(partition);
}
public void setPartitions(List<SQLAssignItem> partitions) {
this.partitions = partitions;
}
public SQLExprTableSource getTableSource() {
return tableSource;
}
public void setTableSource(SQLExprTableSource tableSource) {
if (tableSource != null) {
tableSource.setParent(this);
}
this.tableSource = tableSource;
}
public void setTableSource(SQLName tableName) {
this.setTableSource(new SQLExprTableSource(tableName));
}
public SQLSelect getQuery() {
return query;
}
public void setQuery(SQLSelect query) {
if (query != null) {
query.setParent(this);
}
this.query = query;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
accept0((OdpsASTVisitor) visitor);
}
protected void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, tableSource);
acceptChild(visitor, partitions);
acceptChild(visitor, query);
}
visitor.endVisit(this);
}
}
@@ -18,10 +18,9 @@ package com.alibaba.druid.sql.dialect.odps.ast;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.SQLStatementImpl;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import com.alibaba.druid.util.JdbcConstants;
@@ -30,7 +29,7 @@ public class OdpsInsertStatement extends SQLStatementImpl {
private SQLTableSource from;
private List<OdpsInsert> items = new ArrayList<OdpsInsert>();
private List<HiveInsert> items = new ArrayList<HiveInsert>();
public OdpsInsertStatement() {
super (JdbcConstants.ODPS);
@@ -44,11 +43,11 @@ public class OdpsInsertStatement extends SQLStatementImpl {
return from;
}
public List<OdpsInsert> getItems() {
public List<HiveInsert> getItems() {
return items;
}
public void addItem(OdpsInsert item) {
public void addItem(HiveInsert item) {
if (item != null) {
item.setParent(this);
}
@@ -34,7 +34,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsAddStatisticStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsGrantStmt;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsListStmt;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsReadStatement;
@@ -254,7 +254,7 @@ public class OdpsStatementParser extends SQLStatementParser {
}
for (;;) {
OdpsInsert insert = parseOdpsInsert();
HiveInsert insert = parseHiveInsert();
stmt.addItem(insert);
if (lexer.token() != Token.INSERT) {
@@ -269,54 +269,6 @@ public class OdpsStatementParser extends SQLStatementParser {
return new OdpsSelectParser(this.exprParser, selectListCache);
}
public OdpsInsert parseOdpsInsert() {
OdpsInsert insert = new OdpsInsert();
if (lexer.isKeepComments() && lexer.hasComment()) {
insert.addBeforeComment(lexer.readAndResetComments());
}
SQLSelectParser selectParser = createSQLSelectParser();
accept(Token.INSERT);
if (lexer.token() == Token.INTO) {
lexer.nextToken();
} else {
accept(Token.OVERWRITE);
insert.setOverwrite(true);
}
accept(Token.TABLE);
insert.setTableSource(this.exprParser.name());
if (lexer.token() == Token.PARTITION) {
lexer.nextToken();
accept(Token.LPAREN);
for (;;) {
SQLAssignItem ptExpr = new SQLAssignItem();
ptExpr.setTarget(this.exprParser.name());
if (lexer.token() == Token.EQ) {
lexer.nextToken();
SQLExpr ptValue = this.exprParser.expr();
ptExpr.setValue(ptValue);
}
insert.addPartition(ptExpr);
if (!(lexer.token() == (Token.COMMA))) {
break;
} else {
lexer.nextToken();
}
}
accept(Token.RPAREN);
}
SQLSelect query = selectParser.select();
insert.setQuery(query);
return insert;
}
public SQLStatement parseShow() {
accept(Token.SHOW);
@@ -15,7 +15,7 @@
*/
package com.alibaba.druid.sql.dialect.odps.visitor;
import com.alibaba.druid.sql.ast.statement.SQLLateralViewTableSource;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.odps.ast.*;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
@@ -29,9 +29,9 @@ public interface OdpsASTVisitor extends SQLASTVisitor {
boolean visit(OdpsInsertStatement x);
void endVisit(OdpsInsert x);
void endVisit(HiveInsert x);
boolean visit(OdpsInsert x);
boolean visit(HiveInsert x);
void endVisit(OdpsUDTFSQLSelectItem x);
@@ -15,6 +15,10 @@
*/
package com.alibaba.druid.sql.dialect.odps.visitor;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsertStatement;
import com.alibaba.druid.sql.dialect.hive.ast.HiveMultiInsertStatement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.dialect.odps.ast.*;
import com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter;
@@ -42,12 +46,36 @@ public class OdpsASTVisitorAdapter extends SQLASTVisitorAdapter implements OdpsA
}
@Override
public void endVisit(OdpsInsert x) {
public void endVisit(HiveInsert x) {
}
public boolean visit(HiveCreateTableStatement x) {
return false;
}
public void endVisit(HiveCreateTableStatement x) {
}
public boolean visit(HiveMultiInsertStatement x) {
return false;
}
public void endVisit(HiveMultiInsertStatement x) {
}
public boolean visit(HiveInsertStatement x) {
return false;
}
public void endVisit(HiveInsertStatement x) {
}
@Override
public boolean visit(OdpsInsert x) {
public boolean visit(HiveInsert x) {
return true;
}
@@ -23,6 +23,7 @@ import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.odps.ast.*;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.JdbcConstants;
@@ -223,7 +224,7 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
}
for (int i = 0; i < x.getItems().size(); ++i) {
OdpsInsert insert = x.getItems().get(i);
HiveInsert insert = x.getItems().get(i);
if (i != 0) {
println();
}
@@ -233,12 +234,12 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
}
@Override
public void endVisit(OdpsInsert x) {
public void endVisit(HiveInsert x) {
}
@Override
public boolean visit(OdpsInsert x) {
public boolean visit(HiveInsert x) {
if (x.hasBeforeComment()) {
printlnComments(x.getBeforeCommentsDirect());
}
@@ -18,6 +18,7 @@ package com.alibaba.druid.sql.dialect.odps.visitor;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.hive.ast.HiveInsert;
import com.alibaba.druid.sql.dialect.odps.ast.*;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.stat.TableStat;
@@ -54,12 +55,12 @@ public class OdpsSchemaStatVisitor extends SchemaStatVisitor implements OdpsASTV
}
@Override
public void endVisit(OdpsInsert x) {
public void endVisit(HiveInsert x) {
}
@Override
public boolean visit(OdpsInsert x) {
public boolean visit(HiveInsert x) {
setMode(x, TableStat.Mode.Insert);
SQLExprTableSource tableSource = x.getTableSource();