improved hive sql parser

This commit is contained in:
高铁
2019-11-02 22:46:16 +08:00
parent 481a0c8380
commit 1cdd3f28eb
20 changed files with 669 additions and 36 deletions
@@ -72,7 +72,7 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS
// for hive & odps
protected SQLExternalRecordFormat rowFormat;
protected final List<SQLColumnDefinition> partitionColumns = new ArrayList<SQLColumnDefinition>(2);
protected final List<SQLName> clusteredBy = new ArrayList<SQLName>();
protected final List<SQLSelectOrderByItem> clusteredBy = new ArrayList<SQLSelectOrderByItem>();
protected final List<SQLSelectOrderByItem> sortedBy = new ArrayList<SQLSelectOrderByItem>();
protected int buckets;
@@ -1074,7 +1074,7 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS
this.onCommitPreserveRows = onCommitPreserveRows;
}
public List<SQLName> getClusteredBy() {
public List<SQLSelectOrderByItem> getClusteredBy() {
return clusteredBy;
}
@@ -1125,4 +1125,9 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS
return pk.containsColumn(columnNameHash);
}
public void addClusteredByItem(SQLSelectOrderByItem item) {
item.setParent(this);
this.clusteredBy.add(item);
}
}
@@ -21,6 +21,8 @@ import com.alibaba.druid.sql.ast.SQLOrderingSpecification;
import com.alibaba.druid.sql.ast.SQLReplaceable;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import java.util.List;
public final class SQLSelectOrderByItem extends SQLObjectImpl implements SQLReplaceable {
protected SQLExpr expr;
@@ -155,4 +157,41 @@ public final class SQLSelectOrderByItem extends SQLObjectImpl implements SQLRepl
public void setResolvedSelectItem(SQLSelectItem resolvedSelectItem) {
this.resolvedSelectItem = resolvedSelectItem;
}
public boolean isClusterBy() {
if (parent instanceof SQLCreateTableStatement) {
List<SQLSelectOrderByItem> clusteredBy = ((SQLCreateTableStatement) parent).getClusteredBy();
return clusteredBy.indexOf(this) != -1;
}
if (parent instanceof SQLSelectQueryBlock) {
List<SQLSelectOrderByItem> clusterBy = ((SQLSelectQueryBlock) parent).getClusterByDirect();
return clusterBy != null && clusterBy.indexOf(this) != -1;
}
return false;
}
public boolean isSortBy() {
if (parent instanceof SQLCreateTableStatement) {
List<SQLSelectOrderByItem> sortedBy = ((SQLCreateTableStatement) parent).getSortedBy();
return sortedBy.indexOf(this) != -1;
}
if (parent instanceof SQLSelectQueryBlock) {
List<SQLSelectOrderByItem> sortedBy = ((SQLSelectQueryBlock) parent).getSortByDirect();
return sortedBy != null && sortedBy.indexOf(this) != -1;
}
return false;
}
public boolean isDistributeBy() {
if (parent instanceof SQLSelectQueryBlock) {
List<SQLSelectOrderByItem> distributeBy = ((SQLSelectQueryBlock) parent).getDistributeBy();
return distributeBy.indexOf(this) != -1;
}
return false;
}
}
@@ -176,7 +176,15 @@ public class HiveCreateTableParser extends SQLCreateTableParser {
lexer.nextToken();
accept(Token.BY);
accept(Token.LPAREN);
this.exprParser.names(stmt.getClusteredBy());
for (; ; ) {
SQLSelectOrderByItem item = this.exprParser.parseSelectOrderByItem();
stmt.addClusteredByItem(item);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
}
break;
}
accept(Token.RPAREN);
}
@@ -48,34 +48,6 @@ public class HiveSelectParser extends SQLSelectParser {
return new HiveExprParser(lexer);
}
protected void parseSortBy(SQLSelectQueryBlock queryBlock) {
if (lexer.token() == Token.SORT) {
lexer.nextToken();
accept(Token.BY);
for (;;) {
SQLExpr expr = this.expr();
SQLSelectOrderByItem sortByItem = new SQLSelectOrderByItem(expr);
if (lexer.token() == Token.ASC) {
sortByItem.setType(SQLOrderingSpecification.ASC);
lexer.nextToken();
} else if (lexer.token() == Token.DESC) {
sortByItem.setType(SQLOrderingSpecification.DESC);
lexer.nextToken();
}
queryBlock.addSortBy(sortByItem);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
} else {
break;
}
}
}
}
protected SQLTableSource parseTableSourceRest(SQLTableSource tableSource) {
if (lexer.identifierEquals(FnvHash.Constants.TABLESAMPLE) && tableSource instanceof SQLExprTableSource) {
Lexer.SavePoint mark = lexer.mark();
@@ -118,7 +118,7 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
print(')');
}
List<SQLName> clusteredBy = x.getClusteredBy();
List<SQLSelectOrderByItem> clusteredBy = x.getClusteredBy();
if (clusteredBy.size() > 0) {
println();
print0(ucase ? "CLUSTERED BY (" : "clustered by (");
@@ -144,7 +144,15 @@ public class OdpsCreateTableParser extends SQLCreateTableParser {
lexer.nextToken();
accept(Token.BY);
accept(Token.LPAREN);
this.exprParser.names(stmt.getClusteredBy());
for (; ; ) {
SQLSelectOrderByItem item = this.exprParser.parseSelectOrderByItem();
stmt.addClusteredByItem(item);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
}
break;
}
accept(Token.RPAREN);
}
@@ -162,7 +162,7 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
print(')');
}
List<SQLName> clusteredBy = x.getClusteredBy();
List<SQLSelectOrderByItem> clusteredBy = x.getClusteredBy();
if (clusteredBy.size() > 0) {
println();
print0(ucase ? "CLUSTERED BY (" : "clustered by (");
@@ -17,6 +17,7 @@ package com.alibaba.druid.sql.parser;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.util.FnvHash;
import com.alibaba.druid.util.StringUtils;
public class SQLParser {
protected final Lexer lexer;
@@ -79,16 +80,71 @@ public class SQLParser {
}
if (token == Token.IDENTIFIER) {
String ident = lexer.stringVal;
long hash = lexer.hash_lower;
if (isEnabled(SQLParserFeature.IgnoreNameQuotes) && ident.length() > 1) {
ident = StringUtils.removeNameQuotes(ident);
}
if (hash == FnvHash.Constants.START
|| hash == FnvHash.Constants.CONNECT
|| hash == FnvHash.Constants.NATURAL
|| hash == FnvHash.Constants.CROSS
|| hash == FnvHash.Constants.OFFSET
|| hash == FnvHash.Constants.LIMIT) {
if (must) {
throw new ParserException("illegal alias. " + lexer.info());
}
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
switch (lexer.token) {
case EOF:
case COMMA:
case WHERE:
case INNER:
return ident;
default:
lexer.reset(mark);
break;
}
return null;
}
if (!must) {
if (hash == FnvHash.Constants.MODEL) {
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
if (lexer.token == Token.PARTITION
|| lexer.token == Token.UNION
|| lexer.identifierEquals(FnvHash.Constants.DIMENSION)
|| lexer.identifierEquals(FnvHash.Constants.IGNORE)
|| lexer.identifierEquals(FnvHash.Constants.KEEP)) {
lexer.reset(mark);
return null;
}
return ident;
} else if (hash == FnvHash.Constants.WINDOW) {
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
if (lexer.token == Token.IDENTIFIER) {
lexer.reset(mark);
return null;
}
return ident;
} else if (hash == FnvHash.Constants.DISTRIBUTE
|| hash == FnvHash.Constants.SORT
|| hash == FnvHash.Constants.CLUSTER) {
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
if (lexer.token == Token.BY) {
lexer.reset(mark);
return null;
}
return ident;
}
}
}
return this.as();
@@ -715,7 +715,7 @@ public class SQLSelectParser extends SQLParser {
}
}
if (lexer.identifierEquals(FnvHash.Constants.SORT)) {
if (lexer.token == Token.SORT || lexer.identifierEquals(FnvHash.Constants.SORT)) {
lexer.nextToken();
accept(Token.BY);
@@ -2294,6 +2294,20 @@ class SchemaResolveVisitorFactory {
}
}
List<SQLSelectOrderByItem> distributeBy = x.getDistributeBy();
if (distributeBy != null) {
for (SQLSelectOrderByItem item : distributeBy) {
item.accept(visitor);
}
}
List<SQLSelectOrderByItem> sortBy = x.getSortBy();
if (sortBy != null) {
for (SQLSelectOrderByItem item : sortBy) {
item.accept(visitor);
}
}
visitor.popContext();
}
@@ -1994,6 +1994,27 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
orderBy.accept(this);
}
final List<SQLSelectOrderByItem> distributeBy = x.getDistributeByDirect();
if (distributeBy != null && distributeBy.size() > 0) {
println();
print0(ucase ? "DISTRIBUTE BY " : "distribute by ");
printAndAccept(distributeBy, ", ");
}
List<SQLSelectOrderByItem> sortBy = x.getSortByDirect();
if (sortBy != null && sortBy.size() > 0) {
println();
print0(ucase ? "SORT BY " : "sort by ");
printAndAccept(sortBy, ", ");
}
final List<SQLSelectOrderByItem> clusterBy = x.getClusterByDirect();
if (clusterBy != null && clusterBy.size() > 0) {
println();
print0(ucase ? "CLUSTER BY " : "cluster by ");
printAndAccept(clusterBy, ", ");
}
if (!informix) {
printFetchFirst(x);
}