add odps sql parser support

This commit is contained in:
wenshao
2013-12-23 23:59:03 +08:00
parent 7b1d6bfbcc
commit 6f5bb55384
26 changed files with 3356 additions and 1677 deletions
@@ -0,0 +1,69 @@
package com.alibaba.druid.sql.dialect.odps.ast;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class OdpsCreateTableStatement extends SQLCreateTableStatement {
private boolean ifNotExiists = false;
private SQLExprTableSource like;
protected SQLExpr comment;
protected List<SQLColumnDefinition> partitionColumns = new ArrayList<SQLColumnDefinition>(2);
public boolean isIfNotExiists() {
return ifNotExiists;
}
public void setIfNotExiists(boolean ifNotExiists) {
this.ifNotExiists = ifNotExiists;
}
public SQLExprTableSource getLike() {
return like;
}
public void setLike(SQLName like) {
this.setLike(new SQLExprTableSource(like));
}
public void setLike(SQLExprTableSource like) {
this.like = like;
}
public SQLExpr getComment() {
return comment;
}
public void setComment(SQLExpr comment) {
this.comment = comment;
}
public List<SQLColumnDefinition> getPartitionColumns() {
return partitionColumns;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
accept0((OdpsASTVisitor) visitor);
}
protected void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
this.acceptChild(visitor, tableSource);
this.acceptChild(visitor, tableElementList);
this.acceptChild(visitor, partitionColumns);
}
visitor.endVisit(this);
}
}
@@ -0,0 +1,79 @@
package com.alibaba.druid.sql.dialect.odps.ast;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLObjectImpl;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class OdpsInsert extends SQLObjectImpl {
private boolean overwrite = false;
private List<SQLAssignItem> partitions = new ArrayList<SQLAssignItem>();
protected SQLExprTableSource tableSource;
private SQLSelect query;
public boolean isOverwrite() {
return overwrite;
}
public void setOverwrite(boolean overwrite) {
this.overwrite = overwrite;
}
public List<SQLAssignItem> getPartitions() {
return partitions;
}
public void setPartitions(List<SQLAssignItem> partitions) {
this.partitions = partitions;
}
public SQLExprTableSource getTableSource() {
return tableSource;
}
public void setTableSource(SQLExprTableSource tableSource) {
if (tableSource != null) {
tableSource.setParent(this);
}
this.tableSource = tableSource;
}
public void setTableSource(SQLName tableName) {
this.setTableSource(new SQLExprTableSource(tableName));
}
public SQLSelect getQuery() {
return query;
}
public void setQuery(SQLSelect query) {
if (query != null) {
query.setParent(this);
}
this.query = query;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
accept0((OdpsASTVisitor) visitor);
}
protected void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, tableSource);
acceptChild(visitor, partitions);
acceptChild(visitor, query);
}
visitor.endVisit(this);
}
}
@@ -0,0 +1,42 @@
package com.alibaba.druid.sql.dialect.odps.ast;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.SQLStatementImpl;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class OdpsInsertStatement extends SQLStatementImpl implements SQLStatement {
private SQLSubqueryTableSource from;
private List<OdpsInsert> items = new ArrayList<OdpsInsert>();
public void setFrom(SQLSubqueryTableSource from) {
this.from = from;
}
public SQLSubqueryTableSource getFrom() {
return from;
}
public List<OdpsInsert> getItems() {
return items;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
accept0((OdpsASTVisitor) visitor);
}
protected void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, from);
acceptChild(visitor, items);
}
visitor.endVisit(this);
}
}
@@ -0,0 +1,95 @@
package com.alibaba.druid.sql.dialect.odps.parser;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement;
import com.alibaba.druid.sql.parser.ParserException;
import com.alibaba.druid.sql.parser.SQLCreateTableParser;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.Token;
public class OdpsCreateTableParser extends SQLCreateTableParser {
public OdpsCreateTableParser(String sql){
super(new OdpsExprParser(sql));
}
public OdpsCreateTableParser(SQLExprParser exprParser){
super(exprParser);
}
public SQLCreateTableStatement parseCrateTable(boolean acceptCreate) {
OdpsCreateTableStatement stmt = new OdpsCreateTableStatement();
if (acceptCreate) {
accept(Token.CREATE);
}
accept(Token.TABLE);
if (lexer.token() == Token.IF || identifierEquals("IF")) {
lexer.nextToken();
accept(Token.NOT);
accept(Token.EXISTS);
stmt.setIfNotExiists(true);
}
stmt.setName(this.exprParser.name());
if (lexer.token() == Token.LIKE) {
lexer.nextToken();
SQLName name = this.exprParser.name();
stmt.setLike(name);
} else {
accept(Token.LPAREN);
for (;;) {
if (lexer.token() != Token.IDENTIFIER) {
throw new ParserException("expect identifier");
}
SQLColumnDefinition column = this.exprParser.parseColumn();
stmt.getTableElementList().add(column);
if (!(lexer.token() == (Token.COMMA))) {
break;
} else {
lexer.nextToken();
}
}
accept(Token.RPAREN);
}
if (lexer.token() == Token.COMMENT) {
lexer.nextToken();
stmt.setComment(this.exprParser.primary());
}
if (lexer.token() == Token.PARTITIONED) {
lexer.nextToken();
accept(Token.BY);
accept(Token.LPAREN);
for (;;) {
if (lexer.token() != Token.IDENTIFIER) {
throw new ParserException("expect identifier");
}
SQLColumnDefinition column = this.exprParser.parseColumn();
stmt.getPartitionColumns().add(column);
if (!(lexer.token() == (Token.COMMA))) {
break;
} else {
lexer.nextToken();
}
}
accept(Token.RPAREN);
}
return stmt;
}
}
@@ -0,0 +1,29 @@
package com.alibaba.druid.sql.dialect.odps.parser;
import com.alibaba.druid.sql.parser.Lexer;
import com.alibaba.druid.sql.parser.SQLExprParser;
public class OdpsExprParser extends SQLExprParser {
public final static String[] AGGREGATE_FUNCTIONS = { "AVG", //
"COUNT", //
"LAG",
"LEAD",
"MAX", //
"MIN", //
"STDDEV", //
"SUM", //
"ROW_NUMBER"//
};
public OdpsExprParser(Lexer lexer){
super(lexer);
this.aggregateFunctions = AGGREGATE_FUNCTIONS;
}
public OdpsExprParser(String sql){
this(new OdpsLexer(sql));
this.lexer.nextToken();
}
}
@@ -0,0 +1,126 @@
package com.alibaba.druid.sql.dialect.odps.parser;
import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI;
import java.util.HashMap;
import java.util.Map;
import com.alibaba.druid.sql.parser.Keywords;
import com.alibaba.druid.sql.parser.Lexer;
import com.alibaba.druid.sql.parser.NotAllowCommentException;
import com.alibaba.druid.sql.parser.Token;
public class OdpsLexer extends Lexer {
public final static Keywords DEFAULT_ODPS_KEYWORDS;
static {
Map<String, Token> map = new HashMap<String, Token>();
map.putAll(Keywords.DEFAULT_KEYWORDS.getKeywords());
map.put("PARTITION", Token.PARTITION);
map.put("PARTITIONED", Token.PARTITIONED);
map.put("OVERWRITE", Token.OVERWRITE);
map.put("OVER", Token.OVER);
DEFAULT_ODPS_KEYWORDS = new Keywords(map);
}
public OdpsLexer(String input){
super(input);
super.keywods = DEFAULT_ODPS_KEYWORDS;
}
public void scanComment() {
if (ch != '/' && ch != '-') {
throw new IllegalStateException();
}
mark = pos;
bufPos = 0;
scanChar();
// /*+ */
if (ch == '*') {
scanChar();
bufPos++;
while (ch == ' ') {
scanChar();
bufPos++;
}
boolean isHint = false;
int startHintSp = bufPos + 1;
if (ch == '+') {
isHint = true;
scanChar();
bufPos++;
}
for (;;) {
if (ch == '*' && charAt(pos + 1) == '/') {
bufPos += 2;
scanChar();
scanChar();
break;
}
scanChar();
bufPos++;
}
if (isHint) {
stringVal = subString(mark + startHintSp, (bufPos - startHintSp) - 1);
token = Token.HINT;
} else {
stringVal = subString(mark, bufPos);
token = Token.MULTI_LINE_COMMENT;
}
if (token != Token.HINT && !isAllowComment()) {
throw new NotAllowCommentException();
}
return;
}
if (!isAllowComment()) {
throw new NotAllowCommentException();
}
if (ch == '/' || ch == '-') {
scanChar();
bufPos++;
for (;;) {
if (ch == '\r') {
if (charAt(pos + 1) == '\n') {
bufPos += 2;
scanChar();
break;
}
bufPos++;
break;
} else if (ch == EOI) {
break;
}
if (ch == '\n') {
scanChar();
bufPos++;
break;
}
scanChar();
bufPos++;
}
stringVal = subString(mark + 1, bufPos);
token = Token.LINE_COMMENT;
return;
}
}
}
@@ -0,0 +1,122 @@
package com.alibaba.druid.sql.dialect.odps.parser;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement;
import com.alibaba.druid.sql.parser.SQLCreateTableParser;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.SQLSelectParser;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.parser.Token;
public class OdpsStatementParser extends SQLStatementParser {
public OdpsStatementParser(String sql){
super(new OdpsExprParser(sql));
}
public OdpsStatementParser(SQLExprParser exprParser){
super(exprParser);
}
public SQLCreateTableStatement parseCreateTable() {
SQLCreateTableParser parser = new OdpsCreateTableParser(this.exprParser);
return parser.parseCrateTable();
}
public SQLCreateTableParser getSQLCreateTableParser() {
return new OdpsCreateTableParser(this.exprParser);
}
public boolean parseStatementListDialect(List<SQLStatement> statementList) {
if (lexer.token() == Token.FROM) {
SQLStatement stmt = this.parseInsert();
statementList.add(stmt);
return true;
}
return false;
}
public SQLStatement parseInsert() {
OdpsInsertStatement stmt = new OdpsInsertStatement();
if (lexer.token() == Token.FROM) {
lexer.nextToken();
accept(Token.LPAREN);
SQLSelectParser selectParser = createSQLSelectParser();
SQLSelect select = selectParser.select();
accept(Token.RPAREN);
String alias = lexer.stringVal();
accept(Token.IDENTIFIER);
SQLSubqueryTableSource from = new SQLSubqueryTableSource(select, alias);
stmt.setFrom(from);
}
for (;;) {
OdpsInsert insert = parseOdpsInsert();
stmt.getItems().add(insert);
if (lexer.token() != Token.INSERT) {
break;
}
}
return stmt;
}
public OdpsInsert parseOdpsInsert() {
OdpsInsert insert = new OdpsInsert();
SQLSelectParser selectParser = createSQLSelectParser();
accept(Token.INSERT);
if (lexer.token() == Token.INTO) {
lexer.nextToken();
} else {
accept(Token.OVERWRITE);
insert.setOverwrite(true);
}
accept(Token.TABLE);
insert.setTableSource(this.exprParser.name());
if (lexer.token() == Token.PARTITION) {
lexer.nextToken();
accept(Token.LPAREN);
for (;;) {
SQLAssignItem ptExpr = new SQLAssignItem();
ptExpr.setTarget(this.exprParser.name());
if (lexer.token() == Token.EQ) {
lexer.nextToken();
SQLExpr ptValue = this.exprParser.expr();
ptExpr.setValue(ptValue);
}
insert.getPartitions().add(ptExpr);
if (!(lexer.token() == (Token.COMMA))) {
break;
} else {
lexer.nextToken();
}
}
accept(Token.RPAREN);
}
SQLSelect query = selectParser.select();
insert.setQuery(query);
return insert;
}
}
@@ -0,0 +1,21 @@
package com.alibaba.druid.sql.dialect.odps.visitor;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public interface OdpsASTVisitor extends SQLASTVisitor {
void endVisit(OdpsCreateTableStatement x);
boolean visit(OdpsCreateTableStatement x);
void endVisit(OdpsInsertStatement x);
boolean visit(OdpsInsertStatement x);
void endVisit(OdpsInsert x);
boolean visit(OdpsInsert x);
}
@@ -0,0 +1,263 @@
package com.alibaba.druid.sql.dialect.odps.visitor;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLCaseExpr;
import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource;
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType;
import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause;
import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVisitor {
public OdpsOutputVisitor(Appendable appender){
super(appender);
}
public boolean visit(OdpsCreateTableStatement x) {
if (x.isIfNotExiists()) {
print("CREATE TABLE IF NOT EXISTS ");
} else {
print("CREATE TABLE ");
}
x.getName().accept(this);
x.getName().accept(this);
if (x.getLike() != null) {
print(" LIKE ");
x.getLike().accept(this);
}
int size = x.getTableElementList().size();
if (size > 0) {
print(" (");
incrementIndent();
println();
for (int i = 0; i < size; ++i) {
if (i != 0) {
print(", ");
println();
}
x.getTableElementList().get(i).accept(this);
}
decrementIndent();
println();
print(")");
}
if (x.getComment() != null) {
println();
print("COMMENT ");
x.getComment().accept(this);
}
int partitionSize = x.getPartitionColumns().size();
if (partitionSize > 0) {
println();
print("PARTITIONED (");
for (int i = 0; i < partitionSize; ++i) {
if (i != 0) {
print(", ");
}
x.getPartitionColumns().get(i).accept(this);
}
print(")");
}
return false;
}
@Override
public void endVisit(OdpsCreateTableStatement x) {
super.endVisit((SQLCreateTableStatement) x);
}
public SQLStatement parseInsert() {
OdpsInsertStatement stmt = new OdpsInsertStatement();
return stmt;
}
@Override
public void endVisit(OdpsInsertStatement x) {
}
@Override
public boolean visit(OdpsInsertStatement x) {
if (x.getFrom() != null) {
print("FROM (");
incrementIndent();
println();
x.getFrom().getSelect().accept(this);
decrementIndent();
println();
print(") ");
print(x.getFrom().getAlias());
}
for (OdpsInsert insert : x.getItems()) {
println();
println();
insert.accept(this);
}
return false;
}
@Override
public void endVisit(OdpsInsert x) {
}
@Override
public boolean visit(OdpsInsert x) {
if (x.isOverwrite()) {
print("INSERT OVERWRITE TABLE ");
} else {
print("INSERT INTO OVERWRITE ");
}
x.getTableSource().accept(this);
int partitions = x.getPartitions().size();
if (partitions > 0) {
print(" PARTITION (");
for (int i = 0; i < partitions; ++i) {
if (i != 0) {
print(", ");
}
SQLAssignItem assign = x.getPartitions().get(i);
assign.getTarget().accept(this);
if (assign.getValue() != null) {
print("=");
assign.getValue().accept(this);
}
}
print(")");
}
println();
x.getQuery().accept(this);
return false;
}
public boolean visit(SQLCaseExpr x) {
incrementIndent();
print("CASE ");
if (x.getValueExpr() != null) {
x.getValueExpr().accept(this);
println();
}
printAndAccept(x.getItems(), " ");
if (x.getElseExpr() != null) {
println();
print("ELSE ");
x.getElseExpr().accept(this);
}
println();
print("END");
decrementIndent();
return false;
}
public boolean visit(SQLSelectGroupByClause x) {
int itemSize = x.getItems().size();
if (itemSize > 0) {
print("GROUP BY ");
incrementIndent();
for (int i = 0; i < itemSize; ++i) {
if (i != 0) {
println(", ");
}
x.getItems().get(i).accept(this);
}
decrementIndent();
}
if (x.getHaving() != null) {
println();
print("HAVING ");
x.getHaving().accept(this);
}
return false;
}
protected void printSelectList(List<SQLSelectItem> selectList) {
incrementIndent();
for (int i = 0, size = selectList.size(); i < size; ++i) {
if (i != 0) {
print(", ");
println();
}
selectList.get(i).accept(this);
}
decrementIndent();
}
@Override
public boolean visit(SQLSubqueryTableSource x) {
print("(");
incrementIndent();
println();
x.getSelect().accept(this);
decrementIndent();
println();
print(")");
if (x.getAlias() != null) {
print(' ');
print(x.getAlias());
}
return false;
}
@Override
public boolean visit(SQLJoinTableSource x) {
x.getLeft().accept(this);
if (x.getJoinType() == JoinType.COMMA) {
print(",");
} else {
println();
print(JoinType.toString(x.getJoinType()));
}
print(" ");
x.getRight().accept(this);
if (x.getCondition() != null) {
incrementIndent();
print(" ON ");
x.getCondition().accept(this);
decrementIndent();
}
if (x.getUsing().size() > 0) {
print(" USING (");
printAndAccept(x.getUsing(), ", ");
print(")");
}
if (x.getAlias() != null) {
print(" AS ");
print(x.getAlias());
}
return false;
}
}
@@ -0,0 +1,43 @@
package com.alibaba.druid.sql.dialect.odps.visitor;
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
public class OdpsSchemaStatVisitor extends SchemaStatVisitor implements OdpsASTVisitor {
@Override
public void endVisit(OdpsCreateTableStatement x) {
super.endVisit((SQLCreateTableStatement) x);
}
@Override
public boolean visit(OdpsCreateTableStatement x) {
return super.visit((SQLCreateTableStatement) x);
}
@Override
public void endVisit(OdpsInsertStatement x) {
}
@Override
public boolean visit(OdpsInsertStatement x) {
return false;
}
@Override
public void endVisit(OdpsInsert x) {
}
@Override
public boolean visit(OdpsInsert x) {
return false;
}
}