From a12207516e78c25a6960f369ed9210ffb6fe2937 Mon Sep 17 00:00:00 2001 From: "shaojin.wensj" Date: Sun, 12 Jul 2015 22:39:36 +0800 Subject: [PATCH] improved odps sql parser support 'read ...' --- .../dialect/odps/ast/OdpsReadStatement.java | 61 ++++++++++ .../odps/parser/OdpsStatementParser.java | 28 +++++ .../dialect/odps/visitor/OdpsASTVisitor.java | 5 + .../odps/visitor/OdpsASTVisitorAdapter.java | 11 ++ .../odps/visitor/OdpsOutputVisitor.java | 107 +++++++++++------- .../odps/visitor/OdpsSchemaStatVisitor.java | 11 ++ .../druid/bvt/sql/odps/OdpsReadTest.java | 37 ++++++ 7 files changed, 222 insertions(+), 38 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsReadStatement.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsReadTest.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsReadStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsReadStatement.java new file mode 100644 index 0000000000..e82a7a1702 --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsReadStatement.java @@ -0,0 +1,61 @@ +package com.alibaba.druid.sql.dialect.odps.ast; + +import java.util.ArrayList; +import java.util.List; + +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.SQLName; +import com.alibaba.druid.sql.ast.statement.SQLAssignItem; +import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; +import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor; + +public class OdpsReadStatement extends OdpsStatementImpl { + + private SQLExprTableSource table; + private List partition = new ArrayList(); + private List columns = new ArrayList(); + private SQLExpr rowCount; + + @Override + protected void accept0(OdpsASTVisitor visitor) { + if (visitor.visit(this)) { + this.acceptChild(visitor, table); + this.acceptChild(visitor, columns); + this.acceptChild(visitor, partition); + this.acceptChild(visitor, rowCount); + } + visitor.endVisit(this); + } + + public SQLExprTableSource getTable() { + return table; + } + + public void setTable(SQLExprTableSource table) { + if (table != null) { + table.setParent(table); + } + this.table = table; + } + + public void setTable(SQLName table) { + this.setTable(new SQLExprTableSource(table)); + } + + public List getPartition() { + return partition; + } + + public List getColumns() { + return columns; + } + + public SQLExpr getRowCount() { + return rowCount; + } + + public void setRowCount(SQLExpr rowCount) { + this.rowCount = rowCount; + } + +} diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java index aeb0321f8b..5d19f5b689 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java @@ -32,6 +32,7 @@ import com.alibaba.druid.sql.dialect.odps.ast.OdpsAddStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsReadStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsRemoveStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSetLabelStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsShowPartitionsStmt; @@ -131,6 +132,33 @@ public class OdpsStatementParser extends SQLStatementParser { throw new ParserException("TODO " + lexer.token() + " " + lexer.stringVal()); } + + if (identifierEquals("READ")) { + lexer.nextToken(); + OdpsReadStatement stmt = new OdpsReadStatement(); + stmt.setTable(this.exprParser.name()); + + if (lexer.token() == Token.LPAREN) { + lexer.nextToken(); + this.exprParser.names(stmt.getColumns(), stmt); + accept(Token.RPAREN); + } + + if (lexer.token() == Token.PARTITION) { + lexer.nextToken(); + + accept(Token.LPAREN); + parseAssignItems(stmt.getPartition(), stmt); + accept(Token.RPAREN); + } + + if (lexer.token() == Token.LITERAL_INT) { + stmt.setRowCount(this.exprParser.primary()); + } + + statementList.add(stmt); + return true; + } return false; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java index 8d373f5c32..bc3c3d9d51 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java @@ -20,6 +20,7 @@ import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsReadStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsRemoveStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSelectQueryBlock; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSetLabelStatement; @@ -98,4 +99,8 @@ public interface OdpsASTVisitor extends SQLASTVisitor { void endVisit(OdpsStatisticClause.ColumnMin x); boolean visit(OdpsStatisticClause.ColumnMin x); + + void endVisit(OdpsReadStatement x); + + boolean visit(OdpsReadStatement x); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java index 70bc2676fd..801fd07e35 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java @@ -20,6 +20,7 @@ import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsReadStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsRemoveStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSelectQueryBlock; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSetLabelStatement; @@ -201,5 +202,15 @@ public class OdpsASTVisitorAdapter extends SQLASTVisitorAdapter implements OdpsA public boolean visit(OdpsStatisticClause.ColumnMin x) { return true; } + + @Override + public void endVisit(OdpsReadStatement x) { + + } + + @Override + public boolean visit(OdpsReadStatement x) { + return true; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index 943ea29faf..e7e6cc3127 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -34,6 +34,7 @@ import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsReadStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsRemoveStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSelectQueryBlock; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSetLabelStatement; @@ -106,7 +107,7 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis print("LIFECYCLE "); x.getLifecycle().accept(this); } - + if (x.getSelect() != null) { println(); x.getSelect().accept(this); @@ -241,17 +242,17 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis incrementIndent(); for (int i = 0, size = selectList.size(); i < size; ++i) { SQLSelectItem selectItem = selectList.get(i); - + if (i != 0) { print(", "); - + printComment(selectList.get(i - 1).getAfterCommentsDirect(), "\n"); - + println(); } - + selectItem.accept(this); - + if (i == selectList.size() - 1 && selectItem.hasAfterComment()) { print(' '); printComment(selectItem.getAfterCommentsDirect(), "\n"); @@ -397,10 +398,10 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis printComment(x.getBeforeCommentsDirect(), "\n"); println(); } - + print("SELECT "); - List hints= x.getHintsDirect(); + List hints = x.getHintsDirect(); if (hints != null) { printAndAccept(hints, " "); print(' '); @@ -442,7 +443,7 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis println(); x.getOrderBy().accept(this); } - + if (x.getLimit() != null) { println(); print("LIMIT "); @@ -451,7 +452,7 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis return false; } - + public boolean visit(SQLOrderBy x) { int itemSize = x.getItems().size(); if (itemSize > 0) { @@ -471,28 +472,28 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis @Override public void endVisit(OdpsAnalyzeTableStatement x) { - + } @Override public boolean visit(OdpsAnalyzeTableStatement x) { print("ANALYZE TABLE "); x.getTable().accept(this); - + if (x.getPartition().size() > 0) { print(" PARTITION ("); printAndAccept(x.getPartition(), ", "); print(")"); } - + print(" COMPUTE STATISTICS"); - + return false; } @Override public void endVisit(OdpsAddStatisticStatement x) { - + } @Override @@ -501,93 +502,123 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis x.getTable().accept(this); print(" "); x.getStatisticClause().accept(this); - + return false; } - + @Override public void endVisit(OdpsRemoveStatisticStatement x) { - + } - + @Override public boolean visit(OdpsRemoveStatisticStatement x) { print("REMOVE STATISTIC "); x.getTable().accept(this); print(" "); x.getStatisticClause().accept(this); - + return false; } - + @Override public void endVisit(OdpsStatisticClause.TableCount x) { - + } - + @Override public boolean visit(OdpsStatisticClause.TableCount x) { print("TABLE_COUNT"); return false; } - + @Override public void endVisit(OdpsStatisticClause.ExpressionCondition x) { - + } - + @Override public boolean visit(OdpsStatisticClause.ExpressionCondition x) { print("EXPRESSION_CONDITION "); x.getExpr().accept(this); return false; } - + @Override public void endVisit(OdpsStatisticClause.NullValue x) { - + } - + @Override public boolean visit(OdpsStatisticClause.NullValue x) { print("NULL_VALUE "); x.getColumn().accept(this); return false; } - + @Override public void endVisit(OdpsStatisticClause.ColumnSum x) { - + } - + @Override public boolean visit(OdpsStatisticClause.ColumnSum x) { print("COLUMN_SUM "); x.getColumn().accept(this); return false; } - + @Override public void endVisit(OdpsStatisticClause.ColumnMax x) { - + } - + @Override public boolean visit(OdpsStatisticClause.ColumnMax x) { print("COLUMN_MAX "); x.getColumn().accept(this); return false; } - + @Override public void endVisit(OdpsStatisticClause.ColumnMin x) { - + } - + @Override public boolean visit(OdpsStatisticClause.ColumnMin x) { print("COLUMN_MIN "); x.getColumn().accept(this); return false; } + + @Override + public void endVisit(OdpsReadStatement x) { + + } + + @Override + public boolean visit(OdpsReadStatement x) { + print("READ "); + x.getTable().accept(this); + + if (x.getColumns().size() > 0) { + print(" ("); + printAndAccept(x.getColumns(), ", "); + print(")"); + } + + if (x.getPartition().size() > 0) { + print(" PARTITION ("); + printAndAccept(x.getPartition(), ", "); + print(")"); + } + + if (x.getRowCount() != null) { + print(' '); + x.getRowCount().accept(this); + } + + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java index 22bb86d69e..b44ffe1727 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java @@ -22,6 +22,7 @@ import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsReadStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsRemoveStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSelectQueryBlock; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSetLabelStatement; @@ -205,4 +206,14 @@ public class OdpsSchemaStatVisitor extends SchemaStatVisitor implements OdpsASTV public boolean visit(OdpsStatisticClause.ColumnMin x) { return true; } + + @Override + public void endVisit(OdpsReadStatement x) { + + } + + @Override + public boolean visit(OdpsReadStatement x) { + return true; + } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsReadTest.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsReadTest.java new file mode 100644 index 0000000000..9ec268ea68 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsReadTest.java @@ -0,0 +1,37 @@ +package com.alibaba.druid.bvt.sql.odps; + +import org.junit.Assert; + +import com.alibaba.druid.sql.SQLUtils; + +import junit.framework.TestCase; + +public class OdpsReadTest extends TestCase { + + public void test_read() throws Exception { + String sql = "read sale_detail"; + Assert.assertEquals("READ sale_detail", SQLUtils.formatOdps(sql)); + } + + + public void test_read_column() throws Exception { + String sql = "read sale_detail(f0, f1, f2) 10"; + Assert.assertEquals("READ sale_detail (f0, f1, f2) 10", SQLUtils.formatOdps(sql)); + } + + public void test_read_limit() throws Exception { + String sql = "read sale_detail 10"; + Assert.assertEquals("READ sale_detail 10", SQLUtils.formatOdps(sql)); + } + + public void test_read_partition() throws Exception { + String sql = "read sale_detail partition(ds='20150701')"; + Assert.assertEquals("READ sale_detail PARTITION (ds = '20150701')", SQLUtils.formatOdps(sql)); + } + + public void test_read_partition_limit() throws Exception { + String sql = "read sale_detail partition(ds='20150701') 10"; + Assert.assertEquals("READ sale_detail PARTITION (ds = '20150701') 10", SQLUtils.formatOdps(sql)); + } + +}