From 404ca1fa58e4f89501742807a40e2f8a7472af5c Mon Sep 17 00:00:00 2001 From: Yako Date: Fri, 6 Nov 2015 15:26:45 +0800 Subject: [PATCH] support odps desc --- .../sql/dialect/odps/ast/OdpsDescStmt.java | 61 +++++++++ .../odps/parser/OdpsStatementParser.java | 126 ++++++++++++------ .../dialect/odps/visitor/OdpsASTVisitor.java | 5 + .../odps/visitor/OdpsASTVisitorAdapter.java | 11 ++ .../odps/visitor/OdpsOutputVisitor.java | 26 ++++ .../odps/visitor/OdpsSchemaStatVisitor.java | 11 ++ .../druid/bvt/sql/odps/OdpsDescTest.java | 64 +++++++++ 7 files changed, 264 insertions(+), 40 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsDescStmt.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsDescTest.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsDescStmt.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsDescStmt.java new file mode 100644 index 0000000000..10b9f5b3c3 --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsDescStmt.java @@ -0,0 +1,61 @@ +/* + * Copyright 1999-2101 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.dialect.odps.ast; + +import java.util.ArrayList; +import java.util.List; + +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.statement.SQLDescribeStatement; +import com.alibaba.druid.sql.ast.statement.SQLObjectType; +import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; + +public class OdpsDescStmt extends SQLDescribeStatement { + + private SQLObjectType objectType; + + private List partition = new ArrayList(); + + @Override + protected void accept0(SQLASTVisitor visitor) { + accept0((OdpsASTVisitor) visitor); + } + + protected void accept0(OdpsASTVisitor visitor) { + if (visitor.visit(this)) { + acceptChild(visitor, object); + } + visitor.endVisit(this); + } + + public SQLObjectType getObjectType() { + return objectType; + } + + public void setObjectType(SQLObjectType objectType) { + this.objectType = objectType; + } + + public List getPartition() { + return partition; + } + + public void setPartition(List partition) { + this.partition = partition; + } + +} diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java index 6fff3b0208..277cfb3ce1 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java @@ -33,6 +33,7 @@ import com.alibaba.druid.sql.ast.statement.SQLShowTablesStatement; import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAddStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsDescStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsGrantStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; @@ -53,8 +54,9 @@ import com.alibaba.druid.sql.parser.Token; import com.alibaba.druid.util.JdbcConstants; public class OdpsStatementParser extends SQLStatementParser { + public OdpsStatementParser(String sql){ - super (new OdpsLexer(sql, true, true), JdbcConstants.ODPS); + super(new OdpsLexer(sql, true, true), JdbcConstants.ODPS); this.exprParser = new OdpsExprParser(this.lexer); this.lexer.nextToken(); } @@ -83,16 +85,16 @@ public class OdpsStatementParser extends SQLStatementParser { statementList.add(stmt); return true; } - + if (identifierEquals("ANALYZE")) { lexer.nextToken(); accept(Token.TABLE); - + OdpsAnalyzeTableStatement stmt = new OdpsAnalyzeTableStatement(); - + SQLName table = this.exprParser.name(); stmt.setTable(table); - + if (lexer.token() == Token.PARTITION) { lexer.nextToken(); @@ -100,17 +102,17 @@ public class OdpsStatementParser extends SQLStatementParser { parseAssignItems(stmt.getPartition(), stmt); accept(Token.RPAREN); } - + accept(Token.COMPUTE); acceptIdentifier("STATISTICS"); - + statementList.add(stmt); return true; } - + if (identifierEquals("ADD")) { lexer.nextToken(); - + if (identifierEquals("STATISTIC")) { lexer.nextToken(); OdpsAddStatisticStatement stmt = new OdpsAddStatisticStatement(); @@ -119,13 +121,13 @@ public class OdpsStatementParser extends SQLStatementParser { statementList.add(stmt); return true; } - + throw new ParserException("TODO " + lexer.token() + " " + lexer.stringVal()); } - + if (identifierEquals("REMOVE")) { lexer.nextToken(); - + if (identifierEquals("STATISTIC")) { lexer.nextToken(); OdpsRemoveStatisticStatement stmt = new OdpsRemoveStatisticStatement(); @@ -134,21 +136,21 @@ public class OdpsStatementParser extends SQLStatementParser { statementList.add(stmt); return true; } - + throw new ParserException("TODO " + lexer.token() + " " + lexer.stringVal()); } - + if (identifierEquals("READ")) { lexer.nextToken(); OdpsReadStatement stmt = new OdpsReadStatement(); stmt.setTable(this.exprParser.name()); - + if (lexer.token() == Token.LPAREN) { lexer.nextToken(); this.exprParser.names(stmt.getColumns(), stmt); accept(Token.RPAREN); } - + if (lexer.token() == Token.PARTITION) { lexer.nextToken(); @@ -156,26 +158,32 @@ public class OdpsStatementParser extends SQLStatementParser { parseAssignItems(stmt.getPartition(), stmt); accept(Token.RPAREN); } - + if (lexer.token() == Token.LITERAL_INT) { stmt.setRowCount(this.exprParser.primary()); } - + statementList.add(stmt); return true; } - - if(identifierEquals("LIST")) { + + if (identifierEquals("LIST")) { OdpsListStmt stmt = new OdpsListStmt(); lexer.nextToken(); stmt.setObject(this.exprParser.expr()); - + statementList.add(stmt); return true; } - + + if (lexer.token() == Token.DESC || identifierEquals("DESCRIBE")) { + OdpsDescStmt stmt = parseDescribe(); + statementList.add(stmt); + return true; + } + return false; } @@ -247,11 +255,11 @@ public class OdpsStatementParser extends SQLStatementParser { public OdpsInsert parseOdpsInsert() { OdpsInsert insert = new OdpsInsert(); - + if (lexer.isKeepComments() && lexer.hasComment()) { insert.addBeforeComment(lexer.readAndResetComments()); } - + SQLSelectParser selectParser = createSQLSelectParser(); accept(Token.INSERT); @@ -317,17 +325,17 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } - + if (identifierEquals("TABLES")) { lexer.nextToken(); SQLShowTablesStatement stmt = new SQLShowTablesStatement(); - + if (lexer.token() == Token.FROM) { lexer.nextToken(); stmt.setDatabase(this.exprParser.name()); } - + if (lexer.token() == Token.LIKE) { lexer.nextToken(); stmt.setLike(this.exprParser.expr()); @@ -335,7 +343,7 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } - + if (identifierEquals("GRANTS")) { lexer.nextToken(); OdpsShowGrantsStmt stmt = new OdpsShowGrantsStmt(); @@ -344,7 +352,7 @@ public class OdpsStatementParser extends SQLStatementParser { lexer.nextToken(); stmt.setUser(this.exprParser.expr()); } - + if (lexer.token() == Token.ON) { lexer.nextToken(); acceptIdentifier("type"); @@ -353,7 +361,7 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } - + throw new ParserException("TODO " + lexer.token() + " " + lexer.stringVal()); } @@ -362,18 +370,18 @@ public class OdpsStatementParser extends SQLStatementParser { if (lexer.isKeepComments() && lexer.hasComment()) { comments = lexer.readAndResetComments(); } - + accept(Token.SET); if (identifierEquals("LABEL")) { OdpsSetLabelStatement stmt = new OdpsSetLabelStatement(); - + if (comments != null) { stmt.addBeforeComment(comments); } - + lexer.nextToken(); - + stmt.setLabel(lexer.stringVal()); lexer.nextToken(); accept(Token.TO); @@ -397,7 +405,7 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } else { SQLSetStatement stmt = new SQLSetStatement(getDbType()); - + if (comments != null) { stmt.addBeforeComment(comments); } @@ -407,7 +415,7 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } } - + public OdpsGrantStmt parseGrant() { accept(Token.GRANT); OdpsGrantStmt stmt = new OdpsGrantStmt(); @@ -488,7 +496,7 @@ public class OdpsStatementParser extends SQLStatementParser { return stmt; } - + protected void parsePrivileages(List privileges, SQLObject parent) { for (;;) { String privilege = null; @@ -545,14 +553,14 @@ public class OdpsStatementParser extends SQLStatementParser { privilege = "CreateJob"; } else if (identifierEquals("CreateVolume")) { lexer.nextToken(); - privilege = "CreateVolume"; - } else if (identifierEquals("CreateOfflineModel")) { + privilege = "CreateVolume"; + } else if (identifierEquals("CreateOfflineModel")) { lexer.nextToken(); privilege = "CreateOfflineModel"; } else if (identifierEquals("CreateXflow")) { lexer.nextToken(); privilege = "CreateXflow"; - } + } SQLExpr expr = null; if (privilege != null) { @@ -570,4 +578,42 @@ public class OdpsStatementParser extends SQLStatementParser { break; } } + + public OdpsDescStmt parseDescribe() { + if (lexer.token() == Token.DESC || identifierEquals("DESCRIBE")) { + lexer.nextToken(); + } else { + throw new ParserException("expect DESC, actual " + lexer.token()); + } + + OdpsDescStmt stmt = new OdpsDescStmt(); + if (identifierEquals("ROLE")) { + lexer.nextToken(); + stmt.setObjectType(SQLObjectType.ROLE); + } else if (identifierEquals("PACKAGE")) { + lexer.nextToken(); + stmt.setObjectType(SQLObjectType.PACKAGE); + } else if (identifierEquals("INSTANCE")) { + lexer.nextToken(); + stmt.setObjectType(SQLObjectType.INSTANCE); + } + stmt.setObject(this.exprParser.name()); + + if (lexer.token() == Token.PARTITION) { + lexer.nextToken(); + this.accept(Token.LPAREN); + for (;;) { + stmt.getPartition().add(this.exprParser.expr()); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + if (lexer.token() == Token.RPAREN) { + lexer.nextToken(); + break; + } + } + } + return stmt; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java index 7f6b41ab04..75d5d2fc49 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitor.java @@ -18,6 +18,7 @@ package com.alibaba.druid.sql.dialect.odps.visitor; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAddStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsDescStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsGrantStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; @@ -118,4 +119,8 @@ public interface OdpsASTVisitor extends SQLASTVisitor { void endVisit(OdpsGrantStmt x); boolean visit(OdpsGrantStmt x); + + void endVisit(OdpsDescStmt x); + + boolean visit(OdpsDescStmt x); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java index 04cab8cf4e..9d36b6f1e7 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsASTVisitorAdapter.java @@ -18,6 +18,7 @@ package com.alibaba.druid.sql.dialect.odps.visitor; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAddStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsDescStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsGrantStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; @@ -246,4 +247,14 @@ public class OdpsASTVisitorAdapter extends SQLASTVisitorAdapter implements OdpsA return true; } + @Override + public void endVisit(OdpsDescStmt x) { + + } + + @Override + public boolean visit(OdpsDescStmt x) { + return true; + } + } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index af4b8184ae..372eb33a7a 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -38,6 +38,7 @@ import com.alibaba.druid.sql.ast.statement.SQLTableElement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAddStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsDescStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsGrantStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; @@ -841,4 +842,29 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis return false; } + + @Override + public void endVisit(OdpsDescStmt x) { + + } + + @Override + public boolean visit(OdpsDescStmt x) { + print("DESC "); + if (x.getObjectType() != null) { + print(x.getObjectType().name()); + print(' '); + } + + if(x.getObject() != null) { + x.getObject().accept(this); + } + + if (x.getPartition().size() > 0) { + print(" PARTITION ("); + printAndAccept(x.getPartition(), ", "); + print(")"); + } + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java index f462f60f64..99ddaed85c 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsSchemaStatVisitor.java @@ -21,6 +21,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAddStatisticStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsAnalyzeTableStatement; import com.alibaba.druid.sql.dialect.odps.ast.OdpsCreateTableStatement; +import com.alibaba.druid.sql.dialect.odps.ast.OdpsDescStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsGrantStmt; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert; import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement; @@ -250,4 +251,14 @@ public class OdpsSchemaStatVisitor extends SchemaStatVisitor implements OdpsASTV public boolean visit(OdpsGrantStmt x) { return super.visit((SQLGrantStatement) x); } + + @Override + public void endVisit(OdpsDescStmt x) { + + } + + @Override + public boolean visit(OdpsDescStmt x) { + return true; + } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsDescTest.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsDescTest.java new file mode 100644 index 0000000000..1ac08d9f75 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsDescTest.java @@ -0,0 +1,64 @@ +package com.alibaba.druid.bvt.sql.odps; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.odps.parser.OdpsStatementParser; +import com.alibaba.druid.sql.parser.Token; + +public class OdpsDescTest extends TestCase { + + public void test_0() throws Exception { + String sql = "desc coupon_dataset_4_feature"; + OdpsStatementParser parser = new OdpsStatementParser(sql); + SQLStatement stmt = parser.parseStatementList().get(0); + parser.match(Token.EOF); + String output = SQLUtils.toOdpsString(stmt); + System.out.println(output); + Assert.assertEquals("DESC coupon_dataset_4_feature", output); + } + + public void test_1() throws Exception { + String sql = "desc role admin"; + OdpsStatementParser parser = new OdpsStatementParser(sql); + SQLStatement stmt = parser.parseStatementList().get(0); + parser.match(Token.EOF); + String output = SQLUtils.toOdpsString(stmt); + System.out.println(output); + Assert.assertEquals("DESC ROLE admin", output); + } + + public void test_2() throws Exception { + String sql = "desc instance 20150715103441522gond1qa2"; + OdpsStatementParser parser = new OdpsStatementParser(sql); + SQLStatement stmt = parser.parseStatementList().get(0); + parser.match(Token.EOF); + String output = SQLUtils.toOdpsString(stmt); + System.out.println(output); + Assert.assertEquals("DESC INSTANCE 20150715103441522gond1qa2", output); + } + + public void test_3() throws Exception { + String sql = "desc idl_cheka_ent_sql_fht partition (ds='20151010',hh='10')"; + OdpsStatementParser parser = new OdpsStatementParser(sql); + SQLStatement stmt = parser.parseStatementList().get(0); + parser.match(Token.EOF); + String output = SQLUtils.toOdpsString(stmt); + System.out.println(output); + Assert.assertEquals("DESC idl_cheka_ent_sql_fht PARTITION (ds = '20151010', hh = '10')", output); + } + + public void test_4() throws Exception { + String sql = "desc idl_cheka_ent_sql_fdt partition (ds='20151010')"; + OdpsStatementParser parser = new OdpsStatementParser(sql); + SQLStatement stmt = parser.parseStatementList().get(0); + parser.match(Token.EOF); + String output = SQLUtils.toOdpsString(stmt); + System.out.println(output); + Assert.assertEquals("DESC idl_cheka_ent_sql_fdt PARTITION (ds = '20151010')", output); + } + +}