diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java index e65c0bd094..0aaaaea3df 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java @@ -29,10 +29,13 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { private SQLOrderBy orderBy; - private SQLExpr limit; - + private SQLExpr limit; + protected List hints; + protected SQLExpr distributeBy; + protected SQLExpr sortBy; + public OdpsSelectQueryBlock(){ } @@ -45,6 +48,22 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { this.orderBy = orderBy; } + public SQLExpr getDistributeBy() { + return distributeBy; + } + + public void setDistributeBy(SQLExpr distributeBy) { + this.distributeBy = distributeBy; + } + + public SQLExpr getSortBy() { + return sortBy; + } + + public void setSortBy(SQLExpr sortBy) { + this.sortBy = sortBy; + } + public SQLExpr getLimit() { return limit; } @@ -75,11 +94,11 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { } else if (!limit.equals(other.limit)) return false; return true; } - + public List getHintsDirect() { return hints; } - + public List getHints() { if (hints == null) { hints = new ArrayList(2); @@ -109,6 +128,8 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { acceptChild(visitor, this.where); acceptChild(visitor, this.groupBy); acceptChild(visitor, this.orderBy); + acceptChild(visitor, this.distributeBy); + acceptChild(visitor, this.sortBy); acceptChild(visitor, this.limit); acceptChild(visitor, this.into); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java index 1481544c57..d11d4a69b5 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java @@ -44,6 +44,7 @@ public class OdpsLexer extends Lexer { map.put("OVER", Token.OVER); map.put("LIMIT", Token.LIMIT); map.put("IF", Token.IF); + map.put("DISTRIBUTE", Token.DISTRIBUTE); DEFAULT_ODPS_KEYWORDS = new Keywords(map); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java index d1a30fadfd..ca518fd350 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java @@ -15,6 +15,7 @@ */ package com.alibaba.druid.sql.dialect.odps.parser; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLSetQuantifier; import com.alibaba.druid.sql.ast.statement.SQLSelectQuery; import com.alibaba.druid.sql.dialect.odps.ast.OdpsSelectQueryBlock; @@ -76,6 +77,21 @@ public class OdpsSelectParser extends SQLSelectParser { parseGroupBy(queryBlock); queryBlock.setOrderBy(this.exprParser.parseOrderBy()); + + if (lexer.token() == Token.DISTRIBUTE) { + lexer.nextToken(); + accept(Token.BY); + SQLExpr distributeBy = this.expr(); + queryBlock.setDistributeBy(distributeBy); + + + if (identifierEquals("SORT")) { + lexer.nextToken(); + accept(Token.BY); + SQLExpr sortBy = this.expr(); + queryBlock.setSortBy(sortBy); + } + } if (lexer.token() == Token.LIMIT) { lexer.nextToken(); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index ab9185bf06..424c5a59a1 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -520,6 +520,17 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis println(); x.getOrderBy().accept(this); } + + if (x.getDistributeBy() != null) { + println(); + print("DISTRIBUTE BY "); + x.getDistributeBy().accept(this); + + if (x.getSortBy() != null) { + print(" SORT BY "); + x.getSortBy().accept(this); + } + } if (x.getLimit() != null) { println(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/Token.java b/src/main/java/com/alibaba/druid/sql/parser/Token.java index d4263b7334..401a05ab4d 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Token.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Token.java @@ -223,6 +223,7 @@ public enum Token { TOP("TOP"), ARRAY("ARRAY"), + DISTRIBUTE("DISTRIBUTE"), // hive diff --git a/src/main/java/com/alibaba/druid/support/opds/udf/ExportColumns.java b/src/main/java/com/alibaba/druid/support/opds/udf/ExportColumns.java index 6a59b3ce2a..d7f2235f0d 100644 --- a/src/main/java/com/alibaba/druid/support/opds/udf/ExportColumns.java +++ b/src/main/java/com/alibaba/druid/support/opds/udf/ExportColumns.java @@ -9,27 +9,34 @@ import com.alibaba.druid.stat.TableStat; import com.aliyun.odps.udf.UDF; public class ExportColumns extends UDF { + public String evaluate(String sql) { return evaluate(sql, null); } - + public String evaluate(String sql, String dbType) { - List statementList = SQLUtils.parseStatements(sql, dbType); - SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); - - for (SQLStatement stmt : statementList) { - stmt.accept(visitor); - } - - StringBuffer buf = new StringBuffer(); - - for (TableStat.Column column : visitor.getColumns()) { - if (buf.length() != 0) { - buf.append(','); + try { + List statementList = SQLUtils.parseStatements(sql, dbType); + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); + + for (SQLStatement stmt : statementList) { + stmt.accept(visitor); } - buf.append(column.toString()); + + StringBuffer buf = new StringBuffer(); + + for (TableStat.Column column : visitor.getColumns()) { + if (buf.length() != 0) { + buf.append(','); + } + buf.append(column.toString()); + } + + return buf.toString(); + } catch (Exception ex) { + System.err.println("error sql : " + sql); + ex.printStackTrace(); + return null; } - - return buf.toString(); } } diff --git a/src/main/java/com/alibaba/druid/support/opds/udf/ExportTables.java b/src/main/java/com/alibaba/druid/support/opds/udf/ExportTables.java index 1e751f7b11..f864268e4c 100644 --- a/src/main/java/com/alibaba/druid/support/opds/udf/ExportTables.java +++ b/src/main/java/com/alibaba/druid/support/opds/udf/ExportTables.java @@ -10,29 +10,36 @@ import com.alibaba.druid.stat.TableStat; import com.aliyun.odps.udf.UDF; public class ExportTables extends UDF { + public String evaluate(String sql) { return evaluate(sql, null); } - + public String evaluate(String sql, String dbType) { - List statementList = SQLUtils.parseStatements(sql, dbType); - SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); - - for (SQLStatement stmt : statementList) { - stmt.accept(visitor); - } - - StringBuffer buf = new StringBuffer(); - - for (Map.Entry entry : visitor.getTables().entrySet()) { - TableStat.Name name = entry.getKey(); - - if (buf.length() != 0) { - buf.append(','); + try { + List statementList = SQLUtils.parseStatements(sql, dbType); + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); + + for (SQLStatement stmt : statementList) { + stmt.accept(visitor); } - buf.append(name.toString()); + + StringBuffer buf = new StringBuffer(); + + for (Map.Entry entry : visitor.getTables().entrySet()) { + TableStat.Name name = entry.getKey(); + + if (buf.length() != 0) { + buf.append(','); + } + buf.append(name.toString()); + } + + return buf.toString(); + } catch (Exception ex) { + System.err.println("error sql : " + sql); + ex.printStackTrace(); + return null; } - - return buf.toString(); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest.java index cc00405bbc..fc049ba4bf 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest.java @@ -15,79 +15,26 @@ */ package com.alibaba.druid.bvt.sql.odps; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.List; - -import junit.framework.TestCase; - import org.junit.Assert; import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLStatement; -import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; -import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlParameterizedOutputVisitor; -import com.alibaba.druid.sql.dialect.odps.parser.OdpsStatementParser; -import com.alibaba.druid.sql.dialect.odps.visitor.OdpsSchemaStatVisitor; -import com.alibaba.druid.sql.visitor.SchemaStatVisitor; -import com.alibaba.druid.util.JdbcUtils; -import com.alibaba.druid.util.Utils; + +import junit.framework.TestCase; public class OdpsSelectTest extends TestCase { - public void test_0() throws Exception { - exec_test("bvt/parser/odps-1.sql"); - exec_test("bvt/parser/odps-2.sql"); - exec_test("bvt/parser/odps-3.sql"); + public void test_distribute_by() throws Exception { + String sql = "select region from sale_detail distribute by region;";// + Assert.assertEquals("SELECT region" // + + "\nFROM sale_detail" // + + "\nDISTRIBUTE BY region;", SQLUtils.formatOdps(sql)); } - - public void exec_test(String resource) throws Exception { - System.out.println(resource); - InputStream is = null; - - is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); - Reader reader = new InputStreamReader(is, "UTF-8"); - String input = Utils.read(reader); - JdbcUtils.close(reader); - String[] items = input.split("---------------------------"); - String sql = items[0].trim(); - String expect = items[1].trim(); - - OdpsStatementParser parser = new OdpsStatementParser(sql); - List statementList = parser.parseStatementList(); - SQLStatement stmt = statementList.get(0); - - Assert.assertEquals(1, statementList.size()); - - SchemaStatVisitor visitor = new OdpsSchemaStatVisitor(); - stmt.accept(visitor); - - System.out.println(sql); - System.out.println("Tables : " + visitor.getTables()); - System.out.println("fields : " + visitor.getColumns()); - - System.out.println(); - System.out.println("---------------------------"); - System.out.println(SQLUtils.toOdpsString(stmt)); + + public void test_distribute_by_1() throws Exception { + String sql = " select region from sale_detail distribute by region sort by f1;";// + Assert.assertEquals("SELECT region" // + + "\nFROM sale_detail" // + + "\nDISTRIBUTE BY region SORT BY f1;", SQLUtils.formatOdps(sql)); } - void mergValidate(String sql, String expect) { - - MySqlStatementParser parser = new MySqlStatementParser(sql); - List statementList = parser.parseStatementList(); - SQLStatement statemen = statementList.get(0); - - Assert.assertEquals(1, statementList.size()); - - StringBuilder out = new StringBuilder(); - MySqlParameterizedOutputVisitor visitor = new MySqlParameterizedOutputVisitor(out); - statemen.accept(visitor); - - System.out.println(out.toString()); - - Assert.assertEquals(expect, out.toString()); - } - - } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelect_distribute_by_Test.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelect_distribute_by_Test.java new file mode 100644 index 0000000000..72dc8e87d9 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelect_distribute_by_Test.java @@ -0,0 +1,93 @@ +/* + * Copyright 1999-2101 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.odps; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.List; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlParameterizedOutputVisitor; +import com.alibaba.druid.sql.dialect.odps.parser.OdpsStatementParser; +import com.alibaba.druid.sql.dialect.odps.visitor.OdpsSchemaStatVisitor; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcUtils; +import com.alibaba.druid.util.Utils; + +public class OdpsSelect_distribute_by_Test extends TestCase { + + public void test_0() throws Exception { + exec_test("bvt/parser/odps-1.sql"); + exec_test("bvt/parser/odps-2.sql"); + exec_test("bvt/parser/odps-3.sql"); + } + + public void exec_test(String resource) throws Exception { + System.out.println(resource); + InputStream is = null; + + is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); + Reader reader = new InputStreamReader(is, "UTF-8"); + String input = Utils.read(reader); + JdbcUtils.close(reader); + String[] items = input.split("---------------------------"); + String sql = items[0].trim(); + String expect = items[1].trim(); + + OdpsStatementParser parser = new OdpsStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + + Assert.assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = new OdpsSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println(sql); + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + + System.out.println(); + System.out.println("---------------------------"); + System.out.println(SQLUtils.toOdpsString(stmt)); + } + + void mergValidate(String sql, String expect) { + + MySqlStatementParser parser = new MySqlStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement statemen = statementList.get(0); + + Assert.assertEquals(1, statementList.size()); + + StringBuilder out = new StringBuilder(); + MySqlParameterizedOutputVisitor visitor = new MySqlParameterizedOutputVisitor(out); + statemen.accept(visitor); + + System.out.println(out.toString()); + + Assert.assertEquals(expect, out.toString()); + } + + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/udf/ExportColumnsTest.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/udf/ExportColumnsTest.java new file mode 100644 index 0000000000..4b67821a80 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/udf/ExportColumnsTest.java @@ -0,0 +1,14 @@ +package com.alibaba.druid.bvt.sql.odps.udf; + +import com.alibaba.druid.support.opds.udf.ExportColumns; + +import junit.framework.TestCase; + +public class ExportColumnsTest extends TestCase { + + private ExportColumns udf = new ExportColumns(); + + public ExportColumnsTest() { + + } +} diff --git a/src/test/java/com/alibaba/druid/sql/odps/OdpsFileTest.java b/src/test/java/com/alibaba/druid/sql/odps/OdpsFileTest.java new file mode 100644 index 0000000000..7a08619a69 --- /dev/null +++ b/src/test/java/com/alibaba/druid/sql/odps/OdpsFileTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 1999-2101 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.odps; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.List; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlParameterizedOutputVisitor; +import com.alibaba.druid.sql.dialect.odps.parser.OdpsStatementParser; +import com.alibaba.druid.sql.dialect.odps.visitor.OdpsSchemaStatVisitor; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcUtils; +import com.alibaba.druid.util.Utils; + +public class OdpsFileTest extends TestCase { + + public void test_0() throws Exception { + exec_test("bvt/parser/odps-0.sql"); + } + + public void exec_test(String resource) throws Exception { + System.out.println(resource); + InputStream is = null; + + is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); + Reader reader = new InputStreamReader(is, "UTF-8"); + String input = Utils.read(reader); + JdbcUtils.close(reader); + String[] items = input.split("---------------------------"); + String sql = items[0].trim(); + String expect = items[1].trim(); + + OdpsStatementParser parser = new OdpsStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + + Assert.assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = new OdpsSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println(sql); + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + + System.out.println(); + System.out.println("---------------------------"); + System.out.println(SQLUtils.toOdpsString(stmt)); + } + + void mergValidate(String sql, String expect) { + + MySqlStatementParser parser = new MySqlStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement statemen = statementList.get(0); + + Assert.assertEquals(1, statementList.size()); + + StringBuilder out = new StringBuilder(); + MySqlParameterizedOutputVisitor visitor = new MySqlParameterizedOutputVisitor(out); + statemen.accept(visitor); + + System.out.println(out.toString()); + + Assert.assertEquals(expect, out.toString()); + } + + +}