From 1cdd3f28eba88430c7dce30256bba1ea6d873fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Sat, 2 Nov 2019 22:46:16 +0800 Subject: [PATCH] improved hive sql parser --- .../druid/pool/DruidPooledConnection.java | 1 - .../statement/SQLCreateTableStatement.java | 9 +- .../ast/statement/SQLSelectOrderByItem.java | 39 ++++++ .../hive/parser/HiveCreateTableParser.java | 10 +- .../dialect/hive/parser/HiveSelectParser.java | 28 ----- .../hive/visitor/HiveOutputVisitor.java | 2 +- .../odps/parser/OdpsCreateTableParser.java | 10 +- .../odps/visitor/OdpsOutputVisitor.java | 2 +- .../alibaba/druid/sql/parser/SQLParser.java | 56 +++++++++ .../druid/sql/parser/SQLSelectParser.java | 2 +- .../SchemaResolveVisitorFactory.java | 14 +++ .../sql/visitor/SQLASTOutputVisitor.java | 21 ++++ .../java/com/alibaba/druid/util/FnvHash.java | 4 + .../hive/HiveSelectTest_13_cluster_by.java | 72 +++++++++++ .../bvt/sql/hive/HiveSelectTest_3_array.java | 44 +++++++ .../bvt/sql/hive/HiveSelectTest_4_array.java | 44 +++++++ .../sql/hive/HiveSelectTest_cluster_by.java | 62 ++++++++++ .../sql/hive/HiveSelectTest_distribute.java | 66 ++++++++++ .../select/MySqlSelectTest_with_20433301.java | 106 ++++++++++++++++ .../sql/postgresql/select/PGUnionTest.java | 113 ++++++++++++++++++ 20 files changed, 669 insertions(+), 36 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_13_cluster_by.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_3_array.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_4_array.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_cluster_by.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_distribute.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_with_20433301.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGUnionTest.java diff --git a/src/main/java/com/alibaba/druid/pool/DruidPooledConnection.java b/src/main/java/com/alibaba/druid/pool/DruidPooledConnection.java index 3051cadb41..7ee1d6d129 100644 --- a/src/main/java/com/alibaba/druid/pool/DruidPooledConnection.java +++ b/src/main/java/com/alibaba/druid/pool/DruidPooledConnection.java @@ -768,7 +768,6 @@ public class DruidPooledConnection extends PoolableWrapper implements javax.sql. DruidAbstractDataSource dataSource = holder.getDataSource(); dataSource.incrementStartTransactionCount(); transactionInfo = new TransactionInfo(dataSource.createTransactionId()); - holder.setLastExecTimeMillis(System.currentTimeMillis()); } @Override diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java index 5eadac2df8..1c8ec8bbf7 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java @@ -72,7 +72,7 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS // for hive & odps protected SQLExternalRecordFormat rowFormat; protected final List partitionColumns = new ArrayList(2); - protected final List clusteredBy = new ArrayList(); + protected final List clusteredBy = new ArrayList(); protected final List sortedBy = new ArrayList(); protected int buckets; @@ -1074,7 +1074,7 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS this.onCommitPreserveRows = onCommitPreserveRows; } - public List getClusteredBy() { + public List getClusteredBy() { return clusteredBy; } @@ -1125,4 +1125,9 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS return pk.containsColumn(columnNameHash); } + + public void addClusteredByItem(SQLSelectOrderByItem item) { + item.setParent(this); + this.clusteredBy.add(item); + } } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectOrderByItem.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectOrderByItem.java index 5db19d5bd6..58f9621cfe 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectOrderByItem.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectOrderByItem.java @@ -21,6 +21,8 @@ import com.alibaba.druid.sql.ast.SQLOrderingSpecification; import com.alibaba.druid.sql.ast.SQLReplaceable; import com.alibaba.druid.sql.visitor.SQLASTVisitor; +import java.util.List; + public final class SQLSelectOrderByItem extends SQLObjectImpl implements SQLReplaceable { protected SQLExpr expr; @@ -155,4 +157,41 @@ public final class SQLSelectOrderByItem extends SQLObjectImpl implements SQLRepl public void setResolvedSelectItem(SQLSelectItem resolvedSelectItem) { this.resolvedSelectItem = resolvedSelectItem; } + + public boolean isClusterBy() { + if (parent instanceof SQLCreateTableStatement) { + List clusteredBy = ((SQLCreateTableStatement) parent).getClusteredBy(); + return clusteredBy.indexOf(this) != -1; + } + + if (parent instanceof SQLSelectQueryBlock) { + List clusterBy = ((SQLSelectQueryBlock) parent).getClusterByDirect(); + return clusterBy != null && clusterBy.indexOf(this) != -1; + } + + return false; + } + + public boolean isSortBy() { + if (parent instanceof SQLCreateTableStatement) { + List sortedBy = ((SQLCreateTableStatement) parent).getSortedBy(); + return sortedBy.indexOf(this) != -1; + } + + if (parent instanceof SQLSelectQueryBlock) { + List sortedBy = ((SQLSelectQueryBlock) parent).getSortByDirect(); + return sortedBy != null && sortedBy.indexOf(this) != -1; + } + + return false; + } + + public boolean isDistributeBy() { + if (parent instanceof SQLSelectQueryBlock) { + List distributeBy = ((SQLSelectQueryBlock) parent).getDistributeBy(); + return distributeBy.indexOf(this) != -1; + } + + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java index 84cbcd9391..a230566608 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java @@ -176,7 +176,15 @@ public class HiveCreateTableParser extends SQLCreateTableParser { lexer.nextToken(); accept(Token.BY); accept(Token.LPAREN); - this.exprParser.names(stmt.getClusteredBy()); + for (; ; ) { + SQLSelectOrderByItem item = this.exprParser.parseSelectOrderByItem(); + stmt.addClusteredByItem(item); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } accept(Token.RPAREN); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java index 86d4ca969a..8611b92a4d 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java @@ -48,34 +48,6 @@ public class HiveSelectParser extends SQLSelectParser { return new HiveExprParser(lexer); } - protected void parseSortBy(SQLSelectQueryBlock queryBlock) { - if (lexer.token() == Token.SORT) { - lexer.nextToken(); - accept(Token.BY); - for (;;) { - SQLExpr expr = this.expr(); - - SQLSelectOrderByItem sortByItem = new SQLSelectOrderByItem(expr); - - if (lexer.token() == Token.ASC) { - sortByItem.setType(SQLOrderingSpecification.ASC); - lexer.nextToken(); - } else if (lexer.token() == Token.DESC) { - sortByItem.setType(SQLOrderingSpecification.DESC); - lexer.nextToken(); - } - - queryBlock.addSortBy(sortByItem); - - if (lexer.token() == Token.COMMA) { - lexer.nextToken(); - } else { - break; - } - } - } - } - protected SQLTableSource parseTableSourceRest(SQLTableSource tableSource) { if (lexer.identifierEquals(FnvHash.Constants.TABLESAMPLE) && tableSource instanceof SQLExprTableSource) { Lexer.SavePoint mark = lexer.mark(); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java index 29b04ba11d..347b996383 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java @@ -118,7 +118,7 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis print(')'); } - List clusteredBy = x.getClusteredBy(); + List clusteredBy = x.getClusteredBy(); if (clusteredBy.size() > 0) { println(); print0(ucase ? "CLUSTERED BY (" : "clustered by ("); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java index b63fc18e68..cc8a4d8ef9 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsCreateTableParser.java @@ -144,7 +144,15 @@ public class OdpsCreateTableParser extends SQLCreateTableParser { lexer.nextToken(); accept(Token.BY); accept(Token.LPAREN); - this.exprParser.names(stmt.getClusteredBy()); + for (; ; ) { + SQLSelectOrderByItem item = this.exprParser.parseSelectOrderByItem(); + stmt.addClusteredByItem(item); + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } accept(Token.RPAREN); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index 0431bc1cdf..2cb03e662f 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -162,7 +162,7 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis print(')'); } - List clusteredBy = x.getClusteredBy(); + List clusteredBy = x.getClusteredBy(); if (clusteredBy.size() > 0) { println(); print0(ucase ? "CLUSTERED BY (" : "clustered by ("); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java index ec372410e1..51e8ddb17a 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParser.java @@ -17,6 +17,7 @@ package com.alibaba.druid.sql.parser; import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; import com.alibaba.druid.util.FnvHash; +import com.alibaba.druid.util.StringUtils; public class SQLParser { protected final Lexer lexer; @@ -79,16 +80,71 @@ public class SQLParser { } if (token == Token.IDENTIFIER) { + String ident = lexer.stringVal; long hash = lexer.hash_lower; + if (isEnabled(SQLParserFeature.IgnoreNameQuotes) && ident.length() > 1) { + ident = StringUtils.removeNameQuotes(ident); + } + if (hash == FnvHash.Constants.START || hash == FnvHash.Constants.CONNECT + || hash == FnvHash.Constants.NATURAL + || hash == FnvHash.Constants.CROSS || hash == FnvHash.Constants.OFFSET || hash == FnvHash.Constants.LIMIT) { if (must) { throw new ParserException("illegal alias. " + lexer.info()); } + + Lexer.SavePoint mark = lexer.mark(); + lexer.nextToken(); + switch (lexer.token) { + case EOF: + case COMMA: + case WHERE: + case INNER: + return ident; + default: + lexer.reset(mark); + break; + } + return null; } + + if (!must) { + if (hash == FnvHash.Constants.MODEL) { + Lexer.SavePoint mark = lexer.mark(); + lexer.nextToken(); + if (lexer.token == Token.PARTITION + || lexer.token == Token.UNION + || lexer.identifierEquals(FnvHash.Constants.DIMENSION) + || lexer.identifierEquals(FnvHash.Constants.IGNORE) + || lexer.identifierEquals(FnvHash.Constants.KEEP)) { + lexer.reset(mark); + return null; + } + return ident; + } else if (hash == FnvHash.Constants.WINDOW) { + Lexer.SavePoint mark = lexer.mark(); + lexer.nextToken(); + if (lexer.token == Token.IDENTIFIER) { + lexer.reset(mark); + return null; + } + return ident; + } else if (hash == FnvHash.Constants.DISTRIBUTE + || hash == FnvHash.Constants.SORT + || hash == FnvHash.Constants.CLUSTER) { + Lexer.SavePoint mark = lexer.mark(); + lexer.nextToken(); + if (lexer.token == Token.BY) { + lexer.reset(mark); + return null; + } + return ident; + } + } } return this.as(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java index a4ecdb7f1d..9c5a2421c4 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java @@ -715,7 +715,7 @@ public class SQLSelectParser extends SQLParser { } } - if (lexer.identifierEquals(FnvHash.Constants.SORT)) { + if (lexer.token == Token.SORT || lexer.identifierEquals(FnvHash.Constants.SORT)) { lexer.nextToken(); accept(Token.BY); diff --git a/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java b/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java index 2c71e33504..469d7efe82 100644 --- a/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java +++ b/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java @@ -2294,6 +2294,20 @@ class SchemaResolveVisitorFactory { } } + List distributeBy = x.getDistributeBy(); + if (distributeBy != null) { + for (SQLSelectOrderByItem item : distributeBy) { + item.accept(visitor); + } + } + + List sortBy = x.getSortBy(); + if (sortBy != null) { + for (SQLSelectOrderByItem item : sortBy) { + item.accept(visitor); + } + } + visitor.popContext(); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index c284f154e2..3cadd09394 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -1994,6 +1994,27 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet orderBy.accept(this); } + final List distributeBy = x.getDistributeByDirect(); + if (distributeBy != null && distributeBy.size() > 0) { + println(); + print0(ucase ? "DISTRIBUTE BY " : "distribute by "); + printAndAccept(distributeBy, ", "); + } + + List sortBy = x.getSortByDirect(); + if (sortBy != null && sortBy.size() > 0) { + println(); + print0(ucase ? "SORT BY " : "sort by "); + printAndAccept(sortBy, ", "); + } + + final List clusterBy = x.getClusterByDirect(); + if (clusterBy != null && clusterBy.size() > 0) { + println(); + print0(ucase ? "CLUSTER BY " : "cluster by "); + printAndAccept(clusterBy, ", "); + } + if (!informix) { printFetchFirst(x); } diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index 99ff70e44b..93210d77e1 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -689,5 +689,9 @@ public final class FnvHash { long OUTLINE = fnv1a_64_lower("OUTLINE"); long BD = fnv1a_64_lower("BD"); long CURRENT_DATE = fnv1a_64_lower("CURRENT_DATE"); + long PARTITION = fnv1a_64_lower("PARTITION"); + long MODEL = fnv1a_64_lower("MODEL"); + long DIMENSION = fnv1a_64_lower("DIMENSION"); + long KEEP = fnv1a_64_lower("KEEP"); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_13_cluster_by.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_13_cluster_by.java new file mode 100644 index 0000000000..1f906112dc --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_13_cluster_by.java @@ -0,0 +1,72 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class HiveSelectTest_13_cluster_by + extends TestCase { + + public void test_0() throws Exception + { + String sql = "select buyer_id,seller_id ,order_id,div_pay_amt,t1.dim_seller.member_id,t1.dim_buyer.member_id from LD_aly.fct_pay_ord_cn_di t1 cluster by buyer_id,seller_id \n"; + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE); + SQLSelectStatement stmt = (SQLSelectStatement) statementList.get(0); + + assertEquals("SELECT buyer_id, seller_id, order_id, div_pay_amt, t1.dim_seller.member_id\n" + + "\t, t1.dim_buyer.member_id\n" + + "FROM LD_aly.fct_pay_ord_cn_di t1\n" + + "CLUSTER BY buyer_id, seller_id", stmt.toString()); + + assertEquals("SELECT buyer_id, seller_id, order_id, div_pay_amt, t1.dim_seller.member_id\n" + + "\t, t1.dim_buyer.member_id\n" + + "FROM LD_aly.fct_pay_ord_cn_di t1\n" + + "CLUSTER BY buyer_id, seller_id", SQLUtils.toSQLString(stmt)); + + assertEquals(1, statementList.size()); + + for (SQLSelectOrderByItem item : stmt.getSelect().getQueryBlock().getClusterBy()) { + assertFalse(item.isSortBy()); + assertFalse(item.isDistributeBy()); + assertTrue(item.isClusterBy()); + } + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + System.out.println("groupBy : " + visitor.getGroupByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(5, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + assertEquals(0, visitor.getGroupByColumns().size()); + + assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "buyer_id")); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_3_array.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_3_array.java new file mode 100644 index 0000000000..e499770ae3 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_3_array.java @@ -0,0 +1,44 @@ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class HiveSelectTest_3_array + extends TestCase { + public void test_select() throws Exception + { + String sql = "select languages[0] from json_nested_test;";// + assertEquals("SELECT languages[0]\n" + + "FROM json_nested_test;", SQLUtils.formatHive(sql)); + assertEquals("select languages[0]\n" + + "from json_nested_test;", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + + assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE); + stmt.accept(visitor); + + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(1, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + + assertTrue(visitor.containsColumn("json_nested_test", "languages")); +// assertTrue(visitor.containsColumn("customers", "isxx")); +// assertTrue(visitor.containsColumn("customers", "*")); + + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_4_array.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_4_array.java new file mode 100644 index 0000000000..6745eec901 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_4_array.java @@ -0,0 +1,44 @@ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class HiveSelectTest_4_array + extends TestCase { + public void test_select() throws Exception + { + String sql = "select religions['catholic'][0] from json_nested_test;";// + assertEquals("SELECT religions['catholic'][0]\n" + + "FROM json_nested_test;", SQLUtils.formatHive(sql)); + assertEquals("select religions['catholic'][0]\n" + + "from json_nested_test;", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION)); + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + + assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE); + stmt.accept(visitor); + + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(1, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + + assertTrue(visitor.containsColumn("json_nested_test", "religions")); +// assertTrue(visitor.containsColumn("customers", "isxx")); +// assertTrue(visitor.containsColumn("customers", "*")); + + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_cluster_by.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_cluster_by.java new file mode 100644 index 0000000000..33baf3ff45 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_cluster_by.java @@ -0,0 +1,62 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class HiveSelectTest_cluster_by + extends TestCase { + + public void test_0() throws Exception + { + String sql = "SELECT a from x cluster by a;"; + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + + assertEquals("SELECT a\n" + + "FROM x\n" + + "CLUSTER BY a;", stmt.toString()); + + assertEquals("SELECT a\n" + + "FROM x\n" + + "CLUSTER BY a;", SQLUtils.toSQLString(stmt)); + + assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + System.out.println("groupBy : " + visitor.getGroupByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(1, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + assertEquals(0, visitor.getGroupByColumns().size()); + + assertTrue(visitor.containsColumn("x", "a")); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_distribute.java b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_distribute.java new file mode 100644 index 0000000000..67074f0bc9 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/hive/HiveSelectTest_distribute.java @@ -0,0 +1,66 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.hive; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class HiveSelectTest_distribute + extends TestCase { + + public void test_0() throws Exception + { + String sql = "select * from LD_aly.fct_pay_ord_cn_di t1 distribute by buyer_id sort by seller_id"; + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE); + SQLStatement stmt = statementList.get(0); + + assertEquals("SELECT *\n" + + "FROM LD_aly.fct_pay_ord_cn_di t1\n" + + "DISTRIBUTE BY buyer_id\n" + + "SORT BY seller_id", stmt.toString()); + + assertEquals("SELECT *\n" + + "FROM LD_aly.fct_pay_ord_cn_di t1\n" + + "DISTRIBUTE BY buyer_id\n" + + "SORT BY seller_id", SQLUtils.toSQLString(stmt)); + + assertEquals(1, statementList.size()); + + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + System.out.println("groupBy : " + visitor.getGroupByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(3, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + assertEquals(0, visitor.getGroupByColumns().size()); + + assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "*")); + assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "buyer_id")); + assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "seller_id")); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_with_20433301.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_with_20433301.java new file mode 100644 index 0000000000..16b226eb72 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_with_20433301.java @@ -0,0 +1,106 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.mysql.select; + +import com.alibaba.druid.sql.MysqlTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; + + +public class MySqlSelectTest_with_20433301 extends MysqlTest { + + public void test_0() throws Exception { + String sql = "WITH total AS(\n" + + "SELECT cinema_id, count(1) as scheduleCnt, sum(hall_seat_cnt) as hall_seat_cnt, sum(sold_seat_cnt) as sold_seat_cnt, sum(avg_ticket_price*sold_seat_cnt) as box_office\n" + + "  FROM gw_ads_schedule_rt\n" + + " WHERE show_date= 20190513\n" + + "   and stat_month= 201905\n" + + "   and stat_date= 20190508\n" + + "   and scd_is_valid= 1\n" + + "   and scd_status= 4\n" + + " GROUP BY cinema_id),\n" + + "         detail AS(\n" + + "SELECT show_id, cinema_id, count(1) as scheduleCnt, sum(hall_seat_cnt) as hall_seat_cnt, sum(sold_seat_cnt) as sold_seat_cnt, sum(avg_ticket_price*sold_seat_cnt) as box_office\n" + + "  FROM gw_ads_schedule_rt\n" + + " WHERE show_date= 20190513\n" + + "   and stat_month= 201905\n" + + "   and stat_date= 20190508\n" + + "   and scd_is_valid= 1\n" + + "   and scd_status= 4\n" + + "   AND show_id IN(1229077)\n" + + " GROUP BY show_id, cinema_id)\n" + + "SELECT detail.cinema_id,\n" + + "       detail.show_id,\n" + + "       detail.scheduleCnt,\n" + + "       detail.hall_seat_cnt,\n" + + "       detail.sold_seat_cnt,\n" + + "       detail.box_office,\n" + + "       total.scheduleCnt as totalScheduleCnt,\n" + + "       total.hall_seat_cnt as totalHallSeatCnt,\n" + + "       total.sold_seat_cnt as totalSoldSeatCnt,\n" + + "       total.box_office as totalBoxOffice,\n" + + "       fcst.scd_cnt_fcst as predictScheduleCnt\n" + + "  FROM detail join total on detail.cinema_id= total.cinema_id left join gw_ads_cinema_schedule_fcst fcst on fcst.cinema_id= detail.cinema_id\n" + + " where fcst.week_day= 1\n" + + " order by detail.scheduleCnt desc"; + + + SQLSelectStatement stmt = (SQLSelectStatement) SQLUtils.parseSingleStatement(sql, JdbcConstants.MYSQL); + + assertEquals("WITH total AS (\n" + + "\t\tSELECT cinema_id, count(1) AS scheduleCnt, sum(hall_seat_cnt) AS hall_seat_cnt\n" + + "\t\t\t, sum(sold_seat_cnt) AS sold_seat_cnt\n" + + "\t\t\t, sum(avg_ticket_price * sold_seat_cnt) AS box_office\n" + + "\t\tFROM gw_ads_schedule_rt\n" + + "\t\tWHERE show_date = 20190513\n" + + "\t\t\tAND stat_month = 201905\n" + + "\t\t\tAND stat_date = 20190508\n" + + "\t\t\tAND scd_is_valid = 1\n" + + "\t\t\tAND scd_status = 4\n" + + "\t\tGROUP BY cinema_id\n" + + "\t), \n" + + "\tdetail AS (\n" + + "\t\tSELECT show_id, cinema_id, count(1) AS scheduleCnt\n" + + "\t\t\t, sum(hall_seat_cnt) AS hall_seat_cnt, sum(sold_seat_cnt) AS sold_seat_cnt\n" + + "\t\t\t, sum(avg_ticket_price * sold_seat_cnt) AS box_office\n" + + "\t\tFROM gw_ads_schedule_rt\n" + + "\t\tWHERE show_date = 20190513\n" + + "\t\t\tAND stat_month = 201905\n" + + "\t\t\tAND stat_date = 20190508\n" + + "\t\t\tAND scd_is_valid = 1\n" + + "\t\t\tAND scd_status = 4\n" + + "\t\t\tAND show_id IN (1229077)\n" + + "\t\tGROUP BY show_id, cinema_id\n" + + "\t)\n" + + "SELECT detail.cinema_id, detail.show_id, detail.scheduleCnt, detail.hall_seat_cnt, detail.sold_seat_cnt\n" + + "\t, detail.box_office, total.scheduleCnt AS totalScheduleCnt, total.hall_seat_cnt AS totalHallSeatCnt, total.sold_seat_cnt AS totalSoldSeatCnt, total.box_office AS totalBoxOffice\n" + + "\t, fcst.scd_cnt_fcst AS predictScheduleCnt\n" + + "FROM detail\n" + + "\tJOIN total ON detail.cinema_id = total.cinema_id\n" + + "\tLEFT JOIN gw_ads_cinema_schedule_fcst fcst ON fcst.cinema_id = detail.cinema_id\n" + + "WHERE fcst.week_day = 1\n" + + "ORDER BY detail.scheduleCnt DESC", stmt.toString()); + + + MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor(); + stmt.accept(visitor); + } + + + +} \ No newline at end of file diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGUnionTest.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGUnionTest.java new file mode 100644 index 0000000000..ca4310375b --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGUnionTest.java @@ -0,0 +1,113 @@ +package com.alibaba.druid.bvt.sql.postgresql.select; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; +import junit.framework.TestCase; +import org.junit.Assert; + +public class PGUnionTest + extends TestCase { + + public void testUnion(){ + String sql = "(select id,name from t1) union (select id,name from t2)"; + String targetSql = "(SELECT id, name\n" + + "FROM t1)\n" + + "UNION\n" + + "(SELECT id, name\n" + + "FROM t2)"; + + PGSQLStatementParser pgparser=new PGSQLStatementParser(sql); + SQLStatement pgstatement = pgparser.parseStatement(); + Assert.assertEquals(targetSql, pgstatement.toString()); + System.out.println(pgstatement.toString()); + } +// +// public void testUnion1(){ +// String sql = "select * from test_a where id=(select id from test_b where id = 1) union all select * from test_c"; +// String targetSql = "SELECT *\n" + +// "FROM test_a\n" + +// "WHERE id = (\n" + +// "\tSELECT id\n" + +// "\tFROM test_b\n" + +// "\tWHERE id = 1\n" + +// ")\n" + +// "UNION ALL\n" + +// "SELECT *\n" + +// "FROM test_c"; +// +// PGSQLStatementParser pgparser=new PGSQLStatementParser(sql); +// SQLStatement pgstatement = pgparser.parseStatement(); +// Assert.assertEquals(targetSql, pgstatement.toString()); +// System.out.println(pgstatement.toString()); +// } + + public void testUnion2(){ + String sql = "select \n" + + "( \n" + + " ( \n" + + " select 'y' from dual \n" + + " where exists ( select 1 from dual where 1 = 0 ) \n" + + " ) \n" + + " union \n" + + " ( \n" + + " select 'n' from dual \n" + + " where not exists ( select 1 from dual where 1 = 0 ) \n" + + " ) \n" + + ") \n" + + "as yes_no \n" + + "from dual"; + String targetSql = "SELECT (\n" + + "\t\tSELECT 'y'\n" + + "\t\tFROM dual\n" + + "\t\tWHERE EXISTS (\n" + + "\t\t\tSELECT 1\n" + + "\t\t\tFROM dual\n" + + "\t\t\tWHERE 1 = 0\n" + + "\t\t)\n" + + "\t\tUNION\n" + + "\t\t(SELECT 'n'\n" + + "\t\tFROM dual\n" + + "\t\tWHERE NOT EXISTS (\n" + + "\t\t\tSELECT 1\n" + + "\t\t\tFROM dual\n" + + "\t\t\tWHERE 1 = 0\n" + + "\t\t))\n" + + "\t) AS yes_no\n" + + "FROM dual"; + + PGSQLStatementParser pgparser=new PGSQLStatementParser(sql); + SQLStatement pgstatement = pgparser.parseStatement(); + Assert.assertEquals(targetSql, pgstatement.toString()); + System.out.println(pgstatement.toString()); + } + + public void testUnion3(){ + String sql = "select * from v.e\n" + + "where\n" + + "\tcid <> rid\n" + + "\tand rid not in\n" + + "\t(\n" + + "\t\t(select distinct rid from v.s )\n" + + "\t\tunion\n" + + "\t\t(select distinct rid from v.p )\n" + + "\t)\n" + + "\tand \"timestamp\" <= 1298505600000"; + String targetSql = "SELECT *\n" + + "FROM v.e\n" + + "WHERE cid <> rid\n" + + "\tAND rid NOT IN (\n" + + "\t\tSELECT DISTINCT rid\n" + + "\t\tFROM v.s\n" + + "\t\tUNION\n" + + "\t\t(SELECT DISTINCT rid\n" + + "\t\tFROM v.p)\n" + + "\t)\n" + + "\tAND 'timestamp' <= 1298505600000"; + + SQLStatement pgstatement = SQLUtils.parseSingleMysqlStatement(sql); + Assert.assertEquals(targetSql, pgstatement.toString()); + System.out.println(pgstatement.toString()); + } + +} \ No newline at end of file