From 3df15cbaed5362ae3f2fa438b62bdf41535b603a Mon Sep 17 00:00:00 2001 From: wenshao Date: Sun, 1 Apr 2018 13:39:38 +0800 Subject: [PATCH] improved sql parser. bug fixed for issue #2446 --- .../ast/statement/SQLSelectQueryBlock.java | 36 ++++ .../ast/statement/SQLValuesTableSource.java | 83 +++++++++ .../dialect/db2/parser/DB2SelectParser.java | 21 +-- .../dialect/hive/parser/HiveSelectParser.java | 30 ++++ .../ast/statement/MySqlSelectQueryBlock.java | 20 --- .../mysql/parser/MySqlSelectParser.java | 22 ++- .../odps/ast/OdpsSelectQueryBlock.java | 17 -- .../odps/visitor/OdpsOutputVisitor.java | 2 +- .../ast/stmt/OracleSelectQueryBlock.java | 19 -- .../parser/SQLServerSelectParser.java | 7 +- .../alibaba/druid/sql/parser/Keywords.java | 3 +- .../druid/sql/parser/SQLSelectParser.java | 167 +++++++++++++++--- .../sql/visitor/SQLASTOutputVisitor.java | 116 +++++++++++- .../druid/sql/visitor/SQLASTVisitor.java | 3 + .../sql/visitor/SQLASTVisitorAdapter.java | 9 + .../sql/clickhouse/ClickHouse_select_0.java | 44 +++++ .../bvt/sql/cobar/DMLSelectParserTest.java | 3 +- .../bvt/sql/postgresql/PGDeleteTest10.java | 56 ++++++ .../bvt/sql/schemaStat/SchemaStatTest14.java | 6 +- 19 files changed, 547 insertions(+), 117 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/sql/ast/statement/SQLValuesTableSource.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouse_select_0.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGDeleteTest10.java diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java index 6e639f2b30..fd82f41bb7 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java @@ -56,6 +56,7 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery protected String cachedSelectList; // optimized for SelectListCache protected long cachedSelectListHash; // optimized for SelectListCache + protected List hints; protected String dbType; public SQLSelectQueryBlock(){ @@ -322,6 +323,16 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery return sortBy; } + public void addSortBy(SQLSelectOrderByItem item) { + if (sortBy == null) { + sortBy = new ArrayList(); + } + if (item != null) { + item.setParent(this); + } + this.sortBy.add(item); + } + @Override protected void accept0(SQLASTVisitor visitor) { if (visitor.visit(this)) { @@ -333,6 +344,8 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery acceptChild(visitor, this.connectBy); acceptChild(visitor, this.groupBy); acceptChild(visitor, this.orderBy); + acceptChild(visitor, this.distributeBy); + acceptChild(visitor, this.sortBy); acceptChild(visitor, this.waitTime); acceptChild(visitor, this.limit); } @@ -676,6 +689,29 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery return cachedSelectListHash; } + public List getHintsDirect() { + return hints; + } + + public List getHints() { + if (hints == null) { + hints = new ArrayList(2); + } + return hints; + } + + public void setHints(List hints) { + this.hints = hints; + } + + public int getHintsSize() { + if (hints == null) { + return 0; + } + + return hints.size(); + } + public String getDbType() { return dbType; } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLValuesTableSource.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLValuesTableSource.java new file mode 100644 index 0000000000..b8c9f3422f --- /dev/null +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLValuesTableSource.java @@ -0,0 +1,83 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.ast.statement; + +import com.alibaba.druid.sql.ast.SQLHint; +import com.alibaba.druid.sql.ast.SQLName; +import com.alibaba.druid.sql.ast.expr.SQLListExpr; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by wenshao on 23/02/2017. + */ +public class SQLValuesTableSource extends SQLTableSourceImpl { + private List values = new ArrayList(); + private List columns = new ArrayList(); + + public List getValues() { + return values; + } + + public List getColumns() { + return columns; + } + + @Override + public void accept0(SQLASTVisitor visitor) { + if (visitor.visit(this)) { + acceptChild(visitor, values); + acceptChild(visitor, columns); + } + visitor.endVisit(this); + } + + @Override + public SQLValuesTableSource clone() { + + SQLValuesTableSource x = new SQLValuesTableSource(); + + x.setAlias(this.alias); + + for (SQLListExpr e : this.values) { + SQLListExpr e2 = e.clone(); + e2.setParent(x); + x.getValues().add(e2); + } + + for (SQLName e : this.columns) { + SQLName e2 = e.clone(); + e2.setParent(x); + x.getColumns().add(e2); + } + + if (this.flashback != null) { + x.setFlashback(this.flashback.clone()); + } + + if (this.hints != null) { + for (SQLHint e : this.hints) { + SQLHint e2 = e.clone(); + e2.setParent(x); + x.getHints().add(e2); + } + } + + return x; + } +} diff --git a/src/main/java/com/alibaba/druid/sql/dialect/db2/parser/DB2SelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/db2/parser/DB2SelectParser.java index c5b46e9bcb..0fafa6fa33 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/db2/parser/DB2SelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/db2/parser/DB2SelectParser.java @@ -16,6 +16,7 @@ package com.alibaba.druid.sql.dialect.db2.parser; import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.SQLOrderBy; import com.alibaba.druid.sql.ast.SQLSetQuantifier; import com.alibaba.druid.sql.ast.statement.SQLSelectQuery; @@ -45,14 +46,14 @@ public class DB2SelectParser extends SQLSelectParser { } @Override - public SQLSelectQuery query() { + public SQLSelectQuery query(SQLObject parent, boolean acceptUnion) { if (lexer.token() == Token.LPAREN) { lexer.nextToken(); SQLSelectQuery select = query(); accept(Token.RPAREN); - return queryRest(select); + return queryRest(select, acceptUnion); } accept(Token.SELECT); @@ -83,7 +84,7 @@ public class DB2SelectParser extends SQLSelectParser { parseHierachical(queryBlock); parseGroupBy(queryBlock); - + if (lexer.token() == Token.ORDER) { SQLOrderBy orderBy = parseOrderBy(); queryBlock.setOrderBy(orderBy); @@ -102,7 +103,7 @@ public class DB2SelectParser extends SQLSelectParser { accept(Token.ONLY); continue; } - + if (lexer.token() == Token.WITH) { lexer.nextToken(); if (lexer.identifierEquals("RR")) { @@ -119,10 +120,10 @@ public class DB2SelectParser extends SQLSelectParser { lexer.nextToken(); continue; } - + if (lexer.token() == Token.FOR) { lexer.nextToken(); - + if (lexer.token() == Token.UPDATE) { queryBlock.setForUpdate(true); lexer.nextToken(); @@ -132,11 +133,11 @@ public class DB2SelectParser extends SQLSelectParser { queryBlock.setForReadOnly(true); } } - + if (lexer.token() == Token.OPTIMIZE) { lexer.nextToken(); accept(Token.FOR); - + queryBlock.setOptimizeFor(this.expr()); if (lexer.identifierEquals("ROW")) { lexer.nextToken(); @@ -144,10 +145,10 @@ public class DB2SelectParser extends SQLSelectParser { acceptIdentifier("ROWS"); } } - + break; } - return queryRest(queryBlock); + return queryRest(queryBlock, acceptUnion); } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java index c24aacd5b9..ec0ac34484 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveSelectParser.java @@ -15,7 +15,9 @@ */ package com.alibaba.druid.sql.dialect.hive.parser; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLOrderBy; +import com.alibaba.druid.sql.ast.SQLOrderingSpecification; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.SQLSelectListCache; @@ -40,4 +42,32 @@ public class HiveSelectParser extends SQLSelectParser { return new HiveExprParser(lexer); } + protected void parseSortBy(SQLSelectQueryBlock queryBlock) { + if (lexer.token() == Token.SORT) { + lexer.nextToken(); + accept(Token.BY); + for (;;) { + SQLExpr expr = this.expr(); + + SQLSelectOrderByItem sortByItem = new SQLSelectOrderByItem(expr); + + if (lexer.token() == Token.ASC) { + sortByItem.setType(SQLOrderingSpecification.ASC); + lexer.nextToken(); + } else if (lexer.token() == Token.DESC) { + sortByItem.setType(SQLOrderingSpecification.DESC); + lexer.nextToken(); + } + + queryBlock.addSortBy(sortByItem); + + if (lexer.token() == Token.COMMA) { + lexer.nextToken(); + } else { + break; + } + } + } + } + } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlSelectQueryBlock.java index ab9eee1a72..5a69e857d6 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlSelectQueryBlock.java @@ -39,7 +39,6 @@ public class MySqlSelectQueryBlock extends SQLSelectQueryBlock implements MySqlO private SQLName procedureName; private List procedureArgumentList; private boolean lockInShareMode; - private List hints; private SQLName forcePartition; // for petadata public MySqlSelectQueryBlock(){ @@ -74,25 +73,6 @@ public class MySqlSelectQueryBlock extends SQLSelectQueryBlock implements MySqlO return x; } - public int getHintsSize() { - if (hints == null) { - return 0; - } - - return hints.size(); - } - - public List getHints() { - if (hints == null) { - hints = new ArrayList(2); - } - return hints; - } - - public void setHints(List hints) { - this.hints = hints; - } - public boolean isLockInShareMode() { return lockInShareMode; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java index d4645d35c5..42f46eee7c 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java @@ -17,6 +17,7 @@ package com.alibaba.druid.sql.dialect.mysql.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; +import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.SQLSetQuantifier; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLListExpr; @@ -75,9 +76,9 @@ public class MySqlSelectParser extends SQLSelectParser { queryBlock.setFrom(parseTableSource()); } - + @Override - public SQLSelectQuery query() { + public SQLSelectQuery query(SQLObject parent, boolean acceptUnion) { if (lexer.token() == Token.LPAREN) { lexer.nextToken(); @@ -85,10 +86,11 @@ public class MySqlSelectParser extends SQLSelectParser { select.setBracket(true); accept(Token.RPAREN); - return queryRest(select); + return queryRest(select, acceptUnion); } MySqlSelectQueryBlock queryBlock = new MySqlSelectQueryBlock(); + queryBlock.setParent(parent); if (lexer.hasComment() && lexer.isKeepComments()) { queryBlock.addBeforeComment(lexer.readAndResetComments()); @@ -103,8 +105,12 @@ public class MySqlSelectParser extends SQLSelectParser { if (lexer.token() == Token.SELECT) { lexer.nextTokenValue(); - if (lexer.token() == Token.HINT) { - this.exprParser.parseHints(queryBlock.getHints()); + for(;;) { + if (lexer.token() == Token.HINT) { + this.exprParser.parseHints(queryBlock.getHints()); + } else { + break; + } } Token token = lexer.token(); @@ -167,7 +173,7 @@ public class MySqlSelectParser extends SQLSelectParser { SQLName partition = this.exprParser.name(); queryBlock.setForcePartition(partition); } - + parseInto(queryBlock); } @@ -197,7 +203,7 @@ public class MySqlSelectParser extends SQLSelectParser { accept(Token.UPDATE); queryBlock.setForUpdate(true); - + if (lexer.identifierEquals(FnvHash.Constants.NO_WAIT) || lexer.identifierEquals(FnvHash.Constants.NOWAIT)) { lexer.nextToken(); queryBlock.setNoWait(true); @@ -216,7 +222,7 @@ public class MySqlSelectParser extends SQLSelectParser { queryBlock.setLockInShareMode(true); } - return queryRest(queryBlock); + return queryRest(queryBlock, acceptUnion); } public SQLTableSource parseTableSource() { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java index 1fa640b212..9892f90cb3 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java @@ -34,8 +34,6 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { private SQLOrderBy orderBy; - protected List hints; - public OdpsSelectQueryBlock(){ dbType = JdbcConstants.ODPS; @@ -72,21 +70,6 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock { return true; } - public List getHintsDirect() { - return hints; - } - - public List getHints() { - if (hints == null) { - hints = new ArrayList(2); - } - return hints; - } - - public void setHints(List hints) { - this.hints = hints; - } - @Override protected void accept0(SQLASTVisitor visitor) { if (visitor instanceof OdpsASTVisitor) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java index a27ecd41ea..25665b6148 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java @@ -455,7 +455,7 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis print0(ucase ? "SELECT " : "select "); - List hints = x.getHintsDirect(); + List hints = x.getHintsDirect(); if (hints != null) { printAndAccept(hints, " "); print(' '); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleSelectQueryBlock.java index 8526f67c8e..838b5fcff9 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleSelectQueryBlock.java @@ -35,11 +35,7 @@ import com.alibaba.druid.util.JdbcConstants; public class OracleSelectQueryBlock extends SQLSelectQueryBlock implements OracleSQLObject { - private List hints; - private ModelClause modelClause; - - private boolean skipLocked = false; public OracleSelectQueryBlock clone() { @@ -84,21 +80,6 @@ public class OracleSelectQueryBlock extends SQLSelectQueryBlock implements Oracl this.modelClause = modelClause; } - public List getHints() { - if (hints == null) { - hints = new ArrayList(1); - } - return this.hints; - } - - public int getHintsSize() { - if (hints == null) { - return 0; - } - - return hints.size(); - } - public boolean isSkipLocked() { return skipLocked; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerSelectParser.java index b0a8be6bba..c90e5f1086 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerSelectParser.java @@ -16,6 +16,7 @@ package com.alibaba.druid.sql.dialect.sqlserver.parser; import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.SQLSetQuantifier; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.sqlserver.ast.SQLServerSelectQueryBlock; @@ -115,14 +116,14 @@ public class SQLServerSelectParser extends SQLSelectParser { return select; } - public SQLSelectQuery query() { + public SQLSelectQuery query(SQLObject parent, boolean acceptUnion) { if (lexer.token() == Token.LPAREN) { lexer.nextToken(); SQLSelectQuery select = query(); accept(Token.RPAREN); - return queryRest(select); + return queryRest(select, acceptUnion); } SQLServerSelectQueryBlock queryBlock = new SQLServerSelectQueryBlock(); @@ -167,7 +168,7 @@ public class SQLServerSelectParser extends SQLSelectParser { parseFetchClause(queryBlock); - return queryRest(queryBlock); + return queryRest(queryBlock, acceptUnion); } protected SQLServerExprParser createExprParser() { diff --git a/src/main/java/com/alibaba/druid/sql/parser/Keywords.java b/src/main/java/com/alibaba/druid/sql/parser/Keywords.java index 3c00b37989..a8cbd5d391 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Keywords.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Keywords.java @@ -160,7 +160,8 @@ public class Keywords { map.put("FETCH", Token.FETCH); map.put("OUT", Token.OUT); map.put("INOUT", Token.INOUT); - + map.put("LIMIT", Token.LIMIT); + DEFAULT_KEYWORDS = new Keywords(map); Map sqlitemap = new HashMap(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java index 283260f467..8e1f396290 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java @@ -100,23 +100,62 @@ public class SQLSelectParser extends SQLParser { } public SQLSelectQuery queryRest(SQLSelectQuery selectQuery) { + return queryRest(selectQuery, true); + } + + public SQLSelectQuery queryRest(SQLSelectQuery selectQuery, boolean acceptUnion) { + if (!acceptUnion) { + return selectQuery; + } + if (lexer.token == Token.UNION) { - lexer.nextToken(); - - SQLUnionQuery union = createSQLUnionQuery(); - union.setLeft(selectQuery); - - if (lexer.token == Token.ALL) { - union.setOperator(SQLUnionOperator.UNION_ALL); + do { lexer.nextToken(); - } else if (lexer.token == Token.DISTINCT) { - union.setOperator(SQLUnionOperator.DISTINCT); - lexer.nextToken(); - } - SQLSelectQuery right = this.query(); - union.setRight(right); - return unionRest(union); + SQLUnionQuery union = createSQLUnionQuery(); + if (union.getLeft() == null) { + union.setLeft(selectQuery); + } + + boolean paren = lexer.token == Token.LPAREN; + + if (lexer.token == Token.ALL) { + union.setOperator(SQLUnionOperator.UNION_ALL); + lexer.nextToken(); + } else if (lexer.token == Token.DISTINCT) { + union.setOperator(SQLUnionOperator.DISTINCT); + lexer.nextToken(); + } + SQLSelectQuery right = this.query(paren ? null : union, false); + union.setRight(right); + + if (!paren) { + if (right instanceof SQLSelectQueryBlock) { + SQLSelectQueryBlock rightQuery = (SQLSelectQueryBlock) right; + SQLOrderBy orderBy = rightQuery.getOrderBy(); + SQLLimit limit = rightQuery.getLimit(); + if (orderBy != null && limit == null) { + union.setOrderBy(orderBy); + rightQuery.setOrderBy(null); + } + } else if (right instanceof SQLUnionQuery) { + SQLUnionQuery rightUnion = (SQLUnionQuery) right; + if (rightUnion.getOrderBy() != null) { + union.setOrderBy(rightUnion.getOrderBy()); + rightUnion.setOrderBy(null); + } + } + } + + union = unionRest(union); + + selectQuery = union; + + } while (lexer.token() == Token.UNION); + + selectQuery = queryRest(selectQuery, true); + + return selectQuery; } if (lexer.token == Token.EXCEPT) { @@ -127,10 +166,10 @@ public class SQLSelectParser extends SQLParser { union.setOperator(SQLUnionOperator.EXCEPT); - SQLSelectQuery right = this.query(); + SQLSelectQuery right = this.query(union, false); union.setRight(right); - return union; + return queryRest(union, true); } if (lexer.token == Token.INTERSECT) { @@ -141,13 +180,13 @@ public class SQLSelectParser extends SQLParser { union.setOperator(SQLUnionOperator.INTERSECT); - SQLSelectQuery right = this.query(); + SQLSelectQuery right = this.query(union, false); union.setRight(right); - return union; + return queryRest(union, true); } - if (lexer.token == Token.MINUS) { + if (acceptUnion && lexer.token == Token.MINUS) { lexer.nextToken(); SQLUnionQuery union = new SQLUnionQuery(); @@ -155,33 +194,45 @@ public class SQLSelectParser extends SQLParser { union.setOperator(SQLUnionOperator.MINUS); - SQLSelectQuery right = this.query(); + SQLSelectQuery right = this.query(union, false); union.setRight(right); - return union; + return queryRest(union, true); } return selectQuery; } public SQLSelectQuery query() { + return query(null, true); + } + + public SQLSelectQuery query(SQLObject parent) { + return query(parent, true); + } + + public SQLSelectQuery query(SQLObject parent, boolean acceptUnion) { if (lexer.token == Token.LPAREN) { lexer.nextToken(); SQLSelectQuery select = query(); accept(Token.RPAREN); - return queryRest(select); + return queryRest(select, acceptUnion); } - + SQLSelectQueryBlock queryBlock = new SQLSelectQueryBlock(); - + if (lexer.hasComment() && lexer.isKeepComments()) { queryBlock.addBeforeComment(lexer.readAndResetComments()); } accept(Token.SELECT); + if (lexer.token() == Token.HINT) { + this.exprParser.parseHints(queryBlock.getHints()); + } + if (lexer.token == Token.COMMENT) { lexer.nextToken(); } @@ -219,9 +270,15 @@ public class SQLSelectParser extends SQLParser { parseGroupBy(queryBlock); + parseSortBy(queryBlock); + parseFetchClause(queryBlock); - return queryRest(queryBlock); + return queryRest(queryBlock, acceptUnion); + } + + protected void parseSortBy(SQLSelectQueryBlock queryBlock) { + } protected void withSubquery(SQLSelect select) { @@ -533,10 +590,10 @@ public class SQLSelectParser extends SQLParser { lexer.nextToken(); SQLTableSource tableSource; if (lexer.token == Token.SELECT || lexer.token == Token.WITH - || lexer.token == Token.SEL) { + || lexer.token == Token.SEL) { SQLSelect select = select(); accept(Token.RPAREN); - SQLSelectQuery query = queryRest(select.getQuery()); + SQLSelectQuery query = queryRest(select.getQuery(), true); if (query instanceof SQLUnionQuery) { tableSource = new SQLUnionQueryTableSource((SQLUnionQuery) query); } else { @@ -550,9 +607,61 @@ public class SQLSelectParser extends SQLParser { accept(Token.RPAREN); } + if (lexer.token == Token.AS + && tableSource instanceof SQLValuesTableSource + && ((SQLValuesTableSource) tableSource).getColumns().size() == 0) + { + lexer.nextToken(); + + String alias = this.tableAlias(); + tableSource.setAlias(alias); + + SQLValuesTableSource values = (SQLValuesTableSource) tableSource; + accept(Token.LPAREN); + this.exprParser.names(values.getColumns(), values); + accept(Token.RPAREN); + } + return parseTableSourceRest(tableSource); } + if (lexer.token() == Token.VALUES) { + lexer.nextToken(); + SQLValuesTableSource tableSource = new SQLValuesTableSource(); + + for (;;) { + accept(Token.LPAREN); + SQLListExpr listExpr = new SQLListExpr(); + this.exprParser.exprList(listExpr.getItems(), listExpr); + accept(Token.RPAREN); + + listExpr.setParent(tableSource); + + tableSource.getValues().add(listExpr); + + if (lexer.token == Token.COMMA) { + lexer.nextToken(); + continue; + } + break; + } + + if (lexer.token == Token.RPAREN) { + return tableSource; + } + + String alias = this.tableAlias(); + if (alias != null) { + tableSource.setAlias(alias); + } + + accept(Token.LPAREN); + this.exprParser.names(tableSource.getColumns(), tableSource); + accept(Token.RPAREN); + + return tableSource; + } + if (lexer.token == Token.SELECT) { throw new ParserException("TODO " + lexer.info()); } @@ -562,11 +671,11 @@ public class SQLSelectParser extends SQLParser { parseTableSourceQueryTableExpr(tableReference); SQLTableSource tableSrc = parseTableSourceRest(tableReference); - + if (lexer.hasComment() && lexer.isKeepComments()) { tableSrc.addAfterComment(lexer.readAndResetComments()); } - + return tableSrc; } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 85e1711793..2de225c01b 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -2708,21 +2708,106 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet @Override public boolean visit(SQLUnionQuery x) { SQLUnionOperator operator = x.getOperator(); + SQLSelectQuery left = x.getLeft(); + SQLSelectQuery right = x.getRight(); boolean bracket = x.isBracket() && !(x.getParent() instanceof SQLUnionQueryTableSource); + SQLOrderBy orderBy = x.getOrderBy(); + if ((!bracket) + && left instanceof SQLUnionQuery + && ((SQLUnionQuery) left).getOperator() == operator + && !right.isBracket() + && orderBy == null) { + + SQLUnionQuery leftUnion = (SQLUnionQuery) left; + + List rights = new ArrayList(); + rights.add(right); + + for (;;) { + SQLSelectQuery leftLeft = leftUnion.getLeft(); + SQLSelectQuery leftRight = leftUnion.getRight(); + + if ((!leftUnion.isBracket()) + && leftUnion.getOrderBy() == null + && (!leftLeft.isBracket()) + && (!leftRight.isBracket()) + && leftLeft instanceof SQLUnionQuery + && ((SQLUnionQuery) leftLeft).getOperator() == operator) { + rights.add(leftRight); + leftUnion = (SQLUnionQuery) leftLeft; + continue; + } else { + rights.add(leftRight); + rights.add(leftLeft); + } + break; + } + + for (int i = rights.size() - 1; i >= 0; i--) { + SQLSelectQuery item = rights.get(i); + item.accept(this); + + if (i > 0) { + println(); + print0(ucase ? operator.name : operator.name_lcase); + println(); + } + } + return false; + } + if (bracket) { print('('); } - x.getLeft().accept(this); + + if (left != null) { + for (;;) { + if (left.getClass() == SQLUnionQuery.class) { + SQLUnionQuery leftUnion = (SQLUnionQuery) left; + SQLSelectQuery leftLeft = leftUnion.getLeft(); + SQLSelectQuery leftRigt = leftUnion.getRight(); + if ((!leftUnion.isBracket()) + && leftUnion.getRight() instanceof SQLSelectQueryBlock + && leftUnion.getLeft() != null + && leftUnion.getOrderBy() == null) + { + if (leftLeft.getClass() == SQLUnionQuery.class) { + visit((SQLUnionQuery) leftLeft); + } else { + printQuery(leftLeft); + } + println(); + print0(ucase ? leftUnion.getOperator().name : leftUnion.getOperator().name_lcase); + println(); + leftRigt.accept(this); + } else { + visit(leftUnion); + } + } else { + left.accept(this); + } + break; + } + } + + if (right == null) { + return false; + } println(); print0(ucase ? operator.name : operator.name_lcase); println(); - SQLSelectQuery right = x.getRight(); - SQLOrderBy orderBy = x.getOrderBy(); - boolean needParen = orderBy != null && !right.isBracket(); + boolean needParen = false; + if (orderBy != null + && (!right.isBracket()) && right instanceof SQLSelectQueryBlock) { + SQLSelectQueryBlock rightQuery = (SQLSelectQueryBlock) right; + if (rightQuery.getOrderBy() != null || rightQuery.getLimit() != null) { + needParen = true; + } + } if (needParen) { print('('); @@ -6318,4 +6403,27 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet printAndAccept(x.getValues(), ", "); return false; } + + @Override + public boolean visit(SQLValuesTableSource x) { + List columns = x.getColumns(); + + if (columns.size() > 0) { + print('('); + } + print0(ucase ? "VALUES " : "values "); + printAndAccept(x.getValues(), ", "); + + if (columns.size() > 0) { + print(") "); + } + + print0(ucase ? "AS " : "as "); + print0(x.getAlias()); + print0(" ("); + printAndAccept(columns, ", "); + print(')'); + + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index ba8b646acb..ce544d62ad 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -829,4 +829,7 @@ public interface SQLASTVisitor { boolean visit(SQLValuesExpr x); void endVisit(SQLValuesExpr x); + + boolean visit(SQLValuesTableSource x); + void endVisit(SQLValuesTableSource x); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java index b341bb681f..3b05206bb9 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java @@ -1964,6 +1964,15 @@ public class SQLASTVisitorAdapter implements SQLASTVisitor { } + @Override + public boolean visit(SQLValuesTableSource x) { + return true; + } + + public void endVisit(SQLValuesTableSource x) { + + } + public final boolean isEnabled(VisitorFeature feature) { return VisitorFeature.isEnabled(this.features, feature); } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouse_select_0.java b/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouse_select_0.java new file mode 100644 index 0000000000..dce9d9725d --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/clickhouse/ClickHouse_select_0.java @@ -0,0 +1,44 @@ +package com.alibaba.druid.bvt.sql.clickhouse; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.db2.visitor.DB2SchemaStatVisitor; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class ClickHouse_select_0 extends TestCase { + public void test_0() throws Exception { + String sql = "SELECT date, transactionChannel, tranactionType FROM preComp_3All_20180322 limit 1,10"; + + List statementList = SQLUtils.parseStatements(sql, JdbcConstants.CLICKHOUSE); + SQLStatement stmt = statementList.get(0); + + assertEquals(1, statementList.size()); + + DB2SchemaStatVisitor visitor = new DB2SchemaStatVisitor(); + stmt.accept(visitor); + +// System.out.println("Tables : " + visitor.getTables()); +// System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); +// System.out.println("orderBy : " + visitor.getOrderByColumns()); + + assertEquals(1, visitor.getTables().size()); + assertEquals(3, visitor.getColumns().size()); + assertEquals(0, visitor.getConditions().size()); + + assertTrue(visitor.containsTable("preComp_3All_20180322")); + + // assertTrue(visitor.getColumns().contains(new Column("mytable", "last_name"))); + // assertTrue(visitor.getColumns().contains(new Column("mytable", "first_name"))); + // assertTrue(visitor.getColumns().contains(new Column("mytable", "full_name"))); + + String output = SQLUtils.toSQLString(stmt, JdbcConstants.CLICKHOUSE); + assertEquals("SELECT date, transactionChannel, tranactionType\n" + + "FROM preComp_3All_20180322\n" + + "LIMIT 1, 10", // + output); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLSelectParserTest.java b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLSelectParserTest.java index a601d56601..6cd6f33218 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLSelectParserTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLSelectParserTest.java @@ -39,7 +39,8 @@ public class DMLSelectParserTest extends TestCase { "UNION ALL\n" + "(SELECT id\n" + "FROM t3)\n" + - "ORDER BY d DESC", output); + "ORDER BY d DESC\n" + + "LIMIT ?, 1", output); } public void test_union_1() throws Exception { diff --git a/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGDeleteTest10.java b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGDeleteTest10.java new file mode 100644 index 0000000000..d76e3715a1 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGDeleteTest10.java @@ -0,0 +1,56 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.postgresql; + +import com.alibaba.druid.sql.PGTest; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; +import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor; +import com.alibaba.druid.stat.TableStat; + +import java.util.List; + +public class PGDeleteTest10 extends PGTest { + + public void test_0() throws Exception { + String sql = "delete from account a using (values (1), (7)) as tmp(id) WHERE a.id = tmp.id"; + + PGSQLStatementParser parser = new PGSQLStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + + assertEquals("DELETE FROM account AS a\n" + + "USING (VALUES (1), (7)) AS tmp (id)\n" + + "WHERE a.id = tmp.id", stmt.toString()); + + assertEquals(1, statementList.size()); + + PGSchemaStatVisitor visitor = new PGSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); +// System.out.println("coditions : " + visitor.getConditions()); + + assertTrue(visitor.getTables().containsKey(new TableStat.Name("account"))); + + assertEquals(2, visitor.getColumns().size() ); + + assertTrue(visitor.containsColumn("account", "id")); + } + + +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest14.java b/src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest14.java index f4400d27c3..f0ae03170b 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest14.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest14.java @@ -17,13 +17,11 @@ public class SchemaStatTest14 extends TestCase { public void test_schemaStat() throws Exception { String sql = "delete r from t_res r where id=1 "; - String dbType = JdbcConstants.MYSQL; - SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType); - SQLStatement stmt = parser.parseStatementList().get(0); + SQLStatement stmt = SQLUtils.parseStatements(sql, JdbcConstants.MYSQL).get(0); System.out.println(stmt); - SchemaStatVisitor statVisitor = SQLUtils.createSchemaStatVisitor(dbType); + SchemaStatVisitor statVisitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.MYSQL); stmt.accept(statVisitor); Set relationships = statVisitor.getRelationships();