Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improved sql parser. for issue #1637

Browse Source
This commit is contained in:
wenshao
2017-03-14 20:34:25 +08:00
parent aa8459e192
commit aa8cfcf4f8
8 changed files with 281 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectStatement.java
+13
View File
@@ -15,13 +15,18 @@
*/
package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLCommentHint;
import com.alibaba.druid.sql.ast.SQLStatementImpl;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import java.util.List;
public class SQLSelectStatement extends SQLStatementImpl {
protected SQLSelect select;
private List<SQLCommentHint> headHints;
public SQLSelectStatement(){
}
@@ -60,4 +65,12 @@ public class SQLSelectStatement extends SQLStatementImpl {
}
visitor.endVisit(this);
}
public List<SQLCommentHint> getHeadHintsDirect() {
return headHints;
}
public void setHeadHints(List<SQLCommentHint> headHints) {
this.headHints = headHints;
}
}
src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java
+39 -9
View File
@@ -692,7 +692,29 @@ public class MySqlStatementParser extends SQLStatementParser {
}
if (lexer.token() == Token.HINT) {
statementList.add(this.parseHint());
List<SQLCommentHint> hints = this.exprParser.parseHints();
boolean tddlSelectHints = false;
if (hints.size() == 1 && statementList.size() == 0 && lexer.token() == Token.SELECT) {
SQLCommentHint hint = hints.get(0);
String hintText = hint.getText();
if (hintText.startsWith("+TDDL")) {
tddlSelectHints = true;
}
}
if (tddlSelectHints) {
SQLSelectStatement stmt = (SQLSelectStatement) this.parseStatement();
stmt.setHeadHints(hints);
statementList.add(stmt);
return true;
}
MySqlHintStatement stmt = new MySqlHintStatement();
stmt.setHints(hints);
statementList.add(stmt);
return true;
}
@@ -1105,7 +1127,8 @@ public class MySqlStatementParser extends SQLStatementParser {
if (identifierEquals("STORAGE")) {
lexer.nextToken();
acceptIdentifier(ENGINES);
accept(Token.EQ);
accept(Token.DEFAULT);
MySqlShowEnginesStatement stmt = new MySqlShowEnginesStatement();
stmt.setStorage(true);
return stmt;
@@ -2755,6 +2778,20 @@ public class MySqlStatementParser extends SQLStatementParser {
SQLTableSource tableSrc = this.createSQLSelectParser().parseTableSource();
stmt.getTableOptions().put("UNION", tableSrc);
accept(Token.RPAREN);
} else if (identifierEquals("ROW_FORMAT")) {
lexer.nextToken();
if (lexer.token() == Token.EQ) {
lexer.nextToken();
}
if (lexer.token() == Token.DEFAULT || lexer.token() == Token.IDENTIFIER) {
SQLIdentifierExpr rowFormat = new SQLIdentifierExpr(lexer.stringVal());
lexer.nextToken();
stmt.getTableOptions().put("ROW_FORMAT", rowFormat);
} else {
throw new ParserException("illegal syntax.");
}
} else {
break;
}
@@ -3025,13 +3062,6 @@ public class MySqlStatementParser extends SQLStatementParser {
return (MySqlExprParser) exprParser;
}
public MySqlHintStatement parseHint() {
// accept(Token.HINT);
MySqlHintStatement stmt = new MySqlHintStatement();
stmt.setHints(this.exprParser.parseHints());
return stmt;
}
/**
* parse create procedure statement
src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java
-30
View File
@@ -864,36 +864,6 @@ public class OracleExprParser extends SQLExprParser {
return new OracleSelectParser(this);
}
@Override
public SQLSelectOrderByItem parseSelectOrderByItem() {
SQLSelectOrderByItem item = new SQLSelectOrderByItem();
item.setExpr(expr());
if (lexer.token() == (Token.ASC)) {
lexer.nextToken();
item.setType(SQLOrderingSpecification.ASC);
} else if (lexer.token() == (Token.DESC)) {
lexer.nextToken();
item.setType(SQLOrderingSpecification.DESC);
}
if (identifierEquals("NULLS")) {
lexer.nextToken();
if (identifierEquals("FIRST")) {
lexer.nextToken();
item.setNullsOrderType(SQLSelectOrderByItem.NullsOrderType.NullsFirst);
} else if (identifierEquals("LAST")) {
lexer.nextToken();
item.setNullsOrderType(SQLSelectOrderByItem.NullsOrderType.NullsLast);
} else {
throw new ParserException("TODO " + lexer.token());
}
}
return item;
}
protected SQLExpr parseInterval() {
accept(Token.INTERVAL);
src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+13
View File
@@ -1220,6 +1220,19 @@ public class SQLExprParser extends SQLParser {
item.setType(SQLOrderingSpecification.DESC);
}
if (identifierEquals("NULLS")) {
lexer.nextToken();
if (identifierEquals("FIRST")) {
lexer.nextToken();
item.setNullsOrderType(SQLSelectOrderByItem.NullsOrderType.NullsFirst);
} else if (identifierEquals("LAST")) {
lexer.nextToken();
item.setNullsOrderType(SQLSelectOrderByItem.NullsOrderType.NullsLast);
} else {
throw new ParserException("TODO " + lexer.token());
}
}
return item;
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java
+13
View File
@@ -1192,6 +1192,11 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
print0(x.getCollate());
}
if (x.getNullsOrderType() != null) {
print(' ');
print0(x.getNullsOrderType().toFormalString());
}
return false;
}
@@ -1261,6 +1266,14 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
}
public boolean visit(SQLSelectStatement stmt) {
List<SQLCommentHint> headHints = stmt.getHeadHintsDirect();
if (headHints != null) {
for (SQLCommentHint hint : headHints) {
hint.accept(this);
println();
}
}
SQLSelect select = stmt.getSelect();
select.accept(this);
src/test/java/com/alibaba/druid/bvt/sql/mysql/alter/MySqlAlterTableTest27.java
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 1999-2101 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.mysql.alter;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.parser.Token;
import junit.framework.TestCase;
import org.junit.Assert;
public class MySqlAlterTableTest27 extends TestCase {
public void test_alter_add_key() throws Exception {
String sql = "ALTER TABLE app_key ROW_FORMAT = DEFAULT";
MySqlStatementParser parser = new MySqlStatementParser(sql);
SQLStatement stmt = parser.parseStatementList().get(0);
parser.match(Token.EOF);
String output = SQLUtils.toMySqlString(stmt);
Assert.assertEquals("ALTER TABLE app_key" +
"\n\tROW_FORMAT = DEFAULT", output);
}
}
src/test/java/com/alibaba/druid/bvt/sql/mysql/param/MySqlParameterizedOutputVisitorTest_35.java
+75
View File
@@ -0,0 +1,75 @@
package com.alibaba.druid.bvt.sql.mysql.param;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.ArrayList;
import java.util.List;
/**
* Created by wenshao on 16/8/23.
*/
public class MySqlParameterizedOutputVisitorTest_35 extends TestCase {
public void test_for_parameterize() throws Exception {
final String dbType = JdbcConstants.MYSQL;
String sql = "/*+TDDL({'extra':{'SOCKET_TIMEOUT':'3600000'}})*/\n" +
"select sample_table_schema, \n" +
"sample_table_name,\n" +
"sample_table_orig_size,\n" +
"sample_table_sample_size,\n" +
"col.id as id,\n" +
"sample_column_name, \n" +
"sample_column_type,\n" +
"sample_string, \n" +
"sample_column_highkey, \n" +
"sample_column_high2key, \n" +
"sample_column_lowkey,\n" +
"sample_column_low2key,\n" +
"sample_column_cardinality,\n" +
"sample_avg_length,\n" +
"sample_column_dist_type as type, \n" +
"sample_column_dist_quantileno as quantileno,\n" +
"sample_column_dist_highkey,\n" +
"sample_column_dist_lowkey, \n" +
"sample_column_dist_value,\n" +
"sample_column_dist_cardinality,\n" +
"sample_column_dist_count,\n" +
"col.gmt_create as time\n" +
"from sample_tables tab, sample_columns col, sample_column_distribution dist\n" +
"where \n" +
"tab.id = col.sample_column_table_id and \n" +
"col.id = dist.sample_column_dist_column_id and \n" +
"col.id = ( \n" +
" SELECT id FROM sample_columns col \n" +
" WHERE sample_column_name = 'gmt_modified'\n" +
" AND sample_column_table_schema = 'SC_PRODUCT_03'\n" +
" AND sample_column_table_name = 'product_0096'\n" +
" ORDER BY id DESC LIMIT 1 \n" +
" order by type, quantileno)";
SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType);
List<SQLStatement> stmtList = parser.parseStatementList();
SQLStatement statement = stmtList.get(0);
StringBuilder out = new StringBuilder();
// List<Object> parameters = new ArrayList<Object>();
SQLASTOutputVisitor visitor = SQLUtils.createOutputVisitor(out, JdbcConstants.MYSQL);
visitor.setParameterized(true);
visitor.setParameterizedMergeInList(true);
// visitor.setParameters(parameters);
visitor.setExportTables(true);
visitor.setPrettyFormat(false);
statement.accept(visitor);
System.out.println(out);
}
}
src/test/java/com/alibaba/druid/bvt/sql/postgresql/select/PGSelectTest47.java
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 1999-2101 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.postgresql.select;
import com.alibaba.druid.sql.PGTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import org.junit.Assert;
import java.util.List;
public class PGSelectTest47 extends PGTest {
private String dbType = JdbcConstants.POSTGRESQL;
public void test_0() throws Exception {
String sql = "SELECT\n" +
"u.userid,\n" +
"u.phonenumber,\n" +
"u.cdate,\n" +
"u.udate,\n" +
"u.wechatid,\n" +
"u.wechatnickname,\n" +
"u.wechattime,\n" +
"b. ID,\n" +
"b.username serviceStaffName,\n" +
"u.appid,\n" +
"s. ID servicegroupid,\n" +
"n.total_amount blackNumber\n" +
"FROM\n" +
"users u\n" +
"LEFT JOIN user_servicegroup s ON u.userid = s.userid\n" +
"LEFT JOIN jcs_user_base b ON s.servicestaffid = b. ID\n" +
"LEFT JOIN jcs_user_purchase_record r ON u.userid = r.user_id\n" +
"LEFT JOIN user_black_article_number n ON u.userid = n.userid\n" +
"WHERE\n" +
"u.isactive = TRUE\n" +
"ORDER BY\n" +
"n.total_amount DESC NULLS LAST";
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, dbType);
SQLStatement stmt = statementList.get(0);
Assert.assertEquals("SELECT u.userid, u.phonenumber, u.cdate, u.udate, u.wechatid\n" +
"\t, u.wechatnickname, u.wechattime, b.ID, b.username AS serviceStaffName, u.appid\n" +
"\t, s.ID AS servicegroupid, n.total_amount AS blackNumber\n" +
"FROM users u\n" +
"\tLEFT JOIN user_servicegroup s ON u.userid = s.userid\n" +
"\tLEFT JOIN jcs_user_base b ON s.servicestaffid = b.ID\n" +
"\tLEFT JOIN jcs_user_purchase_record r ON u.userid = r.user_id\n" +
"\tLEFT JOIN user_black_article_number n ON u.userid = n.userid\n" +
"WHERE u.isactive = true\n" +
"ORDER BY n.total_amount DESC NULLS LAST", SQLUtils.toPGString(stmt));
Assert.assertEquals("select u.userid, u.phonenumber, u.cdate, u.udate, u.wechatid\n" +
"\t, u.wechatnickname, u.wechattime, b.ID, b.username as serviceStaffName, u.appid\n" +
"\t, s.ID as servicegroupid, n.total_amount as blackNumber\n" +
"from users u\n" +
"\tleft join user_servicegroup s on u.userid = s.userid\n" +
"\tleft join jcs_user_base b on s.servicestaffid = b.ID\n" +
"\tleft join jcs_user_purchase_record r on u.userid = r.user_id\n" +
"\tleft join user_black_article_number n on u.userid = n.userid\n" +
"where u.isactive = true\n" +
"order by n.total_amount desc NULLS LAST", SQLUtils.toPGString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
Assert.assertEquals(1, statementList.size());
PGSchemaStatVisitor visitor = new PGSchemaStatVisitor();
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
// System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
Assert.assertEquals(17, visitor.getColumns().size());
Assert.assertEquals(5, visitor.getTables().size());
}
}