improved sql parser.

This commit is contained in:
wenshao
2017-08-25 00:29:12 +08:00
parent 026295ddbd
commit 76c441949f
35 changed files with 1442 additions and 211 deletions
@@ -0,0 +1,48 @@
package com.alibaba.druid.benckmark.sql;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
public class MySqlParameterizedBenchmark extends TestCase {
public void test_perf() throws Exception {
String sql = "select id as id, gmt_create as gmtCreate, gmt_modified as gmtModified, name as name, owner as owner, type as type, statement as statement, datasource as datasource, meta as meta, param_file as paramFile, sharable as sharable, data_type as dataType, status as status, config as config, project_id as projectId, plugins as plugins, field_compare as fieldCompare, field_ext as fieldExt, open as open from quark_s_dataset where id = 12569434";
for (int i = 0; i < 5; ++i) {
// perf(sql); // 6740 6201
perf_parse(sql); // 4643 4377
}
}
public void perf(String sql) {
long startMillis = System.currentTimeMillis();
for (int i = 0; i < 1000 * 1000; ++i) {
String psql = SqlHolder.of(sql).parameterize();
// stmt.toString();
}
long millis = System.currentTimeMillis() - startMillis;
System.out.println("millis : " + millis);
}
public void perf_parse(String sql) {
long startMillis = System.currentTimeMillis();
for (int i = 0; i < 1000 * 1000; ++i) {
SqlHolder.of(sql).ensureParsed();
// stmt.toString();
}
long millis = System.currentTimeMillis() - startMillis;
System.out.println("millis : " + millis);
}
}
@@ -0,0 +1,115 @@
package com.alibaba.druid.benckmark.sql;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.druid.sql.parser.ParserException;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.JdbcConstants;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.serializer.SerializerFeature;
import org.apache.commons.lang3.StringUtils;
import java.util.*;
/**
* Created by kaiwang.ckw on 15/05/2017.
*/
public class SqlHolder {
private String text;
private String dialect;
private boolean parsed;
private SQLStatement ast;
public static SqlHolder of(String sql) {
return new SqlHolder(sql);
}
SqlHolder(String text) {
this(text, JdbcConstants.MYSQL);
}
SqlHolder(String text, String dialect) {
if (!"mysql".equalsIgnoreCase(dialect)) {
throw new IllegalArgumentException("only mysql is");
}
this.text = text;
this.dialect = dialect;
}
public String format() {
try {
return SQLUtils.format(text, dialect);
} catch (ParserException e) {
throw new UnsupportedOperationException(e);
}
}
public void ensureParsed() {
if (parsed) {
return;
}
// ast = SQLUtils.parseStatements(text, dialect).get(0);
try {
ast = new MySqlStatementParser(text).parseStatement();
} catch (ParserException e) {
throw new UnsupportedOperationException(e);
}
parsed = true;
}
// returns rewritten sql, or original string object if not rewritten
public String select() {
ensureParsed();
SQLStatement stmt = ast;
if (stmt instanceof SQLSelectStatement) {
boolean rewritten = StatementConverter.rewriteSelect((SQLSelectStatement) stmt);
if (rewritten) {
return SQLUtils.toMySqlString(stmt);
} else {
return text;
}
} else {
SQLSelectStatement selectStatement = StatementConverter.rewrite(stmt);
if (stmt == selectStatement) {
return text;
} else {
return SQLUtils.toMySqlString(selectStatement);
}
}
}
public String parameterize() {
return parameterize(null, null);
}
public String parameterize(Set<String> physicalNames) {
ensureParsed();
return Templates.parameterize(ast, physicalNames, null);
}
public String parameterize(Set<String> physicalNames, List<Object> params) {
ensureParsed();
return Templates.parameterize(ast, physicalNames, params);
}
public String getParams() {
ensureParsed();
StringBuilder out = new StringBuilder();
SQLASTOutputVisitor visitor = SQLUtils.createOutputVisitor(out, JdbcConstants.MYSQL);
List<Object> parameters = new ArrayList<Object>();
visitor.setParameterized(true);
visitor.setParameterizedMergeInList(true);
visitor.setParameters(parameters);
ast.accept(visitor);
String params = JSONArray.toJSONString(parameters, SerializerFeature.WriteClassName);
params = StringUtils.replace(params, "\"", "\\\"");
return params;
}
}
@@ -0,0 +1,84 @@
package com.alibaba.druid.benckmark.sql;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr;
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlReplaceStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
/**
* Created by kaiwang.ckw on 15/05/2017.
*/
public class StatementConverter {
// remove 'for update' from 'select ... for update'
public static boolean rewriteSelect(SQLSelectStatement stmt) {
SQLSelectQuery q = stmt.getSelect().getQuery();
if (q instanceof MySqlSelectQueryBlock) {
MySqlSelectQueryBlock qb = (MySqlSelectQueryBlock) q;
if (qb.isForUpdate()) {
qb.setForUpdate(false);
return true;
}
}
return false;
}
// construct a 'select' from other types of dml
public static SQLSelectStatement rewrite(SQLStatement stmt) {
SQLSelectStatement selectStmt;
if (stmt instanceof SQLSelectStatement) {
throw new RuntimeException("please use rewriteSelect which does ast modification instead of construction");
} else if (stmt instanceof SQLUpdateStatement) {
SQLTableSource tableSource = ((SQLUpdateStatement) stmt).getTableSource();
SQLExpr where = ((SQLUpdateStatement) stmt).getWhere();
String dbType = ((SQLUpdateStatement) stmt).getDbType();
selectStmt = buildSelect(tableSource, where, dbType);
} else if (stmt instanceof SQLDeleteStatement) {
SQLTableSource tableSource = ((SQLDeleteStatement) stmt).getTableSource();
SQLExpr where = ((SQLDeleteStatement) stmt).getWhere();
String dbType = ((SQLDeleteStatement) stmt).getDbType();
selectStmt = buildSelect(tableSource, where, dbType);
} else if (stmt instanceof SQLInsertStatement) {
SQLSelect sqlSelect = ((SQLInsertStatement) stmt).getQuery();
if (sqlSelect != null) {
selectStmt = new SQLSelectStatement();
selectStmt.setSelect(sqlSelect);
} else {
throw new UnsupportedOperationException("only insert..select.. is supported");
}
} else if (stmt instanceof MySqlReplaceStatement) {
SQLQueryExpr sqlQueryExpr = ((MySqlReplaceStatement) stmt).getQuery();
if (sqlQueryExpr != null) {
SQLSelect sqlSelect = ((MySqlReplaceStatement) stmt).getQuery().getSubQuery();
selectStmt = new SQLSelectStatement();
selectStmt.setSelect(sqlSelect);
} else {
throw new UnsupportedOperationException("only replace..select.. is supported");
}
} else {
throw new UnsupportedOperationException("only select/update/delete/insert/replace are supported");
}
return selectStmt;
}
// build 'select * from ...'
static SQLSelectStatement buildSelect(SQLTableSource tableSource, SQLExpr where, String dbType) {
SQLSelectQueryBlock sqlSelectQuery = new SQLSelectQueryBlock();
sqlSelectQuery.addSelectItem(new SQLSelectItem(new SQLAllColumnExpr()));
sqlSelectQuery.setWhere(where);
sqlSelectQuery.setFrom(tableSource);
tableSource.setParent(sqlSelectQuery);
SQLSelect sqlSelect = new SQLSelect();
sqlSelect.setQuery(sqlSelectQuery);
SQLSelectStatement selectStmt = new SQLSelectStatement();
selectStmt.setSelect(sqlSelect);
selectStmt.setDbType(dbType);
return selectStmt;
}
}
@@ -0,0 +1,51 @@
package com.alibaba.druid.benckmark.sql;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* Created by kaiwang.ckw on 15/05/2017.
*/
public class Templates {
public static final String UNKNOWN = "UNKNOWN";
public static String parameterize(SQLStatement ast, Set<String> physicalNames, List<Object> params) {
List<Object> parameters = null;
StringBuilder out = new StringBuilder();
SQLASTOutputVisitor visitor = new MySqlOutputVisitor(out);
visitor.setParameterized(true);
visitor.setParameterizedMergeInList(true);
if (physicalNames != null) {
visitor.setExportTables(true);
}
if (params != null) {
parameters = new ArrayList<Object>();
visitor.setParameters(parameters);
}
visitor.setPrettyFormat(false);
ast.accept(visitor);
if (physicalNames != null) {
Set<String> tableSet = visitor.getTables();
if (tableSet != null) {
physicalNames.addAll(tableSet);
}
}
if (params != null) {
if (!parameters.isEmpty()) {
params.addAll(parameters);
}
}
String sqlTemplate = out.toString();
return sqlTemplate;
}
}
@@ -0,0 +1,68 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.oracle.select;
import com.alibaba.druid.sql.OracleTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor;
import org.junit.Assert;
import java.util.List;
public class OracleSelectTest75_comment extends OracleTest {
public void test_0() throws Exception {
String sql = //
"--CMS???'?o??\n" +
"SELECT *\n" +
"FROM CDM.CDM_CON_ISSUE_DATE_CONTROL@ERPCMS idc\n" +
"WHERE idc.ecc_contract_header_id = ?"; //
OracleStatementParser parser = new OracleStatementParser(sql);
List<SQLStatement> statementList = parser.parseStatementList();
SQLStatement stmt = statementList.get(0);
print(statementList);
Assert.assertEquals(1, statementList.size());
OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor();
stmt.accept(visitor);
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(2, visitor.getColumns().size());
{
String text = SQLUtils.toOracleString(stmt);
assertEquals("-- CMS???'?o??\n" +
"SELECT *\n" +
"FROM CDM.CDM_CON_ISSUE_DATE_CONTROL@ERPCMS idc\n" +
"WHERE idc.ecc_contract_header_id = ?", text);
}
// Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("acduser.vw_acd_info", "xzqh")));
// Assert.assertTrue(visitor.getOrderByColumns().contains(new TableStat.Column("employees", "last_name")));
}
}
@@ -0,0 +1,12 @@
package com.alibaba.druid.bvt.utils;
import com.alibaba.druid.util.MySqlUtils;
import junit.framework.TestCase;
public class MySqlUtils_1_builtin_dataTypes extends TestCase {
public void test_builtin_dataTypes() throws Exception {
assertTrue(MySqlUtils.isBuiltinDataType("decimal"));
assertTrue(MySqlUtils.isBuiltinDataType("INT"));
assertFalse(MySqlUtils.isBuiltinDataType("decimalx"));
}
}