mirror of
https://github.com/alibaba/druid.git
synced 2023-03-04 19:17:16 +00:00
improved sql parser.
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
package com.alibaba.druid.benckmark.sql;
|
||||
|
||||
import com.alibaba.druid.sql.SQLUtils;
|
||||
import com.alibaba.druid.sql.ast.SQLStatement;
|
||||
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
|
||||
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
|
||||
import com.alibaba.druid.util.JdbcConstants;
|
||||
import junit.framework.TestCase;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class MySqlParameterizedBenchmark extends TestCase {
|
||||
|
||||
public void test_perf() throws Exception {
|
||||
String sql = "select id as id, gmt_create as gmtCreate, gmt_modified as gmtModified, name as name, owner as owner, type as type, statement as statement, datasource as datasource, meta as meta, param_file as paramFile, sharable as sharable, data_type as dataType, status as status, config as config, project_id as projectId, plugins as plugins, field_compare as fieldCompare, field_ext as fieldExt, open as open from quark_s_dataset where id = 12569434";
|
||||
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
// perf(sql); // 6740 6201
|
||||
perf_parse(sql); // 4643 4377
|
||||
}
|
||||
}
|
||||
public void perf(String sql) {
|
||||
long startMillis = System.currentTimeMillis();
|
||||
for (int i = 0; i < 1000 * 1000; ++i) {
|
||||
|
||||
|
||||
String psql = SqlHolder.of(sql).parameterize();
|
||||
// stmt.toString();
|
||||
}
|
||||
long millis = System.currentTimeMillis() - startMillis;
|
||||
System.out.println("millis : " + millis);
|
||||
}
|
||||
|
||||
public void perf_parse(String sql) {
|
||||
long startMillis = System.currentTimeMillis();
|
||||
for (int i = 0; i < 1000 * 1000; ++i) {
|
||||
|
||||
|
||||
SqlHolder.of(sql).ensureParsed();
|
||||
// stmt.toString();
|
||||
}
|
||||
long millis = System.currentTimeMillis() - startMillis;
|
||||
System.out.println("millis : " + millis);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package com.alibaba.druid.benckmark.sql;
|
||||
|
||||
import com.alibaba.druid.sql.SQLUtils;
|
||||
import com.alibaba.druid.sql.ast.SQLStatement;
|
||||
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
|
||||
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
|
||||
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
|
||||
import com.alibaba.druid.sql.parser.ParserException;
|
||||
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
|
||||
import com.alibaba.druid.util.JdbcConstants;
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Created by kaiwang.ckw on 15/05/2017.
|
||||
*/
|
||||
public class SqlHolder {
|
||||
private String text;
|
||||
private String dialect;
|
||||
|
||||
private boolean parsed;
|
||||
private SQLStatement ast;
|
||||
|
||||
public static SqlHolder of(String sql) {
|
||||
return new SqlHolder(sql);
|
||||
}
|
||||
|
||||
SqlHolder(String text) {
|
||||
this(text, JdbcConstants.MYSQL);
|
||||
}
|
||||
|
||||
SqlHolder(String text, String dialect) {
|
||||
if (!"mysql".equalsIgnoreCase(dialect)) {
|
||||
throw new IllegalArgumentException("only mysql is");
|
||||
}
|
||||
|
||||
this.text = text;
|
||||
this.dialect = dialect;
|
||||
}
|
||||
|
||||
public String format() {
|
||||
try {
|
||||
return SQLUtils.format(text, dialect);
|
||||
} catch (ParserException e) {
|
||||
throw new UnsupportedOperationException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void ensureParsed() {
|
||||
if (parsed) {
|
||||
return;
|
||||
}
|
||||
// ast = SQLUtils.parseStatements(text, dialect).get(0);
|
||||
try {
|
||||
ast = new MySqlStatementParser(text).parseStatement();
|
||||
} catch (ParserException e) {
|
||||
throw new UnsupportedOperationException(e);
|
||||
}
|
||||
parsed = true;
|
||||
}
|
||||
|
||||
// returns rewritten sql, or original string object if not rewritten
|
||||
public String select() {
|
||||
ensureParsed();
|
||||
|
||||
SQLStatement stmt = ast;
|
||||
if (stmt instanceof SQLSelectStatement) {
|
||||
boolean rewritten = StatementConverter.rewriteSelect((SQLSelectStatement) stmt);
|
||||
if (rewritten) {
|
||||
return SQLUtils.toMySqlString(stmt);
|
||||
} else {
|
||||
return text;
|
||||
}
|
||||
} else {
|
||||
SQLSelectStatement selectStatement = StatementConverter.rewrite(stmt);
|
||||
if (stmt == selectStatement) {
|
||||
return text;
|
||||
} else {
|
||||
return SQLUtils.toMySqlString(selectStatement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String parameterize() {
|
||||
return parameterize(null, null);
|
||||
}
|
||||
|
||||
public String parameterize(Set<String> physicalNames) {
|
||||
ensureParsed();
|
||||
return Templates.parameterize(ast, physicalNames, null);
|
||||
}
|
||||
|
||||
public String parameterize(Set<String> physicalNames, List<Object> params) {
|
||||
ensureParsed();
|
||||
return Templates.parameterize(ast, physicalNames, params);
|
||||
}
|
||||
|
||||
|
||||
public String getParams() {
|
||||
ensureParsed();
|
||||
StringBuilder out = new StringBuilder();
|
||||
SQLASTOutputVisitor visitor = SQLUtils.createOutputVisitor(out, JdbcConstants.MYSQL);
|
||||
List<Object> parameters = new ArrayList<Object>();
|
||||
visitor.setParameterized(true);
|
||||
visitor.setParameterizedMergeInList(true);
|
||||
visitor.setParameters(parameters);
|
||||
ast.accept(visitor);
|
||||
String params = JSONArray.toJSONString(parameters, SerializerFeature.WriteClassName);
|
||||
params = StringUtils.replace(params, "\"", "\\\"");
|
||||
return params;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.alibaba.druid.benckmark.sql;
|
||||
|
||||
import com.alibaba.druid.sql.ast.SQLExpr;
|
||||
import com.alibaba.druid.sql.ast.SQLStatement;
|
||||
import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr;
|
||||
import com.alibaba.druid.sql.ast.expr.SQLQueryExpr;
|
||||
import com.alibaba.druid.sql.ast.statement.*;
|
||||
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlReplaceStatement;
|
||||
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
|
||||
|
||||
/**
|
||||
* Created by kaiwang.ckw on 15/05/2017.
|
||||
*/
|
||||
public class StatementConverter {
|
||||
|
||||
// remove 'for update' from 'select ... for update'
|
||||
public static boolean rewriteSelect(SQLSelectStatement stmt) {
|
||||
SQLSelectQuery q = stmt.getSelect().getQuery();
|
||||
if (q instanceof MySqlSelectQueryBlock) {
|
||||
MySqlSelectQueryBlock qb = (MySqlSelectQueryBlock) q;
|
||||
if (qb.isForUpdate()) {
|
||||
qb.setForUpdate(false);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// construct a 'select' from other types of dml
|
||||
public static SQLSelectStatement rewrite(SQLStatement stmt) {
|
||||
SQLSelectStatement selectStmt;
|
||||
if (stmt instanceof SQLSelectStatement) {
|
||||
throw new RuntimeException("please use rewriteSelect which does ast modification instead of construction");
|
||||
} else if (stmt instanceof SQLUpdateStatement) {
|
||||
SQLTableSource tableSource = ((SQLUpdateStatement) stmt).getTableSource();
|
||||
SQLExpr where = ((SQLUpdateStatement) stmt).getWhere();
|
||||
String dbType = ((SQLUpdateStatement) stmt).getDbType();
|
||||
selectStmt = buildSelect(tableSource, where, dbType);
|
||||
} else if (stmt instanceof SQLDeleteStatement) {
|
||||
SQLTableSource tableSource = ((SQLDeleteStatement) stmt).getTableSource();
|
||||
SQLExpr where = ((SQLDeleteStatement) stmt).getWhere();
|
||||
String dbType = ((SQLDeleteStatement) stmt).getDbType();
|
||||
selectStmt = buildSelect(tableSource, where, dbType);
|
||||
} else if (stmt instanceof SQLInsertStatement) {
|
||||
SQLSelect sqlSelect = ((SQLInsertStatement) stmt).getQuery();
|
||||
if (sqlSelect != null) {
|
||||
selectStmt = new SQLSelectStatement();
|
||||
selectStmt.setSelect(sqlSelect);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("only insert..select.. is supported");
|
||||
}
|
||||
} else if (stmt instanceof MySqlReplaceStatement) {
|
||||
SQLQueryExpr sqlQueryExpr = ((MySqlReplaceStatement) stmt).getQuery();
|
||||
if (sqlQueryExpr != null) {
|
||||
SQLSelect sqlSelect = ((MySqlReplaceStatement) stmt).getQuery().getSubQuery();
|
||||
selectStmt = new SQLSelectStatement();
|
||||
selectStmt.setSelect(sqlSelect);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("only replace..select.. is supported");
|
||||
}
|
||||
} else {
|
||||
throw new UnsupportedOperationException("only select/update/delete/insert/replace are supported");
|
||||
}
|
||||
return selectStmt;
|
||||
}
|
||||
|
||||
// build 'select * from ...'
|
||||
static SQLSelectStatement buildSelect(SQLTableSource tableSource, SQLExpr where, String dbType) {
|
||||
SQLSelectQueryBlock sqlSelectQuery = new SQLSelectQueryBlock();
|
||||
sqlSelectQuery.addSelectItem(new SQLSelectItem(new SQLAllColumnExpr()));
|
||||
sqlSelectQuery.setWhere(where);
|
||||
sqlSelectQuery.setFrom(tableSource);
|
||||
tableSource.setParent(sqlSelectQuery);
|
||||
|
||||
SQLSelect sqlSelect = new SQLSelect();
|
||||
sqlSelect.setQuery(sqlSelectQuery);
|
||||
|
||||
SQLSelectStatement selectStmt = new SQLSelectStatement();
|
||||
selectStmt.setSelect(sqlSelect);
|
||||
selectStmt.setDbType(dbType);
|
||||
|
||||
return selectStmt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.alibaba.druid.benckmark.sql;
|
||||
|
||||
import com.alibaba.druid.sql.ast.SQLStatement;
|
||||
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
|
||||
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Created by kaiwang.ckw on 15/05/2017.
|
||||
*/
|
||||
public class Templates {
|
||||
|
||||
public static final String UNKNOWN = "UNKNOWN";
|
||||
|
||||
public static String parameterize(SQLStatement ast, Set<String> physicalNames, List<Object> params) {
|
||||
|
||||
List<Object> parameters = null;
|
||||
StringBuilder out = new StringBuilder();
|
||||
SQLASTOutputVisitor visitor = new MySqlOutputVisitor(out);
|
||||
visitor.setParameterized(true);
|
||||
visitor.setParameterizedMergeInList(true);
|
||||
if (physicalNames != null) {
|
||||
visitor.setExportTables(true);
|
||||
}
|
||||
if (params != null) {
|
||||
parameters = new ArrayList<Object>();
|
||||
visitor.setParameters(parameters);
|
||||
}
|
||||
visitor.setPrettyFormat(false);
|
||||
ast.accept(visitor);
|
||||
|
||||
if (physicalNames != null) {
|
||||
Set<String> tableSet = visitor.getTables();
|
||||
if (tableSet != null) {
|
||||
physicalNames.addAll(tableSet);
|
||||
}
|
||||
}
|
||||
if (params != null) {
|
||||
if (!parameters.isEmpty()) {
|
||||
params.addAll(parameters);
|
||||
}
|
||||
}
|
||||
|
||||
String sqlTemplate = out.toString();
|
||||
return sqlTemplate;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 1999-2017 Alibaba Group Holding Ltd.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.alibaba.druid.bvt.sql.oracle.select;
|
||||
|
||||
import com.alibaba.druid.sql.OracleTest;
|
||||
import com.alibaba.druid.sql.SQLUtils;
|
||||
import com.alibaba.druid.sql.ast.SQLStatement;
|
||||
import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
|
||||
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor;
|
||||
import org.junit.Assert;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class OracleSelectTest75_comment extends OracleTest {
|
||||
|
||||
public void test_0() throws Exception {
|
||||
String sql = //
|
||||
"--CMS???'?o??\n" +
|
||||
"SELECT *\n" +
|
||||
"FROM CDM.CDM_CON_ISSUE_DATE_CONTROL@ERPCMS idc\n" +
|
||||
"WHERE idc.ecc_contract_header_id = ?"; //
|
||||
|
||||
OracleStatementParser parser = new OracleStatementParser(sql);
|
||||
List<SQLStatement> statementList = parser.parseStatementList();
|
||||
SQLStatement stmt = statementList.get(0);
|
||||
print(statementList);
|
||||
|
||||
Assert.assertEquals(1, statementList.size());
|
||||
|
||||
OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor();
|
||||
stmt.accept(visitor);
|
||||
|
||||
System.out.println("Tables : " + visitor.getTables());
|
||||
System.out.println("fields : " + visitor.getColumns());
|
||||
System.out.println("coditions : " + visitor.getConditions());
|
||||
System.out.println("relationships : " + visitor.getRelationships());
|
||||
System.out.println("orderBy : " + visitor.getOrderByColumns());
|
||||
|
||||
assertEquals(1, visitor.getTables().size());
|
||||
|
||||
assertEquals(2, visitor.getColumns().size());
|
||||
|
||||
{
|
||||
String text = SQLUtils.toOracleString(stmt);
|
||||
|
||||
assertEquals("-- CMS???'?o??\n" +
|
||||
"SELECT *\n" +
|
||||
"FROM CDM.CDM_CON_ISSUE_DATE_CONTROL@ERPCMS idc\n" +
|
||||
"WHERE idc.ecc_contract_header_id = ?", text);
|
||||
}
|
||||
// Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("acduser.vw_acd_info", "xzqh")));
|
||||
|
||||
// Assert.assertTrue(visitor.getOrderByColumns().contains(new TableStat.Column("employees", "last_name")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.alibaba.druid.bvt.utils;
|
||||
|
||||
import com.alibaba.druid.util.MySqlUtils;
|
||||
import junit.framework.TestCase;
|
||||
|
||||
public class MySqlUtils_1_builtin_dataTypes extends TestCase {
|
||||
public void test_builtin_dataTypes() throws Exception {
|
||||
assertTrue(MySqlUtils.isBuiltinDataType("decimal"));
|
||||
assertTrue(MySqlUtils.isBuiltinDataType("INT"));
|
||||
assertFalse(MySqlUtils.isBuiltinDataType("decimalx"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user