Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

Merge pull request #251 from wenshao/master

Browse Source 
improve sql parser perforamnce
This commit is contained in:
温高铁
2013-01-14 04:46:33 -08:00
parent 9899806c33 7aa2038498
commit b8ea7cb1fe
11 changed files with 376 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java
+14 -9
View File
@@ -204,24 +204,29 @@ public class MySqlLexer extends Lexer {
}
}
}
protected final void scanString() {
{
boolean hasSpecial = false;
int startIndex = pos + 1;
int endIndex = indexOf('\'', startIndex);
int endIndex = -1; //text.indexOf('\'', startIndex);
for (int i = startIndex; i < text.length(); ++i) {
final char ch = text.charAt(i);
if (ch == '\\') {
hasSpecial = true;
}
if (ch == '\'') {
endIndex = i;
break;
}
}
if (endIndex == -1) {
throw new ParserException("unclosed str");
}
String stringVal = subString(startIndex, endIndex - startIndex);
for (int i = 0; i < stringVal.length(); ++i) {
char ch = stringVal.charAt(i);
if (ch == '\\') {
hasSpecial = true;
break;
}
}
// hasSpecial = stringVal.indexOf('\\') != -1;
if (!hasSpecial) {
this.stringVal = stringVal;
src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java
+74 -83
View File
@@ -38,6 +38,7 @@ import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLDropTableStatement;
import com.alibaba.druid.sql.ast.statement.SQLDropViewStatement;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause;
import com.alibaba.druid.sql.ast.statement.SQLPrimaryKey;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
@@ -1488,21 +1489,8 @@ public class MySqlStatementParser extends SQLStatementParser {
if (lexer.token() == Token.VALUES || identifierEquals("VALUE")) {
lexer.nextToken();
for (;;) {
accept(Token.LPAREN);
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause();
this.exprParser.exprList(values.getValues());
stmt.getValuesList().add(values);
accept(Token.RPAREN);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
} else {
break;
}
}
parseValueClause(stmt.getValuesList(), 0);
} else if (lexer.token() == Token.SELECT) {
SQLQueryExpr queryExpr = (SQLQueryExpr) this.exprParser.expr();
stmt.setQuery(queryExpr);
@@ -1781,74 +1769,7 @@ public class MySqlStatementParser extends SQLStatementParser {
if (lexer.token() == Token.VALUES || identifierEquals("VALUE")) {
lexer.nextTokenLParen();
for (;;) {
if (lexer.token() != Token.LPAREN) {
throw new SQLParseException("syntax error, expect ')'");
}
lexer.nextTokenValue();
if (lexer.token() != Token.RPAREN) {
List<SQLExpr> valueExprList;
if (columnSize > 0) {
valueExprList = new ArrayList<SQLExpr>(columnSize);
} else {
valueExprList = new ArrayList<SQLExpr>();
}
for (;;) {
SQLExpr expr;
if (lexer.token() == Token.LITERAL_INT) {
expr = new SQLIntegerExpr(lexer.integerValue());
lexer.nextTokenComma();
} else if (lexer.token() == Token.LITERAL_CHARS) {
expr = new SQLCharExpr(lexer.stringVal());
lexer.nextTokenComma();
} else if (lexer.token() == Token.LITERAL_CHARS) {
expr = new SQLCharExpr(lexer.stringVal());
lexer.nextTokenComma();
} else {
expr = exprParser.expr();
}
if (lexer.token() == Token.COMMA) {
valueExprList.add(expr);
lexer.nextTokenValue();
continue;
} else if (lexer.token() == Token.RPAREN) {
valueExprList.add(expr);
break;
} else {
expr = this.exprParser.primaryRest(expr);
valueExprList.add(expr);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
} else {
break;
}
}
}
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause(valueExprList);
insertStatement.getValuesList().add(values);
} else {
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause(new ArrayList<SQLExpr>(0));
insertStatement.getValuesList().add(values);
}
if (lexer.token() != Token.RPAREN) {
throw new SQLParseException("syntax error");
}
lexer.nextTokenComma();
if (lexer.token() == Token.COMMA) {
lexer.nextTokenLParen();
continue;
} else {
break;
}
}
parseValueClause(insertStatement.getValuesList(), columnSize);
} else if (lexer.token() == Token.SET) {
lexer.nextToken();
@@ -1897,6 +1818,76 @@ public class MySqlStatementParser extends SQLStatementParser {
return insertStatement;
}
private void parseValueClause(List<ValuesClause> valueClauseList, int columnSize) {
for (;;) {
if (lexer.token() != Token.LPAREN) {
throw new SQLParseException("syntax error, expect ')'");
}
lexer.nextTokenValue();
if (lexer.token() != Token.RPAREN) {
List<SQLExpr> valueExprList;
if (columnSize > 0) {
valueExprList = new ArrayList<SQLExpr>(columnSize);
} else {
valueExprList = new ArrayList<SQLExpr>();
}
for (;;) {
SQLExpr expr;
if (lexer.token() == Token.LITERAL_INT) {
expr = new SQLIntegerExpr(lexer.integerValue());
lexer.nextTokenComma();
} else if (lexer.token() == Token.LITERAL_CHARS) {
expr = new SQLCharExpr(lexer.stringVal());
lexer.nextTokenComma();
} else if (lexer.token() == Token.LITERAL_CHARS) {
expr = new SQLCharExpr(lexer.stringVal());
lexer.nextTokenComma();
} else {
expr = exprParser.expr();
}
if (lexer.token() == Token.COMMA) {
valueExprList.add(expr);
lexer.nextTokenValue();
continue;
} else if (lexer.token() == Token.RPAREN) {
valueExprList.add(expr);
break;
} else {
expr = this.exprParser.primaryRest(expr);
valueExprList.add(expr);
if (lexer.token() == Token.COMMA) {
lexer.nextToken();
continue;
} else {
break;
}
}
}
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause(valueExprList);
valueClauseList.add(values);
} else {
SQLInsertStatement.ValuesClause values = new SQLInsertStatement.ValuesClause(new ArrayList<SQLExpr>(0));
valueClauseList.add(values);
}
if (lexer.token() != Token.RPAREN) {
throw new SQLParseException("syntax error");
}
lexer.nextTokenComma();
if (lexer.token() == Token.COMMA) {
lexer.nextTokenLParen();
continue;
} else {
break;
}
}
}
public SQLStatement parseDropUser() {
acceptIdentifier(USER);
src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java
+39 -44
View File
@@ -79,6 +79,7 @@ public class OracleExprParser extends SQLExprParser {
public boolean allowStringAdditive = false;
/**
@@ -389,12 +390,6 @@ public class OracleExprParser extends SQLExprParser {
sqlExpr = cursorExpr;
return primaryRest(sqlExpr);
default:
if (identifierEquals("DATE")) {
return primaryRest(parseDate());
}
if (identifierEquals("TIMESTAMP")) {
return primaryRest(parseTimestamp());
}
return super.primary();
}
}
@@ -413,8 +408,46 @@ public class OracleExprParser extends SQLExprParser {
}
public SQLExpr primaryRest(SQLExpr expr) {
if (expr.getClass() == SQLIdentifierExpr.class) {
String ident = ((SQLIdentifierExpr)expr).getName();
if ("DATE".equalsIgnoreCase(ident)) {
OracleDateExpr timestamp = new OracleDateExpr();
String literal = lexer.stringVal();
timestamp.setLiteral(literal);
accept(Token.LITERAL_CHARS);
return primaryRest(timestamp);
}
if ("TIMESTAMP".equalsIgnoreCase(ident)) {
if (lexer.token() != Token.LITERAL_ALIAS && lexer.token() != Token.LITERAL_CHARS) {
return new SQLIdentifierExpr("TIMESTAMP");
}
OracleTimestampExpr timestamp = new OracleTimestampExpr();
String literal = lexer.stringVal();
timestamp.setLiteral(literal);
accept(Token.LITERAL_CHARS);
if (identifierEquals("AT")) {
lexer.nextToken();
acceptIdentifier("TIME");
acceptIdentifier("ZONE");
String timezone = lexer.stringVal();
timestamp.setTimeZone(timezone);
accept(Token.LITERAL_CHARS);
}
return primaryRest(timestamp);
}
}
if (lexer.token() == Token.IDENTIFIER && expr instanceof SQLNumericLiteralExpr) {
String ident = lexer.stringVal();
if (ident.length() == 1) {
char unit = ident.charAt(0);
switch (unit) {
@@ -570,44 +603,6 @@ public class OracleExprParser extends SQLExprParser {
return super.dotRest(expr);
}
public OracleDateExpr parseDate() {
acceptIdentifier("DATE");
OracleDateExpr timestamp = new OracleDateExpr();
String literal = lexer.stringVal();
timestamp.setLiteral(literal);
accept(Token.LITERAL_CHARS);
return timestamp;
}
public SQLExpr parseTimestamp() {
acceptIdentifier("TIMESTAMP");
if (lexer.token() != Token.LITERAL_ALIAS && lexer.token() != Token.LITERAL_CHARS) {
return new SQLIdentifierExpr("TIMESTAMP");
}
OracleTimestampExpr timestamp = new OracleTimestampExpr();
String literal = lexer.stringVal();
timestamp.setLiteral(literal);
accept(Token.LITERAL_CHARS);
if (identifierEquals("AT")) {
lexer.nextToken();
acceptIdentifier("TIME");
acceptIdentifier("ZONE");
String timezone = lexer.stringVal();
timestamp.setTimeZone(timezone);
accept(Token.LITERAL_CHARS);
}
return timestamp;
}
@Override
public OracleOrderBy parseOrderBy() {
if (lexer.token() == (Token.ORDER)) {
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+6 -5
View File
@@ -79,11 +79,7 @@ public class Lexer {
return text.charAt(index);
}
public final int indexOf(char ch, int fromIndex) {
return text.indexOf(ch, fromIndex);
}
public String addSymbol() {
public final String addSymbol() {
return subString(mark, bufPos);
}
@@ -248,6 +244,11 @@ public class Lexer {
return;
}
if (isFirstIdentifierChar(ch) && ch != 'N') {
scanIdentifier();
return;
}
nextToken();
}
src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java
+25 -5
View File
@@ -15,6 +15,8 @@
*/
package com.alibaba.druid.sql.parser;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLOrderBy;
import com.alibaba.druid.sql.ast.SQLSetQuantifier;
@@ -243,12 +245,30 @@ public class SQLSelectParser extends SQLParser {
}
}
protected void parseSelectList(SQLSelectQueryBlock queryBlock) {
queryBlock.getSelectList().add(new SQLSelectItem(expr(), as()));
while (lexer.token() == Token.COMMA) {
protected final void parseSelectList(SQLSelectQueryBlock queryBlock) {
final List<SQLSelectItem> selectList = queryBlock.getSelectList();
for (;;) {
SQLExpr expr;
if (lexer.token() == Token.IDENTIFIER) {
expr = new SQLIdentifierExpr(lexer.stringVal());
lexer.nextTokenComma();
if (lexer.token() != Token.COMMA) {
expr = this.exprParser.primaryRest(expr);
expr = this.exprParser.exprRest(expr);
}
} else {
expr = expr();
}
final String alias = as();
final SQLSelectItem selectItem = new SQLSelectItem(expr, alias);
selectList.add(selectItem);
if (lexer.token() != Token.COMMA) {
break;
}
lexer.nextToken();
queryBlock.getSelectList().add(new SQLSelectItem(expr(), as()));
}
}
src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java
+24 -3
View File
@@ -137,9 +137,30 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter {
}
private String handleName(String ident) {
ident = ident.replaceAll("\"", "");
ident = ident.replaceAll("`", "");
ident = ident.replaceAll(" ", "");
boolean flag0 = false;
boolean flag1 = false;
boolean flag2 = false;
for (int i = 0; i < ident.length(); ++i) {
final char ch = ident.charAt(i);
if (ch == '\"') {
flag0 = true;
} else if (ch == '`') {
flag1 = true;
} else if (ch == ' ') {
flag2 = true;
}
}
if (flag0) {
ident = ident.replaceAll("\"", "");
}
if (flag1) {
ident = ident.replaceAll("`", "");
}
if (flag2) {
ident = ident.replaceAll(" ", "");
}
ident = aliasWrap(ident);
src/main/java/com/alibaba/druid/stat/TableStat.java
+5 -3
View File
@@ -20,6 +20,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.alibaba.druid.util.StringUtils;
public class TableStat {
int selectCount = 0;
@@ -170,7 +172,7 @@ public class TableStat {
}
public int hashCode() {
return this.name.toLowerCase().hashCode();
return StringUtils.lowerHashCode(name);
}
public boolean equals(Object o) {
@@ -401,8 +403,8 @@ public class TableStat {
}
public int hashCode() {
int tableHashCode = table != null ? table.toLowerCase().hashCode() : 0;
int nameHashCode = name != null ? name.toLowerCase().hashCode() : 0;
int tableHashCode = table != null ? StringUtils.lowerHashCode(table) : 0;
int nameHashCode = name != null ? StringUtils.lowerHashCode(name) : 0;
return tableHashCode + nameHashCode;
}
src/main/java/com/alibaba/druid/util/StringUtils.java
+14
View File
@@ -91,4 +91,18 @@ public class StringUtils {
return false;
}
public static int lowerHashCode(String text) {
// return text.toLowerCase().hashCode();
int h = 0;
for (int i = 0; i < text.length(); ++i) {
char ch = text.charAt(i);
if (ch >= 'A' && ch <= 'Z') {
ch = (char) (ch + 32);
}
h = 31 * h + ch;
}
return h;
}
}
src/test/java/com/alibaba/druid/benckmark/sql/MySqlPerfMain_select.java
+73
View File
@@ -0,0 +1,73 @@
/*
* Copyright 1999-2011 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.benckmark.sql;
import java.lang.management.ManagementFactory;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter;
import com.alibaba.druid.sql.test.TestUtils;
public class MySqlPerfMain_select {
public static void main(String[] args) throws Exception {
System.out.println(System.getProperty("java.vm.name") + " " + System.getProperty("java.runtime.version"));
List<String> arguments = ManagementFactory.getRuntimeMXBean().getInputArguments();
System.out.println(arguments);
String sql = "SELECT ID, NAME, AGE FROM USER WHERE ID = ?";
for (int i = 0; i < 10; ++i) {
perfMySql(sql);
}
}
static long perfMySql(String sql) {
long startYGC = TestUtils.getYoungGC();
long startYGCTime = TestUtils.getYoungGCTime();
long startFGC = TestUtils.getFullGC();
long startMillis = System.currentTimeMillis();
for (int i = 0; i < 1000 * 1000; ++i) {
execMySql(sql);
}
long millis = System.currentTimeMillis() - startMillis;
long ygc = TestUtils.getYoungGC() - startYGC;
long ygct = TestUtils.getYoungGCTime() - startYGCTime;
long fgc = TestUtils.getFullGC() - startFGC;
System.out.println("MySql\t" + millis + ", ygc " + ygc + ", ygct " + ygct + ", fgc " + fgc);
return millis;
}
static void execMySql(String sql) {
MySqlStatementParser parser = new MySqlStatementParser(sql);
List<SQLStatement> statementList = parser.parseStatementList();
// StringBuilder out = new StringBuilder();
// MySqlOutputVisitor visitor = new MySqlOutputVisitor(out);
// MySqlASTVisitorAdapter visitor = new MySqlASTVisitorAdapter();
//
// for (SQLStatement statement : statementList) {
// statement.accept(visitor);
// }
// out.toString();
}
}
src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlSelectTest_8.java
+60
View File
@@ -0,0 +1,60 @@
/*
* Copyright 1999-2011 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.mysql;
import java.util.List;
import junit.framework.Assert;
import com.alibaba.druid.sql.MysqlTest;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
public class MySqlSelectTest_8 extends MysqlTest {
public void test_0() throws Exception {
String sql = "select 1";
MySqlStatementParser parser = new MySqlStatementParser(sql);
List<SQLStatement> statementList = parser.parseStatementList();
SQLStatement stmt = statementList.get(0);
SQLSelectStatement selectStmt = (SQLSelectStatement) stmt;
SQLSelect select = selectStmt.getSelect();
Assert.assertNotNull(select.getQuery());
MySqlSelectQueryBlock queryBlock = (MySqlSelectQueryBlock) select.getQuery();
Assert.assertNull(queryBlock.getOrderBy());
print(statementList);
Assert.assertEquals(1, statementList.size());
MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor();
stmt.accept(visitor);
Assert.assertEquals(0, visitor.getTables().size());
Assert.assertEquals(0, visitor.getColumns().size());
Assert.assertEquals(0, visitor.getConditions().size());
Assert.assertEquals(0, visitor.getOrderByColumns().size());
// Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("mytable")));
}
}
src/test/java/com/alibaba/druid/bvt/sql/sqlserver/SQLServerSelectTest11.java
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright 1999-2011 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.sqlserver;
import junit.framework.TestCase;
import org.junit.Assert;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser;
import com.alibaba.druid.sql.test.TestUtils;
public class SQLServerSelectTest11 extends TestCase {
public void test_simple() throws Exception {
String sql = "SELECT 1";
String expect = "SELECT 1";
SQLServerStatementParser parser = new SQLServerStatementParser(sql);
SQLStatement stmt = parser.parseStatementList().get(0);
String text = TestUtils.outputSqlServer(stmt);
Assert.assertEquals(expect, text);
System.out.println(text);
}
}