From 5804eea0c9b01411babac031bfaee089bf34bd37 Mon Sep 17 00:00:00 2001 From: wenshao Date: Mon, 21 Aug 2017 15:34:10 +0800 Subject: [PATCH] improved parser performance. --- .../druid/sql/ast/expr/SQLNumberExpr.java | 39 ++++- .../mysql/visitor/MySqlOutputVisitor.java | 116 +++++++++------ .../oracle/ast/stmt/OracleAlterStatement.java | 15 ++ .../stmt/OracleCreateSynonymStatement.java | 15 ++ .../oracle/parser/OracleExprParser.java | 4 +- .../com/alibaba/druid/sql/parser/Lexer.java | 38 ++++- .../druid/sql/parser/SQLExprParser.java | 6 +- .../sql/visitor/SQLASTOutputVisitor.java | 99 +++++++++---- .../sql/visitor/SQLASTOutputVisitorUtils.java | 46 ------ .../com/alibaba/druid/util/StringUtils.java | 135 +++++++++++++++--- .../benckmark/sql/MySqlInsertBenchmark.java | 48 +++++++ .../benckmark/sql/MySqlInsertBenchmark_2.java | 50 +++++++ .../bvt/sql/cobar/DMLInsertParserTest.java | 2 +- .../bvt/sql/cobar/DMLReplaceParserTest.java | 2 +- .../sql/mysql/MathematicalFunctionsTest.java | 4 +- .../druid/bvt/sql/mysql/NumberTest.java | 2 +- .../druid/bvt/sql/odps/OdpsSelectTest22.java | 8 +- .../sql/oracle/OracleNumberLiteralTest.java | 2 +- .../sql/oracle/insert/OracleInsertTest12.java | 2 +- .../sql/oracle/insert/OracleInsertTest14.java | 2 +- 20 files changed, 484 insertions(+), 151 deletions(-) delete mode 100644 src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitorUtils.java create mode 100644 src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark.java create mode 100644 src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark_2.java diff --git a/src/main/java/com/alibaba/druid/sql/ast/expr/SQLNumberExpr.java b/src/main/java/com/alibaba/druid/sql/ast/expr/SQLNumberExpr.java index 5ea6bd6c77..22c97d1926 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/expr/SQLNumberExpr.java +++ b/src/main/java/com/alibaba/druid/sql/ast/expr/SQLNumberExpr.java @@ -19,30 +19,54 @@ import com.alibaba.druid.sql.ast.SQLDataType; import com.alibaba.druid.sql.ast.SQLDataTypeImpl; import com.alibaba.druid.sql.visitor.SQLASTVisitor; +import java.io.IOException; +import java.math.BigDecimal; + public class SQLNumberExpr extends SQLNumericLiteralExpr { private static SQLDataType defaultDataType = new SQLDataTypeImpl("number"); private Number number; + private char[] chars; + public SQLNumberExpr(){ } public SQLNumberExpr(Number number){ - this.number = number; } + public SQLNumberExpr(char[] chars){ + this.chars = chars; + } + public Number getNumber() { + if (chars != null && number == null) { + this.number = new BigDecimal(chars); + } return this.number; } public void setNumber(Number number) { this.number = number; + this.chars = null; + } + + public void output(StringBuilder buf) { + if (chars != null) { + buf.append(chars); + } else { + buf.append(this.number.toString()); + } } public void output(StringBuffer buf) { - buf.append(this.number.toString()); + if (chars != null) { + buf.append(chars); + } else { + buf.append(this.number.toString()); + } } protected void accept0(SQLASTVisitor visitor) { @@ -52,6 +76,9 @@ public class SQLNumberExpr extends SQLNumericLiteralExpr { @Override public int hashCode() { + if (chars != null && number == null) { + this.number = new BigDecimal(chars); + } final int prime = 31; int result = 1; result = prime * result + ((number == null) ? 0 : number.hashCode()); @@ -60,6 +87,10 @@ public class SQLNumberExpr extends SQLNumericLiteralExpr { @Override public boolean equals(Object obj) { + if (chars != null && number == null) { + this.number = new BigDecimal(chars); + } + if (this == obj) { return true; } @@ -70,6 +101,10 @@ public class SQLNumberExpr extends SQLNumericLiteralExpr { return false; } SQLNumberExpr other = (SQLNumberExpr) obj; + if (other.chars != null && other.number == null) { + other.number = new BigDecimal(other.chars); + } + if (number == null) { if (other.number != null) { return false; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java index ab0670c298..a10b4d0d57 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlOutputVisitor.java @@ -53,6 +53,7 @@ import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; import com.alibaba.druid.util.JdbcConstants; +import java.io.IOException; import java.security.AccessControlException; import java.util.List; import java.util.Map; @@ -567,44 +568,59 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV } public boolean visit(SQLCharExpr x) { - if (this.parameterized - && ParameterizedOutputVisitorUtils.checkParameterize(x)) { - print('?'); - incrementReplaceCunt(); - if (this instanceof ExportParameterVisitor || this.parameters != null) { - ExportParameterVisitorUtils.exportParameter(this.parameters, x); - } + if (this.appender == null) { return false; } - print('\''); - - String text = x.getText(); - - StringBuilder buf = new StringBuilder(text.length()); - for (int i = 0; i < text.length(); ++i) { - char ch = text.charAt(i); - if (ch == '\'') { - buf.append('\''); - buf.append('\''); - } else if (ch == '\\') { - buf.append('\\'); - buf.append('\\'); - } else if (ch == '\0') { - buf.append('\\'); - buf.append('0'); - } else { - buf.append(ch); + try { + if (this.parameterized + && ParameterizedOutputVisitorUtils.checkParameterize(x)) { + this.appender.append('?'); + incrementReplaceCunt(); + if (this instanceof ExportParameterVisitor || this.parameters != null) { + ExportParameterVisitorUtils.exportParameter(this.parameters, x); + } + return false; } - } - if (buf.length() != text.length()) { - text = buf.toString(); - } - print0(text); + this.appender.append('\''); - print('\''); + String text = x.getText(); + + boolean hasSpecial = false; + for (int i = 0; i < text.length(); ++i) { + char ch = text.charAt(i); + if (ch == '\'' || ch == '\\' || ch == '\0') { + hasSpecial = true; + break; + } + } + + if (hasSpecial) { + for (int i = 0; i < text.length(); ++i) { + char ch = text.charAt(i); + if (ch == '\'') { + appender.append('\''); + appender.append('\''); + } else if (ch == '\\') { + appender.append('\\'); + appender.append('\\'); + } else if (ch == '\0') { + appender.append('\\'); + appender.append('0'); + } else { + appender.append(ch); + } + } + } else { + appender.append(text); + } + + appender.append('\''); return false; + } catch (IOException e) { + throw new RuntimeException("println error", e); + } } public boolean visit(SQLVariantRefExpr x) { @@ -924,12 +940,18 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV print0(ucase ? "INTO " : "into "); - x.getTableSource().accept(this); + SQLExprTableSource tableSource = x.getTableSource(); + if (tableSource.getClass() == SQLExprTableSource.class) { + visit(tableSource); + } else { + tableSource.accept(this); + } - if (x.getColumns().size() > 0) { + List columns = x.getColumns(); + if (columns.size() > 0) { incrementIndent(); print0(" ("); - for (int i = 0, size = x.getColumns().size(); i < size; ++i) { + for (int i = 0, size = columns.size(); i < size; ++i) { if (i != 0) { if (i % 5 == 0) { println(); @@ -937,15 +959,22 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV print0(", "); } - x.getColumns().get(i).accept(this); + SQLExpr columnn = columns.get(i); + if (columnn instanceof SQLIdentifierExpr) { + visit((SQLIdentifierExpr) columnn); + } + else { + columnn.accept(this); + } } print(')'); decrementIndent(); } - if (!x.getValuesList().isEmpty()) { + List valuesList = x.getValuesList(); + if (!valuesList.isEmpty()) { println(); - printValuesList(x); + printValuesList(valuesList); } if (x.getQuery() != null) { @@ -970,8 +999,7 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV return false; } - protected void printValuesList(MySqlInsertStatement x) { - List valuesList = x.getValuesList(); + protected void printValuesList(List valuesList) { if (parameterized) { print0(ucase ? "VALUES " : "values "); @@ -985,18 +1013,20 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV } print0(ucase ? "VALUES " : "values "); - if (x.getValuesList().size() > 1) { - incrementIndent(); + if (valuesList.size() > 1) { + this.indentCount++; } for (int i = 0, size = valuesList.size(); i < size; ++i) { if (i != 0) { print(','); println(); } - valuesList.get(i).accept(this); + + SQLInsertStatement.ValuesClause item = valuesList.get(i); + visit(item); } if (valuesList.size() > 1) { - decrementIndent(); + this.indentCount--; } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleAlterStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleAlterStatement.java index d1b9eb777c..99f0c21773 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleAlterStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleAlterStatement.java @@ -1,3 +1,18 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.alibaba.druid.sql.dialect.oracle.ast.stmt; import com.alibaba.druid.sql.ast.SQLStatement; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateSynonymStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateSynonymStatement.java index 81218a9108..4311496f1e 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateSynonymStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateSynonymStatement.java @@ -1,3 +1,18 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.alibaba.druid.sql.dialect.oracle.ast.stmt; import com.alibaba.druid.sql.ast.SQLName; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java index 93c31a0e63..0cc0489e93 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java @@ -330,7 +330,7 @@ public class OracleExprParser extends SQLExprParser { lexer.nextToken(); break; case LITERAL_FLOAT: - sqlExpr = new SQLNumberExpr(lexer.decimalValue()); + sqlExpr = lexer.numberExpr(); lexer.nextToken(); break; case BINARY_FLOAT: @@ -382,7 +382,7 @@ public class OracleExprParser extends SQLExprParser { lexer.nextToken(); break; case LITERAL_FLOAT: - sqlExpr = new SQLNumberExpr(lexer.decimalValue().negate()); + sqlExpr = lexer.numberExpr(); lexer.nextToken(); break; case BINARY_FLOAT: diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index c1d1c05628..b2b605db3a 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import com.alibaba.druid.sql.ast.expr.SQLNumberExpr; import com.alibaba.druid.util.JdbcConstants; import com.alibaba.druid.util.StringUtils; @@ -89,6 +90,8 @@ public class Lexer { protected String dbType; + protected boolean hasSpecial = false; + private int startPos; private int posLine; private int posColumn; @@ -144,6 +147,12 @@ public class Lexer { return text.substring(offset, offset + count); } + public final char[] sub_chars(int offset, int count) { + char[] chars = new char[count]; + text.getChars(offset, offset + count, chars, 0); + return chars; + } + protected void initBuff(int size) { if (buf == null) { if (size < 32) { @@ -1522,11 +1531,36 @@ public class Lexer { } public BigDecimal decimalValue() { - String value = subString(mark, bufPos); + char[] value = sub_chars(mark, bufPos); if (!StringUtils.isNumber(value)){ throw new ParserException(value+" is not a number! " + info()); } - return new BigDecimal(value.toCharArray()); + return new BigDecimal(value); + } + + public SQLNumberExpr numberExpr() { + char[] value = sub_chars(mark, bufPos); + if (!StringUtils.isNumber(value)){ + throw new ParserException(value+" is not a number! " + info()); + } + + return new SQLNumberExpr(value); + } + + public SQLNumberExpr numberExpr(boolean negate) { + char[] value = sub_chars(mark, bufPos); + if (!StringUtils.isNumber(value)){ + throw new ParserException(value+" is not a number! " + info()); + } + + if (negate) { + char[] chars = new char[value.length + 1]; + chars[0] = '-'; + System.arraycopy(value, 0, chars, 1, value.length); + return new SQLNumberExpr(chars); + } else { + return new SQLNumberExpr(value); + } } public static interface CommentHandler { diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index ae4bb2a30f..21c9703181 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -297,7 +297,7 @@ public class SQLExprParser extends SQLParser { lexer.nextToken(); break; case LITERAL_FLOAT: - sqlExpr = new SQLNumberExpr(lexer.decimalValue()); + sqlExpr = lexer.numberExpr(); lexer.nextToken(); break; case LITERAL_CHARS: @@ -521,7 +521,7 @@ public class SQLExprParser extends SQLParser { lexer.nextToken(); break; case LITERAL_FLOAT: - sqlExpr = new SQLNumberExpr(lexer.decimalValue().negate()); + sqlExpr = lexer.numberExpr(true); lexer.nextToken(); break; case IDENTIFIER: // 当负号后面为字段的情况 @@ -559,7 +559,7 @@ public class SQLExprParser extends SQLParser { lexer.nextToken(); break; case LITERAL_FLOAT: - sqlExpr = new SQLNumberExpr(lexer.decimalValue()); + sqlExpr = lexer.numberExpr(); lexer.nextToken(); break; case IDENTIFIER: // 当+号后面为字段的情况 diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index d45b61ab3f..258008c1ac 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -53,7 +53,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet protected final Appendable appender; private String indent = "\t"; - private int indentCount = 0; + protected int indentCount = 0; private boolean prettyFormat = true; protected boolean ucase = true; protected int selectListNumberOfLine = 5; @@ -166,11 +166,11 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } public void decrementIndent() { - this.indentCount -= 1; + this.indentCount--; } public void incrementIndent() { - this.indentCount += 1; + this.indentCount++; } public boolean isParameterized() { @@ -214,9 +214,30 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return; } - print0(Integer.toString(value)); + if (appender instanceof StringBuffer) { + ((StringBuffer) appender).append(value); + } else if (appender instanceof StringBuilder) { + ((StringBuilder) appender).append(value); + } else { + print0(Integer.toString(value)); + } } + public void print(long value) { + if (this.appender == null) { + return; + } + + if (appender instanceof StringBuilder) { + ((StringBuilder) appender).append(value); + } else if (appender instanceof StringBuffer) { + ((StringBuffer) appender).append(value); + } else { + print0(Long.toString(value)); + } + } + + public void print(Date date) { if (this.appender == null) { return; @@ -231,13 +252,6 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet print0("'" + dateFormat.format(date) + "'"); } - public void print(long value) { - if (this.appender == null) { - return; - } - print0(Long.toString(value)); - } - public void print(String text) { if (this.appender == null) { return; @@ -707,7 +721,11 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet print0(ucase ? "NULL" : "null"); } else { print('\''); - print0(text.replaceAll("'", "''")); + int index = text.indexOf('\''); + if (index >= 0) { + text = text.replaceAll("'", "''"); + } + print0(text); print('\''); } } @@ -957,7 +975,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } public boolean visit(SQLIntegerExpr x) { - int val = x.getNumber().intValue(); + long val = x.getNumber().longValue(); SQLObject parent = x.getParent(); if (val == 1) { @@ -980,7 +998,8 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet if (this.parameterized && ParameterizedOutputVisitorUtils.checkParameterize(x)) { if (!ParameterizedOutputVisitorUtils.checkParameterize(x)) { - return SQLASTOutputVisitorUtils.visit(this, x); + print(val); + return false; } print('?'); @@ -992,7 +1011,8 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return false; } - return SQLASTOutputVisitorUtils.visit(this, x); + print(val); + return false; } public boolean visit(SQLMethodInvokeExpr x) { @@ -1180,7 +1200,14 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return false; } - return SQLASTOutputVisitorUtils.visit(this, x); + if (appender instanceof StringBuilder) { + x.output((StringBuilder) appender); + } else if (appender instanceof StringBuilder) { + x.output((StringBuilder) appender); + } else { + print0(x.getNumber().toString()); + } + return false; } public boolean visit(SQLPropertyExpr x) { @@ -1612,7 +1639,14 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } } - expr.accept(this); + if (expr instanceof SQLIdentifierExpr) { + visit((SQLIdentifierExpr) expr); + } else if (expr instanceof SQLPropertyExpr) { + visit((SQLPropertyExpr) expr); + } else { + expr.accept(this); + } + } public boolean visit(SQLExprTableSource x) { @@ -2315,8 +2349,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet @Override public boolean visit(ValuesClause x) { print('('); - incrementIndent(); - for (int i = 0, size = x.getValues().size(); i < size; ++i) { + this.indentCount++; + + final List values = x.getValues(); + for (int i = 0, size = values.size(); i < size; ++i) { if (i != 0) { if (i % 5 == 0) { println(); @@ -2324,11 +2360,27 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet print0(", "); } - SQLExpr expr = x.getValues().get(i); - expr.setParent(x); - expr.accept(this); + SQLExpr expr = values.get(i); + if (expr instanceof SQLIntegerExpr) { + visit((SQLIntegerExpr) expr); + } else if (expr instanceof SQLCharExpr) { + visit((SQLCharExpr) expr); + } else if (expr instanceof SQLBooleanExpr) { + visit((SQLBooleanExpr) expr); + } else if (expr instanceof SQLNumberExpr) { + visit((SQLNumberExpr) expr); + } else if (expr instanceof SQLNullExpr) { + visit((SQLNullExpr) expr); + } else if (expr instanceof SQLVariantRefExpr) { + visit((SQLVariantRefExpr) expr); + } else if (expr instanceof SQLNCharExpr) { + visit((SQLNCharExpr) expr); + } else { + expr.accept(this); + } } - decrementIndent(); + + this.indentCount--; print(')'); return false; } @@ -3374,7 +3426,6 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet public boolean visit(SQLBooleanExpr x) { print0(x.getValue() ? "true" : "false"); - return false; } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitorUtils.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitorUtils.java deleted file mode 100644 index 0f2a7ff57f..0000000000 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitorUtils.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 1999-2017 Alibaba Group Holding Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.druid.sql.visitor; - -import com.alibaba.druid.sql.ast.expr.SQLCharExpr; -import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; -import com.alibaba.druid.sql.ast.expr.SQLNumberExpr; - -public class SQLASTOutputVisitorUtils { - - public static boolean visit(PrintableVisitor visitor, SQLIntegerExpr x) { - visitor.print(x.getNumber().toString()); - return false; - } - - public static boolean visit(PrintableVisitor visitor, SQLNumberExpr x) { - visitor.print(x.getNumber().toString()); - return false; - } - - public static boolean visit(PrintableVisitor visitor, SQLCharExpr x) { - visitor.print('\''); - - String text = x.getText(); - text = text.replaceAll("'", "''"); - text = text.replaceAll("\\\\", "\\\\"); - - visitor.print(text); - - visitor.print('\''); - return false; - } -} diff --git a/src/main/java/com/alibaba/druid/util/StringUtils.java b/src/main/java/com/alibaba/druid/util/StringUtils.java index cd17f325b9..20fa117a5b 100644 --- a/src/main/java/com/alibaba/druid/util/StringUtils.java +++ b/src/main/java/com/alibaba/druid/util/StringUtils.java @@ -152,10 +152,108 @@ public class StringUtils { } public static boolean isNumber(String str) { - if (isEmpty(str)) { + if (str.length() == 0) { + return false; + } + int sz = str.length(); + boolean hasExp = false; + boolean hasDecPoint = false; + boolean allowSigns = false; + boolean foundDigit = false; + // deal with any possible sign up front + int start = (str.charAt(0) == '-') ? 1 : 0; + if (sz > start + 1) { + if (str.charAt(start) == '0' && str.charAt(start + 1) == 'x') { + int i = start + 2; + if (i == sz) { + return false; // str == "0x" + } + // checking hex (it can't be anything else) + for (; i < str.length(); i++) { + char ch = str.charAt(i); + if ((ch < '0' || ch > '9') + && (ch < 'a' || ch > 'f') + && (ch < 'A' || ch > 'F')) { + return false; + } + } + return true; + } + } + sz--; // don't want to loop to the last char, check it afterwords + // for type qualifiers + int i = start; + // loop to the next to last char or to the last char if we need another digit to + // make a valid number (e.g. chars[0..5] = "1234E") + while (i < sz || (i < sz + 1 && allowSigns && !foundDigit)) { + char ch = str.charAt(i); + if (ch >= '0' && ch <= '9') { + foundDigit = true; + allowSigns = false; + + } else if (ch == '.') { + if (hasDecPoint || hasExp) { + // two decimal points or dec in exponent + return false; + } + hasDecPoint = true; + } else if (ch == 'e' || ch == 'E') { + // we've already taken care of hex. + if (hasExp) { + // two E's + return false; + } + if (!foundDigit) { + return false; + } + hasExp = true; + allowSigns = true; + } else if (ch == '+' || ch == '-') { + if (!allowSigns) { + return false; + } + allowSigns = false; + foundDigit = false; // we need a digit after the E + } else { + return false; + } + i++; + } + if (i < str.length()) { + char ch = str.charAt(i); + + if (ch >= '0' && ch <= '9') { + // no type qualifier, OK + return true; + } + if (ch == 'e' || ch == 'E') { + // can't have an E at the last byte + return false; + } + if (!allowSigns + && (ch == 'd' + || ch == 'D' + || ch == 'f' + || ch == 'F')) { + return foundDigit; + } + if (ch == 'l' + || ch == 'L') { + // not allowing L with an exponent + return foundDigit && !hasExp; + } + // last character is illegal + return false; + } + // allowSigns is true iff the val ends in 'E' + // found digit it to make sure weird stuff like '.' and '1E-' doesn't pass + return !allowSigns && foundDigit; + } + + public static boolean isNumber(char[] chars) { + if (chars.length == 0) { return false; } - char[] chars = str.toCharArray(); int sz = chars.length; boolean hasExp = false; boolean hasDecPoint = false; @@ -171,9 +269,10 @@ public class StringUtils { } // checking hex (it can't be anything else) for (; i < chars.length; i++) { - if ((chars[i] < '0' || chars[i] > '9') - && (chars[i] < 'a' || chars[i] > 'f') - && (chars[i] < 'A' || chars[i] > 'F')) { + char ch = chars[i]; + if ((ch < '0' || ch > '9') + && (ch < 'a' || ch > 'f') + && (ch < 'A' || ch > 'F')) { return false; } } @@ -186,17 +285,18 @@ public class StringUtils { // loop to the next to last char or to the last char if we need another digit to // make a valid number (e.g. chars[0..5] = "1234E") while (i < sz || (i < sz + 1 && allowSigns && !foundDigit)) { - if (chars[i] >= '0' && chars[i] <= '9') { + char ch = chars[i]; + if (ch >= '0' && ch <= '9') { foundDigit = true; allowSigns = false; - } else if (chars[i] == '.') { + } else if (ch == '.') { if (hasDecPoint || hasExp) { // two decimal points or dec in exponent return false; } hasDecPoint = true; - } else if (chars[i] == 'e' || chars[i] == 'E') { + } else if (ch == 'e' || ch == 'E') { // we've already taken care of hex. if (hasExp) { // two E's @@ -207,7 +307,7 @@ public class StringUtils { } hasExp = true; allowSigns = true; - } else if (chars[i] == '+' || chars[i] == '-') { + } else if (ch == '+' || ch == '-') { if (!allowSigns) { return false; } @@ -219,23 +319,24 @@ public class StringUtils { i++; } if (i < chars.length) { - if (chars[i] >= '0' && chars[i] <= '9') { + char ch = chars[i]; + if (ch >= '0' && ch <= '9') { // no type qualifier, OK return true; } - if (chars[i] == 'e' || chars[i] == 'E') { + if (ch == 'e' || ch == 'E') { // can't have an E at the last byte return false; } if (!allowSigns - && (chars[i] == 'd' - || chars[i] == 'D' - || chars[i] == 'f' - || chars[i] == 'F')) { + && (ch == 'd' + || ch == 'D' + || ch == 'f' + || ch == 'F')) { return foundDigit; } - if (chars[i] == 'l' - || chars[i] == 'L') { + if (ch == 'l' + || ch == 'L') { // not allowing L with an exponent return foundDigit && !hasExp; } diff --git a/src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark.java b/src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark.java new file mode 100644 index 0000000000..b271d894ba --- /dev/null +++ b/src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark.java @@ -0,0 +1,48 @@ +package com.alibaba.druid.benckmark.sql; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; +import org.apache.ibatis.jdbc.SQL; + +import java.util.List; + +public class MySqlInsertBenchmark extends TestCase { + static String sql = "INSERT INTO test_table VALUES (1, '1', '2017-10-10', true, false, '2017-10-10 10:10:10', '10:10:10', 1.111, null);"; + List stmtList; + + protected void setUp() throws Exception { + stmtList = SQLUtils.parseStatements(sql, JdbcConstants.MYSQL); + + } + + public void test_perf() throws Exception { + for (int i = 0; i < 5; ++i) { + perf(); +// perf_toString(); // 425 + } + } + + public void perf_toString() { + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + SQLUtils.toMySqlString(stmtList.get(0)); + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + } + + public void perf() { + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + + List stmt = SQLUtils.parseStatements(sql, JdbcConstants.MYSQL); +// stmt.toString(); + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + } + + +} diff --git a/src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark_2.java b/src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark_2.java new file mode 100644 index 0000000000..4d0487718f --- /dev/null +++ b/src/test/java/com/alibaba/druid/benckmark/sql/MySqlInsertBenchmark_2.java @@ -0,0 +1,50 @@ +package com.alibaba.druid.benckmark.sql; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +public class MySqlInsertBenchmark_2 extends TestCase { + static String sql = "INSERT INTO test_table VALUES (1, '1', '2017-10-10', true, false, '2017-10-10 10:10:10', '10:10:10', 1.111, NULL), (2, '2', '2017-10-10', true, false, '2017-10-10 10:10:10', '10:10:10', 2.222, NULL)" + + ", (2, '2', '2017-09-09', true, false, '2017-10-10 10:10:10', '10:10:10', 3.333, NULL)" + + ", (3, '3', '2017-10-10', true, false, '2017-10-10 10:10:10', '11:11:11', 4.333, NULL)" + + ", (4, '4', '2017-10-10', true, false, '2017-10-10 10:10:10', '11:11:11', 4.333, NULL)" + + ", (5, '5', '2017-10-10', true, false, '2017-10-10 10:10:10', '11:11:11', 4.333, NULL);"; + List stmtList; + + protected void setUp() throws Exception { + stmtList = SQLUtils.parseStatements(sql, JdbcConstants.MYSQL); + + } + + public void test_perf() throws Exception { + System.out.println(sql); + for (int i = 0; i < 5; ++i) { + perf(); // 5043 +// perf_toString(); // 2101 + } + } + + public void perf() { + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + + List stmt = SQLUtils.parseStatements(sql, JdbcConstants.MYSQL); +// stmt.toString(); + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + } + + public void perf_toString() { + long startMillis = System.currentTimeMillis(); + for (int i = 0; i < 1000 * 1000; ++i) { + SQLUtils.toMySqlString(stmtList.get(0)); + } + long millis = System.currentTimeMillis() - startMillis; + System.out.println("millis : " + millis); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLInsertParserTest.java b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLInsertParserTest.java index 8f62c4bfa7..bb9f9a0c95 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLInsertParserTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLInsertParserTest.java @@ -61,7 +61,7 @@ public class DMLInsertParserTest extends TestCase { SQLStatement stmt = parser.parseStatementList().get(0); parser.match(Token.EOF); String output = SQLUtils.toMySqlString(stmt); - Assert.assertEquals("INSERT LOW_PRIORITY INTO t1\nVALUES (0.12, 1, 2)," + // + Assert.assertEquals("INSERT LOW_PRIORITY INTO t1\nVALUES (12e-2, 1, 2)," + // "\n\t(?)," + // "\n\t(DEFAULT)", output); } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLReplaceParserTest.java b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLReplaceParserTest.java index ff36bf11ab..9a74db9f88 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLReplaceParserTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DMLReplaceParserTest.java @@ -59,7 +59,7 @@ public class DMLReplaceParserTest extends TestCase { SQLStatement stmt = parser.parseStatementList().get(0); parser.match(Token.EOF); String output = SQLUtils.toMySqlString(stmt); - Assert.assertEquals("REPLACE LOW_PRIORITY INTO t1\nVALUES (0.12), (?)", output); + Assert.assertEquals("REPLACE LOW_PRIORITY INTO t1\nVALUES (12e-2), (?)", output); } public void testReplace_4() throws Exception { diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/MathematicalFunctionsTest.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MathematicalFunctionsTest.java index 7a2ffb0e85..20e5c10059 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/MathematicalFunctionsTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MathematicalFunctionsTest.java @@ -268,7 +268,7 @@ public class MathematicalFunctionsTest extends TestCase { String text = output(stmtList); - Assert.assertEquals("SELECT PI() + 0E-18", text); + Assert.assertEquals("SELECT PI() + 0.000000000000000000", text); } public void test_22() throws Exception { @@ -323,7 +323,7 @@ public class MathematicalFunctionsTest extends TestCase { String text = output(stmtList); - Assert.assertEquals("SELECT ROUND(2.5), ROUND(2.5)", text); + Assert.assertEquals("SELECT ROUND(2.5), ROUND(25E-1)", text); } private String output(List stmtList) { diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/NumberTest.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/NumberTest.java index e524c15eca..cff50de6b2 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/NumberTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/NumberTest.java @@ -38,7 +38,7 @@ public class NumberTest extends TestCase { String text = SQLUtils.toSQLString(stmtList, JdbcConstants.MYSQL); - Assert.assertEquals("SELECT -3.20326809E+14;", text); + Assert.assertEquals("SELECT -32032.6809e+10;", text); } public void test_1() throws Exception { diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest22.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest22.java index a1530a8a7c..f084498d35 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest22.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest22.java @@ -56,10 +56,10 @@ public class OdpsSelectTest22 extends TestCase { ") A\n" + "group by bucket_id;";// assertEquals("SELECT bucket_id, SUM(pv) AS pv, SUM(clk) AS clk\n" + - "\t, SUM(clk) / (SUM(pv) + 1E-10) AS ctr\n" + + "\t, SUM(clk) / (SUM(pv) + 1e-10) AS ctr\n" + "\t, SUM(ut_ad_clk) AS ut_ad_clk, SUM(ad_clk) AS ad_clk\n" + "\t, SUM(cost) AS cost\n" + - "\t, SUM(cost) * 1000 / (SUM(pv) + 1E-10) / 100 AS rpm\n" + + "\t, SUM(cost) * 1000 / (SUM(pv) + 1e-10) / 100 AS rpm\n" + "FROM (\n" + "\tSELECT bucket_id, UT.pv AS pv, UT.clk AS clk, UT.ad_clk AS ut_ad_clk, CLK.clk AS ad_clk\n" + "\t\t, CLK.cost AS cost\n" + @@ -95,10 +95,10 @@ public class OdpsSelectTest22 extends TestCase { "GROUP BY bucket_id;", SQLUtils.formatOdps(sql)); assertEquals("select bucket_id, sum(pv) as pv, sum(clk) as clk\n" + - "\t, sum(clk) / (sum(pv) + 1E-10) as ctr\n" + + "\t, sum(clk) / (sum(pv) + 1e-10) as ctr\n" + "\t, sum(ut_ad_clk) as ut_ad_clk, sum(ad_clk) as ad_clk\n" + "\t, sum(cost) as cost\n" + - "\t, sum(cost) * 1000 / (sum(pv) + 1E-10) / 100 as rpm\n" + + "\t, sum(cost) * 1000 / (sum(pv) + 1e-10) / 100 as rpm\n" + "from (\n" + "\tselect bucket_id, UT.pv as pv, UT.clk as clk, UT.ad_clk as ut_ad_clk, CLK.clk as ad_clk\n" + "\t\t, CLK.cost as cost\n" + diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleNumberLiteralTest.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleNumberLiteralTest.java index 5ec1b51cf2..ffdbcdc6be 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleNumberLiteralTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleNumberLiteralTest.java @@ -32,7 +32,7 @@ public class OracleNumberLiteralTest extends TestCase { String text = TestUtils.outputOracle(stmt); - Assert.assertEquals("SELECT 7, 255, 0.5, 6.34, 0.025\n\t, 6.34F, 0.5D, -1.0D\nFROM DUAL", text); + Assert.assertEquals("SELECT 7, 255, 0.5, 6.34, 25e-03\n\t, 6.34F, 0.5D, -1.0D\nFROM DUAL", text); System.out.println(text); } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest12.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest12.java index 56d54547ef..7425dff612 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest12.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest12.java @@ -46,7 +46,7 @@ public class OracleInsertTest12 extends OracleTest { + "\n\t(employee_id, last_name, email, hire_date, job_id" // + "\n\t, salary, commission_pct)" // + "\nVALUES (207, 'Gregory', 'pgregory@example.com', SYSDATE, 'PU_CLERK'" // - + "\n\t, 1.2E+3, NULL);",// + + "\n\t, 1.2E3, NULL);",// SQLUtils.toSQLString(stmt, JdbcConstants.ORACLE)); OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest14.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest14.java index 9ad616b7df..0c5fb31623 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest14.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/insert/OracleInsertTest14.java @@ -45,7 +45,7 @@ public class OracleInsertTest14 extends OracleTest { Assert.assertEquals("INSERT INTO raises" // + "\nSELECT employee_id, salary * 1.1"// + "\nFROM employees"// - + "\nWHERE commission_pct > 0.2"// + + "\nWHERE commission_pct > .2"// + "\nLOG ERRORS INTO errlog ('my_bad') REJECT LIMIT 10;",// SQLUtils.toSQLString(stmt, JdbcConstants.ORACLE));