From 91b9b5eeb13e4f214e0ab92a90dc34a6c594804d Mon Sep 17 00:00:00 2001 From: wenshao Date: Sat, 17 Mar 2018 01:00:35 +0800 Subject: [PATCH] bug fixed for mysql concat literal string. --- .../mysql/parser/MySqlCreateTableParser.java | 4 +- .../dialect/mysql/parser/MySqlExprParser.java | 7 +- .../mysql/parser/MySqlSelectParser.java | 2 +- .../com/alibaba/druid/sql/parser/Lexer.java | 218 +++++++++++++++++- .../druid/sql/parser/SQLExprParser.java | 37 ++- .../druid/sql/parser/SQLStatementParser.java | 10 +- .../sql/visitor/SQLASTOutputVisitor.java | 8 +- .../bvt/sql/mysql/StringFunctionsTest.java | 2 +- .../insert/MySqlInsertTest_27_str_concat.java | 117 ++++++++++ .../select/MySqlSelectTest_71_concat.java | 6 +- 10 files changed, 362 insertions(+), 49 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_27_str_concat.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java index 523618918a..30dce4ae6b 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlCreateTableParser.java @@ -89,7 +89,7 @@ public class MySqlCreateTableParser extends SQLCreateTableParser { stmt.setName(this.exprParser.name()); if (lexer.token() == Token.LIKE) { - lexer.nextToken(); + lexer.nextTokenValue(); SQLName name = this.exprParser.name(); stmt.setLike(name); } @@ -98,7 +98,7 @@ public class MySqlCreateTableParser extends SQLCreateTableParser { lexer.nextToken(); if (lexer.token() == Token.LIKE) { - lexer.nextToken(); + lexer.nextTokenValue(); SQLName name = this.exprParser.name(); stmt.setLike(name); } else if (lexer.token() == Token.SELECT) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java index 68985f8b4d..46ec6ae785 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java @@ -228,14 +228,13 @@ public class MySqlExprParser extends SQLExprParser { return primaryRest(expr); } } else if (expr instanceof SQLCharExpr) { - SQLMethodInvokeExpr concat = new SQLMethodInvokeExpr("CONCAT"); - concat.addParameter(expr); + String text2 = ((SQLCharExpr) expr).getText(); do { String chars = lexer.stringVal(); - concat.addParameter(new SQLCharExpr(chars)); + text2 += chars; lexer.nextToken(); } while (lexer.token() == Token.LITERAL_CHARS || lexer.token() == Token.LITERAL_ALIAS); - expr = concat; + expr = new SQLCharExpr(text2); } } else if (lexer.token() == Token.IDENTIFIER) { if (expr instanceof SQLHexExpr) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java index bc68cf0a0b..d4645d35c5 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlSelectParser.java @@ -101,7 +101,7 @@ public class MySqlSelectParser extends SQLSelectParser { } if (lexer.token() == Token.SELECT) { - lexer.nextToken(); + lexer.nextTokenValue(); if (lexer.token() == Token.HINT) { this.exprParser.parseHints(queryBlock.getHints()); diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index e1ab2355cf..443a95316f 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -330,6 +330,46 @@ public class Lexer { nextToken(); } + public final void nextTokenCommaValue() { + if (ch == ' ') { + scanChar(); + } + + if (ch == ',' || ch == ',') { + scanChar(); + token = COMMA; + return; + } + + if (ch == ')' || ch == ')') { + scanChar(); + token = RPAREN; + return; + } + + if (ch == '.') { + scanChar(); + token = DOT; + return; + } + + if (ch == 'a' || ch == 'A') { + char ch_next = charAt(pos + 1); + if (ch_next == 's' || ch_next == 'S') { + char ch_next_2 = charAt(pos + 2); + if (ch_next_2 == ' ') { + pos += 2; + ch = ' '; + token = Token.AS; + stringVal = "AS"; + return; + } + } + } + + nextTokenValue(); + } + public final void nextTokenEq() { if (ch == ' ') { scanChar(); @@ -389,7 +429,25 @@ public class Lexer { return; } - if (ch >= '0' && ch <= '9') { + if (ch == '"') { + bufPos = 0; + scanString2_d(); + return; + } + + if (ch == '0') { + bufPos = 0; + if (charAt(pos + 1) == 'x') { + scanChar(); + scanChar(); + scanHexaDecimal(); + } else { + scanNumber(); + } + return; + } + + if (ch > '0' && ch <= '9') { bufPos = 0; scanNumber(); return; @@ -1156,6 +1214,149 @@ public class Lexer { stringVal = new String(buf, 0, bufPos); } } + + protected final void scanString2_d() { + { + boolean hasSpecial = false; + int startIndex = pos + 1; + int endIndex = -1; // text.indexOf('\'', startIndex); + for (int i = startIndex; i < text.length(); ++i) { + final char ch = text.charAt(i); + if (ch == '\\') { + hasSpecial = true; + continue; + } + if (ch == '"') { + endIndex = i; + break; + } + } + + if (endIndex == -1) { + throw new ParserException("unclosed str. " + info()); + } + + String stringVal; + if (token == Token.AS) { + stringVal = subString(pos, endIndex + 1 - pos); + } else { + stringVal = subString(startIndex, endIndex - startIndex); + } + // hasSpecial = stringVal.indexOf('\\') != -1; + + if (!hasSpecial) { + this.stringVal = stringVal; + int pos = endIndex + 1; + char ch = charAt(pos); + if (ch != '\'') { + this.pos = pos; + this.ch = ch; + token = LITERAL_CHARS; + return; + } + } + } + + mark = pos; + boolean hasSpecial = false; + for (;;) { + if (isEOF()) { + lexError("unclosed.str.lit"); + return; + } + + ch = charAt(++pos); + + if (ch == '\\') { + scanChar(); + if (!hasSpecial) { + initBuff(bufPos); + arraycopy(mark + 1, buf, 0, bufPos); + hasSpecial = true; + } + + + switch (ch) { + case '0': + putChar('\0'); + break; + case '\'': + putChar('\''); + break; + case '"': + putChar('"'); + break; + case 'b': + putChar('\b'); + break; + case 'n': + putChar('\n'); + break; + case 'r': + putChar('\r'); + break; + case 't': + putChar('\t'); + break; + case '\\': + putChar('\\'); + break; + case 'Z': + putChar((char) 0x1A); // ctrl + Z + break; + case '%': + if(JdbcConstants.MYSQL.equals(dbType)) { + putChar('\\'); + } + putChar('%'); + break; + case '_': + if(JdbcConstants.MYSQL.equals(dbType)) { + putChar('\\'); + } + putChar('_'); + break; + default: + putChar(ch); + break; + } + + continue; + } + if (ch == '"') { + scanChar(); + if (ch != '"') { + token = LITERAL_CHARS; + break; + } else { + if (!hasSpecial) { + initBuff(bufPos); + arraycopy(mark + 1, buf, 0, bufPos); + hasSpecial = true; + } + putChar('"'); + continue; + } + } + + if (!hasSpecial) { + bufPos++; + continue; + } + + if (bufPos == buf.length) { + putChar(ch); + } else { + buf[bufPos++] = ch; + } + } + + if (!hasSpecial) { + stringVal = subString(mark + 1, bufPos); + } else { + stringVal = new String(buf, 0, bufPos); + } + } protected final void scanAlias() { { @@ -1260,13 +1461,14 @@ public class Lexer { continue; } - if (ch == '\'') { - char ch_next = charAt(pos + 1); - if (ch_next == '"') { - scanChar(); - continue; - } - } else if (ch == '\"') { +// if (ch == '\'') { +// char ch_next = charAt(pos + 1); +// if (ch_next == '"') { +// scanChar(); +// continue; +// } +// } else + if (ch == '\"') { char ch_next = charAt(pos + 1); if (ch_next == '"' || ch_next == '\'') { scanChar(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index 2d6b06d20d..5fab72d7f3 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -347,36 +347,27 @@ public class SQLExprParser extends SQLParser { break; case LITERAL_CHARS: { sqlExpr = new SQLCharExpr(lexer.stringVal()); - lexer.nextToken(); if (JdbcConstants.MYSQL.equals(dbType)) { - SQLMethodInvokeExpr concat = null; + lexer.nextTokenValue(); + for (; ; ) { if (lexer.token == Token.LITERAL_ALIAS) { - if (concat == null) { - concat = new SQLMethodInvokeExpr("CONCAT"); - concat.addParameter(sqlExpr); - sqlExpr = concat; - } - String alias = lexer.stringVal(); - lexer.nextToken(); - SQLCharExpr concat_right = new SQLCharExpr(alias.substring(1, alias.length() - 1)); - concat.addParameter(concat_right); + String concat = ((SQLCharExpr) sqlExpr).getText(); + concat += lexer.stringVal(); + lexer.nextTokenValue(); + sqlExpr = new SQLCharExpr(concat); } else if (lexer.token == Token.LITERAL_CHARS || lexer.token == Token.LITERAL_NCHARS) { - if (concat == null) { - concat = new SQLMethodInvokeExpr("CONCAT"); - concat.addParameter(sqlExpr); - sqlExpr = concat; - } - - String chars = lexer.stringVal(); - lexer.nextToken(); - SQLCharExpr concat_right = new SQLCharExpr(chars); - concat.addParameter(concat_right); + String concat = ((SQLCharExpr) sqlExpr).getText(); + concat += lexer.stringVal(); + lexer.nextTokenValue(); + sqlExpr = new SQLCharExpr(concat); } else { break; } } + } else { + lexer.nextToken(); } break; } case LITERAL_NCHARS: @@ -2091,7 +2082,7 @@ public class SQLExprParser extends SQLParser { expr = new SQLBinaryOpExpr(expr, SQLBinaryOperator.LessThanOrGreater, rightExp, getDbType()); } else if (token == Token.LIKE) { - lexer.nextToken(); + lexer.nextTokenValue(); rightExp = bitOr(); // rightExp = relationalRest(rightExp); @@ -2232,7 +2223,7 @@ public class SQLExprParser extends SQLParser { public SQLExpr notRationalRest(SQLExpr expr) { if (lexer.token == (Token.LIKE)) { - lexer.nextToken(); + lexer.nextTokenValue(); SQLExpr rightExp = equality(); rightExp = relationalRest(rightExp); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index 7849b9fce2..ca37f328a9 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -3228,7 +3228,7 @@ public class SQLStatementParser extends SQLParser { } else { expr = new SQLIntegerExpr(lexer.integerValue()); } - lexer.nextTokenComma(); + lexer.nextTokenCommaValue(); if (lexer.token != Token.COMMA && lexer.token != Token.RPAREN) { expr = this.exprParser.exprRest(expr); @@ -3240,7 +3240,7 @@ public class SQLStatementParser extends SQLParser { } else { expr = new SQLCharExpr(lexer.stringVal()); } - lexer.nextTokenComma(); + lexer.nextTokenCommaValue(); if (lexer.token != Token.COMMA && lexer.token != Token.RPAREN) { expr = this.exprParser.exprRest(expr); } @@ -3251,7 +3251,7 @@ public class SQLStatementParser extends SQLParser { } else { expr = new SQLNCharExpr(lexer.stringVal()); } - lexer.nextTokenComma(); + lexer.nextTokenCommaValue(); if (lexer.token != Token.COMMA && lexer.token != Token.RPAREN) { expr = this.exprParser.exprRest(expr); } @@ -3262,7 +3262,7 @@ public class SQLStatementParser extends SQLParser { } else { expr = new SQLNullExpr(); } - lexer.nextTokenComma(); + lexer.nextTokenCommaValue(); if (lexer.token != Token.COMMA && lexer.token != Token.RPAREN) { expr = this.exprParser.exprRest(expr); } @@ -3287,7 +3287,7 @@ public class SQLStatementParser extends SQLParser { valueExprList.add(expr); if (lexer.token() == Token.COMMA) { - lexer.nextToken(); + lexer.nextTokenValue(); continue; } else { break; diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 69940b94e2..e1a88994c0 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -55,7 +55,11 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet public static Boolean defaultPrintStatementAfterSemi; static { - defaultPrintStatementAfterSemi = getBoolean(System.getProperties(), "druid.sql.output.printStatementAfterSemi"); // compatible for early versions + try { + defaultPrintStatementAfterSemi = getBoolean(System.getProperties(), "druid.sql.output.printStatementAfterSemi"); // compatible for early versions + } catch (Throwable ex) { + // skip + } } protected final Appendable appender; @@ -1233,7 +1237,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet printExpr(x.getExpr()); if (x.isNot()) { - print(ucase ? " NOT" : " not"); + print(ucase ? " NOT IN" : " not in"); } else { print(ucase ? " IN" : " in"); } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/StringFunctionsTest.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/StringFunctionsTest.java index ca79b17a13..e9616f5c08 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/StringFunctionsTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/StringFunctionsTest.java @@ -139,7 +139,7 @@ public class StringFunctionsTest extends TestCase { String text = output(stmtList); - assertEquals("SELECT CONCAT('My', 'S', 'QL')", text); + assertEquals("SELECT 'MySQL'", text); } public void test_10() throws Exception { diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_27_str_concat.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_27_str_concat.java new file mode 100644 index 0000000000..054b30419e --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/insert/MySqlInsertTest_27_str_concat.java @@ -0,0 +1,117 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.mysql.insert; + +import com.alibaba.druid.sql.MysqlTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlInsertStatement; +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; +import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; +import com.alibaba.druid.util.JdbcConstants; + +import java.util.ArrayList; +import java.util.List; + +public class MySqlInsertTest_27_str_concat extends MysqlTest { + + public void test_insert_concat() throws Exception { + String sql = "insert ignore into ktv_sms_test (cp) values ('a' 'b')"; + + MySqlStatementParser parser = new MySqlStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + + MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt; + + assertEquals("INSERT IGNORE INTO ktv_sms_test (cp)\n" + + "VALUES ('ab')", SQLUtils.toMySqlString(insertStmt)); + + System.out.println(sql); + System.out.println(stmt.toString()); + + { + List outParameters = new ArrayList(); + String psql = ParameterizedOutputVisitorUtils.parameterize(sql, JdbcConstants.MYSQL, outParameters); + assertEquals("INSERT IGNORE INTO ktv_sms_test(cp)\n" + + "VALUES (?)", psql); + + assertEquals(1, outParameters.size()); + + String rsql = ParameterizedOutputVisitorUtils.restore(psql, JdbcConstants.MYSQL, outParameters); + assertEquals("INSERT IGNORE INTO ktv_sms_test (cp)\n" + + "VALUES ('ab')", rsql); + } + } + + public void test_insert_concat_2() throws Exception { + String sql = "insert ignore into ktv_sms_test (cp) values (\"a\" \"b\")"; + + MySqlStatementParser parser = new MySqlStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + + MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt; + + assertEquals("INSERT IGNORE INTO ktv_sms_test (cp)\n" + + "VALUES ('ab')", SQLUtils.toMySqlString(insertStmt)); + + System.out.println(sql); + System.out.println(stmt.toString()); + + { + List outParameters = new ArrayList(); + String psql = ParameterizedOutputVisitorUtils.parameterize(sql, JdbcConstants.MYSQL, outParameters); + assertEquals("INSERT IGNORE INTO ktv_sms_test(cp)\n" + + "VALUES (?)", psql); + + assertEquals(1, outParameters.size()); + + String rsql = ParameterizedOutputVisitorUtils.restore(psql, JdbcConstants.MYSQL, outParameters); + assertEquals("INSERT IGNORE INTO ktv_sms_test (cp)\n" + + "VALUES ('ab')", rsql); + } + } + + public void test_insert_concat_3() throws Exception { + String sql = "insert ignore into ktv_sms_test (cp) values (\"a\" 'b')"; + + MySqlStatementParser parser = new MySqlStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + + MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt; + + assertEquals("INSERT IGNORE INTO ktv_sms_test (cp)\n" + + "VALUES ('ab')", SQLUtils.toMySqlString(insertStmt)); + + System.out.println(sql); + System.out.println(stmt.toString()); + + { + List outParameters = new ArrayList(); + String psql = ParameterizedOutputVisitorUtils.parameterize(sql, JdbcConstants.MYSQL, outParameters); + assertEquals("INSERT IGNORE INTO ktv_sms_test(cp)\n" + + "VALUES (?)", psql); + + assertEquals(1, outParameters.size()); + + String rsql = ParameterizedOutputVisitorUtils.restore(psql, JdbcConstants.MYSQL, outParameters); + assertEquals("INSERT IGNORE INTO ktv_sms_test (cp)\n" + + "VALUES ('ab')", rsql); + } + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_71_concat.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_71_concat.java index b4a31a9cca..1b8b295615 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_71_concat.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/select/MySqlSelectTest_71_concat.java @@ -43,18 +43,18 @@ public class MySqlSelectTest_71_concat extends MysqlTest { stmt.accept(visitor); { String output = SQLUtils.toMySqlString(stmt); - assertEquals("SELECT CONCAT('afdf', 'erwhg', 'ads', 'dgs', 'rter') AS tstDiffQuoteConcat;", // + assertEquals("SELECT 'afdferwhgadsdgsrter' AS tstDiffQuoteConcat;", // output); } { String output = SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION); - assertEquals("select CONCAT('afdf', 'erwhg', 'ads', 'dgs', 'rter') as tstDiffQuoteConcat;", // + assertEquals("select 'afdferwhgadsdgsrter' as tstDiffQuoteConcat;", // output); } { String output = SQLUtils.toMySqlString(stmt, new SQLUtils.FormatOption(true, true, true)); - assertEquals("SELECT CONCAT(?, ?, ?, ?, ?) AS tstDiffQuoteConcat;", // + assertEquals("SELECT ? AS tstDiffQuoteConcat;", // output); } }