From cc58cf8be9aff6c6c812ee7ebd1134981989a93f Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 20 Oct 2016 15:59:10 +0800 Subject: [PATCH] improved sql parser. --- .../java/com/alibaba/druid/sql/SQLUtils.java | 32 ++++++++++++--- .../sql/dialect/mysql/parser/MySqlLexer.java | 3 ++ .../sql/dialect/odps/parser/OdpsLexer.java | 2 + .../dialect/oracle/parser/OracleLexer.java | 2 + .../sqlserver/parser/SQLServerLexer.java | 2 + .../com/alibaba/druid/sql/parser/Lexer.java | 18 +++++++- .../druid/sql/parser/SQLParserUtils.java | 41 +++++++++++++++++++ .../ParameterizedOutputVisitorUtils.java | 7 +++- ...ySqlParameterizedOutputVisitorTest_11.java | 27 ++++++++++++ 9 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlParameterizedOutputVisitorTest_11.java diff --git a/src/main/java/com/alibaba/druid/sql/SQLUtils.java b/src/main/java/com/alibaba/druid/sql/SQLUtils.java index ec054ff761..d676fbc328 100644 --- a/src/main/java/com/alibaba/druid/sql/SQLUtils.java +++ b/src/main/java/com/alibaba/druid/sql/SQLUtils.java @@ -47,17 +47,14 @@ import com.alibaba.druid.sql.dialect.postgresql.visitor.PGOutputVisitor; import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor; import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerOutputVisitor; import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerSchemaStatVisitor; -import com.alibaba.druid.sql.parser.ParserException; -import com.alibaba.druid.sql.parser.SQLExprParser; -import com.alibaba.druid.sql.parser.SQLParserUtils; -import com.alibaba.druid.sql.parser.SQLStatementParser; -import com.alibaba.druid.sql.parser.Token; +import com.alibaba.druid.sql.parser.*; import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor; import com.alibaba.druid.sql.visitor.SchemaStatVisitor; import com.alibaba.druid.support.logging.Log; import com.alibaba.druid.support.logging.LogFactory; import com.alibaba.druid.util.JdbcConstants; import com.alibaba.druid.util.StringUtils; +import com.alibaba.druid.util.Utils; public class SQLUtils { public static FormatOption DEFAULT_FORMAT_OPTION = new FormatOption(); @@ -669,5 +666,30 @@ public class SQLUtils { return false; } + + public static long hash(String sql, String dbType) { + Lexer lexer = SQLParserUtils.createLexer(sql, dbType); + + StringBuilder buf = new StringBuilder(sql.length()); + + for (;;) { + lexer.nextToken(); + + Token token = lexer.token(); + if (token == Token.EOF) { + break; + } + + if (token == Token.ERROR) { + return Utils.murmurhash2_64(sql); + } + + if (buf.length() != 0) { + + } + } + + return buf.hashCode(); + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java index e29800b7a0..711f138ee4 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java @@ -114,6 +114,7 @@ public class MySqlLexer extends Lexer { stringVal = subString(mark - 1, bufPos + 1); token = Token.LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } @@ -353,6 +354,7 @@ public class MySqlLexer extends Lexer { } else { stringVal = subString(mark, bufPos); token = Token.MULTI_LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } @@ -399,6 +401,7 @@ public class MySqlLexer extends Lexer { stringVal = subString(mark, bufPos + 1); token = Token.LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java index 936a975b66..6ad18fa30c 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java @@ -115,6 +115,7 @@ public class OdpsLexer extends Lexer { } else { stringVal = subString(mark, bufPos + 1); token = Token.MULTI_LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } @@ -166,6 +167,7 @@ public class OdpsLexer extends Lexer { stringVal = subString(mark, ch != EOI ? bufPos : bufPos + 1); token = Token.LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleLexer.java index b5e54d3063..dd15983bff 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleLexer.java @@ -232,6 +232,7 @@ public class OracleLexer extends Lexer { } else { stringVal = subString(mark, bufPos); token = Token.MULTI_LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } @@ -277,6 +278,7 @@ public class OracleLexer extends Lexer { stringVal = subString(mark + 1, bufPos); token = Token.LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerLexer.java index 130aefeb24..d82803ddd9 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerLexer.java @@ -105,6 +105,7 @@ public class SQLServerLexer extends Lexer { } else { stringVal = subString(mark, bufPos); token = Token.MULTI_LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } @@ -146,6 +147,7 @@ public class SQLServerLexer extends Lexer { stringVal = subString(mark + 1, bufPos); token = Token.LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index 5d61b7293e..b6fcf8b410 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -61,7 +61,9 @@ public class Lexer { protected Keywords keywods = Keywords.DEFAULT_KEYWORDS; protected String stringVal; - + + protected int commentCount = 0; + protected List comments = new ArrayList(2); protected boolean skipComment = true; @@ -95,6 +97,12 @@ public class Lexer { this(input, true); this.commentHandler = commentHandler; } + + public Lexer(String input, CommentHandler commentHandler, String dbType){ + this(input, true); + this.commentHandler = commentHandler; + this.dbType = dbType; + } public boolean isKeepComments() { return keepComments; @@ -300,7 +308,7 @@ public class Lexer { public final void nextToken() { bufPos = 0; - if (comments != null) { + if (comments != null && comments.size() > 0) { comments = null; } @@ -1104,6 +1112,7 @@ public class Lexer { stringVal = subString(mark, bufPos); token = Token.MULTI_LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } @@ -1153,6 +1162,7 @@ public class Lexer { stringVal = subString(mark, bufPos); token = Token.LINE_COMMENT; + commentCount++; if (keepComments) { addComment(stringVal); } @@ -1445,6 +1455,10 @@ public class Lexer { public boolean hasComment() { return comments != null; } + + public int getCommentCount() { + return commentCount; + } public void skipToEOF() { pos = text.length(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java b/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java index 3c2a85b7d1..b6e7152553 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLParserUtils.java @@ -16,18 +16,25 @@ package com.alibaba.druid.sql.parser; import com.alibaba.druid.sql.dialect.db2.parser.DB2ExprParser; +import com.alibaba.druid.sql.dialect.db2.parser.DB2Lexer; import com.alibaba.druid.sql.dialect.db2.parser.DB2StatementParser; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlExprParser; +import com.alibaba.druid.sql.dialect.mysql.parser.MySqlLexer; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.odps.parser.OdpsExprParser; +import com.alibaba.druid.sql.dialect.odps.parser.OdpsLexer; import com.alibaba.druid.sql.dialect.odps.parser.OdpsStatementParser; import com.alibaba.druid.sql.dialect.oracle.parser.OracleExprParser; +import com.alibaba.druid.sql.dialect.oracle.parser.OracleLexer; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; import com.alibaba.druid.sql.dialect.phoenix.parser.PhoenixExprParser; +import com.alibaba.druid.sql.dialect.phoenix.parser.PhoenixLexer; import com.alibaba.druid.sql.dialect.phoenix.parser.PhoenixStatementParser; import com.alibaba.druid.sql.dialect.postgresql.parser.PGExprParser; +import com.alibaba.druid.sql.dialect.postgresql.parser.PGLexer; import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerExprParser; +import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerLexer; import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser; import com.alibaba.druid.util.JdbcUtils; @@ -106,4 +113,38 @@ public class SQLParserUtils { return new SQLExprParser(sql); } + + public static Lexer createLexer(String sql, String dbType) { + if (JdbcUtils.ORACLE.equals(dbType) || JdbcUtils.ALI_ORACLE.equals(dbType)) { + return new OracleLexer(sql); + } + + if (JdbcUtils.MYSQL.equals(dbType) || // + JdbcUtils.MARIADB.equals(dbType) || // + JdbcUtils.H2.equals(dbType)) { + return new MySqlLexer(sql); + } + + if (JdbcUtils.POSTGRESQL.equals(dbType)) { + return new PGLexer(sql); + } + + if (JdbcUtils.SQL_SERVER.equals(dbType) || JdbcUtils.JTDS.equals(dbType)) { + return new SQLServerLexer(sql); + } + + if (JdbcUtils.DB2.equals(dbType)) { + return new DB2Lexer(sql); + } + + if (JdbcUtils.ODPS.equals(dbType)) { + return new OdpsLexer(sql); + } + + if (JdbcUtils.PHOENIX.equals(dbType)) { + return new PhoenixLexer(sql); + } + + return new Lexer(sql); + } } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java b/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java index 9beb00d7e8..75bbde089a 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java @@ -64,10 +64,15 @@ public class ParameterizedOutputVisitorUtils { out.append(";\n"); } SQLStatement stmt = statementList.get(i); + + if (stmt.hasBeforeComment()) { + stmt.getBeforeCommentsDirect().clear(); + } stmt.accept(visitor); } - if (visitor.getReplaceCount() == 0 && !parser.getLexer().hasComment()) { + if (visitor.getReplaceCount() == 0 + && parser.getLexer().getCommentCount() == 0) { return sql; } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlParameterizedOutputVisitorTest_11.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlParameterizedOutputVisitorTest_11.java new file mode 100644 index 0000000000..c251068bb4 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlParameterizedOutputVisitorTest_11.java @@ -0,0 +1,27 @@ +package com.alibaba.druid.bvt.sql.mysql; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlExportParameterVisitor; +import com.alibaba.druid.sql.parser.SQLParserUtils; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.alibaba.druid.sql.visitor.ExportParameterVisitor; +import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +import java.util.List; + +/** + * Created by wenshao on 16/8/23. + */ +public class MySqlParameterizedOutputVisitorTest_11 extends TestCase { + public void test_for_parameterize() throws Exception { + final String dbType = JdbcConstants.MYSQL; + + String sql = "/* 72582af814768580067726386d39b6/0// */ select id,uid from mytable"; + String psql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType); + assertEquals("SELECT id, uid\n" + + "FROM mytable", psql); + } +}