From 43f2b023e82f3f3c2108670f20951521fca07ee4 Mon Sep 17 00:00:00 2001 From: wenshao Date: Wed, 18 May 2011 09:39:29 +0000 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81Comment=E7=9A=84Parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: http://code.alibabatech.com/svn/druid/trunk@117 b9813039-fb51-4c41-a8b9-e21c2acb5095 --- .../sql/dialect/mysql/parser/MySqlLexer.java | 48 +++++----- .../com/alibaba/druid/sql/parser/Lexer.java | 93 +++++++++++++++---- .../druid/sql/parser/SQLExprParser.java | 12 +-- .../alibaba/druid/sql/parser/SQLLexer.java | 10 +- .../com/alibaba/druid/sql/parser/Token.java | 31 +++++-- .../alibaba/druid/sql/parser/CommentTest.java | 49 ++++++++++ 6 files changed, 184 insertions(+), 59 deletions(-) create mode 100644 src/test/java/com/alibaba/druid/sql/parser/CommentTest.java diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java index 7b2b054d59..651714df67 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java @@ -124,15 +124,15 @@ public class MySqlLexer extends Lexer { if (ch == '`') { int hash = first; - offsetCache = curIndex; - sizeCache = 1; + np = bp; + sp = 1; char ch; for (;;) { - ch = sql[++curIndex]; + ch = buf[++bp]; if (ch == '`') { - sizeCache++; - ch = sql[++curIndex]; + sp++; + ch = buf[++bp]; break; } else if (ch == EOI) { throw new SQLParseException("illegal identifier"); @@ -140,13 +140,13 @@ public class MySqlLexer extends Lexer { hash = 31 * hash + ch; - sizeCache++; + sp++; continue; } - this.ch = sql[curIndex]; + this.ch = buf[bp]; - stringVal = symbolTable.addSymbol(sql, offsetCache, sizeCache, hash); + stringVal = symbolTable.addSymbol(buf, np, sp, hash); Token tok = keywods.getKeyword(stringVal); if (tok != null) { token = tok; @@ -162,11 +162,11 @@ public class MySqlLexer extends Lexer { int hash = first; - offsetCache = curIndex; - sizeCache = 1; + np = bp; + sp = 1; char ch; for (;;) { - ch = sql[++curIndex]; + ch = buf[++bp]; if (!isIdentifierChar(ch)) { break; @@ -174,13 +174,13 @@ public class MySqlLexer extends Lexer { hash = 31 * hash + ch; - sizeCache++; + sp++; continue; } - this.ch = sql[curIndex]; + this.ch = buf[bp]; - stringVal = symbolTable.addSymbol(sql, offsetCache, sizeCache, hash); + stringVal = symbolTable.addSymbol(buf, np, sp, hash); Token tok = keywods.getKeyword(stringVal); if (tok != null) { token = tok; @@ -191,21 +191,21 @@ public class MySqlLexer extends Lexer { } protected void scanString() { - offsetCache = curIndex; + np = bp; boolean hasSpecial = false; for (;;) { - if (curIndex >= sqlLength) { + if (bp >= buflen) { lexError(tokenPos, "unclosed.str.lit"); return; } - ch = sql[++curIndex]; + ch = buf[++bp]; if (ch == '\\') { scanChar(); if (!hasSpecial) { - System.arraycopy(sql, offsetCache + 1, sbuf, 0, sizeCache); + System.arraycopy(buf, np + 1, sbuf, 0, sp); hasSpecial = true; } @@ -250,7 +250,7 @@ public class MySqlLexer extends Lexer { token = LITERAL_CHARS; break; } else { - System.arraycopy(sql, offsetCache + 1, sbuf, 0, sizeCache); + System.arraycopy(buf, np + 1, sbuf, 0, sp); hasSpecial = true; putChar('\''); continue; @@ -258,21 +258,21 @@ public class MySqlLexer extends Lexer { } if (!hasSpecial) { - sizeCache++; + sp++; continue; } - if (sizeCache == sbuf.length) { + if (sp == sbuf.length) { putChar(ch); } else { - sbuf[sizeCache++] = ch; + sbuf[sp++] = ch; } } if (!hasSpecial) { - stringVal = new String(sql, offsetCache + 1, sizeCache); + stringVal = new String(buf, np + 1, sp); } else { - stringVal = new String(sbuf, 0, sizeCache); + stringVal = new String(sbuf, 0, sp); } } } diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index 6295aac2a2..9a5407edc8 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -61,7 +61,7 @@ public class Lexer { protected int sp; /** string point as offset */ - protected int offsetCache; + protected int np; protected SymbolTable symbolTable = new SymbolTable(); @@ -386,7 +386,7 @@ public class Lexer { } protected void scanString() { - offsetCache = bp; + np = bp; boolean hasSpecial = false; for (;;) { @@ -403,7 +403,7 @@ public class Lexer { token = LITERAL_CHARS; break; } else { - System.arraycopy(buf, offsetCache + 1, sbuf, 0, sp); + System.arraycopy(buf, np + 1, sbuf, 0, sp); hasSpecial = true; putChar('\''); continue; @@ -423,7 +423,7 @@ public class Lexer { } if (!hasSpecial) { - stringVal = new String(buf, offsetCache + 1, sp); + stringVal = new String(buf, np + 1, sp); } else { stringVal = new String(sbuf, 0, sp); } @@ -461,7 +461,7 @@ public class Lexer { int hash = first; - offsetCache = bp; + np = bp; sp = 1; char ch; for (;;) { @@ -479,7 +479,7 @@ public class Lexer { this.ch = buf[bp]; - stringVal = symbolTable.addSymbol(buf, offsetCache, sp, hash); + stringVal = symbolTable.addSymbol(buf, np, sp, hash); Token tok = keywods.getKeyword(stringVal); if (tok != null) { token = tok; @@ -489,7 +489,64 @@ public class Lexer { } public void scanComment() { + if (ch != '/') { + throw new IllegalStateException(); + } + np = bp; + sp = 0; + scanChar(); + sp++; + + if (ch == '*') { + scanChar(); + sp++; + + for (;;) { + if (ch == '*' && buf[bp + 1] == '/') { + sp += 2; + scanChar(); + break; + } + + scanChar(); + sp++; + } + + stringVal = new String(buf, np, sp); + token = Token.MULTI_LINE_COMMENT; + return; + } + + if (ch == '/') { + scanChar(); + sp++; + + for (;;) { + if (ch == '\r') { + if (buf[bp + 1] == '\n') { + sp += 2; + scanChar(); + break; + } + sp++; + break; + } + + if (ch == '\r') { + scanChar(); + sp++; + break; + } + + scanChar(); + sp++; + } + + stringVal = new String(buf, np + 1, sp); + token = Token.LINE_COMMENT; + return; + } } public void scanIdentifier() { @@ -502,7 +559,7 @@ public class Lexer { int hash = first; - offsetCache = bp; + np = bp; sp = 1; char ch; for (;;) { @@ -520,7 +577,7 @@ public class Lexer { this.ch = buf[bp]; - stringVal = symbolTable.addSymbol(buf, offsetCache, sp, hash); + stringVal = symbolTable.addSymbol(buf, np, sp, hash); Token tok = keywods.getKeyword(stringVal); if (tok != null) { token = tok; @@ -530,7 +587,7 @@ public class Lexer { } public void scanNumber() { - offsetCache = bp; + np = bp; if (ch == '-') { sp++; @@ -585,14 +642,14 @@ public class Lexer { } if (isDouble) { - token = Token.LITERAL_NUM_MIX_DIGIT; + token = Token.LITERAL_FLOAT; } else { - token = Token.LITERAL_NUM_PURE_DIGIT; + token = Token.LITERAL_INT; } } public void scanHexaDecimal() { - offsetCache = bp; + np = bp; if (ch == '-') { sp++; @@ -612,7 +669,7 @@ public class Lexer { } public String hexString() throws NumberFormatException { - return new String(buf, offsetCache, sp); + return new String(buf, np, sp); } public final boolean isDigit(char ch) { @@ -682,12 +739,12 @@ public class Lexer { public Number integerValue() throws NumberFormatException { long result = 0; boolean negative = false; - int i = offsetCache, max = offsetCache + sp; + int i = np, max = np + sp; long limit; long multmin; int digit; - if (buf[offsetCache] == '-') { + if (buf[np] == '-') { negative = true; limit = Long.MIN_VALUE; i++; @@ -713,7 +770,7 @@ public class Lexer { } if (negative) { - if (i > offsetCache + 1) { + if (i > np + 1) { if (result >= Integer.MIN_VALUE) { return (int) result; } @@ -731,10 +788,10 @@ public class Lexer { } public final String numberString() { - return new String(buf, offsetCache, sp); + return new String(buf, np, sp); } public BigDecimal decimalValue() { - return new BigDecimal(buf, offsetCache, sp); + return new BigDecimal(buf, np, sp); } } diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java index 749c1aeba2..45cc0f9823 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java @@ -163,11 +163,11 @@ public class SQLExprParser extends SQLParser { break; case NEW: throw new ParserException("TODO"); - case LITERAL_NUM_PURE_DIGIT: + case LITERAL_INT: sqlExpr = new SQLIntegerExpr(lexer.integerValue()); lexer.nextToken(); break; - case LITERAL_NUM_MIX_DIGIT: + case LITERAL_FLOAT: sqlExpr = new SQLNumberExpr(lexer.decimalValue()); lexer.nextToken(); break; @@ -261,7 +261,7 @@ public class SQLExprParser extends SQLParser { case SUB: lexer.nextToken(); switch (lexer.token()) { - case LITERAL_NUM_PURE_DIGIT: + case LITERAL_INT: Number integerValue = lexer.integerValue(); if (integerValue instanceof Integer) { int intVal = ((Integer) integerValue).intValue(); @@ -283,7 +283,7 @@ public class SQLExprParser extends SQLParser { sqlExpr = new SQLIntegerExpr(integerValue); lexer.nextToken(); break; - case LITERAL_NUM_MIX_DIGIT: + case LITERAL_FLOAT: sqlExpr = new SQLNumberExpr(lexer.decimalValue().negate()); lexer.nextToken(); break; @@ -294,11 +294,11 @@ public class SQLExprParser extends SQLParser { case PLUS: lexer.nextToken(); switch (lexer.token()) { - case LITERAL_NUM_PURE_DIGIT: + case LITERAL_INT: sqlExpr = new SQLIntegerExpr(lexer.integerValue()); lexer.nextToken(); break; - case LITERAL_NUM_MIX_DIGIT: + case LITERAL_FLOAT: sqlExpr = new SQLNumberExpr(lexer.decimalValue()); lexer.nextToken(); break; diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java b/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java index 3de580d4f3..825dc15430 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java @@ -503,7 +503,7 @@ public class SQLLexer { scanIdentifierFromNumber(offsetCache, sizeCache); return; } else { - token = Token.LITERAL_NUM_PURE_DIGIT; + token = Token.LITERAL_INT; return; } break; @@ -522,7 +522,7 @@ public class SQLLexer { throw err("invalid char after '.': " + ch); } } else { - token = Token.LITERAL_NUM_MIX_DIGIT; + token = Token.LITERAL_FLOAT; return; } break; @@ -540,7 +540,7 @@ public class SQLLexer { throw err("invalid char after '.' for as part of number: " + ch); } } else { - token = Token.LITERAL_NUM_MIX_DIGIT; + token = Token.LITERAL_FLOAT; return; } break; @@ -607,12 +607,12 @@ public class SQLLexer { throw err("expect digit char after SIGN for 'e': " + ch); } } else { - token = Token.LITERAL_NUM_MIX_DIGIT; + token = Token.LITERAL_FLOAT; } return; } } - token = state == 0 ? Token.LITERAL_NUM_PURE_DIGIT : Token.LITERAL_NUM_MIX_DIGIT; + token = state == 0 ? Token.LITERAL_INT : Token.LITERAL_FLOAT; } /** diff --git a/src/main/java/com/alibaba/druid/sql/parser/Token.java b/src/main/java/com/alibaba/druid/sql/parser/Token.java index e4dfb4655a..ed1c29b2ed 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Token.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Token.java @@ -15,9 +15,28 @@ */ package com.alibaba.druid.sql.parser; +/** + * + * SQL Token + * @author wenshao 2011-5-18 下午05:16:49 + * @formatter:off + */ public enum Token { - FOR("FOR"), INDEX("INDEX"), PRIMARY("PRIMARY"), KEY("KEY"), DEFAULT("DEFAULT"), CONSTRAINT("CONSTRAINT"), CHECK("CHECK"), VIEW("VIEW"), CREATE("CREATE"), - ALTER("ALTER"), DROP("DROP"), TABLE("TABLE"), UPDATE("UPDATE"), SET("SET"), SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER("ORDER"), BY("BY"), + FOR("FOR"), + INDEX("INDEX"), + PRIMARY("PRIMARY"), + KEY("KEY"), + DEFAULT("DEFAULT"), + CONSTRAINT("CONSTRAINT"), + CHECK("CHECK"), + VIEW("VIEW"), + CREATE("CREATE"), + ALTER("ALTER"), + DROP("DROP"), + TABLE("TABLE"), + UPDATE("UPDATE"), + SET("SET"), + SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER("ORDER"), BY("BY"), GROUP("GROUP"), HAVING("HAVING"), INSERT("INSERT"), INTO("INTO"), NULL("NULL"), NOT("NOT"), AS("AS"), DELETE("DELETE"), DISTINCT("DISTINCT"), UNIQUE("UNIQUE"), FOREIGN("FOREIGN"), REFERENCE("REFERENCE"), REFERENCES("REFERENCES"), ALL("ALL"), UNION("UNION"), INTERSECT("INTERSECT"), MINUS("MINUS"), INNER("INNER"), LEFT("LEFT"), RIGHT("RIGHT"), FULL("FULL"), OUTER("OUTER"), JOIN("JOIN"), ON("ON"), SCHEMA("SCHEMA"), CAST("CAST"), @@ -35,10 +54,8 @@ public enum Token { HINT, // QS_TODO add support in Lexer SYS_VAR, USR_VAR, - /** number composed purely of digit */ - LITERAL_NUM_PURE_DIGIT, - /** number composed of digit mixed with . or e */ - LITERAL_NUM_MIX_DIGIT, + LITERAL_INT, + LITERAL_FLOAT, LITERAL_HEX, // QS_TODO add syntax support LITERAL_BIT, @@ -46,6 +63,8 @@ public enum Token { LITERAL_NCHARS, // QS_TODO remove alias token LITERAL_ALIAS, + LINE_COMMENT, + MULTI_LINE_COMMENT, LPAREN("("), RPAREN(")"), LBRACE("{"), RBRACE("}"), LBRACKET("["), RBRACKET("]"), SEMI(";"), COMMA(","), DOT("."), EQ("="), GT(">"), LT("<"), BANG("!"), TILDE("~"), QUES("?"), COLON(":"), COLONEQ(":="), EQEQ("=="), LTEQ("<="), LTEQGT("<=>"), LTGT("<>"), GTEQ(">="), BANGEQ("!="), BANGGT("!>"), BANGLT("!<"), diff --git a/src/test/java/com/alibaba/druid/sql/parser/CommentTest.java b/src/test/java/com/alibaba/druid/sql/parser/CommentTest.java new file mode 100644 index 0000000000..cf674f4390 --- /dev/null +++ b/src/test/java/com/alibaba/druid/sql/parser/CommentTest.java @@ -0,0 +1,49 @@ +package com.alibaba.druid.sql.parser; + +import java.util.List; + +import junit.framework.TestCase; + +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.ast.visitor.OracleOutputVisitor; +import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; + +public class CommentTest extends TestCase { + public void test_0 () throws Exception { + String sql = "SELECT /*mark for picman*/ * FROM WP_ALBUM WHERE MEMBER_ID = ? AND ID IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + + Lexer lexer = new Lexer(sql); + for (;;) { + lexer.nextToken(); + Token tok = lexer.token(); + + if (tok == Token.IDENTIFIER) { + System.out.println(tok.name() + "\t\t" + lexer.stringVal); + } else if (tok == Token.MULTI_LINE_COMMENT) { + System.out.println(tok.name() + "\t\t" + lexer.stringVal); + } else { + System.out.println(tok.name() + "\t\t\t" + tok.name); + } + + if (tok == Token.EOF) { + break; + } + } + } + + public void f_test_1 () throws Exception { + String sql = "SELECT /*mark for picman*/ * FROM WP_ALBUM WHERE MEMBER_ID = ? AND ID IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + + StringBuilder out = new StringBuilder(); + OracleOutputVisitor visitor = new OracleOutputVisitor(out); + OracleStatementParser parser = new OracleStatementParser(sql); + List statementList = parser.parseStatementList(); + for (SQLStatement statement : statementList) { + statement.accept(visitor); + visitor.print(";"); + visitor.println(); + } + + System.out.println(out.toString()); + } +}