diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java
index 7b2b054d59..651714df67 100644
--- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java
+++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java
@@ -124,15 +124,15 @@ public class MySqlLexer extends Lexer {
if (ch == '`') {
int hash = first;
- offsetCache = curIndex;
- sizeCache = 1;
+ np = bp;
+ sp = 1;
char ch;
for (;;) {
- ch = sql[++curIndex];
+ ch = buf[++bp];
if (ch == '`') {
- sizeCache++;
- ch = sql[++curIndex];
+ sp++;
+ ch = buf[++bp];
break;
} else if (ch == EOI) {
throw new SQLParseException("illegal identifier");
@@ -140,13 +140,13 @@ public class MySqlLexer extends Lexer {
hash = 31 * hash + ch;
- sizeCache++;
+ sp++;
continue;
}
- this.ch = sql[curIndex];
+ this.ch = buf[bp];
- stringVal = symbolTable.addSymbol(sql, offsetCache, sizeCache, hash);
+ stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -162,11 +162,11 @@ public class MySqlLexer extends Lexer {
int hash = first;
- offsetCache = curIndex;
- sizeCache = 1;
+ np = bp;
+ sp = 1;
char ch;
for (;;) {
- ch = sql[++curIndex];
+ ch = buf[++bp];
if (!isIdentifierChar(ch)) {
break;
@@ -174,13 +174,13 @@ public class MySqlLexer extends Lexer {
hash = 31 * hash + ch;
- sizeCache++;
+ sp++;
continue;
}
- this.ch = sql[curIndex];
+ this.ch = buf[bp];
- stringVal = symbolTable.addSymbol(sql, offsetCache, sizeCache, hash);
+ stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -191,21 +191,21 @@ public class MySqlLexer extends Lexer {
}
protected void scanString() {
- offsetCache = curIndex;
+ np = bp;
boolean hasSpecial = false;
for (;;) {
- if (curIndex >= sqlLength) {
+ if (bp >= buflen) {
lexError(tokenPos, "unclosed.str.lit");
return;
}
- ch = sql[++curIndex];
+ ch = buf[++bp];
if (ch == '\\') {
scanChar();
if (!hasSpecial) {
- System.arraycopy(sql, offsetCache + 1, sbuf, 0, sizeCache);
+ System.arraycopy(buf, np + 1, sbuf, 0, sp);
hasSpecial = true;
}
@@ -250,7 +250,7 @@ public class MySqlLexer extends Lexer {
token = LITERAL_CHARS;
break;
} else {
- System.arraycopy(sql, offsetCache + 1, sbuf, 0, sizeCache);
+ System.arraycopy(buf, np + 1, sbuf, 0, sp);
hasSpecial = true;
putChar('\'');
continue;
@@ -258,21 +258,21 @@ public class MySqlLexer extends Lexer {
}
if (!hasSpecial) {
- sizeCache++;
+ sp++;
continue;
}
- if (sizeCache == sbuf.length) {
+ if (sp == sbuf.length) {
putChar(ch);
} else {
- sbuf[sizeCache++] = ch;
+ sbuf[sp++] = ch;
}
}
if (!hasSpecial) {
- stringVal = new String(sql, offsetCache + 1, sizeCache);
+ stringVal = new String(buf, np + 1, sp);
} else {
- stringVal = new String(sbuf, 0, sizeCache);
+ stringVal = new String(sbuf, 0, sp);
}
}
}
diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java
index 6295aac2a2..9a5407edc8 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java
@@ -61,7 +61,7 @@ public class Lexer {
protected int sp;
/** string point as offset */
- protected int offsetCache;
+ protected int np;
protected SymbolTable symbolTable = new SymbolTable();
@@ -386,7 +386,7 @@ public class Lexer {
}
protected void scanString() {
- offsetCache = bp;
+ np = bp;
boolean hasSpecial = false;
for (;;) {
@@ -403,7 +403,7 @@ public class Lexer {
token = LITERAL_CHARS;
break;
} else {
- System.arraycopy(buf, offsetCache + 1, sbuf, 0, sp);
+ System.arraycopy(buf, np + 1, sbuf, 0, sp);
hasSpecial = true;
putChar('\'');
continue;
@@ -423,7 +423,7 @@ public class Lexer {
}
if (!hasSpecial) {
- stringVal = new String(buf, offsetCache + 1, sp);
+ stringVal = new String(buf, np + 1, sp);
} else {
stringVal = new String(sbuf, 0, sp);
}
@@ -461,7 +461,7 @@ public class Lexer {
int hash = first;
- offsetCache = bp;
+ np = bp;
sp = 1;
char ch;
for (;;) {
@@ -479,7 +479,7 @@ public class Lexer {
this.ch = buf[bp];
- stringVal = symbolTable.addSymbol(buf, offsetCache, sp, hash);
+ stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -489,7 +489,64 @@ public class Lexer {
}
public void scanComment() {
+ if (ch != '/') {
+ throw new IllegalStateException();
+ }
+ np = bp;
+ sp = 0;
+ scanChar();
+ sp++;
+
+ if (ch == '*') {
+ scanChar();
+ sp++;
+
+ for (;;) {
+ if (ch == '*' && buf[bp + 1] == '/') {
+ sp += 2;
+ scanChar();
+ break;
+ }
+
+ scanChar();
+ sp++;
+ }
+
+ stringVal = new String(buf, np, sp);
+ token = Token.MULTI_LINE_COMMENT;
+ return;
+ }
+
+ if (ch == '/') {
+ scanChar();
+ sp++;
+
+ for (;;) {
+ if (ch == '\r') {
+ if (buf[bp + 1] == '\n') {
+ sp += 2;
+ scanChar();
+ break;
+ }
+ sp++;
+ break;
+ }
+
+ if (ch == '\r') {
+ scanChar();
+ sp++;
+ break;
+ }
+
+ scanChar();
+ sp++;
+ }
+
+ stringVal = new String(buf, np + 1, sp);
+ token = Token.LINE_COMMENT;
+ return;
+ }
}
public void scanIdentifier() {
@@ -502,7 +559,7 @@ public class Lexer {
int hash = first;
- offsetCache = bp;
+ np = bp;
sp = 1;
char ch;
for (;;) {
@@ -520,7 +577,7 @@ public class Lexer {
this.ch = buf[bp];
- stringVal = symbolTable.addSymbol(buf, offsetCache, sp, hash);
+ stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -530,7 +587,7 @@ public class Lexer {
}
public void scanNumber() {
- offsetCache = bp;
+ np = bp;
if (ch == '-') {
sp++;
@@ -585,14 +642,14 @@ public class Lexer {
}
if (isDouble) {
- token = Token.LITERAL_NUM_MIX_DIGIT;
+ token = Token.LITERAL_FLOAT;
} else {
- token = Token.LITERAL_NUM_PURE_DIGIT;
+ token = Token.LITERAL_INT;
}
}
public void scanHexaDecimal() {
- offsetCache = bp;
+ np = bp;
if (ch == '-') {
sp++;
@@ -612,7 +669,7 @@ public class Lexer {
}
public String hexString() throws NumberFormatException {
- return new String(buf, offsetCache, sp);
+ return new String(buf, np, sp);
}
public final boolean isDigit(char ch) {
@@ -682,12 +739,12 @@ public class Lexer {
public Number integerValue() throws NumberFormatException {
long result = 0;
boolean negative = false;
- int i = offsetCache, max = offsetCache + sp;
+ int i = np, max = np + sp;
long limit;
long multmin;
int digit;
- if (buf[offsetCache] == '-') {
+ if (buf[np] == '-') {
negative = true;
limit = Long.MIN_VALUE;
i++;
@@ -713,7 +770,7 @@ public class Lexer {
}
if (negative) {
- if (i > offsetCache + 1) {
+ if (i > np + 1) {
if (result >= Integer.MIN_VALUE) {
return (int) result;
}
@@ -731,10 +788,10 @@ public class Lexer {
}
public final String numberString() {
- return new String(buf, offsetCache, sp);
+ return new String(buf, np, sp);
}
public BigDecimal decimalValue() {
- return new BigDecimal(buf, offsetCache, sp);
+ return new BigDecimal(buf, np, sp);
}
}
diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
index 749c1aeba2..45cc0f9823 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
@@ -163,11 +163,11 @@ public class SQLExprParser extends SQLParser {
break;
case NEW:
throw new ParserException("TODO");
- case LITERAL_NUM_PURE_DIGIT:
+ case LITERAL_INT:
sqlExpr = new SQLIntegerExpr(lexer.integerValue());
lexer.nextToken();
break;
- case LITERAL_NUM_MIX_DIGIT:
+ case LITERAL_FLOAT:
sqlExpr = new SQLNumberExpr(lexer.decimalValue());
lexer.nextToken();
break;
@@ -261,7 +261,7 @@ public class SQLExprParser extends SQLParser {
case SUB:
lexer.nextToken();
switch (lexer.token()) {
- case LITERAL_NUM_PURE_DIGIT:
+ case LITERAL_INT:
Number integerValue = lexer.integerValue();
if (integerValue instanceof Integer) {
int intVal = ((Integer) integerValue).intValue();
@@ -283,7 +283,7 @@ public class SQLExprParser extends SQLParser {
sqlExpr = new SQLIntegerExpr(integerValue);
lexer.nextToken();
break;
- case LITERAL_NUM_MIX_DIGIT:
+ case LITERAL_FLOAT:
sqlExpr = new SQLNumberExpr(lexer.decimalValue().negate());
lexer.nextToken();
break;
@@ -294,11 +294,11 @@ public class SQLExprParser extends SQLParser {
case PLUS:
lexer.nextToken();
switch (lexer.token()) {
- case LITERAL_NUM_PURE_DIGIT:
+ case LITERAL_INT:
sqlExpr = new SQLIntegerExpr(lexer.integerValue());
lexer.nextToken();
break;
- case LITERAL_NUM_MIX_DIGIT:
+ case LITERAL_FLOAT:
sqlExpr = new SQLNumberExpr(lexer.decimalValue());
lexer.nextToken();
break;
diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java b/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java
index 3de580d4f3..825dc15430 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java
@@ -503,7 +503,7 @@ public class SQLLexer {
scanIdentifierFromNumber(offsetCache, sizeCache);
return;
} else {
- token = Token.LITERAL_NUM_PURE_DIGIT;
+ token = Token.LITERAL_INT;
return;
}
break;
@@ -522,7 +522,7 @@ public class SQLLexer {
throw err("invalid char after '.': " + ch);
}
} else {
- token = Token.LITERAL_NUM_MIX_DIGIT;
+ token = Token.LITERAL_FLOAT;
return;
}
break;
@@ -540,7 +540,7 @@ public class SQLLexer {
throw err("invalid char after '.' for as part of number: " + ch);
}
} else {
- token = Token.LITERAL_NUM_MIX_DIGIT;
+ token = Token.LITERAL_FLOAT;
return;
}
break;
@@ -607,12 +607,12 @@ public class SQLLexer {
throw err("expect digit char after SIGN for 'e': " + ch);
}
} else {
- token = Token.LITERAL_NUM_MIX_DIGIT;
+ token = Token.LITERAL_FLOAT;
}
return;
}
}
- token = state == 0 ? Token.LITERAL_NUM_PURE_DIGIT : Token.LITERAL_NUM_MIX_DIGIT;
+ token = state == 0 ? Token.LITERAL_INT : Token.LITERAL_FLOAT;
}
/**
diff --git a/src/main/java/com/alibaba/druid/sql/parser/Token.java b/src/main/java/com/alibaba/druid/sql/parser/Token.java
index e4dfb4655a..ed1c29b2ed 100644
--- a/src/main/java/com/alibaba/druid/sql/parser/Token.java
+++ b/src/main/java/com/alibaba/druid/sql/parser/Token.java
@@ -15,9 +15,28 @@
*/
package com.alibaba.druid.sql.parser;
+/**
+ *
+ * SQL Token
+ * @author wenshao 2011-5-18 下午05:16:49
+ * @formatter:off
+ */
public enum Token {
- FOR("FOR"), INDEX("INDEX"), PRIMARY("PRIMARY"), KEY("KEY"), DEFAULT("DEFAULT"), CONSTRAINT("CONSTRAINT"), CHECK("CHECK"), VIEW("VIEW"), CREATE("CREATE"),
- ALTER("ALTER"), DROP("DROP"), TABLE("TABLE"), UPDATE("UPDATE"), SET("SET"), SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER("ORDER"), BY("BY"),
+ FOR("FOR"),
+ INDEX("INDEX"),
+ PRIMARY("PRIMARY"),
+ KEY("KEY"),
+ DEFAULT("DEFAULT"),
+ CONSTRAINT("CONSTRAINT"),
+ CHECK("CHECK"),
+ VIEW("VIEW"),
+ CREATE("CREATE"),
+ ALTER("ALTER"),
+ DROP("DROP"),
+ TABLE("TABLE"),
+ UPDATE("UPDATE"),
+ SET("SET"),
+ SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER("ORDER"), BY("BY"),
GROUP("GROUP"), HAVING("HAVING"), INSERT("INSERT"), INTO("INTO"), NULL("NULL"), NOT("NOT"), AS("AS"), DELETE("DELETE"), DISTINCT("DISTINCT"),
UNIQUE("UNIQUE"), FOREIGN("FOREIGN"), REFERENCE("REFERENCE"), REFERENCES("REFERENCES"), ALL("ALL"), UNION("UNION"), INTERSECT("INTERSECT"), MINUS("MINUS"),
INNER("INNER"), LEFT("LEFT"), RIGHT("RIGHT"), FULL("FULL"), OUTER("OUTER"), JOIN("JOIN"), ON("ON"), SCHEMA("SCHEMA"), CAST("CAST"),
@@ -35,10 +54,8 @@ public enum Token {
HINT,
// QS_TODO add support in Lexer
SYS_VAR, USR_VAR,
- /** number composed purely of digit */
- LITERAL_NUM_PURE_DIGIT,
- /** number composed of digit mixed with . or e */
- LITERAL_NUM_MIX_DIGIT,
+ LITERAL_INT,
+ LITERAL_FLOAT,
LITERAL_HEX,
// QS_TODO add syntax support
LITERAL_BIT,
@@ -46,6 +63,8 @@ public enum Token {
LITERAL_NCHARS,
// QS_TODO remove alias token
LITERAL_ALIAS,
+ LINE_COMMENT,
+ MULTI_LINE_COMMENT,
LPAREN("("), RPAREN(")"), LBRACE("{"), RBRACE("}"), LBRACKET("["), RBRACKET("]"), SEMI(";"), COMMA(","), DOT("."), EQ("="), GT(">"), LT("<"), BANG("!"),
TILDE("~"), QUES("?"), COLON(":"), COLONEQ(":="), EQEQ("=="), LTEQ("<="), LTEQGT("<=>"), LTGT("<>"), GTEQ(">="), BANGEQ("!="), BANGGT("!>"), BANGLT("!<"),
diff --git a/src/test/java/com/alibaba/druid/sql/parser/CommentTest.java b/src/test/java/com/alibaba/druid/sql/parser/CommentTest.java
new file mode 100644
index 0000000000..cf674f4390
--- /dev/null
+++ b/src/test/java/com/alibaba/druid/sql/parser/CommentTest.java
@@ -0,0 +1,49 @@
+package com.alibaba.druid.sql.parser;
+
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.dialect.oracle.ast.visitor.OracleOutputVisitor;
+import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
+
+public class CommentTest extends TestCase {
+ public void test_0 () throws Exception {
+ String sql = "SELECT /*mark for picman*/ * FROM WP_ALBUM WHERE MEMBER_ID = ? AND ID IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+
+ Lexer lexer = new Lexer(sql);
+ for (;;) {
+ lexer.nextToken();
+ Token tok = lexer.token();
+
+ if (tok == Token.IDENTIFIER) {
+ System.out.println(tok.name() + "\t\t" + lexer.stringVal);
+ } else if (tok == Token.MULTI_LINE_COMMENT) {
+ System.out.println(tok.name() + "\t\t" + lexer.stringVal);
+ } else {
+ System.out.println(tok.name() + "\t\t\t" + tok.name);
+ }
+
+ if (tok == Token.EOF) {
+ break;
+ }
+ }
+ }
+
+ public void f_test_1 () throws Exception {
+ String sql = "SELECT /*mark for picman*/ * FROM WP_ALBUM WHERE MEMBER_ID = ? AND ID IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+
+ StringBuilder out = new StringBuilder();
+ OracleOutputVisitor visitor = new OracleOutputVisitor(out);
+ OracleStatementParser parser = new OracleStatementParser(sql);
+ List statementList = parser.parseStatementList();
+ for (SQLStatement statement : statementList) {
+ statement.accept(visitor);
+ visitor.print(";");
+ visitor.println();
+ }
+
+ System.out.println(out.toString());
+ }
+}