Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

支持Comment的Parse

Browse Source 
git-svn-id: http://code.alibabatech.com/svn/druid/trunk@117 b9813039-fb51-4c41-a8b9-e21c2acb5095
This commit is contained in:
wenshao
2011-05-18 09:39:29 +00:00
parent 72df0be9e3
commit 43f2b023e8
6 changed files with 184 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java
+24 -24
View File
@@ -124,15 +124,15 @@ public class MySqlLexer extends Lexer {
if (ch == '`') {
int hash = first;
offsetCache = curIndex;
sizeCache = 1;
np = bp;
sp = 1;
char ch;
for (;;) {
ch = sql[++curIndex];
ch = buf[++bp];
if (ch == '`') {
sizeCache++;
ch = sql[++curIndex];
sp++;
ch = buf[++bp];
break;
} else if (ch == EOI) {
throw new SQLParseException("illegal identifier");
@@ -140,13 +140,13 @@ public class MySqlLexer extends Lexer {
hash = 31 * hash + ch;
sizeCache++;
sp++;
continue;
}
this.ch = sql[curIndex];
this.ch = buf[bp];
stringVal = symbolTable.addSymbol(sql, offsetCache, sizeCache, hash);
stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -162,11 +162,11 @@ public class MySqlLexer extends Lexer {
int hash = first;
offsetCache = curIndex;
sizeCache = 1;
np = bp;
sp = 1;
char ch;
for (;;) {
ch = sql[++curIndex];
ch = buf[++bp];
if (!isIdentifierChar(ch)) {
break;
@@ -174,13 +174,13 @@ public class MySqlLexer extends Lexer {
hash = 31 * hash + ch;
sizeCache++;
sp++;
continue;
}
this.ch = sql[curIndex];
this.ch = buf[bp];
stringVal = symbolTable.addSymbol(sql, offsetCache, sizeCache, hash);
stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -191,21 +191,21 @@ public class MySqlLexer extends Lexer {
}
protected void scanString() {
offsetCache = curIndex;
np = bp;
boolean hasSpecial = false;
for (;;) {
if (curIndex >= sqlLength) {
if (bp >= buflen) {
lexError(tokenPos, "unclosed.str.lit");
return;
}
ch = sql[++curIndex];
ch = buf[++bp];
if (ch == '\\') {
scanChar();
if (!hasSpecial) {
System.arraycopy(sql, offsetCache + 1, sbuf, 0, sizeCache);
System.arraycopy(buf, np + 1, sbuf, 0, sp);
hasSpecial = true;
}
@@ -250,7 +250,7 @@ public class MySqlLexer extends Lexer {
token = LITERAL_CHARS;
break;
} else {
System.arraycopy(sql, offsetCache + 1, sbuf, 0, sizeCache);
System.arraycopy(buf, np + 1, sbuf, 0, sp);
hasSpecial = true;
putChar('\'');
continue;
@@ -258,21 +258,21 @@ public class MySqlLexer extends Lexer {
}
if (!hasSpecial) {
sizeCache++;
sp++;
continue;
}
if (sizeCache == sbuf.length) {
if (sp == sbuf.length) {
putChar(ch);
} else {
sbuf[sizeCache++] = ch;
sbuf[sp++] = ch;
}
}
if (!hasSpecial) {
stringVal = new String(sql, offsetCache + 1, sizeCache);
stringVal = new String(buf, np + 1, sp);
} else {
stringVal = new String(sbuf, 0, sizeCache);
stringVal = new String(sbuf, 0, sp);
}
}
}
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+75 -18
View File
@@ -61,7 +61,7 @@ public class Lexer {
protected int sp;
/** string point as offset */
protected int offsetCache;
protected int np;
protected SymbolTable symbolTable = new SymbolTable();
@@ -386,7 +386,7 @@ public class Lexer {
}
protected void scanString() {
offsetCache = bp;
np = bp;
boolean hasSpecial = false;
for (;;) {
@@ -403,7 +403,7 @@ public class Lexer {
token = LITERAL_CHARS;
break;
} else {
System.arraycopy(buf, offsetCache + 1, sbuf, 0, sp);
System.arraycopy(buf, np + 1, sbuf, 0, sp);
hasSpecial = true;
putChar('\'');
continue;
@@ -423,7 +423,7 @@ public class Lexer {
}
if (!hasSpecial) {
stringVal = new String(buf, offsetCache + 1, sp);
stringVal = new String(buf, np + 1, sp);
} else {
stringVal = new String(sbuf, 0, sp);
}
@@ -461,7 +461,7 @@ public class Lexer {
int hash = first;
offsetCache = bp;
np = bp;
sp = 1;
char ch;
for (;;) {
@@ -479,7 +479,7 @@ public class Lexer {
this.ch = buf[bp];
stringVal = symbolTable.addSymbol(buf, offsetCache, sp, hash);
stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -489,7 +489,64 @@ public class Lexer {
}
public void scanComment() {
if (ch != '/') {
throw new IllegalStateException();
}
np = bp;
sp = 0;
scanChar();
sp++;
if (ch == '*') {
scanChar();
sp++;
for (;;) {
if (ch == '*' && buf[bp + 1] == '/') {
sp += 2;
scanChar();
break;
}
scanChar();
sp++;
}
stringVal = new String(buf, np, sp);
token = Token.MULTI_LINE_COMMENT;
return;
}
if (ch == '/') {
scanChar();
sp++;
for (;;) {
if (ch == '\r') {
if (buf[bp + 1] == '\n') {
sp += 2;
scanChar();
break;
}
sp++;
break;
}
if (ch == '\r') {
scanChar();
sp++;
break;
}
scanChar();
sp++;
}
stringVal = new String(buf, np + 1, sp);
token = Token.LINE_COMMENT;
return;
}
}
public void scanIdentifier() {
@@ -502,7 +559,7 @@ public class Lexer {
int hash = first;
offsetCache = bp;
np = bp;
sp = 1;
char ch;
for (;;) {
@@ -520,7 +577,7 @@ public class Lexer {
this.ch = buf[bp];
stringVal = symbolTable.addSymbol(buf, offsetCache, sp, hash);
stringVal = symbolTable.addSymbol(buf, np, sp, hash);
Token tok = keywods.getKeyword(stringVal);
if (tok != null) {
token = tok;
@@ -530,7 +587,7 @@ public class Lexer {
}
public void scanNumber() {
offsetCache = bp;
np = bp;
if (ch == '-') {
sp++;
@@ -585,14 +642,14 @@ public class Lexer {
}
if (isDouble) {
token = Token.LITERAL_NUM_MIX_DIGIT;
token = Token.LITERAL_FLOAT;
} else {
token = Token.LITERAL_NUM_PURE_DIGIT;
token = Token.LITERAL_INT;
}
}
public void scanHexaDecimal() {
offsetCache = bp;
np = bp;
if (ch == '-') {
sp++;
@@ -612,7 +669,7 @@ public class Lexer {
}
public String hexString() throws NumberFormatException {
return new String(buf, offsetCache, sp);
return new String(buf, np, sp);
}
public final boolean isDigit(char ch) {
@@ -682,12 +739,12 @@ public class Lexer {
public Number integerValue() throws NumberFormatException {
long result = 0;
boolean negative = false;
int i = offsetCache, max = offsetCache + sp;
int i = np, max = np + sp;
long limit;
long multmin;
int digit;
if (buf[offsetCache] == '-') {
if (buf[np] == '-') {
negative = true;
limit = Long.MIN_VALUE;
i++;
@@ -713,7 +770,7 @@ public class Lexer {
}
if (negative) {
if (i > offsetCache + 1) {
if (i > np + 1) {
if (result >= Integer.MIN_VALUE) {
return (int) result;
}
@@ -731,10 +788,10 @@ public class Lexer {
}
public final String numberString() {
return new String(buf, offsetCache, sp);
return new String(buf, np, sp);
}
public BigDecimal decimalValue() {
return new BigDecimal(buf, offsetCache, sp);
return new BigDecimal(buf, np, sp);
}
}
src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+6 -6
View File
@@ -163,11 +163,11 @@ public class SQLExprParser extends SQLParser {
break;
case NEW:
throw new ParserException("TODO");
case LITERAL_NUM_PURE_DIGIT:
case LITERAL_INT:
sqlExpr = new SQLIntegerExpr(lexer.integerValue());
lexer.nextToken();
break;
case LITERAL_NUM_MIX_DIGIT:
case LITERAL_FLOAT:
sqlExpr = new SQLNumberExpr(lexer.decimalValue());
lexer.nextToken();
break;
@@ -261,7 +261,7 @@ public class SQLExprParser extends SQLParser {
case SUB:
lexer.nextToken();
switch (lexer.token()) {
case LITERAL_NUM_PURE_DIGIT:
case LITERAL_INT:
Number integerValue = lexer.integerValue();
if (integerValue instanceof Integer) {
int intVal = ((Integer) integerValue).intValue();
@@ -283,7 +283,7 @@ public class SQLExprParser extends SQLParser {
sqlExpr = new SQLIntegerExpr(integerValue);
lexer.nextToken();
break;
case LITERAL_NUM_MIX_DIGIT:
case LITERAL_FLOAT:
sqlExpr = new SQLNumberExpr(lexer.decimalValue().negate());
lexer.nextToken();
break;
@@ -294,11 +294,11 @@ public class SQLExprParser extends SQLParser {
case PLUS:
lexer.nextToken();
switch (lexer.token()) {
case LITERAL_NUM_PURE_DIGIT:
case LITERAL_INT:
sqlExpr = new SQLIntegerExpr(lexer.integerValue());
lexer.nextToken();
break;
case LITERAL_NUM_MIX_DIGIT:
case LITERAL_FLOAT:
sqlExpr = new SQLNumberExpr(lexer.decimalValue());
lexer.nextToken();
break;
src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java
+5 -5
View File
@@ -503,7 +503,7 @@ public class SQLLexer {
scanIdentifierFromNumber(offsetCache, sizeCache);
return;
} else {
token = Token.LITERAL_NUM_PURE_DIGIT;
token = Token.LITERAL_INT;
return;
}
break;
@@ -522,7 +522,7 @@ public class SQLLexer {
throw err("invalid char after '.': " + ch);
}
} else {
token = Token.LITERAL_NUM_MIX_DIGIT;
token = Token.LITERAL_FLOAT;
return;
}
break;
@@ -540,7 +540,7 @@ public class SQLLexer {
throw err("invalid char after '.' for as part of number: " + ch);
}
} else {
token = Token.LITERAL_NUM_MIX_DIGIT;
token = Token.LITERAL_FLOAT;
return;
}
break;
@@ -607,12 +607,12 @@ public class SQLLexer {
throw err("expect digit char after SIGN for 'e': " + ch);
}
} else {
token = Token.LITERAL_NUM_MIX_DIGIT;
token = Token.LITERAL_FLOAT;
}
return;
}
}
token = state == 0 ? Token.LITERAL_NUM_PURE_DIGIT : Token.LITERAL_NUM_MIX_DIGIT;
token = state == 0 ? Token.LITERAL_INT : Token.LITERAL_FLOAT;
}
/**
src/main/java/com/alibaba/druid/sql/parser/Token.java
+25 -6
View File
@@ -15,9 +15,28 @@
*/
package com.alibaba.druid.sql.parser;
/**
*
* SQL Token
* @author wenshao 2011-5-18 下午05:16:49
* @formatter:off
*/
public enum Token {
FOR("FOR"), INDEX("INDEX"), PRIMARY("PRIMARY"), KEY("KEY"), DEFAULT("DEFAULT"), CONSTRAINT("CONSTRAINT"), CHECK("CHECK"), VIEW("VIEW"), CREATE("CREATE"),
ALTER("ALTER"), DROP("DROP"), TABLE("TABLE"), UPDATE("UPDATE"), SET("SET"), SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER("ORDER"), BY("BY"),
FOR("FOR"),
INDEX("INDEX"),
PRIMARY("PRIMARY"),
KEY("KEY"),
DEFAULT("DEFAULT"),
CONSTRAINT("CONSTRAINT"),
CHECK("CHECK"),
VIEW("VIEW"),
CREATE("CREATE"),
ALTER("ALTER"),
DROP("DROP"),
TABLE("TABLE"),
UPDATE("UPDATE"),
SET("SET"),
SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER("ORDER"), BY("BY"),
GROUP("GROUP"), HAVING("HAVING"), INSERT("INSERT"), INTO("INTO"), NULL("NULL"), NOT("NOT"), AS("AS"), DELETE("DELETE"), DISTINCT("DISTINCT"),
UNIQUE("UNIQUE"), FOREIGN("FOREIGN"), REFERENCE("REFERENCE"), REFERENCES("REFERENCES"), ALL("ALL"), UNION("UNION"), INTERSECT("INTERSECT"), MINUS("MINUS"),
INNER("INNER"), LEFT("LEFT"), RIGHT("RIGHT"), FULL("FULL"), OUTER("OUTER"), JOIN("JOIN"), ON("ON"), SCHEMA("SCHEMA"), CAST("CAST"),
@@ -35,10 +54,8 @@ public enum Token {
HINT,
// QS_TODO add support in Lexer
SYS_VAR, USR_VAR,
/** number composed purely of digit */
LITERAL_NUM_PURE_DIGIT,
/** number composed of digit mixed with <code>.</code> or <code>e</code> */
LITERAL_NUM_MIX_DIGIT,
LITERAL_INT,
LITERAL_FLOAT,
LITERAL_HEX,
// QS_TODO add syntax support
LITERAL_BIT,
@@ -46,6 +63,8 @@ public enum Token {
LITERAL_NCHARS,
// QS_TODO remove alias token
LITERAL_ALIAS,
LINE_COMMENT,
MULTI_LINE_COMMENT,
LPAREN("("), RPAREN(")"), LBRACE("{"), RBRACE("}"), LBRACKET("["), RBRACKET("]"), SEMI(";"), COMMA(","), DOT("."), EQ("="), GT(">"), LT("<"), BANG("!"),
TILDE("~"), QUES("?"), COLON(":"), COLONEQ(":="), EQEQ("=="), LTEQ("<="), LTEQGT("<=>"), LTGT("<>"), GTEQ(">="), BANGEQ("!="), BANGGT("!>"), BANGLT("!<"),