Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

refactor Lexer

Browse Source
This commit is contained in:
wenshao
2012-11-10 18:55:31 +08:00
parent ccb314e3b8
commit 799fe95703
6 changed files with 266 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+113 -132
View File
@@ -41,29 +41,14 @@ import java.math.BigInteger;
public class Lexer {
private final String text;
protected int bp;
// QS_TODO what is the purpose?
protected int eofPos;
protected int pos;
protected int mark;
/** The current character. */
protected char ch;
/** The token's position, 0-based offset from beginning of text. */
protected int tokenPos;
protected char[] buf;
protected int bufPos;
protected char[] sbuf;
/** string point as size */
protected int sp;
/** string point as offset */
protected int np;
protected SymbolTable symbolTable = new SymbolTable();
/**
* The token, set by nextToken().
*/
protected Token token;
protected Keywords keywods = Keywords.DEFAULT_KEYWORDS;
@@ -94,7 +79,7 @@ public class Lexer {
}
public String addSymbol(int hash) {
return subString(np, sp);
return subString(mark, bufPos);
}
public final String subString(int offset, int count) {
@@ -118,7 +103,6 @@ public class Lexer {
}
private static class SavePoint {
int bp;
int sp;
int np;
@@ -132,18 +116,18 @@ public class Lexer {
public void mark() {
SavePoint savePoint = new SavePoint();
savePoint.bp = bp;
savePoint.sp = sp;
savePoint.np = np;
savePoint.bp = pos;
savePoint.sp = bufPos;
savePoint.np = mark;
savePoint.ch = ch;
savePoint.token = token;
this.savePoint = savePoint;
}
public void reset() {
this.bp = savePoint.bp;
this.sp = savePoint.sp;
this.np = savePoint.np;
this.pos = savePoint.bp;
this.bufPos = savePoint.sp;
this.mark = savePoint.np;
this.ch = savePoint.ch;
this.token = savePoint.token;
}
@@ -152,7 +136,7 @@ public class Lexer {
this.skipComment = skipComment;
this.text = input;
this.bp = -1;
this.pos = -1;
scanChar();
}
@@ -162,21 +146,21 @@ public class Lexer {
}
protected final void scanChar() {
ch = charAt(++bp);
ch = charAt(++pos);
}
protected void unscan() {
ch = charAt(--bp);
ch = charAt(--pos);
}
public boolean isEOF() {
return bp >= text.length();
return pos >= text.length();
}
/**
* Report an error at the given position using the provided arguments.
*/
protected void lexError(int pos, String key, Object... args) {
protected void lexError(String key, Object... args) {
token = ERROR;
}
@@ -192,25 +176,23 @@ public class Lexer {
}
public final void nextToken() {
sp = 0;
bufPos = 0;
for (;;) {
tokenPos = bp;
if (isWhitespace(ch)) {
scanChar();
continue;
}
if (ch == '$' && charAt(bp + 1) == '{') {
if (ch == '$' && charAt(pos + 1) == '{') {
scanVariable();
return;
}
if (isFirstIdentifierChar(ch)) {
if (ch == 'N') {
if (charAt(bp + 1) == '\'') {
++bp;
if (charAt(pos + 1) == '\'') {
++pos;
ch = '\'';
scanString();
token = Token.LITERAL_NCHARS;
@@ -224,7 +206,7 @@ public class Lexer {
switch (ch) {
case '0':
if (charAt(bp + 1) == 'x') {
if (charAt(pos + 1) == 'x') {
scanChar();
scanChar();
scanHexaDecimal();
@@ -331,11 +313,11 @@ public class Lexer {
scanVariable();
return;
case '-':
int subNextChar = charAt(bp + 1);
int subNextChar = charAt(pos + 1);
if (subNextChar == '-') {
scanComment();
if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) {
sp = 0;
bufPos = 0;
continue;
}
} else {
@@ -343,11 +325,11 @@ public class Lexer {
}
return;
case '/':
int nextChar = charAt(bp + 1);
int nextChar = charAt(pos + 1);
if (nextChar == '/' || nextChar == '*') {
scanComment();
if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) {
sp = 0;
bufPos = 0;
continue;
}
} else {
@@ -369,9 +351,8 @@ public class Lexer {
// QS_TODO ?
if (isEOF()) { // JLS
token = EOF;
tokenPos = bp = eofPos;
} else {
lexError(tokenPos, "illegal.char", String.valueOf((int) ch));
lexError("illegal.char", String.valueOf((int) ch));
scanChar();
}
@@ -495,16 +476,16 @@ public class Lexer {
}
protected void scanString() {
np = bp;
mark = pos;
boolean hasSpecial = false;
for (;;) {
if (isEOF()) {
lexError(tokenPos, "unclosed.str.lit");
lexError("unclosed.str.lit");
return;
}
ch = charAt(++bp);
ch = charAt(++pos);
if (ch == '\'') {
scanChar();
@@ -513,10 +494,10 @@ public class Lexer {
break;
} else {
if (!hasSpecial) {
if (sbuf == null) {
sbuf = new char[32];
if (buf == null) {
buf = new char[32];
}
arraycopy(np + 1, sbuf, 0, sp);
arraycopy(mark + 1, buf, 0, bufPos);
hasSpecial = true;
}
putChar('\'');
@@ -525,38 +506,38 @@ public class Lexer {
}
if (!hasSpecial) {
sp++;
bufPos++;
continue;
}
if (sp == sbuf.length) {
if (bufPos == buf.length) {
putChar(ch);
} else {
sbuf[sp++] = ch;
buf[bufPos++] = ch;
}
}
if (!hasSpecial) {
stringVal = subString(np + 1, sp);
stringVal = subString(mark + 1, bufPos);
} else {
stringVal = new String(sbuf, 0, sp);
stringVal = new String(buf, 0, bufPos);
}
}
private final void scanAlias() {
np = bp;
mark = pos;
if (sbuf == null) {
sbuf = new char[32];
if (buf == null) {
buf = new char[32];
}
for (;;) {
if (isEOF()) {
lexError(tokenPos, "unclosed.str.lit");
lexError("unclosed.str.lit");
return;
}
ch = charAt(++bp);
ch = charAt(++pos);
if (ch == '\"') {
scanChar();
@@ -564,14 +545,14 @@ public class Lexer {
break;
}
if (sp == sbuf.length) {
if (bufPos == buf.length) {
putChar(ch);
} else {
sbuf[sp++] = ch;
buf[bufPos++] = ch;
}
}
stringVal = subString(np + 1, sp);
stringVal = subString(mark + 1, bufPos);
}
public void scanVariable() {
@@ -583,25 +564,25 @@ public class Lexer {
int hash = first;
np = bp;
sp = 1;
mark = pos;
bufPos = 1;
char ch;
boolean mybatisFlag = false;
if (charAt(bp + 1) == '@') {
ch = charAt(++bp);
if (charAt(pos + 1) == '@') {
ch = charAt(++pos);
hash = 31 * hash + ch;
sp++;
} else if (charAt(bp + 1) == '{') {
bufPos++;
} else if (charAt(pos + 1) == '{') {
hash = 31 * hash + '"';
bp++;
sp++;
pos++;
bufPos++;
mybatisFlag = true;
}
for (;;) {
ch = charAt(++bp);
ch = charAt(++pos);
if (!isIdentifierChar(ch)) {
break;
@@ -609,7 +590,7 @@ public class Lexer {
hash = 31 * hash + ch;
sp++;
bufPos++;
continue;
}
@@ -618,11 +599,11 @@ public class Lexer {
throw new SQLParseException("syntax error");
}
hash = 31 * hash + '"';
++bp;
sp++;
++pos;
bufPos++;
}
this.ch = charAt(bp);
this.ch = charAt(pos);
stringVal = addSymbol(hash);
token = Token.VARIANT;
@@ -637,57 +618,57 @@ public class Lexer {
throw new IllegalStateException();
}
np = bp;
sp = 0;
mark = pos;
bufPos = 0;
scanChar();
if (ch == '*') {
scanChar();
sp++;
bufPos++;
for (;;) {
if (ch == '*' && charAt(bp + 1) == '/') {
sp += 2;
if (ch == '*' && charAt(pos + 1) == '/') {
bufPos += 2;
scanChar();
scanChar();
break;
}
scanChar();
sp++;
bufPos++;
}
stringVal = subString(np, sp);
stringVal = subString(mark, bufPos);
token = Token.MULTI_LINE_COMMENT;
return;
}
if (ch == '/') {
scanChar();
sp++;
bufPos++;
for (;;) {
if (ch == '\r') {
if (charAt(bp + 1) == '\n') {
sp += 2;
if (charAt(pos + 1) == '\n') {
bufPos += 2;
scanChar();
break;
}
sp++;
bufPos++;
break;
}
if (ch == '\n') {
scanChar();
sp++;
bufPos++;
break;
}
scanChar();
sp++;
bufPos++;
}
stringVal = subString(np + 1, sp);
stringVal = subString(mark + 1, bufPos);
token = Token.LINE_COMMENT;
return;
}
@@ -703,11 +684,11 @@ public class Lexer {
int hash = first;
np = bp;
sp = 1;
mark = pos;
bufPos = 1;
char ch;
for (;;) {
ch = charAt(++bp);
ch = charAt(++pos);
if (!isIdentifierChar(ch)) {
break;
@@ -715,11 +696,11 @@ public class Lexer {
hash = 31 * hash + ch;
sp++;
bufPos++;
continue;
}
this.ch = charAt(bp);
this.ch = charAt(pos);
stringVal = addSymbol(hash);
Token tok = keywods.getKeyword(stringVal);
@@ -731,59 +712,59 @@ public class Lexer {
}
public void scanNumber() {
np = bp;
mark = pos;
if (ch == '-') {
sp++;
ch = charAt(++bp);
bufPos++;
ch = charAt(++pos);
}
for (;;) {
if (ch >= '0' && ch <= '9') {
sp++;
bufPos++;
} else {
break;
}
ch = charAt(++bp);
ch = charAt(++pos);
}
boolean isDouble = false;
if (ch == '.') {
if (charAt(bp + 1) == '.') {
if (charAt(pos + 1) == '.') {
token = Token.LITERAL_INT;
return;
}
sp++;
ch = charAt(++bp);
bufPos++;
ch = charAt(++pos);
isDouble = true;
for (;;) {
if (ch >= '0' && ch <= '9') {
sp++;
bufPos++;
} else {
break;
}
ch = charAt(++bp);
ch = charAt(++pos);
}
}
if (ch == 'e' || ch == 'E') {
sp++;
ch = charAt(++bp);
bufPos++;
ch = charAt(++pos);
if (ch == '+' || ch == '-') {
sp++;
ch = charAt(++bp);
bufPos++;
ch = charAt(++pos);
}
for (;;) {
if (ch >= '0' && ch <= '9') {
sp++;
bufPos++;
} else {
break;
}
ch = charAt(++bp);
ch = charAt(++pos);
}
isDouble = true;
@@ -797,27 +778,27 @@ public class Lexer {
}
public void scanHexaDecimal() {
np = bp;
mark = pos;
if (ch == '-') {
sp++;
ch = charAt(++bp);
bufPos++;
ch = charAt(++pos);
}
for (;;) {
if (CharTypes.isHex(ch)) {
sp++;
bufPos++;
} else {
break;
}
ch = charAt(++bp);
ch = charAt(++pos);
}
token = Token.LITERAL_HEX;
}
public String hexString() {
return subString(np, sp);
return subString(mark, bufPos);
}
public final boolean isDigit(char ch) {
@@ -828,12 +809,12 @@ public class Lexer {
* Append a character to sbuf.
*/
protected final void putChar(char ch) {
if (sp == sbuf.length) {
char[] newsbuf = new char[sbuf.length * 2];
System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
sbuf = newsbuf;
if (bufPos == buf.length) {
char[] newsbuf = new char[buf.length * 2];
System.arraycopy(buf, 0, newsbuf, 0, buf.length);
buf = newsbuf;
}
sbuf[sp++] = ch;
buf[bufPos++] = ch;
}
/**
@@ -841,7 +822,7 @@ public class Lexer {
* translation)
*/
public final int pos() {
return tokenPos;
return 0;
}
/**
@@ -887,12 +868,12 @@ public class Lexer {
public Number integerValue() {
long result = 0;
boolean negative = false;
int i = np, max = np + sp;
int i = mark, max = mark + bufPos;
long limit;
long multmin;
int digit;
if (charAt(np) == '-') {
if (charAt(mark) == '-') {
negative = true;
limit = Long.MIN_VALUE;
i++;
@@ -918,7 +899,7 @@ public class Lexer {
}
if (negative) {
if (i > np + 1) {
if (i > mark + 1) {
if (result >= Integer.MIN_VALUE) {
return (int) result;
}
@@ -936,7 +917,7 @@ public class Lexer {
}
public int bp() {
return this.bp;
return this.pos;
}
public char current() {
@@ -944,16 +925,16 @@ public class Lexer {
}
public void reset(int mark, char markChar, Token token) {
this.bp = mark;
this.pos = mark;
this.ch = markChar;
this.token = token;
}
public final String numberString() {
return subString(np, sp);
return subString(mark, bufPos);
}
public BigDecimal decimalValue() {
return new BigDecimal(text.toCharArray(), np, sp);
return new BigDecimal(text.toCharArray(), mark, bufPos);
}
}