From f37c65f6b35b06db1d82b351861e0a94985d9109 Mon Sep 17 00:00:00 2001 From: wenshao Date: Wed, 18 May 2011 15:26:39 +0000 Subject: [PATCH] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: http://code.alibabatech.com/svn/druid/trunk@125 b9813039-fb51-4c41-a8b9-e21c2acb5095 --- .../alibaba/druid/sql/parser/SQLLexer.java | 764 ------------------ .../com/alibaba/druid/sql/parser/Token.java | 63 +- .../druid/sql/parser/SQLLexerTest.java | 34 - 3 files changed, 55 insertions(+), 806 deletions(-) delete mode 100644 src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java delete mode 100644 src/test/java/com/alibaba/druid/sql/parser/SQLLexerTest.java diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java b/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java deleted file mode 100644 index 825dc15430..0000000000 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLLexer.java +++ /dev/null @@ -1,764 +0,0 @@ -/* - * Copyright 1999-2011 Alibaba Group Holding Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.druid.sql.parser; - -import java.sql.SQLSyntaxErrorException; - -/** - * support MySQL 5.5 token - * - * @author QIU Shuo - * @author wenshao - */ -public class SQLLexer { - - protected final char[] buf; - /** always be {@link #buf}.length - 1 */ - protected final int eofIndex; - - /** current index of {@link #buf} */ - protected int curIndex = -1; - /** always be {@link #buf}[{@link #curIndex}] */ - protected char ch; - - /** current token, set by {@link #nextToken()} */ - protected int tokenPos = 0; - protected Token token; - - /** A character buffer for literals. */ - protected final static ThreadLocal sbufRef = new ThreadLocal(); - protected char[] sbuf; - - protected String stringVal; - protected SymbolTable symbolTable = new SymbolTable(); - - public SQLLexer(char[] sql){ - super(); - if ((this.sbuf = sbufRef.get()) == null) { - this.sbuf = new char[1024]; - sbufRef.set(this.sbuf); - } - if (CharTypes.isWhitespace(sql[sql.length - 1])) { - this.buf = sql; - } else { - this.buf = new char[sql.length + 1]; - System.arraycopy(sql, 0, this.buf, 0, sql.length); - } - this.eofIndex = this.buf.length - 1; - this.buf[this.eofIndex] = LayoutCharacters.EOI; - scanChar(); - } - - public SQLLexer(String sql){ - this(sql.toCharArray()); - } - - protected final Token token() { - return token; - } - - protected final char scanChar() { - return ch = buf[++curIndex]; - } - - /** - * @param skip if 1, then equals to {@link #scanChar()} - */ - protected final char scanChar(int skip) { - return ch = buf[curIndex += skip]; - } - - protected final boolean hasChars(int howMany) { - return curIndex + howMany <= eofIndex; - } - - protected final boolean eof() { - return curIndex >= eofIndex; - } - - public void nextToken() throws SQLSyntaxErrorException { - skipSeparator(); - tokenPos = curIndex; - switch (ch) { - case '0': - switch (buf[curIndex + 1]) { - case 'x': - scanChar(2); - scanHexaDecimal(false); - return; - case 'b': - scanChar(2); - scanBitField(false); - return; - } - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - scanNumber(); - return; - case '.': - if (CharTypes.isDigit(buf[curIndex + 1])) { - scanNumber(); - } else { - scanChar(); - token = Token.DOT; - } - return; - case '\'': - case '"': - scanString(); - return; - case 'n': - case 'N': - if (buf[curIndex + 1] == '\'') { - scanChar(); - scanString(); - token = Token.LITERAL_NCHARS; - return; - } - scanIdentifier(); - return; - case 'x': - case 'X': - if (buf[curIndex + 1] == '\'') { - scanChar(2); - scanHexaDecimal(true); - return; - } - scanIdentifier(); - return; - case 'b': - case 'B': - if (buf[curIndex + 1] == '\'') { - scanChar(2); - scanBitField(true); - return; - } - scanIdentifier(); - return; - case '@': - if (buf[curIndex + 1] == '@') { - scanSystemVariable(); - return; - } - scanUserVariable(); - return; - case '?': - scanChar(); - token = Token.QUES; - return; - case '(': - scanChar(); - token = Token.LPAREN; - return; - case ')': - scanChar(); - token = Token.RPAREN; - return; - case '[': - scanChar(); - token = Token.LBRACKET; - return; - case ']': - scanChar(); - token = Token.RBRACKET; - return; - case '{': - scanChar(); - token = Token.LBRACE; - return; - case '}': - scanChar(); - token = Token.RBRACE; - return; - case ',': - scanChar(); - token = Token.COMMA; - return; - case ';': - scanChar(); - token = Token.SEMI; - return; - case ':': - if (buf[curIndex + 1] == '=') { - scanChar(2); - token = Token.COLONEQ; - return; - } - scanChar(); - token = Token.COLON; - return; - case '=': - scanChar(); - token = Token.EQ; - return; - case '~': - scanChar(); - token = Token.TILDE; - return; - case '*': - scanChar(); - token = Token.STAR; - return; - case '-': - scanChar(); - token = Token.SUB; - return; - case '+': - scanChar(); - token = Token.PLUS; - return; - case '^': - scanChar(); - token = Token.CARET; - return; - case '/': - scanChar(); - token = Token.SLASH; - return; - case '%': - scanChar(); - token = Token.PERCENT; - return; - case '&': - if (buf[curIndex + 1] == '&') { - scanChar(2); - token = Token.AMPAMP; - return; - } - scanChar(); - token = Token.AMP; - return; - case '|': - if (buf[curIndex + 1] == '|') { - scanChar(2); - token = Token.BARBAR; - return; - } - scanChar(); - token = Token.BAR; - return; - case '!': - if (buf[curIndex + 1] == '=') { - scanChar(2); - token = Token.BANGEQ; - return; - } - scanChar(); - token = Token.BANG; - return; - case '>': - switch (buf[curIndex + 1]) { - case '=': - scanChar(2); - token = Token.GTEQ; - return; - case '>': - scanChar(2); - token = Token.GTGT; - return; - default: - scanChar(); - token = Token.GT; - return; - } - case '<': - switch (buf[curIndex + 1]) { - case '=': - if (buf[curIndex + 2] == '>') { - scanChar(3); - token = Token.LTEQGT; - return; - } - scanChar(2); - token = Token.LTEQ; - return; - case '>': - scanChar(2); - token = Token.LTGT; - return; - case '<': - scanChar(2); - token = Token.LTLT; - return; - default: - scanChar(); - token = Token.LT; - return; - } - case '`': - scanIdentifierWithAccent(); - return; - default: - if (CharTypes.isIdentifierChar(ch)) { - scanIdentifier(); - } else if (eof()) { - token = Token.EOF; - tokenPos = curIndex = eofIndex; - } else { - throw err("unsupported character: " + ch); - } - return; - } - } - - protected int offsetCache; - protected int sizeCache; - - /** - * first @ is included - */ - protected void scanUserVariable() throws SQLSyntaxErrorException { - if (ch != '@') throw err("first char must be @"); - offsetCache = curIndex; - sizeCache = 1; - - int hash = ch; - for (; CharTypes.isIdentifierChar(scanChar()); ++sizeCache) { - hash = 31 * hash + ch; - } - - stringVal = symbolTable.addSymbol(buf, offsetCache, sizeCache, hash); - token = Token.USR_VAR; - } - - /** - * first @@ is included - */ - protected void scanSystemVariable() throws SQLSyntaxErrorException { - if (ch != '@' || buf[curIndex + 1] != '@') throw err("first char must be @@"); - offsetCache = curIndex; - sizeCache = 2; - - int hash = 31 * ch + ch; - for (scanChar(); CharTypes.isIdentifierChar(scanChar()); ++sizeCache) { - hash = 31 * hash + ch; - } - - stringVal = symbolTable.addSymbol(buf, offsetCache, sizeCache, hash); - token = Token.SYS_VAR; - } - - protected void scanString() throws SQLSyntaxErrorException { - boolean dq = false; - if (ch == '\'') { - } else if (ch == '"') { - dq = true; - } else { - throw err("first char must be \" or '"); - } - - offsetCache = curIndex; - int size = 1; - sbuf[0] = '\''; - if (dq) { - loop: while (true) { - switch (scanChar()) { - case '\'': - putChar('\\', size++); - putChar('\'', size++); - break; - case '\\': - putChar('\\', size++); - putChar(scanChar(), size++); - continue; - case '"': - if (buf[curIndex + 1] == '"') { - putChar('"', size++); - scanChar(); - continue; - } - putChar('\'', size++); - scanChar(); - break loop; - default: - if (eof()) { - throw err("unclosed string"); - } - putChar(ch, size++); - continue; - } - } - } else { - loop: while (true) { - switch (scanChar()) { - case '\\': - putChar('\\', size++); - putChar(scanChar(), size++); - continue; - case '\'': - if (buf[curIndex + 1] == '\'') { - putChar('\\', size++); - putChar(scanChar(), size++); - continue; - } - putChar('\'', size++); - scanChar(); - break loop; - default: - if (eof()) { - throw err("unclosed string"); - } - putChar(ch, size++); - continue; - } - } - } - - sizeCache = curIndex - offsetCache; - stringVal = new String(sbuf, 0, size); - token = Token.LITERAL_CHARS; - } - - /** - * @param quoteMode if false: first 0x has been skipped; if true: first x' has been - * skipped - */ - protected void scanHexaDecimal(boolean quoteMode) throws SQLSyntaxErrorException { - offsetCache = curIndex; - for (; CharTypes.isHex(ch); scanChar()) - ; - - if ((sizeCache = curIndex - offsetCache) <= 0) { - throw err("expect at least one hexdigit"); - } - if (quoteMode) { - if (ch != '\'') { - throw err("invalid char for hex: " + ch); - } - } else if (CharTypes.isIdentifierChar(ch)) { - ch = buf[curIndex = offsetCache -= 2]; - scanIdentifierFromNumber(offsetCache, sizeCache + 2); - return; - } - - token = Token.LITERAL_HEX; - } - - /** - * @param quoteMode if false: first 0b has been skipped; if true: first b' has been - * skipped - */ - protected void scanBitField(boolean quoteMode) throws SQLSyntaxErrorException { - offsetCache = curIndex; - for (; ch == '0' || ch == '1'; scanChar()) - ; - - if ((sizeCache = curIndex - offsetCache) <= 0) { - throw err("expect at least one bit"); - } - if (quoteMode) { - if (ch != '\'') { - throw err("invalid char for bit: " + ch); - } - } else if (CharTypes.isIdentifierChar(ch)) { - ch = buf[curIndex = offsetCache -= 2]; - scanIdentifierFromNumber(offsetCache, sizeCache + 2); - return; - } - - token = Token.LITERAL_BIT; - } - - /** - * if first char is ., token may be {@link Token#DOT} if invalid char is presented after . - */ - protected void scanNumber() throws SQLSyntaxErrorException { - offsetCache = curIndex; - sizeCache = 1; - final boolean fstDot = ch == '.'; - boolean dot = fstDot; - boolean sign = false; - int state = fstDot ? 1 : 0; - - for (; scanChar() != LayoutCharacters.EOI; ++sizeCache) { - switch (state) { - case 0: - if (CharTypes.isDigit(ch)) { - } else if (ch == '.') { - dot = true; - state = 1; - } else if (ch == 'e' || ch == 'E') { - state = 3; - } else if (CharTypes.isIdentifierChar(ch)) { - scanIdentifierFromNumber(offsetCache, sizeCache); - return; - } else { - token = Token.LITERAL_INT; - return; - } - break; - case 1: - if (CharTypes.isDigit(ch)) { - state = 2; - } else if (ch == 'e' || ch == 'E') { - state = 3; - } else if (CharTypes.isIdentifierChar(ch)) { - if (fstDot) { - sizeCache = 1; - ch = buf[curIndex = offsetCache + 1]; - token = Token.DOT; - return; - } else { - throw err("invalid char after '.': " + ch); - } - } else { - token = Token.LITERAL_FLOAT; - return; - } - break; - case 2: - if (CharTypes.isDigit(ch)) { - } else if (ch == 'e' || ch == 'E') { - state = 3; - } else if (CharTypes.isIdentifierChar(ch)) { - if (fstDot) { - sizeCache = 1; - ch = buf[curIndex = offsetCache + 1]; - token = Token.DOT; - return; - } else { - throw err("invalid char after '.' for as part of number: " + ch); - } - } else { - token = Token.LITERAL_FLOAT; - return; - } - break; - case 3: - if (CharTypes.isDigit(ch)) { - state = 5; - } else if (ch == '+' || ch == '-') { - sign = true; - state = 4; - } else if (CharTypes.isIdentifierChar(ch)) { - if (fstDot) { - sizeCache = 1; - ch = buf[curIndex = offsetCache + 1]; - token = Token.DOT; - return; - } else if (!dot) { - scanIdentifierFromNumber(offsetCache, sizeCache); - return; - } else { - throw err("invalid char after '.' and 'e' for as part of number: " + ch); - } - } else if (!dot) { - token = Token.IDENTIFIER; - stringVal = symbolTable.addSymbol(buf, offsetCache, sizeCache); - return; - } else { - throw err("invalid char after '.' and 'e' for as part of number: " + ch); - } - break; - case 4: - if (CharTypes.isDigit(ch)) { - state = 5; - break; - } else if (fstDot) { - sizeCache = 1; - ch = buf[curIndex = offsetCache + 1]; - token = Token.DOT; - } else if (!dot) { - ch = buf[--curIndex]; - --sizeCache; - token = Token.IDENTIFIER; - stringVal = symbolTable.addSymbol(buf, offsetCache, sizeCache); - } else { - throw err("expect digit char after SIGN for 'e': " + ch); - } - return; - case 5: - if (CharTypes.isDigit(ch)) { - break; - } else if (CharTypes.isIdentifierChar(ch)) { - if (fstDot) { - sizeCache = 1; - ch = buf[curIndex = offsetCache + 1]; - token = Token.DOT; - } else if (!dot) { - if (sign) { - sizeCache = 1; - ch = buf[curIndex = offsetCache]; - scanIdentifierFromNumber(curIndex, 0); - } else { - scanIdentifierFromNumber(offsetCache, sizeCache); - } - } else { - throw err("expect digit char after SIGN for 'e': " + ch); - } - } else { - token = Token.LITERAL_FLOAT; - } - return; - } - } - token = state == 0 ? Token.LITERAL_INT : Token.LITERAL_FLOAT; - } - - /** - * @param initSize how many char has already been consumed - */ - private void scanIdentifierFromNumber(int initOffset, int initSize) throws SQLSyntaxErrorException { - offsetCache = initOffset; - sizeCache = initSize; - for (; CharTypes.isIdentifierChar(ch); ++sizeCache) { - scanChar(); - } - stringVal = symbolTable.addSymbol(buf, offsetCache, sizeCache); - token = Token.IDENTIFIER; - } - - /** - * id is NOT included in `. - */ - protected void scanIdentifier() throws SQLSyntaxErrorException { - scanIdentifierFromNumber(curIndex, 0); - } - - /** - * id is included in `. first ` is included - */ - protected void scanIdentifierWithAccent() throws SQLSyntaxErrorException { - offsetCache = curIndex; - for (; scanChar() != LayoutCharacters.EOI;) { - if (ch == '`') { - if (scanChar() != '`') { - break; - } - } - } - stringVal = symbolTable.addSymbol(buf, offsetCache, sizeCache = curIndex - offsetCache); - token = Token.IDENTIFIER; - } - - /** - * Append a character to sbuf. - */ - protected final void putChar(char ch, int index) { - if (index >= sbuf.length) { - char[] newsbuf = new char[sbuf.length * 2]; - System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length); - sbuf = newsbuf; - } - sbuf[index] = ch; - } - - /** - * skip whitespace and comment - */ - protected void skipSeparator() { - while (!eof()) { - while (CharTypes.isWhitespace(ch)) { // imply: ch!=eof - scanChar(); - continue; - } - - switch (ch) { - case '#': // MySQL specified - while (!eof()) { - if (scanChar() == '\n') { - scanChar(); - continue; - } - } - continue; - case '/': - if (hasChars(2) && '*' == buf[curIndex + 1] && '!' != buf[curIndex + 2]) { - scanChar(2); - for (int state = 0; !eof(); scanChar()) { - if (state == 0) { - if ('*' == ch) { - state = 1; - } - } else { - if ('*' != ch) { - state = 0; - } else if ('/' == ch) { - scanChar(); - continue; - } - } - } - continue; - } - return; - case '-': - if (hasChars(3) && '-' == buf[curIndex + 1] && CharTypes.isWhitespace(buf[curIndex + 2])) { - scanChar(3); - for (; !eof(); scanChar()) { - if ('\n' == ch) { - scanChar(); - continue; - } - } - } - return; - } - return; - } - } - - /** - * always throw SQLSyntaxErrorException - */ - protected SQLSyntaxErrorException err(String msg) throws SQLSyntaxErrorException { - String errMsg = msg + ". " + toString(); - throw new SQLSyntaxErrorException(errMsg); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getSimpleName()).append('@').append(hashCode()).append('{'); - String sqlLeft = new String(buf, curIndex, buf.length - curIndex); - sb.append("curIndex=").append(curIndex).append(", ch=").append(ch).append(", token=").append(token).append(", sqlLeft=").append(sqlLeft).append(", sql=").append(buf); - sb.append('}'); - return sb.toString(); - } - - // - // public static void main(String[] args) { - // char[] dd = new char[333]; - // for (int i = 0; i < dd.length; ++i) { - // dd[i] = 'd'; - // } - // char t = 'c'; - // char t1 = 'd'; - // long start = System.currentTimeMillis(); - // long end = System.currentTimeMillis(); - // for (int i = 0; i < 10; ++i) { - // int r = i % dd.length; - // t = dd[r]; - // } - // - // int loop = 500000000; - // start = System.currentTimeMillis(); - // for (int i = 0; i < loop; ++i) { - // int r = i % dd.length; - // t = t1; - // } - // end = System.currentTimeMillis(); - // System.out.println((end - start) * 1.0d / (loop / 1000000)); - // } - -} diff --git a/src/main/java/com/alibaba/druid/sql/parser/Token.java b/src/main/java/com/alibaba/druid/sql/parser/Token.java index ed1c29b2ed..66fe7fc59c 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Token.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Token.java @@ -36,18 +36,65 @@ public enum Token { TABLE("TABLE"), UPDATE("UPDATE"), SET("SET"), - SELECT("SELECT"), FROM("FROM"), WHERE("WHERE"), ORDER("ORDER"), BY("BY"), - GROUP("GROUP"), HAVING("HAVING"), INSERT("INSERT"), INTO("INTO"), NULL("NULL"), NOT("NOT"), AS("AS"), DELETE("DELETE"), DISTINCT("DISTINCT"), - UNIQUE("UNIQUE"), FOREIGN("FOREIGN"), REFERENCE("REFERENCE"), REFERENCES("REFERENCES"), ALL("ALL"), UNION("UNION"), INTERSECT("INTERSECT"), MINUS("MINUS"), - INNER("INNER"), LEFT("LEFT"), RIGHT("RIGHT"), FULL("FULL"), OUTER("OUTER"), JOIN("JOIN"), ON("ON"), SCHEMA("SCHEMA"), CAST("CAST"), + SELECT("SELECT"), + FROM("FROM"), + WHERE("WHERE"), + ORDER("ORDER"), + BY("BY"), + GROUP("GROUP"), + HAVING("HAVING"), + INSERT("INSERT"), + INTO("INTO"), + NULL("NULL"), + NOT("NOT"), + AS("AS"), + DELETE("DELETE"), + DISTINCT("DISTINCT"), + UNIQUE("UNIQUE"), + FOREIGN("FOREIGN"), + REFERENCE("REFERENCE"), + REFERENCES("REFERENCES"), + ALL("ALL"), + UNION("UNION"), + INTERSECT("INTERSECT"), + MINUS("MINUS"), + INNER("INNER"), + LEFT("LEFT"), + RIGHT("RIGHT"), + FULL("FULL"), + OUTER("OUTER"), + JOIN("JOIN"), + ON("ON"), + SCHEMA("SCHEMA"), + CAST("CAST"), - AND("AND"), OR("OR"), XOR("XOR"), CASE("CASE"), WHEN("WHEN"), THEN("THEN"), ELSE("ELSE"), END("END"), EXISTS("EXISTS"), IN("IN"), + AND("AND"), + OR("OR"), + XOR("XOR"), + CASE("CASE"), + WHEN("WHEN"), + THEN("THEN"), + ELSE("ELSE"), + END("END"), + EXISTS("EXISTS"), + IN("IN"), - NEW("NEW"), ASC("ASC"), DESC("DESC"), IS("IS"), LIKE("LIKE"), ESCAPE("ESCAPE"), BETWEEN("BETWEEN"), VALUES("VALUES"), INTERVAL("INTERVAL"), + NEW("NEW"), + ASC("ASC"), + DESC("DESC"), + IS("IS"), + LIKE("LIKE"), + ESCAPE("ESCAPE"), + BETWEEN("BETWEEN"), + VALUES("VALUES"), + INTERVAL("INTERVAL"), - LOCK("LOCK"), SOME("SOME"), ANY("ANY"), + LOCK("LOCK"), + SOME("SOME"), + ANY("ANY"), - TRUE("TRUE"), FALSE("FALSE"), + TRUE("TRUE"), + FALSE("FALSE"), EOF, ERROR, IDENTIFIER, diff --git a/src/test/java/com/alibaba/druid/sql/parser/SQLLexerTest.java b/src/test/java/com/alibaba/druid/sql/parser/SQLLexerTest.java deleted file mode 100644 index d33b08e57c..0000000000 --- a/src/test/java/com/alibaba/druid/sql/parser/SQLLexerTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 1999-2011 Alibaba Group Holding Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.druid.sql.parser; - -import com.alibaba.druid.sql.parser.SQLLexer; - -import junit.framework.Assert; -import junit.framework.TestCase; - -/** - * @author QIU Shuo - */ -public class SQLLexerTest extends TestCase { - - public void testSkipSeparator() { - SQLLexer sut = new SQLLexer("/**/ \t\n\r\n -- \n#\n"); - sut.skipSeparator(); - Assert.assertEquals(sut.eofIndex, sut.curIndex); - Assert.assertEquals(sut.buf[sut.eofIndex], sut.ch); - } -}