From 63574c47020b85560f9d4d6649cdc778273cc6e5 Mon Sep 17 00:00:00 2001 From: wenshao Date: Tue, 30 Jul 2013 20:00:40 +0800 Subject: [PATCH] =?UTF-8?q?sql=20parser=E6=94=AF=E6=8C=81=E4=B8=AD?= =?UTF-8?q?=E6=96=87=E7=A9=BA=E6=A0=BC=E5=92=8C=E9=80=97=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../alibaba/druid/sql/parser/CharTypes.java | 12 +- .../com/alibaba/druid/sql/parser/Lexer.java | 2069 +++++++++-------- .../bvt/sql/oracle/OracleSelectTest45.java | 2 +- 3 files changed, 1045 insertions(+), 1038 deletions(-) diff --git a/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java b/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java index 4ecd057264..84d601e7b5 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java +++ b/src/main/java/com/alibaba/druid/sql/parser/CharTypes.java @@ -53,7 +53,10 @@ public class CharTypes { } public static boolean isFirstIdentifierChar(char c) { - return c > firstIdentifierFlags.length || firstIdentifierFlags[c]; + if (c <= firstIdentifierFlags.length) { + return firstIdentifierFlags[c]; + } + return c != '　' && c != '，'; } private final static boolean[] identifierFlags = new boolean[256]; @@ -74,7 +77,10 @@ public class CharTypes { } public static boolean isIdentifierChar(char c) { - return c > identifierFlags.length || identifierFlags[c]; + if (c <= identifierFlags.length) { + return identifierFlags[c]; + } + return c != '　' && c != '，'; } private final static boolean[] whitespaceFlags = new boolean[256]; @@ -93,7 +99,7 @@ public class CharTypes { */ public static boolean isWhitespace(char c) { return (c <= whitespaceFlags.length && whitespaceFlags[c]) // - || c == '　'; // Chinese space + || c == '　'; // Chinese space } } diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index 53707630fb..2d2ae4b53d 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -1,1034 +1,1035 @@ -/* - * Copyright 1999-2011 Alibaba Group Holding Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.druid.sql.parser; - -import static com.alibaba.druid.sql.parser.CharTypes.isFirstIdentifierChar; -import static com.alibaba.druid.sql.parser.CharTypes.isIdentifierChar; -import static com.alibaba.druid.sql.parser.CharTypes.isWhitespace; -import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI; -import static com.alibaba.druid.sql.parser.Token.COLONCOLON; -import static com.alibaba.druid.sql.parser.Token.COLONEQ; -import static com.alibaba.druid.sql.parser.Token.COMMA; -import static com.alibaba.druid.sql.parser.Token.EOF; -import static com.alibaba.druid.sql.parser.Token.ERROR; -import static com.alibaba.druid.sql.parser.Token.LBRACE; -import static com.alibaba.druid.sql.parser.Token.LBRACKET; -import static com.alibaba.druid.sql.parser.Token.LITERAL_ALIAS; -import static com.alibaba.druid.sql.parser.Token.LITERAL_CHARS; -import static com.alibaba.druid.sql.parser.Token.LPAREN; -import static com.alibaba.druid.sql.parser.Token.RBRACE; -import static com.alibaba.druid.sql.parser.Token.RBRACKET; -import static com.alibaba.druid.sql.parser.Token.RPAREN; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.Arrays; - -/** - * @author wenshao - */ -public class Lexer { - - protected final String text; - protected int pos; - protected int mark; - - protected char ch; - - protected char[] buf; - protected int bufPos; - - protected Token token; - - protected Keywords keywods = Keywords.DEFAULT_KEYWORDS; - - protected String stringVal; - - protected boolean skipComment = true; - - private SavePoint savePoint = null; - - /* - * anti sql injection - */ - private boolean allowComment = true; - - private int varIndex = -1; - - protected CommentHandler commentHandler; - - public Lexer(String input){ - this(input, true); - } - - public CommentHandler getCommentHandler() { - return commentHandler; - } - - public void setCommentHandler(CommentHandler commentHandler) { - this.commentHandler = commentHandler; - } - - public final char charAt(int index) { - if (index >= text.length()) { - return EOI; - } - - return text.charAt(index); - } - - public final String addSymbol() { - return subString(mark, bufPos); - } - - public final String subString(int offset, int count) { - return text.substring(offset, offset + count); - } - - protected void initBuff(int size) { - if (buf == null) { - if (size < 32) { - buf = new char[32]; - } else { - buf = new char[size + 32]; - } - } else if (buf.length < size) { - buf = Arrays.copyOf(buf, size); - } - } - - public void arraycopy(int srcPos, char[] dest, int destPos, int length) { - text.getChars(srcPos, srcPos + length, dest, destPos); - } - - public boolean isAllowComment() { - return allowComment; - } - - public void setAllowComment(boolean allowComment) { - this.allowComment = allowComment; - } - - public int nextVarIndex() { - return ++varIndex; - } - - private static class SavePoint { - - int bp; - int sp; - int np; - char ch; - Token token; - } - - public Keywords getKeywods() { - return keywods; - } - - public void mark() { - SavePoint savePoint = new SavePoint(); - savePoint.bp = pos; - savePoint.sp = bufPos; - savePoint.np = mark; - savePoint.ch = ch; - savePoint.token = token; - this.savePoint = savePoint; - } - - public void reset() { - this.pos = savePoint.bp; - this.bufPos = savePoint.sp; - this.mark = savePoint.np; - this.ch = savePoint.ch; - this.token = savePoint.token; - } - - public Lexer(String input, boolean skipComment){ - this.skipComment = skipComment; - - this.text = input; - this.pos = -1; - - scanChar(); - } - - public Lexer(char[] input, int inputLength, boolean skipComment){ - this(new String(input, 0, inputLength), skipComment); - } - - protected final void scanChar() { - ch = charAt(++pos); - } - - protected void unscan() { - ch = charAt(--pos); - } - - public boolean isEOF() { - return pos >= text.length(); - } - - /** - * Report an error at the given position using the provided arguments. - */ - protected void lexError(String key, Object... args) { - token = ERROR; - } - - /** - * Return the current token, set by nextToken(). - */ - public final Token token() { - return token; - } - - public String info() { - return this.token + " " + this.stringVal(); - } - - public final void nextTokenComma() { - if (ch == ' ') { - scanChar(); - } - - if (ch == ',') { - scanChar(); - token = COMMA; - return; - } - - if (ch == ')') { - scanChar(); - token = RPAREN; - return; - } - - nextToken(); - } - - public final void nextTokenLParen() { - if (ch == ' ') { - scanChar(); - } - - if (ch == '(') { - scanChar(); - token = LPAREN; - return; - } - nextToken(); - } - - public final void nextTokenValue() { - if (ch == ' ') { - scanChar(); - } - - if (ch == '\'') { - bufPos = 0; - scanString(); - return; - } - - if (ch >= '0' && ch <= '9') { - bufPos = 0; - scanNumber(); - return; - } - - if (ch == '?') { - scanChar(); - token = Token.QUES; - return; - } - - if (isFirstIdentifierChar(ch) && ch != 'N') { - scanIdentifier(); - return; - } - - nextToken(); - } - - public final void nextToken() { - bufPos = 0; - - for (;;) { - if (isWhitespace(ch)) { - scanChar(); - continue; - } - - if (ch == '$' && charAt(pos + 1) == '{') { - scanVariable(); - return; - } - - if (isFirstIdentifierChar(ch)) { - if (ch == 'N') { - if (charAt(pos + 1) == '\'') { - ++pos; - ch = '\''; - scanString(); - token = Token.LITERAL_NCHARS; - return; - } - } - - scanIdentifier(); - return; - } - - switch (ch) { - case '0': - if (charAt(pos + 1) == 'x') { - scanChar(); - scanChar(); - scanHexaDecimal(); - } else { - scanNumber(); - } - return; - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - scanNumber(); - return; - case ',': - scanChar(); - token = COMMA; - return; - case '(': - scanChar(); - token = LPAREN; - return; - case ')': - scanChar(); - token = RPAREN; - return; - case '[': - scanLBracket(); - return; - case ']': - scanChar(); - token = RBRACKET; - return; - case '{': - scanChar(); - token = LBRACE; - return; - case '}': - scanChar(); - token = RBRACE; - return; - case ':': - scanChar(); - if (ch == '=') { - scanChar(); - token = COLONEQ; - } else if (ch == ':') { - scanChar(); - token = COLONCOLON; - } else { - if (isDigit(ch)) { - unscan(); - scanVariable(); - } else { - unscan(); - scanVariable(); - } - } - return; - case '#': - scanSharp(); - if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) { - bufPos = 0; - continue; - } - return; - case '.': - scanChar(); - if (isDigit(ch)) { - unscan(); - scanNumber(); - return; - } else if (ch == '.') { - scanChar(); - if (ch == '.') { - scanChar(); - token = Token.DOTDOTDOT; - } else { - token = Token.DOTDOT; - } - } else { - token = Token.DOT; - } - return; - case '\'': - scanString(); - return; - case '\"': - scanAlias(); - return; - case '*': - scanChar(); - token = Token.STAR; - return; - case '?': - scanChar(); - token = Token.QUES; - return; - case ';': - scanChar(); - token = Token.SEMI; - return; - case '`': - throw new ParserException("TODO"); // TODO - case '@': - scanVariable(); - return; - case '-': - int subNextChar = charAt(pos + 1); - if (subNextChar == '-') { - scanComment(); - if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) { - bufPos = 0; - continue; - } - } else { - scanOperator(); - } - return; - case '/': - int nextChar = charAt(pos + 1); - if (nextChar == '/' || nextChar == '*') { - scanComment(); - if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) { - bufPos = 0; - continue; - } - } else { - token = Token.SLASH; - scanChar(); - } - return; - default: - if (Character.isLetter(ch)) { - scanIdentifier(); - return; - } - - if (isOperator(ch)) { - scanOperator(); - return; - } - - // QS_TODO ? - if (isEOF()) { // JLS - token = EOF; - } else { - lexError("illegal.char", String.valueOf((int) ch)); - scanChar(); - } - - return; - } - } - - } - - protected void scanLBracket() { - scanChar(); - token = LBRACKET; - } - - private final void scanOperator() { - switch (ch) { - case '+': - scanChar(); - token = Token.PLUS; - break; - case '-': - scanChar(); - token = Token.SUB; - break; - case '*': - scanChar(); - token = Token.STAR; - break; - case '/': - scanChar(); - token = Token.SLASH; - break; - case '&': - scanChar(); - if (ch == '&') { - scanChar(); - token = Token.AMPAMP; - } else { - token = Token.AMP; - } - break; - case '|': - scanChar(); - if (ch == '|') { - scanChar(); - token = Token.BARBAR; - } else { - token = Token.BAR; - } - break; - case '^': - scanChar(); - token = Token.CARET; - break; - case '%': - scanChar(); - token = Token.PERCENT; - break; - case '=': - scanChar(); - if (ch == '=') { - scanChar(); - token = Token.EQEQ; - } else { - token = Token.EQ; - } - break; - case '>': - scanChar(); - if (ch == '=') { - scanChar(); - token = Token.GTEQ; - } else if (ch == '>') { - scanChar(); - token = Token.GTGT; - } else { - token = Token.GT; - } - break; - case '<': - scanChar(); - if (ch == '=') { - scanChar(); - if (ch == '>') { - token = Token.LTEQGT; - scanChar(); - } else { - token = Token.LTEQ; - } - } else if (ch == '>') { - scanChar(); - token = Token.LTGT; - } else if (ch == '<') { - scanChar(); - token = Token.LTLT; - } else { - token = Token.LT; - } - break; - case '!': - scanChar(); - if (ch == '=') { - scanChar(); - token = Token.BANGEQ; - } else if (ch == '>') { - scanChar(); - token = Token.BANGGT; - } else if (ch == '<') { - scanChar(); - token = Token.BANGLT; - } else { - token = Token.BANG; - } - break; - case '?': - scanChar(); - token = Token.QUES; - break; - case '~': - scanChar(); - token = Token.TILDE; - break; - default: - throw new ParserException("TODO"); - } - } - - protected void scanString() { - mark = pos; - boolean hasSpecial = false; - - for (;;) { - if (isEOF()) { - lexError("unclosed.str.lit"); - return; - } - - ch = charAt(++pos); - - if (ch == '\'') { - scanChar(); - if (ch != '\'') { - token = LITERAL_CHARS; - break; - } else { - if (!hasSpecial) { - initBuff(bufPos); - arraycopy(mark + 1, buf, 0, bufPos); - hasSpecial = true; - } - putChar('\''); - continue; - } - } - - if (!hasSpecial) { - bufPos++; - continue; - } - - if (bufPos == buf.length) { - putChar(ch); - } else { - buf[bufPos++] = ch; - } - } - - if (!hasSpecial) { - stringVal = subString(mark + 1, bufPos); - } else { - stringVal = new String(buf, 0, bufPos); - } - } - - private final void scanAlias() { - mark = pos; - - if (buf == null) { - buf = new char[32]; - } - - for (;;) { - if (isEOF()) { - lexError("unclosed.str.lit"); - return; - } - - ch = charAt(++pos); - - if (ch == '\"') { - scanChar(); - token = LITERAL_ALIAS; - break; - } - - if (bufPos == buf.length) { - putChar(ch); - } else { - buf[bufPos++] = ch; - } - } - - stringVal = subString(mark + 1, bufPos); - } - - public void scanSharp() { - scanVariable(); - } - - public void scanVariable() { - if (ch != '@' && ch != ':' && ch != '#' && ch != '$') { - throw new ParserException("illegal variable"); - } - - mark = pos; - bufPos = 1; - char ch; - - boolean mybatisFlag = false; - if (charAt(pos + 1) == '@') { - ch = charAt(++pos); - - bufPos++; - } else if (charAt(pos + 1) == '{') { - pos++; - bufPos++; - mybatisFlag = true; - } - - for (;;) { - ch = charAt(++pos); - - if (!isIdentifierChar(ch)) { - break; - } - - bufPos++; - continue; - } - - if (mybatisFlag) { - if (ch != '}') { - throw new ParserException("syntax error"); - } - ++pos; - bufPos++; - } - - this.ch = charAt(pos); - - stringVal = addSymbol(); - token = Token.VARIANT; - } - - public void scanComment() { - if (!allowComment) { - throw new NotAllowCommentException(); - } - - if (ch != '/') { - throw new IllegalStateException(); - } - - mark = pos; - bufPos = 0; - scanChar(); - - if (ch == '*') { - scanChar(); - bufPos++; - - for (;;) { - if (ch == '*' && charAt(pos + 1) == '/') { - bufPos += 2; - scanChar(); - scanChar(); - break; - } - - scanChar(); - bufPos++; - } - - stringVal = subString(mark, bufPos); - token = Token.MULTI_LINE_COMMENT; - return; - } - - if (ch == '/') { - scanChar(); - bufPos++; - - for (;;) { - if (ch == '\r') { - if (charAt(pos + 1) == '\n') { - bufPos += 2; - scanChar(); - break; - } - bufPos++; - break; - } - - if (ch == '\n') { - scanChar(); - bufPos++; - break; - } - - scanChar(); - bufPos++; - } - - stringVal = subString(mark + 1, bufPos); - token = Token.LINE_COMMENT; - return; - } - } - - public void scanIdentifier() { - final char first = ch; - - final boolean firstFlag = isFirstIdentifierChar(first); - if (!firstFlag) { - throw new ParserException("illegal identifier"); - } - - mark = pos; - bufPos = 1; - char ch; - for (;;) { - ch = charAt(++pos); - - if (!isIdentifierChar(ch)) { - break; - } - - bufPos++; - continue; - } - - this.ch = charAt(pos); - - stringVal = addSymbol(); - Token tok = keywods.getKeyword(stringVal); - if (tok != null) { - token = tok; - } else { - token = Token.IDENTIFIER; - } - } - - public void scanNumber() { - mark = pos; - - if (ch == '-') { - bufPos++; - ch = charAt(++pos); - } - - for (;;) { - if (ch >= '0' && ch <= '9') { - bufPos++; - } else { - break; - } - ch = charAt(++pos); - } - - boolean isDouble = false; - - if (ch == '.') { - if (charAt(pos + 1) == '.') { - token = Token.LITERAL_INT; - return; - } - bufPos++; - ch = charAt(++pos); - isDouble = true; - - for (;;) { - if (ch >= '0' && ch <= '9') { - bufPos++; - } else { - break; - } - ch = charAt(++pos); - } - } - - if (ch == 'e' || ch == 'E') { - bufPos++; - ch = charAt(++pos); - - if (ch == '+' || ch == '-') { - bufPos++; - ch = charAt(++pos); - } - - for (;;) { - if (ch >= '0' && ch <= '9') { - bufPos++; - } else { - break; - } - ch = charAt(++pos); - } - - isDouble = true; - } - - if (isDouble) { - token = Token.LITERAL_FLOAT; - } else { - token = Token.LITERAL_INT; - } - } - - public void scanHexaDecimal() { - mark = pos; - - if (ch == '-') { - bufPos++; - ch = charAt(++pos); - } - - for (;;) { - if (CharTypes.isHex(ch)) { - bufPos++; - } else { - break; - } - ch = charAt(++pos); - } - - token = Token.LITERAL_HEX; - } - - public String hexString() { - return subString(mark, bufPos); - } - - public final boolean isDigit(char ch) { - return ch >= '0' && ch <= '9'; - } - - /** - * Append a character to sbuf. - */ - protected final void putChar(char ch) { - if (bufPos == buf.length) { - char[] newsbuf = new char[buf.length * 2]; - System.arraycopy(buf, 0, newsbuf, 0, buf.length); - buf = newsbuf; - } - buf[bufPos++] = ch; - } - - /** - * Return the current token's position: a 0-based offset from beginning of the raw input stream (before unicode - * translation) - */ - public final int pos() { - return pos; - } - - /** - * The value of a literal token, recorded as a string. For integers, leading 0x and 'l' suffixes are suppressed. - */ - public final String stringVal() { - return stringVal; - } - - private boolean isOperator(char ch) { - switch (ch) { - case '!': - case '%': - case '&': - case '*': - case '+': - case '-': - case '<': - case '=': - case '>': - case '^': - case '|': - case '~': - case ';': - return true; - default: - return false; - } - } - - private static final long MULTMIN_RADIX_TEN = Long.MIN_VALUE / 10; - private static final long N_MULTMAX_RADIX_TEN = -Long.MAX_VALUE / 10; - - private final static int[] digits = new int[(int) '9' + 1]; - - static { - for (int i = '0'; i <= '9'; ++i) { - digits[i] = i - '0'; - } - } - - // QS_TODO negative number is invisible for lexer - public Number integerValue() { - long result = 0; - boolean negative = false; - int i = mark, max = mark + bufPos; - long limit; - long multmin; - int digit; - - if (charAt(mark) == '-') { - negative = true; - limit = Long.MIN_VALUE; - i++; - } else { - limit = -Long.MAX_VALUE; - } - multmin = negative ? MULTMIN_RADIX_TEN : N_MULTMAX_RADIX_TEN; - if (i < max) { - digit = digits[charAt(i++)]; - result = -digit; - } - while (i < max) { - // Accumulating negatively avoids surprises near MAX_VALUE - digit = digits[charAt(i++)]; - if (result < multmin) { - return new BigInteger(numberString()); - } - result *= 10; - if (result < limit + digit) { - return new BigInteger(numberString()); - } - result -= digit; - } - - if (negative) { - if (i > mark + 1) { - if (result >= Integer.MIN_VALUE) { - return (int) result; - } - return result; - } else { /* Only got "-" */ - throw new NumberFormatException(numberString()); - } - } else { - result = -result; - if (result <= Integer.MAX_VALUE) { - return (int) result; - } - return result; - } - } - - public int bp() { - return this.pos; - } - - public char current() { - return this.ch; - } - - public void reset(int mark, char markChar, Token token) { - this.pos = mark; - this.ch = markChar; - this.token = token; - } - - public final String numberString() { - return subString(mark, bufPos); - } - - public BigDecimal decimalValue() { - return new BigDecimal(text.toCharArray(), mark, bufPos); - } - - public static interface CommentHandler { - - boolean handle(Token lastToken, String comment); - } -} +/* + * Copyright 1999-2011 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.sql.parser; + +import static com.alibaba.druid.sql.parser.CharTypes.isFirstIdentifierChar; +import static com.alibaba.druid.sql.parser.CharTypes.isIdentifierChar; +import static com.alibaba.druid.sql.parser.CharTypes.isWhitespace; +import static com.alibaba.druid.sql.parser.LayoutCharacters.EOI; +import static com.alibaba.druid.sql.parser.Token.COLONCOLON; +import static com.alibaba.druid.sql.parser.Token.COLONEQ; +import static com.alibaba.druid.sql.parser.Token.COMMA; +import static com.alibaba.druid.sql.parser.Token.EOF; +import static com.alibaba.druid.sql.parser.Token.ERROR; +import static com.alibaba.druid.sql.parser.Token.LBRACE; +import static com.alibaba.druid.sql.parser.Token.LBRACKET; +import static com.alibaba.druid.sql.parser.Token.LITERAL_ALIAS; +import static com.alibaba.druid.sql.parser.Token.LITERAL_CHARS; +import static com.alibaba.druid.sql.parser.Token.LPAREN; +import static com.alibaba.druid.sql.parser.Token.RBRACE; +import static com.alibaba.druid.sql.parser.Token.RBRACKET; +import static com.alibaba.druid.sql.parser.Token.RPAREN; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; + +/** + * @author wenshao + */ +public class Lexer { + + protected final String text; + protected int pos; + protected int mark; + + protected char ch; + + protected char[] buf; + protected int bufPos; + + protected Token token; + + protected Keywords keywods = Keywords.DEFAULT_KEYWORDS; + + protected String stringVal; + + protected boolean skipComment = true; + + private SavePoint savePoint = null; + + /* + * anti sql injection + */ + private boolean allowComment = true; + + private int varIndex = -1; + + protected CommentHandler commentHandler; + + public Lexer(String input){ + this(input, true); + } + + public CommentHandler getCommentHandler() { + return commentHandler; + } + + public void setCommentHandler(CommentHandler commentHandler) { + this.commentHandler = commentHandler; + } + + public final char charAt(int index) { + if (index >= text.length()) { + return EOI; + } + + return text.charAt(index); + } + + public final String addSymbol() { + return subString(mark, bufPos); + } + + public final String subString(int offset, int count) { + return text.substring(offset, offset + count); + } + + protected void initBuff(int size) { + if (buf == null) { + if (size < 32) { + buf = new char[32]; + } else { + buf = new char[size + 32]; + } + } else if (buf.length < size) { + buf = Arrays.copyOf(buf, size); + } + } + + public void arraycopy(int srcPos, char[] dest, int destPos, int length) { + text.getChars(srcPos, srcPos + length, dest, destPos); + } + + public boolean isAllowComment() { + return allowComment; + } + + public void setAllowComment(boolean allowComment) { + this.allowComment = allowComment; + } + + public int nextVarIndex() { + return ++varIndex; + } + + private static class SavePoint { + + int bp; + int sp; + int np; + char ch; + Token token; + } + + public Keywords getKeywods() { + return keywods; + } + + public void mark() { + SavePoint savePoint = new SavePoint(); + savePoint.bp = pos; + savePoint.sp = bufPos; + savePoint.np = mark; + savePoint.ch = ch; + savePoint.token = token; + this.savePoint = savePoint; + } + + public void reset() { + this.pos = savePoint.bp; + this.bufPos = savePoint.sp; + this.mark = savePoint.np; + this.ch = savePoint.ch; + this.token = savePoint.token; + } + + public Lexer(String input, boolean skipComment){ + this.skipComment = skipComment; + + this.text = input; + this.pos = -1; + + scanChar(); + } + + public Lexer(char[] input, int inputLength, boolean skipComment){ + this(new String(input, 0, inputLength), skipComment); + } + + protected final void scanChar() { + ch = charAt(++pos); + } + + protected void unscan() { + ch = charAt(--pos); + } + + public boolean isEOF() { + return pos >= text.length(); + } + + /** + * Report an error at the given position using the provided arguments. + */ + protected void lexError(String key, Object... args) { + token = ERROR; + } + + /** + * Return the current token, set by nextToken(). + */ + public final Token token() { + return token; + } + + public String info() { + return this.token + " " + this.stringVal(); + } + + public final void nextTokenComma() { + if (ch == ' ') { + scanChar(); + } + + if (ch == ',' || ch == '，') { + scanChar(); + token = COMMA; + return; + } + + if (ch == ')') { + scanChar(); + token = RPAREN; + return; + } + + nextToken(); + } + + public final void nextTokenLParen() { + if (ch == ' ') { + scanChar(); + } + + if (ch == '(') { + scanChar(); + token = LPAREN; + return; + } + nextToken(); + } + + public final void nextTokenValue() { + if (ch == ' ') { + scanChar(); + } + + if (ch == '\'') { + bufPos = 0; + scanString(); + return; + } + + if (ch >= '0' && ch <= '9') { + bufPos = 0; + scanNumber(); + return; + } + + if (ch == '?') { + scanChar(); + token = Token.QUES; + return; + } + + if (isFirstIdentifierChar(ch) && ch != 'N') { + scanIdentifier(); + return; + } + + nextToken(); + } + + public final void nextToken() { + bufPos = 0; + + for (;;) { + if (isWhitespace(ch)) { + scanChar(); + continue; + } + + if (ch == '$' && charAt(pos + 1) == '{') { + scanVariable(); + return; + } + + if (isFirstIdentifierChar(ch)) { + if (ch == 'N') { + if (charAt(pos + 1) == '\'') { + ++pos; + ch = '\''; + scanString(); + token = Token.LITERAL_NCHARS; + return; + } + } + + scanIdentifier(); + return; + } + + switch (ch) { + case '0': + if (charAt(pos + 1) == 'x') { + scanChar(); + scanChar(); + scanHexaDecimal(); + } else { + scanNumber(); + } + return; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + scanNumber(); + return; + case ',': + case '，': + scanChar(); + token = COMMA; + return; + case '(': + scanChar(); + token = LPAREN; + return; + case ')': + scanChar(); + token = RPAREN; + return; + case '[': + scanLBracket(); + return; + case ']': + scanChar(); + token = RBRACKET; + return; + case '{': + scanChar(); + token = LBRACE; + return; + case '}': + scanChar(); + token = RBRACE; + return; + case ':': + scanChar(); + if (ch == '=') { + scanChar(); + token = COLONEQ; + } else if (ch == ':') { + scanChar(); + token = COLONCOLON; + } else { + if (isDigit(ch)) { + unscan(); + scanVariable(); + } else { + unscan(); + scanVariable(); + } + } + return; + case '#': + scanSharp(); + if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) { + bufPos = 0; + continue; + } + return; + case '.': + scanChar(); + if (isDigit(ch)) { + unscan(); + scanNumber(); + return; + } else if (ch == '.') { + scanChar(); + if (ch == '.') { + scanChar(); + token = Token.DOTDOTDOT; + } else { + token = Token.DOTDOT; + } + } else { + token = Token.DOT; + } + return; + case '\'': + scanString(); + return; + case '\"': + scanAlias(); + return; + case '*': + scanChar(); + token = Token.STAR; + return; + case '?': + scanChar(); + token = Token.QUES; + return; + case ';': + scanChar(); + token = Token.SEMI; + return; + case '`': + throw new ParserException("TODO"); // TODO + case '@': + scanVariable(); + return; + case '-': + int subNextChar = charAt(pos + 1); + if (subNextChar == '-') { + scanComment(); + if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) { + bufPos = 0; + continue; + } + } else { + scanOperator(); + } + return; + case '/': + int nextChar = charAt(pos + 1); + if (nextChar == '/' || nextChar == '*') { + scanComment(); + if ((token() == Token.LINE_COMMENT || token() == Token.MULTI_LINE_COMMENT) && skipComment) { + bufPos = 0; + continue; + } + } else { + token = Token.SLASH; + scanChar(); + } + return; + default: + if (Character.isLetter(ch)) { + scanIdentifier(); + return; + } + + if (isOperator(ch)) { + scanOperator(); + return; + } + + // QS_TODO ? + if (isEOF()) { // JLS + token = EOF; + } else { + lexError("illegal.char", String.valueOf((int) ch)); + scanChar(); + } + + return; + } + } + + } + + protected void scanLBracket() { + scanChar(); + token = LBRACKET; + } + + private final void scanOperator() { + switch (ch) { + case '+': + scanChar(); + token = Token.PLUS; + break; + case '-': + scanChar(); + token = Token.SUB; + break; + case '*': + scanChar(); + token = Token.STAR; + break; + case '/': + scanChar(); + token = Token.SLASH; + break; + case '&': + scanChar(); + if (ch == '&') { + scanChar(); + token = Token.AMPAMP; + } else { + token = Token.AMP; + } + break; + case '|': + scanChar(); + if (ch == '|') { + scanChar(); + token = Token.BARBAR; + } else { + token = Token.BAR; + } + break; + case '^': + scanChar(); + token = Token.CARET; + break; + case '%': + scanChar(); + token = Token.PERCENT; + break; + case '=': + scanChar(); + if (ch == '=') { + scanChar(); + token = Token.EQEQ; + } else { + token = Token.EQ; + } + break; + case '>': + scanChar(); + if (ch == '=') { + scanChar(); + token = Token.GTEQ; + } else if (ch == '>') { + scanChar(); + token = Token.GTGT; + } else { + token = Token.GT; + } + break; + case '<': + scanChar(); + if (ch == '=') { + scanChar(); + if (ch == '>') { + token = Token.LTEQGT; + scanChar(); + } else { + token = Token.LTEQ; + } + } else if (ch == '>') { + scanChar(); + token = Token.LTGT; + } else if (ch == '<') { + scanChar(); + token = Token.LTLT; + } else { + token = Token.LT; + } + break; + case '!': + scanChar(); + if (ch == '=') { + scanChar(); + token = Token.BANGEQ; + } else if (ch == '>') { + scanChar(); + token = Token.BANGGT; + } else if (ch == '<') { + scanChar(); + token = Token.BANGLT; + } else { + token = Token.BANG; + } + break; + case '?': + scanChar(); + token = Token.QUES; + break; + case '~': + scanChar(); + token = Token.TILDE; + break; + default: + throw new ParserException("TODO"); + } + } + + protected void scanString() { + mark = pos; + boolean hasSpecial = false; + + for (;;) { + if (isEOF()) { + lexError("unclosed.str.lit"); + return; + } + + ch = charAt(++pos); + + if (ch == '\'') { + scanChar(); + if (ch != '\'') { + token = LITERAL_CHARS; + break; + } else { + if (!hasSpecial) { + initBuff(bufPos); + arraycopy(mark + 1, buf, 0, bufPos); + hasSpecial = true; + } + putChar('\''); + continue; + } + } + + if (!hasSpecial) { + bufPos++; + continue; + } + + if (bufPos == buf.length) { + putChar(ch); + } else { + buf[bufPos++] = ch; + } + } + + if (!hasSpecial) { + stringVal = subString(mark + 1, bufPos); + } else { + stringVal = new String(buf, 0, bufPos); + } + } + + private final void scanAlias() { + mark = pos; + + if (buf == null) { + buf = new char[32]; + } + + for (;;) { + if (isEOF()) { + lexError("unclosed.str.lit"); + return; + } + + ch = charAt(++pos); + + if (ch == '\"') { + scanChar(); + token = LITERAL_ALIAS; + break; + } + + if (bufPos == buf.length) { + putChar(ch); + } else { + buf[bufPos++] = ch; + } + } + + stringVal = subString(mark + 1, bufPos); + } + + public void scanSharp() { + scanVariable(); + } + + public void scanVariable() { + if (ch != '@' && ch != ':' && ch != '#' && ch != '$') { + throw new ParserException("illegal variable"); + } + + mark = pos; + bufPos = 1; + char ch; + + boolean mybatisFlag = false; + if (charAt(pos + 1) == '@') { + ch = charAt(++pos); + + bufPos++; + } else if (charAt(pos + 1) == '{') { + pos++; + bufPos++; + mybatisFlag = true; + } + + for (;;) { + ch = charAt(++pos); + + if (!isIdentifierChar(ch)) { + break; + } + + bufPos++; + continue; + } + + if (mybatisFlag) { + if (ch != '}') { + throw new ParserException("syntax error"); + } + ++pos; + bufPos++; + } + + this.ch = charAt(pos); + + stringVal = addSymbol(); + token = Token.VARIANT; + } + + public void scanComment() { + if (!allowComment) { + throw new NotAllowCommentException(); + } + + if (ch != '/') { + throw new IllegalStateException(); + } + + mark = pos; + bufPos = 0; + scanChar(); + + if (ch == '*') { + scanChar(); + bufPos++; + + for (;;) { + if (ch == '*' && charAt(pos + 1) == '/') { + bufPos += 2; + scanChar(); + scanChar(); + break; + } + + scanChar(); + bufPos++; + } + + stringVal = subString(mark, bufPos); + token = Token.MULTI_LINE_COMMENT; + return; + } + + if (ch == '/') { + scanChar(); + bufPos++; + + for (;;) { + if (ch == '\r') { + if (charAt(pos + 1) == '\n') { + bufPos += 2; + scanChar(); + break; + } + bufPos++; + break; + } + + if (ch == '\n') { + scanChar(); + bufPos++; + break; + } + + scanChar(); + bufPos++; + } + + stringVal = subString(mark + 1, bufPos); + token = Token.LINE_COMMENT; + return; + } + } + + public void scanIdentifier() { + final char first = ch; + + final boolean firstFlag = isFirstIdentifierChar(first); + if (!firstFlag) { + throw new ParserException("illegal identifier"); + } + + mark = pos; + bufPos = 1; + char ch; + for (;;) { + ch = charAt(++pos); + + if (!isIdentifierChar(ch)) { + break; + } + + bufPos++; + continue; + } + + this.ch = charAt(pos); + + stringVal = addSymbol(); + Token tok = keywods.getKeyword(stringVal); + if (tok != null) { + token = tok; + } else { + token = Token.IDENTIFIER; + } + } + + public void scanNumber() { + mark = pos; + + if (ch == '-') { + bufPos++; + ch = charAt(++pos); + } + + for (;;) { + if (ch >= '0' && ch <= '9') { + bufPos++; + } else { + break; + } + ch = charAt(++pos); + } + + boolean isDouble = false; + + if (ch == '.') { + if (charAt(pos + 1) == '.') { + token = Token.LITERAL_INT; + return; + } + bufPos++; + ch = charAt(++pos); + isDouble = true; + + for (;;) { + if (ch >= '0' && ch <= '9') { + bufPos++; + } else { + break; + } + ch = charAt(++pos); + } + } + + if (ch == 'e' || ch == 'E') { + bufPos++; + ch = charAt(++pos); + + if (ch == '+' || ch == '-') { + bufPos++; + ch = charAt(++pos); + } + + for (;;) { + if (ch >= '0' && ch <= '9') { + bufPos++; + } else { + break; + } + ch = charAt(++pos); + } + + isDouble = true; + } + + if (isDouble) { + token = Token.LITERAL_FLOAT; + } else { + token = Token.LITERAL_INT; + } + } + + public void scanHexaDecimal() { + mark = pos; + + if (ch == '-') { + bufPos++; + ch = charAt(++pos); + } + + for (;;) { + if (CharTypes.isHex(ch)) { + bufPos++; + } else { + break; + } + ch = charAt(++pos); + } + + token = Token.LITERAL_HEX; + } + + public String hexString() { + return subString(mark, bufPos); + } + + public final boolean isDigit(char ch) { + return ch >= '0' && ch <= '9'; + } + + /** + * Append a character to sbuf. + */ + protected final void putChar(char ch) { + if (bufPos == buf.length) { + char[] newsbuf = new char[buf.length * 2]; + System.arraycopy(buf, 0, newsbuf, 0, buf.length); + buf = newsbuf; + } + buf[bufPos++] = ch; + } + + /** + * Return the current token's position: a 0-based offset from beginning of the raw input stream (before unicode + * translation) + */ + public final int pos() { + return pos; + } + + /** + * The value of a literal token, recorded as a string. For integers, leading 0x and 'l' suffixes are suppressed. + */ + public final String stringVal() { + return stringVal; + } + + private boolean isOperator(char ch) { + switch (ch) { + case '!': + case '%': + case '&': + case '*': + case '+': + case '-': + case '<': + case '=': + case '>': + case '^': + case '|': + case '~': + case ';': + return true; + default: + return false; + } + } + + private static final long MULTMIN_RADIX_TEN = Long.MIN_VALUE / 10; + private static final long N_MULTMAX_RADIX_TEN = -Long.MAX_VALUE / 10; + + private final static int[] digits = new int[(int) '9' + 1]; + + static { + for (int i = '0'; i <= '9'; ++i) { + digits[i] = i - '0'; + } + } + + // QS_TODO negative number is invisible for lexer + public Number integerValue() { + long result = 0; + boolean negative = false; + int i = mark, max = mark + bufPos; + long limit; + long multmin; + int digit; + + if (charAt(mark) == '-') { + negative = true; + limit = Long.MIN_VALUE; + i++; + } else { + limit = -Long.MAX_VALUE; + } + multmin = negative ? MULTMIN_RADIX_TEN : N_MULTMAX_RADIX_TEN; + if (i < max) { + digit = digits[charAt(i++)]; + result = -digit; + } + while (i < max) { + // Accumulating negatively avoids surprises near MAX_VALUE + digit = digits[charAt(i++)]; + if (result < multmin) { + return new BigInteger(numberString()); + } + result *= 10; + if (result < limit + digit) { + return new BigInteger(numberString()); + } + result -= digit; + } + + if (negative) { + if (i > mark + 1) { + if (result >= Integer.MIN_VALUE) { + return (int) result; + } + return result; + } else { /* Only got "-" */ + throw new NumberFormatException(numberString()); + } + } else { + result = -result; + if (result <= Integer.MAX_VALUE) { + return (int) result; + } + return result; + } + } + + public int bp() { + return this.pos; + } + + public char current() { + return this.ch; + } + + public void reset(int mark, char markChar, Token token) { + this.pos = mark; + this.ch = markChar; + this.token = token; + } + + public final String numberString() { + return subString(mark, bufPos); + } + + public BigDecimal decimalValue() { + return new BigDecimal(text.toCharArray(), mark, bufPos); + } + + public static interface CommentHandler { + + boolean handle(Token lastToken, String comment); + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest45.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest45.java index 4853e48a97..1ce8bea5fb 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest45.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/OracleSelectTest45.java @@ -29,7 +29,7 @@ public class OracleSelectTest45 extends OracleTest { public void test_0() throws Exception { String sql = // - "SELECT T1.BC_ID AS BCID,SUM(PRODUCT_NUM) AS COUNT " // + "SELECT T1.BC_ID AS BCID，SUM(PRODUCT_NUM) AS COUNT " // + "　　FROM MT_PRODUCT_ORDER T1 ,MT_ORDER T2 " // + "　　WHERE T1.MT_ORDER_ID= T2.MT_ORDER_ID " // + "AND T2.PUBLISH_TIME>= ? " //