improved sql parser.

2023-03-04 19:17:16 +00:00 · 2017-09-17 17:11:26 +08:00
parent b54a987cc8
commit 3c98a1eee5
23 changed files with 955 additions and 31 deletions
@@ -446,7 +446,7 @@ public class SQLTransformUtils {
                int precision, scale = 0;
                if (arg0 instanceof SQLAllColumnExpr) {
                    precision = 19;
-                    scale = 0;
+                    scale = -1;
                } else {
                    precision = ((SQLIntegerExpr) arg0).getNumber().intValue();
                }
@@ -476,6 +476,8 @@ public class SQLTransformUtils {
                    } else {
                        dataType = new SQLDataTypeImpl(SQLDataType.Constants.DECIMAL, precision);
                    }
+                } else if (scale == -1) {
+                    dataType = new SQLDataTypeImpl(SQLDataType.Constants.DOUBLE_PRECISION);
                } else {
                    dataType = new SQLDataTypeImpl(SQLDataType.Constants.DECIMAL, precision, scale);
                }
@@ -622,4 +624,13 @@ public class SQLTransformUtils {

        return dataType;
    }
+
+    public static SQLExpr transformOracleToPostgresql(SQLMethodInvokeExpr x) {
+        long nameHashCode64 = x.methodNameHashCode64();
+        if (nameHashCode64 == FnvHash.Constants.SYS_GUID) {
+            SQLMethodInvokeExpr uuid_generate_v4 = new SQLMethodInvokeExpr("uuid_generate_v4");
+            return uuid_generate_v4;
+        }
+        return x;
+    }
 }
@@ -41,6 +41,7 @@ public interface SQLDataType extends SQLObject {
        String CHAR = "CHAR";
        String NCHAR = "NCHAR";
        String VARCHAR = "VARCHAR";
+        String DATE = "DATE";
        String TIMESTAMP = "TIMESTAMP";
        String XML = "XML";

@@ -26,11 +26,12 @@ import com.alibaba.druid.sql.ast.SQLExprImpl;
 import com.alibaba.druid.sql.ast.SQLReplaceable;
 import com.alibaba.druid.sql.dialect.oracle.visitor.OracleASTVisitor;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
+import com.alibaba.druid.util.FnvHash;

 public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable, Serializable {

    private static final long   serialVersionUID = 1L;
-    private String              methodName;
+    private String              name;
    private SQLExpr             owner;
    private final List<SQLExpr> parameters       = new ArrayList<SQLExpr>();

@@ -40,34 +41,50 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,

    private String              trimOption;

+    private long                nameHashCode64;
+
    public SQLMethodInvokeExpr(){

    }

    public SQLMethodInvokeExpr(String methodName){
-        this.methodName = methodName;
+        this.name = methodName;
+    }
+
+    public SQLMethodInvokeExpr(String methodName, long nameHashCode64){
+        this.name = methodName;
+        this.nameHashCode64 = nameHashCode64;
    }

    public SQLMethodInvokeExpr(String methodName, SQLExpr owner){

-        this.methodName = methodName;
+        this.name = methodName;
        setOwner(owner);
    }

    public SQLMethodInvokeExpr(String methodName, SQLExpr owner, SQLExpr... params){
-        this.methodName = methodName;
+        this.name = methodName;
        setOwner(owner);
        for (SQLExpr param : params) {
            this.addParameter(param);
        }
    }

+    public long methodNameHashCode64() {
+        if (nameHashCode64 == 0
+                && name != null) {
+            nameHashCode64 = FnvHash.hashCode64(name);
+        }
+        return nameHashCode64;
+    }
+
    public String getMethodName() {
-        return this.methodName;
+        return this.name;
    }

    public void setMethodName(String methodName) {
-        this.methodName = methodName;
+        this.name = methodName;
+        this.nameHashCode64 = 0L;
    }

    public SQLExpr getOwner() {
@@ -106,7 +123,7 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,
            buf.append(".");
        }

-        buf.append(this.methodName);
+        buf.append(this.name);
        buf.append("(");
        for (int i = 0, size = this.parameters.size(); i < size; ++i) {
            if (i != 0) {
@@ -144,7 +161,7 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,

        SQLMethodInvokeExpr that = (SQLMethodInvokeExpr) o;

-        if (methodName != null ? !methodName.equals(that.methodName) : that.methodName != null) return false;
+        if (name != null ? !name.equals(that.name) : that.name != null) return false;
        if (owner != null ? !owner.equals(that.owner) : that.owner != null) return false;
        if (parameters != null ? !parameters.equals(that.parameters) : that.parameters != null) return false;
        return from != null ? from.equals(that.from) : that.from == null;
@@ -153,7 +170,7 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,

    @Override
    public int hashCode() {
-        int result = methodName != null ? methodName.hashCode() : 0;
+        int result = name != null ? name.hashCode() : 0;
        result = 31 * result + (owner != null ? owner.hashCode() : 0);
        result = 31 * result + (parameters != null ? parameters.hashCode() : 0);
        result = 31 * result + (from != null ? from.hashCode() : 0);
@@ -163,7 +180,7 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,
    public SQLMethodInvokeExpr clone() {
        SQLMethodInvokeExpr x = new SQLMethodInvokeExpr();

-        x.methodName = methodName;
+        x.name = name;

        if (owner != null) {
            x.setOwner(owner.clone());
@@ -204,7 +221,7 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,
            return false;
        }

-        if (!SQLUtils.nameEquals(function, methodName)) {
+        if (!SQLUtils.nameEquals(function, name)) {
            return false;
        }

@@ -224,19 +241,19 @@ public class SQLMethodInvokeExpr extends SQLExprImpl implements SQLReplaceable,
    }

    public SQLDataType computeDataType() {
-        if (SQLUtils.nameEquals("to_date", methodName)
-                || SQLUtils.nameEquals("add_months", methodName)) {
+        if (SQLUtils.nameEquals("to_date", name)
+                || SQLUtils.nameEquals("add_months", name)) {
            return SQLDateExpr.DEFAULT_DATA_TYPE;
        }

        if (parameters.size() == 1) {
-            if (SQLUtils.nameEquals("trunc", methodName)) {
+            if (SQLUtils.nameEquals("trunc", name)) {
                return parameters.get(0).computeDataType();
            }
        } else if (parameters.size() == 2) {
            SQLExpr param0 = parameters.get(0);
            SQLExpr param1 = parameters.get(1);
-            if (SQLUtils.nameEquals("nvl", methodName) || SQLUtils.nameEquals("ifnull", methodName)) {
+            if (SQLUtils.nameEquals("nvl", name) || SQLUtils.nameEquals("ifnull", name)) {
                SQLDataType dataType = param0.computeDataType();
                if (dataType != null) {
                    return dataType;
@@ -24,7 +24,7 @@ import java.io.IOException;
 import java.math.BigDecimal;

 public class SQLNumberExpr extends SQLNumericLiteralExpr implements SQLValuableExpr {
-    private static SQLDataType defaultDataType = new SQLDataTypeImpl("number");
+    public final static SQLDataType defaultDataType = new SQLDataTypeImpl("number");

    private Number number;

@@ -26,7 +26,7 @@ import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 import com.alibaba.druid.util.JdbcConstants;

-public class SQLColumnDefinition extends SQLObjectImpl implements SQLTableElement, SQLObjectWithDataType {
+public class SQLColumnDefinition extends SQLObjectImpl implements SQLTableElement, SQLObjectWithDataType, SQLReplaceable {
    protected String                          dbType;

    protected SQLName                         name;
@@ -250,6 +250,21 @@ public class SQLColumnDefinition extends SQLObjectImpl implements SQLTableElemen
        this.storage = storage;
    }

+    @Override
+    public boolean replace(SQLExpr expr, SQLExpr target) {
+        if (defaultExpr == expr) {
+            setDefaultExpr(target);
+            return true;
+        }
+
+        if (name == expr) {
+            setName((SQLName) target);
+            return true;
+        }
+
+        return false;
+    }
+
    public static class Identity extends SQLObjectImpl {

        private Integer seed;
@@ -124,6 +124,12 @@ public class MySqlExprParser extends SQLExprParser {
                lexer.reset(6, ' ', Token.UPDATE);
                return;
            }
+
+            if (c0 == '/' && c1 == '*' && isEnabled(SQLParserFeature.OptimizedForParameterized)) {
+                MySqlLexer mySqlLexer = (MySqlLexer) lexer;
+                mySqlLexer.skipFirstHintsOrMultiCommentAndNextToken();
+                return;
+            }
        }
        this.lexer.nextToken();

@@ -401,6 +401,89 @@ public class MySqlLexer extends Lexer {
        scanString2();
    }

+    public void skipFirstHintsOrMultiCommentAndNextToken() {
+        int starIndex = pos + 2;
+
+        for (;;) {
+            starIndex = text.indexOf('*', starIndex);
+            if (starIndex == -1 || starIndex == text.length() - 1) {
+                this.token = Token.ERROR;
+                return;
+            }
+
+            int slashIndex = starIndex + 1;
+            if (charAt(slashIndex) == '/') {
+                pos = slashIndex + 1;
+                ch = text.charAt(pos);
+                if (pos < text.length() - 6) {
+                    int pos_6 = pos + 6;
+                    char c0 = ch;
+                    char c1 = text.charAt(pos + 1);
+                    char c2 = text.charAt(pos + 2);
+                    char c3 = text.charAt(pos + 3);
+                    char c4 = text.charAt(pos + 4);
+                    char c5 = text.charAt(pos + 5);
+                    char c6 = text.charAt(pos_6);
+                    if (c0 == 's' && c1 == 'e' && c2 == 'l' && c3 == 'e' && c4 == 'c' && c5 == 't' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.SELECT);
+                        return;
+                    }
+
+                    if (c0 == 'i' && c1 == 'n' && c2 == 's' && c3 == 'e' && c4 == 'r' && c5 == 't' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.INSERT);
+                        return;
+                    }
+
+                    if (c0 == 'u' && c1 == 'p' && c2 == 'd' && c3 == 'a' && c4 == 't' && c5 == 'e' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.UPDATE);
+                        return;
+                    }
+
+
+                    if (c0 == 'd' && c1 == 'e' && c2 == 'l' && c3 == 'e' && c4 == 't' && c5 == 'e' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.DELETE);
+                        return;
+                    }
+
+                    if (c0 == 'S' && c1 == 'E' && c2 == 'L' && c3 == 'E' && c4 == 'C' && c5 == 'T' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.SELECT);
+                        return;
+                    }
+
+                    if (c0 == 'I' && c1 == 'N' && c2 == 'S' && c3 == 'E' && c4 == 'R' && c5 == 'T' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.INSERT);
+                        return;
+                    }
+
+                    if (c0 == 'U' && c1 == 'P' && c2 == 'D' && c3 == 'A' && c4 == 'T' && c5 == 'E' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.UPDATE);
+                        return;
+                    }
+
+                    if (c0 == 'D' && c1 == 'E' && c2 == 'L' && c3 == 'E' && c4 == 'T' && c5 == 'E' && c6 == ' ') {
+                        this.comments = null;
+                        reset(pos_6, ' ', Token.DELETE);
+                        return;
+                    }
+
+                    nextToken();
+                    return;
+                } else {
+                    nextToken();
+                    return;
+                }
+            }
+            starIndex++;
+        }
+    }
+
    public void scanComment() {
        Token lastToken = this.token;
        
@@ -60,7 +60,7 @@ public class Lexer {
    protected long         hash;

    protected int            commentCount = 0;
-    protected List<String>   comments     = new ArrayList<String>(2);
+    protected List<String>   comments     = null;
    protected boolean        skipComment  = true;
    private SavePoint        savePoint    = null;

@@ -913,7 +913,7 @@ public class SQLExprParser extends SQLParser {
        SQLExpr owner = null;
        String trimOption = null;

-        long hash_lower;
+        long hash_lower = 0L;
        if (expr instanceof SQLIdentifierExpr) {
            SQLIdentifierExpr identifierExpr = (SQLIdentifierExpr) expr;
            methodName = identifierExpr.getName();
@@ -964,7 +964,7 @@ public class SQLExprParser extends SQLParser {
            return aggregateExpr;
        }

-        methodInvokeExpr = new SQLMethodInvokeExpr(methodName);
+        methodInvokeExpr = new SQLMethodInvokeExpr(methodName, hash_lower);
        if (owner != null) {
            methodInvokeExpr.setOwner(owner);
        }
@@ -565,10 +565,16 @@ public class SQLSelectParser extends SQLParser {

    protected SQLTableSource parseTableSourceRest(SQLTableSource tableSource) {
        if ((tableSource.getAlias() == null) || (tableSource.getAlias().length() == 0)) {
-            if (lexer.token != Token.LEFT && lexer.token != Token.RIGHT && lexer.token != Token.FULL
-                    && (!lexer.identifierEquals(FnvHash.Constants.STRAIGHT_JOIN))
-                    && (!lexer.identifierEquals(FnvHash.Constants.CROSS))
-                    && lexer.token != Token.OUTER) {
+            Token token = lexer.token;
+            long hash;
+            if (token != Token.LEFT
+                    && token != Token.RIGHT
+                    && token != Token.FULL
+                    && token != Token.OUTER
+                    && !(token == Token.IDENTIFIER
+                        && ((hash = lexer.hash_lower()) == FnvHash.Constants.STRAIGHT_JOIN
+                            || hash == FnvHash.Constants.CROSS)))
+            {
                String alias = tableAlias();
                if (alias != null) {
                    tableSource.setAlias(alias);
@@ -15,10 +15,14 @@
 */
 package com.alibaba.druid.sql.parser;

+import java.nio.charset.Charset;
+
 /**
 * @author wenshao[szujobs@hotmail.com]
 */
 public class SymbolTable {
+    private static final Charset UTF8 = Charset.forName("UTF-8");
+
    public static SymbolTable global = new SymbolTable(8192);

    private final Entry[] entries;
@@ -29,15 +33,12 @@ public class SymbolTable {
        this.entries = new Entry[tableSize];
    }

-
-
    public String addSymbol(String buffer, int offset, int len, long hash) {
        final int bucket = ((int) hash) & indexMask;

        Entry entry = entries[bucket];
        if (entry != null) {
-            if (hash == entry.hash //
-                    && len == entry.len) {
+            if (hash == entry.hash) {
                return entry.value;
            }

@@ -52,12 +53,64 @@ public class SymbolTable {
        return str;
    }

+    public String addSymbol(byte[] buffer, int offset, int len, long hash) {
+        final int bucket = ((int) hash) & indexMask;
+
+        Entry entry = entries[bucket];
+        if (entry != null) {
+            if (hash == entry.hash) {
+                return entry.value;
+            }
+
+            String str = subString(buffer, offset, len);
+
+            return str;
+        }
+
+        String str = subString(buffer, offset, len);
+        entry = new Entry(hash, len, str);
+        entries[bucket] = entry;
+        return str;
+    }
+
+    public String addSymbol(String symbol, long hash) {
+        final int bucket = ((int) hash) & indexMask;
+
+        Entry entry = entries[bucket];
+        if (entry != null) {
+            if (hash == entry.hash) {
+                return entry.value;
+            }
+
+            return symbol;
+        }
+
+        entry = new Entry(hash, symbol.length(), symbol);
+        entries[bucket] = entry;
+        return symbol;
+    }
+
+    public String findSymbol(long hash) {
+        final int bucket = ((int) hash) & indexMask;
+        Entry entry = entries[bucket];
+        if (entry != null && entry.hash == hash) {
+            return entry.value;
+        }
+        return null;
+    }
+
    private static String subString(String src, int offset, int len) {
        char[] chars = new char[len];
        src.getChars(offset, offset + len, chars, 0);
        return new String(chars);
    }

+    private static String subString(byte[] bytes, int from, int len) {
+        byte[] strBytes = new byte[len];
+        System.arraycopy(bytes, from, strBytes, 0, len);
+        return new String(strBytes, UTF8);
+    }
+
    private static class Entry {
        public final long hash;
        public final int len;
@@ -65,7 +65,7 @@ public class ParameterizedOutputVisitorUtils {
        }

        if (visitor.getReplaceCount() == 0
-                && parser.getLexer().getCommentCount() == 0) {
+                && parser.getLexer().getCommentCount() == 0 && sql.charAt(0) != '/') {
            return sql;
        }

@@ -55,6 +55,25 @@ public final class FnvHash {
        return hash;
    }

+    public static long fnv1a_64(byte[] input, int offset, int end) {
+        if (input == null) {
+            return 0;
+        }
+
+        if (input.length < end) {
+            end = input.length;
+        }
+
+        long hash = BASIC;
+        for (int i = offset; i < end; ++i) {
+            byte c = input[i];
+            hash ^= c;
+            hash *= PRIME;
+        }
+
+        return hash;
+    }
+
    public static long fnv1a_64(char[] chars) {
        if (chars == null) {
            return 0;
@@ -468,5 +487,13 @@ public final class FnvHash {
        long SOURCE = fnv1a_64_lower("SOURCE");

        long STAR = fnv1a_64_lower("*");
+
+        long TO_CHAR = fnv1a_64_lower("TO_CHAR");
+        long SYS_GUID = fnv1a_64_lower("SYS_GUID");
+
+        long STATISTICS = fnv1a_64_lower("STATISTICS");
+        long TRANSACTION = fnv1a_64_lower("TRANSACTION");
+        long OFF = fnv1a_64_lower("OFF");
+        long IDENTITY_INSERT = fnv1a_64_lower("IDENTITY_INSERT");
    }
 }
@@ -0,0 +1,50 @@
+package com.alibaba.druid.bvt.sql;
+
+import com.alibaba.druid.sql.parser.SymbolTable;
+import com.alibaba.druid.util.FnvHash;
+import junit.framework.TestCase;
+
+public class SymbolTableTest extends TestCase {
+    public void test_symbols() throws Exception {
+        SymbolTable symbols = new SymbolTable(65535);
+
+        String[] strings = new String[10];
+        for (int i = 0; i < strings.length; ++i) {
+            String str = "abc" + i;
+            strings[i] = str;
+            symbols.addSymbol(str, FnvHash.fnv1a_64(str));
+        }
+
+        for (int i = 0; i < strings.length; ++i) {
+            String str = strings[i];
+            long hash = FnvHash.fnv1a_64(str);
+            String symbol = symbols.findSymbol(hash);
+            assertSame(str, symbol);
+        }
+        {
+            byte[] bytes = "kkk#abc0#aa".getBytes();
+            long hash = FnvHash.fnv1a_64(bytes, 4, 8);
+            String symbol = symbols.addSymbol(bytes, 4, 8, hash);
+            assertSame(strings[0], symbol);
+        }
+
+        byte[] bytes = "xab#time:3333".getBytes();
+        System.out.println(indexOfTime(bytes, 1));
+        System.out.println("xab#time:3333".indexOf("#time"));
+    }
+
+    public static int indexOfTime(byte[] bytes, int fromIndex) {
+        int end = bytes.length - 5;
+        for (int i = fromIndex; i < end; ++i) {
+            if (bytes[i] == '#'
+                    && bytes[i + 1] == 't'
+                    && bytes[i + 2] == 'i'
+                    && bytes[i + 3] == 'm'
+                    && bytes[i + 4] == 'e') {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+}
@@ -0,0 +1,346 @@
+package com.alibaba.druid.bvt.sql;
+
+import com.alibaba.druid.sql.parser.SymbolTable;
+import com.alibaba.druid.util.FnvHash;
+
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Created by yunning on 16/7/10.
+ */
+public class TestNewSpout {
+
+
+    public static int indexOf_nsharp(byte[] bytes, int fromIndex) {
+        int end = bytes.length - 2;
+        for (int i = fromIndex; i < end; ++i) {
+            if (bytes[i] == '\n'
+                    && bytes[i + 1] == '#') {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    public static int indexOf_nReturn(byte[] bytes, int fromIndex) {
+        for (int i = fromIndex; i < bytes.length; ++i) {
+            if (bytes[i] == '\n') {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    public static int indexOfTime(byte[] bytes, int fromIndex) {
+        int end = bytes.length - 5;
+        for (int i = fromIndex; i < end; ++i) {
+            if (bytes[i] == '#'
+                    && bytes[i + 1] == 't'
+                    && bytes[i + 2] == 'i'
+                    && bytes[i + 3] == 'm'
+                    && bytes[i + 4] == 'e') {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    public static final SymbolTable dbSymbols = new SymbolTable(1024 * 256);
+
+    public static String substring(byte[] bytes, int from, int to) {
+        int newLength = to - from;
+        byte[] copy = new byte[newLength];
+        System.arraycopy(bytes, from, copy, 0, newLength);
+        return new String(copy, Charset.forName("UTF-8"));
+    }
+
+    public static void main(String[] args) throws Exception {
+        final String datass = "#time:1470191976686205\n" +
+                "#user@host:monitor[monitor] @ [127.0.0.1]\n" +
+                "#db:test\n" +
+                "#table_name:aa\n" +
+                "set session time_zone='+08:00';\n" +
+                "#query_time:0.000028123456789987654321\n" +
+                "#lock_time:0.000000\n" +
+                "#rows_sent:0\n" +
+                "#rows_examined:0\n" +
+                "#rows_affected:0\n" +
+                "#innodb_pages_read:1\n" +
+                "#innodb_pages_io_read:2\n" +
+                "#id:55438\n" +
+                "#time:1470191976686207\n" +
+                "#user@host:monitor[monitor] @ [127.0.0.1]\n" +
+                "#db:test\n" +
+                "#table_name:aa\n" +
+                "#time:1470191976686206\n" +
+                "#user@host:monitor[monitor] @ [127.0.0.1]\n" +
+                "#db:test\n" +
+                "#table_name:aa\n" +
+                "set session time_zone='+0800';\n" +
+                "#query_time:0.000028123456789987654322\n" +
+                "#lock_time:0.000000\n" +
+                "#rows_sent:0\n" +
+                "#rows_examined:0\n" +
+                "#rows_affected:0\n" +
+                "#innodb_pages_read:0\n" +
+                "#innodb_pages_io_read:0\n" +
+                "#id:123";
+
+        byte[] datas = datass.getBytes();
+        long start = System.currentTimeMillis();
+        for (int i = 0; i < 1; i++) {
+            final int len = datas.length;
+            int offset = indexOfTime(datas, 0);
+            long now = System.nanoTime();
+            while (offset != -1) {
+                int rowOffset = offset;
+                int k = 0;
+                final int[] offsets = new int[12];
+                boolean isNoFirst = false;
+                while (rowOffset != -1 && k < 12) {
+                    if (isNoFirst && rowOffset + 5 <= len - 1) {
+                        if (datas[rowOffset + 2] == 't' &&
+                                datas[rowOffset + 3] == 'i' &&
+                                datas[rowOffset + 4] == 'm' &&
+                                datas[rowOffset + 5] == 'e') {
+                            break;
+                        }
+                    }
+                    offsets[k] = rowOffset;
+                    rowOffset = indexOf_nsharp(datas, rowOffset + 1);
+                    k = k + 1;
+                    isNoFirst = true;
+                }
+                offset = rowOffset == -1 ? -1 : indexOfTime(datas, rowOffset + 1);
+                if (k < 12) {
+                    continue;
+                }
+                SqlInfo sqlInfo = new SqlInfo();
+                //  sqlInfo.setHostName(hostName);
+
+                boolean isError = false;
+                String sql = null;
+
+                //time
+                int currItemOffset = offsets[0];
+                int nextItemOffset = offsets[1];
+                           /* String item = datas.substring(currItemOffset + 6, nextItemOffset);
+                            long itemTime = Long.parseLong(item);*/
+                long itemTime = MathUtil.str2Long(datas, currItemOffset + 6, nextItemOffset);
+                System.out.println("TIME : " + itemTime);
+                if (itemTime < 0 || itemTime > 2000000000000000L) {
+                }
+                sqlInfo.setTime(itemTime);
+
+                //user@host
+                currItemOffset = offsets[1];
+                nextItemOffset = offsets[2];
+                String item = substring(datas, currItemOffset + 12, nextItemOffset);
+                System.out.println("user@host : " + item);
+//                sqlInfo.setUserHost(item);
+
+                //db
+                currItemOffset = offsets[2];
+                nextItemOffset = offsets[3];
+                int dbOffset = currItemOffset + 5;
+                int dbLen = nextItemOffset - dbOffset;
+                long dbHash = FnvHash.fnv1a_64(datas, dbOffset, nextItemOffset);
+                String db = dbSymbols.addSymbol(datas, dbOffset, dbLen, dbHash);
+
+                //  String db = substring(datas, currItemOffset + 5, nextItemOffset);
+                System.out.println("db : " + db);
+                if (dbLen > 0 && SqlUtil.filterDbs.indexOf(db) != -1) {
+//                    sqlInfo.setNotValiDB(true);
+                } else {
+                }
+
+                //table_name
+                currItemOffset = offsets[3];
+                nextItemOffset = offsets[4];
+                int tsqlPos = currItemOffset + 13;
+                int idx2 = indexOf_nReturn(datas, tsqlPos);
+                String table = substring(datas, tsqlPos, idx2);
+                sql = substring(datas, idx2 + 1, nextItemOffset);
+                System.out.println("table : " + table + " : " + sql);
+
+                int totalLen = sql.length();
+                if (totalLen < 9 ||
+                        totalLen >= 8000) {
+//                    sqlInfo.setParseRes(0);
+                } else {
+                }
+
+                //query_time
+                currItemOffset = offsets[4];
+                nextItemOffset = offsets[5];
+                int rtBound = Math.min(nextItemOffset - currItemOffset, 24);
+                double itemRt = MathUtil.str2Double(datas, currItemOffset + 13, currItemOffset + rtBound);
+                sqlInfo.setQueryTime(itemRt);
+
+                //lock_time
+                currItemOffset = offsets[5];
+                nextItemOffset = offsets[6];
+                int lockTimeBound = Math.min(nextItemOffset - currItemOffset, 23);
+                double itemLockTime = MathUtil.str2Double(datas, currItemOffset + 12, currItemOffset + lockTimeBound);
+                sqlInfo.setLockTime(itemLockTime);
+
+                //rows_sent
+                currItemOffset = offsets[6];
+                nextItemOffset = offsets[7];
+                long itemRowsSent = MathUtil.str2Long(datas, currItemOffset + 12, nextItemOffset);
+                if (itemRowsSent < 0 || itemRowsSent > 1000000000) {
+                }
+//                sqlInfo.setRowsSent(itemRowsSent);
+
+                //rows_examined
+                currItemOffset = offsets[7];
+                nextItemOffset = offsets[8];
+                long itemRowsExam = MathUtil.str2Long(datas, currItemOffset + 16, nextItemOffset);
+                if (itemRowsExam < 0 || itemRowsExam > 1000000000) {
+                }
+//                sqlInfo.setRowsExamined(itemRowsExam);
+
+                //rows_affected
+                currItemOffset = offsets[8];
+                nextItemOffset = offsets[9];
+                long itemRowsAffected = MathUtil.str2Long(datas, currItemOffset + 16, nextItemOffset);
+                if (itemRowsAffected < 0 || itemRowsAffected > 1000000000) {
+                }
+                sqlInfo.setRowsAffected(itemRowsAffected);
+
+                //innodb_pages_read
+                currItemOffset = offsets[9];
+                nextItemOffset = offsets[10];
+                long itemPageRead = MathUtil.str2Long(datas, currItemOffset + 20, nextItemOffset);
+                if (itemPageRead < 0 || itemPageRead > 1000000000) {
+                }
+                sqlInfo.setInnodbPagesRead(itemPageRead);
+
+                //innodb_pages_io_read
+                currItemOffset = offsets[10];
+                nextItemOffset = offsets[11];
+                long itemPageIORead = MathUtil.str2Long(datas, currItemOffset + 23, nextItemOffset);
+                if (itemPageIORead < 0 || itemPageIORead > 1000000000) {
+                }
+//                sqlInfo.setInnodbPagesIoRead(itemPageIORead);
+                // currItemOffset = nextItemOffset;
+
+                            /*nextItemOffset = datas.indexOf("\n#", currItemOffset + 1);
+                            if (nextItemOffset >= end) {
+                                return null;
+                            }
+                            itemEnd = nextItemOffset == -1 ? end : nextItemOffset;
+                            item = datas.substring(currItemOffset + 1, itemEnd);
+                            sqlInfo.setId(Long.parseLong(item.trim().substring(4)));
+                            num++;*/
+                // sqlInfo.setId(0);
+            }
+
+        }
+
+        System.out.println(System.currentTimeMillis() - start);
+    }
+
+    static class SqlInfo {
+        private long innodbPagesRead;
+        private long time;
+        private double queryTime;
+        private double lockTime;
+        public long rowsAffected;
+
+        public long getInnodbPagesRead() {
+            return innodbPagesRead;
+        }
+
+        public void setInnodbPagesRead(long innodbPagesRead) {
+            this.innodbPagesRead = innodbPagesRead;
+        }
+
+        public long getTime() {
+            return time;
+        }
+
+        public void setTime(long time) {
+            this.time = time;
+        }
+
+        public void setQueryTime(double queryTime) {
+            this.queryTime = queryTime;
+        }
+
+        public void setLockTime(double lockTime) {
+            this.lockTime = lockTime;
+        }
+
+        public void setRowsAffected(long rowsAffected) {
+            this.rowsAffected = rowsAffected;
+        }
+    }
+
+    static class MathUtil {
+        public static long str2Long(byte[] data, int start, int end) {
+            long value = 0;
+            for (int i = start; i < end; ++i) {
+                byte ch = data[i];
+                if (ch == '.') break;
+                int digit = data[i] - '0';
+                value = value * 10 + digit;
+            }
+            return value;
+        }
+
+        public static double str2Double(byte[] data,int start,int end) {
+            long value = 0;
+            long power = 0;
+            for (int i = start; i < end; ++i) {
+                byte ch = data[i];
+                if (ch == '.') {
+                    power = 1;
+                    continue;
+                }
+                int digit = ch - '0';
+                value = value * 10 + digit;
+                power *= 10;
+            }
+            double doubleValue = (double) value;
+            if (power == 0) {
+                return doubleValue;
+            } else {
+                return doubleValue / power;
+            }
+        }
+    }
+
+    static class SqlUtil {
+
+        public static Set<String> filterDb = new HashSet<String>();
+
+        private static int[] orders = new int[]{2, 3, 0, 1};
+
+        static {
+            filterDb.add("DRC");
+            filterDb.add("INFORMATION_SCHEMA");
+            filterDb.add("INNODB_MEMCACHE");
+            filterDb.add("MYSQL");
+            filterDb.add("PERFORMANCE_SCHEMA");
+            filterDb.add("RECYCLE_BIN");
+            filterDb.add("ROCKSDB");
+            filterDb.add("TEST");
+            filterDb.add("DRC".toLowerCase());
+            filterDb.add("INFORMATION_SCHEMA".toLowerCase());
+            filterDb.add("INNODB_MEMCACHE".toLowerCase());
+            filterDb.add("MYSQL".toLowerCase());
+            filterDb.add("PERFORMANCE_SCHEMA".toLowerCase());
+            filterDb.add("RECYCLE_BIN".toLowerCase());
+            filterDb.add("ROCKSDB".toLowerCase());
+            filterDb.add("TEST".toLowerCase());
+        }
+
+        public static final String filterDbs = org.apache.commons.lang3.StringUtils.join(SqlUtil.filterDb, ",");
+    }
+}
@@ -0,0 +1,67 @@
+/*
+ * Copyright 1999-2017 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.mysql.select;
+
+import com.alibaba.druid.sql.MysqlTest;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.parser.SQLParserFeature;
+import com.alibaba.druid.sql.parser.SQLParserUtils;
+import com.alibaba.druid.sql.parser.SQLStatementParser;
+import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+import com.alibaba.druid.util.JdbcConstants;
+
+import java.util.List;
+
+public class MySqlSelectTest_60_firstComment extends MysqlTest {
+
+    public void test_0() throws Exception {
+        String sql =  "/* xxx * xxxx */SELECT * from t where t !  = 1";
+
+        System.out.println(sql);
+
+        SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, JdbcConstants.MYSQL, SQLParserFeature.OptimizedForParameterized);
+        List<SQLStatement> statementList = parser.parseStatementList();
+        SQLStatement stmt = statementList.get(0);
+
+        assertEquals(1, statementList.size());
+
+        SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.MYSQL);
+        stmt.accept(visitor);
+        {
+            String output = SQLUtils.toMySqlString(stmt);
+            assertEquals("SELECT *\n" +
+                            "FROM t\n" +
+                            "WHERE t != 1", //
+                                output);
+        }
+        {
+            String output = SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION);
+            assertEquals("select *\n" +
+                            "from t\n" +
+                            "where t != 1", //
+                                output);
+        }
+
+        {
+            String output = SQLUtils.toMySqlString(stmt, new SQLUtils.FormatOption(true, true, true));
+            assertEquals("SELECT *\n" +
+                            "FROM t\n" +
+                            "WHERE t != ?", //
+                    output);
+        }
+    }
+}
@@ -0,0 +1,67 @@
+/*
+ * Copyright 1999-2017 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.mysql.select;
+
+import com.alibaba.druid.sql.MysqlTest;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.parser.SQLParserFeature;
+import com.alibaba.druid.sql.parser.SQLParserUtils;
+import com.alibaba.druid.sql.parser.SQLStatementParser;
+import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+import com.alibaba.druid.util.JdbcConstants;
+
+import java.util.List;
+
+public class MySqlSelectTest_61_firstComment_2 extends MysqlTest {
+
+    public void test_0() throws Exception {
+        String sql =  "/* xxx * xxxx */ SELECT * from t where t !  = 1";
+
+        System.out.println(sql);
+
+        SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, JdbcConstants.MYSQL, SQLParserFeature.OptimizedForParameterized);
+        List<SQLStatement> statementList = parser.parseStatementList();
+        SQLStatement stmt = statementList.get(0);
+
+        assertEquals(1, statementList.size());
+
+        SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.MYSQL);
+        stmt.accept(visitor);
+        {
+            String output = SQLUtils.toMySqlString(stmt);
+            assertEquals("SELECT *\n" +
+                            "FROM t\n" +
+                            "WHERE t != 1", //
+                                output);
+        }
+        {
+            String output = SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION);
+            assertEquals("select *\n" +
+                            "from t\n" +
+                            "where t != 1", //
+                                output);
+        }
+
+        {
+            String output = SQLUtils.toMySqlString(stmt, new SQLUtils.FormatOption(true, true, true));
+            assertEquals("SELECT *\n" +
+                            "FROM t\n" +
+                            "WHERE t != ?", //
+                    output);
+        }
+    }
+}
@@ -0,0 +1,67 @@
+/*
+ * Copyright 1999-2017 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.mysql.select;
+
+import com.alibaba.druid.sql.MysqlTest;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.parser.SQLParserFeature;
+import com.alibaba.druid.sql.parser.SQLParserUtils;
+import com.alibaba.druid.sql.parser.SQLStatementParser;
+import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+import com.alibaba.druid.util.JdbcConstants;
+
+import java.util.List;
+
+public class MySqlSelectTest_62_optimized extends MysqlTest {
+
+    public void test_0() throws Exception {
+        String sql =  "SELECT * from t where t !  = 1";
+
+        System.out.println(sql);
+
+        SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, JdbcConstants.MYSQL, SQLParserFeature.OptimizedForParameterized);
+        List<SQLStatement> statementList = parser.parseStatementList();
+        SQLStatement stmt = statementList.get(0);
+
+        assertEquals(1, statementList.size());
+
+        SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.MYSQL);
+        stmt.accept(visitor);
+        {
+            String output = SQLUtils.toMySqlString(stmt);
+            assertEquals("SELECT *\n" +
+                            "FROM t\n" +
+                            "WHERE t != 1", //
+                                output);
+        }
+        {
+            String output = SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION);
+            assertEquals("select *\n" +
+                            "from t\n" +
+                            "where t != 1", //
+                                output);
+        }
+
+        {
+            String output = SQLUtils.toMySqlString(stmt, new SQLUtils.FormatOption(true, true, true));
+            assertEquals("SELECT *\n" +
+                            "FROM t\n" +
+                            "WHERE t != ?", //
+                    output);
+        }
+    }
+}
@@ -0,0 +1,74 @@
+/*
+ * Copyright 1999-2017 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.postgresql.select;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
+import com.alibaba.druid.util.JdbcConstants;
+import junit.framework.TestCase;
+
+import java.util.List;
+
+public class PGSelectTest57 extends TestCase {
+    private final String dbType = JdbcConstants.POSTGRESQL;
+
+    public void test_0() throws Exception {
+        String sql = "SELECT ts#, file#, block#, cols, IFNULL(size$, -1), pctfree$, pctused$\n" +
+                "  , initrans, maxtrans, hashkeys, func, extind\n" +
+                "  , avgchn, IFNULL(degree, 1)\n" +
+                "  , IFNULL(instances, 1)\n" +
+                "  , IFNULL(flags, 0)\n" +
+                "  , IFNULL(spare1, 0)\n" +
+                "FROM clu$\n" +
+                "WHERE obj# ='633'";
+
+        List<SQLStatement> stmtList = SQLUtils.parseStatements(sql, dbType);
+        SQLStatement stmt = stmtList.get(0);
+
+        assertEquals("SELECT ts#, file#, block#, cols\n" +
+                "\t, IFNULL(size$, -1), pctfree$\n" +
+                "\t, pctused$, initrans, maxtrans, hashkeys, func\n" +
+                "\t, extind, avgchn, IFNULL(degree, 1)\n" +
+                "\t, IFNULL(instances, 1)\n" +
+                "\t, IFNULL(flags, 0)\n" +
+                "\t, IFNULL(spare1, 0)\n" +
+                "FROM clu$\n" +
+                "WHERE obj# = '633'", SQLUtils.toPGString(stmt));
+        
+        assertEquals("select ts#, file#, block#, cols\n" +
+                "\t, IFNULL(size$, -1), pctfree$\n" +
+                "\t, pctused$, initrans, maxtrans, hashkeys, func\n" +
+                "\t, extind, avgchn, IFNULL(degree, 1)\n" +
+                "\t, IFNULL(instances, 1)\n" +
+                "\t, IFNULL(flags, 0)\n" +
+                "\t, IFNULL(spare1, 0)\n" +
+                "from clu$\n" +
+                "where obj# = '633'", SQLUtils.toPGString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
+
+        assertEquals(1, stmtList.size());
+
+        SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType);
+        stmt.accept(visitor);
+
+//        System.out.println("Tables : " + visitor.getTables());
+//        System.out.println("fields : " + visitor.getColumns());
+//        System.out.println("coditions : " + visitor.getConditions());
+
+        assertEquals(18, visitor.getColumns().size());
+        assertEquals(1, visitor.getTables().size());
+    }
+}
@@ -0,0 +1,18 @@
+package com.alibaba.druid.bvt.sql.transform.datatype.oracle2pg;
+
+import com.alibaba.druid.sql.SQLTransformUtils;
+import com.alibaba.druid.sql.ast.SQLDataType;
+import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
+import com.alibaba.druid.sql.parser.SQLParserUtils;
+import com.alibaba.druid.util.JdbcConstants;
+import junit.framework.TestCase;
+
+public class Oracle2MySql_method_sys_guid extends TestCase {
+    public void test_oracle2pg_int_1() throws Exception {
+        String sql = "sys_guid()";
+        SQLMethodInvokeExpr expr = (SQLMethodInvokeExpr) SQLParserUtils.createExprParser(sql, JdbcConstants.ORACLE).expr();
+        SQLExpr targetExpr = SQLTransformUtils.transformOracleToPostgresql(expr);
+        assertEquals("uuid_generate_v4()", targetExpr.toString());
+    }
+}
@@ -98,4 +98,12 @@ public class Oracle2PG_DataTypeTest_chars extends TestCase {
        SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
        assertEquals("TEXT", pgDataType.toString());
    }
+
+
+    public void test_oracle2pg_bfile() throws Exception {
+        String sql = "bfile";
+        SQLDataType dataType = SQLParserUtils.createExprParser(sql, JdbcConstants.ORACLE).parseDataType();
+        SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
+        assertEquals("VARCHAR(255)", pgDataType.toString());
+    }
 }
@@ -48,5 +48,13 @@ public class Oracle2PG_DataTypeTest_double extends TestCase {
        SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
        assertEquals("DOUBLE PRECISION", pgDataType.toString());
    }
+
+
+    public void test_oracle2pg_number_star() throws Exception {
+        String sql = "number(*)";
+        SQLDataType dataType = SQLParserUtils.createExprParser(sql, JdbcConstants.ORACLE).parseDataType();
+        SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
+        assertEquals("DOUBLE PRECISION", pgDataType.toString());
+    }
 }

@@ -158,7 +158,7 @@ public class Oracle2PG_DataTypeTest_number_int extends TestCase {
        String sql = "number(*)";
        SQLDataType dataType = SQLParserUtils.createExprParser(sql, JdbcConstants.ORACLE).parseDataType();
        SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
-        assertEquals("BIGINT", pgDataType.toString());
+        assertEquals("DOUBLE PRECISION", pgDataType.toString());
    }

    public void test_oracle2pg_int_21() throws Exception {