Merge pull request #238 from wenshao/master

improve sql parser & LogFilter

src/main/java/com/alibaba/druid/filter/logging/LogFilter.java

@@ -77,6 +77,18 @@ public abstract class LogFilter extends FilterEventAdapter implements LogFilterM
     protected DataSourceProxy dataSource;
 
     public LogFilter(){
+        {
+            String prop = System.getProperty("druid.log.stmt");
+            if (prop == "false") {
+                statementLogEnabled = false;
+            }
+        }
+        {
+            String prop = System.getProperty("druid.log.rs");
+            if (prop == "false") {
+                resultSetLogEnabled = false;
+            }
+        }
     }
 
     @Override

src/main/java/com/alibaba/druid/sql/ast/expr/SQLUnaryOperator.java

@@ -16,7 +16,15 @@
 package com.alibaba.druid.sql.ast.expr;
 
 public enum SQLUnaryOperator {
-    Plus("+"), Negative("-"), Not("!"), Compl("~"), Prior("PRIOR"), ConnectByRoot("CONNECT BY"), NOT("NOT");
+    Plus("+"), //
+    Negative("-"), //
+    Not("!"), //
+    Compl("~"), //
+    Prior("PRIOR"), //
+    ConnectByRoot("CONNECT BY"), //
+    BINARY("BINARY"), //
+    RAW("RAW"), //
+    NOT("NOT");
 
     public final String name;
 

src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java

@@ -24,6 +24,8 @@ import com.alibaba.druid.sql.ast.expr.SQLHexExpr;
 import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
 import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
 import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
+import com.alibaba.druid.sql.ast.expr.SQLUnaryExpr;
+import com.alibaba.druid.sql.ast.expr.SQLUnaryOperator;
 import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr;
 import com.alibaba.druid.sql.ast.statement.SQLAssignItem;
 import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
@@ -160,6 +162,10 @@ public class MySqlExprParser extends SQLExprParser {
                     throw new ParserException("syntax error, illegal values clause");
                 }
                 return this.methodRest(new SQLIdentifierExpr("VALUES"), true);
+            case BINARY:
+                lexer.nextToken();
+                SQLUnaryExpr binaryExpr = new SQLUnaryExpr(SQLUnaryOperator.BINARY, expr());
+                return primaryRest(binaryExpr);
             default:
                 return super.primary();
         }
@@ -204,15 +210,6 @@ public class MySqlExprParser extends SQLExprParser {
 
                     expr = mysqlCharExpr;
 
-                    return primaryRest(expr);
-                } else if (ident.equalsIgnoreCase("BINARY")) {
-                    String charValue = lexer.stringVal();
-                    lexer.nextToken();
-
-                    MySqlCharExpr mysqlCharExpr = new MySqlCharExpr(charValue);
-                    mysqlCharExpr.setCharset("BINARY");
-                    expr = mysqlCharExpr;
-
                     return primaryRest(expr);
                 }
             } else if (expr instanceof SQLCharExpr) {

src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java

@@ -46,6 +46,7 @@ public class MySqlLexer extends Lexer {
 
         map.put("LIMIT", Token.LIMIT);
         map.put("TRUE", Token.TRUE);
+        map.put("BINARY", Token.BINARY);
 
         DEFAULT_MYSQL_KEYWORDS = new Keywords(map);
     }

src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java

@@ -755,7 +755,7 @@ public class MySqlStatementParser extends SQLStatementParser {
             return new MySqlShowAuthorsStatement();
         }
 
-        if (identifierEquals("BINARY")) {
+        if (lexer.token() == Token.BINARY) {
             lexer.nextToken();
             acceptIdentifier("LOGS");
             return new MySqlShowBinaryLogsStatement();

src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java

@@ -449,6 +449,10 @@ public class SQLExprParser extends SQLParser {
 
                 sqlExpr = allExpr;
                 break;
+            case LITERAL_ALIAS:
+                sqlExpr = new SQLIdentifierExpr('"' + lexer.stringVal() + '"');
+                lexer.nextToken();
+                break;
             default:
                 throw new ParserException("ERROR. token : " + tok + " " + lexer.stringVal());
         }

src/main/java/com/alibaba/druid/sql/parser/Token.java

@@ -105,6 +105,7 @@ public enum Token {
     IDENTIFIED("IDENTIFIED"),
     PASSWORD("PASSWORD"),
     DUAL("DUAL"),
+    BINARY("BINARY"),
     
     //postgresql
     WINDOW("WINDOW"),

src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java

@@ -212,7 +212,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
             print(' ');
             return;
         }
-        
+
         print("\n");
         printIndent();
     }
@@ -455,7 +455,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
 
         printAndAccept(x.getArguments(), ", ");
         print(")");
-        
+
         if (x.getOver() != null) {
             print(" ");
             x.getOver().accept(this);
@@ -551,12 +551,12 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
 
     public boolean visit(SQLSelect x) {
         x.getQuery().setParent(x);
-        
+
         if (x.getWithSubQuery() != null) {
             x.getWithSubQuery().accept(this);
             println();
         }
-        
+
         x.getQuery().accept(this);
 
         if (x.getOrderBy() != null) {
@@ -885,7 +885,20 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
     @Override
     public boolean visit(SQLUnaryExpr x) {
         print(x.getOperator().name);
+
         SQLExpr expr = x.getExpr();
+        
+        switch (x.getOperator()) {
+            case BINARY:
+            case Prior:
+            case ConnectByRoot:
+                print(' ');
+                expr.accept(this);
+                return false;
+            default:
+                break;
+        }
+
         if (expr instanceof SQLBinaryOpExpr) {
             print('(');
             expr.accept(this);
@@ -943,7 +956,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
     public boolean visit(SQLJoinTableSource x) {
         x.getLeft().accept(this);
         incrementIndent();
-        
+
         if (x.getJoinType() == JoinType.COMMA) {
             print(",");
         } else {
@@ -959,7 +972,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
             x.getCondition().accept(this);
             decrementIndent();
         }
-        
+
         decrementIndent();
 
         return false;
@@ -1168,21 +1181,21 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
         x.getName().accept(this);
         return false;
     }
-    
+
     @Override
     public boolean visit(SQLAlterTableDropIndex x) {
         print("DROP INDEX ");
         x.getIndexName().accept(this);
         return false;
     }
-    
+
     @Override
     public boolean visit(SQLAlterTableAddPrimaryKey x) {
         print("ADD ");
         x.getPrimaryKey().accept(this);
         return false;
     }
-    
+
     @Override
     public boolean visit(SQLOver x) {
         print("OVER (");
@@ -1194,19 +1207,19 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
         }
         return false;
     }
-    
+
     @Override
     public boolean visit(SQLColumnPrimaryKey x) {
         print(" PRIMARY KEY");
         return false;
     }
-    
+
     @Override
     public boolean visit(SQLColumnUniqueIndex x) {
         print(" UNIQUE INDEX");
         return false;
     }
-    
+
     @Override
     public boolean visit(SQLWithSubqueryClause x) {
         print("WITH");
@@ -1219,7 +1232,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter {
         decrementIndent();
         return false;
     }
-    
+
     @Override
     public boolean visit(SQLWithSubqueryClause.Entry x) {
         x.getName().accept(this);

src/main/java/com/alibaba/druid/wall/WallConfig.java

@@ -61,7 +61,7 @@ public class WallConfig implements WallConfigMBean {
 
     private boolean             mustParameterized          = false;
 
-    private boolean             allowDoPrivileged          = false;
+    private boolean             doPrivilegedAllow          = false;
 
     protected final Set<String> permitFunctions            = new HashSet<String>();
     protected final Set<String> permitTables               = new HashSet<String>();
@@ -117,12 +117,12 @@ public class WallConfig implements WallConfigMBean {
         this.wrapAllow = wrapAllow;
     }
 
-    public boolean isAllowDoPrivileged() {
-        return allowDoPrivileged;
+    public boolean isDoPrivilegedAllow() {
+        return doPrivilegedAllow;
     }
 
-    public void setAllowDoPrivileged(boolean allowDoPrivileged) {
-        this.allowDoPrivileged = allowDoPrivileged;
+    public void setDoPrivilegedAllow(boolean doPrivilegedAllow) {
+        this.doPrivilegedAllow = doPrivilegedAllow;
     }
 
     public boolean isSelectAllColumnAllow() {

src/main/java/com/alibaba/druid/wall/WallFilter.java

@@ -320,7 +320,7 @@ public class WallFilter extends FilterAdapter implements WallFilterMBean {
 
     @Override
     public boolean isWrapperFor(FilterChain chain, Wrapper wrapper, Class<?> iface) throws SQLException {
-        if (config.isAllowDoPrivileged() && WallProvider.ispPivileged()) {
+        if (config.isDoPrivilegedAllow() && WallProvider.ispPivileged()) {
             return chain.isWrapperFor(wrapper, iface);
         }
 
@@ -332,7 +332,7 @@ public class WallFilter extends FilterAdapter implements WallFilterMBean {
 
     @Override
     public <T> T unwrap(FilterChain chain, Wrapper wrapper, Class<T> iface) throws SQLException {
-        if (config.isAllowDoPrivileged() && WallProvider.ispPivileged()) {
+        if (config.isDoPrivilegedAllow() && WallProvider.ispPivileged()) {
             return chain.unwrap(wrapper, iface);
         }
 
@@ -346,7 +346,7 @@ public class WallFilter extends FilterAdapter implements WallFilterMBean {
 
     @Override
     public DatabaseMetaData connection_getMetaData(FilterChain chain, ConnectionProxy connection) throws SQLException {
-        if (config.isAllowDoPrivileged() && WallProvider.ispPivileged()) {
+        if (config.isDoPrivilegedAllow() && WallProvider.ispPivileged()) {
             return chain.connection_getMetaData(connection);
         }
 

src/main/java/com/alibaba/druid/wall/WallProvider.java

@@ -119,7 +119,7 @@ public abstract class WallProvider {
     public WallCheckResult check(String sql) {
         WallCheckResult result = new WallCheckResult();
 
-        if (config.isAllowDoPrivileged() && privileged.get() == Boolean.TRUE) {
+        if (config.isDoPrivilegedAllow() && ispPivileged()) {
             return result;
         }
 

src/test/java/com/alibaba/druid/bvt/filter/wall/DoPrivilegedTest.java

@@ -33,7 +33,7 @@ public class DoPrivilegedTest extends TestCase {
 
     public void test_1() throws Exception {
         final WallConfig config = new WallConfig();
-        config.setAllowDoPrivileged(true);
+        config.setDoPrivilegedAllow(true);
         
         WallProvider.doPrivileged(new PrivilegedAction<Object>() {
             @Override

src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlSelectTest_BinaryConcat.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 1999-2011 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.mysql;
+
+import java.util.List;
+
+import junit.framework.Assert;
+
+import com.alibaba.druid.sql.MysqlTest;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.ast.statement.SQLSelect;
+import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
+import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
+import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
+
+public class MySqlSelectTest_BinaryConcat extends MysqlTest {
+
+    public void test_0() throws Exception {
+        String sql = "select campaignId from Campaign" + //
+                     "   where   advertiserId =   ?" + //
+                     "   and   deleteStatus = false" + //
+                     "   and   campaignName like binary CONCAT('%',?,'%')";
+
+        MySqlStatementParser parser = new MySqlStatementParser(sql);
+        List<SQLStatement> statementList = parser.parseStatementList();
+        SQLStatement stmt = statementList.get(0);
+
+        SQLSelectStatement selectStmt = (SQLSelectStatement) stmt;
+
+        SQLSelect select = selectStmt.getSelect();
+        Assert.assertNotNull(select.getQuery());
+        MySqlSelectQueryBlock queryBlock = (MySqlSelectQueryBlock) select.getQuery();
+        Assert.assertNull(queryBlock.getOrderBy());
+
+        print(statementList);
+
+        Assert.assertEquals(1, statementList.size());
+
+        MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor();
+        stmt.accept(visitor);
+
+        System.out.println("Tables : " + visitor.getTables());
+        System.out.println("fields : " + visitor.getColumns());
+        System.out.println("coditions : " + visitor.getConditions());
+        System.out.println("orderBy : " + visitor.getOrderByColumns());
+
+        Assert.assertEquals(1, visitor.getTables().size());
+        Assert.assertEquals(4, visitor.getColumns().size());
+        Assert.assertEquals(3, visitor.getConditions().size());
+        Assert.assertEquals(0, visitor.getOrderByColumns().size());
+
+        // Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("mytable")));
+    }
+}

src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGSelectTest7.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 1999-2011 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.postgresql;
+
+import java.util.List;
+
+import junit.framework.Assert;
+
+import com.alibaba.druid.sql.PGTest;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
+import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor;
+
+public class PGSelectTest7 extends PGTest {
+
+    public void test_0() throws Exception {
+        String sql = "select id, name, parameters, \"algModuleRule.id\", \"algModuleRule.name\"" + //
+                     ", \"algModuleRule.assembleType\", \"algModuleRule.algmIds\", \"algModuleRule.subAlgmIds\"" + //
+                     ", \"algModuleRule.algmRatio\", \"algModuleRule.algmMaxRatio\", \"algModuleRule.objectType\"" + //
+                     ", \"cateRule.id\", \"cateRule.name\", \"cateRule.assembleType\", \"cateRule.algmIds\"" + //
+                     ", \"cateRule.algmRatio\", \"cateRule.algmMaxRatio\", \"appObject.id\",\"appObject.name\"" + //
+                     ", \"appObject.beanId\" " + //
+                     "from cnres.function_select_irecom_scene()" + //
+                     " as a( id text,name text,parameters text, \"algModuleRule.id\" text,\"algModuleRule.name\" text" + //
+                     "    , \"algModuleRule.assembleType\" text,\"algModuleRule.algmIds\" text" + //
+                     "    , \"algModuleRule.subAlgmIds\" text,\"algModuleRule.algmRatio\" text" + //
+                     "    , \"algModuleRule.algmMaxRatio\" text, \"algModuleRule.objectType\" text" + //
+                     "    , \"cateRule.id\" text, \"cateRule.name\" text, \"cateRule.assembleType\" text" + //
+                     "    , \"cateRule.algmIds\" text, \"cateRule.algmRatio\" text, \"cateRule.algmMaxRatio\" text" + //
+                     "    , \"appObject.id\" text,\"appObject.name\" text, \"appObject.beanId\" text)";
+
+        PGSQLStatementParser parser = new PGSQLStatementParser(sql);
+        List<SQLStatement> statementList = parser.parseStatementList();
+        SQLStatement statemen = statementList.get(0);
+        print(statementList);
+
+        Assert.assertEquals(1, statementList.size());
+
+        PGSchemaStatVisitor visitor = new PGSchemaStatVisitor();
+        statemen.accept(visitor);
+
+        System.out.println("Tables : " + visitor.getTables());
+        System.out.println("fields : " + visitor.getColumns());
+        System.out.println("coditions : " + visitor.getConditions());
+
+        Assert.assertEquals(20, visitor.getColumns().size());
+        Assert.assertEquals(0, visitor.getTables().size());
+    }
+}
+
+// select categoryId , offerIds from cnres.function_select_get_spt_p4p_offer_list (' 1031918 , 1031919 , 1037004 ') as
+// a(categoryId numeric,offerIds character varying(4000))
+// select memberId , offerIds from cnres.function_select_get_seller_hot_offer_list('\'gzyyd168\'') as a(memberId
+// character varying(20),offerIds character varying(4000))
+

src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGSelectTest8.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 1999-2011 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.postgresql;
+
+import java.util.List;
+
+import junit.framework.Assert;
+
+import com.alibaba.druid.sql.PGTest;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
+import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor;
+
+public class PGSelectTest8 extends PGTest {
+
+    public void test_0() throws Exception {
+        String sql = "select id, name, beanId, \"algLable.id\", \"algLable.name\"" + //
+                     ", \"algLable.schemaName\", \"algLable.tableName\", \"algLable.fieldName\"" + //
+                     " from cnres.function_select_algmodule_rule() " + //
+                     " as a( id text,name text,beanId text, \"algLable.id\" text,\"algLable.name\" text" + //
+                     ", \"algLable.schemaName\" text,\"algLable.tableName\" text, \"algLable.fieldName\" text)";
+
+        PGSQLStatementParser parser = new PGSQLStatementParser(sql);
+        List<SQLStatement> statementList = parser.parseStatementList();
+        SQLStatement statemen = statementList.get(0);
+        print(statementList);
+
+        Assert.assertEquals(1, statementList.size());
+
+        PGSchemaStatVisitor visitor = new PGSchemaStatVisitor();
+        statemen.accept(visitor);
+
+        System.out.println("Tables : " + visitor.getTables());
+        System.out.println("fields : " + visitor.getColumns());
+        System.out.println("coditions : " + visitor.getConditions());
+
+        Assert.assertEquals(8, visitor.getColumns().size());
+        Assert.assertEquals(0, visitor.getTables().size());
+    }
+}
+
+// select categoryId , offerIds from cnres.function_select_get_spt_p4p_offer_list (' 1031918 , 1031919 , 1037004 ') as
+// a(categoryId numeric,offerIds character varying(4000))
+// select memberId , offerIds from cnres.function_select_get_seller_hot_offer_list('\'gzyyd168\'') as a(memberId
+// character varying(20),offerIds character varying(4000))
+