Merge pull request #947 from wenshao/master

bug fixed for mysql connection validation checker, and improved odps …

src/main/java/com/alibaba/druid/pool/vendor/MySqlValidConnectionChecker.java

@@ -35,12 +35,14 @@ import java.util.Properties;
 
 public class MySqlValidConnectionChecker extends ValidConnectionCheckerAdapter implements ValidConnectionChecker, Serializable {
 
+    public static final int DEFAULT_VALIDATION_QUERY_TIMEOUT = 1000;
+
     private static final long serialVersionUID = 1L;
     private static final Log  LOG              = LogFactory.getLog(MySqlValidConnectionChecker.class);
 
-    private Class<?>          clazz;
-    private Method            ping;
-    private boolean           usePingMethod    = false;
+    private Class<?> clazz;
+    private Method   ping;
+    private boolean  usePingMethod = false;
 
     public MySqlValidConnectionChecker(){
         try {
@@ -52,7 +54,7 @@ public class MySqlValidConnectionChecker extends ValidConnectionCheckerAdapter i
         } catch (Exception e) {
             LOG.warn("Cannot resolve com.mysq.jdbc.Connection.ping method.  Will use 'SELECT 1' instead.", e);
         }
-        
+
         configFromProperties(System.getProperties());
     }
 
@@ -94,6 +96,10 @@ public class MySqlValidConnectionChecker extends ValidConnectionCheckerAdapter i
             }
 
             if (clazz.isAssignableFrom(conn.getClass())) {
+                if (validationQueryTimeout < 0) {
+                    validationQueryTimeout = DEFAULT_VALIDATION_QUERY_TIMEOUT;
+                }
+
                 try {
                     ping.invoke(conn, true, validationQueryTimeout);
                     return true;

src/main/java/com/alibaba/druid/sql/SQLUtils.java

@@ -38,6 +38,7 @@ import com.alibaba.druid.sql.dialect.db2.visitor.DB2SchemaStatVisitor;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
 import com.alibaba.druid.sql.dialect.odps.visitor.OdpsOutputVisitor;
+import com.alibaba.druid.sql.dialect.odps.visitor.OdpsSchemaStatVisitor;
 import com.alibaba.druid.sql.dialect.oracle.visitor.OracleOutputVisitor;
 import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor;
 import com.alibaba.druid.sql.dialect.oracle.visitor.OracleToMySqlOutputVisitor;
@@ -370,6 +371,10 @@ public class SQLUtils {
         if (JdbcUtils.DB2.equals(dbType)) {
             return new DB2SchemaStatVisitor();
         }
+        
+        if (JdbcUtils.ODPS.equals(dbType)) {
+            return new OdpsSchemaStatVisitor();
+        }
 
         return new SchemaStatVisitor();
     }

src/main/java/com/alibaba/druid/sql/ast/expr/SQLCharExpr.java

@@ -15,6 +15,7 @@
  */
 package com.alibaba.druid.sql.ast.expr;
 
+import com.alibaba.druid.sql.SQLUtils;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 
 public class SQLCharExpr extends SQLTextLiteralExpr implements SQLValuableExpr{
@@ -47,4 +48,8 @@ public class SQLCharExpr extends SQLTextLiteralExpr implements SQLValuableExpr{
     public Object getValue() {
         return this.text;
     }
+    
+    public String toString() {
+        return SQLUtils.toSQLString(this);
+    }
 }

src/main/java/com/alibaba/druid/sql/ast/expr/SQLIdentifierExpr.java

@@ -97,4 +97,7 @@ public class SQLIdentifierExpr extends SQLExprImpl implements SQLName {
         return true;
     }
 
+    public String toString() {
+        return this.name;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/expr/MySqlCharExpr.java

@@ -15,6 +15,7 @@
  */
 package com.alibaba.druid.sql.dialect.mysql.ast.expr;
 
+import com.alibaba.druid.sql.SQLUtils;
 import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitor;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
@@ -76,4 +77,10 @@ public class MySqlCharExpr extends SQLCharExpr implements MySqlExpr {
         visitor.visit(this);
         visitor.endVisit(this);
     }
+    
+    public String toString() {
+        StringBuffer buf = new StringBuffer();
+        output(buf);
+        return buf.toString();
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsSelectQueryBlock.java

@@ -18,6 +18,7 @@ package com.alibaba.druid.sql.dialect.odps.ast;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.alibaba.druid.sql.SQLUtils;
 import com.alibaba.druid.sql.ast.SQLExpr;
 import com.alibaba.druid.sql.ast.SQLHint;
 import com.alibaba.druid.sql.ast.SQLOrderBy;
@@ -34,8 +35,8 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock {
 
     protected List<SQLHint> hints;
 
-    protected SQLExpr distributeBy;
-    protected SQLSelectOrderByItem sortBy;
+    protected SQLExpr                    distributeBy;
+    protected List<SQLSelectOrderByItem> sortBy = new ArrayList<SQLSelectOrderByItem>(2);
 
     public OdpsSelectQueryBlock(){
 
@@ -57,14 +58,10 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock {
         this.distributeBy = distributeBy;
     }
 
-    public SQLSelectOrderByItem getSortBy() {
+    public List<SQLSelectOrderByItem> getSortBy() {
         return sortBy;
     }
 
-    public void setSortBy(SQLSelectOrderByItem sortBy) {
-        this.sortBy = sortBy;
-    }
-
     public SQLExpr getLimit() {
         return limit;
     }
@@ -138,4 +135,7 @@ public class OdpsSelectQueryBlock extends SQLSelectQueryBlock {
         visitor.endVisit(this);
     }
 
+    public String toString() {
+        return SQLUtils.toOdpsString(this);
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java

@@ -17,11 +17,15 @@ package com.alibaba.druid.sql.dialect.odps.parser;
 
 import com.alibaba.druid.sql.ast.SQLExpr;
 import com.alibaba.druid.sql.ast.expr.SQLArrayExpr;
+import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
+import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator;
 import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
 import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
 import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
 import com.alibaba.druid.sql.dialect.odps.ast.OdpsUDTFSQLSelectItem;
+import com.alibaba.druid.sql.parser.EOFParserException;
 import com.alibaba.druid.sql.parser.Lexer;
+import com.alibaba.druid.sql.parser.ParserException;
 import com.alibaba.druid.sql.parser.SQLExprParser;
 import com.alibaba.druid.sql.parser.Token;
 
@@ -131,4 +135,23 @@ public class OdpsExprParser extends SQLExprParser {
         
         return super.primaryRest(expr);
     }
+    
+    public SQLExpr equalityRest(SQLExpr expr) {
+        if (lexer.token() == Token.EQEQ) {
+            SQLExpr rightExp;
+            lexer.nextToken();
+            try {
+                rightExp = bitOr();
+            } catch (EOFParserException e) {
+                throw new ParserException("EOF, " + expr + "=", e);
+            }
+            rightExp = equalityRest(rightExp);
+
+            expr = new SQLBinaryOpExpr(expr, SQLBinaryOperator.Equality, rightExp, getDbType());
+            
+            return expr;
+        }
+        
+        return super.equalityRest(expr);
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java

@@ -90,19 +90,28 @@ public class OdpsSelectParser extends SQLSelectParser {
             if (identifierEquals("SORT")) {
                 lexer.nextToken();
                 accept(Token.BY);
-                SQLExpr expr = this.expr();
                 
-                SQLSelectOrderByItem sortBy = new SQLSelectOrderByItem(expr);
-                
-                if (lexer.token() == Token.ASC) {
-                    sortBy.setType(SQLOrderingSpecification.ASC);
-                    lexer.nextToken();
-                } else if (lexer.token() == Token.DESC) {
-                    sortBy.setType(SQLOrderingSpecification.DESC);
-                    lexer.nextToken();
+                for (;;) {
+                    SQLExpr expr = this.expr();
+                    
+                    SQLSelectOrderByItem sortByItem = new SQLSelectOrderByItem(expr);
+                    
+                    if (lexer.token() == Token.ASC) {
+                        sortByItem.setType(SQLOrderingSpecification.ASC);
+                        lexer.nextToken();
+                    } else if (lexer.token() == Token.DESC) {
+                        sortByItem.setType(SQLOrderingSpecification.DESC);
+                        lexer.nextToken();
+                    }
+                    
+                    queryBlock.getSortBy().add(sortByItem);
+                    
+                    if (lexer.token() == Token.COMMA) {
+                        lexer.nextToken();
+                    } else {
+                        break;
+                    }
                 }
-                
-                queryBlock.setSortBy(sortBy);
             }
         }
 

src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java

@@ -526,9 +526,9 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
             print("DISTRIBUTE BY ");
             x.getDistributeBy().accept(this);
 
-            if (x.getSortBy() != null) {
+            if (!x.getSortBy().isEmpty()) {
                 print(" SORT BY ");
-                x.getSortBy().accept(this);
+                printAndAccept(x.getSortBy(), ", ");
             }
         }
 

src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java

@@ -438,8 +438,13 @@ public class SQLExprParser extends SQLParser {
                         break;
                     case IDENTIFIER: // 当负号后面为字段的情况
                         sqlExpr = new SQLIdentifierExpr(lexer.stringVal());
-                        sqlExpr = new SQLUnaryExpr(SQLUnaryOperator.Negative, sqlExpr);
                         lexer.nextToken();
+                        
+                        if (lexer.token() == Token.LPAREN || lexer.token() == Token.LBRACKET) {
+                            sqlExpr = primaryRest(sqlExpr);
+                        }
+                        sqlExpr = new SQLUnaryExpr(SQLUnaryOperator.Negative, sqlExpr);
+                        
                         break;
                     case QUES:
                         sqlExpr = new SQLUnaryExpr(SQLUnaryOperator.Negative, new SQLVariantRefExpr("?"));
@@ -705,6 +710,14 @@ public class SQLExprParser extends SQLParser {
             }
 
             accept(Token.RPAREN);
+            
+            if (lexer.token() == Token.OVER) {
+                SQLAggregateExpr aggregateExpr = new SQLAggregateExpr(methodName);
+                aggregateExpr.getArguments().addAll(methodInvokeExpr.getParameters());
+                over(aggregateExpr);
+                
+                return primaryRest(aggregateExpr);
+            }
 
             return primaryRest(methodInvokeExpr);
         }
@@ -947,33 +960,37 @@ public class SQLExprParser extends SQLParser {
         accept(Token.RPAREN);
 
         if (lexer.token() == Token.OVER) {
-            lexer.nextToken();
-            SQLOver over = new SQLOver();
-            accept(Token.LPAREN);
-
-            if (lexer.token() == Token.PARTITION || identifierEquals("PARTITION")) {
-                lexer.nextToken();
-                accept(Token.BY);
-
-                if (lexer.token() == (Token.LPAREN)) {
-                    lexer.nextToken();
-                    exprList(over.getPartitionBy(), over);
-                    accept(Token.RPAREN);
-                } else {
-                    exprList(over.getPartitionBy(), over);
-                }
-            }
-
-            over.setOrderBy(parseOrderBy());
-
-            accept(Token.RPAREN);
-            aggregateExpr.setOver(over);
+            over(aggregateExpr);
 
         }
 
         return aggregateExpr;
     }
 
+    protected void over(SQLAggregateExpr aggregateExpr) {
+        lexer.nextToken();
+        SQLOver over = new SQLOver();
+        accept(Token.LPAREN);
+
+        if (lexer.token() == Token.PARTITION || identifierEquals("PARTITION")) {
+            lexer.nextToken();
+            accept(Token.BY);
+
+            if (lexer.token() == (Token.LPAREN)) {
+                lexer.nextToken();
+                exprList(over.getPartitionBy(), over);
+                accept(Token.RPAREN);
+            } else {
+                exprList(over.getPartitionBy(), over);
+            }
+        }
+
+        over.setOrderBy(parseOrderBy());
+
+        accept(Token.RPAREN);
+        aggregateExpr.setOver(over);
+    }
+
     protected SQLAggregateExpr parseAggregateExprRest(SQLAggregateExpr aggregateExpr) {
         return aggregateExpr;
     }

src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java

@@ -736,6 +736,10 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter {
 
         if (selectList != null) {
             for (SQLSelectItem item : selectList) {
+                if (!item.getClass().equals(SQLSelectItem.class)) {
+                    continue;
+                }
+                
                 String itemAlias = item.getAlias();
                 SQLExpr itemExpr = item.getExpr();
                 if (itemAlias == null) {

src/main/java/com/alibaba/druid/support/opds/udf/ExportColumns.java

@@ -48,7 +48,7 @@ public class ExportColumns extends UDF {
             }
 
             return buf.toString();
-        } catch (Exception ex) {
+        } catch (Throwable ex) {
             System.err.println("error sql : " + sql);
             ex.printStackTrace();
             return null;

src/main/java/com/alibaba/druid/support/opds/udf/ExportTables.java

@@ -51,7 +51,7 @@ public class ExportTables extends UDF {
             }
 
             return buf.toString();
-        } catch (Exception ex) {
+        } catch (Throwable ex) {
             System.err.println("error sql : " + sql);
             ex.printStackTrace();
             return null;

src/test/java/com/alibaba/druid/bvt/filter/wall/PGWallTest.java

@@ -9,7 +9,7 @@ import com.alibaba.druid.wall.WallUtils;
 public class PGWallTest extends TestCase {
 
     public void test_false() throws Exception {
-        Assert.assertFalse(WallUtils.isValidatePostgres(//
+        Assert.assertTrue(WallUtils.isValidatePostgres(//
         "select wm_concat(article_id) over() from t_nds_web_article")); //
     }
 }

src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsResourceTest.java

@@ -50,8 +50,12 @@ public class OdpsResourceTest extends TestCase {
         JdbcUtils.close(reader);
         String[] items = input.split("---------------------------");
         String sql = items[0].trim();
-        String expect = items[1].trim();
-
+        String expect = null;
+        
+        if (items.length > 1) {
+            expect = items[1].trim();
+        }
+        
         OdpsStatementParser parser = new OdpsStatementParser(sql);
         List<SQLStatement> statementList = parser.parseStatementList();
         SQLStatement stmt = statementList.get(0);

src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest2.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 1999-2101 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.odps;
+
+import org.junit.Assert;
+
+import com.alibaba.druid.sql.SQLUtils;
+
+import junit.framework.TestCase;
+
+public class OdpsSelectTest2 extends TestCase {
+
+    public void test_distribute_by() throws Exception {
+        String sql = "SELECT user_id,user_flag,cat1_id,wireless_client_type,alipay_num,last_buy_time,md5,my_udf_001('t_datax_odps2ots_resource_tcif_dmp_user_topup_d',md5) AS datax_pt FROM mytable_001 WHERE ds='20150819'  DISTRIBUTE BY cast (datax_pt as BIGINT) SORT BY md5,user_id,user_flag,cat1_id,wireless_client_type";//
+        Assert.assertEquals("SELECT user_id"
+                + "\n\t, user_flag"
+                + "\n\t, cat1_id"
+                + "\n\t, wireless_client_type"
+                + "\n\t, alipay_num"
+                + "\n\t, last_buy_time"
+                + "\n\t, md5"
+                + "\n\t, my_udf_001('t_datax_odps2ots_resource_tcif_dmp_user_topup_d', md5) AS datax_pt"
+                + "\nFROM mytable_001"
+                + "\nWHERE ds = '20150819'"
+                + "\nDISTRIBUTE BY CAST(datax_pt AS BIGINT) SORT BY md5, user_id, user_flag, cat1_id, wireless_client_type", SQLUtils.formatOdps(sql));
+    }
+    
+
+}

src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest3.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 1999-2101 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.odps;
+
+import org.junit.Assert;
+
+import com.alibaba.druid.sql.SQLUtils;
+
+import junit.framework.TestCase;
+
+public class OdpsSelectTest3 extends TestCase {
+
+    public void test_distribute_by() throws Exception {
+        String sql = "select RANK() OVER (PARTITION BY ui ORDER BY duration DESC) rank from dual";//
+        Assert.assertEquals("SELECT RANK() OVER (PARTITION BY ui ORDER BY duration DESC) AS rank"
+                + "\nFROM dual", SQLUtils.formatOdps(sql));
+    }
+    
+
+}

src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsSelectTest4.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 1999-2101 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.odps;
+
+import org.junit.Assert;
+
+import com.alibaba.druid.sql.SQLUtils;
+
+import junit.framework.TestCase;
+
+public class OdpsSelectTest4 extends TestCase {
+
+    public void test_distribute_by() throws Exception {
+        String sql = "select total_day_cnt * EXP(-datediff(to_date('20150819', 'yyyymmdd'), last_time, 'dd') / 60) from dual";//
+        Assert.assertEquals("SELECT total_day_cnt * EXP(-datediff(TO_DATE('20150819', 'yyyymmdd'), last_time, 'dd') / 60)"
+                + "\nFROM dual", SQLUtils.formatOdps(sql));
+    }
+    
+
+}

src/test/java/com/alibaba/druid/bvt/sql/odps/udf/ExportColumnsTest0.java

@@ -0,0 +1,20 @@
+package com.alibaba.druid.bvt.sql.odps.udf;
+
+import com.alibaba.druid.support.opds.udf.ExportColumns;
+
+import junit.framework.TestCase;
+
+public class ExportColumnsTest0 extends TestCase {
+
+    private ExportColumns udf = new ExportColumns();
+
+    public void test_0() throws Exception {
+        String sql = "SELECT fund_base_cv_creative_ocr_judge_control_words_1422189630695(a.tfs, b.ocr_text, a.cates)"
+                + "\n AS (tfs, control_word, ocr_word, control_type, cates)"
+                + "\n FROM fund_base_cv_ad_auction_ocr_pv_tfs a"
+                + "\n JOIN fund_base_cv_ad_auction_ocr_pv_tfs_ocr b ON a.tfs = b.tfs"
+                + "\n WHERE a.ds=20150819 and b.ds=20150819;";
+        
+        udf.evaluate(sql, "odps");
+    }
+}

src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGSelectTest24.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 1999-2101 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.bvt.sql.postgresql;
+
+import java.util.List;
+
+import org.junit.Assert;
+
+import com.alibaba.druid.sql.PGTest;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
+import com.alibaba.druid.sql.dialect.postgresql.visitor.PGSchemaStatVisitor;
+
+public class PGSelectTest24 extends PGTest {
+
+    public void test_0() throws Exception {
+        String sql = "select wm_concat(article_id) over() from t_nds_web_article";
+
+        PGSQLStatementParser parser = new PGSQLStatementParser(sql);
+        List<SQLStatement> statementList = parser.parseStatementList();
+        SQLStatement statemen = statementList.get(0);
+        print(statementList);
+
+        Assert.assertEquals("SELECT wm_concat(article_id) OVER ()"
+                + "\nFROM t_nds_web_article", output(statementList));
+
+        Assert.assertEquals(1, statementList.size());
+
+        PGSchemaStatVisitor visitor = new PGSchemaStatVisitor();
+        statemen.accept(visitor);
+
+        System.out.println("Tables : " + visitor.getTables());
+        System.out.println("fields : " + visitor.getColumns());
+        System.out.println("coditions : " + visitor.getConditions());
+
+        Assert.assertEquals(1, visitor.getColumns().size());
+        Assert.assertEquals(1, visitor.getTables().size());
+    }
+}