improved odps sql formatter

src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java

@@ -106,11 +106,11 @@ public class MySqlLexer extends Lexer {
             bufPos++;
         }
 
-        stringVal = subString(mark, bufPos);
+        stringVal = subString(mark - 1, bufPos + 1);
         token = Token.LINE_COMMENT;
         hasComment = true;
         if (keepComments) {
-            commentVal = stringVal;
+            addComment(stringVal);
         }
 
         if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
@@ -452,7 +452,7 @@ public class MySqlLexer extends Lexer {
                 token = Token.MULTI_LINE_COMMENT;
                 hasComment = true;
                 if (keepComments) {
-                    commentVal = stringVal;
+                    addComment(stringVal);
                 }
             }
 
@@ -499,7 +499,7 @@ public class MySqlLexer extends Lexer {
             token = Token.LINE_COMMENT;
             hasComment = true;
             if (keepComments) {
-                commentVal = stringVal;
+                addComment(stringVal);
             }
 
             if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {

src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsObject.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 1999-2011 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.druid.sql.dialect.odps.ast;
+
+import com.alibaba.druid.sql.ast.SQLObject;
+import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
+
+public interface OdpsObject extends SQLObject {
+    void accept0(OdpsASTVisitor visitor);
+}

src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsObjectImpl.java

@@ -20,13 +20,13 @@ import com.alibaba.druid.sql.ast.SQLObjectImpl;
 import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
 import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 
-public abstract class OdpsObjectImpl extends SQLObjectImpl {
+public abstract class OdpsObjectImpl extends SQLObjectImpl implements OdpsObject {
     @Override
     protected void accept0(SQLASTVisitor visitor) {
         accept0((OdpsASTVisitor) visitor);
     }
     
-    protected abstract void accept0(OdpsASTVisitor visitor);
+    public abstract void accept0(OdpsASTVisitor visitor);
     
     public String toString() {
         return SQLUtils.toOdpsString(this);

src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsStatisticClause.java

@@ -40,7 +40,7 @@ public abstract class OdpsStatisticClause extends OdpsObjectImpl {
 
     public static class NullValue extends ColumnStatisticClause {
         @Override
-        protected void accept0(OdpsASTVisitor visitor) {
+        public void accept0(OdpsASTVisitor visitor) {
             if (visitor.visit(this)) {
                 acceptChild(visitor, column);
             }
@@ -50,7 +50,7 @@ public abstract class OdpsStatisticClause extends OdpsObjectImpl {
 
     public static class ColumnSum extends ColumnStatisticClause {
         @Override
-        protected void accept0(OdpsASTVisitor visitor) {
+        public void accept0(OdpsASTVisitor visitor) {
             if (visitor.visit(this)) {
                 acceptChild(visitor, column);
             }
@@ -60,7 +60,7 @@ public abstract class OdpsStatisticClause extends OdpsObjectImpl {
 
     public static class ColumnMin extends ColumnStatisticClause {
         @Override
-        protected void accept0(OdpsASTVisitor visitor) {
+        public void accept0(OdpsASTVisitor visitor) {
             if (visitor.visit(this)) {
                 acceptChild(visitor, column);
             }
@@ -70,7 +70,7 @@ public abstract class OdpsStatisticClause extends OdpsObjectImpl {
 
     public static class ColumnMax extends ColumnStatisticClause {
         @Override
-        protected void accept0(OdpsASTVisitor visitor) {
+        public void accept0(OdpsASTVisitor visitor) {
             if (visitor.visit(this)) {
                 acceptChild(visitor, column);
             }
@@ -94,7 +94,7 @@ public abstract class OdpsStatisticClause extends OdpsObjectImpl {
         }
     
         @Override
-        protected void accept0(OdpsASTVisitor visitor) {
+        public void accept0(OdpsASTVisitor visitor) {
             if (visitor.visit(this)) {
                 acceptChild(visitor, expr);
             }
@@ -105,7 +105,7 @@ public abstract class OdpsStatisticClause extends OdpsObjectImpl {
     public static class TableCount extends OdpsStatisticClause {
     
         @Override
-        protected void accept0(OdpsASTVisitor visitor) {
+        public void accept0(OdpsASTVisitor visitor) {
             if (visitor.visit(this)) {
     
             }

src/main/java/com/alibaba/druid/sql/dialect/odps/ast/OdpsUDTFSQLSelectItem.java

@@ -19,8 +19,10 @@ import java.util.ArrayList;
 import java.util.List;
 
 import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
+import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
+import com.alibaba.druid.sql.visitor.SQLASTVisitor;
 
-public class OdpsUDTFSQLSelectItem extends SQLSelectItem {
+public class OdpsUDTFSQLSelectItem extends SQLSelectItem implements OdpsObject {
 
     private final List<String> aliasList = new ArrayList<String>();
 
@@ -36,4 +38,15 @@ public class OdpsUDTFSQLSelectItem extends SQLSelectItem {
         return aliasList;
     }
 
+    protected void accept0(SQLASTVisitor visitor) {
+        accept0((OdpsASTVisitor) visitor);
+    }
+    
+    public void accept0(OdpsASTVisitor visitor) {
+        if (visitor.visit(this)) {
+            acceptChild(visitor, this.expr);
+        }
+        visitor.endVisit(this);
+    }
+
 }

src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java

@@ -21,7 +21,6 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
 import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
 import com.alibaba.druid.sql.dialect.odps.ast.OdpsUDTFSQLSelectItem;
 import com.alibaba.druid.sql.parser.Lexer;
-import com.alibaba.druid.sql.parser.ParserException;
 import com.alibaba.druid.sql.parser.SQLExprParser;
 import com.alibaba.druid.sql.parser.Token;
 
@@ -49,8 +48,8 @@ public class OdpsExprParser extends SQLExprParser {
         this.lexer.nextToken();
     }
     
-    public OdpsExprParser(String sql, Lexer.CommentHandler commentHandler){
-        this(new OdpsLexer(sql, commentHandler));
+    public OdpsExprParser(String sql, boolean skipComments, boolean keepComments){
+        this(new OdpsLexer(sql, skipComments, keepComments));
         this.lexer.nextToken();
     }
     
@@ -107,7 +106,7 @@ public class OdpsExprParser extends SQLExprParser {
         SQLSelectItem item = new SQLSelectItem(expr, alias);
         
         if (lexer.hasComment() && lexer.isKeepComments()) {
-            item.addAfterComment(lexer.commentVal());
+            item.addAfterComment(lexer.readAndResetComments());
         }
 
         return item;

src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java

@@ -53,6 +53,13 @@ public class OdpsLexer extends Lexer {
         super.keywods = DEFAULT_ODPS_KEYWORDS;
     }
     
+    public OdpsLexer(String input, boolean skipComment, boolean keepComments){
+        super(input, skipComment);
+        this.skipComment = skipComment;
+        this.keepComments = keepComments;
+        super.keywods = DEFAULT_ODPS_KEYWORDS;
+    }
+    
     public OdpsLexer(String input, CommentHandler commentHandler){
         super(input, commentHandler);
         super.keywods = DEFAULT_ODPS_KEYWORDS;
@@ -107,7 +114,7 @@ public class OdpsLexer extends Lexer {
                 token = Token.MULTI_LINE_COMMENT;
                 hasComment = true;
                 if (keepComments) {
-                    commentVal = stringVal;
+                    addComment(stringVal);
                 }
             }
             
@@ -157,7 +164,7 @@ public class OdpsLexer extends Lexer {
             token = Token.LINE_COMMENT;
             hasComment = true;
             if (keepComments) {
-                commentVal = stringVal;
+                addComment(stringVal);
             }
             endOfComment = isEOF();
             

src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java

@@ -43,7 +43,7 @@ public class OdpsSelectParser extends SQLSelectParser {
         OdpsSelectQueryBlock queryBlock = new OdpsSelectQueryBlock();
         
         if (lexer.hasComment() && lexer.isKeepComments()) {
-            queryBlock.addBeforeComment(lexer.commentVal());
+            queryBlock.addBeforeComment(lexer.readAndResetComments());
         }
         
         accept(Token.SELECT);

src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java

@@ -48,9 +48,8 @@ import com.alibaba.druid.util.JdbcConstants;
 
 public class OdpsStatementParser extends SQLStatementParser {
     public OdpsStatementParser(String sql){
-        super (new OdpsLexer(sql), JdbcConstants.ODPS);
+        super (new OdpsLexer(sql, true, true), JdbcConstants.ODPS);
         this.exprParser = new OdpsExprParser(this.lexer);
-        this.lexer.setCommentHandler(commentCallBack);
         this.lexer.nextToken();
     }
 
@@ -231,8 +230,8 @@ public class OdpsStatementParser extends SQLStatementParser {
     public OdpsInsert parseOdpsInsert() {
         OdpsInsert insert = new OdpsInsert();
         
-        if (lexer.hasComment()) {
-            insert.addBeforeComment(lexer.commentVal());
+        if (lexer.isKeepComments() && lexer.hasComment()) {
+            insert.addBeforeComment(lexer.readAndResetComments());
         }
         
         SQLSelectParser selectParser = createSQLSelectParser();
@@ -323,11 +322,22 @@ public class OdpsStatementParser extends SQLStatementParser {
     }
 
     public SQLStatement parseSet() {
+        List<String> comments = null;
+        if (lexer.isKeepComments() && lexer.hasComment()) {
+            comments = lexer.readAndResetComments();
+        }
+        
         accept(Token.SET);
 
         if (identifierEquals("LABEL")) {
-            lexer.nextToken();
             OdpsSetLabelStatement stmt = new OdpsSetLabelStatement();
+            
+            if (comments != null) {
+                stmt.addBeforeComment(comments);
+            }
+            
+            lexer.nextToken();
+            
             stmt.setLabel(lexer.stringVal());
             lexer.nextToken();
             accept(Token.TO);
@@ -351,6 +361,10 @@ public class OdpsStatementParser extends SQLStatementParser {
             return stmt;
         } else {
             SQLSetStatement stmt = new SQLSetStatement(getDbType());
+            
+            if (comments != null) {
+                stmt.addBeforeComment(comments);
+            }
 
             parseAssignItems(stmt.getItems(), stmt);
 

src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java

@@ -144,10 +144,14 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
             println();
             print(") ");
             print(x.getFrom().getAlias());
+            println();
         }
 
-        for (OdpsInsert insert : x.getItems()) {
-            println();
+        for (int i = 0; i < x.getItems().size(); ++i) {
+            OdpsInsert insert = x.getItems().get(i);
+            if (i != 0) {
+                println();
+            }
             insert.accept(this);
         }
         return false;
@@ -327,15 +331,30 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
     public boolean visit(OdpsUDTFSQLSelectItem x) {
         x.getExpr().accept(this);
 
-        print(" AS (");
+        println();
+        print("AS (");
+        
+        int aliasSize = x.getAliasList().size();
+        if (aliasSize > 5) {
+            incrementIndent();
+            println();
+        }
 
-        for (int i = 0; i < x.getAliasList().size(); ++i) {
+        for (int i = 0; i < aliasSize; ++i) {
             if (i != 0) {
-                print(", ");
+                if (aliasSize > 5) {
+                    println(",");
+                } else {
+                    print(", ");    
+                }
             }
             print(x.getAliasList().get(i));
         }
-
+        
+        if (aliasSize > 5) {
+            decrementIndent();
+            println();
+        }
         print(")");
 
         return false;

src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleSetTransactionStatement.java

@@ -22,6 +22,7 @@ import com.alibaba.druid.sql.dialect.oracle.visitor.OracleASTVisitor;
 public class OracleSetTransactionStatement extends OracleSQLObjectImpl implements OracleStatement {
 
     private boolean readOnly = false;
+    private String  dbType;
 
     private SQLExpr name;
 
@@ -49,4 +50,12 @@ public class OracleSetTransactionStatement extends OracleSQLObjectImpl implement
         this.readOnly = readOnly;
     }
 
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
+    
 }

src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleLexer.java

@@ -232,7 +232,7 @@ public class OracleLexer extends Lexer {
                 token = Token.MULTI_LINE_COMMENT;
                 hasComment = true;
                 if (keepComments) {
-                    commentVal = stringVal;
+                    addComment(stringVal);
                 }
             }
 
@@ -278,7 +278,7 @@ public class OracleLexer extends Lexer {
             token = Token.LINE_COMMENT;
             hasComment = true;
             if (keepComments) {
-                commentVal = stringVal;
+                addComment(stringVal);
             }
             endOfComment = isEOF();
             return;

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerBlockStatement.java

@@ -26,6 +26,8 @@ import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerASTVisitor;
 public class SQLServerBlockStatement  extends SQLServerObjectImpl implements SQLServerStatement  {
 
     private List<SQLStatement>    statementList = new ArrayList<SQLStatement>();
+    
+    private String dbType;
 
     public List<SQLStatement> getStatementList() {
         return statementList;
@@ -43,4 +45,11 @@ public class SQLServerBlockStatement  extends SQLServerObjectImpl implements SQL
         visitor.endVisit(this);
     }
 
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerCommitStatement.java

@@ -27,6 +27,8 @@ public class SQLServerCommitStatement extends SQLServerObjectImpl implements SQL
     private SQLExpr transactionName;
 
     private SQLExpr delayedDurability;
+    
+    private String  dbType;
 
     public boolean isWork() {
         return work;
@@ -59,4 +61,12 @@ public class SQLServerCommitStatement extends SQLServerObjectImpl implements SQL
     public void setDelayedDurability(SQLExpr delayedDurability) {
         this.delayedDurability = delayedDurability;
     }
+    
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerDeclareStatement.java

@@ -26,6 +26,8 @@ import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerASTVisitor;
 public class SQLServerDeclareStatement extends SQLServerObjectImpl implements SQLServerStatement {
 
     protected List<SQLServerDeclareItem> items = new ArrayList<SQLServerDeclareItem>();
+    
+    private String dbType;
 
     @Override
     public void accept0(SQLServerASTVisitor visitor) {
@@ -43,4 +45,11 @@ public class SQLServerDeclareStatement extends SQLServerObjectImpl implements SQ
         this.items = items;
     }
 
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerExecStatement.java

@@ -29,6 +29,7 @@ public class SQLServerExecStatement extends SQLServerObjectImpl implements SQLSe
     private SQLName       returnStatus;
     private SQLName       moduleName;
     private List<SQLExpr> parameters = new ArrayList<SQLExpr>();
+    private String        dbType;
 
     public SQLName getModuleName() {
         return moduleName;
@@ -58,4 +59,12 @@ public class SQLServerExecStatement extends SQLServerObjectImpl implements SQLSe
     public void setReturnStatus(SQLName returnStatus) {
         this.returnStatus = returnStatus;
     }
+    
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerIfStatement.java

@@ -29,6 +29,7 @@ public class SQLServerIfStatement extends SQLServerObjectImpl implements SQLServ
     private SQLExpr            condition;
     private List<SQLStatement> statements = new ArrayList<SQLStatement>();
     private Else               elseItem;
+    private String             dbType;
 
     @Override
     public void accept0(SQLServerASTVisitor visitor) {
@@ -86,4 +87,11 @@ public class SQLServerIfStatement extends SQLServerObjectImpl implements SQLServ
 
     }
 
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerSetStatement.java

@@ -24,7 +24,7 @@ import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerASTVisitor;
 public class SQLServerSetStatement extends SQLServerObjectImpl implements SQLServerStatement {
 
     private SQLAssignItem item = new SQLAssignItem();
-    
+    private String dbType;
 
     public SQLServerSetStatement(){
     }
@@ -53,4 +53,12 @@ public class SQLServerSetStatement extends SQLServerObjectImpl implements SQLSer
         }
         visitor.endVisit(this);
     }
+    
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerSetTransactionIsolationLevelStatement.java

@@ -22,6 +22,7 @@ import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerASTVisitor;
 public class SQLServerSetTransactionIsolationLevelStatement extends SQLServerObjectImpl implements SQLServerStatement {
 
     private String level;
+    private String dbType;
 
     @Override
     public void accept0(SQLServerASTVisitor visitor) {
@@ -37,4 +38,11 @@ public class SQLServerSetTransactionIsolationLevelStatement extends SQLServerObj
         this.level = level;
     }
 
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/ast/stmt/SQLServerWaitForStatement.java

@@ -30,6 +30,8 @@ public class SQLServerWaitForStatement extends SQLServerObjectImpl implements SQ
     private SQLStatement statement;
 
     private SQLExpr      timeout;
+    
+    private String       dbType;
 
     @Override
     public void accept0(SQLServerASTVisitor visitor) {
@@ -74,4 +76,11 @@ public class SQLServerWaitForStatement extends SQLServerObjectImpl implements SQ
         this.timeout = timeout;
     }
 
+    public String getDbType() {
+        return dbType;
+    }
+    
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
 }

src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerLexer.java

@@ -107,7 +107,7 @@ public class SQLServerLexer extends Lexer {
                 token = Token.MULTI_LINE_COMMENT;
                 hasComment = true;
                 if (keepComments) {
-                    commentVal = stringVal;
+                    addComment(stringVal);
                 }
             }
 
@@ -149,7 +149,7 @@ public class SQLServerLexer extends Lexer {
             token = Token.LINE_COMMENT;
             hasComment = true;
             if (keepComments) {
-                commentVal = stringVal;
+                addComment(stringVal);
             }
             endOfComment = isEOF();