improved parser

src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java

@@ -19,9 +19,12 @@ import com.alibaba.druid.sql.ast.*;
 import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
 public abstract class SQLInsertInto extends SQLStatementImpl implements SQLReplaceable {
+    protected List<String>              insertBeforeComments;
+
     protected SQLExprTableSource        tableSource;
     protected final List<SQLExpr>       columns = new ArrayList<SQLExpr>();
     protected transient String          columnsString;
@@ -65,6 +68,19 @@ public abstract class SQLInsertInto extends SQLStatementImpl implements SQLRepla
         }
     }
 
+    public void addInsertBeforeComment(List<String> comments) {
+        if (insertBeforeComments == null) {
+            insertBeforeComments = new ArrayList<>();
+        }
+
+        insertBeforeComments.addAll(comments);
+    }
+
+    @SuppressWarnings("unchecked")
+    public List<String> getInsertBeforeCommentsDirect() {
+        return insertBeforeComments;
+    }
+
     public boolean replace(SQLExpr expr, SQLExpr target) {
         for (int i = 0; i < columns.size(); i++) {
             if (columns.get(i) == expr) {

src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java

@@ -53,6 +53,11 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
         if (x.hasBeforeComment()) {
             printlnComments(x.getBeforeCommentsDirect());
         }
+
+        if (x.getInsertBeforeCommentsDirect() != null) {
+            printlnComments(x.getInsertBeforeCommentsDirect());
+        }
+
         SQLExprTableSource tableSource = x.getTableSource();
 
         if (tableSource != null) {

src/main/java/com/alibaba/druid/sql/parser/Lexer.java

@@ -2245,10 +2245,25 @@ public class Lexer {
             pos++;
             bufPos++;
 
+            boolean ident = false;
             for (;;) {
                 ch = charAt(++pos);
+                if (isEOF()) {
+                    pos--;
+                    bufPos--;
+                    break;
+                }
 
-                if (ch == '}') {
+                if (ch == '}' && !ident) {
+                    if (isIdentifierChar(charAt(pos + 1))) {
+                        bufPos++;
+                        ident = true;
+                        continue;
+                    }
+                    break;
+                }
+
+                if (ident && isWhitespace(ch)) {
                     break;
                 }
 
@@ -2256,7 +2271,7 @@ public class Lexer {
                 continue;
             }
 
-            if (ch != '}') {
+            if (ch != '}' && !ident) {
                 throw new ParserException("syntax error. " + info());
             }
             ++pos;
@@ -2273,7 +2288,7 @@ public class Lexer {
             }
 
             stringVal = addSymbol();
-            token = Token.VARIANT;
+            token = ident ? IDENTIFIER : Token.VARIANT;
             return;
         } else if (c1 == '$' && charAt(pos + 2) == '{') {
             pos += 2;
@@ -2836,12 +2851,30 @@ public class Lexer {
                     && !(ch == 'b' && bufPos == 1 && charAt(pos - 1) == '0' && dbType != DbType.odps)
             ) {
                 bufPos++;
+                boolean brace = false;
                 for (;;) {
                     char c0 = ch;
                     ch = charAt(++pos);
 
+                    if (isEOF()) {
+                        break;
+                    }
+
                     if (!isIdentifierChar(ch)) {
-                        if (ch == '（'  || ch == '）' && c0 > 256) {
+                        if (ch == '{' && charAt(pos - 1) == '$' && !brace) {
+                            bufPos++;
+                            brace = true;
+                            continue;
+                        }
+
+                        if (ch == '}' && brace) {
+                            bufPos++;
+                            brace = false;
+                            continue;
+                        }
+
+                        if ((ch == '（'  || ch == '）')
+                                && c0 > 256) {
                             bufPos++;
                             continue;
                         }
@@ -3213,7 +3246,7 @@ public class Lexer {
                 || comment.indexOf('&') != -1 //
                 || comment.indexOf('|') != -1 //
                 || comment.indexOf('^') != -1 //
-                ) {
+        ) {
             return false;
         }
         return true;

src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java

@@ -6408,7 +6408,7 @@ public class SQLStatementParser extends SQLParser {
         insert.setDbType(dbType);
 
         if (lexer.isKeepComments() && lexer.hasComment()) {
-            insert.addBeforeComment(lexer.readAndResetComments());
+            insert.addInsertBeforeComment(lexer.readAndResetComments());
         }
 
         SQLSelectParser selectParser = createSQLSelectParser();

src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java

@@ -3377,6 +3377,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
             }
         }
 
+        if (x.getInsertBeforeCommentsDirect() != null) {
+            printlnComments(x.getInsertBeforeCommentsDirect());
+        }
+
         SQLWithSubqueryClause with = x.getWith();
         if (with != null) {
             visit(with);
@@ -6481,7 +6485,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
             return;
         }
 
-        if (comment.startsWith("--") && comment.length() > 2 && comment.charAt(2) != ' ') {
+        if (comment.startsWith("--")
+                && comment.length() > 2
+                && comment.charAt(2) != ' '
+        ) {
             print0("-- ");
             print0(comment.substring(2));
         } else {
@@ -6511,7 +6518,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
         }
 
         SQLName changeOwnerTo = x.getChangeOwnerTo();
-        if (to != null) {
+        if (changeOwnerTo != null) {
             print0(ucase ? " CHANGEOWNER TO " : " changeowner to ");
             printExpr(changeOwnerTo);
         }

src/test/java/com/alibaba/druid/bvt/sql/cobar/DALParserTest.java

@@ -1243,7 +1243,7 @@ public class DALParserTest extends TestCase {
     public void test_show_variants() throws Exception {
         String sql = "SHOW VARIABLES LIKE '%size%';";
         MySqlStatementParser parser = new MySqlStatementParser(sql);
-        MySqlShowVariantsStatement show = (MySqlShowVariantsStatement) parser.parseStatementList().get(0);
+        SQLShowVariantsStatement show = (SQLShowVariantsStatement) parser.parseStatementList().get(0);
         parser.match(Token.EOF);
         String output = SQLUtils.toMySqlString(show);
         assertEquals("SHOW VARIABLES LIKE '%size%';", output);
@@ -1252,7 +1252,7 @@ public class DALParserTest extends TestCase {
     public void test_show_variants_1() throws Exception {
         String sql = "SHOW GLOBAL VARIABLES LIKE '%size%';";
         MySqlStatementParser parser = new MySqlStatementParser(sql);
-        MySqlShowVariantsStatement show = (MySqlShowVariantsStatement) parser.parseStatementList().get(0);
+        SQLShowVariantsStatement show = (SQLShowVariantsStatement) parser.parseStatementList().get(0);
         parser.match(Token.EOF);
         String output = SQLUtils.toMySqlString(show);
         assertEquals("SHOW GLOBAL VARIABLES LIKE '%size%';", output);
@@ -1261,7 +1261,7 @@ public class DALParserTest extends TestCase {
     public void test_show_variants_2() throws Exception {
         String sql = "SHOW SESSION VARIABLES LIKE '%size%';";
         MySqlStatementParser parser = new MySqlStatementParser(sql);
-        MySqlShowVariantsStatement show = (MySqlShowVariantsStatement) parser.parseStatementList().get(0);
+        SQLShowVariantsStatement show = (SQLShowVariantsStatement) parser.parseStatementList().get(0);
         parser.match(Token.EOF);
         String output = SQLUtils.toMySqlString(show);
         assertEquals("SHOW SESSION VARIABLES LIKE '%size%';", output);

src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlError_test_3.java

@@ -38,6 +38,6 @@ public class MySqlError_test_3 extends MysqlTest {
 
         Assert.assertNotNull(error);
 //        error.printStackTrace();
-        Assert.assertEquals("syntax error, error in :'us = LIMIT ?, ?, pos 248, line 1, column 248, token ?", error.getMessage());
+        Assert.assertEquals("not supported.pos 248, line 1, column 248, token ?", error.getMessage());
     }
 }

src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsListResourcesTest.java

@@ -20,7 +20,11 @@ import java.io.InputStreamReader;
 import java.io.Reader;
 import java.util.List;
 
+import com.alibaba.druid.DbType;
+import com.alibaba.druid.sql.SQLUtils;
 import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
+import com.alibaba.druid.sql.parser.SQLParserFeature;
+import com.alibaba.druid.support.opds.udf.SqlCodeStat;
 import junit.framework.TestCase;
 
 import org.junit.Assert;
@@ -39,7 +43,30 @@ public class OdpsListResourcesTest extends TestCase {
         exec_test("bvt/parser/odps-3.sql");
     }
 
+    public static String readResource(String resource) throws Exception {
+        InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource);
+        Reader reader = new InputStreamReader(is, "UTF-8");
+        String input = Utils.read(reader);
+        return input;
+    }
+
+    public void test_8() throws Exception {
+        exec_test("bvt/parser/odps-8.txt");
+    }
+
+    public void format_resource(String resource) throws Exception {
+        InputStream is = null;
+
+        is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource);
+        Reader reader = new InputStreamReader(is, "UTF-8");
+        String input = Utils.read(reader);
+        String formattedSql = SQLUtils.format(input, DbType.odps);
+        System.out.println(formattedSql);
+
+    }
+
     public void exec_test(String resource) throws Exception {
+//        System.out.println(resource);
         InputStream is = null;
 
         is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource);
@@ -48,13 +75,28 @@ public class OdpsListResourcesTest extends TestCase {
         JdbcUtils.close(reader);
         String[] items = input.split("---------------------------");
         String sql = items[0].trim();
-        String expect = items[1].trim();
+        String expect = null;
+        if (items.length > 1) {
+            expect = items[1].trim();
+            if (expect != null) {
+                expect = expect.replaceAll("\\r\\n", "\n");
+            }
+        }
 
-        OdpsStatementParser parser = new OdpsStatementParser(sql);
+        new SqlCodeStat()
+                .evaluate(sql, "odps");
+
+        OdpsStatementParser parser = new OdpsStatementParser(sql
+                , SQLParserFeature.EnableSQLBinaryOpExprGroup
+                , SQLParserFeature.EnableMultiUnion
+                , SQLParserFeature.KeepComments);
         List<SQLStatement> statementList = parser.parseStatementList();
+        if (statementList.size() == 0) {
+            throw new Exception("empty");
+        }
         SQLStatement stmt = statementList.get(0);
 
-        Assert.assertEquals(1, statementList.size());
+//        Assert.assertEquals(1, statementList.size());
 
         SchemaStatVisitor visitor = new OdpsSchemaStatVisitor();
         stmt.accept(visitor);
@@ -65,7 +107,16 @@ public class OdpsListResourcesTest extends TestCase {
 //
 //        System.out.println();
 //        System.out.println("---------------------------");
-//        System.out.println(SQLUtils.toOdpsString(stmt));
+        System.out.println(SQLUtils.toOdpsString(stmt));
+
+        if (expect != null && !expect.isEmpty()) {
+            assertEquals(expect, stmt.toString());
+        }
+
+        SchemaStatVisitor schemaStatVisitor = SQLUtils.createSchemaStatVisitor(DbType.odps);
+        for (SQLStatement item : statementList) {
+            item.accept(schemaStatVisitor);
+        }
     }
 
     void mergValidate(String sql, String expect) {
@@ -86,4 +137,5 @@ public class OdpsListResourcesTest extends TestCase {
     }
 
 
+
 }

src/test/resources/bvt/parser/odps-8.txt

@@ -0,0 +1,16 @@
+select *
+from (
+    select id, collectId, 3 as c_0
+    from (
+        SELECT *
+        FROM tc_merchant
+        where collectId in (select collect_id from cte_merchant)
+        UNION
+        SELECT *
+        FROM st_merchant
+        where collectId in (select collect_id from st_merchant_tmp)
+        )
+        union
+        select id, collectId, c2
+        from (select id, collectId, c2 from table_2)
+)