From 0801840a1fb4ccc240dae8628dec779218d1fba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Sat, 2 Oct 2021 22:09:31 +0800 Subject: [PATCH] improved parser --- .../sql/ast/statement/SQLInsertInto.java | 16 +++++ .../hive/visitor/HiveOutputVisitor.java | 5 ++ .../com/alibaba/druid/sql/parser/Lexer.java | 43 +++++++++++-- .../druid/sql/parser/SQLStatementParser.java | 2 +- .../sql/visitor/SQLASTOutputVisitor.java | 11 +++- .../druid/bvt/sql/cobar/DALParserTest.java | 6 +- .../bvt/sql/mysql/MySqlError_test_3.java | 2 +- .../bvt/sql/odps/OdpsListResourcesTest.java | 60 +++++++++++++++++-- src/test/resources/bvt/parser/odps-8.txt | 16 +++++ 9 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 src/test/resources/bvt/parser/odps-8.txt diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java index 2081a9578a..7453724de9 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLInsertInto.java @@ -19,9 +19,12 @@ import com.alibaba.druid.sql.ast.*; import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; public abstract class SQLInsertInto extends SQLStatementImpl implements SQLReplaceable { + protected List insertBeforeComments; + protected SQLExprTableSource tableSource; protected final List columns = new ArrayList(); protected transient String columnsString; @@ -65,6 +68,19 @@ public abstract class SQLInsertInto extends SQLStatementImpl implements SQLRepla } } + public void addInsertBeforeComment(List comments) { + if (insertBeforeComments == null) { + insertBeforeComments = new ArrayList<>(); + } + + insertBeforeComments.addAll(comments); + } + + @SuppressWarnings("unchecked") + public List getInsertBeforeCommentsDirect() { + return insertBeforeComments; + } + public boolean replace(SQLExpr expr, SQLExpr target) { for (int i = 0; i < columns.size(); i++) { if (columns.get(i) == expr) { diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java index 3eada42abe..65e0b387b1 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java @@ -53,6 +53,11 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis if (x.hasBeforeComment()) { printlnComments(x.getBeforeCommentsDirect()); } + + if (x.getInsertBeforeCommentsDirect() != null) { + printlnComments(x.getInsertBeforeCommentsDirect()); + } + SQLExprTableSource tableSource = x.getTableSource(); if (tableSource != null) { diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index 065a7ed499..17ede2f1bd 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -2245,10 +2245,25 @@ public class Lexer { pos++; bufPos++; + boolean ident = false; for (;;) { ch = charAt(++pos); + if (isEOF()) { + pos--; + bufPos--; + break; + } - if (ch == '}') { + if (ch == '}' && !ident) { + if (isIdentifierChar(charAt(pos + 1))) { + bufPos++; + ident = true; + continue; + } + break; + } + + if (ident && isWhitespace(ch)) { break; } @@ -2256,7 +2271,7 @@ public class Lexer { continue; } - if (ch != '}') { + if (ch != '}' && !ident) { throw new ParserException("syntax error. " + info()); } ++pos; @@ -2273,7 +2288,7 @@ public class Lexer { } stringVal = addSymbol(); - token = Token.VARIANT; + token = ident ? IDENTIFIER : Token.VARIANT; return; } else if (c1 == '$' && charAt(pos + 2) == '{') { pos += 2; @@ -2836,12 +2851,30 @@ public class Lexer { && !(ch == 'b' && bufPos == 1 && charAt(pos - 1) == '0' && dbType != DbType.odps) ) { bufPos++; + boolean brace = false; for (;;) { char c0 = ch; ch = charAt(++pos); + if (isEOF()) { + break; + } + if (!isIdentifierChar(ch)) { - if (ch == '（' || ch == '）' && c0 > 256) { + if (ch == '{' && charAt(pos - 1) == '$' && !brace) { + bufPos++; + brace = true; + continue; + } + + if (ch == '}' && brace) { + bufPos++; + brace = false; + continue; + } + + if ((ch == '（' || ch == '）') + && c0 > 256) { bufPos++; continue; } @@ -3213,7 +3246,7 @@ public class Lexer { || comment.indexOf('&') != -1 // || comment.indexOf('|') != -1 // || comment.indexOf('^') != -1 // - ) { + ) { return false; } return true; diff --git a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java index 92687a2f62..c6e3e98ee8 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java +++ b/src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java @@ -6408,7 +6408,7 @@ public class SQLStatementParser extends SQLParser { insert.setDbType(dbType); if (lexer.isKeepComments() && lexer.hasComment()) { - insert.addBeforeComment(lexer.readAndResetComments()); + insert.addInsertBeforeComment(lexer.readAndResetComments()); } SQLSelectParser selectParser = createSQLSelectParser(); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index 4d23ea5b25..d26344fcaa 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -3377,6 +3377,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } } + if (x.getInsertBeforeCommentsDirect() != null) { + printlnComments(x.getInsertBeforeCommentsDirect()); + } + SQLWithSubqueryClause with = x.getWith(); if (with != null) { visit(with); @@ -6481,7 +6485,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet return; } - if (comment.startsWith("--") && comment.length() > 2 && comment.charAt(2) != ' ') { + if (comment.startsWith("--") + && comment.length() > 2 + && comment.charAt(2) != ' ' + ) { print0("-- "); print0(comment.substring(2)); } else { @@ -6511,7 +6518,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } SQLName changeOwnerTo = x.getChangeOwnerTo(); - if (to != null) { + if (changeOwnerTo != null) { print0(ucase ? " CHANGEOWNER TO " : " changeowner to "); printExpr(changeOwnerTo); } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DALParserTest.java b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DALParserTest.java index 65b5deaed0..af3c43ca1b 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/cobar/DALParserTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/cobar/DALParserTest.java @@ -1243,7 +1243,7 @@ public class DALParserTest extends TestCase { public void test_show_variants() throws Exception { String sql = "SHOW VARIABLES LIKE '%size%';"; MySqlStatementParser parser = new MySqlStatementParser(sql); - MySqlShowVariantsStatement show = (MySqlShowVariantsStatement) parser.parseStatementList().get(0); + SQLShowVariantsStatement show = (SQLShowVariantsStatement) parser.parseStatementList().get(0); parser.match(Token.EOF); String output = SQLUtils.toMySqlString(show); assertEquals("SHOW VARIABLES LIKE '%size%';", output); @@ -1252,7 +1252,7 @@ public class DALParserTest extends TestCase { public void test_show_variants_1() throws Exception { String sql = "SHOW GLOBAL VARIABLES LIKE '%size%';"; MySqlStatementParser parser = new MySqlStatementParser(sql); - MySqlShowVariantsStatement show = (MySqlShowVariantsStatement) parser.parseStatementList().get(0); + SQLShowVariantsStatement show = (SQLShowVariantsStatement) parser.parseStatementList().get(0); parser.match(Token.EOF); String output = SQLUtils.toMySqlString(show); assertEquals("SHOW GLOBAL VARIABLES LIKE '%size%';", output); @@ -1261,7 +1261,7 @@ public class DALParserTest extends TestCase { public void test_show_variants_2() throws Exception { String sql = "SHOW SESSION VARIABLES LIKE '%size%';"; MySqlStatementParser parser = new MySqlStatementParser(sql); - MySqlShowVariantsStatement show = (MySqlShowVariantsStatement) parser.parseStatementList().get(0); + SQLShowVariantsStatement show = (SQLShowVariantsStatement) parser.parseStatementList().get(0); parser.match(Token.EOF); String output = SQLUtils.toMySqlString(show); assertEquals("SHOW SESSION VARIABLES LIKE '%size%';", output); diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlError_test_3.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlError_test_3.java index aee1705f2a..900c37c982 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlError_test_3.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlError_test_3.java @@ -38,6 +38,6 @@ public class MySqlError_test_3 extends MysqlTest { Assert.assertNotNull(error); // error.printStackTrace(); - Assert.assertEquals("syntax error, error in :'us = LIMIT ?, ?, pos 248, line 1, column 248, token ?", error.getMessage()); + Assert.assertEquals("not supported.pos 248, line 1, column 248, token ?", error.getMessage()); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsListResourcesTest.java b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsListResourcesTest.java index 4eb0d67a2c..f27d5ae06a 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsListResourcesTest.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsListResourcesTest.java @@ -20,7 +20,11 @@ import java.io.InputStreamReader; import java.io.Reader; import java.util.List; +import com.alibaba.druid.DbType; +import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor; +import com.alibaba.druid.sql.parser.SQLParserFeature; +import com.alibaba.druid.support.opds.udf.SqlCodeStat; import junit.framework.TestCase; import org.junit.Assert; @@ -39,7 +43,30 @@ public class OdpsListResourcesTest extends TestCase { exec_test("bvt/parser/odps-3.sql"); } + public static String readResource(String resource) throws Exception { + InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); + Reader reader = new InputStreamReader(is, "UTF-8"); + String input = Utils.read(reader); + return input; + } + + public void test_8() throws Exception { + exec_test("bvt/parser/odps-8.txt"); + } + + public void format_resource(String resource) throws Exception { + InputStream is = null; + + is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); + Reader reader = new InputStreamReader(is, "UTF-8"); + String input = Utils.read(reader); + String formattedSql = SQLUtils.format(input, DbType.odps); + System.out.println(formattedSql); + + } + public void exec_test(String resource) throws Exception { +// System.out.println(resource); InputStream is = null; is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); @@ -48,13 +75,28 @@ public class OdpsListResourcesTest extends TestCase { JdbcUtils.close(reader); String[] items = input.split("---------------------------"); String sql = items[0].trim(); - String expect = items[1].trim(); + String expect = null; + if (items.length > 1) { + expect = items[1].trim(); + if (expect != null) { + expect = expect.replaceAll("\\r\\n", "\n"); + } + } - OdpsStatementParser parser = new OdpsStatementParser(sql); + new SqlCodeStat() + .evaluate(sql, "odps"); + + OdpsStatementParser parser = new OdpsStatementParser(sql + , SQLParserFeature.EnableSQLBinaryOpExprGroup + , SQLParserFeature.EnableMultiUnion + , SQLParserFeature.KeepComments); List statementList = parser.parseStatementList(); + if (statementList.size() == 0) { + throw new Exception("empty"); + } SQLStatement stmt = statementList.get(0); - Assert.assertEquals(1, statementList.size()); +// Assert.assertEquals(1, statementList.size()); SchemaStatVisitor visitor = new OdpsSchemaStatVisitor(); stmt.accept(visitor); @@ -65,7 +107,16 @@ public class OdpsListResourcesTest extends TestCase { // // System.out.println(); // System.out.println("---------------------------"); -// System.out.println(SQLUtils.toOdpsString(stmt)); + System.out.println(SQLUtils.toOdpsString(stmt)); + + if (expect != null && !expect.isEmpty()) { + assertEquals(expect, stmt.toString()); + } + + SchemaStatVisitor schemaStatVisitor = SQLUtils.createSchemaStatVisitor(DbType.odps); + for (SQLStatement item : statementList) { + item.accept(schemaStatVisitor); + } } void mergValidate(String sql, String expect) { @@ -86,4 +137,5 @@ public class OdpsListResourcesTest extends TestCase { } + } diff --git a/src/test/resources/bvt/parser/odps-8.txt b/src/test/resources/bvt/parser/odps-8.txt new file mode 100644 index 0000000000..d288734841 --- /dev/null +++ b/src/test/resources/bvt/parser/odps-8.txt @@ -0,0 +1,16 @@ +select * +from ( + select id, collectId, 3 as c_0 + from ( + SELECT * + FROM tc_merchant + where collectId in (select collect_id from cte_merchant) + UNION + SELECT * + FROM st_merchant + where collectId in (select collect_id from st_merchant_tmp) + ) + union + select id, collectId, c2 + from (select id, collectId, c2 from table_2) +) \ No newline at end of file