fix : sql parser & output miss comments

This commit is contained in:
shaojin.wensj
2023-02-18 22:06:29 +08:00
parent e1c3b6870f
commit a8aa5c29c7
5 changed files with 85 additions and 2 deletions
@@ -163,6 +163,11 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis
println();
}
List<String> insertBeforeComments = x.getInsertBeforeCommentsDirect();
if (insertBeforeComments != null) {
printlnComments(insertBeforeComments);
}
if (x.isOverwrite()) {
print0(ucase ? "INSERT OVERWRITE TABLE " : "insert overwrite table ");
} else {
@@ -856,13 +856,29 @@ public class SQLSelectParser extends SQLParser {
}
for (; ; ) {
List<String> comments = null;
if (lexer.hasComment()) {
comments = lexer.readAndResetComments();
}
SQLExpr item = parseGroupByItem();
if (comments != null) {
item.addBeforeComment(comments);
}
item.setParent(groupBy);
groupBy.addItem(item);
if (lexer.token == Token.COMMA) {
int line = lexer.line;
lexer.nextToken();
if (lexer.hasComment()
&& lexer.isKeepComments()
&& lexer.getComments().size() == 1
&& lexer.getComments().get(0).startsWith("--")
&& lexer.line == line + 1) {
item.addAfterComment(lexer.readAndResetComments());
}
continue;
} else if (lexer.identifierEquals(FnvHash.Constants.GROUPING)) {
continue;
@@ -2378,6 +2378,13 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
this.indentCount++;
for (int i = 0; i < itemSize; ++i) {
SQLExpr item = items.get(i);
if (isPrettyFormat() && item.hasBeforeComment()) {
if (i != 0) {
print(' ');
}
printlnComments(item.getBeforeCommentsDirect());
}
if (i != 0) {
if (groupItemSingleLine) {
println(", ");
@@ -47,11 +47,23 @@ public class OdpsResourceTest extends TestCase {
public void test_10() throws Exception {
exec_test("bvt/parser/odps-10.txt");
}
public void test_11() throws Exception {
exec_test("bvt/parser/odps-11.txt");
}
public void test_12() throws Exception {
exec_test("bvt/parser/odps-12.txt");
}
//
// public void test_13() throws Exception {
// exec_test("bvt/parser/odps-13.txt");
// }
public void test_14() throws Exception {
exec_test("bvt/parser/odps-14.txt");
}
public void exec_test(String resource) throws Exception {
String input = TestUtil.getResource(resource);
String[] items = input.split("---------------------------");
@@ -62,8 +74,7 @@ public class OdpsResourceTest extends TestCase {
expect = items[1].trim();
}
OdpsStatementParser parser = new OdpsStatementParser(sql);
parser.config(SQLParserFeature.KeepComments, true);
OdpsStatementParser parser = new OdpsStatementParser(sql, SQLParserFeature.KeepComments);
List<SQLStatement> statementList = parser.parseStatementList();
SQLStatement stmt = statementList.get(0);
@@ -0,0 +1,44 @@
-- 1 开头注释 ok
insert into table tmp.abc
-- 1.1 test ok
select -- 2 select注释
-- 3 空白注释
a,b,-- 4 空白注释 ok
c, -- 5 空白注释 ok
d,e,f -- 6 空白注释
-- 7 空白注释
from tmp t1 -- 8 空白注释 ok
-- 9 空白注释 ok
group by -- 10 空白注释 ok
a,b,c,
d,-- 11 空白注释 ok
-- 12 空白注释 ok
e,f
-- 13 空白注释 ok
grouping sets( -- 14 空白注释
-- 15 空白注释
(a,b,c),(b,c,d),(d,e,f))
-- 16 空白注释 ok
---------------------------
-- 1 开头注释 ok
INSERT INTO TABLE tmp.abc
-- 1.1 test ok
SELECT a -- 2 select注释
-- 3 空白注释
, b -- 4 空白注释 ok
, c -- 5 空白注释 ok
, d, e, f -- 6 空白注释
-- 7 空白注释
FROM tmp t1 -- 8 空白注释 ok
-- 9 空白注释 ok
GROUP BY -- 10 空白注释 ok
a,
b,
c,
d -- 11 空白注释 ok
-- 12 空白注释 ok
,
e,
f -- 13 空白注释 ok
,
GROUPING SETS ((a, b, c), (b, c, d), (d, e, f))