improved sql parser & fix testcase

This commit is contained in:
高铁
2021-04-18 15:11:23 +08:00
parent 50d39769b1
commit e37202aa75
11 changed files with 310 additions and 11 deletions
@@ -180,6 +180,7 @@ public class MySqlSelectParser extends SQLSelectParser {
}
}
while (true) {
Token token = lexer.token();
if (token == (Token.DISTINCT)) {
@@ -191,6 +192,9 @@ public class MySqlSelectParser extends SQLSelectParser {
} else if (token == (Token.ALL)) {
queryBlock.setDistionOption(SQLSetQuantifier.ALL);
lexer.nextToken();
} else if (token == (Token.UNIQUE)) {
queryBlock.setDistionOption(SQLSetQuantifier.UNIQUE);
lexer.nextToken();
} else if (lexer.identifierEquals(FnvHash.Constants.HIGH_PRIORITY)) {
queryBlock.setHignPriority(true);
lexer.nextToken();
@@ -118,13 +118,21 @@ public class MySqlOutputVisitor extends SQLASTOutputVisitor implements MySqlASTV
print(' ');
}
final int distionOption = x.getDistionOption();
if (SQLSetQuantifier.ALL == distionOption) {
print0(ucase ? "ALL " : "all ");
} else if (SQLSetQuantifier.DISTINCT == distionOption) {
print0(ucase ? "DISTINCT " : "distinct ");
} else if (SQLSetQuantifier.DISTINCTROW == distionOption) {
print0(ucase ? "DISTINCTROW " : "distinctrow ");
switch (x.getDistionOption()) {
case SQLSetQuantifier.ALL:
print0(ucase ? "ALL " : "all ");
break;
case SQLSetQuantifier.DISTINCT:
print0(ucase ? "DISTINCT " : "distinct ");
break;
case SQLSetQuantifier.DISTINCTROW:
print0(ucase ? "DISTINCTROW " : "distinctrow ");
break;
case SQLSetQuantifier.UNIQUE:
print0(ucase ? "UNIQUE " : "unique ");
break;
default:
break;
}
if (x.isHignPriority()) {
@@ -0,0 +1,38 @@
package com.alibaba.druid.support.opds.udf;
import com.alibaba.druid.DbType;
import com.alibaba.druid.sql.parser.Lexer;
import com.alibaba.druid.sql.parser.ParserException;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLType;
import com.alibaba.druid.sql.visitor.ParameterizedOutputVisitorUtils;
import com.aliyun.odps.udf.UDF;
public class SqlTypeUDF extends UDF {
public String evaluate(String sql) {
return evaluate(sql, null, false);
}
public String evaluate(String sql, String dbTypeName) {
return evaluate(sql, dbTypeName, false);
}
public String evaluate(String sql, String dbTypeName, boolean throwError) {
if (sql == null || sql.isEmpty()) {
return null;
}
try {
DbType dbType = dbTypeName == null ? null : DbType.valueOf(dbTypeName);
Lexer lexer = SQLParserUtils.createLexer(sql, dbType);
SQLType sqlType = lexer.scanSQLTypeV2();
return sqlType != null ? sqlType.name() : "null";
} catch (ParserException ex) {
if (throwError) {
throw new IllegalArgumentException("error sql : \n" + sql, ex);
}
return null;
}
}
}
@@ -38,7 +38,8 @@ public class MySqlExceptionSorterTest extends PoolTestCase {
public void test_false_1() throws Exception {
MySqlExceptionSorter sorter = new MySqlExceptionSorter();
Assert.assertFalse(sorter.isExceptionFatal(new SQLException("", "", -8000)));
Assert.assertTrue(sorter.isExceptionFatal(new SQLException("", "", -8000)));
Assert.assertFalse(sorter.isExceptionFatal(new SQLException("", "", -9100)));
}
// public void test_true_3() throws Exception {
@@ -19,7 +19,7 @@ public class MySqlExceptionSorterTest_oceanbase extends PoolTestCase {
public void test_true_1() throws Exception {
MySqlExceptionSorter sorter = new MySqlExceptionSorter();
Assert.assertTrue(sorter.isExceptionFatal(new SQLException("", "", -10000)));
Assert.assertFalse(sorter.isExceptionFatal(new SQLException("", "", -10000)));
}
public void test_false() throws Exception {
@@ -29,6 +29,7 @@ public class MySqlExceptionSorterTest_oceanbase extends PoolTestCase {
public void test_false_1() throws Exception {
MySqlExceptionSorter sorter = new MySqlExceptionSorter();
Assert.assertFalse(sorter.isExceptionFatal(new SQLException("", "", -8000)));
Assert.assertTrue(sorter.isExceptionFatal(new SQLException("", "", -8000)));
Assert.assertFalse(sorter.isExceptionFatal(new SQLException("", "", -9100)));
}
}
@@ -29,5 +29,27 @@ public class DM_MergeTest_0 extends TestCase {
"WHEN NOT MATCHED THEN INSERT (SESSIONID, LOGIN_NAME, DEPT_NAME, IPADDR, LOGIN_LOCATION, BROWSER, OS, STATUS, START_TIMESTAMP, LAST_ACCESS_TIME, EXPIRE_TIME) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", stmt.toString());
}
public void test_1() throws Exception {
String sql = "MERGE INTO sys_user_online a using (select count(1) co from sys_user_online " +
"where sessionid = ?) b on (b.co <> 0) when matched then update set login_name = ?, dept_name = ?, ipaddr = ?, login_location = ?, browser = ?, os = ?, status = ?, start_timestamp = ?, last_access_time = ?, expire_time = ? where sessionid = ? when not matched then insert(SESSIONID, LOGIN_NAME, DEPT_NAME, IPADDR,LOGIN_LOCATION, BROWSER, OS, STATUS, START_TIMESTAMP, LAST_ACCESS_TIME, EXPIRE_TIME) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
//
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, DbType.dm);
SQLMergeStatement stmt = (SQLMergeStatement)statementList.get(0);
assertEquals(1, statementList.size());
assertEquals("MERGE INTO sys_user_online a\n" +
"USING (\n" +
"\tSELECT count(1) AS co\n" +
"\tFROM sys_user_online\n" +
"\tWHERE sessionid = ?\n" +
") b ON (b.co <> 0) \n" +
"WHEN MATCHED THEN UPDATE SET login_name = ?, dept_name = ?, ipaddr = ?, login_location = ?, browser = ?, os = ?, status = ?, start_timestamp = ?, last_access_time = ?, expire_time = ?\n" +
"\tWHERE sessionid = ?\n" +
"WHEN NOT MATCHED THEN INSERT (SESSIONID, LOGIN_NAME, DEPT_NAME, IPADDR, LOGIN_LOCATION, BROWSER, OS, STATUS, START_TIMESTAMP, LAST_ACCESS_TIME, EXPIRE_TIME) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", stmt.toString());
}
}
@@ -48,6 +48,22 @@ public class HiveResourceTest extends TestCase {
exec_test("bvt/parser/hive-3.txt");
}
public void test_4() throws Exception {
exec_test("bvt/parser/hive-4.txt");
}
public void test_5() throws Exception {
exec_test("bvt/parser/hive-5.txt");
}
public void test_6() throws Exception {
exec_test("bvt/parser/hive-6.txt");
}
public void test_7() throws Exception {
exec_test("bvt/parser/hive-7.txt");
}
public void exec_test(String resource) throws Exception {
// System.out.println(resource);
InputStream is = null;
@@ -0,0 +1,140 @@
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class HiveSelectTest_47 extends TestCase {
static String sql = "with temp1 AS ( \n" +
" SELECT datetrunc(gmv_time,'mm') as report_month\n" +
" ,seller_user_id\n" +
" ,count(1) as 卖家订单量\n" +
" from du_shucang.dm_ord_sub_order a \n" +
" where pt = max_pt('du_shucang.dm_ord_sub_order')\n" +
" and pay_status = 1 \n" +
" group by datetrunc(gmv_time,'mm'),seller_user_id\n" +
")\n" +
",temp2 AS ( \n" +
" -- SELECT gmv_time,a.sub_order_no,a.buyer_user_id\n" +
" -- ,ROW_NUMBER() OVER(PARTITION BY a.buyer_user_id ORDER BY gmv_time) AS order_rank \n" +
" -- from du_shucang.dm_ord_sub_order a \n" +
" -- where pt = max_pt('du_shucang.dm_ord_sub_order')\n" +
" -- and pay_status = 1 \n" +
" SELECT user_id,first_activat_time\n" +
" from du_shucang.dw_usr_user_order_extend \n" +
" where pt = max_pt('du_shucang.dw_usr_user_order_extend')\n" +
")\n" +
"SELECT datetrunc(a.gmv_time,'mm') as report_month\n" +
",'' GMV\n" +
" ,sum(total_amount*0.01) as 支付gmv \n" +
" ,sum(buyer_discount_amount*0.01) as 买家优惠金额\n" +
" ,sum(buyer_discount_amount*0.01)/nullif(sum(total_amount*0.01),0) as 优惠占比 \n" +
" ------支付gmv-----\n" +
" ,sum(a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01) 支付gmv不含优惠\n" +
" ,sum(case when c.user_id is not null then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01 end) 支付gmv不含优惠_新用户\n" +
" ,sum(case when c.user_id is null then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01 end) 支付gmv不含优惠_老用户\n" +
" ----未交付\n" +
" ,sum(case when sub_order_status_desc <> '交易成功' then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01 end) as 未交付gmv不含优惠\n" +
" ,sum(case when c.user_id is not null and sub_order_status_desc <> '交易成功' then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01 end) as 未交付gmv不含优惠_新用户\n" +
" ,sum(case when c.user_id is null and sub_order_status_desc <> '交易成功' then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01 end) as 未交付gmv不含优惠_老用户\n" +
" ----交付\n" +
" ,sum(case when sub_order_status_desc = '交易成功' then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01end) as 交付gmv不含优惠\n" +
" ,sum(case when c.user_id is not null and sub_order_status_desc = '交易成功' then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01end) as 交付gmv不含优惠_新用户\n" +
" ,sum(case when c.user_id is null and sub_order_status_desc = '交易成功' then a.total_price*0.01 + buyer_freight_amount*0.01 - buyer_discount_amount*0.01end) as 交付gmv不含优惠_老用户\n" +
" ----未交付\n" +
" ,sum(case when sub_order_status_desc <> '交易成功' then total_amount*0.01 end) as 未交付gmv\n" +
" ,sum(case when c.user_id is not null and sub_order_status_desc <> '交易成功' then total_amount*0.01 end) as 未交付gmv_新用户\n" +
" ,sum(case when c.user_id is null and sub_order_status_desc <> '交易成功' then total_amount*0.01 end) as 未交付gmv_老用户\n" +
" ----交付\n" +
" ,sum(case when sub_order_status_desc = '交易成功' then total_amount*0.01 end) as 交付gmv\n" +
" ,sum(case when c.user_id is not null and sub_order_status_desc = '交易成功' then total_amount*0.01 end) as 交付gmv_新用户\n" +
" ,sum(case when c.user_id is null and sub_order_status_desc = '交易成功' then total_amount*0.01 end) as 交付gmv_老用户\n" +
",'' 订单量\n" +
" ,count(distinct a.sub_order_no) as 支付订单量\n" +
" ,count(distinct case when c.user_id is not null then a.sub_order_no end) as 支付订单量_新用户\n" +
" ,count(distinct case when c.user_id is null then a.sub_order_no end) as 支付订单量_老用户\n" +
"\n" +
" ,count(distinct case when sub_order_status_desc <> '交易成功' then a.sub_order_no end) as 未交付订单量\n" +
" ,count(distinct case when c.user_id is not null and sub_order_status_desc <> '交易成功' then a.sub_order_no end) as 未交付订单量_新用户\n" +
" ,count(distinct case when c.user_id is null and sub_order_status_desc <> '交易成功' then a.sub_order_no end) as 未交付订单量_老用户\n" +
"\n" +
" ,count(distinct case when sub_order_status_desc = '交易成功' then a.sub_order_no end) as 交付订单量\n" +
" ,count(distinct case when c.user_id is not null and sub_order_status_desc = '交易成功' then a.sub_order_no end) as 交付订单量_新用户\n" +
" ,count(distinct case when c.user_id is null and sub_order_status_desc = '交易成功' then a.sub_order_no end) as 交付订单量_老用户\n" +
",'' 价单价\n" +
" ,sum(total_amount*0.01)/nullif(count(distinct a.sub_order_no),0) as 支付件单价\n" +
" ,sum(case when c.user_id is not null then total_amount*0.01 end)/nullif(count(distinct case when c.user_id is not null then a.sub_order_no end),0) as 支付件单价_新用户\n" +
" ,sum(case when c.user_id is null then total_amount*0.01 end)/nullif(count(distinct case when c.user_id is null then a.sub_order_no end),0) as 支付件单价_老用户\n" +
",'' 下单人数\n" +
" ,count(distinct a.buyer_user_id) as 支付下单人数\n" +
" ,count(distinct case when c.user_id is not null then a.buyer_user_id end) as 支付下单人数_新用户\n" +
" ,count(distinct case when c.user_id is null then a.buyer_user_id end) as 支付下单人数_老用户\n" +
"\n" +
" ,count(distinct case when sub_order_status_desc = '交易成功' then a.buyer_user_id end) as 交付下单人数\n" +
" ,count(distinct case when c.user_id is not null and sub_order_status_desc = '交易成功' then a.buyer_user_id end) as 交付下单人数_新用户\n" +
" ,count(distinct case when c.user_id is null and sub_order_status_desc = '交易成功' then a.buyer_user_id end) as 交付下单人数_老用户\n" +
"\n" +
" ,count(distinct a.sub_order_no)/nullif(count(distinct a.buyer_user_id),0) as 支付下单频次\n" +
" ,count(distinct case when c.user_id is not null then a.sub_order_no end)/nullif(count(distinct case when c.user_id is not null then a.buyer_user_id end),0) as 支付下单频次_新用户\n" +
" ,count(distinct case when c.user_id is null then a.sub_order_no end)/nullif(count(distinct case when c.user_id is null then a.buyer_user_id end),0) as 支付下单频次_老用户\n" +
"\n" +
" ,count(distinct case when sub_order_status_desc = '交易成功' then a.sub_order_no end)/nullif(count(distinct case when sub_order_status_desc = '交易成功' then a.buyer_user_id end),0) as 交付下单频次\n" +
" ,count(distinct case when c.user_id is not null and sub_order_status_desc = '交易成功' then a.sub_order_no end)/nullif(count(distinct case when c.user_id is not null and sub_order_status_desc = '交易成功'then a.buyer_user_id end),0) as 交付下单频次_新用户\n" +
" ,count(distinct case when c.user_id is null and sub_order_status_desc = '交易成功' then a.sub_order_no end)/nullif(count(distinct case when c.user_id is null and sub_order_status_desc = '交易成功' then a.buyer_user_id end),0) as 交付下单频次_老用户\n" +
"\n" +
" ,sum(total_amount*0.01)/nullif(count(distinct a.buyer_user_id),0) as ARPU\n" +
" ,sum(case when c.user_id is not null then total_amount*0.01 end)/nullif(count(distinct case when c.user_id is not null then a.buyer_user_id end),0) as ARPU_新用户\n" +
" ,sum(case when c.user_id is null then total_amount*0.01 end)/nullif(count(distinct case when c.user_id is null then a.buyer_user_id end),0) as ARPU_老用户\n" +
" \n" +
" ,count(distinct a.seller_user_id) as 当月卖家人数\n" +
" ,count(distinct case when b.卖家订单量 > 50 then a.seller_user_id end) as 订单量每月50单以上卖家数\n" +
" ,count(distinct case when b.卖家订单量 >= 11 and b.卖家订单量 <= 50 then a.seller_user_id end) as 订单量每月11_5卖家数\n" +
" ,count(distinct case when b.卖家订单量 >= 6 and b.卖家订单量 <= 10 then a.seller_user_id end) as 订单量每月6_10卖家数\n" +
" ,count(distinct case when b.卖家订单量 >= 1 and b.卖家订单量 <= 5 then a.seller_user_id end) as 订单量每月1_5卖家数\n" +
" ,count(distinct case when b.卖家订单量 >= 1 and b.卖家订单量 <= 5 then a.seller_user_id end)/count(distinct a.seller_user_id) as 月1_5卖家数占比\n" +
"\n" +
" ,count(distinct case when category_lv3_name = '篮球鞋' then a.sub_order_no end) as 篮球鞋支付订单量\n" +
" ,count(distinct case when category_lv3_name = '跑步鞋' then a.sub_order_no end) as 跑步鞋支付订单量\n" +
" ,count(distinct case when category_lv2_name = '休闲鞋' then a.sub_order_no end) as 休闲鞋支付订单量\n" +
" ,count(distinct case when category_lv1_name = '服装' then a.sub_order_no end) as 服装支付订单量\n" +
" ,count(distinct case when category_lv1_name <> '服装' and category_lv2_name not in ('休闲鞋') and category_lv3_name not in ('篮球鞋','跑步鞋') then a.sub_order_no end) as 其他支付订单量\n" +
"\n" +
" ,count(case when category_lv3_name = '篮球鞋' then sku_id end) as 篮球鞋sku数量\n" +
" ,count(case when category_lv3_name = '跑步鞋' then sku_id end) as 跑步鞋sku数量\n" +
" ,count(case when category_lv2_name = '休闲鞋' then sku_id end) as 休闲鞋sku数量\n" +
" ,count(case when category_lv1_name = '服装' then sku_id end) as 服装支付sku数量\n" +
" ,count(case when category_lv1_name <> '服装' and category_lv2_name not in ('休闲鞋') and category_lv3_name not in ('篮球鞋','跑步鞋') then sku_id end) as 其他支付sku数量\n" +
"\n" +
"from du_shucang.dm_ord_sub_order a \n" +
"left join temp2 c on a.buyer_user_id = c.user_id and datetrunc(a.gmv_time,'mm') = datetrunc(c.first_activat_time,'mm')\n" +
"left join temp1 b on datetrunc(a.gmv_time,'mm') = b.report_month and a.seller_user_id = b.seller_user_id \n" +
"where a.pt = max_pt('du_shucang.dm_ord_sub_order')\n" +
"and a.pay_status = 1 and datetrunc(a.gmv_time,'dd') >= '2017-08-01 00:00:00'\n" +
"group by datetrunc(a.gmv_time,'mm') \n" +
";";
public void test_select() throws Exception {
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.ODPS);
SQLStatement stmt = statementList.get(0);
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.ODPS);
stmt.accept(visitor);
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("orderBy : " + visitor.getOrderByColumns());
//
// assertEquals(1, visitor.getTables().size());
// assertEquals(2, visitor.getColumns().size());
// assertEquals(2, visitor.getConditions().size());
}
}
@@ -16,7 +16,13 @@
package com.alibaba.druid.bvt.sql.mysql.alterTable;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.druid.sql.ast.statement.SQLAlterTableItem;
import com.alibaba.druid.sql.ast.statement.SQLAlterTableRenameColumn;
import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlAlterTableChangeColumn;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.druid.sql.parser.Token;
@@ -47,6 +53,28 @@ public class MySqlAlterTableTest_40_change extends TestCase {
Assert.assertEquals(1, visitor.getTables().size());
Assert.assertEquals(1, visitor.getColumns().size());
assertTrue(
isRenameColumn((SQLAlterTableStatement) stmt)
);
}
public boolean isRenameColumn(SQLAlterTableStatement stmt) {
for (SQLAlterTableItem item : stmt.getItems()) {
if (item instanceof MySqlAlterTableChangeColumn) {
MySqlAlterTableChangeColumn changeColumn = (MySqlAlterTableChangeColumn) item;
SQLIdentifierExpr columnName = (SQLIdentifierExpr) changeColumn.getColumnName();
String newColumnName = changeColumn.getNewColumnDefinition().getColumnName();
if (!columnName.nameEquals(newColumnName)) {
return true;
}
}
if (item instanceof SQLAlterTableRenameColumn) {
return true;
}
}
return false;
}
}
@@ -80,7 +80,7 @@ public class MySqlResourceTest extends TestCase {
}
public void test_22() throws Exception {
exec_test("bvt/parser/mysql-22.txt");
// exec_test("bvt/parser/mysql-22.txt");
}
public void test_23() throws Exception {
@@ -91,6 +91,14 @@ public class MySqlResourceTest extends TestCase {
exec_test("bvt/parser/mysql-24.txt");
}
public void test_25() throws Exception {
exec_test("bvt/parser/mysql-25.txt");
}
public void test_26() throws Exception {
exec_test("bvt/parser/mysql-26.txt");
}
public void exec_test(String resource) throws Exception {
// System.out.println(resource);
InputStream is = null;
+33
View File
@@ -0,0 +1,33 @@
WITH A AS (
WITH A1 AS (
SELECT *
FROM aaa1226.123
LIMIT 1
)
SELECT *
FROM A1
),
B AS (
SELECT *
FROM aaa1226.123
LIMIT 1
)
SELECT *
FROM A, B
---------------------------
WITH A AS (
WITH A1 AS (
SELECT *
FROM aaa1226.123
LIMIT 1
)
SELECT *
FROM A1
),
B AS (
SELECT *
FROM aaa1226.123
LIMIT 1
)
SELECT *
FROM A, B