improved hive sql parser

This commit is contained in:
高铁
2019-11-02 22:46:16 +08:00
parent 481a0c8380
commit 1cdd3f28eb
20 changed files with 669 additions and 36 deletions
@@ -0,0 +1,72 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class HiveSelectTest_13_cluster_by
extends TestCase {
public void test_0() throws Exception
{
String sql = "select buyer_id,seller_id ,order_id,div_pay_amt,t1.dim_seller.member_id,t1.dim_buyer.member_id from LD_aly.fct_pay_ord_cn_di t1 cluster by buyer_id,seller_id \n";
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
SQLSelectStatement stmt = (SQLSelectStatement) statementList.get(0);
assertEquals("SELECT buyer_id, seller_id, order_id, div_pay_amt, t1.dim_seller.member_id\n" +
"\t, t1.dim_buyer.member_id\n" +
"FROM LD_aly.fct_pay_ord_cn_di t1\n" +
"CLUSTER BY buyer_id, seller_id", stmt.toString());
assertEquals("SELECT buyer_id, seller_id, order_id, div_pay_amt, t1.dim_seller.member_id\n" +
"\t, t1.dim_buyer.member_id\n" +
"FROM LD_aly.fct_pay_ord_cn_di t1\n" +
"CLUSTER BY buyer_id, seller_id", SQLUtils.toSQLString(stmt));
assertEquals(1, statementList.size());
for (SQLSelectOrderByItem item : stmt.getSelect().getQueryBlock().getClusterBy()) {
assertFalse(item.isSortBy());
assertFalse(item.isDistributeBy());
assertTrue(item.isClusterBy());
}
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("orderBy : " + visitor.getOrderByColumns());
System.out.println("groupBy : " + visitor.getGroupByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(5, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertEquals(0, visitor.getGroupByColumns().size());
assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "buyer_id"));
}
}
@@ -0,0 +1,44 @@
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class HiveSelectTest_3_array
extends TestCase {
public void test_select() throws Exception
{
String sql = "select languages[0] from json_nested_test;";//
assertEquals("SELECT languages[0]\n" +
"FROM json_nested_test;", SQLUtils.formatHive(sql));
assertEquals("select languages[0]\n" +
"from json_nested_test;", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
SQLStatement stmt = statementList.get(0);
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
stmt.accept(visitor);
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(1, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertTrue(visitor.containsColumn("json_nested_test", "languages"));
// assertTrue(visitor.containsColumn("customers", "isxx"));
// assertTrue(visitor.containsColumn("customers", "*"));
}
}
@@ -0,0 +1,44 @@
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class HiveSelectTest_4_array
extends TestCase {
public void test_select() throws Exception
{
String sql = "select religions['catholic'][0] from json_nested_test;";//
assertEquals("SELECT religions['catholic'][0]\n" +
"FROM json_nested_test;", SQLUtils.formatHive(sql));
assertEquals("select religions['catholic'][0]\n" +
"from json_nested_test;", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
SQLStatement stmt = statementList.get(0);
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
stmt.accept(visitor);
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(1, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertTrue(visitor.containsColumn("json_nested_test", "religions"));
// assertTrue(visitor.containsColumn("customers", "isxx"));
// assertTrue(visitor.containsColumn("customers", "*"));
}
}
@@ -0,0 +1,62 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class HiveSelectTest_cluster_by
extends TestCase {
public void test_0() throws Exception
{
String sql = "SELECT a from x cluster by a;";
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
SQLStatement stmt = statementList.get(0);
assertEquals("SELECT a\n" +
"FROM x\n" +
"CLUSTER BY a;", stmt.toString());
assertEquals("SELECT a\n" +
"FROM x\n" +
"CLUSTER BY a;", SQLUtils.toSQLString(stmt));
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("orderBy : " + visitor.getOrderByColumns());
System.out.println("groupBy : " + visitor.getGroupByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(1, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertEquals(0, visitor.getGroupByColumns().size());
assertTrue(visitor.containsColumn("x", "a"));
}
}
@@ -0,0 +1,66 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.List;
public class HiveSelectTest_distribute
extends TestCase {
public void test_0() throws Exception
{
String sql = "select * from LD_aly.fct_pay_ord_cn_di t1 distribute by buyer_id sort by seller_id";
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
SQLStatement stmt = statementList.get(0);
assertEquals("SELECT *\n" +
"FROM LD_aly.fct_pay_ord_cn_di t1\n" +
"DISTRIBUTE BY buyer_id\n" +
"SORT BY seller_id", stmt.toString());
assertEquals("SELECT *\n" +
"FROM LD_aly.fct_pay_ord_cn_di t1\n" +
"DISTRIBUTE BY buyer_id\n" +
"SORT BY seller_id", SQLUtils.toSQLString(stmt));
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("orderBy : " + visitor.getOrderByColumns());
System.out.println("groupBy : " + visitor.getGroupByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(3, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertEquals(0, visitor.getGroupByColumns().size());
assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "*"));
assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "buyer_id"));
assertTrue(visitor.containsColumn("LD_aly.fct_pay_ord_cn_di", "seller_id"));
}
}
@@ -0,0 +1,106 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.bvt.sql.mysql.select;
import com.alibaba.druid.sql.MysqlTest;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
public class MySqlSelectTest_with_20433301 extends MysqlTest {
public void test_0() throws Exception {
String sql = "WITH total AS(\n" +
"SELECT cinema_id, count(1) as scheduleCnt, sum(hall_seat_cnt) as hall_seat_cnt, sum(sold_seat_cnt) as sold_seat_cnt, sum(avg_ticket_price*sold_seat_cnt) as box_office\n" +
"  FROM gw_ads_schedule_rt\n" +
" WHERE show_date= 20190513\n" +
"   and stat_month= 201905\n" +
"   and stat_date= 20190508\n" +
"   and scd_is_valid= 1\n" +
"   and scd_status= 4\n" +
" GROUP BY cinema_id),\n" +
"         detail AS(\n" +
"SELECT show_id, cinema_id, count(1) as scheduleCnt, sum(hall_seat_cnt) as hall_seat_cnt, sum(sold_seat_cnt) as sold_seat_cnt, sum(avg_ticket_price*sold_seat_cnt) as box_office\n" +
"  FROM gw_ads_schedule_rt\n" +
" WHERE show_date= 20190513\n" +
"   and stat_month= 201905\n" +
"   and stat_date= 20190508\n" +
"   and scd_is_valid= 1\n" +
"   and scd_status= 4\n" +
"   AND show_id IN(1229077)\n" +
" GROUP BY show_id, cinema_id)\n" +
"SELECT detail.cinema_id,\n" +
"       detail.show_id,\n" +
"       detail.scheduleCnt,\n" +
"       detail.hall_seat_cnt,\n" +
"       detail.sold_seat_cnt,\n" +
"       detail.box_office,\n" +
"       total.scheduleCnt as totalScheduleCnt,\n" +
"       total.hall_seat_cnt as totalHallSeatCnt,\n" +
"       total.sold_seat_cnt as totalSoldSeatCnt,\n" +
"       total.box_office as totalBoxOffice,\n" +
"       fcst.scd_cnt_fcst as predictScheduleCnt\n" +
"  FROM detail join total on detail.cinema_id= total.cinema_id left join gw_ads_cinema_schedule_fcst fcst on fcst.cinema_id= detail.cinema_id\n" +
" where fcst.week_day= 1\n" +
" order by detail.scheduleCnt desc";
SQLSelectStatement stmt = (SQLSelectStatement) SQLUtils.parseSingleStatement(sql, JdbcConstants.MYSQL);
assertEquals("WITH total AS (\n" +
"\t\tSELECT cinema_id, count(1) AS scheduleCnt, sum(hall_seat_cnt) AS hall_seat_cnt\n" +
"\t\t\t, sum(sold_seat_cnt) AS sold_seat_cnt\n" +
"\t\t\t, sum(avg_ticket_price * sold_seat_cnt) AS box_office\n" +
"\t\tFROM gw_ads_schedule_rt\n" +
"\t\tWHERE show_date = 20190513\n" +
"\t\t\tAND stat_month = 201905\n" +
"\t\t\tAND stat_date = 20190508\n" +
"\t\t\tAND scd_is_valid = 1\n" +
"\t\t\tAND scd_status = 4\n" +
"\t\tGROUP BY cinema_id\n" +
"\t), \n" +
"\tdetail AS (\n" +
"\t\tSELECT show_id, cinema_id, count(1) AS scheduleCnt\n" +
"\t\t\t, sum(hall_seat_cnt) AS hall_seat_cnt, sum(sold_seat_cnt) AS sold_seat_cnt\n" +
"\t\t\t, sum(avg_ticket_price * sold_seat_cnt) AS box_office\n" +
"\t\tFROM gw_ads_schedule_rt\n" +
"\t\tWHERE show_date = 20190513\n" +
"\t\t\tAND stat_month = 201905\n" +
"\t\t\tAND stat_date = 20190508\n" +
"\t\t\tAND scd_is_valid = 1\n" +
"\t\t\tAND scd_status = 4\n" +
"\t\t\tAND show_id IN (1229077)\n" +
"\t\tGROUP BY show_id, cinema_id\n" +
"\t)\n" +
"SELECT detail.cinema_id, detail.show_id, detail.scheduleCnt, detail.hall_seat_cnt, detail.sold_seat_cnt\n" +
"\t, detail.box_office, total.scheduleCnt AS totalScheduleCnt, total.hall_seat_cnt AS totalHallSeatCnt, total.sold_seat_cnt AS totalSoldSeatCnt, total.box_office AS totalBoxOffice\n" +
"\t, fcst.scd_cnt_fcst AS predictScheduleCnt\n" +
"FROM detail\n" +
"\tJOIN total ON detail.cinema_id = total.cinema_id\n" +
"\tLEFT JOIN gw_ads_cinema_schedule_fcst fcst ON fcst.cinema_id = detail.cinema_id\n" +
"WHERE fcst.week_day = 1\n" +
"ORDER BY detail.scheduleCnt DESC", stmt.toString());
MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor();
stmt.accept(visitor);
}
}
@@ -0,0 +1,113 @@
package com.alibaba.druid.bvt.sql.postgresql.select;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
import junit.framework.TestCase;
import org.junit.Assert;
public class PGUnionTest
extends TestCase {
public void testUnion(){
String sql = "(select id,name from t1) union (select id,name from t2)";
String targetSql = "(SELECT id, name\n"
+ "FROM t1)\n"
+ "UNION\n"
+ "(SELECT id, name\n"
+ "FROM t2)";
PGSQLStatementParser pgparser=new PGSQLStatementParser(sql);
SQLStatement pgstatement = pgparser.parseStatement();
Assert.assertEquals(targetSql, pgstatement.toString());
System.out.println(pgstatement.toString());
}
//
// public void testUnion1(){
// String sql = "select * from test_a where id=(select id from test_b where id = 1) union all select * from test_c";
// String targetSql = "SELECT *\n" +
// "FROM test_a\n" +
// "WHERE id = (\n" +
// "\tSELECT id\n" +
// "\tFROM test_b\n" +
// "\tWHERE id = 1\n" +
// ")\n" +
// "UNION ALL\n" +
// "SELECT *\n" +
// "FROM test_c";
//
// PGSQLStatementParser pgparser=new PGSQLStatementParser(sql);
// SQLStatement pgstatement = pgparser.parseStatement();
// Assert.assertEquals(targetSql, pgstatement.toString());
// System.out.println(pgstatement.toString());
// }
public void testUnion2(){
String sql = "select \n" +
"( \n" +
" ( \n" +
" select 'y' from dual \n" +
" where exists ( select 1 from dual where 1 = 0 ) \n" +
" ) \n" +
" union \n" +
" ( \n" +
" select 'n' from dual \n" +
" where not exists ( select 1 from dual where 1 = 0 ) \n" +
" ) \n" +
") \n" +
"as yes_no \n" +
"from dual";
String targetSql = "SELECT (\n" +
"\t\tSELECT 'y'\n" +
"\t\tFROM dual\n" +
"\t\tWHERE EXISTS (\n" +
"\t\t\tSELECT 1\n" +
"\t\t\tFROM dual\n" +
"\t\t\tWHERE 1 = 0\n" +
"\t\t)\n" +
"\t\tUNION\n" +
"\t\t(SELECT 'n'\n" +
"\t\tFROM dual\n" +
"\t\tWHERE NOT EXISTS (\n" +
"\t\t\tSELECT 1\n" +
"\t\t\tFROM dual\n" +
"\t\t\tWHERE 1 = 0\n" +
"\t\t))\n" +
"\t) AS yes_no\n" +
"FROM dual";
PGSQLStatementParser pgparser=new PGSQLStatementParser(sql);
SQLStatement pgstatement = pgparser.parseStatement();
Assert.assertEquals(targetSql, pgstatement.toString());
System.out.println(pgstatement.toString());
}
public void testUnion3(){
String sql = "select * from v.e\n" +
"where\n" +
"\tcid <> rid\n" +
"\tand rid not in\n" +
"\t(\n" +
"\t\t(select distinct rid from v.s )\n" +
"\t\tunion\n" +
"\t\t(select distinct rid from v.p )\n" +
"\t)\n" +
"\tand \"timestamp\" <= 1298505600000";
String targetSql = "SELECT *\n" +
"FROM v.e\n" +
"WHERE cid <> rid\n" +
"\tAND rid NOT IN (\n" +
"\t\tSELECT DISTINCT rid\n" +
"\t\tFROM v.s\n" +
"\t\tUNION\n" +
"\t\t(SELECT DISTINCT rid\n" +
"\t\tFROM v.p)\n" +
"\t)\n" +
"\tAND 'timestamp' <= 1298505600000";
SQLStatement pgstatement = SQLUtils.parseSingleMysqlStatement(sql);
Assert.assertEquals(targetSql, pgstatement.toString());
System.out.println(pgstatement.toString());
}
}