improved sql parser.

This commit is contained in:
wenshao
2017-09-13 20:56:20 +08:00
parent c49078c650
commit f208f42e01
33 changed files with 853 additions and 29 deletions
@@ -0,0 +1,42 @@
package com.alibaba.druid.bvt.sql.elastic;
import com.alibaba.druid.filter.stat.MergeStatFilter;
import com.alibaba.druid.filter.stat.StatFilter;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
public class Elastic_StatFilter_merge_0 extends TestCase {
public void test_for_stat_merge() throws Exception {
String sql = "/* ///9fe39e72/ */SELECT fkc1.fkc_order_id, fkc1.out_order_id, fkc1.seller_nick, fkc1.buyer_nick, fkc1.seller_id, fkc1.buyer_id, fkc1.auction_id, fkc1.auction_title, fkc1.auction_price, fkc1.buy_amount, fkc1.fkc_type, fkc1.sub_fkc_type, fkc1.fail_reason, fkc1.pay_status, fkc1.logistics_status, fkc1.out_trade_status, fkc1.snap_path, fkc1.gmt_create, fkc1.status, ifnull(fkc1.buyer_rate_status, 4) AS buyer_rate_status, ifnull(fkc1.seller_rate_status, 4) AS seller_rate_status, fkc1.auction_pict_url, fkc1.seller_memo, fkc1.buyer_memo, fkc1.seller_flag, fkc1.buyer_flag, fkc1.buyer_message_path, fkc1.refund_status, fkc1.attributes, fkc1.attributes_cc, fkc1.gmt_modified, fkc1.ip, fkc1.end_time, fkc1.pay_time, fkc1.is_main, fkc1.is_detail, fkc1.point_rate, fkc1.parent_id, fkc1.adjust_fee, fkc1.discount_fee, fkc1.refund_fee, fkc1.confirm_paid_fee, fkc1.cod_status, fkc1.trade_tag, fkc1.shop_id, fkc1.options, fkc1.ignore_sold_quantity, fkc1.from_group, fkc1.gmt_create_tc FROM (SELECT fkc_order_id FROM tc_fkc_order AS tc_fkc_order IGNORE INDEX (ind_fkc_order_status, idx_sid_bt_st_im) WHERE is_main = 1 AND (fkc_type = 2700 OR fkc_type = 200 OR fkc_type = 10000 OR fkc_type = 610 OR fkc_type = 600 OR fkc_type = 300 OR fkc_type = 100 OR fkc_type = 500 OR fkc_type = 900 OR fkc_type = 620 OR fkc_type = 630 OR fkc_type = 650 OR fkc_type = 1000 OR fkc_type = 1001 OR fkc_type = 710 OR fkc_type = 1200 OR fkc_type = 1201 OR fkc_type = 1500 OR fkc_type = 1600 OR fkc_type = 2000 OR fkc_type = 2400) AND seller_id = 2169996291 AND (sub_fkc_type = 1 OR sub_fkc_type = 901 OR sub_fkc_type = 801 OR sub_fkc_type = 701 OR sub_fkc_type = 601 OR sub_fkc_type = 2) AND status = 0 AND gmt_create >= DATE_FORMAT('2017-04-22 00:00:00', '%Y-%m-%d %T') AND gmt_create <= DATE_FORMAT('2017-05-07 23:59:59', '%Y-%m-%d %T') AND from_group = 0 ORDER BY gmt_create DESC LIMIT 900, 100) AS fkc2, tc_fkc_order AS fkc1 WHERE fkc2.fkc_order_id = 1";
StatFilter filter = new MergeStatFilter();
filter.setDbType(JdbcConstants.ELASTIC_SEARCH);
String mergedSql = filter.mergeSql(sql, JdbcConstants.ELASTIC_SEARCH);
assertEquals("SELECT fkc1.fkc_order_id, fkc1.out_order_id, fkc1.seller_nick, fkc1.buyer_nick, fkc1.seller_id\n" +
"\t, fkc1.buyer_id, fkc1.auction_id, fkc1.auction_title, fkc1.auction_price, fkc1.buy_amount\n" +
"\t, fkc1.fkc_type, fkc1.sub_fkc_type, fkc1.fail_reason, fkc1.pay_status, fkc1.logistics_status\n" +
"\t, fkc1.out_trade_status, fkc1.snap_path, fkc1.gmt_create, fkc1.status\n" +
"\t, ifnull(fkc1.buyer_rate_status, ?) AS buyer_rate_status\n" +
"\t, ifnull(fkc1.seller_rate_status, ?) AS seller_rate_status, fkc1.auction_pict_url\n" +
"\t, fkc1.seller_memo, fkc1.buyer_memo, fkc1.seller_flag, fkc1.buyer_flag, fkc1.buyer_message_path\n" +
"\t, fkc1.refund_status, fkc1.attributes, fkc1.attributes_cc, fkc1.gmt_modified, fkc1.ip\n" +
"\t, fkc1.end_time, fkc1.pay_time, fkc1.is_main, fkc1.is_detail, fkc1.point_rate\n" +
"\t, fkc1.parent_id, fkc1.adjust_fee, fkc1.discount_fee, fkc1.refund_fee, fkc1.confirm_paid_fee\n" +
"\t, fkc1.cod_status, fkc1.trade_tag, fkc1.shop_id, fkc1.options, fkc1.ignore_sold_quantity\n" +
"\t, fkc1.from_group, fkc1.gmt_create_tc\n" +
"FROM (\n" +
"\tSELECT fkc_order_id\n" +
"\tFROM tc_fkc_order tc_fkc_order IGNORE INDEX (ind_fkc_order_status, idx_sid_bt_st_im)\n" +
"\tWHERE is_main = ?\n" +
"\t\tAND (fkc_type = ?)\n" +
"\t\tAND seller_id = ?\n" +
"\t\tAND (sub_fkc_type = ?)\n" +
"\t\tAND status = ?\n" +
"\t\tAND gmt_create >= DATE_FORMAT(?, '%Y-%m-%d %T')\n" +
"\t\tAND gmt_create <= DATE_FORMAT(?, '%Y-%m-%d %T')\n" +
"\t\tAND from_group = ?\n" +
"\tORDER BY gmt_create DESC\n" +
"\tLIMIT ?, ?\n" +
") fkc2, tc_fkc_order fkc1\n" +
"WHERE fkc2.fkc_order_id = ?", mergedSql);
}
}
@@ -0,0 +1,47 @@
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.stat.TableStat;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import org.junit.Assert;
import java.util.List;
public class HiveSelectTest_0 extends TestCase {
public void test_select() throws Exception {
String sql = "SELECT page_views.*\n" +
"FROM page_views\n" +
"WHERE page_views.date >= '2008-03-01' AND page_views.date <= '2008-03-31'";//
Assert.assertEquals("SELECT page_views.*\n" +
"FROM page_views\n" +
"WHERE page_views.date >= '2008-03-01'\n" +
"\tAND page_views.date <= '2008-03-31'", SQLUtils.formatHive(sql));
Assert.assertEquals("select page_views.*\n" +
"from page_views\n" +
"where page_views.date >= '2008-03-01'\n" +
"\tand page_views.date <= '2008-03-31'", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
SQLStatement stmt = statementList.get(0);
Assert.assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
// System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
Assert.assertEquals(1, visitor.getTables().size());
Assert.assertEquals(2, visitor.getColumns().size());
Assert.assertEquals(2, visitor.getConditions().size());
Assert.assertTrue(visitor.containsColumn("page_views", "date"));
Assert.assertTrue(visitor.containsColumn("page_views", "*"));
}
}
@@ -0,0 +1,44 @@
package com.alibaba.druid.bvt.sql.hive;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import org.junit.Assert;
import java.util.List;
public class HiveSelectTest_1_limit extends TestCase {
public void test_select() throws Exception {
String sql = "SELECT * FROM customers ORDER BY create_date LIMIT 2,5";//
assertEquals("SELECT *\n" +
"FROM customers\n" +
"ORDER BY create_date\n" +
"LIMIT 2, 5", SQLUtils.formatHive(sql));
assertEquals("select *\n" +
"from customers\n" +
"order by create_date\n" +
"limit 2, 5", SQLUtils.formatHive(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.HIVE);
SQLStatement stmt = statementList.get(0);
assertEquals(1, statementList.size());
SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.HIVE);
stmt.accept(visitor);
// System.out.println("Tables : " + visitor.getTables());
// System.out.println("fields : " + visitor.getColumns());
// System.out.println("coditions : " + visitor.getConditions());
// System.out.println("orderBy : " + visitor.getOrderByColumns());
assertEquals(1, visitor.getTables().size());
assertEquals(2, visitor.getColumns().size());
assertEquals(0, visitor.getConditions().size());
assertTrue(visitor.containsColumn("customers", "create_date"));
assertTrue(visitor.containsColumn("customers", "*"));
}
}
@@ -17,9 +17,9 @@ import java.util.List;
public class MySqlParameterizedOutputVisitorTest_52_schema extends TestCase {
public void test_for_parameterize() throws Exception {
final String dbType = JdbcConstants.MYSQL;
String sql = "UPDATE `buyer_0158`.xc_yy_order\n" +
"SET `ALIPAY_BUYER_ID` = ?, `GMT_MODIFIED` = ?\n" +
"WHERE `PAY_ORDER_ID` = ?";
String sql = "UPDATE `buyer_0158`.xc_yy_order_1082\n" +
"SET `ALIPAY_BUYER_ID` = 1025, `GMT_MODIFIED` = 'aaaa'\n" +
"WHERE `PAY_ORDER_ID` = 'kkk'";
SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType);
List<SQLStatement> stmtList = parser.parseStatementList();
@@ -0,0 +1,72 @@
package com.alibaba.druid.bvt.sql.mysql.param;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.ArrayList;
import java.util.List;
/**
* Created by wenshao on 16/8/23.
*/
public class MySqlParameterizedOutputVisitorTest_55 extends TestCase {
public void test_for_parameterize() throws Exception {
final String dbType = JdbcConstants.MYSQL;
String sql = "/* ///9fe39e72/ */SELECT fck1.fck_order_id, fck1.out_order_id, fck1.seller_nick, fck1.buyer_nick, fck1.seller_id, fck1.buyer_id, fck1.auction_id, fck1.auction_title, fck1.auction_price, fck1.buy_amount, fck1.fck_type, fck1.sub_fck_type, fck1.fail_reason, fck1.pay_status, fck1.logistics_status, fck1.out_trade_status, fck1.snap_path, fck1.gmt_create, fck1.status, ifnull(fck1.buyer_rate_status, 4) AS buyer_rate_status, ifnull(fck1.seller_rate_status, 4) AS seller_rate_status, fck1.auction_pict_url, fck1.seller_memo, fck1.buyer_memo, fck1.seller_flag, fck1.buyer_flag, fck1.buyer_message_path, fck1.refund_status, fck1.attributes, fck1.attributes_cc, fck1.gmt_modified, fck1.ip, fck1.end_time, fck1.pay_time, fck1.is_main, fck1.is_detail, fck1.point_rate, fck1.parent_id, fck1.adjust_fee, fck1.discount_fee, fck1.refund_fee, fck1.confirm_paid_fee, fck1.cod_status, fck1.trade_tag, fck1.shop_id, fck1.options, fck1.ignore_sold_quantity, fck1.from_group, fck1.gmt_create_tc FROM (SELECT fck_order_id FROM tc_fck_order AS tc_fck_order IGNORE INDEX (ind_fck_order_status, idx_sid_bt_st_im) WHERE is_main = 1 AND (fck_type = 2700 OR fck_type = 200 OR fck_type = 10000 OR fck_type = 610 OR fck_type = 600 OR fck_type = 300 OR fck_type = 100 OR fck_type = 500 OR fck_type = 900 OR fck_type = 620 OR fck_type = 630 OR fck_type = 650 OR fck_type = 1000 OR fck_type = 1001 OR fck_type = 710 OR fck_type = 1200 OR fck_type = 1201 OR fck_type = 1500 OR fck_type = 1600 OR fck_type = 2000 OR fck_type = 2400) AND seller_id = 2169996291 AND (sub_fck_type = 1 OR sub_fck_type = 901 OR sub_fck_type = 801 OR sub_fck_type = 701 OR sub_fck_type = 601 OR sub_fck_type = 2) AND status = 0 AND gmt_create >= DATE_FORMAT('2017-04-22 00:00:00', '%Y-%m-%d %T') AND gmt_create <= DATE_FORMAT('2017-05-07 23:59:59', '%Y-%m-%d %T') AND from_group = 0 ORDER BY gmt_create DESC LIMIT 900, 100) AS fck2, tc_fck_order AS fck1 WHERE fck2.fck_order_id = fck1.fck_order_id";
SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType);
List<SQLStatement> stmtList = parser.parseStatementList();
SQLStatement statement = stmtList.get(0);
StringBuilder out = new StringBuilder();
SQLASTOutputVisitor visitor = SQLUtils.createOutputVisitor(out, JdbcConstants.MYSQL);
List<Object> parameters = new ArrayList<Object>();
visitor.setParameterized(true);
visitor.setParameterizedMergeInList(true);
visitor.setParameters(parameters);
/*visitor.setPrettyFormat(false);*/
statement.accept(visitor);
/* JSONArray array = new JSONArray();
for(String table : visitor.getTables()){
array.add(table.replaceAll("`",""));
}*/
String psql = out.toString();
System.out.println(psql);
assertEquals("SELECT fck1.fck_order_id, fck1.out_order_id, fck1.seller_nick, fck1.buyer_nick, fck1.seller_id\n" +
"\t, fck1.buyer_id, fck1.auction_id, fck1.auction_title, fck1.auction_price, fck1.buy_amount\n" +
"\t, fck1.fck_type, fck1.sub_fck_type, fck1.fail_reason, fck1.pay_status, fck1.logistics_status\n" +
"\t, fck1.out_trade_status, fck1.snap_path, fck1.gmt_create, fck1.status\n" +
"\t, ifnull(fck1.buyer_rate_status, ?) AS buyer_rate_status\n" +
"\t, ifnull(fck1.seller_rate_status, ?) AS seller_rate_status, fck1.auction_pict_url\n" +
"\t, fck1.seller_memo, fck1.buyer_memo, fck1.seller_flag, fck1.buyer_flag, fck1.buyer_message_path\n" +
"\t, fck1.refund_status, fck1.attributes, fck1.attributes_cc, fck1.gmt_modified, fck1.ip\n" +
"\t, fck1.end_time, fck1.pay_time, fck1.is_main, fck1.is_detail, fck1.point_rate\n" +
"\t, fck1.parent_id, fck1.adjust_fee, fck1.discount_fee, fck1.refund_fee, fck1.confirm_paid_fee\n" +
"\t, fck1.cod_status, fck1.trade_tag, fck1.shop_id, fck1.options, fck1.ignore_sold_quantity\n" +
"\t, fck1.from_group, fck1.gmt_create_tc\n" +
"FROM (\n" +
"\tSELECT fck_order_id\n" +
"\tFROM tc_fck_order tc_fck_order IGNORE INDEX (ind_fck_order_status, idx_sid_bt_st_im)\n" +
"\tWHERE is_main = ?\n" +
"\t\tAND (fck_type = ?)\n" +
"\t\tAND seller_id = ?\n" +
"\t\tAND (sub_fck_type = ?)\n" +
"\t\tAND status = ?\n" +
"\t\tAND gmt_create >= DATE_FORMAT(?, '%Y-%m-%d %T')\n" +
"\t\tAND gmt_create <= DATE_FORMAT(?, '%Y-%m-%d %T')\n" +
"\t\tAND from_group = ?\n" +
"\tORDER BY gmt_create DESC\n" +
"\tLIMIT ?, ?\n" +
") fck2, tc_fck_order fck1\n" +
"WHERE fck2.fck_order_id = fck1.fck_order_id", psql);
}
}
@@ -0,0 +1,60 @@
package com.alibaba.druid.bvt.sql.oracle.visitor;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import com.alibaba.druid.util.JdbcConstants;
import com.sun.org.apache.xerces.internal.xs.datatypes.ObjectList;
import junit.framework.TestCase;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* Created by wenshao on 16/8/23.
*/
public class OracleParameterizedOutputVisitorTest_01 extends TestCase {
public void test_for_parameterize() throws Exception {
String sql = "SELECT dd.synonym_name table_name,'EAIREP' as schema_name, aa.num_rows,bb.comments description,to_char(cc.created,'YYYY-MM-DD HH24:MI:SS') createTime,to_char(cc.last_ddl_time,'YYYY-MM-DD HH24:MI:SS') last_ddl_time, 'SYNONYM' table_type ,lower(aa.owner || '.' || aa.table_name) ref_info\n" +
" from dba_synonyms dd,\n" +
" dba_objects cc,\n" +
" dba_tables aa\n" +
" left join dba_tab_comments bb\n" +
" on aa.owner = bb.owner\n" +
" and aa.table_name = bb.table_name\n" +
" where aa.owner=cc.owner\n" +
" and aa.table_name=cc.object_name\n" +
" and cc.subobject_name is null\n" +
" and cc.object_type='TABLE'\n" +
" and aa.owner = dd.table_owner\n" +
" and aa.table_name = dd.table_name and dd.owner = :1 ";
List<Object> parameters = new ArrayList<Object>();
parameters.add("EAIREP");
List<SQLStatement> stmts = SQLUtils.parseStatements(sql, JdbcConstants.ORACLE);
String tempResult = SQLUtils.toSQLString(stmts, com.alibaba.druid.util.JdbcConstants.ORACLE, parameters);
assertEquals("SELECT dd.synonym_name AS table_name, 'EAIREP' AS schema_name, aa.num_rows, bb.comments AS description\n" +
"\t, to_char(cc.created, 'YYYY-MM-DD HH24:MI:SS') AS createTime\n" +
"\t, to_char(cc.last_ddl_time, 'YYYY-MM-DD HH24:MI:SS') AS last_ddl_time, 'SYNONYM' AS table_type\n" +
"\t, lower(aa.owner || '.' || aa.table_name) AS ref_info\n" +
"FROM dba_synonyms dd, dba_objects cc, dba_tables aa\n" +
"\tLEFT JOIN dba_tab_comments bb ON aa.owner = bb.owner\n" +
"AND aa.table_name = bb.table_name \n" +
"WHERE aa.owner = cc.owner\n" +
"\tAND aa.table_name = cc.object_name\n" +
"\tAND cc.subobject_name IS NULL\n" +
"\tAND cc.object_type = 'TABLE'\n" +
"\tAND aa.owner = dd.table_owner\n" +
"\tAND aa.table_name = dd.table_name\n" +
"\tAND dd.owner = 'EAIREP'", tempResult);
}
static void f() {
String ip = "";
String port = "";
String[] ins = new String[]{ip, "_", port};
}
}
@@ -83,4 +83,19 @@ public class Oracle2PG_DataTypeTest_chars extends TestCase {
SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
assertEquals("varchar(255)", pgDataType.toString());
}
public void test_oracle2pg_varchar2_8000() throws Exception {
String sql = "varchar2(8000)";
SQLDataType dataType = SQLParserUtils.createExprParser(sql, JdbcConstants.ORACLE).parseDataType();
SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
assertEquals("text", pgDataType.toString());
}
public void test_oracle2pg_varchar2_10000() throws Exception {
String sql = "varchar2(10000)";
SQLDataType dataType = SQLParserUtils.createExprParser(sql, JdbcConstants.ORACLE).parseDataType();
SQLDataType pgDataType = SQLTransformUtils.transformOracleToPostgresql(dataType);
assertEquals("text", pgDataType.toString());
}
}
@@ -46,4 +46,19 @@ public class JdbcUtils_driver extends TestCase {
String dbType = JdbcUtils.getDbType(jdbcUrl, "net.sf.log4jdbc.sql.jdbcapi.DriverSpy");
assertEquals("not support log4jdbc mysql, url like jdbc:log4jdbc:derby:...", JdbcConstants.DERBY, dbType);
}
public void test_log4jdbc_es() throws Exception {
assertEquals(JdbcConstants.ELASTIC_SEARCH
, JdbcUtils.getDbType("jdbc:elastic://100.69.202.4:9300/tcbuyer?cluster.name=refund_cluster", null));
}
public void test_log4jdbc_es_1() throws Exception {
assertEquals(JdbcConstants.ELASTIC_SEARCH
, JdbcUtils.getDbType("jdbc:elastic://100.69.202.4:9300/tcbuyer?cluster.name=refund_cluster", "com.alibaba.xdriver.elastic.jdbc.ElasticDriver"));
}
public void test_log4jdbc_es_driver() throws Exception {
assertEquals(JdbcConstants.ELASTIC_SEARCH_DRIVER
, JdbcUtils.getDriverClassName("jdbc:elastic://100.69.202.4:9300/tcbuyer?cluster.name=refund_cluster"));
}
}