sql desensitize support.

This commit is contained in:
wenshao
2017-06-19 16:54:04 +08:00
parent 0528af3207
commit fae4f44239
14 changed files with 525 additions and 16 deletions
@@ -15,11 +15,9 @@
*/
package com.alibaba.druid.sql;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.alibaba.druid.DruidRuntimeException;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.ast.SQLReplaceable;
@@ -277,8 +275,9 @@ public class SQLUtils {
option = DEFAULT_FORMAT_OPTION;
}
visitor.setUppCase(option.isUppCase());
visitor.setPrettyFormat(option.isPrettyFormat());
visitor.setParameterized(option.isParameterized());
visitor.setPrettyFormat(option.prettyFormat);
visitor.setParameterized(option.parameterized);
visitor.setDesensitize(option.desensitize);
if (tableMapping != null) {
visitor.setTableMapping(tableMapping);
@@ -630,6 +629,7 @@ public class SQLUtils {
private boolean ucase = true;
private boolean prettyFormat = true;
private boolean parameterized = false;
private boolean desensitize = false;
public FormatOption() {
@@ -649,6 +649,14 @@ public class SQLUtils {
this.parameterized = parameterized;
}
public boolean isDesensitize() {
return desensitize;
}
public void setDesensitize(boolean desensitize) {
this.desensitize = desensitize;
}
public boolean isUppCase() {
return ucase;
}
@@ -865,5 +873,15 @@ public class SQLUtils {
return false;
}
public static String desensitizeTable(String tableName) {
if (tableName == null) {
return null;
}
tableName = normalize(tableName);
long hash = Utils.fnv_64_lower(tableName);
return Utils.hex_t(hash);
}
}
@@ -30,7 +30,7 @@ public class SQLCreateViewStatement extends SQLStatementImpl implements SQLDDLSt
private boolean orReplace = false;
private boolean force = false;
protected SQLName name;
// protected SQLName name;
protected SQLSelect subQuery;
protected boolean ifNotExists = false;
@@ -38,6 +38,8 @@ public class SQLCreateViewStatement extends SQLStatementImpl implements SQLDDLSt
protected SQLName definer;
protected String sqlSecurity;
protected SQLExprTableSource tableSource;
protected final List<Column> columns = new ArrayList<Column>();
private Level with;
@@ -61,11 +63,26 @@ public class SQLCreateViewStatement extends SQLStatementImpl implements SQLDDLSt
}
public SQLName getName() {
return name;
if (tableSource == null) {
return null;
}
return (SQLName) tableSource.getExpr();
}
public void setName(SQLName name) {
this.name = name;
this.setTableSource(new SQLExprTableSource(name));
}
public SQLExprTableSource getTableSource() {
return tableSource;
}
public void setTableSource(SQLExprTableSource tableSource) {
if (tableSource != null) {
tableSource.setParent(this);
}
this.tableSource = tableSource;
}
public Level getWith() {
@@ -151,7 +168,7 @@ public class SQLCreateViewStatement extends SQLStatementImpl implements SQLDDLSt
public void output(StringBuffer buf) {
buf.append("CREATE VIEW ");
this.name.output(buf);
this.getName().output(buf);
if (this.columns.size() > 0) {
buf.append(" (");
@@ -176,7 +193,7 @@ public class SQLCreateViewStatement extends SQLStatementImpl implements SQLDDLSt
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, this.name);
acceptChild(visitor, this.tableSource);
acceptChild(visitor, this.columns);
acceptChild(visitor, this.comment);
acceptChild(visitor, this.subQuery);
@@ -24,6 +24,7 @@ import java.sql.NClob;
import java.text.SimpleDateFormat;
import java.util.*;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.*;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.*;
@@ -69,6 +70,8 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
protected transient int lines = 0;
protected boolean desensitize = false;
public SQLASTOutputVisitor(Appendable appender){
this.appender = appender;
}
@@ -118,6 +121,14 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
return parameters;
}
public boolean isDesensitize() {
return desensitize;
}
public void setDesensitize(boolean desensitize) {
this.desensitize = desensitize;
}
public Set<String> getTables() {
return this.tables;
}
@@ -1423,6 +1434,26 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
addTable(expr.toString());
}
if (desensitize) {
SQLExpr owner = null;
String ident = null;
if (expr instanceof SQLIdentifierExpr) {
ident = ((SQLIdentifierExpr) expr).getName();
} else if (expr instanceof SQLPropertyExpr) {
SQLPropertyExpr propertyExpr = (SQLPropertyExpr) expr;
propertyExpr.getOwner().accept(this);
print('.');
ident = propertyExpr.getName();
}
if (ident != null) {
String desensitizeTable = SQLUtils.desensitizeTable(ident);
print0(desensitizeTable);
return;
}
}
if (tableMapping != null && expr instanceof SQLName) {
String tableName = expr.toString();
@@ -2446,7 +2477,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
print0(ucase ? "IF NOT EXISTS " : "if not exists ");
}
x.getName().accept(this);
x.getTableSource().accept(this);
if (x.getColumns().size() > 0) {
println();
@@ -426,7 +426,11 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter {
return true;
}
switch (x.getOperator()) {
final SQLBinaryOperator op = x.getOperator();
final SQLExpr left = x.getLeft();
final SQLExpr right = x.getRight();
switch (op) {
case Equality:
case NotEqual:
case GreaterThan:
@@ -439,15 +443,32 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter {
case NotLike:
case Is:
case IsNot:
handleCondition(x.getLeft(), x.getOperator().name, x.getRight());
handleCondition(x.getRight(), x.getOperator().name, x.getLeft());
handleCondition(left, x.getOperator().name, right);
handleCondition(right, x.getOperator().name, left);
handleRelationship(x.getLeft(), x.getOperator().name, x.getRight());
handleRelationship(left, x.getOperator().name, right);
break;
case BooleanOr:{
List<SQLExpr> list = SQLBinaryOpExpr.split(x, op);
for (SQLExpr item : list) {
if (item instanceof SQLBinaryOpExpr) {
visit((SQLBinaryOpExpr) item);
} else {
item.accept(this);
}
}
return false;
}
default:
break;
}
return true;
left.accept(this);
right.accept(this);
return false;
}
protected void handleRelationship(SQLExpr left, String operator, SQLExpr right) {
@@ -334,4 +334,117 @@ public class Utils {
public static boolean equals(Object a, Object b) {
return (a == b) || (a != null && a.equals(b));
}
public static String hex(int hash) {
byte[] bytes = new byte[4];
bytes[3] = (byte) (hash );
bytes[2] = (byte) (hash >>> 8);
bytes[1] = (byte) (hash >>> 16);
bytes[0] = (byte) (hash >>> 24);
char[] chars = new char[8];
for (int i = 0; i < 4; ++i) {
byte b = bytes[i];
int a = b & 0xFF;
int b0 = a >> 4;
int b1 = a & 0xf;
chars[i * 2] = (char) (b0 + (b0 < 10 ? 48 : 55));
chars[i * 2 + 1] = (char) (b1 + (b1 < 10 ? 48 : 55));
}
return new String(chars);
}
public static String hex(long hash) {
byte[] bytes = new byte[8];
bytes[7] = (byte) (hash );
bytes[6] = (byte) (hash >>> 8);
bytes[5] = (byte) (hash >>> 16);
bytes[4] = (byte) (hash >>> 24);
bytes[3] = (byte) (hash >>> 32);
bytes[2] = (byte) (hash >>> 40);
bytes[1] = (byte) (hash >>> 48);
bytes[0] = (byte) (hash >>> 56);
char[] chars = new char[16];
for (int i = 0; i < 8; ++i) {
byte b = bytes[i];
int a = b & 0xFF;
int b0 = a >> 4;
int b1 = a & 0xf;
chars[i * 2] = (char) (b0 + (b0 < 10 ? 48 : 55));
chars[i * 2 + 1] = (char) (b1 + (b1 < 10 ? 48 : 55));
}
return new String(chars);
}
public static String hex_t(long hash) {
byte[] bytes = new byte[8];
bytes[7] = (byte) (hash );
bytes[6] = (byte) (hash >>> 8);
bytes[5] = (byte) (hash >>> 16);
bytes[4] = (byte) (hash >>> 24);
bytes[3] = (byte) (hash >>> 32);
bytes[2] = (byte) (hash >>> 40);
bytes[1] = (byte) (hash >>> 48);
bytes[0] = (byte) (hash >>> 56);
char[] chars = new char[18];
chars[0] = 'T';
chars[1] = '_';
for (int i = 0; i < 8; ++i) {
byte b = bytes[i];
int a = b & 0xFF;
int b0 = a >> 4;
int b1 = a & 0xf;
chars[i * 2 + 2] = (char) (b0 + (b0 < 10 ? 48 : 55));
chars[i * 2 + 3] = (char) (b1 + (b1 < 10 ? 48 : 55));
}
return new String(chars);
}
public static long fnv_64(String input) {
if (input == null) {
return 0;
}
long hash = 0x811c9dc5;
for (int i = 0; i < input.length(); ++i) {
char c = input.charAt(i);
hash ^= c;
hash *= 0x1000193;
}
return hash;
}
public static long fnv_64_lower(String input) {
if (input == null) {
return 0;
}
long hash = 0x811c9dc5;
for (int i = 0; i < input.length(); ++i) {
char c = input.charAt(i);
if (c >= 'A' && c <= 'Z') {
c += 32;
}
hash ^= c;
hash *= 0x1000193;
}
return hash;
}
}
@@ -0,0 +1,84 @@
/*
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.benckmark.sql;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlOutputVisitor;
import com.alibaba.druid.sql.test.TestUtils;
import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.util.JdbcConstants;
import com.alibaba.druid.util.Utils;
import java.lang.management.ManagementFactory;
import java.util.List;
public class MySqlPerfMain_schemaStat {
public static void main(String[] args) throws Exception {
System.out.println(System.getProperty("java.vm.name") + " " + System.getProperty("java.runtime.version"));
List<String> arguments = ManagementFactory.getRuntimeMXBean().getInputArguments();
System.out.println(arguments);
String sql = "select * from t ";
for (int i = 0; i < 300; ++i) {
if (i == 0) {
sql += " WHERE ";
} else {
sql += " OR ";
}
sql += (" id = " + i);
}
for (int i = 0; i < 10; ++i) {
perfMySql(sql);
}
}
static long perfMySql(String sql) {
long startYGC = TestUtils.getYoungGC();
long startYGCTime = TestUtils.getYoungGCTime();
long startFGC = TestUtils.getFullGC();
long startMillis = System.currentTimeMillis();
for (int i = 0; i < 100 * 1; ++i) {
execMySql(sql);
}
long millis = System.currentTimeMillis() - startMillis;
long ygc = TestUtils.getYoungGC() - startYGC;
long ygct = TestUtils.getYoungGCTime() - startYGCTime;
long fgc = TestUtils.getFullGC() - startFGC;
System.out.println("MySql\t" + millis + ", ygc " + ygc + ", ygct " + ygct + ", fgc " + fgc);
return millis;
}
static String execMySql(String sql) {
List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.MYSQL);
SchemaStatVisitor statVisitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.MYSQL);
for (SQLStatement stmt : statementList) {
stmt.accept(statVisitor);
}
// for (SQLStatement statement : statementList) {
// statement.accept(visitor);
// visitor.println();
// }
return "";
}
}
@@ -37,7 +37,7 @@ public class MySqlWallTest87 extends TestCase {
Assert.assertTrue(provider.checkValid(//
"CREATE VIEW test.v AS SELECT * FROM t;"));
Assert.assertEquals(1, provider.getTableStats().size());
Assert.assertEquals(2, provider.getTableStats().size());
Assert.assertTrue(provider.getTableStats().containsKey("t"));
}
@@ -0,0 +1,44 @@
package com.alibaba.druid.bvt.sql.oracle;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
/**
* Created by wenshao on 19/06/2017.
*/
public class DesensitizationTest extends TestCase {
public void test_for_desensitization() throws Exception {
String sql = "select id, name from mytable where id = 3";
SQLUtils.FormatOption option = new SQLUtils.FormatOption();
option.setDesensitize(true);
option.setParameterized(true);
String desens_Sql = SQLUtils.format(sql, JdbcConstants.ORACLE, option);
System.out.println(sql);
System.out.println("-------------------");
System.out.println(desens_Sql);
assertEquals("SELECT id, name\n" +
"FROM T_771DB81F525E1E75\n" +
"WHERE id = ?", desens_Sql);
}
public void test_for_desensitization_1() throws Exception {
String sql = "select id, name from \"mytable\" where id = 3";
SQLUtils.FormatOption option = new SQLUtils.FormatOption();
option.setDesensitize(true);
option.setParameterized(true);
String desens_Sql = SQLUtils.format(sql, JdbcConstants.ORACLE, option);
System.out.println(sql);
System.out.println("-------------------");
System.out.println(desens_Sql);
assertEquals("SELECT id, name\n" +
"FROM T_771DB81F525E1E75\n" +
"WHERE id = ?", desens_Sql);
}
}
@@ -0,0 +1,33 @@
package com.alibaba.druid.bvt.sql.oracle;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
/**
* Created by wenshao on 19/06/2017.
*/
public class DesensitizationTest_createTable extends TestCase {
public void test_for_desensitization() throws Exception {
String sql = "CREATE TABLE customers\n" +
"( customer_id number(10) NOT NULL,\n" +
" customer_name varchar2(50) NOT NULL,\n" +
" city varchar2(50)\n" +
");";
SQLUtils.FormatOption option = new SQLUtils.FormatOption();
option.setDesensitize(true);
option.setParameterized(true);
String desens_Sql = SQLUtils.format(sql, JdbcConstants.ORACLE, option);
System.out.println(sql);
System.out.println("-------------------");
System.out.println(desens_Sql);
assertEquals("CREATE TABLE T_69BFC132C09344DE (\n" +
"\tcustomer_id number(10) NOT NULL,\n" +
"\tcustomer_name varchar2(50) NOT NULL,\n" +
"\tcity varchar2(50)\n" +
");", desens_Sql);
}
}
@@ -0,0 +1,46 @@
package com.alibaba.druid.bvt.sql.oracle;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
import java.util.HashMap;
import java.util.Map;
/**
* Created by wenshao on 19/06/2017.
*/
public class DesensitizationTest_createView extends TestCase {
public void test_for_desensitization() throws Exception {
String sql = "CREATE VIEW sup_orders AS\n" +
" SELECT suppliers.supplier_id, orders.quantity, orders.price\n" +
" FROM suppliers\n" +
" INNER JOIN orders\n" +
" ON suppliers.supplier_id = orders.supplier_id\n" +
" WHERE suppliers.supplier_name = 'Microsoft';";
SQLUtils.FormatOption option = new SQLUtils.FormatOption();
option.setDesensitize(true);
option.setParameterized(true);
String desens_Sql = SQLUtils.format(sql, JdbcConstants.ORACLE, option);
System.out.println(sql);
System.out.println("-------------------");
System.out.println(desens_Sql);
assertEquals("CREATE VIEW T_14856F0E5B66E911\n" +
"AS\n" +
"SELECT suppliers.supplier_id, orders.quantity, orders.price\n" +
"FROM T_126BF2439561E30E\n" +
"\tINNER JOIN T_C5B90BBF325E96EC ON suppliers.supplier_id = orders.supplier_id \n" +
"WHERE suppliers.supplier_name = ?;", desens_Sql);
Map<String, String> tableMapping = new HashMap<String, String>();
tableMapping.put(SQLUtils.desensitizeTable("sup_orders"), "sup_orders");
tableMapping.put(SQLUtils.desensitizeTable("suppliers"), "suppliers");
String restore_sql = SQLUtils.refactor(desens_Sql, JdbcConstants.ORACLE, tableMapping);
System.out.println("-------------------");
System.out.println(restore_sql);
}
}
@@ -0,0 +1,27 @@
package com.alibaba.druid.bvt.sql.oracle;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
/**
* Created by wenshao on 19/06/2017.
*/
public class DesensitizationTest_delete extends TestCase {
public void test_for_desensitization() throws Exception {
String sql = "DELETE FROM customers\n" +
"WHERE last_name = 'Smith';";
SQLUtils.FormatOption option = new SQLUtils.FormatOption();
option.setDesensitize(true);
option.setParameterized(true);
String desens_Sql = SQLUtils.format(sql, JdbcConstants.ORACLE, option);
System.out.println(sql);
System.out.println("-------------------");
System.out.println(desens_Sql);
assertEquals("DELETE FROM T_69BFC132C09344DE\n" +
"WHERE last_name = ?;", desens_Sql);
}
}
@@ -0,0 +1,32 @@
package com.alibaba.druid.bvt.sql.oracle;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
/**
* Created by wenshao on 19/06/2017.
*/
public class DesensitizationTest_insert extends TestCase {
public void test_for_desensitization() throws Exception {
String sql = "INSERT INTO suppliers\n" +
"(supplier_id, supplier_name)\n" +
"SELECT account_no, name\n" +
"FROM customers\n" +
"WHERE customer_id > 5000;";
SQLUtils.FormatOption option = new SQLUtils.FormatOption();
option.setDesensitize(true);
option.setParameterized(true);
String desens_Sql = SQLUtils.format(sql, JdbcConstants.ORACLE, option);
System.out.println(sql);
System.out.println("-------------------");
System.out.println(desens_Sql);
assertEquals("INSERT INTO T_126BF2439561E30E (supplier_id, supplier_name)\n" +
"SELECT account_no, name\n" +
"FROM T_69BFC132C09344DE\n" +
"WHERE customer_id > ?;", desens_Sql);
}
}
@@ -0,0 +1,29 @@
package com.alibaba.druid.bvt.sql.oracle;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.util.JdbcConstants;
import junit.framework.TestCase;
/**
* Created by wenshao on 19/06/2017.
*/
public class DesensitizationTest_update extends TestCase {
public void test_for_desensitization() throws Exception {
String sql = "UPDATE customers\n" +
"SET last_name = 'Anderson'\n" +
"WHERE customer_id = 5000;";
SQLUtils.FormatOption option = new SQLUtils.FormatOption();
option.setDesensitize(true);
option.setParameterized(true);
String desens_Sql = SQLUtils.format(sql, JdbcConstants.ORACLE, option);
System.out.println(sql);
System.out.println("-------------------");
System.out.println(desens_Sql);
assertEquals("UPDATE T_69BFC132C09344DE\n" +
"SET last_name = ?\n" +
"WHERE customer_id = ?;", desens_Sql);
}
}
@@ -0,0 +1,14 @@
package com.alibaba.druid.bvt.utils;
import com.alibaba.druid.util.Utils;
import junit.framework.TestCase;
/**
* Created by wenshao on 19/06/2017.
*/
public class Fnv64_lower_test extends TestCase {
public void test_fnv_lower() throws Exception {
assertEquals(Utils.fnv_64_lower("Id"), Utils.fnv_64_lower("ID"));
assertEquals(Utils.fnv_64_lower("id"), Utils.fnv_64_lower("ID"));
}
}