Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improve sql parser & wall filter

Browse Source
This commit is contained in:
wenshao
2013-04-05 01:59:44 +08:00
parent bec180c36f
commit d268227242
12 changed files with 211 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlStatementParser.java
+2 -1
View File
@@ -642,6 +642,7 @@ public class MySqlStatementParser extends SQLStatementParser {
boolean full = false;
if (lexer.token() == Token.FULL) {
lexer.nextToken();
full = true;
}
if (identifierEquals("PROCESSLIST")) {
@@ -651,7 +652,7 @@ public class MySqlStatementParser extends SQLStatementParser {
return stmt;
}
if (identifierEquals("COLUMNS")) {
if (identifierEquals("COLUMNS") || identifierEquals("FIELDS")) {
lexer.nextToken();
MySqlShowColumnsStatement stmt = parseShowColumns();
src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/MySqlSchemaStatVisitor.java
+2 -2
View File
@@ -422,7 +422,7 @@ public class MySqlSchemaStatVisitor extends SchemaStatVisitor implements MySqlAS
@Override
public boolean visit(MySqlShowColumnsStatement x) {
return true;
return false;
}
@Override
@@ -1283,7 +1283,7 @@ public class MySqlSchemaStatVisitor extends SchemaStatVisitor implements MySqlAS
public boolean visit(MySqlAlterTableImportTablespace x) {
return false;
}
@Override
public void endVisit(MySqlAlterTableImportTablespace x) {
src/main/java/com/alibaba/druid/stat/DruidStatManagerFacade.java
+1 -1
View File
@@ -186,7 +186,7 @@ public final class DruidStatManagerFacade {
}
@SuppressWarnings({ "rawtypes", "unchecked" })
static Map mergWallStat(Map mapA, Map mapB) {
public static Map mergWallStat(Map mapA, Map mapB) {
if (mapA.size() == 0) {
return mapB;
}
src/main/java/com/alibaba/druid/wall/WallCheckResult.java
+47 -6
View File
@@ -15,22 +15,51 @@
*/
package com.alibaba.druid.wall;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import com.alibaba.druid.sql.ast.SQLStatement;
public class WallCheckResult {
private final List<Violation> violations = new ArrayList<Violation>(2);
private final List<SQLStatement> statementList = new ArrayList<SQLStatement>(2);
private final List<SQLStatement> statementList;
private final Map<String, WallSqlTableStat> tableStats;
private final List<Violation> violations;
private final Map<String, WallSqlFunctionStat> functionStats;
private final boolean syntaxError;
public WallCheckResult(){
this(null);
}
public WallCheckResult(Violation violation){
this.violations.add(violation);
public WallCheckResult(WallSqlStat sqlStat){
if (sqlStat != null) {
tableStats = sqlStat.getTableStats();
violations = sqlStat.getViolations();
functionStats = sqlStat.getFunctionStats();
statementList = Collections.emptyList();
syntaxError = sqlStat.isSyntaxError();
} else {
tableStats = Collections.emptyMap();
violations = Collections.emptyList();
functionStats = Collections.emptyMap();
statementList = Collections.emptyList();
syntaxError = false;
}
}
public WallCheckResult(List<Violation> violations, Map<String, WallSqlTableStat> tableStats,
Map<String, WallSqlFunctionStat> functionStats, List<SQLStatement> statementList,
boolean syntaxError){
this.tableStats = tableStats;
this.violations = violations;
this.functionStats = functionStats;
this.statementList = statementList;
this.syntaxError = syntaxError;
}
public List<Violation> getViolations() {
@@ -41,4 +70,16 @@ public class WallCheckResult {
return statementList;
}
public Map<String, WallSqlTableStat> getTableStats() {
return tableStats;
}
public Map<String, WallSqlFunctionStat> getFunctionStats() {
return functionStats;
}
public boolean isSyntaxError() {
return syntaxError;
}
}
src/main/java/com/alibaba/druid/wall/WallFunctionStat.java
+6
View File
@@ -15,6 +15,8 @@
*/
package com.alibaba.druid.wall;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
public class WallFunctionStat {
@@ -39,4 +41,8 @@ public class WallFunctionStat {
return "{\"invokeCount\":" + invokeCount + "}";
}
public Map<String, Object> toMap() {
return Collections.<String, Object>singletonMap("invokeCount", invokeCount);
}
}
src/main/java/com/alibaba/druid/wall/WallProvider.java
+106 -87
View File
@@ -47,7 +47,7 @@ public abstract class WallProvider {
private int MAX_SQL_LENGTH = 2048; // 1k
private int whiteSqlMaxSize = 10000; // 1k
private int whiteSqlMaxSize = 2000; // 1k
private LRUCache<String, WallSqlStat> blackList;
@@ -103,6 +103,10 @@ public abstract class WallProvider {
public WallTableStat getTableStat(String tableName) {
String lowerCaseName = tableName.toLowerCase();
if (lowerCaseName.startsWith("`") && lowerCaseName.endsWith("`")) {
lowerCaseName = lowerCaseName.substring(1, lowerCaseName.length() - 1);
}
return getTableStatWithLowerName(lowerCaseName);
}
@@ -224,7 +228,7 @@ public abstract class WallProvider {
lock.writeLock().unlock();
}
}
public void clearBlackList() {
lock.writeLock().lock();
try {
@@ -371,25 +375,111 @@ public abstract class WallProvider {
}
WallContext.createIfNotExists(this.dbType);
WallCheckResult result = new WallCheckResult();
if (config.isDoPrivilegedAllow() && ispPivileged()) {
return result;
return new WallCheckResult();
}
// first step, check whiteList
boolean mulltiTenant = config.getTenantTablePattern() != null && config.getTenantTablePattern().length() > 0;
if (!mulltiTenant) {
WallCheckResult checkResult = checkWhiteAndBlackList(sql);
if (checkResult != null) {
return checkResult;
}
}
hardCheckCount.incrementAndGet();
final List<Violation> violations = new ArrayList<Violation>();
List<SQLStatement> statementList = new ArrayList<SQLStatement>();
boolean syntaxError = false;
try {
SQLStatementParser parser = createParser(sql);
parser.getLexer().setCommentHandler(WallCommentHandler.instance);
if (!config.isCommentAllow()) {
parser.getLexer().setAllowComment(false); // deny comment
}
parser.parseStatementList(statementList);
final Token lastToken = parser.getLexer().token();
if (lastToken != Token.EOF) {
violations.add(new IllegalSQLObjectViolation("not terminal sql, token " + lastToken, sql));
}
} catch (NotAllowCommentException e) {
violations.add(new SyntaxErrorViolation(e, sql));
incrementCommentDeniedCount();
} catch (ParserException e) {
syntaxErrrorCount.incrementAndGet();
syntaxError = true;
if (config.isStrictSyntaxCheck()) {
violations.add(new SyntaxErrorViolation(e, sql));
}
} catch (Exception e) {
violations.add(new SyntaxErrorViolation(e, sql));
}
if (statementList.size() > 1 && !config.isMultiStatementAllow()) {
violations.add(new IllegalSQLObjectViolation("multi-statement not allow", sql));
}
WallVisitor visitor = createWallVisitor();
if (statementList.size() > 0) {
SQLStatement stmt = statementList.get(0);
try {
stmt.accept(visitor);
} catch (ParserException e) {
violations.add(new SyntaxErrorViolation(e, sql));
}
}
if (visitor.getViolations().size() > 0) {
violations.addAll(visitor.getViolations());
}
WallSqlStat sqlStat = null;
if (violations.size() > 0) {
violationCount.incrementAndGet();
if (sql.length() < MAX_SQL_LENGTH) {
sqlStat = addBlackSql(sql, context.getTableStats(), context.getFunctionStats(), violations, syntaxError);
}
} else {
if (sql.length() < MAX_SQL_LENGTH) {
sqlStat = addWhiteSql(sql, context.getTableStats(), context.getFunctionStats(), syntaxError);
}
}
Map<String, WallSqlTableStat> tableStats = null;
Map<String, WallSqlFunctionStat> functionStats = null;
if (context != null) {
tableStats = context.getTableStats();
functionStats = context.getFunctionStats();
recordStats(tableStats, functionStats);
}
if (sqlStat != null) {
new WallCheckResult(sqlStat);
}
return new WallCheckResult(violations, tableStats, functionStats, statementList, syntaxError);
}
private WallCheckResult checkWhiteAndBlackList(String sql) {
{
WallSqlStat sqlStat = getWhiteSql(sql);
if (sqlStat != null) {
whiteListHitCount.incrementAndGet();
sqlStat.incrementAndGetExecuteCount();
if (sqlStat.isSyntaxError()) {
syntaxErrrorCount.incrementAndGet();
}
recordStats(sqlStat.getTableStats(), sqlStat.getFunctionStats());
return result;
return new WallCheckResult(sqlStat);
}
}
// check black list
@@ -405,80 +495,12 @@ public abstract class WallProvider {
sqlStat.incrementAndGetExecuteCount();
recordStats(sqlStat.getTableStats(), sqlStat.getFunctionStats());
List<Violation> blackViolations = sqlStat.getViolations();
result.getViolations().addAll(blackViolations);
return result;
return new WallCheckResult(sqlStat);
}
}
hardCheckCount.incrementAndGet();
boolean syntaxError = false;
try {
SQLStatementParser parser = createParser(sql);
parser.getLexer().setCommentHandler(WallCommentHandler.instance);
if (!config.isCommentAllow()) {
parser.getLexer().setAllowComment(false); // deny comment
}
parser.parseStatementList(result.getStatementList());
final Token lastToken = parser.getLexer().token();
if (lastToken != Token.EOF) {
result.getViolations().add(new IllegalSQLObjectViolation("not terminal sql, token " + lastToken, sql));
}
} catch (NotAllowCommentException e) {
result.getViolations().add(new SyntaxErrorViolation(e, sql));
incrementCommentDeniedCount();
} catch (ParserException e) {
syntaxErrrorCount.incrementAndGet();
syntaxError = true;
if (config.isStrictSyntaxCheck()) {
result.getViolations().add(new SyntaxErrorViolation(e, sql));
}
} catch (Exception e) {
result.getViolations().add(new SyntaxErrorViolation(e, sql));
}
if (result.getViolations().size() == 0 && result.getStatementList().size() > 1
&& !config.isMultiStatementAllow()) {
result.getViolations().add(new IllegalSQLObjectViolation("multi-statement not allow", sql));
}
WallVisitor visitor = createWallVisitor();
if (result.getStatementList().size() > 0) {
SQLStatement stmt = result.getStatementList().get(0);
try {
stmt.accept(visitor);
} catch (ParserException e) {
result.getViolations().add(new SyntaxErrorViolation(e, sql));
}
}
if (visitor.getViolations().size() > 0) {
result.getViolations().addAll(visitor.getViolations());
}
if (result.getViolations().size() > 0) {
violationCount.incrementAndGet();
if (sql.length() < MAX_SQL_LENGTH) {
addBlackSql(sql, context.getTableStats(), context.getFunctionStats(), result.getViolations(),
syntaxError);
}
} else {
if (sql.length() < MAX_SQL_LENGTH) {
addWhiteSql(sql, context.getTableStats(), context.getFunctionStats(), syntaxError);
}
}
if (context != null) {
recordStats(context.getTableStats(), context.getFunctionStats());
}
return result;
return null;
}
void recordStats(Map<String, WallSqlTableStat> tableStats, Map<String, WallSqlFunctionStat> functionStats) {
@@ -551,7 +573,7 @@ public abstract class WallProvider {
public long getViolationCount() {
return violationCount.get();
}
public long getHardCheckCount() {
return hardCheckCount.get();
}
@@ -589,6 +611,7 @@ public abstract class WallProvider {
Map<String, Object> info = new LinkedHashMap<String, Object>();
info.put("checkCount", this.getCheckCount());
info.put("hardCheckCount", this.getHardCheckCount());
info.put("violationCount", this.getViolationCount());
info.put("blackListHitCount", this.getBlackListHitCount());
info.put("blackListSize", this.getBlackList().size());
@@ -597,25 +620,21 @@ public abstract class WallProvider {
info.put("syntaxErrrorCount", this.getSyntaxErrorCount());
{
List<Map<String, Object>> tables = new ArrayList<Map<String, Object>>();
Map<String, Map<String, Object>> tables = new LinkedHashMap<String, Map<String, Object>>();
for (Map.Entry<String, WallTableStat> entry : this.tableStats.entrySet()) {
Map<String, Object> stat = new LinkedHashMap<String, Object>();
stat.put("name", entry.getKey());
entry.getValue().toMap(stat);
tables.add(stat);
tables.put(entry.getKey(), entry.getValue().toMap());
}
info.put("tables", tables);
}
{
List<Map<String, Object>> functions = new ArrayList<Map<String, Object>>();
Map<String, Map<String, Object>> functions = new LinkedHashMap<String, Map<String, Object>>();
for (Map.Entry<String, WallFunctionStat> entry : this.functionStats.entrySet()) {
Map<String, Object> stat = new HashMap<String, Object>();
stat.put("name", entry.getValue());
stat.put("invokeCount", entry.getValue().getInvokeCount());
functions.add(stat);
functions.put(entry.getKey(), entry.getValue().toMap());
}
info.put("functions", functions);
}
info.put("whiteList", this.getWhiteList());
info.put("blackList", this.getBlackList());
return info;
}
}
src/main/java/com/alibaba/druid/wall/WallSqlFunctionStat.java
+4 -1
View File
@@ -26,5 +26,8 @@ public class WallSqlFunctionStat {
public void incrementInvokeCount() {
this.invokeCount++;
}
public void addInvokeCount(int value) {
this.invokeCount += value;
}
}
src/main/java/com/alibaba/druid/wall/WallSqlTableStat.java
+35
View File
@@ -34,6 +34,10 @@ public class WallSqlTableStat {
public void incrementSelectCount() {
this.selectCount++;
}
public void addSelectCount(int value) {
this.selectCount += value;
}
public int getSelectIntoCount() {
return selectIntoCount;
@@ -42,6 +46,10 @@ public class WallSqlTableStat {
public void incrementSelectIntoCount() {
this.selectIntoCount++;
}
public void addSelectIntoCount(int value) {
this.selectIntoCount += value;
}
public int getInsertCount() {
return insertCount;
@@ -50,6 +58,10 @@ public class WallSqlTableStat {
public void incrementInsertCount() {
this.insertCount++;
}
public void addInsertCount(int value) {
this.insertCount += value;
}
public int getUpdateCount() {
return updateCount;
@@ -59,6 +71,10 @@ public class WallSqlTableStat {
this.updateCount++;
}
public void addUpdateCount(int value) {
this.deleteCount += value;
}
public int getDeleteCount() {
return deleteCount;
}
@@ -67,6 +83,10 @@ public class WallSqlTableStat {
this.deleteCount++;
}
public void addDeleteCount(int value) {
this.deleteCount += value;
}
public int getTruncateCount() {
return truncateCount;
}
@@ -75,6 +95,10 @@ public class WallSqlTableStat {
this.truncateCount++;
}
public void addTruncateCount(int value) {
this.truncateCount += value;
}
public int getCreateCount() {
return createCount;
}
@@ -83,6 +107,10 @@ public class WallSqlTableStat {
this.createCount++;
}
public void addCreateCount(int value) {
this.createCount += value;
}
public int getAlterCount() {
return alterCount;
}
@@ -91,6 +119,10 @@ public class WallSqlTableStat {
this.alterCount++;
}
public void addAlterCount(int value) {
this.alterCount += value;
}
public int getDropCount() {
return dropCount;
}
@@ -99,4 +131,7 @@ public class WallSqlTableStat {
this.dropCount++;
}
public void addDropCount(int value) {
this.dropCount += value;
}
}
src/test/java/com/alibaba/druid/bvt/filter/wall/MySqlWallDescribteTest.java
+1 -1
View File
@@ -38,7 +38,7 @@ public class MySqlWallDescribteTest extends TestCase {
public void test_not_allow() throws Exception {
WallConfig config = new WallConfig();
config.setVariantCheck(true);
config.setDescribeAllow(false);
Assert.assertFalse(WallUtils.isValidateMySql("describe t", config));
}
src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_WhiteList.java
+4 -4
View File
@@ -22,17 +22,17 @@ public class WallStatTest_WhiteList extends TestCase {
public void testMySql() throws Exception {
WallProvider provider = new MySqlWallProvider();
for (int i = 0; i < 1001; ++i) {
for (int i = 0; i < 3001; ++i) {
String sql = "select * from t where id = " + i;
Assert.assertTrue(provider.checkValid(sql));
}
WallTableStat tableStat = provider.getTableStat("t");
Assert.assertEquals(1001, tableStat.getSelectCount());
Assert.assertEquals(3001, tableStat.getSelectCount());
Assert.assertEquals(0, provider.getBlackListHitCount());
Assert.assertEquals(0, provider.getWhiteListHitCount());
Assert.assertEquals(1000, provider.getWhiteList().size());
Assert.assertEquals(1001, provider.getCheckCount());
Assert.assertEquals(2000, provider.getWhiteList().size());
Assert.assertEquals(3001, provider.getCheckCount());
}
}
src/test/java/com/alibaba/druid/bvt/pool/ConfigErrorTestMysql.java
+1 -1
View File
@@ -64,6 +64,6 @@ public class ConfigErrorTestMysql extends TestCase {
Assert.assertEquals(0, LOG.getWarnCount());
dataSource.init();
Assert.assertEquals(1, LOG.getWarnCount());
Assert.assertEquals(0, LOG.getWarnCount());
}
}
src/test/java/com/alibaba/druid/bvt/sql/cobar/DALParserTest.java
+2 -1
View File
@@ -18,9 +18,10 @@
*/
package com.alibaba.druid.bvt.sql.cobar;
import junit.framework.Assert;
import junit.framework.TestCase;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.statement.SQLSetStatement;
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlDescribeStatement;