diff --git a/pom.xml b/pom.xml index 044b53372f..7b8602b5d7 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ 4.11 true - true + true UTF-8 1.6 @@ -351,7 +351,7 @@ ${spring.version} test - + org.springframework spring-ibatis @@ -412,8 +412,8 @@ 9.1-901-1.jdbc4 provided - + com.oracle @@ -533,5 +533,12 @@ 1.1.29 test + + + net.sf.jsqlparser + jsqlparser + 0.8.0 + test + diff --git a/src/main/java/com/alibaba/druid/pool/DruidDataSource.java b/src/main/java/com/alibaba/druid/pool/DruidDataSource.java index 69f4b8a5af..ed1588b2cb 100644 --- a/src/main/java/com/alibaba/druid/pool/DruidDataSource.java +++ b/src/main/java/com/alibaba/druid/pool/DruidDataSource.java @@ -77,6 +77,7 @@ import com.alibaba.druid.sql.parser.SQLStatementParser; import com.alibaba.druid.stat.DruidDataSourceStatManager; import com.alibaba.druid.stat.JdbcDataSourceStat; import com.alibaba.druid.stat.JdbcSqlStat; +import com.alibaba.druid.stat.JdbcSqlStatValue; import com.alibaba.druid.support.logging.Log; import com.alibaba.druid.support.logging.LogFactory; import com.alibaba.druid.util.IOUtils; @@ -144,6 +145,8 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat private boolean mbeanRegistered = false; + public static ThreadLocal waitNanosLocal = new ThreadLocal(); + public DruidDataSource(){ this(false); } @@ -1012,12 +1015,13 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat holder = takeLast(); } - activeCount++; - if (activeCount > activePeak) { - activePeak = activeCount; - activePeakTime = System.currentTimeMillis(); + if (holder != null) { + activeCount++; + if (activeCount > activePeak) { + activePeak = activeCount; + activePeakTime = System.currentTimeMillis(); + } } - } catch (InterruptedException e) { connectErrorCount.incrementAndGet(); throw new SQLException(e.getMessage(), e); @@ -1028,6 +1032,38 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat lock.unlock(); } + if (holder == null) { + long waitNanos = waitNanosLocal.get(); + + StringBuilder buf = new StringBuilder(); + buf.append("wait millis ")// + .append(waitNanos / (1000 * 1000))// + .append(", active " + activeCount)// + ; + + List sqlList = this.getDataSourceStat().getRuningSqlList(); + for (int i = 0; i < sqlList.size(); ++i) { + if (i != 0) { + buf.append('\n'); + } else { + buf.append(", "); + } + JdbcSqlStatValue sql = sqlList.get(i); + buf.append("runningCount "); + buf.append(sql.getRunningCount()); + buf.append(" : "); + buf.append(sql.getSql()); + } + + String errorMessage = buf.toString(); + + if (this.createError == null) { + throw new GetConnectionTimeoutException(errorMessage, createError); + } else { + throw new GetConnectionTimeoutException(errorMessage); + } + } + holder.incrementUseCount(); DruidPooledConnection poolalbeConnection = new DruidPooledConnection(holder); @@ -1313,19 +1349,13 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat DruidConnectionHolder pollLast(long nanos) throws InterruptedException, SQLException { long estimate = nanos; - for (int i = 0;; ++i) { + for (;;) { if (poolingCount == 0) { empty.signal(); // send signal to CreateThread create connection if (estimate <= 0) { - String errorMessage = "loopWaitCount " + i + ", wait millis " + (nanos - estimate) / (1000 * 1000) - + ", active " + activeCount; - - if (this.createError == null) { - throw new GetConnectionTimeoutException(errorMessage, createError); - } else { - throw new GetConnectionTimeoutException(errorMessage); - } + waitNanosLocal.set(nanos - estimate); + return null; } notEmptyWaitThreadCount++; @@ -1358,13 +1388,8 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat continue; } - String errorMessage = "loopWaitCount " + i + ", wait millis " + (nanos - estimate) / (1000 * 1000) - + ", active " + activeCount; - if (createError != null) { - throw new GetConnectionTimeoutException(errorMessage, createError); - } else { - throw new GetConnectionTimeoutException(errorMessage); - } + waitNanosLocal.set(nanos - estimate); + return null; } } diff --git a/src/main/java/com/alibaba/druid/stat/JdbcDataSourceStat.java b/src/main/java/com/alibaba/druid/stat/JdbcDataSourceStat.java index 6acedfe834..eec45fb690 100644 --- a/src/main/java/com/alibaba/druid/stat/JdbcDataSourceStat.java +++ b/src/main/java/com/alibaba/druid/stat/JdbcDataSourceStat.java @@ -97,9 +97,9 @@ public class JdbcDataSourceStat implements JdbcDataSourceStatMBean { public static void setGlobal(JdbcDataSourceStat value) { global = value; } - + public void configFromProperties(Properties properties) { - + } public boolean isResetStatEnable() { @@ -207,7 +207,7 @@ public class JdbcDataSourceStat implements JdbcDataSourceStatMBean { if (!isResetStatEnable()) { return; } - + blobOpenCount.set(0); clobOpenCount.set(0); @@ -375,6 +375,33 @@ public class JdbcDataSourceStat implements JdbcDataSourceStatMBean { return values; } + public List getRuningSqlList() { + List stats = new ArrayList(sqlStatMap.size()); + lock.readLock().lock(); + try { + Iterator> iter = sqlStatMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + JdbcSqlStat stat = entry.getValue(); + if (stat.getRunningCount() >= 0) { + stats.add(entry.getValue()); + } + } + } finally { + lock.readLock().unlock(); + } + + List values = new ArrayList(stats.size()); + for (int i = 0; i < stats.size(); ++i) { + JdbcSqlStat stat = stats.get(i); + JdbcSqlStatValue value = stat.getValue(false); + if (value.getRunningCount() > 0) { + values.add(value); + } + } + return values; + } + public JdbcSqlStat getSqlStat(String sql) { lock.readLock().lock(); try { diff --git a/src/main/java/com/alibaba/druid/wall/WallProvider.java b/src/main/java/com/alibaba/druid/wall/WallProvider.java index 87270e87f4..517bb20f63 100644 --- a/src/main/java/com/alibaba/druid/wall/WallProvider.java +++ b/src/main/java/com/alibaba/druid/wall/WallProvider.java @@ -239,6 +239,49 @@ public abstract class WallProvider { public WallSqlStat addWhiteSql(String sql, Map tableStats, Map functionStats, boolean syntaxError) { + String mergedSql; + try { + mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType); + } catch (Exception ex) { + WallSqlStat stat = new WallSqlStat(tableStats, functionStats, syntaxError); + stat.incrementAndGetExecuteCount(); + return stat; + } + + if (mergedSql != sql) { + WallSqlStat mergedStat; + lock.readLock().lock(); + try { + if (whiteMergedList == null) { + whiteMergedList = new LRUCache(whiteSqlMaxSize); + } + + mergedStat = whiteMergedList.get(mergedSql); + } finally { + lock.readLock().unlock(); + } + + if (mergedStat == null) { + WallSqlStat newStat = new WallSqlStat(tableStats, functionStats, syntaxError); + newStat.setSample(sql); + + lock.writeLock().lock(); + try { + mergedStat = whiteMergedList.get(mergedSql); + if (mergedStat == null) { + whiteMergedList.put(mergedSql, newStat); + mergedStat = newStat; + } + } finally { + lock.writeLock().unlock(); + } + } + + mergedStat.incrementAndGetExecuteCount(); + + return mergedStat; + } + lock.writeLock().lock(); try { if (whiteList == null) { @@ -251,12 +294,6 @@ public abstract class WallProvider { WallSqlStat wallStat = whiteList.get(sql); if (wallStat == null) { - String mergedSql; - try { - mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType); - } catch (Exception ex) { - mergedSql = sql; - } wallStat = whiteMergedList.get(mergedSql); if (wallStat == null) { wallStat = new WallSqlStat(tableStats, functionStats, syntaxError); @@ -277,6 +314,14 @@ public abstract class WallProvider { public WallSqlStat addBlackSql(String sql, Map tableStats, Map functionStats, List violations, boolean syntaxError) { + String mergedSql; + try { + mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType); + } catch (Exception ex) { + // skip + mergedSql = sql; + } + lock.writeLock().lock(); try { if (blackList == null) { @@ -289,12 +334,6 @@ public abstract class WallProvider { WallSqlStat wallStat = blackList.get(sql); if (wallStat == null) { - String mergedSql; - try { - mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType); - } catch (Exception ex) { - mergedSql = sql; - } wallStat = blackMergedList.get(mergedSql); if (wallStat == null) { wallStat = new WallSqlStat(tableStats, functionStats, violations, syntaxError); @@ -401,16 +440,37 @@ public abstract class WallProvider { } public WallSqlStat getWhiteSql(String sql) { + WallSqlStat stat = null; lock.readLock().lock(); try { - if (whiteList == null) { - return null; + if (whiteList != null) { + stat = whiteList.get(sql); } - - return whiteList.get(sql); } finally { lock.readLock().unlock(); } + + if (stat == null) { + String mergedSql; + try { + mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType); + } catch (Exception ex) { + // skip + return null; + } + + lock.readLock().lock(); + try { + if (whiteMergedList == null) { + return null; + } + + stat = whiteMergedList.get(mergedSql); + } finally { + lock.readLock().unlock(); + } + } + return stat; } public WallSqlStat getBlackSql(String sql) { @@ -832,13 +892,13 @@ public abstract class WallProvider { WallTableStat tableStat = entry.getValue(); WallTableStatValue tableStatValue = tableStat.getStatValue(reset); - + if (tableStatValue.getTotalExecuteCount() == 0) { continue; } - + tableStatValue.setName(tableName); - + statValue.getTables().add(tableStatValue); } @@ -847,7 +907,7 @@ public abstract class WallProvider { WallFunctionStat functionStat = entry.getValue(); WallFunctionStatValue functionStatValue = functionStat.getStatValue(reset); - + if (functionStatValue.getInvokeCount() == 0) { continue; } @@ -864,11 +924,11 @@ public abstract class WallProvider { String sql = entry.getKey(); WallSqlStat sqlStat = entry.getValue(); WallSqlStatValue sqlStatValue = sqlStat.getStatValue(reset); - + if (sqlStatValue.getExecuteCount() == 0) { continue; } - + sqlStatValue.setSql(sql); statValue.getWhiteList().add(sqlStatValue); } @@ -879,11 +939,11 @@ public abstract class WallProvider { String sql = entry.getKey(); WallSqlStat sqlStat = entry.getValue(); WallSqlStatValue sqlStatValue = sqlStat.getStatValue(reset); - + if (sqlStatValue.getExecuteCount() == 0) { continue; } - + sqlStatValue.setSql(sql); statValue.getBlackList().add(sqlStatValue); } diff --git a/src/test/java/com/alibaba/druid/bvt/filter/wall/WallProviderTest.java b/src/test/java/com/alibaba/druid/bvt/filter/wall/WallProviderTest.java index 798082fcba..1a5196b1eb 100644 --- a/src/test/java/com/alibaba/druid/bvt/filter/wall/WallProviderTest.java +++ b/src/test/java/com/alibaba/druid/bvt/filter/wall/WallProviderTest.java @@ -52,7 +52,7 @@ public class WallProviderTest extends TestCase { Assert.assertSame(whiteStat_1, provider.getSqlStat(sql_x)); } - Assert.assertEquals(500, provider.getWhiteList().size()); + Assert.assertEquals(0, provider.getWhiteList().size()); Assert.assertEquals(2, provider.getSqlList().size()); Assert.assertFalse(provider.checkValid("slelc")); @@ -66,10 +66,10 @@ public class WallProviderTest extends TestCase { Assert.assertEquals(1009, statValue.getCheckCount()); Assert.assertEquals(2, statValue.getBlackListHitCount()); - Assert.assertEquals(1005, statValue.getHardCheckCount()); + Assert.assertEquals(4, statValue.getHardCheckCount()); Assert.assertEquals(1, statValue.getSyntaxErrorCount()); Assert.assertEquals(5, statValue.getViolationCount()); - Assert.assertEquals(2, statValue.getWhiteListHitCount()); + Assert.assertEquals(1003, statValue.getWhiteListHitCount()); Assert.assertEquals(1, statValue.getWhiteList().size()); Assert.assertEquals(2, statValue.getBlackList().size()); diff --git a/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_WhiteList.java b/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_WhiteList.java index d7d3f9ac33..f93335737d 100644 --- a/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_WhiteList.java +++ b/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_WhiteList.java @@ -30,8 +30,8 @@ public class WallStatTest_WhiteList extends TestCase { WallTableStat tableStat = provider.getTableStat("t"); Assert.assertEquals(3001, tableStat.getSelectCount()); Assert.assertEquals(0, provider.getBlackListHitCount()); - Assert.assertEquals(0, provider.getWhiteListHitCount()); - Assert.assertEquals(500, provider.getWhiteList().size()); + Assert.assertEquals(3000, provider.getWhiteListHitCount()); + Assert.assertEquals(0, provider.getWhiteList().size()); Assert.assertEquals(3001, provider.getCheckCount()); } diff --git a/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_whiteList_syntaxError.java b/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_whiteList_syntaxError.java index 555cbaa954..27473b581f 100644 --- a/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_whiteList_syntaxError.java +++ b/src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_whiteList_syntaxError.java @@ -30,7 +30,7 @@ public class WallStatTest_whiteList_syntaxError extends TestCase { Assert.assertEquals(0, tableStat.getSelectCount()); Assert.assertEquals(0, provider.getBlackListHitCount()); Assert.assertEquals(0, provider.getWhiteListHitCount()); - Assert.assertEquals(1, provider.getWhiteList().size()); + Assert.assertEquals(0, provider.getWhiteList().size()); Assert.assertEquals(0, provider.getBlackList().size()); Assert.assertEquals(1, provider.getCheckCount()); Assert.assertEquals(1, provider.getSyntaxErrorCount()); @@ -42,12 +42,12 @@ public class WallStatTest_whiteList_syntaxError extends TestCase { WallTableStat tableStat = provider.getTableStat("t"); Assert.assertEquals(0, tableStat.getSelectCount()); Assert.assertEquals(0, provider.getBlackListHitCount()); - Assert.assertEquals(1, provider.getWhiteListHitCount()); - Assert.assertEquals(1, provider.getWhiteList().size()); + Assert.assertEquals(0, provider.getWhiteListHitCount()); + Assert.assertEquals(0, provider.getWhiteList().size()); Assert.assertEquals(0, provider.getBlackList().size()); Assert.assertEquals(2, provider.getCheckCount()); Assert.assertEquals(2, provider.getSyntaxErrorCount()); - Assert.assertEquals(1, provider.getHardCheckCount()); + Assert.assertEquals(2, provider.getHardCheckCount()); } } diff --git a/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlCreateTableTest29.java b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlCreateTableTest29.java index 311a4cafbf..d8b9d5c5ac 100644 --- a/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlCreateTableTest29.java +++ b/src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlCreateTableTest29.java @@ -29,7 +29,7 @@ import com.alibaba.druid.stat.TableStat; public class MySqlCreateTableTest29 extends MysqlTest { public void test_0() throws Exception { - String sql = "CREATE TABLE lookup" + // + String sql = "CREATE TABLE user" + // " (id INT, INDEX USING BTREE (id))" + // " MIN_ROWS 1024;"; // @@ -52,10 +52,10 @@ public class MySqlCreateTableTest29 extends MysqlTest { Assert.assertEquals(1, visitor.getColumns().size()); Assert.assertEquals(0, visitor.getConditions().size()); - Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("lookup"))); + Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("user"))); String output = SQLUtils.toMySqlString(stmt); - Assert.assertEquals("CREATE TABLE lookup (" + // + Assert.assertEquals("CREATE TABLE user (" + // "\n\tid INT, " + // "\n\tINDEX USING BTREE(id)" + // "\n) MIN_ROWS = 1024", output); diff --git a/src/test/java/com/alibaba/druid/pool/MaxWaitTest.java b/src/test/java/com/alibaba/druid/pool/MaxWaitTest.java new file mode 100644 index 0000000000..f18988e1a2 --- /dev/null +++ b/src/test/java/com/alibaba/druid/pool/MaxWaitTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 1999-2011 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.pool; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.concurrent.CountDownLatch; + +import EDU.oswego.cs.dl.util.concurrent.CountDown; +import junit.framework.TestCase; + +public class MaxWaitTest extends TestCase { + + private DruidDataSource dataSource; + + protected void setUp() throws Exception { + dataSource = new DruidDataSource(); + dataSource.setUrl("jdbc:mysql://localhost:3306/sonar"); + dataSource.setUsername("sonar"); + dataSource.setPassword("sonar"); + dataSource.setFilters("stat"); + dataSource.setMaxWait(1000); + dataSource.setInitialSize(3); + dataSource.setMinIdle(3); + dataSource.setMaxActive(3); + } + + public void test_maxWait() throws Exception { + final CountDownLatch latch = new CountDownLatch(10); + for (int i = 0; i < 20; ++i) { + Thread thread = new Thread() { + + public void run() { + try { + for (int i = 0; i < 10; ++i) { + try { + Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + stmt.execute("select sleep(" + (i % 5 + 1) + ")"); + stmt.close(); + conn.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } finally { + latch.countDown(); + } + } + }; + thread.start(); + } + + latch.await(); + } +} diff --git a/src/test/java/com/alibaba/druid/sql/JSqlParserTest.java b/src/test/java/com/alibaba/druid/sql/JSqlParserTest.java new file mode 100644 index 0000000000..03a7515532 --- /dev/null +++ b/src/test/java/com/alibaba/druid/sql/JSqlParserTest.java @@ -0,0 +1,16 @@ +package com.alibaba.druid.sql; + +import java.io.StringReader; + +import junit.framework.TestCase; +import net.sf.jsqlparser.parser.CCJSqlParserManager; + +public class JSqlParserTest extends TestCase { + CCJSqlParserManager parserManager = new CCJSqlParserManager(); + + public void test_0() throws Exception { + String statement = "ALTER TABLE `test`.`tb1` ADD INDEX `ix` (`f2` ASC) ;"; + + parserManager.parse(new StringReader(statement)); + } +}