Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

improve sql parser & improve GetConnecitonTimeout error message

Browse Source
This commit is contained in:
wenshao
2013-08-07 19:25:55 +08:00
parent 29e84f2be2
commit 65a963f88f
10 changed files with 268 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+11 -4
View File
@@ -17,7 +17,7 @@
<junit.version>4.11</junit.version>
<gpg.skip>true</gpg.skip>
<javadoc.skip>true</javadoc.skip>
<javadoc.skip>true</javadoc.skip>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jdk.version>1.6</jdk.version>
@@ -351,7 +351,7 @@
<version>${spring.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-ibatis</artifactId>
@@ -412,8 +412,8 @@
<version>9.1-901-1.jdbc4</version>
<scope>provided</scope>
</dependency>
<!-- <dependency> <groupId>com.oracle</groupId> <artifactId>ojdbc5</artifactId>
<version>11.2.0.3</version> <scope>provided</scope> </dependency> -->
<!-- <dependency> <groupId>com.oracle</groupId> <artifactId>ojdbc5</artifactId> <version>11.2.0.3</version> <scope>provided</scope>
</dependency> -->
<dependency>
<groupId>com.oracle</groupId>
@@ -533,5 +533,12 @@
<version>1.1.29</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.sf.jsqlparser</groupId>
<artifactId>jsqlparser</artifactId>
<version>0.8.0</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
src/main/java/com/alibaba/druid/pool/DruidDataSource.java
+46 -21
View File
@@ -77,6 +77,7 @@ import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.stat.DruidDataSourceStatManager;
import com.alibaba.druid.stat.JdbcDataSourceStat;
import com.alibaba.druid.stat.JdbcSqlStat;
import com.alibaba.druid.stat.JdbcSqlStatValue;
import com.alibaba.druid.support.logging.Log;
import com.alibaba.druid.support.logging.LogFactory;
import com.alibaba.druid.util.IOUtils;
@@ -144,6 +145,8 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat
private boolean mbeanRegistered = false;
public static ThreadLocal<Long> waitNanosLocal = new ThreadLocal<Long>();
public DruidDataSource(){
this(false);
}
@@ -1012,12 +1015,13 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat
holder = takeLast();
}
activeCount++;
if (activeCount > activePeak) {
activePeak = activeCount;
activePeakTime = System.currentTimeMillis();
if (holder != null) {
activeCount++;
if (activeCount > activePeak) {
activePeak = activeCount;
activePeakTime = System.currentTimeMillis();
}
}
} catch (InterruptedException e) {
connectErrorCount.incrementAndGet();
throw new SQLException(e.getMessage(), e);
@@ -1028,6 +1032,38 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat
lock.unlock();
}
if (holder == null) {
long waitNanos = waitNanosLocal.get();
StringBuilder buf = new StringBuilder();
buf.append("wait millis ")//
.append(waitNanos / (1000 * 1000))//
.append(", active " + activeCount)//
;
List<JdbcSqlStatValue> sqlList = this.getDataSourceStat().getRuningSqlList();
for (int i = 0; i < sqlList.size(); ++i) {
if (i != 0) {
buf.append('\n');
} else {
buf.append(", ");
}
JdbcSqlStatValue sql = sqlList.get(i);
buf.append("runningCount ");
buf.append(sql.getRunningCount());
buf.append(" : ");
buf.append(sql.getSql());
}
String errorMessage = buf.toString();
if (this.createError == null) {
throw new GetConnectionTimeoutException(errorMessage, createError);
} else {
throw new GetConnectionTimeoutException(errorMessage);
}
}
holder.incrementUseCount();
DruidPooledConnection poolalbeConnection = new DruidPooledConnection(holder);
@@ -1313,19 +1349,13 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat
DruidConnectionHolder pollLast(long nanos) throws InterruptedException, SQLException {
long estimate = nanos;
for (int i = 0;; ++i) {
for (;;) {
if (poolingCount == 0) {
empty.signal(); // send signal to CreateThread create connection
if (estimate <= 0) {
String errorMessage = "loopWaitCount " + i + ", wait millis " + (nanos - estimate) / (1000 * 1000)
+ ", active " + activeCount;
if (this.createError == null) {
throw new GetConnectionTimeoutException(errorMessage, createError);
} else {
throw new GetConnectionTimeoutException(errorMessage);
}
waitNanosLocal.set(nanos - estimate);
return null;
}
notEmptyWaitThreadCount++;
@@ -1358,13 +1388,8 @@ public class DruidDataSource extends DruidAbstractDataSource implements DruidDat
continue;
}
String errorMessage = "loopWaitCount " + i + ", wait millis " + (nanos - estimate) / (1000 * 1000)
+ ", active " + activeCount;
if (createError != null) {
throw new GetConnectionTimeoutException(errorMessage, createError);
} else {
throw new GetConnectionTimeoutException(errorMessage);
}
waitNanosLocal.set(nanos - estimate);
return null;
}
}
src/main/java/com/alibaba/druid/stat/JdbcDataSourceStat.java
+30 -3
View File
@@ -97,9 +97,9 @@ public class JdbcDataSourceStat implements JdbcDataSourceStatMBean {
public static void setGlobal(JdbcDataSourceStat value) {
global = value;
}
public void configFromProperties(Properties properties) {
}
public boolean isResetStatEnable() {
@@ -207,7 +207,7 @@ public class JdbcDataSourceStat implements JdbcDataSourceStatMBean {
if (!isResetStatEnable()) {
return;
}
blobOpenCount.set(0);
clobOpenCount.set(0);
@@ -375,6 +375,33 @@ public class JdbcDataSourceStat implements JdbcDataSourceStatMBean {
return values;
}
public List<JdbcSqlStatValue> getRuningSqlList() {
List<JdbcSqlStat> stats = new ArrayList<JdbcSqlStat>(sqlStatMap.size());
lock.readLock().lock();
try {
Iterator<Map.Entry<String, JdbcSqlStat>> iter = sqlStatMap.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry<String, JdbcSqlStat> entry = iter.next();
JdbcSqlStat stat = entry.getValue();
if (stat.getRunningCount() >= 0) {
stats.add(entry.getValue());
}
}
} finally {
lock.readLock().unlock();
}
List<JdbcSqlStatValue> values = new ArrayList<JdbcSqlStatValue>(stats.size());
for (int i = 0; i < stats.size(); ++i) {
JdbcSqlStat stat = stats.get(i);
JdbcSqlStatValue value = stat.getValue(false);
if (value.getRunningCount() > 0) {
values.add(value);
}
}
return values;
}
public JdbcSqlStat getSqlStat(String sql) {
lock.readLock().lock();
try {
src/main/java/com/alibaba/druid/wall/WallProvider.java
+84 -24
View File
@@ -239,6 +239,49 @@ public abstract class WallProvider {
public WallSqlStat addWhiteSql(String sql, Map<String, WallSqlTableStat> tableStats,
Map<String, WallSqlFunctionStat> functionStats, boolean syntaxError) {
String mergedSql;
try {
mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType);
} catch (Exception ex) {
WallSqlStat stat = new WallSqlStat(tableStats, functionStats, syntaxError);
stat.incrementAndGetExecuteCount();
return stat;
}
if (mergedSql != sql) {
WallSqlStat mergedStat;
lock.readLock().lock();
try {
if (whiteMergedList == null) {
whiteMergedList = new LRUCache<String, WallSqlStat>(whiteSqlMaxSize);
}
mergedStat = whiteMergedList.get(mergedSql);
} finally {
lock.readLock().unlock();
}
if (mergedStat == null) {
WallSqlStat newStat = new WallSqlStat(tableStats, functionStats, syntaxError);
newStat.setSample(sql);
lock.writeLock().lock();
try {
mergedStat = whiteMergedList.get(mergedSql);
if (mergedStat == null) {
whiteMergedList.put(mergedSql, newStat);
mergedStat = newStat;
}
} finally {
lock.writeLock().unlock();
}
}
mergedStat.incrementAndGetExecuteCount();
return mergedStat;
}
lock.writeLock().lock();
try {
if (whiteList == null) {
@@ -251,12 +294,6 @@ public abstract class WallProvider {
WallSqlStat wallStat = whiteList.get(sql);
if (wallStat == null) {
String mergedSql;
try {
mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType);
} catch (Exception ex) {
mergedSql = sql;
}
wallStat = whiteMergedList.get(mergedSql);
if (wallStat == null) {
wallStat = new WallSqlStat(tableStats, functionStats, syntaxError);
@@ -277,6 +314,14 @@ public abstract class WallProvider {
public WallSqlStat addBlackSql(String sql, Map<String, WallSqlTableStat> tableStats,
Map<String, WallSqlFunctionStat> functionStats, List<Violation> violations,
boolean syntaxError) {
String mergedSql;
try {
mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType);
} catch (Exception ex) {
// skip
mergedSql = sql;
}
lock.writeLock().lock();
try {
if (blackList == null) {
@@ -289,12 +334,6 @@ public abstract class WallProvider {
WallSqlStat wallStat = blackList.get(sql);
if (wallStat == null) {
String mergedSql;
try {
mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType);
} catch (Exception ex) {
mergedSql = sql;
}
wallStat = blackMergedList.get(mergedSql);
if (wallStat == null) {
wallStat = new WallSqlStat(tableStats, functionStats, violations, syntaxError);
@@ -401,16 +440,37 @@ public abstract class WallProvider {
}
public WallSqlStat getWhiteSql(String sql) {
WallSqlStat stat = null;
lock.readLock().lock();
try {
if (whiteList == null) {
return null;
if (whiteList != null) {
stat = whiteList.get(sql);
}
return whiteList.get(sql);
} finally {
lock.readLock().unlock();
}
if (stat == null) {
String mergedSql;
try {
mergedSql = ParameterizedOutputVisitorUtils.parameterize(sql, dbType);
} catch (Exception ex) {
// skip
return null;
}
lock.readLock().lock();
try {
if (whiteMergedList == null) {
return null;
}
stat = whiteMergedList.get(mergedSql);
} finally {
lock.readLock().unlock();
}
}
return stat;
}
public WallSqlStat getBlackSql(String sql) {
@@ -832,13 +892,13 @@ public abstract class WallProvider {
WallTableStat tableStat = entry.getValue();
WallTableStatValue tableStatValue = tableStat.getStatValue(reset);
if (tableStatValue.getTotalExecuteCount() == 0) {
continue;
}
tableStatValue.setName(tableName);
statValue.getTables().add(tableStatValue);
}
@@ -847,7 +907,7 @@ public abstract class WallProvider {
WallFunctionStat functionStat = entry.getValue();
WallFunctionStatValue functionStatValue = functionStat.getStatValue(reset);
if (functionStatValue.getInvokeCount() == 0) {
continue;
}
@@ -864,11 +924,11 @@ public abstract class WallProvider {
String sql = entry.getKey();
WallSqlStat sqlStat = entry.getValue();
WallSqlStatValue sqlStatValue = sqlStat.getStatValue(reset);
if (sqlStatValue.getExecuteCount() == 0) {
continue;
}
sqlStatValue.setSql(sql);
statValue.getWhiteList().add(sqlStatValue);
}
@@ -879,11 +939,11 @@ public abstract class WallProvider {
String sql = entry.getKey();
WallSqlStat sqlStat = entry.getValue();
WallSqlStatValue sqlStatValue = sqlStat.getStatValue(reset);
if (sqlStatValue.getExecuteCount() == 0) {
continue;
}
sqlStatValue.setSql(sql);
statValue.getBlackList().add(sqlStatValue);
}
src/test/java/com/alibaba/druid/bvt/filter/wall/WallProviderTest.java
+3 -3
View File
@@ -52,7 +52,7 @@ public class WallProviderTest extends TestCase {
Assert.assertSame(whiteStat_1, provider.getSqlStat(sql_x));
}
Assert.assertEquals(500, provider.getWhiteList().size());
Assert.assertEquals(0, provider.getWhiteList().size());
Assert.assertEquals(2, provider.getSqlList().size());
Assert.assertFalse(provider.checkValid("slelc"));
@@ -66,10 +66,10 @@ public class WallProviderTest extends TestCase {
Assert.assertEquals(1009, statValue.getCheckCount());
Assert.assertEquals(2, statValue.getBlackListHitCount());
Assert.assertEquals(1005, statValue.getHardCheckCount());
Assert.assertEquals(4, statValue.getHardCheckCount());
Assert.assertEquals(1, statValue.getSyntaxErrorCount());
Assert.assertEquals(5, statValue.getViolationCount());
Assert.assertEquals(2, statValue.getWhiteListHitCount());
Assert.assertEquals(1003, statValue.getWhiteListHitCount());
Assert.assertEquals(1, statValue.getWhiteList().size());
Assert.assertEquals(2, statValue.getBlackList().size());
src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_WhiteList.java
+2 -2
View File
@@ -30,8 +30,8 @@ public class WallStatTest_WhiteList extends TestCase {
WallTableStat tableStat = provider.getTableStat("t");
Assert.assertEquals(3001, tableStat.getSelectCount());
Assert.assertEquals(0, provider.getBlackListHitCount());
Assert.assertEquals(0, provider.getWhiteListHitCount());
Assert.assertEquals(500, provider.getWhiteList().size());
Assert.assertEquals(3000, provider.getWhiteListHitCount());
Assert.assertEquals(0, provider.getWhiteList().size());
Assert.assertEquals(3001, provider.getCheckCount());
}
src/test/java/com/alibaba/druid/bvt/filter/wall/WallStatTest_whiteList_syntaxError.java
+4 -4
View File
@@ -30,7 +30,7 @@ public class WallStatTest_whiteList_syntaxError extends TestCase {
Assert.assertEquals(0, tableStat.getSelectCount());
Assert.assertEquals(0, provider.getBlackListHitCount());
Assert.assertEquals(0, provider.getWhiteListHitCount());
Assert.assertEquals(1, provider.getWhiteList().size());
Assert.assertEquals(0, provider.getWhiteList().size());
Assert.assertEquals(0, provider.getBlackList().size());
Assert.assertEquals(1, provider.getCheckCount());
Assert.assertEquals(1, provider.getSyntaxErrorCount());
@@ -42,12 +42,12 @@ public class WallStatTest_whiteList_syntaxError extends TestCase {
WallTableStat tableStat = provider.getTableStat("t");
Assert.assertEquals(0, tableStat.getSelectCount());
Assert.assertEquals(0, provider.getBlackListHitCount());
Assert.assertEquals(1, provider.getWhiteListHitCount());
Assert.assertEquals(1, provider.getWhiteList().size());
Assert.assertEquals(0, provider.getWhiteListHitCount());
Assert.assertEquals(0, provider.getWhiteList().size());
Assert.assertEquals(0, provider.getBlackList().size());
Assert.assertEquals(2, provider.getCheckCount());
Assert.assertEquals(2, provider.getSyntaxErrorCount());
Assert.assertEquals(1, provider.getHardCheckCount());
Assert.assertEquals(2, provider.getHardCheckCount());
}
}
src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlCreateTableTest29.java
+3 -3
View File
@@ -29,7 +29,7 @@ import com.alibaba.druid.stat.TableStat;
public class MySqlCreateTableTest29 extends MysqlTest {
public void test_0() throws Exception {
String sql = "CREATE TABLE lookup" + //
String sql = "CREATE TABLE user" + //
" (id INT, INDEX USING BTREE (id))" + //
" MIN_ROWS 1024;"; //
@@ -52,10 +52,10 @@ public class MySqlCreateTableTest29 extends MysqlTest {
Assert.assertEquals(1, visitor.getColumns().size());
Assert.assertEquals(0, visitor.getConditions().size());
Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("lookup")));
Assert.assertTrue(visitor.getTables().containsKey(new TableStat.Name("user")));
String output = SQLUtils.toMySqlString(stmt);
Assert.assertEquals("CREATE TABLE lookup (" + //
Assert.assertEquals("CREATE TABLE user (" + //
"\n\tid INT, " + //
"\n\tINDEX USING BTREE(id)" + //
"\n) MIN_ROWS = 1024", output);
src/test/java/com/alibaba/druid/pool/MaxWaitTest.java
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright 1999-2011 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.pool;
import java.sql.Connection;
import java.sql.Statement;
import java.util.concurrent.CountDownLatch;
import EDU.oswego.cs.dl.util.concurrent.CountDown;
import junit.framework.TestCase;
public class MaxWaitTest extends TestCase {
private DruidDataSource dataSource;
protected void setUp() throws Exception {
dataSource = new DruidDataSource();
dataSource.setUrl("jdbc:mysql://localhost:3306/sonar");
dataSource.setUsername("sonar");
dataSource.setPassword("sonar");
dataSource.setFilters("stat");
dataSource.setMaxWait(1000);
dataSource.setInitialSize(3);
dataSource.setMinIdle(3);
dataSource.setMaxActive(3);
}
public void test_maxWait() throws Exception {
final CountDownLatch latch = new CountDownLatch(10);
for (int i = 0; i < 20; ++i) {
Thread thread = new Thread() {
public void run() {
try {
for (int i = 0; i < 10; ++i) {
try {
Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
stmt.execute("select sleep(" + (i % 5 + 1) + ")");
stmt.close();
conn.close();
} catch (Exception e) {
e.printStackTrace();
}
}
} finally {
latch.countDown();
}
}
};
thread.start();
}
latch.await();
}
}
src/test/java/com/alibaba/druid/sql/JSqlParserTest.java
+16
View File
@@ -0,0 +1,16 @@
package com.alibaba.druid.sql;
import java.io.StringReader;
import junit.framework.TestCase;
import net.sf.jsqlparser.parser.CCJSqlParserManager;
public class JSqlParserTest extends TestCase {
CCJSqlParserManager parserManager = new CCJSqlParserManager();
public void test_0() throws Exception {
String statement = "ALTER TABLE `test`.`tb1` ADD INDEX `ix` (`f2` ASC) ;";
parserManager.parse(new StringReader(statement));
}
}