From 89c36661cb8fe453792a359c23158d29e86b8a75 Mon Sep 17 00:00:00 2001 From: DigitalSonic Date: Sun, 13 Oct 2019 16:22:21 +0800 Subject: [PATCH] Optimize the Validation logic of RandomDataSource treating normal SQL execution in the pool as validation, so we can skip some validation operations --- .../RandomDataSourceValidateFilter.java | 53 +++++++++++++++++ .../RandomDataSourceValidateThread.java | 57 +++++++++++++++---- .../META-INF/druid-filter.properties | 3 +- 3 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateFilter.java diff --git a/src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateFilter.java b/src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateFilter.java new file mode 100644 index 0000000000..f0b1541ec4 --- /dev/null +++ b/src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateFilter.java @@ -0,0 +1,53 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.pool.ha.selector; + +import com.alibaba.druid.filter.FilterEventAdapter; +import com.alibaba.druid.proxy.jdbc.ConnectionProxy; +import com.alibaba.druid.proxy.jdbc.DataSourceProxy; +import com.alibaba.druid.proxy.jdbc.ResultSetProxy; +import com.alibaba.druid.proxy.jdbc.StatementProxy; + +/** + * A Druid Filter that records the last success execute time. + * + * @author DigitalSonic + */ +public class RandomDataSourceValidateFilter extends FilterEventAdapter { + protected void statementExecuteUpdateAfter(StatementProxy statement, String sql, int updateCount) { + recordTime(statement); + } + + protected void statementExecuteQueryAfter(StatementProxy statement, String sql, ResultSetProxy resultSet) { + recordTime(statement); + } + + protected void statementExecuteAfter(StatementProxy statement, String sql, boolean result) { + recordTime(statement); + } + + protected void statementExecuteBatchAfter(StatementProxy statement, int[] result) { + recordTime(statement); + } + + private void recordTime(StatementProxy statement) { + ConnectionProxy conn = statement.getConnectionProxy(); + if (conn != null) { + DataSourceProxy dataSource = conn.getDirectDataSource(); + RandomDataSourceValidateThread.logSuccessTime(dataSource); + } + } +} diff --git a/src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateThread.java b/src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateThread.java index 2cc268f468..5a64b5c856 100644 --- a/src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateThread.java +++ b/src/main/java/com/alibaba/druid/pool/ha/selector/RandomDataSourceValidateThread.java @@ -17,7 +17,6 @@ package com.alibaba.druid.pool.ha.selector; import java.sql.Connection; import java.sql.Driver; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -30,9 +29,11 @@ import java.util.concurrent.Executors; import javax.sql.DataSource; import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.proxy.jdbc.DataSourceProxy; import com.alibaba.druid.support.logging.Log; import com.alibaba.druid.support.logging.LogFactory; import com.alibaba.druid.util.JdbcUtils; +import com.alibaba.druid.util.StringUtils; /** * A Thread trying to test all DataSource provided by HADataSource. @@ -42,12 +43,24 @@ import com.alibaba.druid.util.JdbcUtils; */ public class RandomDataSourceValidateThread implements Runnable { private final static Log LOG = LogFactory.getLog(RandomDataSourceValidateThread.class); - private int checkingIntervalSeconds = 15; + private final static Map SUCCESS_TIMES = new ConcurrentHashMap(); + + private int checkingIntervalSeconds = 10; // This value should NOT be too small. private int validationSleepSeconds = 0; private int blacklistThreshold = 3; private RandomDataSourceSelector selector; private ExecutorService checkExecutor = Executors.newFixedThreadPool(5); private Map errorCounts = new ConcurrentHashMap(); + private Map lastCheckTimes = new ConcurrentHashMap(); + + /** + * Provide a static method to record the last success time of a DataSource + */ + public static void logSuccessTime(DataSourceProxy dataSource) { + if (dataSource != null && !StringUtils.isEmpty(dataSource.getName())) { + SUCCESS_TIMES.put(dataSource.getName(), System.currentTimeMillis()); + } + } public RandomDataSourceValidateThread(RandomDataSourceSelector selector) { this.selector = selector; @@ -88,12 +101,13 @@ public class RandomDataSourceValidateThread implements Runnable { private void maintainBlacklist() { Map dataSourceMap = selector.getDataSourceMap(); for (Map.Entry e : errorCounts.entrySet()) { + DataSource dataSource = dataSourceMap.get(e.getKey()); if (e.getValue() <= 0) { - selector.removeBlacklist(dataSourceMap.get(e.getKey())); + selector.removeBlacklist(dataSource); } else if (e.getValue() >= blacklistThreshold - && !selector.containInBlacklist(dataSourceMap.get(e.getKey()))) { + && !selector.containInBlacklist(dataSource)) { LOG.warn("Adding " + e.getKey() + " to blacklist."); - selector.addBlacklist(dataSourceMap.get(e.getKey())); + selector.addBlacklist(dataSource); } } } @@ -115,17 +129,26 @@ public class RandomDataSourceValidateThread implements Runnable { tasks.add(new Callable() { @Override public Boolean call() { - boolean flag = check(e.getKey(), (DruidDataSource) e.getValue()); + String key = e.getKey(); + DruidDataSource dataSource = (DruidDataSource) e.getValue(); + + if (isSkipChecking(key, dataSource)) { + return true; + } + + boolean flag = check(key, dataSource); if (flag) { - errorCounts.put(e.getKey(), 0); + logSuccessTime(dataSource); + errorCounts.put(key, 0); } else { - if (!errorCounts.containsKey(e.getKey())) { - errorCounts.put(e.getKey(), 0); + if (!errorCounts.containsKey(key)) { + errorCounts.put(key, 0); } - int count = errorCounts.get(e.getKey()); - errorCounts.put(e.getKey(), count + 1); + int count = errorCounts.get(key); + errorCounts.put(key, count + 1); } + lastCheckTimes.put(dataSource.getName(), System.currentTimeMillis()); return flag; } @@ -138,6 +161,16 @@ public class RandomDataSourceValidateThread implements Runnable { } } + private boolean isSkipChecking(String key, DruidDataSource dataSource) { + Long lastSuccessTime = SUCCESS_TIMES.get(dataSource.getName()); + Long lastCheckTime = lastCheckTimes.get(dataSource.getName()); + + return lastSuccessTime != null && lastCheckTime != null + && (System.currentTimeMillis() - lastSuccessTime) <= checkingIntervalSeconds * 1000 + && (System.currentTimeMillis() - lastCheckTime) <= 5 * checkingIntervalSeconds * 1000 + && (!errorCounts.containsKey(key) || errorCounts.get(key) < 1); + } + private boolean check(String name, DruidDataSource dataSource) { boolean result = true; Driver driver = dataSource.getRawDriver(); @@ -158,7 +191,7 @@ public class RandomDataSourceValidateThread implements Runnable { conn = driver.connect(url, info); sleepBeforeValidation(); dataSource.validateConnection(conn); - } catch (SQLException e) { + } catch (Exception e) { LOG.warn("Validation FAILED for " + name + " with url [" + url + "] and username [" + info.getProperty("user") + "]. Exception: " + e.getMessage()); result = false; diff --git a/src/main/resources/META-INF/druid-filter.properties b/src/main/resources/META-INF/druid-filter.properties index 5f8cbac322..ca9f2537e1 100644 --- a/src/main/resources/META-INF/druid-filter.properties +++ b/src/main/resources/META-INF/druid-filter.properties @@ -9,4 +9,5 @@ druid.filters.slf4j=com.alibaba.druid.filter.logging.Slf4jLogFilter druid.filters.commonlogging=com.alibaba.druid.filter.logging.CommonsLogFilter druid.filters.commonLogging=com.alibaba.druid.filter.logging.CommonsLogFilter druid.filters.wall=com.alibaba.druid.wall.WallFilter -druid.filters.config=com.alibaba.druid.filter.config.ConfigFilter \ No newline at end of file +druid.filters.config=com.alibaba.druid.filter.config.ConfigFilter +druid.filters.haRandomValidator=com.alibaba.druid.pool.ha.selector.RandomDataSourceValidateFilter \ No newline at end of file