Enhance RandomDataSourceSelector

1. shorten the checking interval when validate failed
2. provide some parameters in higher level
3. add a sleep before validation (default 0)
4. validate the DataSource concurrently
This commit is contained in:
DigitalSonic
2019-02-01 18:21:16 +08:00
parent 65fafdb67a
commit b6c90db170
6 changed files with 147 additions and 27 deletions
@@ -105,6 +105,7 @@ public class HighAvailableDataSource extends WrapperAdapter implements DataSourc
}
if (selector == null) {
selector = new RandomDataSourceSelector(this);
selector.init();
}
if (dataSourceMap == null || dataSourceMap.isEmpty()) {
LOG.warn("There is NO DataSource available!!! Please check your configuration.");
@@ -29,4 +29,5 @@ public interface DataSourceSelector {
void setTarget(String name);
boolean isSame(String name);
String getName();
void init();
}
@@ -35,6 +35,10 @@ public class NamedDataSourceSelector implements DataSourceSelector {
this.highAvailableDataSource = highAvailableDataSource;
}
@Override
public void init() {
}
@Override
public boolean isSame(String name) {
return getName().equalsIgnoreCase(name);
@@ -43,12 +43,31 @@ public class RandomDataSourceSelector implements DataSourceSelector {
private RandomDataSourceValidateThread validateThread;
private RandomDataSourceRecoverThread recoverThread;
private int checkingIntervalSeconds = 15;
private int recoveryIntervalSeconds = 30;
private int validationSleepSeconds = 0;
private int blacklistThreshold = 3;
public RandomDataSourceSelector(HighAvailableDataSource highAvailableDataSource) {
this.highAvailableDataSource = highAvailableDataSource;
}
@Override
public void init() {
if (highAvailableDataSource == null) {
LOG.warn("highAvailableDataSource is NULL!");
return;
}
if (!highAvailableDataSource.isTestOnBorrow() && !highAvailableDataSource.isTestOnReturn()) {
validateThread = new RandomDataSourceValidateThread(this);
validateThread.setCheckingIntervalSeconds(checkingIntervalSeconds);
validateThread.setValidationSleepSeconds(validationSleepSeconds);
validateThread.setBlacklistThreshold(blacklistThreshold);
new Thread(validateThread, "RandomDataSourceSelector-isValid-thread").start();
recoverThread = new RandomDataSourceRecoverThread(this);
new Thread(validateThread, "RandomDataSourceSelector-validate-thread").start();
recoverThread.setSleepSeconds(recoveryIntervalSeconds);
new Thread(recoverThread, "RandomDataSourceSelector-recover-thread").start();
} else {
LOG.info("testOnBorrow or testOnReturn has been set to true, ignore validateThread");
@@ -123,11 +142,55 @@ public class RandomDataSourceSelector implements DataSourceSelector {
}
}
public HighAvailableDataSource getHighAvailableDataSource() {
return highAvailableDataSource;
}
public RandomDataSourceValidateThread getValidateThread() {
return validateThread;
}
public void setValidateThread(RandomDataSourceValidateThread validateThread) {
this.validateThread = validateThread;
}
public RandomDataSourceRecoverThread getRecoverThread() {
return recoverThread;
}
public void setRecoverThread(RandomDataSourceRecoverThread recoverThread) {
this.recoverThread = recoverThread;
}
public int getCheckingIntervalSeconds() {
return checkingIntervalSeconds;
}
public void setCheckingIntervalSeconds(int checkingIntervalSeconds) {
this.checkingIntervalSeconds = checkingIntervalSeconds;
}
public int getRecoveryIntervalSeconds() {
return recoveryIntervalSeconds;
}
public void setRecoveryIntervalSeconds(int recoveryIntervalSeconds) {
this.recoveryIntervalSeconds = recoveryIntervalSeconds;
}
public int getValidationSleepSeconds() {
return validationSleepSeconds;
}
public void setValidationSleepSeconds(int validationSleepSeconds) {
this.validationSleepSeconds = validationSleepSeconds;
}
public int getBlacklistThreshold() {
return blacklistThreshold;
}
public void setBlacklistThreshold(int blacklistThreshold) {
this.blacklistThreshold = blacklistThreshold;
}
}
@@ -24,22 +24,22 @@ import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import java.util.concurrent.*;
/**
* A Thread trying to test all DataSource provided by HADataSource.
* If a DataSource failed this test for 3 times, it will be put into a blacklist.
* If a DataSource failed this test for 3 times (default value), it will be put into a blacklist.
*
* @author DigitalSonic
*/
public class RandomDataSourceValidateThread implements Runnable {
private final static Log LOG = LogFactory.getLog(RandomDataSourceValidateThread.class);
private int sleepSeconds = 30;
private int checkingIntervalSeconds = 15;
private int validationSleepSeconds = 0;
private int blacklistThreshold = 3;
private RandomDataSourceSelector selector;
private ExecutorService checkExecutor = Executors.newFixedThreadPool(5);
private Map<String, Integer> errorCounts = new HashMap<String, Integer>();
public RandomDataSourceValidateThread(RandomDataSourceSelector selector) {
@@ -53,12 +53,29 @@ public class RandomDataSourceValidateThread implements Runnable {
checkAllDataSources();
maintainBlacklist();
}
try {
Thread.sleep(sleepSeconds * 1000);
} catch (InterruptedException e) {
// ignore
sleepForNextValidation();
}
}
private void sleepForNextValidation() {
int errorCountBelowThreshold = 0;
for (int count : errorCounts.values()) {
if (count > 0 && count < blacklistThreshold && count > errorCountBelowThreshold) {
errorCountBelowThreshold = count;
}
}
int newSleepSeconds = checkingIntervalSeconds / (errorCountBelowThreshold + 1);
if (newSleepSeconds < 1) {
newSleepSeconds = 1;
}
try {
LOG.debug("Sleep " + newSleepSeconds + " second(s) until next checking.");
Thread.sleep(newSleepSeconds * 1000);
} catch (InterruptedException e) {
// ignore
}
}
private void maintainBlacklist() {
@@ -75,21 +92,36 @@ public class RandomDataSourceValidateThread implements Runnable {
private void checkAllDataSources() {
Map<String, DataSource> dataSourceMap = selector.getDataSourceMap();
for (Map.Entry<String, DataSource> e : dataSourceMap.entrySet()) {
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
for (final Map.Entry<String, DataSource> e : dataSourceMap.entrySet()) {
if (!(e.getValue() instanceof DruidDataSource)) {
continue;
}
boolean flag = check(e.getKey(), (DruidDataSource) e.getValue());
if (flag) {
errorCounts.put(e.getKey(), 0);
} else {
if (!errorCounts.containsKey(e.getKey())) {
errorCounts.put(e.getKey(), 0);
tasks.add(new Callable<Boolean>() {
@Override
public Boolean call() {
boolean flag = check(e.getKey(), (DruidDataSource) e.getValue());
if (flag) {
errorCounts.put(e.getKey(), 0);
} else {
if (!errorCounts.containsKey(e.getKey())) {
errorCounts.put(e.getKey(), 0);
}
int count = errorCounts.get(e.getKey());
errorCounts.put(e.getKey(), count + 1);
}
return flag;
}
int count = errorCounts.get(e.getKey());
errorCounts.put(e.getKey(), count + 1);
}
});
}
try {
checkExecutor.invokeAll(tasks);
} catch (Exception e) {
LOG.warn("Exception occurred while checking DataSource.", e);
}
}
@@ -109,8 +141,9 @@ public class RandomDataSourceValidateThread implements Runnable {
info.setProperty("password", password);
}
try {
LOG.debug("Validating " + name + " every " + sleepSeconds + " seconds.");
LOG.debug("Validating " + name + " every " + checkingIntervalSeconds + " seconds.");
conn = driver.connect(url, info);
sleepBeforeValidation();
dataSource.validateConnection(conn);
} catch (SQLException e) {
LOG.warn("Validation FAILED for " + name + " with url [" + url + "] and username ["
@@ -123,8 +156,24 @@ public class RandomDataSourceValidateThread implements Runnable {
return result;
}
public void setSleepSeconds(int sleepSeconds) {
this.sleepSeconds = sleepSeconds;
private void sleepBeforeValidation() {
if (validationSleepSeconds <= 0) {
return;
}
try {
LOG.debug("Sleep " + validationSleepSeconds + " second(s) before validation.");
Thread.sleep(validationSleepSeconds * 1000L);
} catch (InterruptedException e) {
// ignore
}
}
public void setCheckingIntervalSeconds(int checkingIntervalSeconds) {
this.checkingIntervalSeconds = checkingIntervalSeconds;
}
public void setValidationSleepSeconds(int validationSleepSeconds) {
this.validationSleepSeconds = validationSleepSeconds;
}
public void setBlacklistThreshold(int blacklistThreshold) {