From 5c50f0f37a2581aec3823138011ece2a1856b9b9 Mon Sep 17 00:00:00 2001 From: DigitalSonic Date: Wed, 11 Apr 2018 09:02:06 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0JDBC=20URL=E5=8D=A0=E4=BD=8D?= =?UTF-8?q?=E7=AC=A6=E6=9B=BF=E6=8D=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根据配置文件,对URL中的占位符进行替换,同时提供定时健康检查的功能,可剔除不可用节点。 --- .../com/alibaba/druid/filter/FilterChain.java | 2 + .../alibaba/druid/filter/FilterChainImpl.java | 4 + .../url/ConnectionConnectFilterChainImpl.java | 65 ++++++++ .../druid/filter/url/HostAndPortHolder.java | 141 ++++++++++++++++++ .../url/HostAndPortValidatorThread.java | 105 +++++++++++++ .../url/JdbcUrlHostAndPortReplaceFilter.java | 67 +++++++++ .../filter/url/HostAndPortHolderTest.java | 63 ++++++++ .../JdbcUrlHostAndPortReplaceFilterTest.java | 50 +++++++ .../alibaba/druid/filter/url/dataSource.xml | 29 ++++ .../druid/filter/url/datasource.properties | 4 + 10 files changed, 530 insertions(+) create mode 100644 src/main/java/com/alibaba/druid/filter/url/ConnectionConnectFilterChainImpl.java create mode 100644 src/main/java/com/alibaba/druid/filter/url/HostAndPortHolder.java create mode 100644 src/main/java/com/alibaba/druid/filter/url/HostAndPortValidatorThread.java create mode 100644 src/main/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilter.java create mode 100644 src/test/java/com/alibaba/druid/filter/url/HostAndPortHolderTest.java create mode 100644 src/test/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilterTest.java create mode 100644 src/test/resources/com/alibaba/druid/filter/url/dataSource.xml create mode 100644 src/test/resources/com/alibaba/druid/filter/url/datasource.properties diff --git a/src/main/java/com/alibaba/druid/filter/FilterChain.java b/src/main/java/com/alibaba/druid/filter/FilterChain.java index 681b211ff8..9b63e082d7 100644 --- a/src/main/java/com/alibaba/druid/filter/FilterChain.java +++ b/src/main/java/com/alibaba/druid/filter/FilterChain.java @@ -62,6 +62,8 @@ public interface FilterChain { int getFilterSize(); + int getPos(); + FilterChain cloneChain(); T unwrap(Wrapper wrapper, java.lang.Class iface) throws java.sql.SQLException; diff --git a/src/main/java/com/alibaba/druid/filter/FilterChainImpl.java b/src/main/java/com/alibaba/druid/filter/FilterChainImpl.java index e1279e3d24..c9542f7c5f 100644 --- a/src/main/java/com/alibaba/druid/filter/FilterChainImpl.java +++ b/src/main/java/com/alibaba/druid/filter/FilterChainImpl.java @@ -92,6 +92,10 @@ public class FilterChainImpl implements FilterChain { return filterSize; } + public int getPos() { + return pos; + } + public void reset() { pos = 0; } diff --git a/src/main/java/com/alibaba/druid/filter/url/ConnectionConnectFilterChainImpl.java b/src/main/java/com/alibaba/druid/filter/url/ConnectionConnectFilterChainImpl.java new file mode 100644 index 0000000000..9892dadc97 --- /dev/null +++ b/src/main/java/com/alibaba/druid/filter/url/ConnectionConnectFilterChainImpl.java @@ -0,0 +1,65 @@ +package com.alibaba.druid.filter.url; + +import com.alibaba.druid.filter.FilterChain; +import com.alibaba.druid.filter.FilterChainImpl; +import com.alibaba.druid.proxy.jdbc.ConnectionProxy; +import com.alibaba.druid.proxy.jdbc.ConnectionProxyImpl; +import com.alibaba.druid.proxy.jdbc.DataSourceProxy; +import com.alibaba.druid.support.logging.Log; +import com.alibaba.druid.support.logging.LogFactory; + +import java.sql.Connection; +import java.sql.Driver; +import java.sql.SQLException; +import java.util.Properties; + +public class ConnectionConnectFilterChainImpl extends FilterChainImpl { + private final static Log LOG = LogFactory.getLog(ConnectionConnectFilterChainImpl.class); + + public ConnectionConnectFilterChainImpl(FilterChain filterChain) { + this(filterChain.getDataSource(), filterChain.getPos()); + } + + public ConnectionConnectFilterChainImpl(DataSourceProxy dataSource, int pos) { + super(dataSource, pos); + } + + @Override + public ConnectionProxy connection_connect(Properties info) throws SQLException { + if (this.pos < getFilterSize()) { + return getFilters().get(pos++).connection_connect(this, info); + } + + DataSourceProxy dataSource = getDataSource(); + Driver driver = dataSource.getRawDriver(); + String url = dataSource.getRawJdbcUrl(); + + url = processUrl(url); + + Connection nativeConnection = driver.connect(url, info); + + if (nativeConnection == null) { + return null; + } + + return new ConnectionProxyImpl(dataSource, nativeConnection, info, dataSource.createConnectionId()); + } + + protected String processUrl(String url) { + if (url == null || (url != null && + (url.indexOf("${") == -1 || url.indexOf("}") == -1))) { + return url; + } + + String processedUrl = url; + String placeHolder = url.substring(url.indexOf("${") + 2, url.indexOf("}")); + String value = HostAndPortHolder.getInstance().get(placeHolder); + if (value != null) { + LOG.info("PlaceHolder ${" + placeHolder + "} is found in url, it will be replace by " + value); + processedUrl = url.replace("${" + placeHolder + "}", value); + } else { + LOG.info("PlaceHolder ${" + placeHolder + "} is found in url, but no value is bind."); + } + return processedUrl; + } +} diff --git a/src/main/java/com/alibaba/druid/filter/url/HostAndPortHolder.java b/src/main/java/com/alibaba/druid/filter/url/HostAndPortHolder.java new file mode 100644 index 0000000000..5ac334075d --- /dev/null +++ b/src/main/java/com/alibaba/druid/filter/url/HostAndPortHolder.java @@ -0,0 +1,141 @@ +package com.alibaba.druid.filter.url; + +import com.alibaba.druid.support.logging.Log; +import com.alibaba.druid.support.logging.LogFactory; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; + +public class HostAndPortHolder { + private static final Log LOG = LogFactory.getLog(ConnectionConnectFilterChainImpl.class); + + public static final String UNAVAILABLE = "UNAVAILABLE_HOST"; + + private static HostAndPortHolder instance = new HostAndPortHolder(); + + private Map> holder = new HashMap>(); + private List blacklist = new ArrayList(); + + private Random random = new Random(); + + public static HostAndPortHolder getInstance() { + return instance; + } + + public static void loadFromProperties(String file) { + Properties properties = new Properties(); + if (file == null) { + return; + } + InputStream is; + try { + LOG.debug("Trying to load " + file + " from FileSystem."); + is = new FileInputStream(file); + } catch(FileNotFoundException e) { + LOG.debug("Trying to load " + file + " from Classpath."); + is = HostAndPortHolder.class.getResourceAsStream(file); + } + if (is != null) { + try { + properties.load(is); + } catch(Exception e) { + LOG.error("Exception occurred while loading " + file, e); + } finally { + if (is != null) { + try { + is.close(); + } catch (Exception e) { + // ignore + } + } + } + } else { + LOG.warn("File " + file + " can't be loaded!"); + } + + for (Object k : properties.keySet()) { + String v = properties.getProperty((String) k); + if (v != null && v.trim().length() > 0) { + LOG.info("Loading placeholder " + k + " = [" + v + "]"); + List values = Arrays.asList(v.split(";")); + getInstance().putHostAndPorts((String) k, values); + } + } + + } + + private HostAndPortHolder() { + } + + public void putHostAndPorts(String key, List values) { + if (key == null || values == null || values.isEmpty()) { + return; + } + holder.put(key, values); + } + + public Map> getHolder() { + return holder; + } + + public void clearBlacklist() { + blacklist.clear(); + } + + public void addBlacklist(String hostAndPort) { + blacklist.add(hostAndPort); + } + + public void removeBlacklist(String hostAndPort) { + if (hostAndPort == null) { + return; + } + blacklist.remove(hostAndPort); + } + + public int getBlacklistSize() { + return blacklist.size(); + } + + public String get(String name) { + if (name == null || !holder.containsKey(name)) { + LOG.warn("Can not find placeholder " + name + "."); + return UNAVAILABLE; + } + + List hostAndPorts = holder.get(name); + int idx = random.nextInt(hostAndPorts.size()); + String returnValue = hostAndPorts.get(idx); + String hostAndPort = returnValue; + for (int i = 0; i < hostAndPorts.size(); i++) { + if (isAvailable(hostAndPort)) { + returnValue = hostAndPort; + break; + } + if (++idx >= hostAndPorts.size()) { + idx = 0; + } + hostAndPort = hostAndPorts.get(idx); + } + return returnValue; + } + + public List getAll(String name) { + if (name == null || !holder.containsKey(name)) { + return new ArrayList(); + } + return holder.get(name); + } + + private boolean isAvailable(String hostAndPort) { + return !blacklist.contains(hostAndPort); + } +} diff --git a/src/main/java/com/alibaba/druid/filter/url/HostAndPortValidatorThread.java b/src/main/java/com/alibaba/druid/filter/url/HostAndPortValidatorThread.java new file mode 100644 index 0000000000..05f426b685 --- /dev/null +++ b/src/main/java/com/alibaba/druid/filter/url/HostAndPortValidatorThread.java @@ -0,0 +1,105 @@ +package com.alibaba.druid.filter.url; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.support.logging.Log; +import com.alibaba.druid.support.logging.LogFactory; + +import java.sql.Connection; +import java.sql.Driver; +import java.sql.SQLException; +import java.util.List; +import java.util.Properties; + +public class HostAndPortValidatorThread implements Runnable { + private final static Log LOG = LogFactory.getLog(HostAndPortValidatorThread.class); + private DruidDataSource dataSource; + private int sleepSeconds = 30; + private boolean modifyTestFlags = false; + private boolean oldTestOnBorrow = false; + private boolean oldTestOnReturn = false; + + @Override + public void run() { + oldTestOnBorrow = dataSource.isTestOnBorrow(); + oldTestOnReturn = dataSource.isTestOnReturn(); + while (true) { + try { + check(); + } catch(Exception e) { + LOG.error("Validation Failed!", e); + } finally { + try { + Thread.sleep(sleepSeconds * 1000); + } catch (InterruptedException e) { + // ignore + } + } + } + } + + protected void check() { + Driver driver = dataSource.getRawDriver(); + Properties info = dataSource.getConnectProperties(); + String url = dataSource.getRawJdbcUrl(); + if (url == null || (url != null && + (url.indexOf("${") == -1 || url.indexOf("}") == -1))) { + return; + } + + String placeHolder = url.substring(url.indexOf("${") + 2, url.indexOf("}")); + List values = HostAndPortHolder.getInstance().getAll(placeHolder); + for (String v : values) { + Connection conn = null; + try { + String realUrl = url.replace("${" + placeHolder + "}", v); + conn = driver.connect(realUrl, info); + dataSource.validateConnection(conn); + HostAndPortHolder.getInstance().removeBlacklist(v); + } catch (SQLException e) { + LOG.warn("Validation FAILED for " + v, e); + HostAndPortHolder.getInstance().addBlacklist(v); + modifyFlagsIfNeeded(); + } finally { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + LOG.error("Can not close connection for HostAndPort Validation.", e); + } + } + } + } + if (HostAndPortHolder.getInstance().getBlacklistSize() == 0) { + resetFlagsIfNeeded(); + } + } + + private void resetFlagsIfNeeded() { + if (modifyTestFlags) { + dataSource.setTestOnBorrow(oldTestOnBorrow); + dataSource.setTestOnReturn(oldTestOnReturn); + } + } + + private void modifyFlagsIfNeeded() { + if (!modifyTestFlags) { + return; + } + if (oldTestOnReturn || oldTestOnBorrow) { + dataSource.setTestOnBorrow(true); + dataSource.setTestOnReturn(true); + } + } + + public void setDataSource(DruidDataSource dataSource) { + this.dataSource = dataSource; + } + + public void setSleepSeconds(int sleepSeconds) { + this.sleepSeconds = sleepSeconds; + } + + public void setModifyTestFlags(boolean modifyTestFlags) { + this.modifyTestFlags = modifyTestFlags; + } +} diff --git a/src/main/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilter.java b/src/main/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilter.java new file mode 100644 index 0000000000..b51d22e088 --- /dev/null +++ b/src/main/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilter.java @@ -0,0 +1,67 @@ +package com.alibaba.druid.filter.url; + +import com.alibaba.druid.filter.FilterAdapter; +import com.alibaba.druid.filter.FilterChain; +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.proxy.jdbc.ConnectionProxy; +import com.alibaba.druid.proxy.jdbc.DataSourceProxy; +import com.alibaba.druid.support.logging.Log; +import com.alibaba.druid.support.logging.LogFactory; + +import java.sql.SQLException; +import java.util.Properties; + +public class JdbcUrlHostAndPortReplaceFilter extends FilterAdapter { + private final static Log LOG = LogFactory.getLog(HostAndPortValidatorThread.class); + + public static final String DEFAULT_FILE = "/druid-datasource.properties"; + + private String file = DEFAULT_FILE; + + private int healthCheckIntervalSeconds = 0; + private HostAndPortValidatorThread validatorThread; + private boolean isValidatorThreadStarted = false; + + public void init() { + if (file == null || file.trim().length() == 0) { + file = DEFAULT_FILE; + } + HostAndPortHolder.loadFromProperties(file); + } + + @Override + public ConnectionProxy connection_connect(FilterChain chain, Properties info) throws SQLException { + startValidationThreadIfNeeded(chain); + return super.connection_connect(new ConnectionConnectFilterChainImpl(chain), info); + } + + private void startValidationThreadIfNeeded(FilterChain chain) { + if (isValidatorThreadStarted || healthCheckIntervalSeconds <= 0) { + return; + } + DataSourceProxy dataSource = chain.getDataSource(); + if (dataSource instanceof DruidDataSource) { + if (validatorThread == null) { + validatorThread = new HostAndPortValidatorThread(); + validatorThread.setSleepSeconds(healthCheckIntervalSeconds); + } + validatorThread.setDataSource((DruidDataSource) dataSource); + new Thread(validatorThread).start(); + isValidatorThreadStarted = true; + } else { + LOG.warn("The DataSource is not a DruidDataSource, ignore HostAndPortValidator."); + } + } + + public void setFile(String file) { + this.file = file; + } + + public void setHealthCheckIntervalSeconds(int healthCheckIntervalSeconds) { + this.healthCheckIntervalSeconds = healthCheckIntervalSeconds; + } + + public void setValidatorThread(HostAndPortValidatorThread validatorThread) { + this.validatorThread = validatorThread; + } +} diff --git a/src/test/java/com/alibaba/druid/filter/url/HostAndPortHolderTest.java b/src/test/java/com/alibaba/druid/filter/url/HostAndPortHolderTest.java new file mode 100644 index 0000000000..e177c9a581 --- /dev/null +++ b/src/test/java/com/alibaba/druid/filter/url/HostAndPortHolderTest.java @@ -0,0 +1,63 @@ +package com.alibaba.druid.filter.url; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class HostAndPortHolderTest { + @Before + public void setUp() { + HostAndPortHolder.loadFromProperties("/com/alibaba/druid/filter/url/datasource.properties"); + } + + @After + public void tearDown() { + HostAndPortHolder.getInstance().clearBlacklist(); + } + + @Test + public void testLoadFromProperties() { + Map> holder = HostAndPortHolder.getInstance().getHolder(); + assertEquals(3, holder.size()); + assertEquals("127.0.0.1:3066", holder.get("datasource.foo").get(0)); + assertEquals("192.168.0.1:3066", holder.get("datasource.foo").get(1)); + assertFalse(holder.containsKey("datasource.empty")); + } + + @Test + public void testRandomGet() { + boolean[] flags = new boolean[] { false, false }; + HostAndPortHolder holder = HostAndPortHolder.getInstance(); + for (int i = 0; i < 10; i++) { + String v = holder.get("datasource.foo"); + if (v.equals("127.0.0.1:3066")) { + flags[0] = true; + } else if (v.equals("192.168.0.1:3066")) { + flags[1] = true; + } + } + assertTrue(flags[0] && flags[1]); + } + + @Test + public void testGetOne() { + HostAndPortHolder holder = HostAndPortHolder.getInstance(); + String value = holder.get("datasource.foo"); + assertTrue(value.equals("127.0.0.1:3066") || value.equals("192.168.0.1:3066")); + assertEquals(HostAndPortHolder.UNAVAILABLE, holder.get("UNAVAILABLE")); + } + + @Test + public void testBlackList() { + HostAndPortHolder holder = HostAndPortHolder.getInstance(); + holder.addBlacklist("192.168.0.1:3066"); + for (int i = 0; i < 100; i++) { + assertNotEquals("192.168.0.1:3066", holder.get("datasource.foo")); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilterTest.java b/src/test/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilterTest.java new file mode 100644 index 0000000000..6e1d4fd409 --- /dev/null +++ b/src/test/java/com/alibaba/druid/filter/url/JdbcUrlHostAndPortReplaceFilterTest.java @@ -0,0 +1,50 @@ +package com.alibaba.druid.filter.url; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidPooledConnection; +import com.alibaba.druid.proxy.jdbc.ConnectionProxyImpl; +import org.apache.derby.impl.jdbc.EmbedConnection; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration("classpath:com/alibaba/druid/filter/url/dataSource.xml") +public class JdbcUrlHostAndPortReplaceFilterTest { + @Autowired + private DruidDataSource dataSource; + + @Test + public void testConnectWithDifferentPlaceholderValue() throws Exception { + int[] count = new int[] { 0,0 }; + List connectionList = new ArrayList(); + + for (int i = 0; i < 10; i++) { + DruidPooledConnection conn = dataSource.getConnection(); + EmbedConnection embed = ((EmbedConnection)((ConnectionProxyImpl)conn.getConnection()).getRawObject()); + String connName = embed.toString(); + + if (connName.contains("foo")) { + count[0]++; + } else if (connName.contains("bar")) { + count[1]++; + } else { + fail("NOT foo or bar"); + } + connectionList.add(conn); + } + assertTrue(count[0] > 0 && count[1] >0); + for (Connection c : connectionList) { + c.close(); + } + } + +} \ No newline at end of file diff --git a/src/test/resources/com/alibaba/druid/filter/url/dataSource.xml b/src/test/resources/com/alibaba/druid/filter/url/dataSource.xml new file mode 100644 index 0000000000..076ab68e7d --- /dev/null +++ b/src/test/resources/com/alibaba/druid/filter/url/dataSource.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/resources/com/alibaba/druid/filter/url/datasource.properties b/src/test/resources/com/alibaba/druid/filter/url/datasource.properties new file mode 100644 index 0000000000..4e05753ccd --- /dev/null +++ b/src/test/resources/com/alibaba/druid/filter/url/datasource.properties @@ -0,0 +1,4 @@ +datasource.foo=127.0.0.1:3066;192.168.0.1:3066 +datasource.bar=198.168.1.1:3166;192.168.1.2:3166 +datasource.empty +datasource.test=foo;bar \ No newline at end of file