#171 Redis,还可以继续改进,比如在测试服务器上运行时支持在命令行用-D定义系统变量改变测试参数

This commit is contained in:
Calvin
2013-04-08 23:30:15 +08:00
parent a4297ce64c
commit 8d3e39abce
13 changed files with 147 additions and 99 deletions
+22
View File
@@ -107,6 +107,10 @@
<!-- PERSISTENCE end -->
<!-- WEB begin -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
@@ -184,6 +188,24 @@
<artifactId>jedis</artifactId>
<version>2.1.0</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-mongodb</artifactId>
<version>1.1.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>2.11.0</version>
<exclusions>
<exclusion>
<groupId>javax.persistence</groupId>
<artifactId>persistence-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- CACHE begin -->
<dependency>
@@ -5,41 +5,42 @@ import org.springside.modules.test.benchmark.ConcurrentBenchmark;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Protocol;
/**
* 测试Redis用于计数器时incr()方法的性能.
*
* 可用-Dbenchmark.thread.count,-Dbenchmark.loop.count 重置测试规模
* 可用-Dbenchmark.host,-Dbenchmark.port,-Dbenchmark.timeout 重置连接参数
*
* @author calvin
*/
public class RedisCounterBenchmark extends ConcurrentBenchmark {
private static final int THREAD_COUNT = 50;
private static final long LOOP_COUNT = 20000;
private static final int DEFAULT_THREAD_COUNT = 50;
private static final long DEFAULT_LOOP_COUNT = 20000;
private static final int PRINT_BETWEEN_SECONDS = 10;
private static final String HOST = "localhost";
private static final int PORT = Protocol.DEFAULT_PORT;
private static final int TIMEOUT = Protocol.DEFAULT_TIMEOUT;
private static final String DEFAULT_HOST = "localhost";
private static final int DEFAULT_PORT = Protocol.DEFAULT_PORT;
private static final int DEFAULT_TIMEOUT = Protocol.DEFAULT_TIMEOUT;
private String counterName = "ss.counter";
private JedisPool pool;
public static void main(String[] args) throws Exception {
RedisCounterBenchmark benchmark = new RedisCounterBenchmark(THREAD_COUNT, LOOP_COUNT);
benchmark.run();
RedisCounterBenchmark benchmark = new RedisCounterBenchmark(DEFAULT_THREAD_COUNT, DEFAULT_LOOP_COUNT,
PRINT_BETWEEN_SECONDS);
benchmark.execute();
}
public RedisCounterBenchmark(int threadCount, long loopCount) {
super(threadCount, loopCount);
public RedisCounterBenchmark(int defaultThreadCount, long defaultLoopCount, int printBetweenSeconds) {
super(defaultThreadCount, defaultLoopCount, printBetweenSeconds);
}
@Override
protected void setUp() {
//create jedis pool
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxActive(THREAD_COUNT);
pool = new JedisPool(poolConfig, HOST, PORT, TIMEOUT);
pool = Utils.createJedisPool(DEFAULT_HOST, DEFAULT_PORT, DEFAULT_TIMEOUT, threadCount);
//reset counter
Jedis jedis = pool.getResource();
@@ -56,14 +57,14 @@ public class RedisCounterBenchmark extends ConcurrentBenchmark {
}
@Override
protected BenchmarkTask createTask(int index) {
return new CounterTask(index, this, PRINT_BETWEEN_SECONDS);
protected BenchmarkTask createTask(int taskSequence) {
return new CounterTask(taskSequence, this);
}
public class CounterTask extends BenchmarkTask {
public CounterTask(int index, ConcurrentBenchmark parent, int printBetweenSeconds) {
super(index, parent, printBetweenSeconds);
public CounterTask(int taskSequence, ConcurrentBenchmark parent) {
super(taskSequence, parent);
}
@Override
@@ -6,44 +6,45 @@ import org.springside.modules.test.benchmark.ConcurrentBenchmark;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Pipeline;
import redis.clients.jedis.Protocol;
/**
* 测试Redis批量插入时的性能, 使用PipeLine加速.
* 测试Redis批量插入时的性能, 使用JSON格式存储数据, 使用PipeLine加速.
*
* 可用系统参数重置相关变量,@see RedisCounterBenchmark
*
* @author calvin
*/
public class RedisMassInsertionBenchmark extends ConcurrentBenchmark {
private static final int THREAD_COUNT = 50;
private static final long LOOP_COUNT = 100000;
private static final int DEFAULT_THREAD_COUNT = 50;
private static final long DEFAULT_LOOP_COUNT = 100000;
private static final int PRINT_BETWEEN_SECONDS = 10;
private static final int BATCH_SIZE = 10;
private static final String HOST = "localhost";
private static final int PORT = Protocol.DEFAULT_PORT;
private static final int TIMEOUT = 40000;
private static final String DEFAULT_HOST = "localhost";
private static final int DEFAULT_PORT = Protocol.DEFAULT_PORT;
private static final int DEFAULT_TIMEOUT = Protocol.DEFAULT_TIMEOUT;
private String keyPrefix = "ss.map:";
private int batchSize = 10;
private JsonMapper jsonMapper = new JsonMapper();
private JedisPool pool;
public static void main(String[] args) throws Exception {
RedisMassInsertionBenchmark benchmark = new RedisMassInsertionBenchmark(THREAD_COUNT, LOOP_COUNT);
benchmark.run();
RedisMassInsertionBenchmark benchmark = new RedisMassInsertionBenchmark(DEFAULT_THREAD_COUNT,
DEFAULT_LOOP_COUNT, PRINT_BETWEEN_SECONDS);
benchmark.execute();
}
public RedisMassInsertionBenchmark(int threadCount, long loopCount) {
super(threadCount, loopCount);
public RedisMassInsertionBenchmark(int defaultThreadCount, long defaultLoopCount, int printBetweenSeconds) {
super(defaultThreadCount, defaultLoopCount, printBetweenSeconds);
}
@Override
protected void setUp() {
//create jedis pool
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxActive(THREAD_COUNT);
pool = new JedisPool(poolConfig, HOST, PORT, TIMEOUT);
pool = Utils.createJedisPool(DEFAULT_HOST, DEFAULT_PORT, DEFAULT_TIMEOUT, threadCount);
//remove all keys
Jedis jedis = pool.getResource();
@@ -60,14 +61,14 @@ public class RedisMassInsertionBenchmark extends ConcurrentBenchmark {
}
@Override
protected BenchmarkTask createTask(int index) {
return new MassInsertionTask(index, this, PRINT_BETWEEN_SECONDS);
protected BenchmarkTask createTask(int taskSequence) {
return new MassInsertionTask(taskSequence, this);
}
public class MassInsertionTask extends BenchmarkTask {
public MassInsertionTask(int index, ConcurrentBenchmark parent, int printBetweenSeconds) {
super(index, parent, printBetweenSeconds);
public MassInsertionTask(int taskSequence, ConcurrentBenchmark parent) {
super(taskSequence, parent);
}
@Override
@@ -89,7 +90,7 @@ public class RedisMassInsertionBenchmark extends ConcurrentBenchmark {
pl.set(session.getId(), jsonMapper.toJson(session));
if (i % BATCH_SIZE == 0) {
if (i % batchSize == 0) {
pl.sync();
printProgressMessage(i);
}
@@ -8,42 +8,42 @@ import org.springside.modules.test.benchmark.ConcurrentBenchmark;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Protocol;
/**
* 测试Redis用于Session管理的setEx()与get()方法性能, 使用JSON格式存储数据.
*
* 可用系统参数重置相关变量,@see RedisCounterBenchmark
*
* @author calvin
*/
public class RedisSessionBenchmark extends ConcurrentBenchmark {
private static final int THREAD_COUNT = 50;
private static final long LOOP_COUNT = 20000;
private static final int DEFAULT_THREAD_COUNT = 50;
private static final long DEFAULT_LOOP_COUNT = 20000;
private static final int PRINT_BETWEEN_SECONDS = 10;
private static final String HOST = "localhost";
private static final int PORT = Protocol.DEFAULT_PORT;
private static final int TIMEOUT = Protocol.DEFAULT_TIMEOUT;
private static final String DEFAULT_HOST = "localhost";
private static final int DEFAULT_PORT = Protocol.DEFAULT_PORT;
private static final int DEFAULT_TIMEOUT = Protocol.DEFAULT_TIMEOUT;
private String keyPrefix = "ss.session:";
private JsonMapper jsonMapper = new JsonMapper();
private JedisPool pool;
public static void main(String[] args) throws Exception {
RedisSessionBenchmark benchmark = new RedisSessionBenchmark(THREAD_COUNT, LOOP_COUNT);
benchmark.run();
RedisSessionBenchmark benchmark = new RedisSessionBenchmark(DEFAULT_THREAD_COUNT, DEFAULT_LOOP_COUNT,
PRINT_BETWEEN_SECONDS);
benchmark.execute();
}
public RedisSessionBenchmark(int threadCount, long loopCount) {
super(threadCount, loopCount);
public RedisSessionBenchmark(int defaultThreadCount, long defaultLoopCount, int printBetweenSeconds) {
super(defaultThreadCount, defaultLoopCount, printBetweenSeconds);
}
@Override
protected void setUp() {
//create jedis pool
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxActive(THREAD_COUNT);
pool = new JedisPool(poolConfig, HOST, PORT, TIMEOUT);
pool = Utils.createJedisPool(DEFAULT_HOST, DEFAULT_PORT, DEFAULT_TIMEOUT, threadCount);
//remove all keys
Jedis jedis = pool.getResource();
@@ -60,15 +60,15 @@ public class RedisSessionBenchmark extends ConcurrentBenchmark {
}
@Override
protected BenchmarkTask createTask(int index) {
return new SessionTask(index, this, PRINT_BETWEEN_SECONDS);
protected BenchmarkTask createTask(int taskSequence) {
return new SessionTask(taskSequence, this);
}
public class SessionTask extends BenchmarkTask {
private SecureRandom random = new SecureRandom();
public SessionTask(int index, ConcurrentBenchmark parent, int printBetweenSeconds) {
super(index, parent, printBetweenSeconds);
public SessionTask(int taskSequence, ConcurrentBenchmark parent) {
super(taskSequence, parent);
}
@Override
@@ -0,0 +1,19 @@
package org.springside.examples.showcase.demos.redis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
public class Utils {
public static JedisPool createJedisPool(String defaultHost, int defaultPort, int defaultTimeout, int threadCount) {
//merge default setting and system properties
String host = System.getProperty("benchmark.host", defaultHost);
int port = new Integer(System.getProperty("benchmark.port", String.valueOf(defaultPort)));
int timeout = new Integer(System.getProperty("benchmark.timeout", String.valueOf(defaultPort)));
//create jedis pool
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxActive(threadCount);
return new JedisPool(poolConfig, host, port, timeout);
}
}
@@ -62,7 +62,7 @@ public class JobConsumer implements Runnable {
public static void setUp() {
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxActive(THREAD_COUNT);
pool = new JedisPool(poolConfig, JobManager.HOST, JobManager.PORT, JobManager.TIMEOUT);
pool = new JedisPool(poolConfig, JobManager.DEFAULT_HOST, JobManager.DEFAULT_PORT, JobManager.DEFAULT_TIMEOUT);
}
public static void tearDown() {
@@ -24,9 +24,9 @@ public class JobManager implements Runnable {
public static final int EXPECT_TPS = 2500;
public static final int DELAY_SECONDS = 10;
public static final String HOST = "localhost";
public static final int PORT = Protocol.DEFAULT_PORT;
public static final int TIMEOUT = 5000;
public static final String DEFAULT_HOST = "localhost";
public static final int DEFAULT_PORT = Protocol.DEFAULT_PORT;
public static final int DEFAULT_TIMEOUT = 5000;
private static final int PRINT_BETWEEN_SECONDS = 20;
@@ -60,7 +60,7 @@ public class JobManager implements Runnable {
}
public void setUp() {
jedis = new Jedis(HOST, PORT, TIMEOUT);
jedis = new Jedis(DEFAULT_HOST, DEFAULT_PORT, DEFAULT_TIMEOUT);
String script = "local jobWithScores=redis.call('zrangebyscore', KEYS[1], '-inf', ARGV[1], 'withscores')\n";
script += " redis.call('zremrangebyscore', KEYS[1], '-inf', ARGV[1])\n";
script += " for i=1,ARGV[2] do \n";
@@ -2,12 +2,12 @@ package org.springside.examples.showcase.demos.redis.job;
import java.util.concurrent.atomic.AtomicLong;
import org.springside.examples.showcase.demos.redis.Utils;
import org.springside.modules.test.benchmark.BenchmarkTask;
import org.springside.modules.test.benchmark.ConcurrentBenchmark;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
/**
* 将Job放入ss.job:schedule(sorted set).
@@ -15,30 +15,29 @@ import redis.clients.jedis.JedisPoolConfig;
* @author calvin
*/
public class JobProducer extends ConcurrentBenchmark {
private static final int THREAD_COUNT = 5;
private static final long LOOP_COUNT = 5000 * 60;
private static final int DEFAULT_THREAD_COUNT = 5;
private static final long DEFAULT_LOOP_COUNT = 5000 * 60;
private static final int PRINT_BETWEEN_SECONDS = 10;
private static AtomicLong idGenerator = new AtomicLong(0);
private static long expireTime;
private JedisPool pool;
public static void main(String[] args) throws Exception {
JobProducer benchmark = new JobProducer(THREAD_COUNT, LOOP_COUNT);
benchmark.run();
JobProducer benchmark = new JobProducer(DEFAULT_THREAD_COUNT, DEFAULT_LOOP_COUNT, PRINT_BETWEEN_SECONDS);
benchmark.execute();
}
public JobProducer(int threadCount, long loopCount) {
super(threadCount, loopCount);
public JobProducer(int defaultThreadCount, long defaultLoopCount, int printBetweenSeconds) {
super(defaultThreadCount, defaultLoopCount, printBetweenSeconds);
}
@Override
protected void setUp() {
//create jedis pool
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxActive(THREAD_COUNT);
pool = new JedisPool(poolConfig, JobManager.HOST, JobManager.PORT, JobManager.TIMEOUT);
pool = Utils.createJedisPool(JobManager.DEFAULT_HOST, JobManager.DEFAULT_PORT, JobManager.DEFAULT_TIMEOUT,
threadCount);
expireTime = System.currentTimeMillis() + JobManager.DELAY_SECONDS * 1000;
}
@@ -48,14 +47,14 @@ public class JobProducer extends ConcurrentBenchmark {
}
@Override
protected BenchmarkTask createTask(int index) {
return new JobProducerTask(index, this, PRINT_BETWEEN_SECONDS);
protected BenchmarkTask createTask(int taskSequence) {
return new JobProducerTask(taskSequence, this);
}
public class JobProducerTask extends BenchmarkTask {
public JobProducerTask(int index, ConcurrentBenchmark parent, int printBetweenSeconds) {
super(index, parent, printBetweenSeconds);
public JobProducerTask(int taskSequence, ConcurrentBenchmark parent) {
super(taskSequence, parent);
}
@Override
+5
View File
@@ -49,6 +49,11 @@
<!-- Security end -->
<!-- Web begin -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
@@ -20,7 +20,6 @@ import javax.xml.bind.annotation.XmlAnyElement;
import javax.xml.namespace.QName;
import org.apache.commons.lang3.StringUtils;
import org.springframework.http.converter.HttpMessageConversionException;
import org.springframework.util.Assert;
import org.springside.modules.utils.Exceptions;
import org.springside.modules.utils.Reflections;
@@ -148,8 +147,8 @@ public class JaxbMapper {
jaxbContext = JAXBContext.newInstance(clazz, CollectionWrapper.class);
jaxbContexts.putIfAbsent(clazz, jaxbContext);
} catch (JAXBException ex) {
throw new HttpMessageConversionException("Could not instantiate JAXBContext for class [" + clazz
+ "]: " + ex.getMessage(), ex);
throw new RuntimeException("Could not instantiate JAXBContext for class [" + clazz + "]: "
+ ex.getMessage(), ex);
}
}
return jaxbContext;
@@ -12,9 +12,6 @@ import org.hibernate.dialect.MySQL5InnoDBDialect;
import org.hibernate.dialect.Oracle10gDialect;
public class Hibernates {
public static final String DATETIME_TYPE = "org.jadira.usertype.dateandtime.joda.PersistentDateTime";
/**
* Initialize the lazy property value.
*
@@ -12,20 +12,18 @@ public abstract class BenchmarkTask implements Runnable {
protected int taskSequence;
protected ConcurrentBenchmark parent;
protected int printBetweenMills;
protected long threadStartTime;
protected long previousPrintTime;
protected long previousLoop = 0L;
public BenchmarkTask(int taskSequence, ConcurrentBenchmark parent, int printBetweenSeconds) {
public BenchmarkTask(int taskSequence, ConcurrentBenchmark parent) {
this.taskSequence = taskSequence;
this.parent = parent;
this.printBetweenMills = printBetweenSeconds * 1000;
}
/**
* Must be invoked when each thread finish the preparation.
* Must be invoked in children class when each thread finish the preparation.
*/
protected void onThreadStart() {
parent.startLock.countDown();
@@ -35,12 +33,11 @@ public abstract class BenchmarkTask implements Runnable {
} catch (InterruptedException e) {
e.printStackTrace();
}
threadStartTime = System.currentTimeMillis();
previousPrintTime = threadStartTime;
previousPrintTime = parent.startTime.getTime();
}
/**
* Must be invoked when the loop in thread finished.
* Must be invoked in children class when the loop in thread finish.
*/
protected void onThreadFinish() {
// notify test finish
@@ -56,14 +53,14 @@ public abstract class BenchmarkTask implements Runnable {
protected void printProgressMessage(int currentLoop) {
long currentTime = System.currentTimeMillis();
if ((currentTime - previousPrintTime) > printBetweenMills) {
if ((currentTime - previousPrintTime) > parent.printBetweenMills) {
long lastTimeInMills = currentTime - previousPrintTime;
previousPrintTime = currentTime;
long totalRequest = currentLoop + 1;
long lastRequests = totalRequest - previousLoop;
long totalTimeInMills = currentTime - threadStartTime;
long totalTimeInMills = currentTime - parent.startTime.getTime();
long totalTimeInSeconds = TimeUnit.MILLISECONDS.toSeconds(totalTimeInMills);
long totalTps = totalRequest * 1000 / totalTimeInMills;
@@ -87,7 +84,7 @@ public abstract class BenchmarkTask implements Runnable {
* 打印线程结果信息.
*/
protected void printThreadFinishMessage() {
long totalTimeInMills = System.currentTimeMillis() - threadStartTime;
long totalTimeInMills = System.currentTimeMillis() - parent.startTime.getTime();
long totalRequest = parent.loopCount;
long totalTps = totalRequest * 1000 / totalTimeInMills;
BigDecimal totalLatency = new BigDecimal(totalTimeInMills).divide(new BigDecimal(totalRequest), 2,
@@ -13,6 +13,9 @@ import java.util.concurrent.Executors;
*/
public abstract class ConcurrentBenchmark {
public static final String THREAD_COUNT_NAME = "benchmark.thread.count";
public static final String LOOP_COUNT_NAME = "benchmark.loop.count";
protected int threadCount;
protected long loopCount;
@@ -20,24 +23,29 @@ public abstract class ConcurrentBenchmark {
protected CountDownLatch finishLock;
protected Date startTime;
protected int printBetweenMills;
public ConcurrentBenchmark(int threadCount, long loopCount) {
this.threadCount = threadCount;
this.loopCount = loopCount;
public ConcurrentBenchmark(int defaultThreadCount, long defaultLoopCount, int printBetweenSeconds) {
//merge default setting and system properties
this.threadCount = Integer.parseInt(System.getProperty(THREAD_COUNT_NAME, String.valueOf(defaultThreadCount)));
this.loopCount = Long.parseLong(System.getProperty(LOOP_COUNT_NAME, String.valueOf(defaultLoopCount)));
startLock = new CountDownLatch(threadCount);
finishLock = new CountDownLatch(threadCount);
this.printBetweenMills = printBetweenSeconds * 1000;
}
public void run() throws Exception {
//override for data prepare
public void execute() throws Exception {
//override for connection & data setup
setUp();
//start threads
ExecutorService threadPool = Executors.newFixedThreadPool(threadCount);
try {
for (int i = 0; i < threadCount; i++) {
threadPool.execute(createTask(i));
BenchmarkTask task = createTask(i);
threadPool.execute(task);
}
//wait for all threads ready
@@ -54,7 +62,7 @@ public abstract class ConcurrentBenchmark {
printFinishMessage();
} finally {
threadPool.shutdownNow();
//override for data cleanup
//override for connection & data cleanup
tearDown();
}
}