mirror of
https://github.com/OpenHFT/Chronicle-Map.git
synced 2021-09-13 01:26:12 +00:00
212 lines
8.2 KiB
Java
Executable File
212 lines
8.2 KiB
Java
Executable File
/*
|
|
* Copyright (C) 2012, 2016 higherfrequencytrading.com
|
|
* Copyright (C) 2016 Roman Leventov
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Lesser General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
package net.openhft.chronicle.map;
|
|
|
|
import org.junit.Ignore;
|
|
import org.junit.Test;
|
|
|
|
import java.beans.XMLEncoder;
|
|
import java.io.ByteArrayOutputStream;
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.util.*;
|
|
import java.util.concurrent.*;
|
|
|
|
import static org.junit.Assert.*;
|
|
|
|
/**
|
|
* Created by peter.lawrey on 06/12/14.
|
|
*/
|
|
public class LargeEntriesTest {
|
|
|
|
@Test
|
|
public void testLargeStrings() throws ExecutionException, InterruptedException, IOException {
|
|
final int ENTRIES = 250;
|
|
final int ENTRY_SIZE = 100 * 1024;
|
|
|
|
File file = File.createTempFile("largeEntries" + System.currentTimeMillis(), ".deleteme");
|
|
file.deleteOnExit();
|
|
try (final ChronicleMap<String, String> map = ChronicleMapBuilder
|
|
.of(String.class, String.class)
|
|
// .valueReaderAndDataAccess(, SnappyStringMarshaller.INSTANCE, )
|
|
.actualSegments(1) // to force an error.
|
|
.entries(ENTRIES)
|
|
.averageKeySize(10)
|
|
.averageValueSize(ENTRY_SIZE)
|
|
.putReturnsNull(true)
|
|
.createPersistedTo(file)) {
|
|
warmUpCompression(ENTRY_SIZE);
|
|
int threads = 4; //Runtime.getRuntime().availableProcessors();
|
|
ExecutorService es = Executors.newFixedThreadPool(threads);
|
|
final int block = ENTRIES / threads;
|
|
for (int i = 0; i < 3; i++) {
|
|
long start = System.currentTimeMillis();
|
|
List<Future<?>> futureList = new ArrayList<>();
|
|
for (int t = 0; t < threads; t++) {
|
|
final int finalT = t;
|
|
futureList.add(es.submit(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
exerciseLargeStrings(map, finalT * block, finalT * block + block,
|
|
ENTRY_SIZE);
|
|
}
|
|
}));
|
|
}
|
|
for (Future<?> future : futureList) {
|
|
future.get();
|
|
}
|
|
long time = System.currentTimeMillis() - start;
|
|
long operations = 3;
|
|
System.out.printf("Put/Get rate was %.1f MB/s%n",
|
|
operations * ENTRIES * ENTRY_SIZE / 1e6 / (time / 1e3));
|
|
}
|
|
es.shutdown();
|
|
es.awaitTermination(1, TimeUnit.MINUTES);
|
|
assertTrue(es.isTerminated());
|
|
}
|
|
file.delete();
|
|
}
|
|
|
|
private void warmUpCompression(int entrySize) {
|
|
// String value = generateValue(entrySize);
|
|
// DirectBytes bytes = DirectStore.allocate(entrySize / 6).bytes();
|
|
// for (int i = 0; i < 5; i++) {
|
|
// // warmup to compression.
|
|
// bytes.clear();
|
|
// SnappyStringMarshaller.INSTANCE.write(bytes, value);
|
|
// bytes.flip();
|
|
// SnappyStringMarshaller.INSTANCE.read(bytes);
|
|
// }
|
|
// bytes.release();
|
|
}
|
|
|
|
@Test
|
|
@Ignore("Performance Test")
|
|
public void testLargeStringsPerf() throws ExecutionException, InterruptedException, IOException {
|
|
doLargeEntryPerf(10000, 100 * 1024);
|
|
doLargeEntryPerf(1000000, 1024);
|
|
doLargeEntryPerf(100000, 10 * 1024);
|
|
doLargeEntryPerf(10000, 100 * 1024);
|
|
doLargeEntryPerf(3000, 1024 * 1024);
|
|
}
|
|
|
|
private void doLargeEntryPerf(int ENTRIES, final int ENTRY_SIZE) throws IOException, InterruptedException, ExecutionException {
|
|
System.out.printf("Testing %,d entries of %,d KB%n", ENTRIES, ENTRY_SIZE / 1024);
|
|
File file = File.createTempFile("largeEntries", ".deleteme");
|
|
file.deleteOnExit();
|
|
final ChronicleMap<String, String> map = ChronicleMapBuilder
|
|
.of(String.class, String.class)
|
|
// .valueReaderAndDataAccess(, SnappyStringMarshaller.INSTANCE, )
|
|
.entries(ENTRIES)
|
|
.averageKeySize(10)
|
|
.averageValueSize(ENTRY_SIZE)
|
|
.putReturnsNull(true)
|
|
.createPersistedTo(file);
|
|
{
|
|
// warmUpCompression(ENTRY_SIZE);
|
|
int threads = Runtime.getRuntime().availableProcessors();
|
|
ExecutorService es = Executors.newFixedThreadPool(threads);
|
|
final int block = ENTRIES / threads;
|
|
for (int i = 0; i < 3; i++) {
|
|
long start = System.currentTimeMillis();
|
|
List<Future<?>> futureList = new ArrayList<>();
|
|
for (int t = 0; t < threads; t++) {
|
|
final int finalT = t;
|
|
futureList.add(es.submit(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
exerciseLargeStrings(map, finalT * block, finalT * block + block,
|
|
ENTRY_SIZE);
|
|
}
|
|
}));
|
|
}
|
|
for (Future<?> future : futureList) {
|
|
future.get();
|
|
}
|
|
long time = System.currentTimeMillis() - start;
|
|
long operations = 3;
|
|
System.out.printf("Put/Get rate was %.1f MB/s%n", operations * ENTRIES * ENTRY_SIZE / 1e6 / (time / 1e3));
|
|
}
|
|
es.shutdown();
|
|
if (es.isTerminated())
|
|
map.close();
|
|
}
|
|
}
|
|
|
|
void exerciseLargeStrings(ChronicleMap<String, String> map, int start, int finish, int entrySize) {
|
|
/*
|
|
final Thread thisThread = Thread.currentThread();
|
|
Thread monitor = new Thread(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.append(thisThread).append("\n");
|
|
try {
|
|
while (!Thread.currentThread().isInterrupted()) {
|
|
Jvm.pause(50);
|
|
StackTraceElement[] stackTrace = thisThread.getStackTrace();
|
|
for (int i = 0; i < 4 && i < stackTrace.length; i++)
|
|
sb.append("\tat ").append(stackTrace[i]).append("\n");
|
|
sb.append("\n");
|
|
}
|
|
} catch (InterruptedException e) {
|
|
}
|
|
System.out.println(sb);
|
|
}
|
|
});
|
|
monitor.start();
|
|
*/
|
|
String value = generateValue(entrySize);
|
|
|
|
for (int i = start; i < finish; i++) {
|
|
String key = "key-" + i;
|
|
String object;
|
|
|
|
map.put(key, value);
|
|
object = map.get(key);
|
|
assertTrue(key, map.containsKey(key));
|
|
|
|
assertNotNull(key, object);
|
|
assertEquals(key, entrySize, object.length());
|
|
}
|
|
// monitor.interrupt();
|
|
|
|
for (int i = start; i < finish; i++) {
|
|
// System.out.println(i);
|
|
String key = "key-" + i;
|
|
|
|
String object = map.get(key);
|
|
|
|
assertNotNull(key, object);
|
|
assertEquals(key, entrySize, object.length());
|
|
}
|
|
}
|
|
|
|
private String generateValue(int entrySize) {
|
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
XMLEncoder xml = new XMLEncoder(baos);
|
|
Map<String, String> map2 = new HashMap<>();
|
|
Random rand = new Random(1);
|
|
for (int i = 0; i < entrySize / 80; i++)
|
|
map2.put("key-" + i, "value-" + rand.nextInt(1000));
|
|
xml.writeObject(map2);
|
|
xml.close();
|
|
return baos.toString().substring(0, entrySize);
|
|
}
|
|
}
|