HCOLL-321 Remote implementation for FilePerKeyMap Refactoring

This commit is contained in:
danielshaya
2015-05-13 15:17:58 +01:00
parent 59a8909fa6
commit 26db780105
6 changed files with 429 additions and 141 deletions
@@ -0,0 +1,143 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.bytes.IORuntimeException;
import net.openhft.lang.io.ByteBufferBytes;
import net.openhft.lang.io.Bytes;
import net.openhft.lang.io.serialization.CompactBytesMarshaller;
import org.jetbrains.annotations.Nullable;
import java.io.*;
import java.lang.reflect.Constructor;
import java.nio.ByteBuffer;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
/**
* Created by peter on 24/10/14.
*/
public enum DeflatorStringMarshaller implements CompactBytesMarshaller<CharSequence> {
INSTANCE;
private static final StringFactory STRING_FACTORY = getStringFactory();
private static final int NULL_LENGTH = -1;
private static StringFactory getStringFactory() {
try {
return new StringFactory17();
} catch (Exception e) {
// do nothing
}
try {
return new StringFactory16();
} catch (Exception e) {
// no more alternatives
throw new AssertionError(e);
}
}
@Override
public byte code() {
return '_';
}
@Override
public void write(Bytes bytes, CharSequence s) {
if (s == null) {
bytes.writeStopBit(NULL_LENGTH);
return;
} else if (s.length() == 0) {
bytes.writeStopBit(0);
return;
}
// write the total length.
int length = s.length();
bytes.writeStopBit(length);
long position = bytes.position();
bytes.writeInt(0);
DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(
new DeflaterOutputStream(bytes.outputStream())));
try {
for (int i = 0; i < s.length(); i++) {
dos.write(s.charAt(i));
}
dos.close();
} catch (IOException e) {
throw new IORuntimeException(e);
}
bytes.writeInt(position, (int) (bytes.position() - position - 4));
}
@Override
public String read(Bytes bytes) {
return read(bytes, null);
}
@Override
public String read(Bytes bytes, @Nullable CharSequence ignored) {
long size = bytes.readStopBit();
if (size == NULL_LENGTH)
return null;
if (size < 0 || size > Integer.MAX_VALUE)
throw new IllegalStateException("Invalid length: " + size);
if (size == 0)
return "";
char[] chars = new char[(int) size];
int length = bytes.readInt();
long limit = bytes.limit();
try {
bytes.limit(bytes.position() + length);
DataInputStream dis = new DataInputStream(new BufferedInputStream(
new InflaterInputStream(bytes.inputStream())));
for (int i = 0; i < size; i++)
chars[i] = (char) (dis.readByte() & 0xff);
} catch (IOException e) {
throw new IORuntimeException(e);
} finally {
bytes.position(bytes.limit());
bytes.limit(limit);
}
try {
return STRING_FACTORY.fromChars(chars);
} catch (Exception e) {
throw new AssertionError(e);
}
}
private static abstract class StringFactory {
abstract String fromChars(char[] chars) throws Exception;
}
private static final class StringFactory16 extends StringFactory {
private final Constructor<String> constructor;
private StringFactory16() throws Exception {
constructor = String.class.getDeclaredConstructor(int.class,
int.class, char[].class);
constructor.setAccessible(true);
}
@Override
String fromChars(char[] chars) throws Exception {
return constructor.newInstance(0, chars.length, chars);
}
}
private static final class StringFactory17 extends StringFactory {
private final Constructor<String> constructor;
private StringFactory17() throws Exception {
constructor = String.class.getDeclaredConstructor(char[].class, boolean.class);
constructor.setAccessible(true);
}
@Override
String fromChars(char[] chars) throws Exception {
return constructor.newInstance(chars, true);
}
}
}
@@ -4,6 +4,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
@@ -15,7 +16,7 @@ import static org.junit.Assert.*;
public class FilePerKeyMapTest {
private static FilePerKeyMap map;
@BeforeClass
public static void createMap(){
public static void createMap() throws IOException {
map = new FilePerKeyMap("/tmp/filepermaptests");
//just in case it hasn't been cleared up last time
map.clear();
@@ -133,7 +134,7 @@ public class FilePerKeyMapTest {
@Ignore //This doesn't work on all OS
@Test
public void eventTest() {
public void eventTest() throws InterruptedException {
final List<FPMEvent> events = new ArrayList<>();
Consumer<FPMEvent> fpmEventConsumer = (FPMEvent e) -> {
@@ -147,7 +148,6 @@ public class FilePerKeyMapTest {
map.remove("one");
map.put("one", "one");
try {
//Allow all events to be played through
Thread.sleep(3000);
//Check the events
@@ -158,13 +158,6 @@ public class FilePerKeyMapTest {
assertEquals(FPMEvent.EventType.DELETE, events.get(2).getEventType());
assertEquals(FPMEvent.EventType.NEW, events.get(3).getEventType());
assertEquals(true, events.get(0).isProgrammatic());
assertEquals(true, events.get(1).isProgrammatic());
assertEquals(true, events.get(2).isProgrammatic());
assertEquals(true, events.get(3).isProgrammatic());
} catch (InterruptedException e) {
e.printStackTrace();
}
map.unregisterForEvents(fpmEventConsumer);
map.clear();
@@ -0,0 +1,106 @@
package net.openhft.chronicle.map;
import net.openhft.lang.io.serialization.impl.SnappyStringMarshaller;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.stream.IntStream;
/**
* Created by peter on 12/05/15.
*/
public class WriteThroughputTest {
@Ignore("Long running")
@Test
public void bandwidthTest() throws IOException {
int count = 2000;
int size = 50 << 10;
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
.of(CharSequence.class, CharSequence.class)
.entries(count * 3 / 2)
.averageValueSize(size)
.putReturnsNull(true)
.create()) {
System.out.println("ChronicleMap.put to memory");
doTest(count, size, map);
}
File file = new File("bandwidthTest" + System.nanoTime() + ".deleteme");
file.deleteOnExit();
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
.of(CharSequence.class, CharSequence.class)
.entries(count * 3 / 2)
.averageValueSize(size)
.putReturnsNull(true)
.createPersistedTo(file)) {
System.out.println("ChronicleMap.put to disk");
doTest(count, size, map);
}
File file2 = new File("bandwidthTest" + System.nanoTime() + ".snappy.deleteme");
file2.deleteOnExit();
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
.of(CharSequence.class, CharSequence.class)
.entries(count * 3 / 2)
.averageValueSize(size / 4)
.valueMarshaller(SnappyStringMarshaller.INSTANCE)
.putReturnsNull(true)
.createPersistedTo(file2)) {
System.out.println("ChronicleMap.put to snappy disk");
doTest(count, size, map);
}
}
@Ignore("TODO")
@Test
public void bandwidthTestZ() throws IOException {
int count = 2 << 10;
int size = 50 << 10;
File file3 = new File("bandwidthTest"+System.nanoTime()+ ".Z.deleteme");
file3.deleteOnExit();
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
.of(CharSequence.class, CharSequence.class)
.entries(count*3/2)
.averageValueSize(size/4)
.valueMarshaller(DeflatorStringMarshaller.INSTANCE)
.putReturnsNull(true)
.createPersistedTo(file3)) {
System.out.println("ChronicleMap.put to deflator disk");
doTest(count, size, map);
}
}
private void doTest(int count, int size,ChronicleMap<CharSequence, CharSequence> map) {
StringBuilder value = new StringBuilder("value");
while (value.length() < size)
value.append(value.length());
for (int t = 0; t < 1; t++) {
StringBuilder key = new StringBuilder("key");
long start = System.nanoTime();
for (int i = 0; i < count; i++)
map.put(key, value);
long time = System.nanoTime() - start;
System.out.printf("Serial %,d MB took %.3f seconds%n", size * count >> 20, time / 1e9);
}
for (int t = 0; t < 3; t++) {
long start = System.nanoTime();
IntStream.range(0, count).parallel().forEach(i -> {
StringBuilder key = new StringBuilder("key");
key.append(i);
map.put(key, value);
});
long time = System.nanoTime() - start;
System.out.printf("Concurrent %,d MB took %.3f seconds%n", size * count >> 20, time / 1e9);
}
}
}