mirror of
https://github.com/OpenHFT/Chronicle-Map.git
synced 2021-09-13 01:26:12 +00:00
HCOLL-321 Remote implementation for FilePerKeyMap Refactoring
This commit is contained in:
@@ -0,0 +1,143 @@
|
||||
package net.openhft.chronicle.map;
|
||||
|
||||
import net.openhft.chronicle.bytes.IORuntimeException;
|
||||
import net.openhft.lang.io.ByteBufferBytes;
|
||||
import net.openhft.lang.io.Bytes;
|
||||
import net.openhft.lang.io.serialization.CompactBytesMarshaller;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.io.*;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.zip.DeflaterOutputStream;
|
||||
import java.util.zip.InflaterInputStream;
|
||||
|
||||
/**
|
||||
* Created by peter on 24/10/14.
|
||||
*/
|
||||
public enum DeflatorStringMarshaller implements CompactBytesMarshaller<CharSequence> {
|
||||
INSTANCE;
|
||||
|
||||
private static final StringFactory STRING_FACTORY = getStringFactory();
|
||||
|
||||
private static final int NULL_LENGTH = -1;
|
||||
|
||||
private static StringFactory getStringFactory() {
|
||||
try {
|
||||
return new StringFactory17();
|
||||
} catch (Exception e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
try {
|
||||
return new StringFactory16();
|
||||
} catch (Exception e) {
|
||||
// no more alternatives
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte code() {
|
||||
return '_';
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(Bytes bytes, CharSequence s) {
|
||||
if (s == null) {
|
||||
bytes.writeStopBit(NULL_LENGTH);
|
||||
return;
|
||||
} else if (s.length() == 0) {
|
||||
bytes.writeStopBit(0);
|
||||
return;
|
||||
}
|
||||
// write the total length.
|
||||
int length = s.length();
|
||||
bytes.writeStopBit(length);
|
||||
|
||||
long position = bytes.position();
|
||||
bytes.writeInt(0);
|
||||
DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(
|
||||
new DeflaterOutputStream(bytes.outputStream())));
|
||||
try {
|
||||
for (int i = 0; i < s.length(); i++) {
|
||||
dos.write(s.charAt(i));
|
||||
}
|
||||
dos.close();
|
||||
} catch (IOException e) {
|
||||
throw new IORuntimeException(e);
|
||||
}
|
||||
bytes.writeInt(position, (int) (bytes.position() - position - 4));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String read(Bytes bytes) {
|
||||
return read(bytes, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String read(Bytes bytes, @Nullable CharSequence ignored) {
|
||||
long size = bytes.readStopBit();
|
||||
if (size == NULL_LENGTH)
|
||||
return null;
|
||||
if (size < 0 || size > Integer.MAX_VALUE)
|
||||
throw new IllegalStateException("Invalid length: " + size);
|
||||
if (size == 0)
|
||||
return "";
|
||||
|
||||
char[] chars = new char[(int) size];
|
||||
int length = bytes.readInt();
|
||||
long limit = bytes.limit();
|
||||
try {
|
||||
bytes.limit(bytes.position() + length);
|
||||
DataInputStream dis = new DataInputStream(new BufferedInputStream(
|
||||
new InflaterInputStream(bytes.inputStream())));
|
||||
for (int i = 0; i < size; i++)
|
||||
chars[i] = (char) (dis.readByte() & 0xff);
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new IORuntimeException(e);
|
||||
} finally {
|
||||
bytes.position(bytes.limit());
|
||||
bytes.limit(limit);
|
||||
}
|
||||
try {
|
||||
return STRING_FACTORY.fromChars(chars);
|
||||
} catch (Exception e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static abstract class StringFactory {
|
||||
abstract String fromChars(char[] chars) throws Exception;
|
||||
}
|
||||
|
||||
private static final class StringFactory16 extends StringFactory {
|
||||
private final Constructor<String> constructor;
|
||||
|
||||
private StringFactory16() throws Exception {
|
||||
constructor = String.class.getDeclaredConstructor(int.class,
|
||||
int.class, char[].class);
|
||||
constructor.setAccessible(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
String fromChars(char[] chars) throws Exception {
|
||||
return constructor.newInstance(0, chars.length, chars);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class StringFactory17 extends StringFactory {
|
||||
private final Constructor<String> constructor;
|
||||
|
||||
private StringFactory17() throws Exception {
|
||||
constructor = String.class.getDeclaredConstructor(char[].class, boolean.class);
|
||||
constructor.setAccessible(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
String fromChars(char[] chars) throws Exception {
|
||||
return constructor.newInstance(chars, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
@@ -15,7 +16,7 @@ import static org.junit.Assert.*;
|
||||
public class FilePerKeyMapTest {
|
||||
private static FilePerKeyMap map;
|
||||
@BeforeClass
|
||||
public static void createMap(){
|
||||
public static void createMap() throws IOException {
|
||||
map = new FilePerKeyMap("/tmp/filepermaptests");
|
||||
//just in case it hasn't been cleared up last time
|
||||
map.clear();
|
||||
@@ -133,7 +134,7 @@ public class FilePerKeyMapTest {
|
||||
|
||||
@Ignore //This doesn't work on all OS
|
||||
@Test
|
||||
public void eventTest() {
|
||||
public void eventTest() throws InterruptedException {
|
||||
final List<FPMEvent> events = new ArrayList<>();
|
||||
|
||||
Consumer<FPMEvent> fpmEventConsumer = (FPMEvent e) -> {
|
||||
@@ -147,7 +148,6 @@ public class FilePerKeyMapTest {
|
||||
map.remove("one");
|
||||
map.put("one", "one");
|
||||
|
||||
try {
|
||||
//Allow all events to be played through
|
||||
Thread.sleep(3000);
|
||||
//Check the events
|
||||
@@ -158,13 +158,6 @@ public class FilePerKeyMapTest {
|
||||
assertEquals(FPMEvent.EventType.DELETE, events.get(2).getEventType());
|
||||
assertEquals(FPMEvent.EventType.NEW, events.get(3).getEventType());
|
||||
|
||||
assertEquals(true, events.get(0).isProgrammatic());
|
||||
assertEquals(true, events.get(1).isProgrammatic());
|
||||
assertEquals(true, events.get(2).isProgrammatic());
|
||||
assertEquals(true, events.get(3).isProgrammatic());
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
map.unregisterForEvents(fpmEventConsumer);
|
||||
map.clear();
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package net.openhft.chronicle.map;
|
||||
|
||||
import net.openhft.lang.io.serialization.impl.SnappyStringMarshaller;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
/**
|
||||
* Created by peter on 12/05/15.
|
||||
*/
|
||||
public class WriteThroughputTest {
|
||||
@Ignore("Long running")
|
||||
@Test
|
||||
public void bandwidthTest() throws IOException {
|
||||
int count = 2000;
|
||||
int size = 50 << 10;
|
||||
|
||||
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
|
||||
.of(CharSequence.class, CharSequence.class)
|
||||
.entries(count * 3 / 2)
|
||||
.averageValueSize(size)
|
||||
.putReturnsNull(true)
|
||||
.create()) {
|
||||
|
||||
System.out.println("ChronicleMap.put to memory");
|
||||
doTest(count, size, map);
|
||||
}
|
||||
|
||||
File file = new File("bandwidthTest" + System.nanoTime() + ".deleteme");
|
||||
file.deleteOnExit();
|
||||
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
|
||||
.of(CharSequence.class, CharSequence.class)
|
||||
.entries(count * 3 / 2)
|
||||
.averageValueSize(size)
|
||||
.putReturnsNull(true)
|
||||
.createPersistedTo(file)) {
|
||||
|
||||
System.out.println("ChronicleMap.put to disk");
|
||||
doTest(count, size, map);
|
||||
}
|
||||
|
||||
File file2 = new File("bandwidthTest" + System.nanoTime() + ".snappy.deleteme");
|
||||
file2.deleteOnExit();
|
||||
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
|
||||
.of(CharSequence.class, CharSequence.class)
|
||||
.entries(count * 3 / 2)
|
||||
.averageValueSize(size / 4)
|
||||
.valueMarshaller(SnappyStringMarshaller.INSTANCE)
|
||||
.putReturnsNull(true)
|
||||
.createPersistedTo(file2)) {
|
||||
|
||||
System.out.println("ChronicleMap.put to snappy disk");
|
||||
doTest(count, size, map);
|
||||
}
|
||||
}
|
||||
@Ignore("TODO")
|
||||
@Test
|
||||
public void bandwidthTestZ() throws IOException {
|
||||
int count = 2 << 10;
|
||||
int size = 50 << 10;
|
||||
|
||||
File file3 = new File("bandwidthTest"+System.nanoTime()+ ".Z.deleteme");
|
||||
file3.deleteOnExit();
|
||||
try (ChronicleMap<CharSequence, CharSequence> map = ChronicleMapBuilder
|
||||
.of(CharSequence.class, CharSequence.class)
|
||||
.entries(count*3/2)
|
||||
.averageValueSize(size/4)
|
||||
.valueMarshaller(DeflatorStringMarshaller.INSTANCE)
|
||||
.putReturnsNull(true)
|
||||
.createPersistedTo(file3)) {
|
||||
|
||||
System.out.println("ChronicleMap.put to deflator disk");
|
||||
doTest(count, size, map);
|
||||
}
|
||||
}
|
||||
|
||||
private void doTest(int count, int size,ChronicleMap<CharSequence, CharSequence> map) {
|
||||
StringBuilder value = new StringBuilder("value");
|
||||
while (value.length() < size)
|
||||
value.append(value.length());
|
||||
|
||||
for (int t = 0; t < 1; t++) {
|
||||
StringBuilder key = new StringBuilder("key");
|
||||
long start = System.nanoTime();
|
||||
for (int i = 0; i < count; i++)
|
||||
map.put(key, value);
|
||||
long time = System.nanoTime() - start;
|
||||
System.out.printf("Serial %,d MB took %.3f seconds%n", size * count >> 20, time / 1e9);
|
||||
}
|
||||
|
||||
for (int t = 0; t < 3; t++) {
|
||||
long start = System.nanoTime();
|
||||
IntStream.range(0, count).parallel().forEach(i -> {
|
||||
StringBuilder key = new StringBuilder("key");
|
||||
key.append(i);
|
||||
map.put(key, value);
|
||||
});
|
||||
long time = System.nanoTime() - start;
|
||||
System.out.printf("Concurrent %,d MB took %.3f seconds%n", size * count >> 20, time / 1e9);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user