From 174b11a0752fa73211f7f9589564e149b9cac683 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 10:28:17 +0100 Subject: [PATCH 01/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ef414560..fbcf3dba8 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ product suite. In addition, The original Chronicle has been renamed to Chronicle ## Getting Started #### Tutorial 1 - Creating an instance of Chronicle Map -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](http://vimeo.com/10893210) ### Simple Construction From 4efa5e2df860925828b575716415c67031d88555 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 10:30:04 +0100 Subject: [PATCH 02/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbcf3dba8..1a62e8954 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ product suite. In addition, The original Chronicle has been renamed to Chronicle ## Getting Started #### Tutorial 1 - Creating an instance of Chronicle Map -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](http://vimeo.com/10893210) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://vimeo.com/108932100) ### Simple Construction From d7de8d773c34e5a217f1099f9db7ef658565ec16 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 10:50:53 +0100 Subject: [PATCH 03/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a62e8954..4f30fa7ca 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ product suite. In addition, The original Chronicle has been renamed to Chronicle ## Getting Started #### Tutorial 1 - Creating an instance of Chronicle Map -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://vimeo.com/108932100) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](http://openhft.net/chronicle-map-video-tutorial-1/) ### Simple Construction From 5c3efb2000dfcef9cb683c298bf1d8b29a2df8ee Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 12:47:35 +0100 Subject: [PATCH 04/30] HCOLL-167 Created a stateless map - added read and write client tests --- .../chronicle/map/StatelessMapClientTest.java | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java new file mode 100644 index 000000000..e03a2d14f --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -0,0 +1,120 @@ +package net.openhft.chronicle.map; + +import junit.framework.TestCase; +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.model.constraints.NotNull; +import org.junit.Assert; +import org.junit.Test; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.Collections; + +public class StatelessMapClientTest extends TestCase { + + + @Test + public void testReadValueWriteIntegerValue() throws Exception { + testReadValueWriteValue("Test"); + testReadValueWriteValue(1); + testReadValueWriteValue(1L); + testReadValueWriteValue(1.0); + testReadValueWriteValue(1.0f); + testReadValueWriteValue(Collections.singleton("Test")); + testReadValueWriteValue(Collections.EMPTY_MAP); + testReadValueWriteValue(new MyTestClass(3)); + testReadValueWriteValue(new MyTestClassMarshallable(3)); + + } + + public void testReadValueWriteValue(Object value) throws Exception { + + Class valueClass = value.getClass(); + + final ByteBufferBytes buffer = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + ByteBufferBytes slice = buffer.slice(); + + ChronicleMapBuilder builder = ChronicleMapBuilder.of(Integer.class, valueClass); + builder.preMapConstruction(); + + StatelessMapClient statelessMap = new StatelessMapClient(buffer, builder); + + statelessMap.writeValue(value); + long position = buffer.position(); + slice.limit(position); + + Object actual = statelessMap.readValue(slice); + Assert.assertEquals(actual, value); + } + + + public static class MyTestClass implements Serializable { + int a; + + MyTestClass(int a) { + this.a = a; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClass myTestClass = (MyTestClass) o; + + if (a != myTestClass.a) return false; + + return true; + } + + @Override + public int hashCode() { + return a; + } + } + + public static class MyTestClassMarshallable implements BytesMarshallable { + int a; + + public MyTestClassMarshallable() { + } + + public MyTestClassMarshallable(int a) { + this.a = a; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClassMarshallable that = (MyTestClassMarshallable) o; + + if (a != that.a) return false; + + return true; + } + + @Override + public int hashCode() { + return a; + } + + public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { + a = in.readInt(); + } + + /** + * write an object to bytes + * + * @param out to write to + */ + public void writeMarshallable(@NotNull Bytes out) { + out.writeInt(a); + } + } + + +} \ No newline at end of file From 120454fce9f6b006404bfd609e3b829abfef481b Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:15:39 +0100 Subject: [PATCH 05/30] added Key Value Object Types --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 4f30fa7ca..154edc2b4 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,29 @@ and define the snapshot version in your pom.xml, for example: 1.0.1-SNAPSHOT ``` + +#### Key Value Object Types + +Unlike HashMap which will support any heap object, Chronicle Map only works with objects that it +can store off heap, so the objects have to be : (one of the following ) + +- AutoBoxed primitives - for good performance. +- Strings - for good performance. +- implements Serializable +- implements Externalizable ( with a public default constructor ) +- Implements our custom interface BytesMarshallable ( with a public default constructor ) - use this for best performance. + +or value objects that are created through, a directClass interface, for example : +``` java + ChronicleMap chm = ChronicleMapBuilder + .of(String.class, directClassFor(BondVOInterface.class)) + .create(file); + +``` + +Object graphs can also be included as long as the outer object supports Serializable, Externalizable or BytesMarshallable. + + #### Java Class Construction Creating an instance of Chronicle Map is a little more complexed than just calling a constructor. From 65a5477f55fbf340fd7de6543cd67c85f7ca14c2 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:21:14 +0100 Subject: [PATCH 06/30] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 154edc2b4..6445ff3d2 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,10 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [JavaDoc](http://openhft.github.io/Chronicle-Map/apidocs) * [Getting Started Guide](https://github.com/OpenHFT/Chronicle-Map#getting-started) * [Simple Construction](https://github.com/OpenHFT/Chronicle-Map#simple-construction) - * [Sharing Data Between Two or More Maps](https://github.com/OpenHFT/Chronicle-Map#sharing-data-between-two-or-more-maps) + * [Maven Download](https://github.com/OpenHFT/Chronicle-Map#maven-artifact-download-1) + * [Snapshot Download](https://github.com/OpenHFT/Chronicle-Map#maven-snapshot-download) + * [Key Value Object Types](https://github.com/OpenHFT/Chronicle-Map#key-value-object-types) + * [Sharing Data Between Two or More Maps](https://github.com/OpenHFT/Chronicle-Map#sharing-data-between-two-or-more-maps) * [Entries](https://github.com/OpenHFT/Chronicle-Map#entries) * [Size of Space Reserved on Disk](https://github.com/OpenHFT/Chronicle-Map#size-of-space-reserved-on-disk) * [Chronicle Map Interface](https://github.com/OpenHFT/Chronicle-Map#chronicle-map-interface) From 1fb48ade341beb09bd59c4e2a6a8dd90c49c06c9 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:22:29 +0100 Subject: [PATCH 07/30] formatting only --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6445ff3d2..8fcdf6bcd 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,8 @@ can store off heap, so the objects have to be : (one of the following ) - Strings - for good performance. - implements Serializable - implements Externalizable ( with a public default constructor ) -- Implements our custom interface BytesMarshallable ( with a public default constructor ) - use this for best performance. +- implements our custom interface BytesMarshallable ( with a public default constructor ) - use +this for best performance. or value objects that are created through, a directClass interface, for example : ``` java From 7719a05aaa344a3d00721f457c9c6d399f6de9d9 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:24:04 +0100 Subject: [PATCH 08/30] added MyTestClassObjectGraph test --- .../chronicle/map/StatelessMapClientTest.java | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java index e03a2d14f..19f5c39a5 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -8,7 +8,7 @@ import net.openhft.lang.model.constraints.NotNull; import org.junit.Assert; import org.junit.Test; -import java.io.Serializable; +import java.io.*; import java.nio.ByteBuffer; import java.util.Collections; @@ -27,6 +27,10 @@ public class StatelessMapClientTest extends TestCase { testReadValueWriteValue(new MyTestClass(3)); testReadValueWriteValue(new MyTestClassMarshallable(3)); + testReadValueWriteValue(new MyTestClassExternalizable(3)); + testReadValueWriteValue(new MyTestClassObjectGraph(3)); + + } public void testReadValueWriteValue(Object value) throws Exception { @@ -50,6 +54,45 @@ public class StatelessMapClientTest extends TestCase { } + public static class MyTestClassExternalizable implements Externalizable { + int a; + + public MyTestClassExternalizable() { + } + + + MyTestClassExternalizable(int a) { + this.a = a; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClassExternalizable that = (MyTestClassExternalizable) o; + + if (a != that.a) return false; + + return true; + } + + @Override + public int hashCode() { + return a; + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.write(a); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + a = in.readInt(); + } + } + public static class MyTestClass implements Serializable { int a; @@ -85,6 +128,10 @@ public class StatelessMapClientTest extends TestCase { this.a = a; } + public void setA(int a) { + this.a = a; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -116,5 +163,36 @@ public class StatelessMapClientTest extends TestCase { } } + public static class MyTestClassObjectGraph implements Serializable { + + MyTestClassMarshallable delegate; + + public MyTestClassObjectGraph() { + + } + + MyTestClassObjectGraph(int a) { + delegate = new MyTestClassMarshallable(a); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClassObjectGraph that = (MyTestClassObjectGraph) o; + + if (delegate != null ? !delegate.equals(that.delegate) : that.delegate != null) + return false; + + return true; + } + + @Override + public int hashCode() { + return delegate != null ? delegate.hashCode() : 0; + } + } + } \ No newline at end of file From a7b213f3add2cf44a3492653c2095ed1a443431d Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:51:36 +0100 Subject: [PATCH 09/30] fixed typo around "processes" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8fcdf6bcd..d0c77e809 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ the instances of Chronicle Map on the same server. The name and location of the up to you. For the best performance on many unix systems we recommend using [tmpfs](http://en.wikipedia.org/wiki/Tmpfs). -If instead, you do not wish to replicate between process on the same server or if you are only +If instead, you do not wish to replicate between processes on the same server or if you are only using TCP replication to replicate between servers, you do not have to provide the "file", so you can call `create()` method on ChronicleMapBuilder without file parameter: ``` From 2e0c5d0c86fc680e5045aa484f2f851f8fdc1adb Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:52:55 +0100 Subject: [PATCH 10/30] fixed typo around --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0c77e809..bfb51bbc2 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,8 @@ up to you. For the best performance on many unix systems we recommend using If instead, you do not wish to replicate between processes on the same server or if you are only using TCP replication to replicate between servers, you do not have to provide the "file", -so you can call `create()` method on ChronicleMapBuilder without file parameter: +so you can call the `create()` method on you ChronicleMapBuilder without providing the file +parameter: ``` ConcurrentMap map = builder.create(); ``` From 0240485c910e5f98d88980707bb7c9dc4fda2f4f Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:53:16 +0100 Subject: [PATCH 11/30] HCOLL-167 Created a stateless map - added tests --- .../chronicle/map/ChronicleMapBuilder.java | 2 +- .../chronicle/map/StatelessMapClient.java | 104 ++++++++++++++---- .../chronicle/map/StatelessMapClientTest.java | 6 +- 3 files changed, 85 insertions(+), 27 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 725815757..504d08b58 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -786,7 +786,7 @@ public class ChronicleMapBuilder implements Cloneable, return identifier != -1; } - private void preMapConstruction() { + void preMapConstruction() { keyBuilder.objectSerializer(objectSerializer()); valueBuilder.objectSerializer(objectSerializer()); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 73cae54ba..31a84a946 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -21,7 +21,6 @@ import java.util.concurrent.atomic.AtomicLong; import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; /** - * * **** THIS IS VERY MUCH IN DRAFT *** * * @author Rob Austin. @@ -141,9 +140,43 @@ public class StatelessMapClient, lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); + keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); + keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); + valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); + valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); + + if (defaultValueProvider instanceof ConstantValueProvider) { + ConstantValueProvider constantValueProvider = + (ConstantValueProvider) defaultValueProvider; + if (constantValueProvider.wasDeserialized()) { + ThreadLocalCopies copies = valueReaderProvider.getCopies(null); + BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); + constantValueProvider.initTransients(valueReader); + } + } } + + void initTransients() { + keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); + keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); + + valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); + valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); + + if (defaultValueProvider instanceof ConstantValueProvider) { + ConstantValueProvider constantValueProvider = + (ConstantValueProvider) defaultValueProvider; + if (constantValueProvider.wasDeserialized()) { + ThreadLocalCopies copies = valueReaderProvider.getCopies(null); + BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); + constantValueProvider.initTransients(valueReader); + } + } + } + + @Override public V putIfAbsent(K key, V value) { target.writeByte(PUT_IF_ABSENT.ordinal()); @@ -151,7 +184,7 @@ public class StatelessMapClient, ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey(key, copies); - writeValue(value, copies); + writeValue(value); return null; } @@ -163,7 +196,7 @@ public class StatelessMapClient, ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey((K) key, copies); - writeValue((V) value, copies); + writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -179,8 +212,8 @@ public class StatelessMapClient, target.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey(key, copies); - writeValue(oldValue, copies); - writeValue(newValue, copies); + writeValue(oldValue); + writeValue(newValue); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -191,12 +224,13 @@ public class StatelessMapClient, } + @Override public V replace(K key, V value) { target.writeByte(REPLACE.ordinal()); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey(key, copies); - writeValue(value, copies); + writeValue(value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -251,8 +285,7 @@ public class StatelessMapClient, @Override public boolean containsValue(Object value) { target.writeByte(CONTAINS_VALUE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeValue((V) value, copies); + writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -289,8 +322,25 @@ public class StatelessMapClient, } - private V readValue(ByteBufferBytes byteBufferBytes) { - throw new UnsupportedOperationException("todo"); + V readValue(ByteBufferBytes source) { + + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + + copies = metaKeyInteropProvider.getCopies(copies); + + copies = valueReaderProvider.getCopies(copies); + BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); + + try { + final V v = valueFactory.create(); + + + long valueSize = valueSizeMarshaller.readSize(source); + return valueReader.read(source, valueSize, v); + } catch (Exception e) { + throw new RuntimeException(e); + } + } @Override @@ -309,7 +359,7 @@ public class StatelessMapClient, ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey((K) key, copies); - writeValue((V) value, copies); + writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -362,7 +412,7 @@ public class StatelessMapClient, for (Entry e : entries) { writeKey(e.getKey(), copies); - writeValue(e.getValue(), copies); + writeValue(e.getValue()); } } @@ -460,31 +510,39 @@ public class StatelessMapClient, } - private void writeValue(V value, ThreadLocalCopies copies) { - + void writeValue(V value) { + boolean byteableValue = value instanceof Byteable; + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); copies = valueWriterProvider.getCopies(copies); long valueSize; MetaBytesWriter metaValueWriter = null; Byteable valueAsByteable = null; - if (value instanceof Byteable) { - valueAsByteable = (Byteable) value; - valueSize = valueAsByteable.maxSize(); - - target.writeLong(valueSize); - target.write(valueAsByteable.bytes()); - } else { + if (!byteableValue) { copies = valueWriterProvider.getCopies(copies); valueWriter = valueWriterProvider.get(copies, originalValueWriter); copies = metaValueWriterProvider.getCopies(copies); metaValueWriter = metaValueWriterProvider.get( copies, originalMetaValueWriter, valueWriter, value); valueSize = metaValueWriter.size(valueWriter, value); - target.writeLong(valueSize); - metaValueWriter.write(valueWriter, target, value); + } else { + valueAsByteable = (Byteable) value; + valueSize = valueAsByteable.maxSize(); } + valueSizeMarshaller.writeSize(target, valueSize); + + + if (metaValueWriter != null) { + metaValueWriter.write(valueWriter, target, value); + } else { + throw new UnsupportedOperationException(""); + /* assert valueAsByteable != null; + long valueOffset = entry.positionAddr() - bytes.address(); + target.zeroOut(valueOffset, valueOffset + valueSize); + valueAsByteable.bytes(target, valueOffset);*/ + } } /** diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java index 19f5c39a5..4fb5c6bdf 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -25,10 +25,10 @@ public class StatelessMapClientTest extends TestCase { testReadValueWriteValue(Collections.singleton("Test")); testReadValueWriteValue(Collections.EMPTY_MAP); testReadValueWriteValue(new MyTestClass(3)); - testReadValueWriteValue(new MyTestClassMarshallable(3)); + testReadValueWriteValue(new MyTestClassMarshallable(3)); - testReadValueWriteValue(new MyTestClassExternalizable(3)); - testReadValueWriteValue(new MyTestClassObjectGraph(3)); + // testReadValueWriteValue(new MyTestClassExternalizable(3)); + // testReadValueWriteValue(new MyTestClassObjectGraph(3)); } From 078a22a1417a04203767f5f886d76bbad71a8452 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:59:43 +0100 Subject: [PATCH 12/30] HCOLL-167 Created a stateless map - added tests --- .../net/openhft/chronicle/map/StatelessMapClientTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java index 4fb5c6bdf..4b9e271f0 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -27,8 +27,8 @@ public class StatelessMapClientTest extends TestCase { testReadValueWriteValue(new MyTestClass(3)); testReadValueWriteValue(new MyTestClassMarshallable(3)); - // testReadValueWriteValue(new MyTestClassExternalizable(3)); - // testReadValueWriteValue(new MyTestClassObjectGraph(3)); + testReadValueWriteValue(new MyTestClassExternalizable(3)); + testReadValueWriteValue(new MyTestClassObjectGraph(3)); } @@ -165,14 +165,14 @@ public class StatelessMapClientTest extends TestCase { public static class MyTestClassObjectGraph implements Serializable { - MyTestClassMarshallable delegate; + MyTestClass delegate; public MyTestClassObjectGraph() { } MyTestClassObjectGraph(int a) { - delegate = new MyTestClassMarshallable(a); + delegate = new MyTestClass(a); } @Override From 3d228b591b00ec7ab23a75aabe9bdd01237c131f Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 15:45:34 +0100 Subject: [PATCH 13/30] HCOLL-167 Created a stateless map - added tests --- .../net/openhft/chronicle/map/Serializer.java | 114 ++++++++++++++++++ ...MapClientTest.java => SerializerTest.java} | 96 +++++++++++---- 2 files changed, 186 insertions(+), 24 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/map/Serializer.java rename src/test/java/net/openhft/chronicle/map/{StatelessMapClientTest.java => SerializerTest.java} (61%) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java new file mode 100644 index 000000000..ec112db76 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -0,0 +1,114 @@ +package net.openhft.chronicle.map; + +import net.openhft.chronicle.map.serialization.BytesReader; +import net.openhft.chronicle.map.serialization.MetaBytesWriter; +import net.openhft.chronicle.map.serialization.MetaProvider; +import net.openhft.chronicle.map.serialization.SizeMarshaller; +import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.io.serialization.ObjectFactory; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.constraints.NotNull; + +/** + * uses a {@code SerializationBuilder} to readMarshallable and writeMarshallable objects + * @author Rob Austin. + */ +public class Serializer> + implements BytesMarshallable { + + final Class clazz; + final SizeMarshaller sizeMarshaller; + final BytesReader originalReader; + transient Provider> readerProvider; + final VW originalWriter; + transient Provider writerProvider; + final MVW originalMetaValueWriter; + final MetaProvider metaValueWriterProvider; + final ObjectFactory objectFactory; + + final ThreadLocal value = new ThreadLocal(); + + public Serializer(final SerializationBuilder serializationBuilder) { + + clazz = serializationBuilder.eClass; + sizeMarshaller = serializationBuilder.sizeMarshaller(); + originalReader = serializationBuilder.reader(); + + originalWriter = (VW) serializationBuilder.interop(); + originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); + metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); + objectFactory = serializationBuilder.factory(); + + readerProvider = Provider.of((Class) originalReader.getClass()); + writerProvider = Provider.of((Class) originalWriter.getClass()); + } + + + public O getObject() { + return value.get(); + } + + public void setObject(O value) { + this.value.set(value); + } + + @Override + public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { + O o = readObject(in); + setObject(o); + } + + @Override + public void writeMarshallable(@NotNull Bytes out) { + O value1 = getObject(); + boolean byteableValue = value1 instanceof Byteable; + ThreadLocalCopies copies; + copies = readerProvider.getCopies(null); + VW valueWriter = writerProvider.get(copies, originalWriter); + copies = writerProvider.getCopies(copies); + long valueSize; + MetaBytesWriter metaValueWriter = null; + Byteable valueAsByteable = null; + + if (!byteableValue) { + copies = writerProvider.getCopies(copies); + valueWriter = writerProvider.get(copies, originalWriter); + copies = metaValueWriterProvider.getCopies(copies); + metaValueWriter = metaValueWriterProvider.get( + copies, originalMetaValueWriter, valueWriter, value1); + valueSize = metaValueWriter.size(valueWriter, value1); + } else { + valueAsByteable = (Byteable) value1; + valueSize = valueAsByteable.maxSize(); + } + + sizeMarshaller.writeSize(out, valueSize); + + + if (metaValueWriter != null) { + metaValueWriter.write(valueWriter, out, value1); + } else { + throw new UnsupportedOperationException(""); + } + } + + private O readObject(Bytes in) { + + ThreadLocalCopies copies; + copies = readerProvider.getCopies(null); + BytesReader valueReader = readerProvider.get(copies, originalReader); + + try { + final O v = objectFactory.create(); + long valueSize = sizeMarshaller.readSize(in); + return valueReader.read(in, valueSize, v); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + +} diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java similarity index 61% rename from src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java rename to src/test/java/net/openhft/chronicle/map/SerializerTest.java index 4b9e271f0..11bcfa046 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -6,54 +6,102 @@ import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.BytesMarshallable; import net.openhft.lang.model.constraints.NotNull; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.io.*; import java.nio.ByteBuffer; import java.util.Collections; -public class StatelessMapClientTest extends TestCase { - +public class SerializerTest extends TestCase { @Test - public void testReadValueWriteIntegerValue() throws Exception { - testReadValueWriteValue("Test"); - testReadValueWriteValue(1); - testReadValueWriteValue(1L); - testReadValueWriteValue(1.0); - testReadValueWriteValue(1.0f); - testReadValueWriteValue(Collections.singleton("Test")); - testReadValueWriteValue(Collections.EMPTY_MAP); - testReadValueWriteValue(new MyTestClass(3)); - testReadValueWriteValue(new MyTestClassMarshallable(3)); - - testReadValueWriteValue(new MyTestClassExternalizable(3)); - testReadValueWriteValue(new MyTestClassObjectGraph(3)); - + public void testValueMarshallable() throws Exception { + testReadWriteValue("Test"); + testReadWriteValue(1); + testReadWriteValue(1L); + testReadWriteValue(1.0); + testReadWriteValue(1.0f); + testReadWriteValue(Collections.singleton("Test")); + testReadWriteValue(Collections.EMPTY_MAP); + testReadWriteValue(new MyTestClass(3)); + testReadWriteValue(new MyTestClassMarshallable(3)); + testReadWriteValue(new MyTestClassExternalizable(3)); + testReadWriteValue(new MyTestClassObjectGraph(3)); } - public void testReadValueWriteValue(Object value) throws Exception { + @Ignore + @Test + public void testKeyMarshallable() throws Exception { + testReadWriteKey("Test"); + testReadWriteKey(1); + testReadWriteKey(1L); + testReadWriteKey(1.0); + testReadWriteKey(1.0f); + testReadWriteKey(Collections.singleton("Test")); + testReadWriteKey(Collections.EMPTY_MAP); + testReadWriteKey(new MyTestClass(3)); + testReadWriteKey(new MyTestClassMarshallable(3)); + + testReadWriteKey(new MyTestClassExternalizable(3)); + testReadWriteKey(new MyTestClassObjectGraph(3)); + } + + + public void testReadWriteValue(Object value) throws Exception { Class valueClass = value.getClass(); - final ByteBufferBytes buffer = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); - ByteBufferBytes slice = buffer.slice(); + final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + ByteBufferBytes in = out.slice(); ChronicleMapBuilder builder = ChronicleMapBuilder.of(Integer.class, valueClass); builder.preMapConstruction(); - StatelessMapClient statelessMap = new StatelessMapClient(buffer, builder); + Serializer v = new Serializer(builder.valueBuilder); - statelessMap.writeValue(value); - long position = buffer.position(); - slice.limit(position); + v.setObject(value); + v.writeMarshallable(out); - Object actual = statelessMap.readValue(slice); + long position = out.position(); + in.limit(position); + + // let just blank the value so that we can test if it works + v.setObject(null); + + v.readMarshallable(in); + Object actual = v.getObject(); Assert.assertEquals(actual, value); } + public void testReadWriteKey(Object key) throws Exception { + + Class clazz = key.getClass(); + + final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + ByteBufferBytes in = out.slice(); + + ChronicleMapBuilder builder = ChronicleMapBuilder.of(clazz, Integer.class); + builder.preMapConstruction(); + + Serializer v = new Serializer(builder.keyBuilder); + + v.setObject(key); + v.writeMarshallable(out); + + long position = out.position(); + in.limit(position); + + // let just blank the value so that we can test if it works + v.setObject(null); + + v.readMarshallable(in); + Object actual = v.getObject(); + Assert.assertEquals(actual, key); + } + public static class MyTestClassExternalizable implements Externalizable { int a; From 3bc0be04c65161610309c8772507bd9b8367170d Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 15:48:31 +0100 Subject: [PATCH 14/30] HCOLL-167 Created a stateless map - woops missed this file --- src/main/java/net/openhft/chronicle/map/StatelessMapClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 31a84a946..8d6108386 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -79,7 +79,7 @@ public class StatelessMapClient, .getSimpleName()); } - enum EventId { + public static enum EventId { LONG_SIZE, SIZE, IS_EMPTY, CONTAINS_KEY, From 2e83944837688d424adf11409cd51c9abaac8a48 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 15:58:04 +0100 Subject: [PATCH 15/30] HCOLL-167 Created a stateless map - corrected @Ignore --- src/test/java/net/openhft/chronicle/map/SerializerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index 11bcfa046..dd6360f28 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -1,6 +1,5 @@ package net.openhft.chronicle.map; -import junit.framework.TestCase; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.BytesMarshallable; @@ -13,7 +12,7 @@ import java.io.*; import java.nio.ByteBuffer; import java.util.Collections; -public class SerializerTest extends TestCase { +public class SerializerTest { @Test public void testValueMarshallable() throws Exception { @@ -31,8 +30,9 @@ public class SerializerTest extends TestCase { testReadWriteValue(new MyTestClassObjectGraph(3)); } - @Ignore + @Test + @Ignore public void testKeyMarshallable() throws Exception { testReadWriteKey("Test"); testReadWriteKey(1); From e48a44e6e9cc62c5fe643144b3fdea4de733881d Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 16:23:57 +0100 Subject: [PATCH 16/30] HCOLL-167 Created a stateless map - fixed test case -net.openhft.chronicle.map.SerializerTest.testKeyMarshallable --- .../net/openhft/chronicle/map/Serializer.java | 65 ++--- .../chronicle/map/StatelessMapClient.java | 273 +++++------------- .../openhft/chronicle/map/SerializerTest.java | 20 +- 3 files changed, 94 insertions(+), 264 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index ec112db76..52643c9f2 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -7,19 +7,16 @@ import net.openhft.chronicle.map.serialization.SizeMarshaller; import net.openhft.chronicle.map.threadlocal.Provider; import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; -import net.openhft.lang.io.serialization.BytesMarshallable; -import net.openhft.lang.io.serialization.ObjectFactory; import net.openhft.lang.model.Byteable; import net.openhft.lang.model.constraints.NotNull; /** * uses a {@code SerializationBuilder} to readMarshallable and writeMarshallable objects + * * @author Rob Austin. */ -public class Serializer> - implements BytesMarshallable { +public class Serializer> { - final Class clazz; final SizeMarshaller sizeMarshaller; final BytesReader originalReader; transient Provider> readerProvider; @@ -27,44 +24,38 @@ public class Serializer> transient Provider writerProvider; final MVW originalMetaValueWriter; final MetaProvider metaValueWriterProvider; - final ObjectFactory objectFactory; - - final ThreadLocal value = new ThreadLocal(); public Serializer(final SerializationBuilder serializationBuilder) { - clazz = serializationBuilder.eClass; + sizeMarshaller = serializationBuilder.sizeMarshaller(); originalReader = serializationBuilder.reader(); originalWriter = (VW) serializationBuilder.interop(); originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); - objectFactory = serializationBuilder.factory(); readerProvider = Provider.of((Class) originalReader.getClass()); writerProvider = Provider.of((Class) originalWriter.getClass()); } + public O readMarshallable(@NotNull Bytes in) throws IllegalStateException { - public O getObject() { - return value.get(); + ThreadLocalCopies copies; + copies = readerProvider.getCopies(null); + BytesReader valueReader = readerProvider.get(copies, originalReader); + + try { + long valueSize = sizeMarshaller.readSize(in); + return valueReader.read(in, valueSize, null); + } catch (Exception e) { + throw new RuntimeException(e); + } } - public void setObject(O value) { - this.value.set(value); - } + public void writeMarshallable(@NotNull Bytes out, O value) { - @Override - public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { - O o = readObject(in); - setObject(o); - } - - @Override - public void writeMarshallable(@NotNull Bytes out) { - O value1 = getObject(); - boolean byteableValue = value1 instanceof Byteable; + boolean byteableValue = value instanceof Byteable; ThreadLocalCopies copies; copies = readerProvider.getCopies(null); VW valueWriter = writerProvider.get(copies, originalWriter); @@ -78,10 +69,10 @@ public class Serializer> valueWriter = writerProvider.get(copies, originalWriter); copies = metaValueWriterProvider.getCopies(copies); metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, value1); - valueSize = metaValueWriter.size(valueWriter, value1); + copies, originalMetaValueWriter, valueWriter, value); + valueSize = metaValueWriter.size(valueWriter, value); } else { - valueAsByteable = (Byteable) value1; + valueAsByteable = (Byteable) value; valueSize = valueAsByteable.maxSize(); } @@ -89,26 +80,10 @@ public class Serializer> if (metaValueWriter != null) { - metaValueWriter.write(valueWriter, out, value1); + metaValueWriter.write(valueWriter, out, value); } else { throw new UnsupportedOperationException(""); } } - private O readObject(Bytes in) { - - ThreadLocalCopies copies; - copies = readerProvider.getCopies(null); - BytesReader valueReader = readerProvider.get(copies, originalReader); - - try { - final O v = objectFactory.create(); - long valueSize = sizeMarshaller.readSize(in); - return valueReader.read(in, valueSize, v); - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 8d6108386..3297e5b20 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -1,15 +1,12 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.map.serialization.*; -import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.serialization.MetaBytesInterop; +import net.openhft.chronicle.map.serialization.MetaBytesWriter; import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.ByteBufferBytes; -import net.openhft.lang.io.BytesStore; -import net.openhft.lang.io.serialization.ObjectFactory; -import net.openhft.lang.model.Byteable; +import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; -import javax.swing.text.Segment; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -29,43 +26,18 @@ public class StatelessMapClient, V, VW, MVW extends MetaBytesWriter> extends AbstractMap implements ChronicleMap, Serializable { - final Class kClass; + private ThreadLocal sourceBuffer = new ThreadLocal(); - final BytesReader originalKeyReader; - transient Provider> keyReaderProvider; - final KI originalKeyInterop; - transient Provider keyInteropProvider; - final MKI originalMetaKeyInterop; - final MetaProvider metaKeyInteropProvider; - - final Class vClass; - final SizeMarshaller valueSizeMarshaller; - final BytesReader originalValueReader; - transient Provider> valueReaderProvider; - final VW originalValueWriter; - transient Provider valueWriterProvider; - final MVW originalMetaValueWriter; - final MetaProvider metaValueWriterProvider; - final ObjectFactory valueFactory; - final DefaultValueProvider defaultValueProvider; Map transactionIDs = new HashMap(); AtomicLong transactionID = new AtomicLong(); - private final long lockTimeOutNS; - - transient Segment[] segments; // non-final for close() - // non-final for close() and because it is initialized out of constructor - transient BytesStore ms; - transient long headerSize; - transient Set> entrySet; - - private int bits; - private int mask; private long timeoutMs = TimeUnit.SECONDS.toMillis(20); + private final Serializer valueSerializer; + private final Serializer keySerializer; @Override @@ -111,79 +83,24 @@ public class StatelessMapClient, } - private final ByteBufferBytes target; - private final SizeMarshaller keySizeMarshaller; - - public StatelessMapClient(ByteBufferBytes target, ChronicleMapBuilder builder) { - this.target = target; - - SerializationBuilder keyBuilder = builder.keyBuilder; - kClass = keyBuilder.eClass; - keySizeMarshaller = keyBuilder.sizeMarshaller(); - originalKeyReader = keyBuilder.reader(); - - originalKeyInterop = (KI) keyBuilder.interop(); - - originalMetaKeyInterop = (MKI) keyBuilder.metaInterop(); - metaKeyInteropProvider = (MetaProvider) keyBuilder.metaInteropProvider(); - - SerializationBuilder valueBuilder = builder.valueBuilder; - vClass = valueBuilder.eClass; - valueSizeMarshaller = valueBuilder.sizeMarshaller(); - originalValueReader = valueBuilder.reader(); - - originalValueWriter = (VW) valueBuilder.interop(); - originalMetaValueWriter = (MVW) valueBuilder.metaInterop(); - metaValueWriterProvider = (MetaProvider) valueBuilder.metaInteropProvider(); - valueFactory = valueBuilder.factory(); - defaultValueProvider = builder.defaultValueProvider(); - - lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); - - keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); - keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); - - valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); - valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); - - if (defaultValueProvider instanceof ConstantValueProvider) { - ConstantValueProvider constantValueProvider = - (ConstantValueProvider) defaultValueProvider; - if (constantValueProvider.wasDeserialized()) { - ThreadLocalCopies copies = valueReaderProvider.getCopies(null); - BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); - constantValueProvider.initTransients(valueReader); - } - } - } + private final Bytes out; - void initTransients() { - keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); - keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); + public StatelessMapClient(Bytes out, ChronicleMapBuilder builder) { - valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); - valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); + valueSerializer = new Serializer(builder.valueBuilder); + keySerializer = new Serializer(builder.keyBuilder); + + this.out = out; - if (defaultValueProvider instanceof ConstantValueProvider) { - ConstantValueProvider constantValueProvider = - (ConstantValueProvider) defaultValueProvider; - if (constantValueProvider.wasDeserialized()) { - ThreadLocalCopies copies = valueReaderProvider.getCopies(null); - BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); - constantValueProvider.initTransients(valueReader); - } - } } @Override public V putIfAbsent(K key, V value) { - target.writeByte(PUT_IF_ABSENT.ordinal()); + out.writeByte(PUT_IF_ABSENT.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - - writeKey(key, copies); + writeKey(key); writeValue(value); return null; @@ -192,15 +109,15 @@ public class StatelessMapClient, @Override public boolean remove(Object key, Object value) { - target.writeByte(REMOVE_WITH_VALUE.ordinal()); + out.writeByte(REMOVE_WITH_VALUE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + + writeKey((K) key); writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -209,15 +126,15 @@ public class StatelessMapClient, @Override public boolean replace(K key, V oldValue, V newValue) { - target.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey(key, copies); + out.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); + + writeKey(key); writeValue(oldValue); writeValue(newValue); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -227,14 +144,14 @@ public class StatelessMapClient, @Override public V replace(K key, V value) { - target.writeByte(REPLACE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey(key, copies); + out.writeByte(REPLACE.ordinal()); + + writeKey(key); writeValue(value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -244,11 +161,11 @@ public class StatelessMapClient, @Override public int size() { - target.writeByte(SIZE.ordinal()); + out.writeByte(SIZE.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readInt(); @@ -257,11 +174,11 @@ public class StatelessMapClient, @Override public boolean isEmpty() { - target.writeByte(IS_EMPTY.ordinal()); + out.writeByte(IS_EMPTY.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -269,13 +186,13 @@ public class StatelessMapClient, @Override public boolean containsKey(Object key) { - target.writeByte(CONTAINS_KEY.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + out.writeByte(CONTAINS_KEY.ordinal()); + + writeKey((K) key); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -284,12 +201,12 @@ public class StatelessMapClient, @Override public boolean containsValue(Object value) { - target.writeByte(CONTAINS_VALUE.ordinal()); + out.writeByte(CONTAINS_VALUE.ordinal()); writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -297,11 +214,11 @@ public class StatelessMapClient, @Override public long longSize() { - target.writeByte(LONG_SIZE.ordinal()); + out.writeByte(LONG_SIZE.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readLong(); @@ -309,13 +226,13 @@ public class StatelessMapClient, @Override public V get(Object key) { - target.writeByte(GET.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + out.writeByte(GET.ordinal()); + + writeKey((K) key); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -324,22 +241,9 @@ public class StatelessMapClient, V readValue(ByteBufferBytes source) { - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - copies = metaKeyInteropProvider.getCopies(copies); + return valueSerializer.readMarshallable(source); - copies = valueReaderProvider.getCopies(copies); - BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); - - try { - final V v = valueFactory.create(); - - - long valueSize = valueSizeMarshaller.readSize(source); - return valueReader.read(source, valueSize, v); - } catch (Exception e) { - throw new RuntimeException(e); - } } @@ -355,15 +259,15 @@ public class StatelessMapClient, @Override public V put(K key, V value) { - target.writeByte(PUT.ordinal()); + out.writeByte(PUT.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + + writeKey((K) key); writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -372,11 +276,11 @@ public class StatelessMapClient, @Override public V remove(Object key) { - target.writeByte(REMOVE.ordinal()); + out.writeByte(REMOVE.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -385,9 +289,10 @@ public class StatelessMapClient, @Override public void putAll(Map map) { - target.writeByte(PUT_ALL.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + throw new UnsupportedOperationException("todo"); + /* out.writeByte(PUT_ALL.ordinal()); + HashMap safeCopy = new HashMap(map); @@ -395,34 +300,34 @@ public class StatelessMapClient, // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server blockingFetch(transactionId); - - return; +*/ + //return; } private void writeEntries(ThreadLocalCopies copies, HashMap safeCopy) { - target.writeLong(safeCopy.size()); + out.writeLong(safeCopy.size()); Set entries = (Set) safeCopy.entrySet(); for (Entry e : entries) { - writeKey(e.getKey(), copies); + writeKey(e.getKey()); writeValue(e.getValue()); } } @Override public void clear() { - target.writeByte(CLEAR.ordinal()); + out.writeByte(CLEAR.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server blockingFetch(transactionId); @@ -433,11 +338,11 @@ public class StatelessMapClient, @NotNull @Override public Set keySet() { - target.writeByte(KEY_SET.ordinal()); + out.writeByte(KEY_SET.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readKeySet(blockingFetch(transactionId)); @@ -456,7 +361,7 @@ public class StatelessMapClient, private K readKey(ByteBufferBytes bufferBytes) { long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readKey(blockingFetch(transactionId)); @@ -466,10 +371,10 @@ public class StatelessMapClient, @NotNull @Override public Collection values() { - target.writeByte(VALUES.ordinal()); + out.writeByte(VALUES.ordinal()); long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server ByteBufferBytes bufferBytes = blockingFetch(transactionId); @@ -487,7 +392,7 @@ public class StatelessMapClient, @NotNull @Override public Set> entrySet() { - target.writeByte(ENTRY_SET.ordinal()); + out.writeByte(ENTRY_SET.ordinal()); long transactionId = nextUniqueTransaction(); @@ -511,56 +416,16 @@ public class StatelessMapClient, void writeValue(V value) { - boolean byteableValue = value instanceof Byteable; - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = valueWriterProvider.getCopies(copies); - long valueSize; - MetaBytesWriter metaValueWriter = null; - Byteable valueAsByteable = null; - - if (!byteableValue) { - copies = valueWriterProvider.getCopies(copies); - valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = metaValueWriterProvider.getCopies(copies); - metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, value); - valueSize = metaValueWriter.size(valueWriter, value); - } else { - valueAsByteable = (Byteable) value; - valueSize = valueAsByteable.maxSize(); - } - - valueSizeMarshaller.writeSize(target, valueSize); - - - if (metaValueWriter != null) { - metaValueWriter.write(valueWriter, target, value); - } else { - throw new UnsupportedOperationException(""); - /* assert valueAsByteable != null; - long valueOffset = entry.positionAddr() - bytes.address(); - target.zeroOut(valueOffset, valueOffset + valueSize); - valueAsByteable.bytes(target, valueOffset);*/ - } + valueSerializer.writeMarshallable(out, value); } /** * write the keysize and the key to the the {@code target} buffer * - * @param key the key of the map - * @param copies + * @param key the key of the map */ - private void writeKey(K key, final ThreadLocalCopies copies) { - KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); - - MKI metaKeyWriter = - metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); - - long keySize = metaKeyWriter.size(keyWriter, key); - - keySizeMarshaller.writeSize(target, keySize); - metaKeyWriter.write(keyWriter, target, key); + private void writeKey(K key) { + keySerializer.writeMarshallable(out, key); } diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index dd6360f28..eeea8b527 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -5,7 +5,6 @@ import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.BytesMarshallable; import net.openhft.lang.model.constraints.NotNull; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import java.io.*; @@ -32,7 +31,6 @@ public class SerializerTest { @Test - @Ignore public void testKeyMarshallable() throws Exception { testReadWriteKey("Test"); testReadWriteKey(1); @@ -61,17 +59,13 @@ public class SerializerTest { Serializer v = new Serializer(builder.valueBuilder); - v.setObject(value); - v.writeMarshallable(out); + v.writeMarshallable(out, value); long position = out.position(); in.limit(position); - // let just blank the value so that we can test if it works - v.setObject(null); - v.readMarshallable(in); - Object actual = v.getObject(); + Object actual = v.readMarshallable(in); Assert.assertEquals(actual, value); } @@ -88,17 +82,13 @@ public class SerializerTest { Serializer v = new Serializer(builder.keyBuilder); - v.setObject(key); - v.writeMarshallable(out); + + v.writeMarshallable(out, key); long position = out.position(); in.limit(position); - // let just blank the value so that we can test if it works - v.setObject(null); - - v.readMarshallable(in); - Object actual = v.getObject(); + Object actual = v.readMarshallable(in); Assert.assertEquals(actual, key); } From cffa0916ab945f6b2e6e81e3bc41bb16282cc4e4 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 17:03:19 +0100 Subject: [PATCH 17/30] HCOLL-167 improved the code in net.openhft.chronicle.map.StatelessMapClient, but its still in draft. --- .../net/openhft/chronicle/map/Serializer.java | 39 ++- .../chronicle/map/StatelessMapClient.java | 282 +++++++----------- 2 files changed, 118 insertions(+), 203 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index 52643c9f2..d91eec3da 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -11,7 +11,7 @@ import net.openhft.lang.model.Byteable; import net.openhft.lang.model.constraints.NotNull; /** - * uses a {@code SerializationBuilder} to readMarshallable and writeMarshallable objects + * uses a {@code SerializationBuilder} to read and write object(s) as bytes * * @author Rob Austin. */ @@ -27,23 +27,21 @@ public class Serializer> { public Serializer(final SerializationBuilder serializationBuilder) { - sizeMarshaller = serializationBuilder.sizeMarshaller(); - originalReader = serializationBuilder.reader(); - - originalWriter = (VW) serializationBuilder.interop(); originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); + originalReader = serializationBuilder.reader(); + originalWriter = (VW) serializationBuilder.interop(); + readerProvider = Provider.of((Class) originalReader.getClass()); writerProvider = Provider.of((Class) originalWriter.getClass()); } public O readMarshallable(@NotNull Bytes in) throws IllegalStateException { - ThreadLocalCopies copies; - copies = readerProvider.getCopies(null); - BytesReader valueReader = readerProvider.get(copies, originalReader); + final ThreadLocalCopies copies = readerProvider.getCopies(null); + final BytesReader valueReader = readerProvider.get(copies, originalReader); try { long valueSize = sizeMarshaller.readSize(in); @@ -55,35 +53,32 @@ public class Serializer> { public void writeMarshallable(@NotNull Bytes out, O value) { - boolean byteableValue = value instanceof Byteable; - ThreadLocalCopies copies; - copies = readerProvider.getCopies(null); + ThreadLocalCopies copies = readerProvider.getCopies(null); VW valueWriter = writerProvider.get(copies, originalWriter); copies = writerProvider.getCopies(copies); - long valueSize; - MetaBytesWriter metaValueWriter = null; - Byteable valueAsByteable = null; - if (!byteableValue) { + final long valueSize; + + MetaBytesWriter metaValueWriter = null; + + if ((value instanceof Byteable)) { + valueSize = ((Byteable) value).maxSize(); + } else { copies = writerProvider.getCopies(copies); valueWriter = writerProvider.get(copies, originalWriter); copies = metaValueWriterProvider.getCopies(copies); metaValueWriter = metaValueWriterProvider.get( copies, originalMetaValueWriter, valueWriter, value); valueSize = metaValueWriter.size(valueWriter, value); - } else { - valueAsByteable = (Byteable) value; - valueSize = valueAsByteable.maxSize(); } sizeMarshaller.writeSize(out, valueSize); - - if (metaValueWriter != null) { + if (metaValueWriter != null) metaValueWriter.write(valueWriter, out, value); - } else { + else throw new UnsupportedOperationException(""); - } + } } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 3297e5b20..5693cc691 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -1,15 +1,11 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.map.serialization.MetaBytesInterop; -import net.openhft.chronicle.map.serialization.MetaBytesWriter; -import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; -import java.io.Serializable; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -22,34 +18,7 @@ import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; * * @author Rob Austin. */ -public class StatelessMapClient, - V, VW, MVW extends MetaBytesWriter> extends AbstractMap - implements ChronicleMap, Serializable { - - - private ThreadLocal sourceBuffer = new ThreadLocal(); - - - Map transactionIDs = new HashMap(); - - - AtomicLong transactionID = new AtomicLong(); - - private long timeoutMs = TimeUnit.SECONDS.toMillis(20); - private final Serializer valueSerializer; - private final Serializer keySerializer; - - - @Override - public File file() { - return null; - } - - @Override - public void close() throws IOException { - throw new UnsupportedOperationException("This is not supported in the " + this.getClass() - .getSimpleName()); - } +public class StatelessMapClient implements ChronicleMap { public static enum EventId { LONG_SIZE, SIZE, @@ -69,6 +38,27 @@ public class StatelessMapClient, REMOVE_WITH_VALUE } + private final ThreadLocal sourceBuffer = new ThreadLocal(); + private final Map transactionIDs = new HashMap(); + + private final AtomicLong transactionID = new AtomicLong(); + + private long timeoutMs = TimeUnit.SECONDS.toMillis(20); + private final Serializer valueSerializer; + private final Serializer keySerializer; + + + @Override + public File file() { + return null; + } + + @Override + public void close() throws IOException { + throw new UnsupportedOperationException("This is not supported in the " + this.getClass() + .getSimpleName()); + } + long nextUniqueTransaction() { long time = System.currentTimeMillis(); @@ -86,165 +76,127 @@ public class StatelessMapClient, private final Bytes out; - public StatelessMapClient(Bytes out, ChronicleMapBuilder builder) { + public StatelessMapClient(Bytes out, Class kClass, Class vClass) { - valueSerializer = new Serializer(builder.valueBuilder); - keySerializer = new Serializer(builder.keyBuilder); + final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, + SerializationBuilder.Role.KEY); + final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, + SerializationBuilder.Role.VALUE); + + keySerializer = new Serializer(keyBuilder); + valueSerializer = new Serializer(valueBuilder); this.out = out; - } @Override public V putIfAbsent(K key, V value) { - out.writeByte(PUT_IF_ABSENT.ordinal()); - + writeEvent(PUT_IF_ABSENT); writeKey(key); writeValue(value); - return null; + return readValue(blockingFetch()); + } @Override public boolean remove(Object key, Object value) { - out.writeByte(REMOVE_WITH_VALUE.ordinal()); - - + writeEvent(REMOVE_WITH_VALUE); writeKey((K) key); writeValue((V) value); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } + @Override public boolean replace(K key, V oldValue, V newValue) { - out.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); - + writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); writeKey(key); writeValue(oldValue); writeValue(newValue); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); + } + private void writeValue(V newValue) { + valueSerializer.writeMarshallable(out, newValue); } @Override public V replace(K key, V value) { - out.writeByte(REPLACE.ordinal()); + writeEvent(REPLACE); writeKey(key); writeValue(value); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } @Override public int size() { - out.writeByte(SIZE.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(SIZE); // get the data back from the server - return blockingFetch(transactionId).readInt(); + return blockingFetch().readInt(); } @Override public boolean isEmpty() { - out.writeByte(IS_EMPTY.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(IS_EMPTY); // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } @Override public boolean containsKey(Object key) { - out.writeByte(CONTAINS_KEY.ordinal()); - + writeEvent(CONTAINS_KEY); writeKey((K) key); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } @Override public boolean containsValue(Object value) { - out.writeByte(CONTAINS_VALUE.ordinal()); + writeEvent(CONTAINS_VALUE); writeValue((V) value); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } @Override public long longSize() { - out.writeByte(LONG_SIZE.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(LONG_SIZE); // get the data back from the server - return blockingFetch(transactionId).readLong(); + return blockingFetch().readLong(); } @Override public V get(Object key) { - out.writeByte(GET.ordinal()); - + writeEvent(GET); writeKey((K) key); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } - V readValue(ByteBufferBytes source) { - - + V readValue(Bytes source) { return valueSerializer.readMarshallable(source); - - } @Override @@ -259,61 +211,43 @@ public class StatelessMapClient, @Override public V put(K key, V value) { - out.writeByte(PUT.ordinal()); + writeEvent(PUT); + writeKey(key); + writeValue(value); - writeKey((K) key); - writeValue((V) value); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } @Override public V remove(Object key) { - out.writeByte(REMOVE.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - + writeEvent(REMOVE); + writeKey((K) key); // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } @Override public void putAll(Map map) { - throw new UnsupportedOperationException("todo"); - /* out.writeByte(PUT_ALL.ordinal()); - - - HashMap safeCopy = new HashMap(map); - - writeEntries(copies, safeCopy); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(PUT_ALL); + writeEntries(map); // get the data back from the server - blockingFetch(transactionId); -*/ - //return; - + blockingFetch(); } - private void writeEntries(ThreadLocalCopies copies, HashMap safeCopy) { + private void writeEntries(Map map) { + + final HashMap safeCopy = new HashMap(map); + out.writeLong(safeCopy.size()); - Set entries = (Set) safeCopy.entrySet(); + final Set entries = (Set) safeCopy.entrySet(); for (Entry e : entries) { writeKey(e.getKey()); @@ -323,68 +257,50 @@ public class StatelessMapClient, @Override public void clear() { - out.writeByte(CLEAR.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(CLEAR); // get the data back from the server - blockingFetch(transactionId); + blockingFetch(); - return; } @NotNull @Override public Set keySet() { - out.writeByte(KEY_SET.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(KEY_SET); // get the data back from the server - return readKeySet(blockingFetch(transactionId)); + return readKeySet(blockingFetch()); } - private Set readKeySet(ByteBufferBytes bufferBytes) { - int size = bufferBytes.readInt(); - - HashSet result = new HashSet<>(); + private Set readKeySet(Bytes in) { + int size = in.readInt(); + final HashSet result = new HashSet<>(); for (int i = 0; i < size; i++) { - result.add(readKey(bufferBytes)); + result.add(readKey(out)); } return result; } - private K readKey(ByteBufferBytes bufferBytes) { - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - - // get the data back from the server - return readKey(blockingFetch(transactionId)); - + private K readKey(Bytes bufferBytes) { + return keySerializer.readMarshallable(bufferBytes); } @NotNull @Override public Collection values() { - out.writeByte(VALUES.ordinal()); - long transactionId = nextUniqueTransaction(); - - out.writeLong(transactionId); + writeEvent(VALUES); // get the data back from the server - ByteBufferBytes bufferBytes = blockingFetch(transactionId); + Bytes in = blockingFetch(); - int size = bufferBytes.readInt(); + int size = in.readInt(); ArrayList result = new ArrayList(); for (int i = 0; i < size; i++) { - result.add(readValue(bufferBytes)); + result.add(readValue(in)); } return result; } @@ -392,33 +308,27 @@ public class StatelessMapClient, @NotNull @Override public Set> entrySet() { - out.writeByte(ENTRY_SET.ordinal()); + writeEvent(ENTRY_SET); - long transactionId = nextUniqueTransaction(); // get the data back from the server - ByteBufferBytes bufferBytes = blockingFetch(transactionId); + Bytes bytes = blockingFetch(); - int size = bufferBytes.readInt(); + int size = bytes.readInt(); Set> result = new HashSet>(); for (int i = 0; i < size; i++) { - K k = readKey(bufferBytes); - V v = readValue(bufferBytes); + K k = keySerializer.readMarshallable(bytes); + V v = valueSerializer.readMarshallable(bytes); result.add(new Entry(k, v)); } - return result; } - void writeValue(V value) { - valueSerializer.writeMarshallable(out, value); - } - /** * write the keysize and the key to the the {@code target} buffer * @@ -428,6 +338,9 @@ public class StatelessMapClient, keySerializer.writeMarshallable(out, key); } + private void writeEvent(EventId event) { + out.write((byte) event.ordinal()); + } public Buffer buffer() { Buffer result = sourceBuffer.get(); @@ -463,7 +376,7 @@ public class StatelessMapClient, * @param transactionId * @return */ - private ByteBufferBytes blockingFetch(long transactionId) { + private Bytes blockingFetch(long transactionId) { final Buffer buffer = buffer(); buffer.set(null); @@ -538,6 +451,13 @@ public class StatelessMapClient, } + + + private Bytes blockingFetch() { + long transactionId = nextUniqueTransaction(); + out.writeLong(transactionId); + return blockingFetch(transactionId); + } } From 3c47a33777073d48a0ab5c67fb09c52ada5dd2a8 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 17:33:39 +0100 Subject: [PATCH 18/30] HCOLL-167 updated to - net.openhft lang 6.4.10 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bb883ecb1..5c1b49bfe 100755 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ net.openhft lang - 6.4.10-SNAPSHOT + 6.4.10 From fe574a33b10e20b22c1b24a8f81808faecad73d4 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 18:29:35 +0100 Subject: [PATCH 19/30] HCOLL-167 created StatelessServerConnector, but its still in draft. --- .../net/openhft/chronicle/map/Serializer.java | 2 +- .../chronicle/map/StatelessMapClient.java | 48 ++-- .../map/StatelessServerConnector.java | 265 ++++++++++++++++++ .../openhft/chronicle/map/SerializerTest.java | 4 +- 4 files changed, 292 insertions(+), 27 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index d91eec3da..00bcdacb6 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -51,7 +51,7 @@ public class Serializer> { } } - public void writeMarshallable(@NotNull Bytes out, O value) { + public void writeMarshallable(O value, @NotNull Bytes out) { ThreadLocalCopies copies = readerProvider.getCopies(null); VW valueWriter = writerProvider.get(copies, originalWriter); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 5693cc691..967a000a8 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -47,6 +47,18 @@ public class StatelessMapClient implements ChronicleMap { private final Serializer valueSerializer; private final Serializer keySerializer; + public StatelessMapClient(Bytes out, Class kClass, Class vClass) { + + final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, + SerializationBuilder.Role.KEY); + final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, + SerializationBuilder.Role.VALUE); + + keySerializer = new Serializer(keyBuilder); + valueSerializer = new Serializer(valueBuilder); + + this.out = out; + } @Override public File file() { @@ -76,20 +88,6 @@ public class StatelessMapClient implements ChronicleMap { private final Bytes out; - public StatelessMapClient(Bytes out, Class kClass, Class vClass) { - - final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, - SerializationBuilder.Role.KEY); - final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, - SerializationBuilder.Role.VALUE); - - keySerializer = new Serializer(keyBuilder); - valueSerializer = new Serializer(valueBuilder); - - this.out = out; - } - - @Override public V putIfAbsent(K key, V value) { writeEvent(PUT_IF_ABSENT); @@ -125,7 +123,7 @@ public class StatelessMapClient implements ChronicleMap { } private void writeValue(V newValue) { - valueSerializer.writeMarshallable(out, newValue); + valueSerializer.writeMarshallable(newValue, out); } @@ -245,7 +243,7 @@ public class StatelessMapClient implements ChronicleMap { final HashMap safeCopy = new HashMap(map); - out.writeLong(safeCopy.size()); + out.writeStopBit(safeCopy.size()); final Set entries = (Set) safeCopy.entrySet(); @@ -274,10 +272,10 @@ public class StatelessMapClient implements ChronicleMap { } private Set readKeySet(Bytes in) { - int size = in.readInt(); + long size = in.readStopBit(); final HashSet result = new HashSet<>(); - for (int i = 0; i < size; i++) { + for (long i = 0; i < size; i++) { result.add(readKey(out)); } return result; @@ -293,11 +291,14 @@ public class StatelessMapClient implements ChronicleMap { writeEvent(VALUES); // get the data back from the server - Bytes in = blockingFetch(); + final Bytes in = blockingFetch(); - int size = in.readInt(); + final long size = in.readStopBit(); - ArrayList result = new ArrayList(); + if (size > Integer.MAX_VALUE) + throw new IllegalStateException("size=" + size + " is too large."); + + final ArrayList result = new ArrayList((int) size); for (int i = 0; i < size; i++) { result.add(readValue(in)); @@ -314,7 +315,7 @@ public class StatelessMapClient implements ChronicleMap { // get the data back from the server Bytes bytes = blockingFetch(); - int size = bytes.readInt(); + long size = bytes.readStopBit(); Set> result = new HashSet>(); @@ -322,7 +323,6 @@ public class StatelessMapClient implements ChronicleMap { K k = keySerializer.readMarshallable(bytes); V v = valueSerializer.readMarshallable(bytes); result.add(new Entry(k, v)); - } return result; @@ -335,7 +335,7 @@ public class StatelessMapClient implements ChronicleMap { * @param key the key of the map */ private void writeKey(K key) { - keySerializer.writeMarshallable(out, key); + keySerializer.writeMarshallable(key, out); } private void writeEvent(EventId event) { diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java new file mode 100644 index 000000000..f40669bbf --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -0,0 +1,265 @@ +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.model.constraints.NotNull; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author Rob Austin. + */ +public class StatelessServerConnector { + + + private final ChronicleMap map; + + + private final Serializer valueSerializer; + private final Serializer keySerializer; + + public StatelessServerConnector(Class kClass, Class vClass, ChronicleMap map) { + + this.map = map; + + final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, + SerializationBuilder.Role.KEY); + final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, + SerializationBuilder.Role.VALUE); + + keySerializer = new Serializer(keyBuilder); + valueSerializer = new Serializer(valueBuilder); + + } + + + public void marshall(@NotNull Bytes in, @NotNull Bytes out) { + + byte b = in.readByte(); + + StatelessMapClient.EventId eventId = StatelessMapClient.EventId.values()[b]; + + switch (eventId) { + + case LONG_SIZE: + longSize(in, out); + break; + + case IS_EMPTY: + isEmpty(in, out); + break; + + case CONTAINS_KEY: + containsKey(in, out); + break; + case CONTAINS_VALUE: + containsValue(in, out); + break; + case GET: + get(in, out); + break; + case PUT: + put(in, out); + break; + case REMOVE: + remove(in, out); + break; + case PUT_ALL: + putAll(in, out); + break; + case CLEAR: + clear(in, out); + break; + case KEY_SET: + keySet(in, out); + break; + case VALUES: + values(in, out); + break; + case ENTRY_SET: + entrySet(in, out); + break; + case REPLACE: + replace(in, out); + break; + case REPLACE_WITH_OLD_AND_NEW_VALUE: + replaceWithOldAndNew(in, out); + break; + case PUT_IF_ABSENT: + putIfAbsent(in, out); + break; + case REMOVE_WITH_VALUE: + removeWithValue(in, out); + break; + + } + + } + + private void removeWithValue(Bytes in, Bytes out) { + boolean result = map.remove(readKey(in), readValue(in)); + reflectTransactionId(in, out); + out.writeBoolean(result); + } + + private void replaceWithOldAndNew(Bytes in, Bytes out) { + final K key = readKey(in); + V oldValue = readValue(in); + V newValue = readValue(in); + reflectTransactionId(in, out); + map.replace(key, oldValue, newValue); + } + + + public void longSize(Bytes in, Bytes out) { + reflectTransactionId(in, out); + out.writeLong(map.longSize()); + } + + + public void size(Bytes in, Bytes out) { + reflectTransactionId(in, out); + out.writeInt(map.size()); + } + + + public void isEmpty(Bytes in, Bytes out) { + reflectTransactionId(in, out); + out.writeBoolean(map.isEmpty()); + } + + + public void containsKey(Bytes in, Bytes out) { + K k = readKey(in); + reflectTransactionId(in, out); + out.writeBoolean(map.containsKey(k)); + } + + + public void containsValue(Bytes in, Bytes out) { + V v = readValue(in); + reflectTransactionId(in, out); + out.writeBoolean(map.containsValue(v)); + } + + + public void get(Bytes in, Bytes out) { + K k = readKey(in); + reflectTransactionId(in, out); + writeValue(map.get(k), out); + } + + + public void put(Bytes in, Bytes out) { + K k = readKey(in); + V v = readValue(in); + reflectTransactionId(in, out); + writeValue(map.put(k, v), out); + } + + + public void remove(Bytes in, Bytes out) { + final V value = map.remove(readKey(in)); + reflectTransactionId(in, out); + writeValue(value, out); + } + + + public void putAll(Bytes in, Bytes out) { + map.putAll(readEntries(in)); + reflectTransactionId(in, out); + } + + private Map readEntries(Bytes in) { + + long size = in.readStopBit(); + final HashMap result = new HashMap(); + + for (long i = 0; i < size; i++) { + result.put(readKey(in), readValue(in)); + } + return result; + + } + + public void clear(Bytes in, Bytes out) { + map.clear(); + reflectTransactionId(in, out); + } + + + public void keySet(Bytes in, Bytes out) { + reflectTransactionId(in, out); + + final Set ks = map.keySet(); + out.writeStopBit(ks.size()); + for (K key : ks) { + writeKey(key, out); + } + } + + public void values(Bytes in, Bytes out) { + reflectTransactionId(in, out); + final Collection values = map.values(); + out.writeStopBit(values.size()); + for (final V value : values) { + writeValue(value, out); + } + } + + public void entrySet(Bytes in, Bytes out) { + reflectTransactionId(in, out); + + final Set> entries = map.entrySet(); + out.writeStopBit(entries.size()); + for (Map.Entry e : entries) { + writeKey(e.getKey(), out); + writeValue(e.getValue(), out); + } + } + + public void putIfAbsent(Bytes in, Bytes out) { + K key = readKey(in); + V v = readValue(in); + reflectTransactionId(in, out); + writeValue(map.putIfAbsent(key, v), out); + } + + + public void replace(Bytes in, Bytes out) { + K k = readKey(in); + V v = readValue(in); + + reflectTransactionId(in, out); + map.replace(k, v); + } + + + /** + * write the keysize and the key to the the {@code target} buffer + * + * @param key the key of the map + */ + private void writeKey(K key, Bytes out) { + keySerializer.writeMarshallable(key, out); + } + + private void reflectTransactionId(Bytes in, Bytes out) { + out.writeLong(in.readLong()); + } + + private void writeValue(V value, final Bytes out) { + valueSerializer.writeMarshallable(value, out); + } + + private K readKey(Bytes in) { + return keySerializer.readMarshallable(in); + } + + private V readValue(Bytes in) { + return valueSerializer.readMarshallable(in); + } + +} diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index eeea8b527..c5d52aa3e 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -59,7 +59,7 @@ public class SerializerTest { Serializer v = new Serializer(builder.valueBuilder); - v.writeMarshallable(out, value); + v.writeMarshallable(value, out); long position = out.position(); in.limit(position); @@ -83,7 +83,7 @@ public class SerializerTest { Serializer v = new Serializer(builder.keyBuilder); - v.writeMarshallable(out, key); + v.writeMarshallable(key, out); long position = out.position(); in.limit(position); From 808e14cf8d7d548ef6db75d47fda1bf9df0a3100 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:30:27 +0100 Subject: [PATCH 20/30] [maven-release-plugin] prepare release chronicle-map-2.0.3a --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5c1b49bfe..381ca0212 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.3a-SNAPSHOT + 2.0.3a OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - master + chronicle-map-2.0.3a From cb5f2e06931a161172b7008fdedf832df5cd5783 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:30:32 +0100 Subject: [PATCH 21/30] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 381ca0212..b5acc57f7 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.3a + 2.0.4a-SNAPSHOT OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - chronicle-map-2.0.3a + master From b7a7d5a3852d6f840e8e317b2ee96c7e7e4b5826 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:41:14 +0100 Subject: [PATCH 22/30] [maven-release-plugin] prepare release chronicle-map-2.0.4a --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b5acc57f7..8288d4918 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.4a-SNAPSHOT + 2.0.4a OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - master + chronicle-map-2.0.4a From 15db660e42d5143f94941f033103566c65578e89 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:41:18 +0100 Subject: [PATCH 23/30] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8288d4918..608f7d39a 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.4a + 2.0.5a-SNAPSHOT OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - chronicle-map-2.0.4a + master From 61c32a0490ca63012a9a26c87f8a17eeed66d2a7 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 18:43:39 +0100 Subject: [PATCH 24/30] updated to dependency> net.openhft affinity 2.1.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5c1b49bfe..badceb380 100755 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ net.openhft affinity - 2.1.0 + 2.1.2 From 6030b6338cd772603ee9d359fdac8d4b2561bf3c Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Thu, 16 Oct 2014 10:35:55 +0100 Subject: [PATCH 25/30] HCOLL-167 improved StatelessServerConnector, but its still in draft. --- .../net/openhft/chronicle/map/Serializer.java | 27 +++---- .../chronicle/map/StatelessMapClient.java | 78 +++++++----------- .../map/StatelessServerConnector.java | 79 ++++++++++++------- .../openhft/chronicle/map/SerializerTest.java | 18 +++-- 4 files changed, 104 insertions(+), 98 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index 00bcdacb6..d30bb388a 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -1,9 +1,6 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.map.serialization.BytesReader; -import net.openhft.chronicle.map.serialization.MetaBytesWriter; -import net.openhft.chronicle.map.serialization.MetaProvider; -import net.openhft.chronicle.map.serialization.SizeMarshaller; +import net.openhft.chronicle.map.serialization.*; import net.openhft.chronicle.map.threadlocal.Provider; import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; @@ -15,24 +12,24 @@ import net.openhft.lang.model.constraints.NotNull; * * @author Rob Austin. */ -public class Serializer> { +class Serializer { final SizeMarshaller sizeMarshaller; final BytesReader originalReader; transient Provider> readerProvider; - final VW originalWriter; - transient Provider writerProvider; - final MVW originalMetaValueWriter; - final MetaProvider metaValueWriterProvider; + final Object originalWriter; + transient Provider writerProvider; + final MetaBytesInterop originalMetaValueWriter; + final MetaProvider metaValueWriterProvider; - public Serializer(final SerializationBuilder serializationBuilder) { + Serializer(final SerializationBuilder serializationBuilder) { sizeMarshaller = serializationBuilder.sizeMarshaller(); - originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); - metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); + originalMetaValueWriter = serializationBuilder.metaInterop(); + metaValueWriterProvider = serializationBuilder.metaInteropProvider(); originalReader = serializationBuilder.reader(); - originalWriter = (VW) serializationBuilder.interop(); + originalWriter = serializationBuilder.interop(); readerProvider = Provider.of((Class) originalReader.getClass()); writerProvider = Provider.of((Class) originalWriter.getClass()); @@ -54,12 +51,12 @@ public class Serializer> { public void writeMarshallable(O value, @NotNull Bytes out) { ThreadLocalCopies copies = readerProvider.getCopies(null); - VW valueWriter = writerProvider.get(copies, originalWriter); + Object valueWriter = writerProvider.get(copies, originalWriter); copies = writerProvider.getCopies(copies); final long valueSize; - MetaBytesWriter metaValueWriter = null; + MetaBytesWriter metaValueWriter = null; if ((value instanceof Byteable)) { valueSize = ((Byteable) value).maxSize(); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 967a000a8..6e2fd77b4 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -18,7 +18,7 @@ import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; * * @author Rob Austin. */ -public class StatelessMapClient implements ChronicleMap { +class StatelessMapClient implements ChronicleMap { public static enum EventId { LONG_SIZE, SIZE, @@ -44,20 +44,11 @@ public class StatelessMapClient implements ChronicleMap { private final AtomicLong transactionID = new AtomicLong(); private long timeoutMs = TimeUnit.SECONDS.toMillis(20); - private final Serializer valueSerializer; - private final Serializer keySerializer; - public StatelessMapClient(Bytes out, Class kClass, Class vClass) { - - final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, - SerializationBuilder.Role.KEY); - final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, - SerializationBuilder.Role.VALUE); - - keySerializer = new Serializer(keyBuilder); - valueSerializer = new Serializer(valueBuilder); + public StatelessMapClient(Bytes out, KeyValueSerializer keyValueSerializer) { this.out = out; + this.keyValueSerializer = keyValueSerializer; } @Override @@ -86,6 +77,7 @@ public class StatelessMapClient implements ChronicleMap { private final Bytes out; + private final KeyValueSerializer keyValueSerializer; @Override @@ -94,8 +86,7 @@ public class StatelessMapClient implements ChronicleMap { writeKey(key); writeValue(value); - return readValue(blockingFetch()); - + return readKey(); } @@ -110,7 +101,6 @@ public class StatelessMapClient implements ChronicleMap { } - @Override public boolean replace(K key, V oldValue, V newValue) { writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); @@ -122,20 +112,19 @@ public class StatelessMapClient implements ChronicleMap { return blockingFetch().readBoolean(); } - private void writeValue(V newValue) { - valueSerializer.writeMarshallable(newValue, out); + private void writeKey(K key) { + keyValueSerializer.writeKey(key, out); } @Override public V replace(K key, V value) { writeEvent(REPLACE); - writeKey(key); writeValue(value); // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } @@ -189,13 +178,10 @@ public class StatelessMapClient implements ChronicleMap { writeEvent(GET); writeKey((K) key); // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } - V readValue(Bytes source) { - return valueSerializer.readMarshallable(source); - } @Override public V getUsing(K key, V value) { @@ -209,22 +195,30 @@ public class StatelessMapClient implements ChronicleMap { @Override public V put(K key, V value) { + writeEvent(PUT); writeKey(key); writeValue(value); - // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } + private V readKey() { + return keyValueSerializer.readValue(blockingFetch()); + } + + private void writeValue(V value) { + keyValueSerializer.writeValue(value, out); + } + @Override public V remove(Object key) { writeEvent(REMOVE); writeKey((K) key); // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } @@ -276,14 +270,11 @@ public class StatelessMapClient implements ChronicleMap { final HashSet result = new HashSet<>(); for (long i = 0; i < size; i++) { - result.add(readKey(out)); + result.add(keyValueSerializer.readKey(in)); } return result; } - private K readKey(Bytes bufferBytes) { - return keySerializer.readMarshallable(bufferBytes); - } @NotNull @Override @@ -291,21 +282,22 @@ public class StatelessMapClient implements ChronicleMap { writeEvent(VALUES); // get the data back from the server - final Bytes in = blockingFetch(); + final Bytes in = blockingFetch(); - final long size = in.readStopBit(); + final long size = in.readStopBit(); if (size > Integer.MAX_VALUE) throw new IllegalStateException("size=" + size + " is too large."); - final ArrayList result = new ArrayList((int) size); + final ArrayList result = new ArrayList((int) size); for (int i = 0; i < size; i++) { - result.add(readValue(in)); + result.add(keyValueSerializer.readValue(in)); } return result; } + @NotNull @Override public Set> entrySet() { @@ -313,15 +305,15 @@ public class StatelessMapClient implements ChronicleMap { // get the data back from the server - Bytes bytes = blockingFetch(); + Bytes in = blockingFetch(); - long size = bytes.readStopBit(); + long size = in.readStopBit(); Set> result = new HashSet>(); for (int i = 0; i < size; i++) { - K k = keySerializer.readMarshallable(bytes); - V v = valueSerializer.readMarshallable(bytes); + K k = keyValueSerializer.readKey(in); + V v = keyValueSerializer.readValue(in); result.add(new Entry(k, v)); } @@ -329,15 +321,6 @@ public class StatelessMapClient implements ChronicleMap { } - /** - * write the keysize and the key to the the {@code target} buffer - * - * @param key the key of the map - */ - private void writeKey(K key) { - keySerializer.writeMarshallable(key, out); - } - private void writeEvent(EventId event) { out.write((byte) event.ordinal()); } @@ -449,7 +432,6 @@ public class StatelessMapClient implements ChronicleMap { return getKey() + "=" + getValue(); } - } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java index f40669bbf..0069e56c7 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -11,27 +11,16 @@ import java.util.Set; /** * @author Rob Austin. */ -public class StatelessServerConnector { +class StatelessServerConnector { + private final KeyValueSerializer keyValueSerializer; private final ChronicleMap map; - private final Serializer valueSerializer; - private final Serializer keySerializer; - - public StatelessServerConnector(Class kClass, Class vClass, ChronicleMap map) { - + public StatelessServerConnector(KeyValueSerializer keyValueSerializer, ChronicleMap map) { + this.keyValueSerializer = keyValueSerializer; this.map = map; - - final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, - SerializationBuilder.Role.KEY); - final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, - SerializationBuilder.Role.VALUE); - - keySerializer = new Serializer(keyBuilder); - valueSerializer = new Serializer(valueBuilder); - } @@ -112,46 +101,39 @@ public class StatelessServerConnector { map.replace(key, oldValue, newValue); } - public void longSize(Bytes in, Bytes out) { reflectTransactionId(in, out); out.writeLong(map.longSize()); } - public void size(Bytes in, Bytes out) { reflectTransactionId(in, out); out.writeInt(map.size()); } - public void isEmpty(Bytes in, Bytes out) { reflectTransactionId(in, out); out.writeBoolean(map.isEmpty()); } - public void containsKey(Bytes in, Bytes out) { K k = readKey(in); reflectTransactionId(in, out); out.writeBoolean(map.containsKey(k)); } - public void containsValue(Bytes in, Bytes out) { V v = readValue(in); reflectTransactionId(in, out); out.writeBoolean(map.containsValue(v)); } - public void get(Bytes in, Bytes out) { K k = readKey(in); reflectTransactionId(in, out); writeValue(map.get(k), out); } - public void put(Bytes in, Bytes out) { K k = readKey(in); V v = readValue(in); @@ -159,14 +141,12 @@ public class StatelessServerConnector { writeValue(map.put(k, v), out); } - public void remove(Bytes in, Bytes out) { final V value = map.remove(readKey(in)); reflectTransactionId(in, out); writeValue(value, out); } - public void putAll(Bytes in, Bytes out) { map.putAll(readEntries(in)); reflectTransactionId(in, out); @@ -236,14 +216,13 @@ public class StatelessServerConnector { map.replace(k, v); } - /** * write the keysize and the key to the the {@code target} buffer * * @param key the key of the map */ private void writeKey(K key, Bytes out) { - keySerializer.writeMarshallable(key, out); + keyValueSerializer.writeKey(key, out); } private void reflectTransactionId(Bytes in, Bytes out) { @@ -251,15 +230,59 @@ public class StatelessServerConnector { } private void writeValue(V value, final Bytes out) { - valueSerializer.writeMarshallable(value, out); + keyValueSerializer.writeValue(value, out); } private K readKey(Bytes in) { - return keySerializer.readMarshallable(in); + return keyValueSerializer.readKey(in); } private V readValue(Bytes in) { + return keyValueSerializer.readValue(in); + } + +} + +class KeyValueSerializer { + + private final Serializer valueSerializer; + private final Serializer keySerializer; + + KeyValueSerializer(SerializationBuilder keySerializer, + SerializationBuilder valueSerializer) { + this.keySerializer = new Serializer(keySerializer); + this.valueSerializer = new Serializer(valueSerializer); + } + + V readValue(Bytes in) { + if (in.readBoolean()) + return null; return valueSerializer.readMarshallable(in); + } + + K readKey(Bytes in) { + if (in.readBoolean()) + return null; + + return keySerializer.readMarshallable(in); + } + + /** + * write the keysize and the key to the the {@code target} buffer + * + * @param key the key of the map + */ + void writeKey(K key, Bytes out) { + + out.writeBoolean(key == null); + if (key != null) + keySerializer.writeMarshallable(key, out); + } + + void writeValue(V value, Bytes out) { + out.writeBoolean(value == null); + if (value != null) + valueSerializer.writeMarshallable(value, out); } } diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index c5d52aa3e..f6f1a00df 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -79,17 +79,21 @@ public class SerializerTest { ChronicleMapBuilder builder = ChronicleMapBuilder.of(clazz, Integer.class); builder.preMapConstruction(); - - Serializer v = new Serializer(builder.keyBuilder); + { + Serializer v = new Serializer(builder.keyBuilder); - v.writeMarshallable(key, out); + v.writeMarshallable(key, out); + + long position = out.position(); + in.limit(position); + + Object actual = v.readMarshallable(in); + Assert.assertEquals(actual, key); + + } - long position = out.position(); - in.limit(position); - Object actual = v.readMarshallable(in); - Assert.assertEquals(actual, key); } public static class MyTestClassExternalizable implements Externalizable { From e6868ea233666f5ceb5488c6b5dc9f40c3f38a00 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Thu, 16 Oct 2014 20:45:05 +0100 Subject: [PATCH 26/30] HCOLL-167 first working draft of StatelessServerConnector --- .../map/AbstractChannelReplicator.java | 22 +- .../chronicle/map/ChannelProvider.java | 2 +- .../chronicle/map/ChannelProviderBuilder.java | 2 +- .../chronicle/map/ChronicleMapBuilder.java | 2 +- .../net/openhft/chronicle/map/Replicator.java | 3 +- .../openhft/chronicle/map/Replicators.java | 16 +- .../chronicle/map/StatelessMapClient.java | 308 +++++++++++------- .../map/StatelessServerConnector.java | 83 +++-- .../openhft/chronicle/map/TcpReplicator.java | 157 ++++++--- .../net/openhft/chronicle/map/Builder.java | 5 +- .../chronicle/map/StatelessClientTest.java | 49 +++ 11 files changed, 451 insertions(+), 198 deletions(-) create mode 100644 src/test/java/net/openhft/chronicle/map/StatelessClientTest.java diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java index 89e741a0b..1f9cfba3e 100644 --- a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java @@ -197,8 +197,9 @@ abstract class AbstractChannelReplicator implements Closeable { /** - * checks the number of bytes written in this interval, if this number of bytes exceeds a threshold, - * the selected will de-register the socket that is being written to, until the interval is finished. + * checks the number of bytes written in this interval, if this number of bytes exceeds a + * threshold, the selected will de-register the socket that is being written to, until the + * interval is finished. * * @throws ClosedChannelException */ @@ -261,12 +262,22 @@ abstract class AbstractChannelReplicator implements Closeable { @Override public boolean onEntry(final Bytes entry, final int chronicleId) { + + long pos0 = in.position(); + + // used to denote that this is not a stateless map event + in.writeByte(StatelessMapClient.EventId.STATEFUL_UPDATE.ordinal()); + + long sizeLocation = in.position(); + + // this is where we will store the size of the entry in.skip(SIZE_OF_SHORT); + final long start = in.position(); externalizable.writeExternalEntry(entry, in, chronicleId); if (in.position() == start) { - in.position(in.position() - SIZE_OF_SHORT); + in.position(pos0); return false; } @@ -276,7 +287,10 @@ abstract class AbstractChannelReplicator implements Closeable { if (entrySize > MAX_UNSIGNED_SHORT) throw new IllegalStateException("entry too large, the entry size=" + entrySize + ", " + "entries are limited to a size of " + MAX_UNSIGNED_SHORT); - in.writeUnsignedShort(start - SIZE_OF_SHORT, entrySize); + + if (LOG.isDebugEnabled()) + LOG.debug("sending entry of entrySize=" + entrySize); + in.writeUnsignedShort(sizeLocation, entrySize); return true; } diff --git a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java index c51a56e8c..a59519c97 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java @@ -407,7 +407,7 @@ public final class ChannelProvider implements Closeable { @Override protected Closeable applyTo(ChronicleMapBuilder builder, - Replica map, EntryExternalizable entryExternalizable) { + Replica map, EntryExternalizable entryExternalizable, final ChronicleMap chronicleMap) { if (builder.entrySize() > maxEntrySize) { throw new IllegalArgumentException("During ReplicatingChannelBuilder setup, " + "maxEntrySize=" + maxEntrySize + " was specified, but map with " + diff --git a/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java b/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java index 0bb05dab4..38726cbff 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java @@ -68,7 +68,7 @@ public final class ChannelProviderBuilder { replicatingCluster.asReplica, replicatingCluster.asEntryExternalizable, tcpReplicationConfig, - maxEntrySize); + maxEntrySize, null); replicatingCluster.add(tcpReplicator); } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 504d08b58..5dc965fa1 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -805,7 +805,7 @@ public class ChronicleMapBuilder implements Cloneable, if (map instanceof ReplicatedChronicleMap) { ReplicatedChronicleMap result = (ReplicatedChronicleMap) map; for (Replicator replicator : replicators.values()) { - Closeable token = replicator.applyTo(this, result, result); + Closeable token = replicator.applyTo(this, result, result, map); if (replicators.size() == 1 && token.getClass() == UdpReplicator.class) { LOG.warn( "MISSING TCP REPLICATION : The UdpReplicator only attempts to read data " + diff --git a/src/main/java/net/openhft/chronicle/map/Replicator.java b/src/main/java/net/openhft/chronicle/map/Replicator.java index 86ef2ed82..3dede414d 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicator.java +++ b/src/main/java/net/openhft/chronicle/map/Replicator.java @@ -41,6 +41,7 @@ public abstract class Replicator { * map configurations, not accessible via the map instance. * @param map a replicated map instance. Provides basic tools for replication implementation. * @param entryExternalizable the callback for ser/deser implementation in the replicator + * @param chronicleMap * @return a {@code Closeable} token to control replication resources. * It should be closed on closing the replicated map. * @throws java.io.IOException if an io error occurred during the replicator setup @@ -49,6 +50,6 @@ public abstract class Replicator { * applied to a map (or the specified number of maps) */ protected abstract Closeable applyTo(ChronicleMapBuilder builder, - Replica map, Replica.EntryExternalizable entryExternalizable) + Replica map, Replica.EntryExternalizable entryExternalizable, final ChronicleMap chronicleMap) throws IOException; } diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java index 023e8a7b2..aeb5118d3 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicators.java +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -30,10 +30,18 @@ final class Replicators { @Override protected Closeable applyTo(ChronicleMapBuilder builder, - Replica map, Replica.EntryExternalizable entryExternalizable) + Replica replica, Replica.EntryExternalizable entryExternalizable, + final ChronicleMap chronicleMap) throws IOException { - return new TcpReplicator(map, entryExternalizable, replicationConfig, - builder.entrySize()); + + final KeyValueSerializer keyValueSerializer = new KeyValueSerializer(builder + .keyBuilder, builder.valueBuilder); + + StatelessServerConnector statelessServerConnector = new StatelessServerConnector + (keyValueSerializer, chronicleMap); + + return new TcpReplicator(replica, entryExternalizable, replicationConfig, + builder.entrySize(), statelessServerConnector); } }; } @@ -44,7 +52,7 @@ final class Replicators { @Override protected Closeable applyTo(ChronicleMapBuilder builder, - Replica map, Replica.EntryExternalizable entryExternalizable) + Replica map, Replica.EntryExternalizable entryExternalizable, final ChronicleMap chronicleMap) throws IOException { return new UdpReplicator(map, entryExternalizable, replicationConfig, builder.entrySize()); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 6e2fd77b4..397a69ee3 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -3,9 +3,14 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -18,16 +23,23 @@ import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; * * @author Rob Austin. */ -class StatelessMapClient implements ChronicleMap { +class StatelessMapClient { + + private static final Logger LOG = LoggerFactory.getLogger(StatelessMapClient.class); + + + private final SocketChannel clientChannel; + public static final ByteBuffer STATELESS_CLIENT_IDENTIFER = ByteBuffer.wrap(new byte[]{-127}); public static enum EventId { + HEARTBEAT, + STATEFUL_UPDATE, LONG_SIZE, SIZE, IS_EMPTY, CONTAINS_KEY, CONTAINS_VALUE, GET, PUT, REMOVE, - PUT_ALL, CLEAR, KEY_SET, VALUES, @@ -46,23 +58,48 @@ class StatelessMapClient implements ChronicleMap { private long timeoutMs = TimeUnit.SECONDS.toMillis(20); - public StatelessMapClient(Bytes out, KeyValueSerializer keyValueSerializer) { - this.out = out; + public StatelessMapClient(final KeyValueSerializer keyValueSerializer, final InetSocketAddress remote) throws IOException { + this.keyValueSerializer = keyValueSerializer; + + clientChannel = SocketChannel.open(); + clientChannel.connect(remote); + + doHandShaking(); } - @Override + private void doHandShaking() throws IOException { + + + while (STATELESS_CLIENT_IDENTIFER.remaining() > 0) { + clientChannel.write(STATELESS_CLIENT_IDENTIFER); + } + + in.buffer().clear(); + + while (in.buffer().position() <= 0) { + clientChannel.read(in.buffer()); + } + + in.limit(in.buffer().position()); + in.position(0); + byte remoteIdentifier = in.readByte(); + + LOG.info("Attached to a map with a remote identifier=" + remoteIdentifier); + + } + + public File file() { return null; } - @Override public void close() throws IOException { throw new UnsupportedOperationException("This is not supported in the " + this.getClass() .getSimpleName()); } - long nextUniqueTransaction() { long time = System.currentTimeMillis(); @@ -76,160 +113,142 @@ class StatelessMapClient implements ChronicleMap { } - private final Bytes out; + private final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocate(1024)); + private final ByteBufferBytes in = new ByteBufferBytes(ByteBuffer.allocate(1024)); private final KeyValueSerializer keyValueSerializer; - @Override - public V putIfAbsent(K key, V value) { - writeEvent(PUT_IF_ABSENT); + public synchronized V putIfAbsent(K key, V value) throws IOException { + + long sizeLocation = writeEvent(PUT_IF_ABSENT); writeKey(key); writeValue(value); - - return readKey(); + return readKey(sizeLocation); } - @Override - public boolean remove(Object key, Object value) { - writeEvent(REMOVE_WITH_VALUE); + public synchronized boolean remove(Object key, Object value) throws IOException { + long sizeLocation = writeEvent(REMOVE_WITH_VALUE); writeKey((K) key); writeValue((V) value); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public boolean replace(K key, V oldValue, V newValue) { - writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); + + public synchronized boolean replace(K key, V oldValue, V newValue) throws IOException { + long sizeLocation = writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); writeKey(key); writeValue(oldValue); writeValue(newValue); // get the data back from the server - return blockingFetch().readBoolean(); - } - - private void writeKey(K key) { - keyValueSerializer.writeKey(key, out); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public V replace(K key, V value) { - writeEvent(REPLACE); + public synchronized V replace(K key, V value) throws IOException { + long sizeLocation = writeEvent(REPLACE); writeKey(key); writeValue(value); // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - @Override - public int size() { + public synchronized int size() throws IOException { - writeEvent(SIZE); + long sizeLocation = writeEvent(SIZE); // get the data back from the server - return blockingFetch().readInt(); + return blockingFetch(sizeLocation).readInt(); } - @Override - public boolean isEmpty() { - writeEvent(IS_EMPTY); + public synchronized boolean isEmpty() throws IOException { + long sizeLocation = writeEvent(IS_EMPTY); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public boolean containsKey(Object key) { - writeEvent(CONTAINS_KEY); + + public synchronized boolean containsKey(Object key) throws IOException { + long sizeLocation = writeEvent(CONTAINS_KEY); writeKey((K) key); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public boolean containsValue(Object value) { - writeEvent(CONTAINS_VALUE); + + public synchronized boolean containsValue(Object value) throws IOException { + long sizeLocation = writeEvent(CONTAINS_VALUE); writeValue((V) value); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public long longSize() { - writeEvent(LONG_SIZE); + public synchronized long longSize() throws IOException { + long sizeLocation = writeEvent(LONG_SIZE); // get the data back from the server - return blockingFetch().readLong(); + return blockingFetch(sizeLocation).readLong(); } - @Override - public V get(Object key) { - writeEvent(GET); + + public synchronized V get(Object key) throws IOException { + long sizeLocation = writeEvent(GET); writeKey((K) key); + // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - @Override - public V getUsing(K key, V value) { + public synchronized V getUsing(K key, V value) { throw new UnsupportedOperationException("getUsing is not supported for stateless clients"); } - @Override - public V acquireUsing(K key, V value) { + + public synchronized V acquireUsing(K key, V value) { throw new UnsupportedOperationException("getUsing is not supported for stateless clients"); } - @Override - public V put(K key, V value) { - writeEvent(PUT); + public synchronized V put(K key, V value) throws IOException { + + long sizeLocation = writeEvent(PUT); writeKey(key); writeValue(value); // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - private V readKey() { - return keyValueSerializer.readValue(blockingFetch()); - } - private void writeValue(V value) { - keyValueSerializer.writeValue(value, out); - } - - @Override - public V remove(Object key) { - writeEvent(REMOVE); + public synchronized V remove(Object key) throws IOException { + long sizeLocation = writeEvent(REMOVE); writeKey((K) key); + // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - @Override - public void putAll(Map map) { - writeEvent(PUT_ALL); - writeEntries(map); + public synchronized void putAll(Map map) throws IOException { - // get the data back from the server - blockingFetch(); + for (Map.Entry e : map.entrySet()) { + put(e.getKey(), e.getValue()); + } } @@ -247,42 +266,32 @@ class StatelessMapClient implements ChronicleMap { } } - @Override - public void clear() { - writeEvent(CLEAR); + + public synchronized void clear() throws IOException { + long sizeLocation = writeEvent(CLEAR); // get the data back from the server - blockingFetch(); + blockingFetch(sizeLocation); } @NotNull - @Override - public Set keySet() { - writeEvent(KEY_SET); + + public synchronized Set keySet() throws IOException { + long sizeLocation = writeEvent(KEY_SET); // get the data back from the server - return readKeySet(blockingFetch()); - } - - private Set readKeySet(Bytes in) { - long size = in.readStopBit(); - final HashSet result = new HashSet<>(); - - for (long i = 0; i < size; i++) { - result.add(keyValueSerializer.readKey(in)); - } - return result; + return readKeySet(blockingFetch(sizeLocation)); } @NotNull - @Override - public Collection values() { - writeEvent(VALUES); + + public synchronized Collection values() throws IOException { + long sizeLocation = writeEvent(VALUES); // get the data back from the server - final Bytes in = blockingFetch(); + final Bytes in = blockingFetch(sizeLocation); final long size = in.readStopBit(); @@ -299,13 +308,12 @@ class StatelessMapClient implements ChronicleMap { @NotNull - @Override - public Set> entrySet() { - writeEvent(ENTRY_SET); + public synchronized Set> entrySet() throws IOException { + long sizeLocation = writeEvent(ENTRY_SET); // get the data back from the server - Bytes in = blockingFetch(); + Bytes in = blockingFetch(sizeLocation); long size = in.readStopBit(); @@ -321,8 +329,13 @@ class StatelessMapClient implements ChronicleMap { } - private void writeEvent(EventId event) { + private long writeEvent(EventId event) { + out.clear(); + out.buffer().clear(); + out.write((byte) event.ordinal()); + long sizeLocation = markSizeLocation(); + return sizeLocation; } public Buffer buffer() { @@ -337,12 +350,12 @@ class StatelessMapClient implements ChronicleMap { volatile ByteBufferBytes buffer = null; - @Override + public void set(ByteBufferBytes source) { buffer = source; } - @Override + public ByteBufferBytes get() { return buffer; } @@ -359,7 +372,7 @@ class StatelessMapClient implements ChronicleMap { * @param transactionId * @return */ - private Bytes blockingFetch(long transactionId) { + private Bytes blockingFetchWithTransaction(long transactionId) { final Buffer buffer = buffer(); buffer.set(null); @@ -404,7 +417,11 @@ class StatelessMapClient implements ChronicleMap { public final V setValue(V newValue) { V oldValue = value; - StatelessMapClient.this.put((K) getKey(), (V) newValue); + try { + StatelessMapClient.this.put((K) getKey(), (V) newValue); + } catch (IOException e) { + LOG.error("", e); + } return oldValue; } @@ -431,15 +448,80 @@ class StatelessMapClient implements ChronicleMap { public final String toString() { return getKey() + "=" + getValue(); } - } - - private Bytes blockingFetch() { + private Bytes blockingFetch(long sizeLocation) throws IOException { long transactionId = nextUniqueTransaction(); + + LOG.info("sending data with transactionId=" + transactionId); out.writeLong(transactionId); - return blockingFetch(transactionId); + writeSizeAt(sizeLocation); + + + out.buffer().limit((int) out.position()); + while (out.buffer().remaining() > 0) { + int write = clientChannel.write(out.buffer()); + System.out.println(write); + } + + in.clear(); + in.buffer().clear(); + + // read size + while (in.buffer().position() < 2) { + clientChannel.read(in.buffer()); + } + + int size = in.readUnsignedShort(); + + while (in.buffer().position() < size + 2) { + clientChannel.read(in.buffer()); + } + + long inTransactionId = in.readLong(); + + if (inTransactionId != transactionId) { + throw new IllegalStateException("the received transaction-id=" + inTransactionId + + ", does not match the expected transaction-id=" + transactionId); + } + + return in; } + + private void writeSizeAt(long locationOfSize) { + long size = out.position() - locationOfSize; + out.writeUnsignedShort(locationOfSize, (int) size - 2); + } + + private long markSizeLocation() { + long position = out.position(); + out.skip(2); + return position; + } + + private void writeKey(K key) { + keyValueSerializer.writeKey(key, out); + } + + private V readKey(final long sizeLocation) throws IOException { + return keyValueSerializer.readValue(blockingFetch(sizeLocation)); + } + + private void writeValue(V value) { + keyValueSerializer.writeValue(value, out); + } + + + private Set readKeySet(Bytes in) { + long size = in.readStopBit(); + final HashSet result = new HashSet<>(); + + for (long i = 0; i < size; i++) { + result.add(keyValueSerializer.readKey(in)); + } + return result; + } + } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java index 0069e56c7..ca3f949e4 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -24,13 +24,11 @@ class StatelessServerConnector { } - public void marshall(@NotNull Bytes in, @NotNull Bytes out) { + public void processStatelessEvent(byte eventId, @NotNull Bytes in, @NotNull Bytes out) { - byte b = in.readByte(); - StatelessMapClient.EventId eventId = StatelessMapClient.EventId.values()[b]; - - switch (eventId) { + StatelessMapClient.EventId event = StatelessMapClient.EventId.values()[eventId]; + switch (event) { case LONG_SIZE: longSize(in, out); @@ -55,9 +53,6 @@ class StatelessServerConnector { case REMOVE: remove(in, out); break; - case PUT_ALL: - putAll(in, out); - break; case CLEAR: clear(in, out); break; @@ -83,73 +78,93 @@ class StatelessServerConnector { removeWithValue(in, out); break; + case SIZE: + size(in, out); + break; + + default: + throw new IllegalStateException("unsupported event=" + event); + } } private void removeWithValue(Bytes in, Bytes out) { boolean result = map.remove(readKey(in), readValue(in)); - reflectTransactionId(in, out); out.writeBoolean(result); } private void replaceWithOldAndNew(Bytes in, Bytes out) { + final K key = readKey(in); V oldValue = readValue(in); V newValue = readValue(in); - reflectTransactionId(in, out); + + long sizeLocation = reflectTransactionId(in, out); map.replace(key, oldValue, newValue); + + writeSizeAt(sizeLocation, out); } + public void longSize(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeLong(map.longSize()); + writeSizeAt(sizeLocation, out); } public void size(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeInt(map.size()); + writeSizeAt(sizeLocation, out); } public void isEmpty(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.isEmpty()); + writeSizeAt(sizeLocation, out); } public void containsKey(Bytes in, Bytes out) { K k = readKey(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.containsKey(k)); + writeSizeAt(sizeLocation, out); } public void containsValue(Bytes in, Bytes out) { V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.containsValue(v)); + writeSizeAt(sizeLocation, out); } public void get(Bytes in, Bytes out) { K k = readKey(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(map.get(k), out); + writeSizeAt(sizeLocation, out); } public void put(Bytes in, Bytes out) { K k = readKey(in); V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(map.put(k, v), out); + writeSizeAt(sizeLocation, out); } public void remove(Bytes in, Bytes out) { final V value = map.remove(readKey(in)); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(value, out); + writeSizeAt(sizeLocation, out); } public void putAll(Bytes in, Bytes out) { map.putAll(readEntries(in)); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); + writeSizeAt(sizeLocation, out); } private Map readEntries(Bytes in) { @@ -166,31 +181,34 @@ class StatelessServerConnector { public void clear(Bytes in, Bytes out) { map.clear(); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); + writeSizeAt(sizeLocation, out); } public void keySet(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); final Set ks = map.keySet(); out.writeStopBit(ks.size()); for (K key : ks) { writeKey(key, out); } + writeSizeAt(sizeLocation, out); } public void values(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); final Collection values = map.values(); out.writeStopBit(values.size()); for (final V value : values) { writeValue(value, out); } + writeSizeAt(sizeLocation, out); } public void entrySet(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); final Set> entries = map.entrySet(); out.writeStopBit(entries.size()); @@ -203,8 +221,9 @@ class StatelessServerConnector { public void putIfAbsent(Bytes in, Bytes out) { K key = readKey(in); V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(map.putIfAbsent(key, v), out); + writeSizeAt(sizeLocation, out); } @@ -212,8 +231,9 @@ class StatelessServerConnector { K k = readKey(in); V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); map.replace(k, v); + writeSizeAt(sizeLocation, out); } /** @@ -225,8 +245,12 @@ class StatelessServerConnector { keyValueSerializer.writeKey(key, out); } - private void reflectTransactionId(Bytes in, Bytes out) { - out.writeLong(in.readLong()); + private long reflectTransactionId(Bytes in, Bytes out) { + long sizeLocation = out.position(); + out.skip(2); + long transactionId = in.readLong(); + out.writeLong(transactionId); + return sizeLocation; } private void writeValue(V value, final Bytes out) { @@ -241,6 +265,10 @@ class StatelessServerConnector { return keyValueSerializer.readValue(in); } + private void writeSizeAt(long locationOfSize, Bytes out) { + long size = out.position() - locationOfSize; + out.writeUnsignedShort((int) size); + } } class KeyValueSerializer { @@ -285,4 +313,5 @@ class KeyValueSerializer { valueSerializer.writeMarshallable(value, out); } + } diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 61a013ce3..be8fd7b70 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -17,6 +17,7 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,10 +33,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static java.nio.channels.SelectionKey.*; +import static net.openhft.chronicle.map.StatelessMapClient.EventId.HEARTBEAT; /** - * Used with a {@see net.openhft.map.ReplicatedSharedHashMap} to send data between the maps using a socket - * connection

{@see net.openhft.map.OutSocketReplicator} + * Used with a {@see net.openhft.map.ReplicatedSharedHashMap} to send data between the maps using a + * socket connection

{@see net.openhft.map.OutSocketReplicator} * * @author Rob Austin. */ @@ -43,6 +45,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName()); private static final int BUFFER_SIZE = 0x100000; // 1MB + public static final int STATELESS_CLIENT = -127; + public static final byte NOT_SET = (byte) HEARTBEAT.ordinal(); private final SelectionKey[] selectionKeysStore = new SelectionKey[Byte.MAX_VALUE + 1]; // used to instruct the selector thread to set OP_WRITE on a key correlated by the bit index in the @@ -57,22 +61,28 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private final Replica.EntryExternalizable externalizable; private final TcpReplicationConfig replicationConfig; + private final StatelessServerConnector statelessServerConnector; + private long selectorTimeout; /** - * @param maxEntrySizeBytes used to check that the last entry will fit into the buffer, it can not be - * smaller than the size of and entry, if it is set smaller the buffer will over - * flow, it can be larger then the entry, but setting it too large reduces the - * workable space in the buffer. + * @param maxEntrySizeBytes used to check that the last entry will fit into the buffer, + * it can not be smaller than the size of and entry, if it is + * set smaller the buffer will over flow, it can be larger then + * the entry, but setting it too large reduces the workable + * space in the buffer. + * @param statelessServerConnector * @throws IOException */ TcpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable externalizable, @NotNull final TcpReplicationConfig replicationConfig, - final int maxEntrySizeBytes) throws IOException { + final int maxEntrySizeBytes, + StatelessServerConnector statelessServerConnector) throws IOException { super("TcpSocketReplicator-" + replica.identifier(), replicationConfig.throttlingConfig(), maxEntrySizeBytes); + this.statelessServerConnector = statelessServerConnector; final ThrottlingConfig throttlingConfig = replicationConfig.throttlingConfig(); long throttleBucketInterval = TimeUnit.MILLISECONDS.convert(throttlingConfig @@ -251,7 +261,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * check to see if we have lost connection with the remote node and if we have attempts a reconnect. + * check to see if we have lost connection with the remote node and if we have attempts a + * reconnect. * * @param key the key relating to the heartbeat that we are checking * @param approxTimeOutTime the approximate time in milliseconds @@ -374,8 +385,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear all the - * old bootstrap information + * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear + * all the old bootstrap information * * @param key the nio SelectionKey */ @@ -387,7 +398,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * used to exchange identifiers and timestamps and heartbeat intervals between the server and client + * used to exchange identifiers and timestamps and heartbeat intervals between the server and + * client * * @param key the SelectionKey relating to the this cha * @param socketChannel @@ -404,6 +416,11 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { final byte remoteIdentifier = reader.identifierFromBuffer(); + if (remoteIdentifier == STATELESS_CLIENT) { + attached.handShakingComplete = true; + return; + } + if (remoteIdentifier == Byte.MIN_VALUE) return; @@ -425,7 +442,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { final SocketAddress remoteAddress = socketChannel.getRemoteAddress(); - if ( (identifierListener != null && !identifierListener.isIdentifierUnique(remoteIdentifier, + if ((identifierListener != null && !identifierListener.isIdentifierUnique(remoteIdentifier, remoteAddress)) || remoteIdentifier == localIdentifier) // throwing this exception will cause us to disconnect, both the client and server @@ -477,7 +494,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { // now we're finished we can get on with reading the entries attached.handShakingComplete = true; attached.remoteModificationIterator.dirtyEntries(attached.remoteBootstrapTimestamp); - reader.entriesFromBuffer(); + reader.entriesFromBuffer(attached); } } @@ -531,17 +548,17 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { attached.entryReader.lastHeartBeatReceived = approxTime; if (attached.isHandShakingComplete()) { - attached.entryReader.entriesFromBuffer(); + attached.entryReader.entriesFromBuffer(attached); } else { doHandShaking(key, socketChannel); } } /** - * sets interestOps to "selector keys",The change to interestOps much be on the same thread as the - * selector. This class, allows via {@link AbstractChannelReplicator .KeyInterestUpdater#set(int)} to - * holds a pending change in interestOps ( via a bitset ), this change is processed later on the same - * thread as the selector + * sets interestOps to "selector keys",The change to interestOps much be on the same thread as + * the selector. This class, allows via {@link AbstractChannelReplicator + * .KeyInterestUpdater#set(int)} to holds a pending change in interestOps ( via a bitset ), + * this change is processed later on the same thread as the selector */ private static class KeyInterestUpdater { @@ -572,8 +589,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * @param keyIndex the index of the key that has changed, the list of keys is provided by the - * constructor {@link KeyInterestUpdater(int, SelectionKey[])} + * @param keyIndex the index of the key that has changed, the list of keys is provided by + * the constructor {@link KeyInterestUpdater(int, SelectionKey[])} */ public void set(int keyIndex) { changeOfOpWriteRequired.set(keyIndex); @@ -773,12 +790,19 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } + /** + * @return the buffer messages can be written to + */ + Bytes buffer() { + return in; + } + /** * writes the timestamp into the buffer * * @param localIdentifier the current nodes identifier */ - void identifierToBuffer(final int localIdentifier) { + void identifierToBuffer(final byte localIdentifier) { in.writeByte(localIdentifier); } @@ -792,7 +816,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * writes all the entries that have changed, to the buffer which will later be written to TCP/IP + * writes all the entries that have changed, to the buffer which will later be written to + * TCP/IP * * @param modificationIterator a record of which entries have modification * @param selectionKey @@ -804,7 +829,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { final SocketChannel socketChannel = (SocketChannel) selectionKey.channel(); final Attached attached = (Attached) selectionKey.attachment(); - // this can occur when new CHM's are added to a cluster + // this can occur when new map are added to a channel final boolean handShakingComplete = attached.isHandShakingComplete(); for (; ; ) { @@ -867,9 +892,15 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * used to send an single zero byte if we have not send any data for up to the localHeartbeatInterval + * used to send an single zero byte if we have not send any data for up to the + * localHeartbeatInterval */ private void writeHeartbeatToBuffer() { + + // denotes the state - 0 for a heartbeat + in.writeByte(HEARTBEAT.ordinal()); + + // denotes the size in bytes in.writeUnsignedShort(0); } @@ -879,8 +910,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { /** - * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will get - * added back in as soon as we have data to write + * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE + * will get added back in as soon as we have data to write * * @param socketChannel the socketChannel we wish to stop writing to * @param attached data associated with the socketChannels key @@ -914,8 +945,9 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private final ByteBuffer in; private final ByteBufferBytes out; public long lastHeartBeatReceived = System.currentTimeMillis(); - // we use Integer.MIN_VALUE as N/A - private int sizeOfNextEntry = Integer.MIN_VALUE; + + private int sizeInBytes; + private byte state; private TcpSocketChannelEntryReader() { in = ByteBuffer.allocateDirect(replicationConfig.packetSize() + maxEntrySizeBytes); @@ -943,45 +975,79 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { /** * reads entries from the buffer till empty * + * @param attached * @throws InterruptedException */ - private void entriesFromBuffer() throws InterruptedException, IOException { - + private void entriesFromBuffer(Attached attached) throws InterruptedException, IOException { for (; ; ) { out.limit(in.position()); // its set to MIN_VALUE when it should be read again - if (sizeOfNextEntry == Integer.MIN_VALUE) { - if (out.remaining() < SIZE_OF_SHORT) { + if (state == NOT_SET) { + if (out.remaining() < SIZE_OF_SHORT + 1 ) { return; } - int value = out.readUnsignedShort(); + // state is used for both heartbeat and stateless + state = out.readByte(); - // this is the heartbeat - if (value == 0) + + + sizeInBytes = out.readUnsignedShort(); + + if (state == 0 && sizeInBytes != 0) { + int i = 1; + } + + // this is the : + // -- heartbeat if its 0 + // -- stateful update if its 1 + // -- the id of the stateful event + if (state == NOT_SET) continue; - - sizeOfNextEntry = value; } - if (out.remaining() < sizeOfNextEntry) { + + if (out.remaining() < sizeInBytes) { return; } - final long nextEntryPos = out.position() + sizeOfNextEntry; + if (sizeInBytes == 0) { + int j = 1; + } + + + final long nextEntryPos = out.position() + sizeInBytes; final long limit = out.limit(); out.limit(nextEntryPos); - externalizable.readExternalEntry(out); + + boolean isStateless = (state != 1); + + if (isStateless) { + if (statelessServerConnector == null) { + + LOG.error("", new IllegalArgumentException("received an event " + + "from a stateless map, stateless maps are not " + + "currently supported when using Chronicle Channels")); + } else + + statelessServerConnector.processStatelessEvent(state, + out, attached.entryWriter.buffer()); + // to allow the sizeInBytes to be read the next time around + + + } else + externalizable.readExternalEntry(out); out.limit(limit); // skip onto the next entry out.position(nextEntryPos); - // to allow the sizeOfNextEntry to be read the next time around - sizeOfNextEntry = Integer.MIN_VALUE; + + state = NOT_SET; + sizeInBytes = 0; } } @@ -1016,7 +1082,12 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { * @return the timestamp or -1 if unsuccessful */ long remoteBootstrapTimestamp() { - return (out.remaining() >= 8) ? out.readLong() : Long.MIN_VALUE; + if (out.remaining() >= 8) + return out.readLong(); + else + return Long.MIN_VALUE; + + // return (out.remaining() >= 8) ? out.readLong() : Long.MIN_VALUE; } public long remoteHeartbeatIntervalFromBuffer() { diff --git a/src/test/java/net/openhft/chronicle/map/Builder.java b/src/test/java/net/openhft/chronicle/map/Builder.java index 4c1abddc9..ae60b1915 100644 --- a/src/test/java/net/openhft/chronicle/map/Builder.java +++ b/src/test/java/net/openhft/chronicle/map/Builder.java @@ -21,8 +21,7 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.net.InetSocketAddress; - -import static java.util.concurrent.TimeUnit.SECONDS; +import java.util.concurrent.TimeUnit; /** * @author Rob Austin. @@ -69,7 +68,7 @@ public class Builder { final int serverPort, final InetSocketAddress... endpoints) throws IOException { TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints) - .heartBeatInterval(1L, SECONDS); + .heartBeatInterval(1L, TimeUnit.SECONDS); return ChronicleMapBuilder.of(kClass, vClass) .entries(20000L) .replicators(identifier, tcpConfig); diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java new file mode 100644 index 000000000..da229954a --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -0,0 +1,49 @@ +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import org.junit.Test; +import org.testng.Assert; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static net.openhft.chronicle.map.TcpReplicationConfig.of; + +/** + * @author Rob Austin. + */ +public class StatelessClientTest { + + + @Test + public void test() throws IOException, InterruptedException { + + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000L) + .replicators((byte) 2, of(8076).heartBeatInterval(1L, SECONDS)); + { + final ChronicleMap map2a = + builder.create(getPersistenceFile()); + map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 + } + + KeyValueSerializer keyValueSerializer = new KeyValueSerializer(builder.keyBuilder, builder.valueBuilder); + + final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + // ByteBufferBytes in = out.slice(); + + InetSocketAddress remote = new InetSocketAddress("localhost", 8076); + StatelessMapClient map = new StatelessMapClient(keyValueSerializer, remote); + + + Assert.assertEquals("EXAMPLE-10", map.get(10)); + Assert.assertEquals("EXAMPLE-10", map.get(10)); + + Assert.assertEquals(1, map.size()); + } +} + From adfec1b61352af03ad3fbb2a2651ca0428d1771b Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Fri, 17 Oct 2014 09:12:25 +0100 Subject: [PATCH 27/30] Give more conservative head room for avoid Map's with an imperfect hash distribution filling up. --- .../net/openhft/chronicle/map/ChronicleMapBuilder.java | 7 ++++++- .../map/serialization/CopyingMetaBytesInterop.java | 8 ++++---- .../java/net/openhft/chronicle/map/OSResizesMain.java | 9 ++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index a5e8cf385..71c83fe3f 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -385,8 +385,13 @@ public class ChronicleMapBuilder implements Cloneable, if (actualEntriesPerSegment > 0) return actualEntriesPerSegment; int as = actualSegments(); + long actualEntries = + as == 1 ? entries + 1 : // The extra 1 might not be needed. + as <= 4 ? entries * 10L / 9 : // 10%, assumes you are trying to be tight. + entries * 5L / 4; // 25% + // round up to the next multiple of 64. - return (int) (Math.max(1, entries * 2L / as) + 63) & ~63; + return (int) (Math.max(1, actualEntries / as) + 63) & ~63; } @Override diff --git a/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java b/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java index a8fc3d368..79e6111e3 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java @@ -16,14 +16,14 @@ package net.openhft.chronicle.map.serialization; +import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.threadlocal.StatefulCopyable; +import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.DirectBytes; import net.openhft.lang.io.DirectStore; import net.openhft.lang.io.serialization.BytesMarshaller; import net.openhft.lang.io.serialization.JDKObjectSerializer; -import net.openhft.chronicle.map.threadlocal.Provider; -import net.openhft.chronicle.map.threadlocal.StatefulCopyable; -import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import java.io.IOException; import java.io.ObjectInputStream; @@ -190,7 +190,7 @@ public abstract class CopyingMetaBytesInterop implements MetaBytesInterop< forBytesWriter = new ForBytesWriter(); } - private Bytes obtain(long maxSize) { + Bytes obtain(long maxSize) { DirectBytes buf; if ((buf = buffer) != null) { if (maxSize <= buf.capacity()) { diff --git a/src/test/java/net/openhft/chronicle/map/OSResizesMain.java b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java index 3792462b5..34f8907a7 100644 --- a/src/test/java/net/openhft/chronicle/map/OSResizesMain.java +++ b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java @@ -34,9 +34,8 @@ public class OSResizesMain { public static void main(String[] args) throws IOException, InterruptedException { File file = File.createTempFile("over-sized", "deleteme"); ChronicleMap map = ChronicleMapBuilder.of(String.class, String.class) - - .entrySize(1024 * 1024) - .entries(1024 * 1024) + .entrySize(100 * 1024 * 1024) + .entries(1000000) .create(file); for (int i = 0; i < 10000; i++) { char[] chars = new char[i]; @@ -48,6 +47,10 @@ public class OSResizesMain { file.length() / 1e9, run("du", "-h", file.getAbsolutePath()).split("\\s")[0], run("grep", "over-sized", "/proc/" + Jvm.getProcessId() + "/maps").split("\\s")[0]); + // show up in top. + long time = System.currentTimeMillis(); + while (time + 20000 > System.currentTimeMillis()) + Thread.yield(); map.close(); file.delete(); } From 275684b65a305f65f86553b0f280dd254e87e60f Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Fri, 17 Oct 2014 15:35:27 +0100 Subject: [PATCH 28/30] HCOLL-167 added isException flag --- .../chronicle/map/StatelessMapClient.java | 4 +- .../map/StatelessServerConnector.java | 37 ++++++++++--------- .../chronicle/map/StatelessClientTest.java | 4 +- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 397a69ee3..f74ac48a7 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -474,10 +474,12 @@ class StatelessMapClient { int size = in.readUnsignedShort(); - while (in.buffer().position() < size + 2) { + while (in.buffer().position() < size) { clientChannel.read(in.buffer()); } + boolean isException = in.readBoolean(); + long inTransactionId = in.readLong(); if (inTransactionId != transactionId) { diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java index ca3f949e4..497c85a64 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -103,47 +103,47 @@ class StatelessServerConnector { long sizeLocation = reflectTransactionId(in, out); map.replace(key, oldValue, newValue); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void longSize(Bytes in, Bytes out) { long sizeLocation = reflectTransactionId(in, out); out.writeLong(map.longSize()); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void size(Bytes in, Bytes out) { long sizeLocation = reflectTransactionId(in, out); out.writeInt(map.size()); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void isEmpty(Bytes in, Bytes out) { long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.isEmpty()); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void containsKey(Bytes in, Bytes out) { K k = readKey(in); long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.containsKey(k)); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void containsValue(Bytes in, Bytes out) { V v = readValue(in); long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.containsValue(v)); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void get(Bytes in, Bytes out) { K k = readKey(in); long sizeLocation = reflectTransactionId(in, out); writeValue(map.get(k), out); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void put(Bytes in, Bytes out) { @@ -151,20 +151,20 @@ class StatelessServerConnector { V v = readValue(in); long sizeLocation = reflectTransactionId(in, out); writeValue(map.put(k, v), out); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void remove(Bytes in, Bytes out) { final V value = map.remove(readKey(in)); long sizeLocation = reflectTransactionId(in, out); writeValue(value, out); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void putAll(Bytes in, Bytes out) { map.putAll(readEntries(in)); long sizeLocation = reflectTransactionId(in, out); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } private Map readEntries(Bytes in) { @@ -182,7 +182,7 @@ class StatelessServerConnector { public void clear(Bytes in, Bytes out) { map.clear(); long sizeLocation = reflectTransactionId(in, out); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } @@ -194,7 +194,7 @@ class StatelessServerConnector { for (K key : ks) { writeKey(key, out); } - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void values(Bytes in, Bytes out) { @@ -204,7 +204,7 @@ class StatelessServerConnector { for (final V value : values) { writeValue(value, out); } - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } public void entrySet(Bytes in, Bytes out) { @@ -223,7 +223,7 @@ class StatelessServerConnector { V v = readValue(in); long sizeLocation = reflectTransactionId(in, out); writeValue(map.putIfAbsent(key, v), out); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } @@ -233,7 +233,7 @@ class StatelessServerConnector { long sizeLocation = reflectTransactionId(in, out); map.replace(k, v); - writeSizeAt(sizeLocation, out); + writeSizeAndFlags(sizeLocation, false, out); } /** @@ -247,7 +247,7 @@ class StatelessServerConnector { private long reflectTransactionId(Bytes in, Bytes out) { long sizeLocation = out.position(); - out.skip(2); + out.skip(3); long transactionId = in.readLong(); out.writeLong(transactionId); return sizeLocation; @@ -265,9 +265,10 @@ class StatelessServerConnector { return keyValueSerializer.readValue(in); } - private void writeSizeAt(long locationOfSize, Bytes out) { + private void writeSizeAndFlags(long locationOfSize, boolean isException, Bytes out) { long size = out.position() - locationOfSize; - out.writeUnsignedShort((int) size); + out.writeUnsignedShort(0L, (int) size); + out.writeBoolean(2L, isException); } } diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index da229954a..87b2b6c7a 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -41,9 +41,9 @@ public class StatelessClientTest { Assert.assertEquals("EXAMPLE-10", map.get(10)); - Assert.assertEquals("EXAMPLE-10", map.get(10)); + // Assert.assertEquals("EXAMPLE-10", map.get(10)); - Assert.assertEquals(1, map.size()); + // Assert.assertEquals(1, map.size()); } } From 3110684e12927cafb1a90a2842223fd674032ab8 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Fri, 17 Oct 2014 16:39:25 +0100 Subject: [PATCH 29/30] HCOLL-167 improved passing errors back --- .../chronicle/map/StatelessMapClient.java | 82 +++++++++++++------ .../map/StatelessServerConnector.java | 28 ++++++- .../chronicle/map/StatelessClientTest.java | 4 +- 3 files changed, 82 insertions(+), 32 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index f74ac48a7..993441d5c 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -23,7 +23,7 @@ import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; * * @author Rob Austin. */ -class StatelessMapClient { +class StatelessMapClient implements ChronicleMap { private static final Logger LOG = LoggerFactory.getLogger(StatelessMapClient.class); @@ -95,7 +95,7 @@ class StatelessMapClient { return null; } - public void close() throws IOException { + public void close() { throw new UnsupportedOperationException("This is not supported in the " + this.getClass() .getSimpleName()); } @@ -118,7 +118,7 @@ class StatelessMapClient { private final KeyValueSerializer keyValueSerializer; - public synchronized V putIfAbsent(K key, V value) throws IOException { + public synchronized V putIfAbsent(K key, V value) { long sizeLocation = writeEvent(PUT_IF_ABSENT); writeKey(key); @@ -127,7 +127,7 @@ class StatelessMapClient { } - public synchronized boolean remove(Object key, Object value) throws IOException { + public synchronized boolean remove(Object key, Object value) { long sizeLocation = writeEvent(REMOVE_WITH_VALUE); writeKey((K) key); writeValue((V) value); @@ -138,7 +138,7 @@ class StatelessMapClient { } - public synchronized boolean replace(K key, V oldValue, V newValue) throws IOException { + public synchronized boolean replace(K key, V oldValue, V newValue) { long sizeLocation = writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); writeKey(key); writeValue(oldValue); @@ -149,7 +149,7 @@ class StatelessMapClient { } - public synchronized V replace(K key, V value) throws IOException { + public synchronized V replace(K key, V value) { long sizeLocation = writeEvent(REPLACE); writeKey(key); writeValue(value); @@ -159,7 +159,7 @@ class StatelessMapClient { } - public synchronized int size() throws IOException { + public synchronized int size() { long sizeLocation = writeEvent(SIZE); @@ -168,7 +168,7 @@ class StatelessMapClient { } - public synchronized boolean isEmpty() throws IOException { + public synchronized boolean isEmpty() { long sizeLocation = writeEvent(IS_EMPTY); // get the data back from the server @@ -176,7 +176,7 @@ class StatelessMapClient { } - public synchronized boolean containsKey(Object key) throws IOException { + public synchronized boolean containsKey(Object key) { long sizeLocation = writeEvent(CONTAINS_KEY); writeKey((K) key); @@ -186,7 +186,7 @@ class StatelessMapClient { } - public synchronized boolean containsValue(Object value) throws IOException { + public synchronized boolean containsValue(Object value) { long sizeLocation = writeEvent(CONTAINS_VALUE); writeValue((V) value); @@ -195,14 +195,14 @@ class StatelessMapClient { } - public synchronized long longSize() throws IOException { + public synchronized long longSize() { long sizeLocation = writeEvent(LONG_SIZE); // get the data back from the server return blockingFetch(sizeLocation).readLong(); } - public synchronized V get(Object key) throws IOException { + public synchronized V get(Object key) { long sizeLocation = writeEvent(GET); writeKey((K) key); @@ -222,19 +222,21 @@ class StatelessMapClient { } - public synchronized V put(K key, V value) throws IOException { + public synchronized V put(K key, V value) { long sizeLocation = writeEvent(PUT); writeKey(key); writeValue(value); // get the data back from the server + return readKey(sizeLocation); + } - public synchronized V remove(Object key) throws IOException { + public synchronized V remove(Object key) { long sizeLocation = writeEvent(REMOVE); writeKey((K) key); @@ -244,7 +246,7 @@ class StatelessMapClient { } - public synchronized void putAll(Map map) throws IOException { + public synchronized void putAll(Map map) { for (Map.Entry e : map.entrySet()) { put(e.getKey(), e.getValue()); @@ -267,17 +269,19 @@ class StatelessMapClient { } - public synchronized void clear() throws IOException { + public synchronized void clear() { long sizeLocation = writeEvent(CLEAR); // get the data back from the server + blockingFetch(sizeLocation); + } @NotNull - public synchronized Set keySet() throws IOException { + public synchronized Set keySet() { long sizeLocation = writeEvent(KEY_SET); // get the data back from the server @@ -287,7 +291,7 @@ class StatelessMapClient { @NotNull - public synchronized Collection values() throws IOException { + public synchronized Collection values() { long sizeLocation = writeEvent(VALUES); // get the data back from the server @@ -309,7 +313,7 @@ class StatelessMapClient { @NotNull - public synchronized Set> entrySet() throws IOException { + public synchronized Set> entrySet() { long sizeLocation = writeEvent(ENTRY_SET); // get the data back from the server @@ -417,11 +421,9 @@ class StatelessMapClient { public final V setValue(V newValue) { V oldValue = value; - try { - StatelessMapClient.this.put((K) getKey(), (V) newValue); - } catch (IOException e) { - LOG.error("", e); - } + + StatelessMapClient.this.put((K) getKey(), (V) newValue); + return oldValue; } @@ -450,7 +452,17 @@ class StatelessMapClient { } } - private Bytes blockingFetch(long sizeLocation) throws IOException { + private Bytes blockingFetch(long sizeLocation) { + try { + return blockingFetchThrowable(sizeLocation); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + + } + + + private Bytes blockingFetchThrowable(long sizeLocation) throws IOException { long transactionId = nextUniqueTransaction(); LOG.info("sending data with transactionId=" + transactionId); @@ -487,6 +499,17 @@ class StatelessMapClient { ", does not match the expected transaction-id=" + transactionId); } + if (isException) { + int len = in.readUnsignedShort(); + + in.limit(in.position() + len); + + Object o = in.readObject(); + RuntimeException runtimeException = (RuntimeException) o; + throw runtimeException; + } + + return in; } @@ -505,7 +528,7 @@ class StatelessMapClient { keyValueSerializer.writeKey(key, out); } - private V readKey(final long sizeLocation) throws IOException { + private V readKey(final long sizeLocation) { return keyValueSerializer.readValue(blockingFetch(sizeLocation)); } @@ -533,3 +556,10 @@ interface Buffer { ByteBufferBytes get(); } + +class RuntimeIOException extends RuntimeException { + + public RuntimeIOException(IOException e) { + super(e); + } +} diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java index 497c85a64..71289cd32 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -101,8 +101,13 @@ class StatelessServerConnector { V newValue = readValue(in); long sizeLocation = reflectTransactionId(in, out); - map.replace(key, oldValue, newValue); - + try { + map.replace(key, oldValue, newValue); + } catch (RuntimeException e) { + writeException(e, out); + writeSizeAndFlags(sizeLocation, true, out); + return; + } writeSizeAndFlags(sizeLocation, false, out); } @@ -142,8 +147,25 @@ class StatelessServerConnector { public void get(Bytes in, Bytes out) { K k = readKey(in); long sizeLocation = reflectTransactionId(in, out); - writeValue(map.get(k), out); + try { + writeValue(map.get(k), out); + } catch (RuntimeException e) { + writeException(e, out); + writeSizeAndFlags(sizeLocation, true, out); + return; + } writeSizeAndFlags(sizeLocation, false, out); + + } + + private void writeException(RuntimeException e, Bytes out) { + + long start = out.position(); + out.skip(2); + out.writeObject(e); + long len = out.position() - (start + 2L); + out.writeUnsignedShort(start,(int) len); + } public void put(Bytes in, Bytes out) { diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index 87b2b6c7a..9601e9032 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -41,9 +41,7 @@ public class StatelessClientTest { Assert.assertEquals("EXAMPLE-10", map.get(10)); - // Assert.assertEquals("EXAMPLE-10", map.get(10)); - - // Assert.assertEquals(1, map.size()); + Assert.assertEquals(1, map.size()); } } From 15ee7367b3290e88ac1dd85d54e6e509b6b2ff04 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Fri, 17 Oct 2014 16:43:26 +0100 Subject: [PATCH 30/30] HCOLL-167 simplified test --- .../java/net/openhft/chronicle/map/StatelessClientTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index 9601e9032..0622ab6ae 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -1,12 +1,10 @@ package net.openhft.chronicle.map; -import net.openhft.lang.io.ByteBufferBytes; import org.junit.Test; import org.testng.Assert; import java.io.IOException; import java.net.InetSocketAddress; -import java.nio.ByteBuffer; import static java.util.concurrent.TimeUnit.SECONDS; import static net.openhft.chronicle.map.Builder.getPersistenceFile; @@ -33,13 +31,9 @@ public class StatelessClientTest { KeyValueSerializer keyValueSerializer = new KeyValueSerializer(builder.keyBuilder, builder.valueBuilder); - final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); - // ByteBufferBytes in = out.slice(); - InetSocketAddress remote = new InetSocketAddress("localhost", 8076); StatelessMapClient map = new StatelessMapClient(keyValueSerializer, remote); - Assert.assertEquals("EXAMPLE-10", map.get(10)); Assert.assertEquals(1, map.size()); }