From 200322d268e78227db803dbec75bdc5921a625be Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:23:24 +0100 Subject: [PATCH 01/64] HCOLL-163 renamed chronicle custer to chronicle channel --- README.md | 37 ++++++++++--------- ...ingChannel.java => ChannelReplicator.java} | 8 ++-- ...der.java => ChannelReplicatorBuilder.java} | 14 +++---- .../chronicle/map/ChronicleMapBuilder.java | 6 +-- ...nTest.java => ChannelReplicationTest.java} | 23 ++++++------ .../map/PostClusterMapCreationTest.java | 21 +++++------ .../chronicle/map/SingleMapClusterTest.java | 15 ++++---- 7 files changed, 62 insertions(+), 62 deletions(-) rename src/main/java/net/openhft/chronicle/map/{ReplicatingChannel.java => ChannelReplicator.java} (98%) rename src/main/java/net/openhft/chronicle/map/{ReplicatingChannelBuilder.java => ChannelReplicatorBuilder.java} (80%) rename src/test/java/net/openhft/chronicle/map/{ClusterReplicationTest.java => ChannelReplicationTest.java} (81%) diff --git a/README.md b/README.md index bcbbbb674..8fee78d4e 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Identifier](https://github.com/OpenHFT/Chronicle-Map#identifier) * [Port](https://github.com/OpenHFT/Chronicle-Map#port) * [Heart Beat Interval](https://github.com/OpenHFT/Chronicle-Map#heart-beat-interval) -* [Clustering](https://github.com/OpenHFT/Chronicle-Map#cluster) +* [Channels](https://github.com/OpenHFT/Chronicle-Map#channels) #### Miscellaneous @@ -512,7 +512,7 @@ A heartbeat will only be send if no data is transmitted, if the maps are constan no heartbeat message is sent. If a map does not receive either data of a heartbeat the connection is dropped and re-established. -# Cluster +# Channels Chronicle Map TCP Replication lets you distribute a single Chronicle Map, to a number of servers across your network. Replication is point to point and the data transfer is bidirectional, so in the @@ -523,34 +523,37 @@ unfortunately with just TCP replication you would have to have two tcp socket co not ideal. This is why we created Chronicle Clustering. Clustering lets you replicate numerous Chronicle Maps via a single point to point socket connection. -Clustering is similar to TCP replication, where each map has to be given a unique identifier, but -when using Chronicle Clustering its the cluster that is given the unique identifier not the map. +Chronicle Channels are similar to TCP replication, where each map has to be given a unique identifier, but +when using Chronicle Channels its the channels that are given the unique identifier not the map. ``` java -ReplicatingClusterBuilder clusterBuilder = new ReplicatingClusterBuilder((byte) 2, 1024); +byte identifier = 2; +ChannelReplicator channelReplicatorA = new ChannelReplicatorBuilder(identifier, 1024) + .create(); ``` -In this example above the cluster is given the identifier of 2 +In this example above the channel is given the identifier of 2 In addition to specifying the identifier we also have to set the maximum entry size, this sets -the size of the memory buffers within the cluster. This has to be set manually, with clusters you -are able to attach additional maps to a cluster once its up and running, so the maximum size of each +the size of the memory buffers within the ChannelReplicator. This has to be set manually, with channels you +are able to attach additional maps to a ChannelReplicator once its up and running, so the maximum size of each entry in the map can not be known in advance and we don’t currently support automatic resizing of buffers. -Once you have created the cluster you should attach your tcpConfig +Once you have created the ChannelReplicator you should attach your tcp configuration ``` java -TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); -clusterBuilder.tcpReplication(tcpConfig); -ReplicatingCluster cluster = clusterBuilder.create(); +byte identifier = 2; +ChannelReplicator channelReplicator = new ChannelReplicatorBuilder(identifier, 1024) + .tcpReplication(tcpConfig) + .create();; ``` -Attaching cluster replication to the map: +Attaching ChannelReplicator replication to the map: ``` java ChronicleMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .addReplicator(cluster.channelReplicator((short) 1)) + .channel(channelReplicator.createChannel((short) 1)) .create(file); ``` @@ -567,11 +570,11 @@ If you inadvertently got the chronicle channels around the wrong way, then chron to replicate the wrong maps data. The chronicle channels don't have to be in order but they must be unique for each map you have. -Once you have created the cluster you may wish to hold onto the reference so that you can call close -once you have finished, this will close everything in the cluster +Once you have created the ChannelReplicator you may wish to hold onto the reference so that you can call close +once you have finished, this will close everything in the ChannelReplicator ``` java -cluster.close(); +channelReplicator.close(); ``` #### Known Issues diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatingChannel.java b/src/main/java/net/openhft/chronicle/map/ChannelReplicator.java similarity index 98% rename from src/main/java/net/openhft/chronicle/map/ReplicatingChannel.java rename to src/main/java/net/openhft/chronicle/map/ChannelReplicator.java index 13ae91e3b..4aaa95158 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatingChannel.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelReplicator.java @@ -45,8 +45,8 @@ import static net.openhft.chronicle.map.Replica.ModificationNotifier.NOP; /** * @author Rob Austin. */ -public final class ReplicatingChannel implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(ReplicatingChannel.class.getName()); +public final class ChannelReplicator implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(ChannelReplicator.class.getName()); private static final byte BOOTSTRAP_MESSAGE = 'B'; final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { @@ -152,7 +152,7 @@ public final class ReplicatingChannel implements Closeable { @Override public void close() throws IOException { - ReplicatingChannel.this.close(); + ChannelReplicator.this.close(); } }; private final int maxEntrySize; @@ -169,7 +169,7 @@ public final class ReplicatingChannel implements Closeable { private final Set replicators = new CopyOnWriteArraySet(); - ReplicatingChannel(ReplicatingChannelBuilder builder) { + ChannelReplicator(ChannelReplicatorBuilder builder) { localIdentifier = builder.identifier; maxEntrySize = builder.maxEntrySize; chronicleChannels = new Replica[builder.maxNumberOfChronicles]; diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatingChannelBuilder.java b/src/main/java/net/openhft/chronicle/map/ChannelReplicatorBuilder.java similarity index 80% rename from src/main/java/net/openhft/chronicle/map/ReplicatingChannelBuilder.java rename to src/main/java/net/openhft/chronicle/map/ChannelReplicatorBuilder.java index 23e095896..537095985 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatingChannelBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelReplicatorBuilder.java @@ -21,7 +21,7 @@ import java.io.IOException; /** * @author Rob Austin. */ -public final class ReplicatingChannelBuilder { +public final class ChannelReplicatorBuilder { final byte identifier; final int maxEntrySize; @@ -29,7 +29,7 @@ public final class ReplicatingChannelBuilder { private UdpReplicationConfig udpReplicationConfig = null; private TcpReplicationConfig tcpReplicationConfig = null; - ReplicatingChannelBuilder(byte identifier, final int maxEntrySize) { + ChannelReplicatorBuilder(byte identifier, final int maxEntrySize) { this.identifier = identifier; this.maxEntrySize = maxEntrySize; if (identifier <= 0) { @@ -38,23 +38,23 @@ public final class ReplicatingChannelBuilder { } } - public ReplicatingChannelBuilder udpReplication(UdpReplicationConfig replicationConfig) { + public ChannelReplicatorBuilder udpReplication(UdpReplicationConfig replicationConfig) { this.udpReplicationConfig = replicationConfig; return this; } - public ReplicatingChannelBuilder tcpReplication(TcpReplicationConfig replicationConfig) { + public ChannelReplicatorBuilder tcpReplication(TcpReplicationConfig replicationConfig) { this.tcpReplicationConfig = replicationConfig; return this; } - public ReplicatingChannelBuilder maxNumberOfChronicles(int maxNumberOfChronicles) { + public ChannelReplicatorBuilder maxNumberOfChronicles(int maxNumberOfChronicles) { this.maxNumberOfChronicles = maxNumberOfChronicles; return this; } - public ReplicatingChannel create() throws IOException { - final ReplicatingChannel replicatingCluster = new ReplicatingChannel(this); + public ChannelReplicator create() throws IOException { + final ChannelReplicator replicatingCluster = new ChannelReplicator(this); if (tcpReplicationConfig != null) { final TcpReplicator tcpReplicator = new TcpReplicator( replicatingCluster.asReplica, diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index a5e8cf385..ff33d1e01 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -562,10 +562,10 @@ public class ChronicleMapBuilder implements Cloneable, } - public ChronicleMapBuilder channel(ReplicatingChannel.ChronicleChannel replicator) { - this.identifier = replicator.identifier(); + public ChronicleMapBuilder channel(ChannelReplicator.ChronicleChannel chronicleChannel) { + this.identifier = chronicleChannel.identifier(); this.replicators.clear(); - replicators.put(replicator.getClass(), replicator); + replicators.put(chronicleChannel.getClass(), chronicleChannel); return this; } diff --git a/src/test/java/net/openhft/chronicle/map/ClusterReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java similarity index 81% rename from src/test/java/net/openhft/chronicle/map/ClusterReplicationTest.java rename to src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java index 941179aef..40344799a 100644 --- a/src/test/java/net/openhft/chronicle/map/ClusterReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java @@ -32,7 +32,7 @@ import static org.junit.Assert.assertTrue; * * @author Rob Austin. */ -public class ClusterReplicationTest { +public class ChannelReplicationTest { private ChronicleMap map1a; private ChronicleMap map2a; @@ -40,8 +40,8 @@ public class ClusterReplicationTest { private ChronicleMap map1b; private ChronicleMap map2b; - private ReplicatingChannel clusterB; - private ReplicatingChannel clusterA; + private ChannelReplicator channelReplicatorA; + private ChannelReplicator channelReplicatorB; @Before @@ -52,12 +52,13 @@ public class ClusterReplicationTest { .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - clusterA = new ReplicatingChannelBuilder((byte) 1, 1024).tcpReplication(tcpConfig) - .create(); + channelReplicatorA = new ChannelReplicatorBuilder((byte) 1, 1024) + .tcpReplication(tcpConfig) + .create(); map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterA.createChannel((short) 1)) + .channel(channelReplicatorA.createChannel((short) 1)) .create(getPersistenceFile()); } @@ -65,12 +66,12 @@ public class ClusterReplicationTest { TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); - clusterB = new ReplicatingChannelBuilder((byte) 2, 1024).tcpReplication(tcpConfig) + channelReplicatorB = new ChannelReplicatorBuilder((byte) 2, 1024).tcpReplication(tcpConfig) .create(); map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterB.createChannel((short) 1)) + .channel(channelReplicatorB.createChannel((short) 1)) .create(getPersistenceFile()); } } @@ -78,7 +79,7 @@ public class ClusterReplicationTest { @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{clusterA, clusterB}) { + for (final Closeable closeable : new Closeable[]{channelReplicatorA, channelReplicatorB}) { try { closeable.close(); } catch (IOException e) { @@ -93,13 +94,13 @@ public class ClusterReplicationTest { map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterB.createChannel((short) 2)) + .channel(channelReplicatorB.createChannel((short) 2)) .create(getPersistenceFile()); map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterA.createChannel((short) 2)) + .channel(channelReplicatorA.createChannel((short) 2)) .create(getPersistenceFile()); map2a.put(1, "EXAMPLE-2"); diff --git a/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java index e9c102f34..7cabde3c1 100644 --- a/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java @@ -43,11 +43,8 @@ public class PostClusterMapCreationTest { private ChronicleMap map1b; private ChronicleMap map2b; - private ReplicatingChannel clusterB; - private ReplicatingChannel clusterA; - - private ReplicatingChannelBuilder replicatingClusterBuilder1; - + private ChannelReplicator channelReplicatorA; + private ChannelReplicator channelReplicatorB; public static File getPersistenceFile() { String TMP = System.getProperty("java.io.tmpdir"); @@ -66,18 +63,18 @@ public class PostClusterMapCreationTest { .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - clusterA = new ReplicatingChannelBuilder((byte) 1, 1024) + channelReplicatorA = new ChannelReplicatorBuilder((byte) 1, 1024) .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterA.createChannel((short) 1)) + .channel(channelReplicatorA.createChannel((short) 1)) .create(getPersistenceFile()); map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterA.createChannel((short) 2)) + .channel(channelReplicatorA.createChannel((short) 2)) .create(getPersistenceFile()); } @@ -87,18 +84,18 @@ public class PostClusterMapCreationTest { .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); - clusterB = new ReplicatingChannelBuilder((byte) 2, 1024) + channelReplicatorB = new ChannelReplicatorBuilder((byte) 2, 1024) .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterB.createChannel((short) 1)) + .channel(channelReplicatorB.createChannel((short) 1)) .create(getPersistenceFile()); map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterB.createChannel((short) 2)) + .channel(channelReplicatorB.createChannel((short) 2)) .create(getPersistenceFile()); } @@ -108,7 +105,7 @@ public class PostClusterMapCreationTest { @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{clusterA, clusterB}) { + for (final Closeable closeable : new Closeable[]{channelReplicatorA, channelReplicatorB}) { try { closeable.close(); } catch (IOException e) { diff --git a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java index af26c450f..cd0e870d0 100644 --- a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java +++ b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java @@ -41,9 +41,8 @@ public class SingleMapClusterTest { private ChronicleMap map1b; - private ReplicatingChannel clusterB; - private ReplicatingChannel clusterA; - + private ChannelReplicator channelReplicatorA; + private ChannelReplicator channelReplicatorB; public static File getPersistenceFile() { String TMP = System.getProperty("java.io.tmpdir"); @@ -61,12 +60,12 @@ public class SingleMapClusterTest { .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - clusterA = new ReplicatingChannelBuilder((byte) 1, 1024) + channelReplicatorA = new ChannelReplicatorBuilder((byte) 1, 1024) .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(clusterA.createChannel((short) 1)) + .channel(channelReplicatorA.createChannel((short) 1)) .create(getPersistenceFile()); } @@ -76,11 +75,11 @@ public class SingleMapClusterTest { .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); - clusterB = new ReplicatingChannelBuilder((byte) 2, 1024) + channelReplicatorB = new ChannelReplicatorBuilder((byte) 2, 1024) .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .channel(clusterB.createChannel((short) 1)) + .channel(channelReplicatorB.createChannel((short) 1)) .entries(1000) .create(getPersistenceFile()); } @@ -89,7 +88,7 @@ public class SingleMapClusterTest { @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{clusterA, clusterB}) { + for (final Closeable closeable : new Closeable[]{channelReplicatorA, channelReplicatorB}) { try { closeable.close(); } catch (IOException e) { From abcd89ec5efc2f8a97964705bfe365eb62224ad9 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:25:12 +0100 Subject: [PATCH 02/64] HCOLL-163 renamed chronicle custer to chronicle channel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8fee78d4e..474dc33d8 100644 --- a/README.md +++ b/README.md @@ -520,7 +520,7 @@ example of just two servers, they only have to be connected via a single tcp soc the data is transferred both ways. Which is great, however what if you wanted to replicate more than just one chronicle map, what if you were going to replicate two chronicle maps across your network, unfortunately with just TCP replication you would have to have two tcp socket connections, which is -not ideal. This is why we created Chronicle Clustering. Clustering lets you replicate numerous +not ideal. This is why we created Chronicle Channels. Channels lets you replicate numerous Chronicle Maps via a single point to point socket connection. Chronicle Channels are similar to TCP replication, where each map has to be given a unique identifier, but From 3dbc62eb369d3175e411fdc91f2460da54a5516a Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:26:32 +0100 Subject: [PATCH 03/64] HCOLL-163 fixed readme typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 474dc33d8..2fb70de32 100644 --- a/README.md +++ b/README.md @@ -520,7 +520,7 @@ example of just two servers, they only have to be connected via a single tcp soc the data is transferred both ways. Which is great, however what if you wanted to replicate more than just one chronicle map, what if you were going to replicate two chronicle maps across your network, unfortunately with just TCP replication you would have to have two tcp socket connections, which is -not ideal. This is why we created Chronicle Channels. Channels lets you replicate numerous +not ideal. This is why we created Chronicle Channels. Channels let you replicate numerous Chronicle Maps via a single point to point socket connection. Chronicle Channels are similar to TCP replication, where each map has to be given a unique identifier, but From 97e768d8560c7b204247e5efe19262e10d6c8514 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:28:33 +0100 Subject: [PATCH 04/64] HCOLL-163 adde "int maxEntrySize =1024;" --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2fb70de32..1f876c659 100644 --- a/README.md +++ b/README.md @@ -528,7 +528,8 @@ when using Chronicle Channels its the channels that are given the unique identif ``` java byte identifier = 2; -ChannelReplicator channelReplicatorA = new ChannelReplicatorBuilder(identifier, 1024) +int maxEntrySize =1024; +ChannelReplicator channelReplicatorA = new ChannelReplicatorBuilder(identifier, maxEntrySize) .create(); ``` @@ -543,6 +544,7 @@ of buffers. Once you have created the ChannelReplicator you should attach your tcp configuration ``` java byte identifier = 2; +int maxEntrySize =1024; ChannelReplicator channelReplicator = new ChannelReplicatorBuilder(identifier, 1024) .tcpReplication(tcpConfig) .create();; From 7f8f745eca8b6eb85b778b1ee370a379fd768c87 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:29:41 +0100 Subject: [PATCH 05/64] HCOLL-163 renamed to "replicator" --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1f876c659..4a1005d59 100644 --- a/README.md +++ b/README.md @@ -529,7 +529,7 @@ when using Chronicle Channels its the channels that are given the unique identif ``` java byte identifier = 2; int maxEntrySize =1024; -ChannelReplicator channelReplicatorA = new ChannelReplicatorBuilder(identifier, maxEntrySize) +ChannelReplicator replicator = new ChannelReplicatorBuilder(identifier, maxEntrySize) .create(); ``` @@ -545,7 +545,7 @@ Once you have created the ChannelReplicator you should attach your tcp configura ``` java byte identifier = 2; int maxEntrySize =1024; -ChannelReplicator channelReplicator = new ChannelReplicatorBuilder(identifier, 1024) +ChannelReplicator replicator = new ChannelReplicatorBuilder(identifier, 1024) .tcpReplication(tcpConfig) .create();; ``` @@ -555,7 +555,7 @@ Attaching ChannelReplicator replication to the map: ``` java ChronicleMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicator.createChannel((short) 1)) + .channel(replicator.createChannel((short) 1)) .create(file); ``` @@ -576,7 +576,7 @@ Once you have created the ChannelReplicator you may wish to hold onto the refere once you have finished, this will close everything in the ChannelReplicator ``` java -channelReplicator.close(); +replicator.close(); ``` #### Known Issues From 9ca17bc56e3bd9356b869452f40ac769ef4a8f7e Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:30:32 +0100 Subject: [PATCH 06/64] HCOLL-163 renamed to "replicator" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a1005d59..4020c21cb 100644 --- a/README.md +++ b/README.md @@ -528,7 +528,7 @@ when using Chronicle Channels its the channels that are given the unique identif ``` java byte identifier = 2; -int maxEntrySize =1024; +int maxEntrySize = 1024; ChannelReplicator replicator = new ChannelReplicatorBuilder(identifier, maxEntrySize) .create(); ``` From 6a640541fc9680b880b1d603763fe665ddf2b6f6 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:31:34 +0100 Subject: [PATCH 07/64] HCOLL-163 readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4020c21cb..db16d2ca7 100644 --- a/README.md +++ b/README.md @@ -512,7 +512,7 @@ A heartbeat will only be send if no data is transmitted, if the maps are constan no heartbeat message is sent. If a map does not receive either data of a heartbeat the connection is dropped and re-established. -# Channels +# Channels and ChannelReplicator Chronicle Map TCP Replication lets you distribute a single Chronicle Map, to a number of servers across your network. Replication is point to point and the data transfer is bidirectional, so in the From cc9f26bfae2129de9c7b3885f1c8cd67ac2c2972 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:32:16 +0100 Subject: [PATCH 08/64] HCOLL-163 readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index db16d2ca7..f730b7679 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,8 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Identifier](https://github.com/OpenHFT/Chronicle-Map#identifier) * [Port](https://github.com/OpenHFT/Chronicle-Map#port) * [Heart Beat Interval](https://github.com/OpenHFT/Chronicle-Map#heart-beat-interval) -* [Channels](https://github.com/OpenHFT/Chronicle-Map#channels) +* [Channels and the Channel Replicator](https://github +.com/OpenHFT/Chronicle-Map#channels-and-channelreplicator) #### Miscellaneous From 02072f0e8a4a0c185f2256aa19f3ca812dad6a19 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:33:34 +0100 Subject: [PATCH 09/64] HCOLL-163 readme - https://github.com/OpenHFT/Chronicle-Map#channels-and-channelreplicator --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index f730b7679..338d01678 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,7 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Identifier](https://github.com/OpenHFT/Chronicle-Map#identifier) * [Port](https://github.com/OpenHFT/Chronicle-Map#port) * [Heart Beat Interval](https://github.com/OpenHFT/Chronicle-Map#heart-beat-interval) -* [Channels and the Channel Replicator](https://github -.com/OpenHFT/Chronicle-Map#channels-and-channelreplicator) +* [Channels and the Channel Replicator](https://github.com/OpenHFT/Chronicle-Map#channels-and-channelreplicator) #### Miscellaneous From 72215a67e87ed18e80634454a97d3b30e2323bb3 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 18:35:33 +0100 Subject: [PATCH 10/64] Update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 338d01678..2135938aa 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,6 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Replicating data between process on different servers with UDP] (https://github.com/OpenHFT/Chronicle-Map/blob/master/README.md#example--replicating-data-between-process-on-different-servers-using-udp) * [Creating a Chronicle Set and adding data to it](https://github.com/OpenHFT/Chronicle-Map/blob/master/README.md#example--creating-a-chronicle-set-and-adding-data-to-it) - #### Performance Topics * [Chronicle Map with Large Data ](https://github.com/OpenHFT/Chronicle-Map#chronicle-map-with-large-data) @@ -71,12 +70,10 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Better to use small keys](https://github.com/OpenHFT/Chronicle-Map#better-to-use-small-keys) * [ConcurrentHashMap v ChronicleMap](https://github.com/OpenHFT/Chronicle-Map#concurrenthashmap-v-chroniclemap) - ### Overview Chronicle Map implements the `java.util.concurrent.ConcurrentMap`, however unlike the standard java map, ChronicleMap is able to share your entries accross processes: - ![](http://openhft.net/wp-content/uploads/2014/07/Chronicle-Map-diagram_04.jpg) ## When to use From 2aff5127672cf157e62349684f843c282ac237f5 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 19:03:43 +0100 Subject: [PATCH 11/64] HCOLL-163 renamed channel replicator to channel provider --- README.md | 20 ++++---- ...elReplicator.java => ChannelProvider.java} | 8 ++-- ...ilder.java => ChannelProviderBuilder.java} | 48 +++++++++++-------- .../chronicle/map/ChronicleMapBuilder.java | 2 +- .../net/openhft/chronicle/map/Config.java | 7 +++ .../chronicle/map/TcpReplicationConfig.java | 2 +- .../chronicle/map/UdpReplicationConfig.java | 3 +- .../chronicle/map/ChannelReplicationTest.java | 27 ++++++----- .../map/PostClusterMapCreationTest.java | 26 +++++----- .../chronicle/map/SingleMapClusterTest.java | 19 ++++---- 10 files changed, 90 insertions(+), 72 deletions(-) rename src/main/java/net/openhft/chronicle/map/{ChannelReplicator.java => ChannelProvider.java} (98%) rename src/main/java/net/openhft/chronicle/map/{ChannelReplicatorBuilder.java => ChannelProviderBuilder.java} (67%) create mode 100644 src/main/java/net/openhft/chronicle/map/Config.java diff --git a/README.md b/README.md index 338d01678..d0dfb17bd 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Identifier](https://github.com/OpenHFT/Chronicle-Map#identifier) * [Port](https://github.com/OpenHFT/Chronicle-Map#port) * [Heart Beat Interval](https://github.com/OpenHFT/Chronicle-Map#heart-beat-interval) -* [Channels and the Channel Replicator](https://github.com/OpenHFT/Chronicle-Map#channels-and-channelreplicator) +* [Channels and the Channel Provider](https://github.com/OpenHFT/Chronicle-Map#channels-and-channelprovider) #### Miscellaneous @@ -512,7 +512,7 @@ A heartbeat will only be send if no data is transmitted, if the maps are constan no heartbeat message is sent. If a map does not receive either data of a heartbeat the connection is dropped and re-established. -# Channels and ChannelReplicator +# Channels and ChannelProvider Chronicle Map TCP Replication lets you distribute a single Chronicle Map, to a number of servers across your network. Replication is point to point and the data transfer is bidirectional, so in the @@ -529,28 +529,28 @@ when using Chronicle Channels its the channels that are given the unique identif ``` java byte identifier = 2; int maxEntrySize = 1024; -ChannelReplicator replicator = new ChannelReplicatorBuilder(identifier, maxEntrySize) +ChannelProvider replicator = new ChannelProviderBuilder(identifier, maxEntrySize) .create(); ``` In this example above the channel is given the identifier of 2 In addition to specifying the identifier we also have to set the maximum entry size, this sets -the size of the memory buffers within the ChannelReplicator. This has to be set manually, with channels you -are able to attach additional maps to a ChannelReplicator once its up and running, so the maximum size of each +the size of the memory buffers within the ChannelProvider. This has to be set manually, with channels you +are able to attach additional maps to a ChannelProvider once its up and running, so the maximum size of each entry in the map can not be known in advance and we don’t currently support automatic resizing of buffers. -Once you have created the ChannelReplicator you should attach your tcp configuration +Once you have created the ChannelProvider you should attach your tcp configuration ``` java byte identifier = 2; int maxEntrySize =1024; -ChannelReplicator replicator = new ChannelReplicatorBuilder(identifier, 1024) +ChannelProvider replicator = new ChannelProviderBuilder(identifier, 1024) .tcpReplication(tcpConfig) .create();; ``` -Attaching ChannelReplicator replication to the map: +Attaching ChannelProvider replication to the map: ``` java ChronicleMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) @@ -572,8 +572,8 @@ If you inadvertently got the chronicle channels around the wrong way, then chron to replicate the wrong maps data. The chronicle channels don't have to be in order but they must be unique for each map you have. -Once you have created the ChannelReplicator you may wish to hold onto the reference so that you can call close -once you have finished, this will close everything in the ChannelReplicator +Once you have created the ChannelProvider you may wish to hold onto the reference so that you can call close +once you have finished, this will close everything in the ChannelProvider ``` java replicator.close(); diff --git a/src/main/java/net/openhft/chronicle/map/ChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java similarity index 98% rename from src/main/java/net/openhft/chronicle/map/ChannelReplicator.java rename to src/main/java/net/openhft/chronicle/map/ChannelProvider.java index 4aaa95158..c51a56e8c 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java @@ -45,8 +45,8 @@ import static net.openhft.chronicle.map.Replica.ModificationNotifier.NOP; /** * @author Rob Austin. */ -public final class ChannelReplicator implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(ChannelReplicator.class.getName()); +public final class ChannelProvider implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(ChannelProvider.class.getName()); private static final byte BOOTSTRAP_MESSAGE = 'B'; final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { @@ -152,7 +152,7 @@ public final class ChannelReplicator implements Closeable { @Override public void close() throws IOException { - ChannelReplicator.this.close(); + ChannelProvider.this.close(); } }; private final int maxEntrySize; @@ -169,7 +169,7 @@ public final class ChannelReplicator implements Closeable { private final Set replicators = new CopyOnWriteArraySet(); - ChannelReplicator(ChannelReplicatorBuilder builder) { + ChannelProvider(ChannelProviderBuilder builder) { localIdentifier = builder.identifier; maxEntrySize = builder.maxEntrySize; chronicleChannels = new Replica[builder.maxNumberOfChronicles]; diff --git a/src/main/java/net/openhft/chronicle/map/ChannelReplicatorBuilder.java b/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java similarity index 67% rename from src/main/java/net/openhft/chronicle/map/ChannelReplicatorBuilder.java rename to src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java index 537095985..ca1a00f00 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelReplicatorBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java @@ -21,40 +21,48 @@ import java.io.IOException; /** * @author Rob Austin. */ -public final class ChannelReplicatorBuilder { +public final class ChannelProviderBuilder { - final byte identifier; - final int maxEntrySize; + byte identifier; + int maxEntrySize = 1024; int maxNumberOfChronicles = 128; private UdpReplicationConfig udpReplicationConfig = null; private TcpReplicationConfig tcpReplicationConfig = null; - ChannelReplicatorBuilder(byte identifier, final int maxEntrySize) { - this.identifier = identifier; + + + public int maxEntrySize() { + return maxEntrySize; + } + + public ChannelProviderBuilder maxEntrySize(int maxEntrySize) { this.maxEntrySize = maxEntrySize; - if (identifier <= 0) { - throw new IllegalArgumentException("Identifier must be positive, identifier=" + - identifier); + return this; + } + + public ChannelProviderBuilder replicators(byte identifier, Config... configs) { + this.identifier = identifier; + + + for (Config config : configs) { + + if (config instanceof TcpReplicationConfig) { + this.tcpReplicationConfig = (TcpReplicationConfig) config; + } else if (config instanceof UdpReplicationConfig) { + this.udpReplicationConfig = (UdpReplicationConfig) config; + } else + throw new UnsupportedOperationException(); } - } - - public ChannelReplicatorBuilder udpReplication(UdpReplicationConfig replicationConfig) { - this.udpReplicationConfig = replicationConfig; return this; } - public ChannelReplicatorBuilder tcpReplication(TcpReplicationConfig replicationConfig) { - this.tcpReplicationConfig = replicationConfig; - return this; - } - - public ChannelReplicatorBuilder maxNumberOfChronicles(int maxNumberOfChronicles) { + public ChannelProviderBuilder maxNumberOfChronicles(int maxNumberOfChronicles) { this.maxNumberOfChronicles = maxNumberOfChronicles; return this; } - public ChannelReplicator create() throws IOException { - final ChannelReplicator replicatingCluster = new ChannelReplicator(this); + public ChannelProvider create() throws IOException { + final ChannelProvider replicatingCluster = new ChannelProvider(this); if (tcpReplicationConfig != null) { final TcpReplicator tcpReplicator = new TcpReplicator( replicatingCluster.asReplica, diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index ff33d1e01..6582ea539 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -562,7 +562,7 @@ public class ChronicleMapBuilder implements Cloneable, } - public ChronicleMapBuilder channel(ChannelReplicator.ChronicleChannel chronicleChannel) { + public ChronicleMapBuilder channel(ChannelProvider.ChronicleChannel chronicleChannel) { this.identifier = chronicleChannel.identifier(); this.replicators.clear(); replicators.put(chronicleChannel.getClass(), chronicleChannel); diff --git a/src/main/java/net/openhft/chronicle/map/Config.java b/src/main/java/net/openhft/chronicle/map/Config.java new file mode 100644 index 000000000..6e0d9a436 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Config.java @@ -0,0 +1,7 @@ +package net.openhft.chronicle.map; + +/** + * @author Rob Austin. + */ +public interface Config { +} diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java b/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java index b213f4455..5ea8cb1c1 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java @@ -29,7 +29,7 @@ import static java.util.Collections.unmodifiableSet; import static java.util.concurrent.TimeUnit.SECONDS; -public class TcpReplicationConfig { +public class TcpReplicationConfig implements Config { private static final int DEFAULT_PACKET_SIZE = 1024 * 8; private static final long DEFAULT_HEART_BEAT_INTERVAL = 20; diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java index 0e02d9e9d..a1e8ad0aa 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java @@ -23,7 +23,7 @@ import java.net.InetAddress; import java.net.NetworkInterface; -public abstract class UdpReplicationConfig { +public abstract class UdpReplicationConfig implements Config { /** * Package-private constructor forbids subclassing from outside of the package @@ -37,7 +37,6 @@ public abstract class UdpReplicationConfig { throw new IllegalArgumentException(); - return create(address, port, null, ThrottlingConfig.noThrottling()); } diff --git a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java index 40344799a..086dffb86 100644 --- a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java @@ -17,7 +17,10 @@ package net.openhft.chronicle.map; -import org.junit.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; import java.io.Closeable; import java.io.IOException; @@ -40,8 +43,8 @@ public class ChannelReplicationTest { private ChronicleMap map1b; private ChronicleMap map2b; - private ChannelReplicator channelReplicatorA; - private ChannelReplicator channelReplicatorB; + private ChannelProvider channelProviderA; + private ChannelProvider channelProviderB; @Before @@ -52,13 +55,13 @@ public class ChannelReplicationTest { .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - channelReplicatorA = new ChannelReplicatorBuilder((byte) 1, 1024) - .tcpReplication(tcpConfig) - .create(); + channelProviderA = new ChannelProviderBuilder() + .replicators((byte) 1, tcpConfig) + .create(); map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorA.createChannel((short) 1)) + .channel(channelProviderA.createChannel((short) 1)) .create(getPersistenceFile()); } @@ -66,12 +69,12 @@ public class ChannelReplicationTest { TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); - channelReplicatorB = new ChannelReplicatorBuilder((byte) 2, 1024).tcpReplication(tcpConfig) + channelProviderB = new ChannelProviderBuilder().replicators((byte) 2, tcpConfig) .create(); map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorB.createChannel((short) 1)) + .channel(channelProviderB.createChannel((short) 1)) .create(getPersistenceFile()); } } @@ -79,7 +82,7 @@ public class ChannelReplicationTest { @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{channelReplicatorA, channelReplicatorB}) { + for (final Closeable closeable : new Closeable[]{channelProviderA, channelProviderB}) { try { closeable.close(); } catch (IOException e) { @@ -94,13 +97,13 @@ public class ChannelReplicationTest { map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorB.createChannel((short) 2)) + .channel(channelProviderB.createChannel((short) 2)) .create(getPersistenceFile()); map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorA.createChannel((short) 2)) + .channel(channelProviderA.createChannel((short) 2)) .create(getPersistenceFile()); map2a.put(1, "EXAMPLE-2"); diff --git a/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java index 7cabde3c1..fba74f285 100644 --- a/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java @@ -43,8 +43,8 @@ public class PostClusterMapCreationTest { private ChronicleMap map1b; private ChronicleMap map2b; - private ChannelReplicator channelReplicatorA; - private ChannelReplicator channelReplicatorB; + private ChannelProvider channelProviderA; + private ChannelProvider channelProviderB; public static File getPersistenceFile() { String TMP = System.getProperty("java.io.tmpdir"); @@ -59,43 +59,43 @@ public class PostClusterMapCreationTest { public void setup() throws IOException { { - final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig + final TcpReplicationConfig tcpConfig = TcpReplicationConfig .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - channelReplicatorA = new ChannelReplicatorBuilder((byte) 1, 1024) - .tcpReplication(tcpReplicationConfig).create(); + channelProviderA = new ChannelProviderBuilder() + .replicators((byte) 1, tcpConfig).create(); // this is how you add maps after the custer is created map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorA.createChannel((short) 1)) + .channel(channelProviderA.createChannel((short) 1)) .create(getPersistenceFile()); map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorA.createChannel((short) 2)) + .channel(channelProviderA.createChannel((short) 2)) .create(getPersistenceFile()); } { - final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig + final TcpReplicationConfig tcpConfig = TcpReplicationConfig .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); - channelReplicatorB = new ChannelReplicatorBuilder((byte) 2, 1024) - .tcpReplication(tcpReplicationConfig).create(); + channelProviderB = new ChannelProviderBuilder() + .replicators((byte) 2, tcpConfig).create(); // this is how you add maps after the custer is created map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorB.createChannel((short) 1)) + .channel(channelProviderB.createChannel((short) 1)) .create(getPersistenceFile()); map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorB.createChannel((short) 2)) + .channel(channelProviderB.createChannel((short) 2)) .create(getPersistenceFile()); } @@ -105,7 +105,7 @@ public class PostClusterMapCreationTest { @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{channelReplicatorA, channelReplicatorB}) { + for (final Closeable closeable : new Closeable[]{channelProviderA, channelProviderB}) { try { closeable.close(); } catch (IOException e) { diff --git a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java index cd0e870d0..82ede45f6 100644 --- a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java +++ b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java @@ -41,8 +41,8 @@ public class SingleMapClusterTest { private ChronicleMap map1b; - private ChannelReplicator channelReplicatorA; - private ChannelReplicator channelReplicatorB; + private ChannelProvider channelProviderA; + private ChannelProvider channelProviderB; public static File getPersistenceFile() { String TMP = System.getProperty("java.io.tmpdir"); @@ -60,12 +60,12 @@ public class SingleMapClusterTest { .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - channelReplicatorA = new ChannelReplicatorBuilder((byte) 1, 1024) - .tcpReplication(tcpReplicationConfig).create(); + channelProviderA = new ChannelProviderBuilder() + .replicators((byte) 1, tcpReplicationConfig).create(); // this is how you add maps after the custer is created map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelReplicatorA.createChannel((short) 1)) + .channel(channelProviderA.createChannel((short) 1)) .create(getPersistenceFile()); } @@ -75,11 +75,12 @@ public class SingleMapClusterTest { .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); - channelReplicatorB = new ChannelReplicatorBuilder((byte) 2, 1024) - .tcpReplication(tcpReplicationConfig).create(); + channelProviderB = new ChannelProviderBuilder() + .replicators((byte) 2, tcpReplicationConfig).create(); + // this is how you add maps after the custer is created map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .channel(channelReplicatorB.createChannel((short) 1)) + .channel(channelProviderB.createChannel((short) 1)) .entries(1000) .create(getPersistenceFile()); } @@ -88,7 +89,7 @@ public class SingleMapClusterTest { @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{channelReplicatorA, channelReplicatorB}) { + for (final Closeable closeable : new Closeable[]{channelProviderA, channelProviderB}) { try { closeable.close(); } catch (IOException e) { From 6699ff682e20cd4d29dcf0dd9acf13aec42dd965 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Mon, 13 Oct 2014 19:06:13 +0100 Subject: [PATCH 12/64] HCOLL-163 renamed channel replicator to channel provider --- ...ClusterMapCreationTest.java => PostChannelCreationTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/java/net/openhft/chronicle/map/{PostClusterMapCreationTest.java => PostChannelCreationTest.java} (99%) diff --git a/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java similarity index 99% rename from src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java rename to src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java index fba74f285..f5292974c 100644 --- a/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java @@ -35,7 +35,7 @@ import static org.junit.Assert.assertTrue; * * @author Rob Austin. */ -public class PostClusterMapCreationTest { +public class PostChannelCreationTest { private ChronicleMap map1a; private ChronicleMap map2a; From 8ae20aa6ccb953c8afea629c4491ef1074116de2 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 14 Oct 2014 00:30:20 +0400 Subject: [PATCH 13/64] HCOLL-164 Made ChronicleSet zero overhead by value; fixed ChronicleSet.add() and remove() implementations; added ChronicleMapBuilder.valueMarshallerAndFactory(AgileBytesMarshaller, ObjectFactory) method --- pom.xml | 2 +- .../chronicle/map/ChronicleMapBuilder.java | 8 ++ .../chronicle/map/SerializationBuilder.java | 2 +- .../chronicle/set/ChronicleSetBuilder.java | 114 ++-------------- .../net/openhft/chronicle/set/DummyValue.java | 21 +++ .../chronicle/set/DummyValueMarshaller.java | 71 ++++++++++ .../net/openhft/chronicle/set/SetFromMap.java | 122 ++++++++++++++++++ 7 files changed, 232 insertions(+), 108 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/set/DummyValue.java create mode 100644 src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java create mode 100644 src/main/java/net/openhft/chronicle/set/SetFromMap.java diff --git a/pom.xml b/pom.xml index c7e64009f..bb883ecb1 100755 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ net.openhft lang - 6.4.9 + 6.4.10-SNAPSHOT diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index bbcaaa7de..fd3cfdda4 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -20,6 +20,7 @@ import net.openhft.chronicle.ChronicleHashBuilder; import net.openhft.chronicle.ChronicleHashErrorListener; import net.openhft.chronicle.ChronicleHashErrorListeners; import net.openhft.chronicle.TimeProvider; +import net.openhft.chronicle.map.serialization.AgileBytesMarshaller; import net.openhft.chronicle.map.serialization.MetaBytesInterop; import net.openhft.chronicle.map.serialization.MetaBytesWriter; import net.openhft.chronicle.map.serialization.MetaProvider; @@ -638,6 +639,13 @@ public class ChronicleMapBuilder implements Cloneable, return this; } + public ChronicleMapBuilder valueMarshallerAndFactory( + @NotNull AgileBytesMarshaller valueMarshaller, + @NotNull ObjectFactory valueFactory) { + valueBuilder.agileMarshaller(valueMarshaller, valueFactory); + return this; + } + public ChronicleMapBuilder valueFactory(@NotNull ObjectFactory valueFactory) { valueBuilder.factory(valueFactory); return this; diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index c55306fd3..4973970bb 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -80,7 +80,7 @@ final class SerializationBuilder implements Cloneable { if (role == Role.VALUE) { factory = concreteClass(eClass) && marshallerUseFactory(eClass) ? new NewInstanceObjectFactory(eClass) : - NullObjectFactory.INSTANCE; + NullObjectFactory.of(); } if (concreteClass(eClass) && Byteable.class.isAssignableFrom(eClass)) { diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index 26a27f8f5..5473edcb5 100644 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -16,35 +16,30 @@ package net.openhft.chronicle.set; -import net.openhft.chronicle.TimeProvider; -import net.openhft.chronicle.map.Alignment; import net.openhft.chronicle.ChronicleHashBuilder; import net.openhft.chronicle.ChronicleHashErrorListener; +import net.openhft.chronicle.TimeProvider; import net.openhft.chronicle.map.*; import net.openhft.lang.io.serialization.BytesMarshaller; import net.openhft.lang.io.serialization.BytesMarshallerFactory; import net.openhft.lang.io.serialization.ObjectSerializer; +import net.openhft.lang.io.serialization.impl.NullObjectFactory; import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; -import java.io.Serializable; -import java.util.AbstractSet; -import java.util.Collection; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; public class ChronicleSetBuilder implements ChronicleHashBuilder, ChronicleSetBuilder> { - private ChronicleMapBuilder chronicleMapBuilder; + private ChronicleMapBuilder chronicleMapBuilder; ChronicleSetBuilder(Class keyClass) { - chronicleMapBuilder = ChronicleMapBuilder.of(keyClass, Void.class) - .entryAndValueAlignment(Alignment.NO_ALIGNMENT); + chronicleMapBuilder = ChronicleMapBuilder.of(keyClass, DummyValue.class) + .entryAndValueAlignment(Alignment.NO_ALIGNMENT) + .valueMarshallerAndFactory(DummyValueMarshaller.INSTANCE, NullObjectFactory.of()); } public static ChronicleSetBuilder of(Class keyClass) { @@ -229,108 +224,15 @@ public class ChronicleSetBuilder @Override public ChronicleSet create(File file) throws IOException { - final ChronicleMap map = chronicleMapBuilder.create(file); + final ChronicleMap map = chronicleMapBuilder.create(file); return new SetFromMap(map); } @Override public ChronicleSet create() throws IOException { - final ChronicleMap map = chronicleMapBuilder.create(); + final ChronicleMap map = chronicleMapBuilder.create(); return new SetFromMap(map); } - private static class SetFromMap extends AbstractSet - implements ChronicleSet, Serializable { - - private final ChronicleMap m; // The backing map - private transient Set s; // Its keySet - - SetFromMap(ChronicleMap map) { - if (!map.isEmpty()) - throw new IllegalArgumentException("Map is non-empty"); - m = map; - s = map.keySet(); - } - - public void clear() { - m.clear(); - } - - public int size() { - return m.size(); - } - - public boolean isEmpty() { - return m.isEmpty(); - } - - public boolean contains(Object o) { - return m.containsKey(o); - } - - public boolean remove(Object o) { - return m.remove(o) != null; - } - - public boolean add(E e) { - return m.put(e, (Void) null) == null; - } - - public Iterator iterator() { - return s.iterator(); - } - - public Object[] toArray() { - return s.toArray(); - } - - public T[] toArray(T[] a) { - return s.toArray(a); - } - - public String toString() { - return s.toString(); - } - - public int hashCode() { - return s.hashCode(); - } - - public boolean equals(Object o) { - return o == this || s.equals(o); - } - - public boolean containsAll(Collection c) { - return s.containsAll(c); - } - - public boolean removeAll(Collection c) { - return s.removeAll(c); - } - - public boolean retainAll(Collection c) { - return s.retainAll(c); - } - // addAll is the only inherited implementation - - private static final long serialVersionUID = 2454657854757543876L; - - private void readObject(java.io.ObjectInputStream stream) - throws IOException, ClassNotFoundException { - stream.defaultReadObject(); - s = m.keySet(); - } - - @Override - public long longSize() { - return m.longSize(); - } - - @Override - public void close() throws IOException { - m.close(); - } - } - } diff --git a/src/main/java/net/openhft/chronicle/set/DummyValue.java b/src/main/java/net/openhft/chronicle/set/DummyValue.java new file mode 100644 index 000000000..1d73a070b --- /dev/null +++ b/src/main/java/net/openhft/chronicle/set/DummyValue.java @@ -0,0 +1,21 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.set; + +enum DummyValue { + DUMMY_VALUE +} diff --git a/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java b/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java new file mode 100644 index 000000000..85bb16462 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java @@ -0,0 +1,71 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.set; + +import net.openhft.chronicle.map.serialization.AgileBytesMarshaller; +import net.openhft.lang.io.Bytes; + +import static net.openhft.chronicle.set.DummyValue.DUMMY_VALUE; + +enum DummyValueMarshaller implements AgileBytesMarshaller { + INSTANCE; + + @Override + public boolean startsWith(Bytes bytes, DummyValue dummyValue) { + throw new UnsupportedOperationException(); + } + + @Override + public long hash(DummyValue dummyValue) { + throw new UnsupportedOperationException(); + } + + @Override + public DummyValue read(Bytes bytes, long size) { + return DUMMY_VALUE; + } + + @Override + public DummyValue read(Bytes bytes, long size, DummyValue dummyValue) { + return DUMMY_VALUE; + } + + @Override + public long size(DummyValue dummyValue) { + return 0L; + } + + @Override + public void write(Bytes bytes, DummyValue dummyValue) { + // do nothing + } + + @Override + public int sizeEncodingSize(long size) { + return 0; + } + + @Override + public void writeSize(Bytes bytes, long size) { + // do nothing + } + + @Override + public long readSize(Bytes bytes) { + return 0L; + } +} diff --git a/src/main/java/net/openhft/chronicle/set/SetFromMap.java b/src/main/java/net/openhft/chronicle/set/SetFromMap.java new file mode 100644 index 000000000..47fd84883 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/set/SetFromMap.java @@ -0,0 +1,122 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.set; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleSet; + +import java.io.IOException; +import java.io.Serializable; +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Iterator; +import java.util.Set; + +import static net.openhft.chronicle.set.DummyValue.DUMMY_VALUE; + +class SetFromMap extends AbstractSet + implements ChronicleSet, Serializable { + + private final ChronicleMap m; // The backing map + private transient Set s; // Its keySet + + SetFromMap(ChronicleMap map) { + if (!map.isEmpty()) + throw new IllegalArgumentException("Map is non-empty"); + m = map; + s = map.keySet(); + } + + public void clear() { + m.clear(); + } + + public int size() { + return m.size(); + } + + public boolean isEmpty() { + return m.isEmpty(); + } + + public boolean contains(Object o) { + return m.containsKey(o); + } + + public boolean remove(Object o) { + return m.remove(o) == DUMMY_VALUE; + } + + public boolean add(E e) { + return m.putIfAbsent(e, DUMMY_VALUE) == null; + } + + public Iterator iterator() { + return s.iterator(); + } + + public Object[] toArray() { + return s.toArray(); + } + + public T[] toArray(T[] a) { + return s.toArray(a); + } + + public String toString() { + return s.toString(); + } + + public int hashCode() { + return s.hashCode(); + } + + public boolean equals(Object o) { + return o == this || s.equals(o); + } + + public boolean containsAll(Collection c) { + return s.containsAll(c); + } + + public boolean removeAll(Collection c) { + return s.removeAll(c); + } + + public boolean retainAll(Collection c) { + return s.retainAll(c); + } + // addAll is the only inherited implementation + + private static final long serialVersionUID = 2454657854757543876L; + + private void readObject(java.io.ObjectInputStream stream) + throws IOException, ClassNotFoundException { + stream.defaultReadObject(); + s = m.keySet(); + } + + @Override + public long longSize() { + return m.longSize(); + } + + @Override + public void close() throws IOException { + m.close(); + } +} From c53f092657ba249843db4742cbeea0a6e039a3d6 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 09:42:50 +0100 Subject: [PATCH 14/64] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f0b4706ff..4b45bcd31 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,9 @@ your JVM to crash. Close MUST BE the last thing that you do with the map. # TCP / UDP Replication + +![TCP/IP Replication](http://openhft.net/wp-content/uploads/2014/07/Chronicle-Map-TCP-Replication_simple_02.jpg) + Chronicle Hash Map supports both TCP and UDP replication ### TCP / UDP Background. From 1d7c89e73ca018fcf73a8f1f5e5ab523d03421f2 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 09:43:35 +0100 Subject: [PATCH 15/64] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b45bcd31..c8dae3923 100644 --- a/README.md +++ b/README.md @@ -346,9 +346,10 @@ your JVM to crash. Close MUST BE the last thing that you do with the map. # TCP / UDP Replication +Chronicle Hash Map supports both TCP and UDP replication + ![TCP/IP Replication](http://openhft.net/wp-content/uploads/2014/07/Chronicle-Map-TCP-Replication_simple_02.jpg) -Chronicle Hash Map supports both TCP and UDP replication ### TCP / UDP Background. TCP/IP is a reliable protocol, what this means is unless you have a network failure or hardware From 1b4da147c07366d9aa5b35e24e0c39ab37588ac5 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 10:14:21 +0100 Subject: [PATCH 16/64] HCOLL-163 updated readme --- README.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f0b4706ff..a5a64ea2d 100644 --- a/README.md +++ b/README.md @@ -524,10 +524,11 @@ Chronicle Channels are similar to TCP replication, where each map has to be give when using Chronicle Channels its the channels that are given the unique identifier not the map. ``` java -byte identifier = 2; int maxEntrySize = 1024; -ChannelProvider replicator = new ChannelProviderBuilder(identifier, maxEntrySize) - .create(); +byte identifier= 1; +ChannelProvider channelProvider = new ChannelProviderBuilder() + .maxEntrySize(maxEntrySize) + .replicators(identifier, tcpReplicationConfig).create(); ``` In this example above the channel is given the identifier of 2 @@ -538,22 +539,22 @@ are able to attach additional maps to a ChannelProvider once its up and running, entry in the map can not be known in advance and we don’t currently support automatic resizing of buffers. -Once you have created the ChannelProvider you should attach your tcp configuration +When creating the ChannelProvider you should attach your tcp or udp configuration ``` java -byte identifier = 2; -int maxEntrySize =1024; -ChannelProvider replicator = new ChannelProviderBuilder(identifier, 1024) - .tcpReplication(tcpConfig) - .create();; +int maxEntrySize = 1024; +byte identifier = 1; +ChannelProvider channelProvider = new ChannelProviderBuilder() + .maxEntrySize(maxEntrySize) + .replicators(identifier, tcpReplicationConfig).create(); ``` Attaching ChannelProvider replication to the map: ``` java ChronicleMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .channel(replicator.createChannel((short) 1)) - .create(file); + .entries(1000) + .channel(channelProviderA.createChannel((short) 1)) + .create(getPersistenceFile()); ``` The chronicle channel is use to identify which map is to be replicated to which other map on From 101483466899110fef54cc0691ffcc3c240c8ea0 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 10:41:36 +0100 Subject: [PATCH 17/64] HCOLL-163 kept the use of tcp() and udp() internal --- .../chronicle/ChronicleHashBuilder.java | 2 +- .../chronicle/map/ChannelProviderBuilder.java | 16 +++++------ .../chronicle/map/ChronicleMapBuilder.java | 14 ++++++++-- .../openhft/chronicle/map/NodeDiscovery.java | 11 ++++---- .../{Config.java => ReplicatorConfig.java} | 4 ++- .../openhft/chronicle/map/Replicators.java | 4 +-- .../chronicle/map/SerializationBuilder.java | 2 +- ...va => TcpReplicationReplicatorConfig.java} | 28 +++++++++---------- .../openhft/chronicle/map/TcpReplicator.java | 4 +-- .../chronicle/map/UdpChannelReplicator.java | 2 +- ...va => UdpReplicationReplicatorConfig.java} | 14 +++++----- ...> UdpReplicationReplicatorConfigBean.java} | 8 +++--- .../openhft/chronicle/map/UdpReplicator.java | 2 +- .../chronicle/set/ChronicleSetBuilder.java | 25 +++++++---------- .../net/openhft/chronicle/map/Builder.java | 5 ++-- .../chronicle/map/ChannelReplicationTest.java | 6 ++-- .../map/PostChannelCreationTest.java | 4 +-- .../chronicle/map/SingleMapClusterTest.java | 10 ++++--- ...cketReplicationTest3wayPutReturnsNull.java | 4 +-- .../map/TwoMapOnDifferentServers.java | 13 ++++----- .../map/UDPSocketReplicationTest.java | 6 ++-- .../SetForTCPSocketReplicationTest3way.java | 7 ++--- 22 files changed, 97 insertions(+), 94 deletions(-) rename src/main/java/net/openhft/chronicle/map/{Config.java => ReplicatorConfig.java} (62%) rename src/main/java/net/openhft/chronicle/map/{TcpReplicationConfig.java => TcpReplicationReplicatorConfig.java} (83%) rename src/main/java/net/openhft/chronicle/map/{UdpReplicationConfig.java => UdpReplicationReplicatorConfig.java} (77%) rename src/main/java/net/openhft/chronicle/map/{UdpReplicationConfigBean.java => UdpReplicationReplicatorConfigBean.java} (88%) diff --git a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java index 20cd595b5..a5dd0306f 100644 --- a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java +++ b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java @@ -189,7 +189,7 @@ public interface ChronicleHashBuilder implements Cloneable, return toString().hashCode(); } - public ChronicleMapBuilder replicators(byte identifier, Replicator... replicators) { + public ChronicleMapBuilder replicators(byte identifier, ReplicatorConfig... replicatorConfigs) { this.identifier = identifier; this.replicators.clear(); - for (Replicator replicator : replicators) { + for (ReplicatorConfig replicatorConfig : replicatorConfigs) { + + Replicator replicator; + if (replicatorConfig instanceof TcpReplicationReplicatorConfig) { + replicator = Replicators.tcp((TcpReplicationReplicatorConfig) replicatorConfig); + } else if (replicatorConfig instanceof UdpReplicationReplicatorConfig) { + replicator = Replicators.udp((UdpReplicationReplicatorConfig) replicatorConfig); + } else + throw new UnsupportedOperationException(); + + this.replicators.put(replicator.getClass(), replicator); } diff --git a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java index 26736295c..ba9fa9d60 100644 --- a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java +++ b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java @@ -25,8 +25,7 @@ import static net.openhft.chronicle.map.ConcurrentExpiryMap.getDefaultAddress; import static net.openhft.chronicle.map.DiscoveryNodeBytesMarshallable.ProposedNodes; import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.BOOTSTRAP_BYTES; import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.LOG; -import static net.openhft.chronicle.map.Replicators.tcp; -import static net.openhft.chronicle.map.UdpReplicationConfig.simple; +import static net.openhft.chronicle.map.UdpReplicationReplicatorConfig.simple; /** * @author Rob Austin. @@ -36,7 +35,7 @@ import static net.openhft.chronicle.map.UdpReplicationConfig.simple; public class NodeDiscovery { private final AddressAndPort ourAddressAndPort; - private final UdpReplicationConfig udpConfig; + private final UdpReplicationReplicatorConfig udpConfig; private final DiscoveryNodeBytesMarshallable discoveryNodeBytesMarshallable; private final AtomicReference nodeDiscoveryEventListenerAtomicReference = new AtomicReference(); @@ -201,7 +200,7 @@ public class NodeDiscovery { // add our identifier and host:port to the list of known identifiers knownNodes.add(ourAddressAndPort, identifier); - final TcpReplicationConfig tcpConfig = TcpReplicationConfig + final TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig .of(ourAddressAndPort.getPort(), toInetSocketArray(knownHostPorts)) .heartBeatInterval(1, SECONDS).nonUniqueIdentifierListener(identifierListener); @@ -212,7 +211,7 @@ public class NodeDiscovery { return ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000L) - .replicators(identifier, tcp(tcpConfig)).create(); + .replicators(identifier, tcpConfig).create(); } @@ -340,7 +339,7 @@ class NodeDiscoveryBroadcaster extends UdpChannelReplicator { * @throws java.io.IOException */ NodeDiscoveryBroadcaster( - @NotNull final UdpReplicationConfig replicationConfig, + @NotNull final UdpReplicationReplicatorConfig replicationConfig, final int serializedEntrySize, final BytesMarshallable externalizable) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/map/Config.java b/src/main/java/net/openhft/chronicle/map/ReplicatorConfig.java similarity index 62% rename from src/main/java/net/openhft/chronicle/map/Config.java rename to src/main/java/net/openhft/chronicle/map/ReplicatorConfig.java index 6e0d9a436..d99514f25 100644 --- a/src/main/java/net/openhft/chronicle/map/Config.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatorConfig.java @@ -3,5 +3,7 @@ package net.openhft.chronicle.map; /** * @author Rob Austin. */ -public interface Config { +public interface ReplicatorConfig { + + } diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java index faa766e85..f335ec416 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicators.java +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -25,7 +25,7 @@ public final class Replicators { private Replicators() { } - public static Replicator tcp(final TcpReplicationConfig replicationConfig) { + public static Replicator tcp(final TcpReplicationReplicatorConfig replicationConfig) { return new Replicator() { @Override @@ -39,7 +39,7 @@ public final class Replicators { } public static Replicator udp( - final UdpReplicationConfig replicationConfig) { + final UdpReplicationReplicatorConfig replicationConfig) { return new Replicator() { @Override diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index 4973970bb..282da7327 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -80,7 +80,7 @@ final class SerializationBuilder implements Cloneable { if (role == Role.VALUE) { factory = concreteClass(eClass) && marshallerUseFactory(eClass) ? new NewInstanceObjectFactory(eClass) : - NullObjectFactory.of(); + (ObjectFactory) NullObjectFactory.of(); } if (concreteClass(eClass) && Byteable.class.isAssignableFrom(eClass)) { diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java b/src/main/java/net/openhft/chronicle/map/TcpReplicationReplicatorConfig.java similarity index 83% rename from src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java rename to src/main/java/net/openhft/chronicle/map/TcpReplicationReplicatorConfig.java index 5ea8cb1c1..53d1a4614 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicationReplicatorConfig.java @@ -29,7 +29,7 @@ import static java.util.Collections.unmodifiableSet; import static java.util.concurrent.TimeUnit.SECONDS; -public class TcpReplicationConfig implements Config { +public class TcpReplicationReplicatorConfig implements ReplicatorConfig { private static final int DEFAULT_PACKET_SIZE = 1024 * 8; private static final long DEFAULT_HEART_BEAT_INTERVAL = 20; @@ -45,7 +45,7 @@ public class TcpReplicationConfig implements Config { private TimeUnit heartBeatIntervalUnit = DEFAULT_HEART_BEAT_INTERVAL_UNIT; public IdentifierListener identifierListener; - TcpReplicationConfig() { + TcpReplicationReplicatorConfig() { } @@ -53,7 +53,7 @@ public class TcpReplicationConfig implements Config { return identifierListener; } - public TcpReplicationConfig nonUniqueIdentifierListener(@NotNull final IdentifierListener + public TcpReplicationReplicatorConfig nonUniqueIdentifierListener(@NotNull final IdentifierListener identifierListener) { this.identifierListener = identifierListener; return this; @@ -63,7 +63,7 @@ public class TcpReplicationConfig implements Config { return autoReconnectedUponDroppedConnection; } - public TcpReplicationConfig autoReconnectedUponDroppedConnection(boolean autoReconnectedUponDroppedConnection) { + public TcpReplicationReplicatorConfig autoReconnectedUponDroppedConnection(boolean autoReconnectedUponDroppedConnection) { this.autoReconnectedUponDroppedConnection = autoReconnectedUponDroppedConnection; return this; } @@ -72,7 +72,7 @@ public class TcpReplicationConfig implements Config { return throttlingConfig; } - public TcpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { + public TcpReplicationReplicatorConfig throttlingConfig(ThrottlingConfig throttlingConfig) { this.throttlingConfig = throttlingConfig; return this; } @@ -90,7 +90,7 @@ public class TcpReplicationConfig implements Config { return serverPort; } - public TcpReplicationConfig serverPort(int serverPort) { + public TcpReplicationReplicatorConfig serverPort(int serverPort) { this.serverPort = serverPort; return this; } @@ -99,7 +99,7 @@ public class TcpReplicationConfig implements Config { return endpoints; } - public TcpReplicationConfig endpoints(Set endpoints) { + public TcpReplicationReplicatorConfig endpoints(Set endpoints) { for (final InetSocketAddress endpoint : endpoints) { if (endpoint.getPort() == serverPort && "localhost".equals(endpoint.getHostName())) @@ -115,11 +115,11 @@ public class TcpReplicationConfig implements Config { return packetSize; } - public static TcpReplicationConfig of(int serverPort, InetSocketAddress... endpoints) { + public static TcpReplicationReplicatorConfig of(int serverPort, InetSocketAddress... endpoints) { return of(serverPort, Arrays.asList(endpoints)); } - public static TcpReplicationConfig of(int serverPort, Collection endpoints) { + public static TcpReplicationReplicatorConfig of(int serverPort, Collection endpoints) { // at least in tests, we configure "receive-only" replication without endpoints. // TODO decide what to do with this case // if (endpoints.isEmpty()) @@ -130,7 +130,7 @@ public class TcpReplicationConfig implements Config { + " can not point to the same port as the server"); } - TcpReplicationConfig tcpReplicationConfig = new TcpReplicationConfig(); + TcpReplicationReplicatorConfig tcpReplicationConfig = new TcpReplicationReplicatorConfig(); tcpReplicationConfig.serverPort(serverPort); tcpReplicationConfig.endpoints = unmodifiableSet(new HashSet(endpoints)); @@ -141,14 +141,14 @@ public class TcpReplicationConfig implements Config { } - public TcpReplicationConfig packetSize(int packetSize) { + public TcpReplicationReplicatorConfig packetSize(int packetSize) { if (packetSize <= 0) throw new IllegalArgumentException(); this.packetSize = packetSize; return this; } - public TcpReplicationConfig throttlingConfig(long heartBeatInterval, TimeUnit heartBeatIntervalUnit) { + public TcpReplicationReplicatorConfig throttlingConfig(long heartBeatInterval, TimeUnit heartBeatIntervalUnit) { this.heartBeatInterval = heartBeatInterval; this.heartBeatIntervalUnit = heartBeatIntervalUnit; return this; @@ -160,7 +160,7 @@ public class TcpReplicationConfig implements Config { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - TcpReplicationConfig that = (TcpReplicationConfig) o; + TcpReplicationReplicatorConfig that = (TcpReplicationReplicatorConfig) o; if (autoReconnectedUponDroppedConnection != that.autoReconnectedUponDroppedConnection) return false; if (heartBeatInterval != that.heartBeatInterval) return false; @@ -199,7 +199,7 @@ public class TcpReplicationConfig implements Config { '}'; } - public TcpReplicationConfig heartBeatInterval(long time, TimeUnit unit) { + public TcpReplicationReplicatorConfig heartBeatInterval(long time, TimeUnit unit) { this.heartBeatInterval = time; this.heartBeatIntervalUnit = unit; return this; diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 61a013ce3..3dc3a5732 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -55,7 +55,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private final byte localIdentifier; private final int maxEntrySizeBytes; private final Replica.EntryExternalizable externalizable; - private final TcpReplicationConfig replicationConfig; + private final TcpReplicationReplicatorConfig replicationConfig; private long selectorTimeout; @@ -68,7 +68,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { */ TcpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable externalizable, - @NotNull final TcpReplicationConfig replicationConfig, + @NotNull final TcpReplicationReplicatorConfig replicationConfig, final int maxEntrySizeBytes) throws IOException { super("TcpSocketReplicator-" + replica.identifier(), replicationConfig.throttlingConfig(), diff --git a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java index 26c20d123..e253c4d26 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java @@ -66,7 +66,7 @@ class UdpChannelReplicator extends AbstractChannelReplicator implements Replica. * @param localIdentifier just used for logging * @throws IOException */ - UdpChannelReplicator(@NotNull final UdpReplicationConfig replicationConfig, + UdpChannelReplicator(@NotNull final UdpReplicationReplicatorConfig replicationConfig, final int maxPayloadSize, final byte localIdentifier) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java b/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfig.java similarity index 77% rename from src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java rename to src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfig.java index a1e8ad0aa..ad4ebd862 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfig.java @@ -23,16 +23,16 @@ import java.net.InetAddress; import java.net.NetworkInterface; -public abstract class UdpReplicationConfig implements Config { +public abstract class UdpReplicationReplicatorConfig implements ReplicatorConfig { /** * Package-private constructor forbids subclassing from outside of the package */ - UdpReplicationConfig() { + UdpReplicationReplicatorConfig() { // nothing to do } - public static UdpReplicationConfig simple(@NotNull InetAddress address, int port) { + public static UdpReplicationReplicatorConfig simple(@NotNull InetAddress address, int port) { if (address.isMulticastAddress()) throw new IllegalArgumentException(); @@ -40,17 +40,17 @@ public abstract class UdpReplicationConfig implements Config { return create(address, port, null, ThrottlingConfig.noThrottling()); } - public static UdpReplicationConfig multiCast(@NotNull InetAddress address, int port, + public static UdpReplicationReplicatorConfig multiCast(@NotNull InetAddress address, int port, @NotNull NetworkInterface networkInterface) { if (!address.isMulticastAddress() || networkInterface == null) throw new IllegalArgumentException(); return create(address, port, networkInterface, ThrottlingConfig.noThrottling()); } - static UdpReplicationConfig create(InetAddress address, int port, + static UdpReplicationReplicatorConfig create(InetAddress address, int port, NetworkInterface networkInterface, ThrottlingConfig throttlingConfig) { - return new UdpReplicationConfigBean(address, port, networkInterface, + return new UdpReplicationReplicatorConfigBean(address, port, networkInterface, throttlingConfig); } @@ -63,7 +63,7 @@ public abstract class UdpReplicationConfig implements Config { public abstract ThrottlingConfig throttlingConfig(); - public UdpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { + public UdpReplicationReplicatorConfig throttlingConfig(ThrottlingConfig throttlingConfig) { ThrottlingConfig.checkMillisecondBucketInterval(throttlingConfig, "UDP"); return create(address(), port(), networkInterface(), throttlingConfig); } diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicationConfigBean.java b/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfigBean.java similarity index 88% rename from src/main/java/net/openhft/chronicle/map/UdpReplicationConfigBean.java rename to src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfigBean.java index 441aa4eac..120c88747 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicationConfigBean.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfigBean.java @@ -4,13 +4,13 @@ import java.net.InetAddress; import java.net.NetworkInterface; -final class UdpReplicationConfigBean extends UdpReplicationConfig { +final class UdpReplicationReplicatorConfigBean extends UdpReplicationReplicatorConfig { private final InetAddress address; private final int port; private final NetworkInterface networkInterface; private final ThrottlingConfig throttlingConfig; - UdpReplicationConfigBean( + UdpReplicationReplicatorConfigBean( InetAddress address, int port, NetworkInterface networkInterface, @@ -63,8 +63,8 @@ final class UdpReplicationConfigBean extends UdpReplicationConfig { if (o == this) { return true; } - if (o instanceof UdpReplicationConfig) { - UdpReplicationConfig that = (UdpReplicationConfig) o; + if (o instanceof UdpReplicationReplicatorConfig) { + UdpReplicationReplicatorConfig that = (UdpReplicationReplicatorConfig) o; return (this.address.equals(that.address())) && (this.port == that.port()) && (this.networkInterface.equals(that.networkInterface())) diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java index e32fac525..bb80dc6b1 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java @@ -40,7 +40,7 @@ class UdpReplicator extends UdpChannelReplicator implements Replica.Modification UdpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable entryExternalizable, - @NotNull final UdpReplicationConfig replicationConfig, + @NotNull final UdpReplicationReplicatorConfig replicationConfig, final int serializedEntrySize) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index d4d2b47c1..ede9d2189 100644 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -87,9 +87,9 @@ public class ChronicleSetBuilder * .keySize(10) * .create();} * - *

(Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded - * (and each character takes 2 bytes on-heap), because default off-heap {@link String} encoding - * is UTF-8 in {@code ChronicleSet}.) + *

(Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded (and each + * character takes 2 bytes on-heap), because default off-heap {@link String} encoding is UTF-8 in {@code + * ChronicleSet}.) * * @see #constantKeySizeBySample(Object) * @see #entrySize(int) @@ -119,13 +119,13 @@ public class ChronicleSetBuilder /** * {@inheritDoc} * - *

In fully default case you can expect entry size to be about 120-130 bytes. But it is - * strongly recommended always to configure {@linkplain #keySize(int) key size}, - * if they couldn't be derived statically. + *

In fully default case you can expect entry size to be about 120-130 bytes. But it is strongly + * recommended always to configure {@linkplain #keySize(int) key size}, if they couldn't be derived + * statically. * *

If entry size is not configured explicitly by calling this method, it is computed based on - * {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size}, - * plus a few bytes required by implementations. + * {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size}, plus a few + * bytes required by implementations. */ @Override public ChronicleSetBuilder entrySize(int entrySize) { @@ -164,11 +164,12 @@ public class ChronicleSetBuilder } @Override - public ChronicleSetBuilder replicators(byte identifier, Replicator... replicators) { + public ChronicleSetBuilder replicators(byte identifier, ReplicatorConfig... replicators) { chronicleMapBuilder.replicators(identifier, replicators); return this; } + @Override public String toString() { return " ChronicleSetBuilder{" + @@ -189,12 +190,6 @@ public class ChronicleSetBuilder return toString().hashCode(); } - public ChronicleSetBuilder replicators(byte identifier, Replicator replicator) { - chronicleMapBuilder.replicators(identifier, replicator); - return this; - } - - @Override public ChronicleSetBuilder timeProvider(TimeProvider timeProvider) { chronicleMapBuilder.timeProvider(timeProvider); diff --git a/src/test/java/net/openhft/chronicle/map/Builder.java b/src/test/java/net/openhft/chronicle/map/Builder.java index 1799e16c3..64a4fbf86 100644 --- a/src/test/java/net/openhft/chronicle/map/Builder.java +++ b/src/test/java/net/openhft/chronicle/map/Builder.java @@ -23,7 +23,6 @@ import java.io.RandomAccessFile; import java.net.InetSocketAddress; import static java.util.concurrent.TimeUnit.SECONDS; -import static net.openhft.chronicle.map.Replicators.tcp; /** * @author Rob Austin. @@ -69,11 +68,11 @@ public class Builder { final byte identifier, final int serverPort, final InetSocketAddress... endpoints) throws IOException { - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints) + TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig.of(serverPort, endpoints) .heartBeatInterval(1L, SECONDS); return ChronicleMapBuilder.of(kClass, vClass) .entries(20000L) - .replicators(identifier, tcp( tcpConfig)); + .replicators(identifier, tcpConfig); } diff --git a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java index 086dffb86..22e2fdfe6 100644 --- a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java @@ -51,7 +51,7 @@ public class ChannelReplicationTest { public void setup() throws IOException { { - TcpReplicationConfig tcpConfig = TcpReplicationConfig + TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); @@ -66,8 +66,8 @@ public class ChannelReplicationTest { } { - TcpReplicationConfig tcpConfig = - TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); + TcpReplicationReplicatorConfig tcpConfig = + TcpReplicationReplicatorConfig.of(8087).heartBeatInterval(1, SECONDS); channelProviderB = new ChannelProviderBuilder().replicators((byte) 2, tcpConfig) .create(); diff --git a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java index f5292974c..e44e674ed 100644 --- a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java @@ -59,7 +59,7 @@ public class PostChannelCreationTest { public void setup() throws IOException { { - final TcpReplicationConfig tcpConfig = TcpReplicationConfig + final TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); @@ -80,7 +80,7 @@ public class PostChannelCreationTest { { - final TcpReplicationConfig tcpConfig = TcpReplicationConfig + final TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); diff --git a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java index 82ede45f6..1c75888ec 100644 --- a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java +++ b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java @@ -56,12 +56,14 @@ public class SingleMapClusterTest { public void setup() throws IOException { { - final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig + final TcpReplicationReplicatorConfig tcpReplicationConfig = TcpReplicationReplicatorConfig .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - + int maxEntrySize = 1024; + byte identifier= 1; channelProviderA = new ChannelProviderBuilder() - .replicators((byte) 1, tcpReplicationConfig).create(); + .maxEntrySize(maxEntrySize) + .replicators(identifier, tcpReplicationConfig).create(); // this is how you add maps after the custer is created map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) @@ -71,7 +73,7 @@ public class SingleMapClusterTest { { - final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig + final TcpReplicationReplicatorConfig tcpReplicationConfig = TcpReplicationReplicatorConfig .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java index d4bd0a076..681550355 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java @@ -44,12 +44,12 @@ public class TCPSocketReplicationTest3wayPutReturnsNull { final byte identifier, final int serverPort, final InetSocketAddress... endpoints) throws IOException { - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints); + TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig.of(serverPort, endpoints); return (T) ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) .putReturnsNull(true) - .replicators(identifier, Replicators.tcp(tcpConfig)) + .replicators(identifier, tcpConfig) .create(Builder.getPersistenceFile()); } diff --git a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java index 59199827d..4616e26d8 100644 --- a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java +++ b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java @@ -27,7 +27,7 @@ import java.net.InetSocketAddress; import static java.util.concurrent.TimeUnit.SECONDS; import static net.openhft.chronicle.map.Builder.getPersistenceFile; -import static net.openhft.chronicle.map.Replicators.tcp; +import static net.openhft.chronicle.map.TcpReplicationReplicatorConfig.of; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -43,21 +43,20 @@ public class TwoMapOnDifferentServers { @Before public void setup() throws IOException { - final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig - .of(8076, new InetSocketAddress("localhost", 8077)) - .heartBeatInterval(1, SECONDS); + final TcpReplicationReplicatorConfig tcpConfig = + of(8076, new InetSocketAddress("localhost", 8077)) + .heartBeatInterval(1, SECONDS); File persistenceFile = getPersistenceFile(); map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000) - .replicators((byte) 1, tcp(tcpReplicationConfig)) + .replicators((byte) 1, tcpConfig) .create(persistenceFile); map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000) - .replicators((byte) 2, tcp( - TcpReplicationConfig.of(8077).heartBeatInterval(1, SECONDS))) + .replicators((byte) 2, of(8077).heartBeatInterval(1, SECONDS)) .create(persistenceFile); } diff --git a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java index 8bec68d2c..b47ee20de 100644 --- a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java @@ -23,8 +23,6 @@ import java.io.Closeable; import java.io.IOException; import java.net.Inet4Address; -import static net.openhft.chronicle.map.Replicators.udp; - /** * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket * @@ -39,12 +37,12 @@ public class UDPSocketReplicationTest { final int identifier, final int udpPort) throws IOException { - UdpReplicationConfig udpConfig = UdpReplicationConfig + UdpReplicationReplicatorConfig udpConfig = UdpReplicationReplicatorConfig .simple(Inet4Address.getByName("255.255.255.255"), udpPort); return ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .replicators((byte) identifier, udp(udpConfig)) + .replicators((byte) identifier, udpConfig) .create(Builder.getPersistenceFile()); } diff --git a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java index 0212db7f6..702d68152 100644 --- a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java +++ b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java @@ -27,8 +27,7 @@ import java.net.InetSocketAddress; import static java.util.concurrent.TimeUnit.SECONDS; import static net.openhft.chronicle.map.Builder.getPersistenceFile; -import static net.openhft.chronicle.map.Replicators.tcp; -import static net.openhft.chronicle.map.TcpReplicationConfig.of; +import static net.openhft.chronicle.map.TcpReplicationReplicatorConfig.of; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -51,8 +50,8 @@ public class SetForTCPSocketReplicationTest3way { final int serverPort, final InetSocketAddress... endpoints) throws IOException { return (T) ChronicleSetBuilder.of(Integer.class) - .replicators(identifier, tcp(of(serverPort, endpoints). - heartBeatInterval(1L, SECONDS))).create(getPersistenceFile()); + .replicators(identifier, of(serverPort, endpoints). + heartBeatInterval(1L, SECONDS)).create(getPersistenceFile()); } From 2d31786e36b013a73ec4312ee8d67f474d0ec5b3 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 10:43:02 +0100 Subject: [PATCH 18/64] HCOLL-163 kept the use of tcp() and udp() internal --- src/main/java/net/openhft/chronicle/map/Replicators.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java index f335ec416..48df2c12a 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicators.java +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -20,12 +20,12 @@ package net.openhft.chronicle.map; import java.io.Closeable; import java.io.IOException; -public final class Replicators { +final class Replicators { private Replicators() { } - public static Replicator tcp(final TcpReplicationReplicatorConfig replicationConfig) { + static Replicator tcp(final TcpReplicationReplicatorConfig replicationConfig) { return new Replicator() { @Override @@ -38,7 +38,7 @@ public final class Replicators { }; } - public static Replicator udp( + static Replicator udp( final UdpReplicationReplicatorConfig replicationConfig) { return new Replicator() { From d5f88842df128206512d570e9906328a1b070673 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 10:46:38 +0100 Subject: [PATCH 19/64] HCOLL-163 corrected bad naming of Udp and TCP ReplicationConfig --- README.md | 2 +- .../chronicle/ChronicleHashBuilder.java | 2 +- .../chronicle/map/ChannelProviderBuilder.java | 16 +++++------ .../chronicle/map/ChronicleMapBuilder.java | 12 ++++---- .../openhft/chronicle/map/NodeDiscovery.java | 8 +++--- ...atorConfig.java => ReplicationConfig.java} | 2 +- .../openhft/chronicle/map/Replicators.java | 4 +-- ...rConfig.java => TcpReplicationConfig.java} | 28 +++++++++---------- .../openhft/chronicle/map/TcpReplicator.java | 4 +-- .../chronicle/map/UdpChannelReplicator.java | 2 +- ...rConfig.java => UdpReplicationConfig.java} | 14 +++++----- ...ean.java => UdpReplicationConfigBean.java} | 8 +++--- .../openhft/chronicle/map/UdpReplicator.java | 2 +- .../chronicle/set/ChronicleSetBuilder.java | 2 +- .../net/openhft/chronicle/map/Builder.java | 2 +- .../chronicle/map/ChannelReplicationTest.java | 6 ++-- .../map/PostChannelCreationTest.java | 4 +-- .../chronicle/map/SingleMapClusterTest.java | 4 +-- ...cketReplicationTest3wayPutReturnsNull.java | 2 +- .../map/TwoMapOnDifferentServers.java | 4 +-- .../map/UDPSocketReplicationTest.java | 2 +- .../SetForTCPSocketReplicationTest3way.java | 2 +- 22 files changed, 66 insertions(+), 66 deletions(-) rename src/main/java/net/openhft/chronicle/map/{ReplicatorConfig.java => ReplicationConfig.java} (63%) rename src/main/java/net/openhft/chronicle/map/{TcpReplicationReplicatorConfig.java => TcpReplicationConfig.java} (83%) rename src/main/java/net/openhft/chronicle/map/{UdpReplicationReplicatorConfig.java => UdpReplicationConfig.java} (77%) rename src/main/java/net/openhft/chronicle/map/{UdpReplicationReplicatorConfigBean.java => UdpReplicationConfigBean.java} (88%) diff --git a/README.md b/README.md index c53dc22f6..c5f86244d 100644 --- a/README.md +++ b/README.md @@ -679,7 +679,7 @@ Map map2; ChronicleMapBuilder map1Builder = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000L) - .addReplicator(tcp((byte) 1, tcpConfig)); + .addReplicator((byte) 1, tcpConfig); map1 = map1Builder.create(); diff --git a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java index a5dd0306f..c37ea2cda 100644 --- a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java +++ b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java @@ -189,7 +189,7 @@ public interface ChronicleHashBuilder implements Cloneable, return toString().hashCode(); } - public ChronicleMapBuilder replicators(byte identifier, ReplicatorConfig... replicatorConfigs) { + public ChronicleMapBuilder replicators(byte identifier, ReplicationConfig... replicationConfigs) { this.identifier = identifier; this.replicators.clear(); - for (ReplicatorConfig replicatorConfig : replicatorConfigs) { + for (ReplicationConfig replicationConfig : replicationConfigs) { Replicator replicator; - if (replicatorConfig instanceof TcpReplicationReplicatorConfig) { - replicator = Replicators.tcp((TcpReplicationReplicatorConfig) replicatorConfig); - } else if (replicatorConfig instanceof UdpReplicationReplicatorConfig) { - replicator = Replicators.udp((UdpReplicationReplicatorConfig) replicatorConfig); + if (replicationConfig instanceof TcpReplicationConfig) { + replicator = Replicators.tcp((TcpReplicationConfig) replicationConfig); + } else if (replicationConfig instanceof UdpReplicationConfig) { + replicator = Replicators.udp((UdpReplicationConfig) replicationConfig); } else throw new UnsupportedOperationException(); diff --git a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java index ba9fa9d60..ad2723a86 100644 --- a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java +++ b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java @@ -25,7 +25,7 @@ import static net.openhft.chronicle.map.ConcurrentExpiryMap.getDefaultAddress; import static net.openhft.chronicle.map.DiscoveryNodeBytesMarshallable.ProposedNodes; import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.BOOTSTRAP_BYTES; import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.LOG; -import static net.openhft.chronicle.map.UdpReplicationReplicatorConfig.simple; +import static net.openhft.chronicle.map.UdpReplicationConfig.simple; /** * @author Rob Austin. @@ -35,7 +35,7 @@ import static net.openhft.chronicle.map.UdpReplicationReplicatorConfig.simple; public class NodeDiscovery { private final AddressAndPort ourAddressAndPort; - private final UdpReplicationReplicatorConfig udpConfig; + private final UdpReplicationConfig udpConfig; private final DiscoveryNodeBytesMarshallable discoveryNodeBytesMarshallable; private final AtomicReference nodeDiscoveryEventListenerAtomicReference = new AtomicReference(); @@ -200,7 +200,7 @@ public class NodeDiscovery { // add our identifier and host:port to the list of known identifiers knownNodes.add(ourAddressAndPort, identifier); - final TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig + final TcpReplicationConfig tcpConfig = TcpReplicationConfig .of(ourAddressAndPort.getPort(), toInetSocketArray(knownHostPorts)) .heartBeatInterval(1, SECONDS).nonUniqueIdentifierListener(identifierListener); @@ -339,7 +339,7 @@ class NodeDiscoveryBroadcaster extends UdpChannelReplicator { * @throws java.io.IOException */ NodeDiscoveryBroadcaster( - @NotNull final UdpReplicationReplicatorConfig replicationConfig, + @NotNull final UdpReplicationConfig replicationConfig, final int serializedEntrySize, final BytesMarshallable externalizable) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatorConfig.java b/src/main/java/net/openhft/chronicle/map/ReplicationConfig.java similarity index 63% rename from src/main/java/net/openhft/chronicle/map/ReplicatorConfig.java rename to src/main/java/net/openhft/chronicle/map/ReplicationConfig.java index d99514f25..a72abed48 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatorConfig.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicationConfig.java @@ -3,7 +3,7 @@ package net.openhft.chronicle.map; /** * @author Rob Austin. */ -public interface ReplicatorConfig { +public interface ReplicationConfig { } diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java index 48df2c12a..023e8a7b2 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicators.java +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -25,7 +25,7 @@ final class Replicators { private Replicators() { } - static Replicator tcp(final TcpReplicationReplicatorConfig replicationConfig) { + static Replicator tcp(final TcpReplicationConfig replicationConfig) { return new Replicator() { @Override @@ -39,7 +39,7 @@ final class Replicators { } static Replicator udp( - final UdpReplicationReplicatorConfig replicationConfig) { + final UdpReplicationConfig replicationConfig) { return new Replicator() { @Override diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicationReplicatorConfig.java b/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java similarity index 83% rename from src/main/java/net/openhft/chronicle/map/TcpReplicationReplicatorConfig.java rename to src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java index 53d1a4614..5bce434ab 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicationReplicatorConfig.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java @@ -29,7 +29,7 @@ import static java.util.Collections.unmodifiableSet; import static java.util.concurrent.TimeUnit.SECONDS; -public class TcpReplicationReplicatorConfig implements ReplicatorConfig { +public class TcpReplicationConfig implements ReplicationConfig { private static final int DEFAULT_PACKET_SIZE = 1024 * 8; private static final long DEFAULT_HEART_BEAT_INTERVAL = 20; @@ -45,7 +45,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { private TimeUnit heartBeatIntervalUnit = DEFAULT_HEART_BEAT_INTERVAL_UNIT; public IdentifierListener identifierListener; - TcpReplicationReplicatorConfig() { + TcpReplicationConfig() { } @@ -53,7 +53,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { return identifierListener; } - public TcpReplicationReplicatorConfig nonUniqueIdentifierListener(@NotNull final IdentifierListener + public TcpReplicationConfig nonUniqueIdentifierListener(@NotNull final IdentifierListener identifierListener) { this.identifierListener = identifierListener; return this; @@ -63,7 +63,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { return autoReconnectedUponDroppedConnection; } - public TcpReplicationReplicatorConfig autoReconnectedUponDroppedConnection(boolean autoReconnectedUponDroppedConnection) { + public TcpReplicationConfig autoReconnectedUponDroppedConnection(boolean autoReconnectedUponDroppedConnection) { this.autoReconnectedUponDroppedConnection = autoReconnectedUponDroppedConnection; return this; } @@ -72,7 +72,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { return throttlingConfig; } - public TcpReplicationReplicatorConfig throttlingConfig(ThrottlingConfig throttlingConfig) { + public TcpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { this.throttlingConfig = throttlingConfig; return this; } @@ -90,7 +90,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { return serverPort; } - public TcpReplicationReplicatorConfig serverPort(int serverPort) { + public TcpReplicationConfig serverPort(int serverPort) { this.serverPort = serverPort; return this; } @@ -99,7 +99,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { return endpoints; } - public TcpReplicationReplicatorConfig endpoints(Set endpoints) { + public TcpReplicationConfig endpoints(Set endpoints) { for (final InetSocketAddress endpoint : endpoints) { if (endpoint.getPort() == serverPort && "localhost".equals(endpoint.getHostName())) @@ -115,11 +115,11 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { return packetSize; } - public static TcpReplicationReplicatorConfig of(int serverPort, InetSocketAddress... endpoints) { + public static TcpReplicationConfig of(int serverPort, InetSocketAddress... endpoints) { return of(serverPort, Arrays.asList(endpoints)); } - public static TcpReplicationReplicatorConfig of(int serverPort, Collection endpoints) { + public static TcpReplicationConfig of(int serverPort, Collection endpoints) { // at least in tests, we configure "receive-only" replication without endpoints. // TODO decide what to do with this case // if (endpoints.isEmpty()) @@ -130,7 +130,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { + " can not point to the same port as the server"); } - TcpReplicationReplicatorConfig tcpReplicationConfig = new TcpReplicationReplicatorConfig(); + TcpReplicationConfig tcpReplicationConfig = new TcpReplicationConfig(); tcpReplicationConfig.serverPort(serverPort); tcpReplicationConfig.endpoints = unmodifiableSet(new HashSet(endpoints)); @@ -141,14 +141,14 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { } - public TcpReplicationReplicatorConfig packetSize(int packetSize) { + public TcpReplicationConfig packetSize(int packetSize) { if (packetSize <= 0) throw new IllegalArgumentException(); this.packetSize = packetSize; return this; } - public TcpReplicationReplicatorConfig throttlingConfig(long heartBeatInterval, TimeUnit heartBeatIntervalUnit) { + public TcpReplicationConfig throttlingConfig(long heartBeatInterval, TimeUnit heartBeatIntervalUnit) { this.heartBeatInterval = heartBeatInterval; this.heartBeatIntervalUnit = heartBeatIntervalUnit; return this; @@ -160,7 +160,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - TcpReplicationReplicatorConfig that = (TcpReplicationReplicatorConfig) o; + TcpReplicationConfig that = (TcpReplicationConfig) o; if (autoReconnectedUponDroppedConnection != that.autoReconnectedUponDroppedConnection) return false; if (heartBeatInterval != that.heartBeatInterval) return false; @@ -199,7 +199,7 @@ public class TcpReplicationReplicatorConfig implements ReplicatorConfig { '}'; } - public TcpReplicationReplicatorConfig heartBeatInterval(long time, TimeUnit unit) { + public TcpReplicationConfig heartBeatInterval(long time, TimeUnit unit) { this.heartBeatInterval = time; this.heartBeatIntervalUnit = unit; return this; diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 3dc3a5732..61a013ce3 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -55,7 +55,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private final byte localIdentifier; private final int maxEntrySizeBytes; private final Replica.EntryExternalizable externalizable; - private final TcpReplicationReplicatorConfig replicationConfig; + private final TcpReplicationConfig replicationConfig; private long selectorTimeout; @@ -68,7 +68,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { */ TcpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable externalizable, - @NotNull final TcpReplicationReplicatorConfig replicationConfig, + @NotNull final TcpReplicationConfig replicationConfig, final int maxEntrySizeBytes) throws IOException { super("TcpSocketReplicator-" + replica.identifier(), replicationConfig.throttlingConfig(), diff --git a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java index e253c4d26..26c20d123 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java @@ -66,7 +66,7 @@ class UdpChannelReplicator extends AbstractChannelReplicator implements Replica. * @param localIdentifier just used for logging * @throws IOException */ - UdpChannelReplicator(@NotNull final UdpReplicationReplicatorConfig replicationConfig, + UdpChannelReplicator(@NotNull final UdpReplicationConfig replicationConfig, final int maxPayloadSize, final byte localIdentifier) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfig.java b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java similarity index 77% rename from src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfig.java rename to src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java index ad4ebd862..15a3382d7 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfig.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java @@ -23,16 +23,16 @@ import java.net.InetAddress; import java.net.NetworkInterface; -public abstract class UdpReplicationReplicatorConfig implements ReplicatorConfig { +public abstract class UdpReplicationConfig implements ReplicationConfig { /** * Package-private constructor forbids subclassing from outside of the package */ - UdpReplicationReplicatorConfig() { + UdpReplicationConfig() { // nothing to do } - public static UdpReplicationReplicatorConfig simple(@NotNull InetAddress address, int port) { + public static UdpReplicationConfig simple(@NotNull InetAddress address, int port) { if (address.isMulticastAddress()) throw new IllegalArgumentException(); @@ -40,17 +40,17 @@ public abstract class UdpReplicationReplicatorConfig implements ReplicatorConfig return create(address, port, null, ThrottlingConfig.noThrottling()); } - public static UdpReplicationReplicatorConfig multiCast(@NotNull InetAddress address, int port, + public static UdpReplicationConfig multiCast(@NotNull InetAddress address, int port, @NotNull NetworkInterface networkInterface) { if (!address.isMulticastAddress() || networkInterface == null) throw new IllegalArgumentException(); return create(address, port, networkInterface, ThrottlingConfig.noThrottling()); } - static UdpReplicationReplicatorConfig create(InetAddress address, int port, + static UdpReplicationConfig create(InetAddress address, int port, NetworkInterface networkInterface, ThrottlingConfig throttlingConfig) { - return new UdpReplicationReplicatorConfigBean(address, port, networkInterface, + return new UdpReplicationConfigBean(address, port, networkInterface, throttlingConfig); } @@ -63,7 +63,7 @@ public abstract class UdpReplicationReplicatorConfig implements ReplicatorConfig public abstract ThrottlingConfig throttlingConfig(); - public UdpReplicationReplicatorConfig throttlingConfig(ThrottlingConfig throttlingConfig) { + public UdpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { ThrottlingConfig.checkMillisecondBucketInterval(throttlingConfig, "UDP"); return create(address(), port(), networkInterface(), throttlingConfig); } diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfigBean.java b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfigBean.java similarity index 88% rename from src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfigBean.java rename to src/main/java/net/openhft/chronicle/map/UdpReplicationConfigBean.java index 120c88747..441aa4eac 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicationReplicatorConfigBean.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfigBean.java @@ -4,13 +4,13 @@ import java.net.InetAddress; import java.net.NetworkInterface; -final class UdpReplicationReplicatorConfigBean extends UdpReplicationReplicatorConfig { +final class UdpReplicationConfigBean extends UdpReplicationConfig { private final InetAddress address; private final int port; private final NetworkInterface networkInterface; private final ThrottlingConfig throttlingConfig; - UdpReplicationReplicatorConfigBean( + UdpReplicationConfigBean( InetAddress address, int port, NetworkInterface networkInterface, @@ -63,8 +63,8 @@ final class UdpReplicationReplicatorConfigBean extends UdpReplicationReplicatorC if (o == this) { return true; } - if (o instanceof UdpReplicationReplicatorConfig) { - UdpReplicationReplicatorConfig that = (UdpReplicationReplicatorConfig) o; + if (o instanceof UdpReplicationConfig) { + UdpReplicationConfig that = (UdpReplicationConfig) o; return (this.address.equals(that.address())) && (this.port == that.port()) && (this.networkInterface.equals(that.networkInterface())) diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java index bb80dc6b1..e32fac525 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java @@ -40,7 +40,7 @@ class UdpReplicator extends UdpChannelReplicator implements Replica.Modification UdpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable entryExternalizable, - @NotNull final UdpReplicationReplicatorConfig replicationConfig, + @NotNull final UdpReplicationConfig replicationConfig, final int serializedEntrySize) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index ede9d2189..1e5eb3310 100644 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -164,7 +164,7 @@ public class ChronicleSetBuilder } @Override - public ChronicleSetBuilder replicators(byte identifier, ReplicatorConfig... replicators) { + public ChronicleSetBuilder replicators(byte identifier, ReplicationConfig... replicators) { chronicleMapBuilder.replicators(identifier, replicators); return this; } diff --git a/src/test/java/net/openhft/chronicle/map/Builder.java b/src/test/java/net/openhft/chronicle/map/Builder.java index 64a4fbf86..4c1abddc9 100644 --- a/src/test/java/net/openhft/chronicle/map/Builder.java +++ b/src/test/java/net/openhft/chronicle/map/Builder.java @@ -68,7 +68,7 @@ public class Builder { final byte identifier, final int serverPort, final InetSocketAddress... endpoints) throws IOException { - TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig.of(serverPort, endpoints) + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints) .heartBeatInterval(1L, SECONDS); return ChronicleMapBuilder.of(kClass, vClass) .entries(20000L) diff --git a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java index 22e2fdfe6..086dffb86 100644 --- a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java @@ -51,7 +51,7 @@ public class ChannelReplicationTest { public void setup() throws IOException { { - TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig + TcpReplicationConfig tcpConfig = TcpReplicationConfig .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); @@ -66,8 +66,8 @@ public class ChannelReplicationTest { } { - TcpReplicationReplicatorConfig tcpConfig = - TcpReplicationReplicatorConfig.of(8087).heartBeatInterval(1, SECONDS); + TcpReplicationConfig tcpConfig = + TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); channelProviderB = new ChannelProviderBuilder().replicators((byte) 2, tcpConfig) .create(); diff --git a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java index e44e674ed..f5292974c 100644 --- a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java @@ -59,7 +59,7 @@ public class PostChannelCreationTest { public void setup() throws IOException { { - final TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig + final TcpReplicationConfig tcpConfig = TcpReplicationConfig .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); @@ -80,7 +80,7 @@ public class PostChannelCreationTest { { - final TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig + final TcpReplicationConfig tcpConfig = TcpReplicationConfig .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); diff --git a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java index 1c75888ec..99c41eefa 100644 --- a/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java +++ b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java @@ -56,7 +56,7 @@ public class SingleMapClusterTest { public void setup() throws IOException { { - final TcpReplicationReplicatorConfig tcpReplicationConfig = TcpReplicationReplicatorConfig + final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig .of(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); int maxEntrySize = 1024; @@ -73,7 +73,7 @@ public class SingleMapClusterTest { { - final TcpReplicationReplicatorConfig tcpReplicationConfig = TcpReplicationReplicatorConfig + final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig .of(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java index 681550355..e8925e4fb 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java @@ -44,7 +44,7 @@ public class TCPSocketReplicationTest3wayPutReturnsNull { final byte identifier, final int serverPort, final InetSocketAddress... endpoints) throws IOException { - TcpReplicationReplicatorConfig tcpConfig = TcpReplicationReplicatorConfig.of(serverPort, endpoints); + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints); return (T) ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) diff --git a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java index 4616e26d8..fd418c0c5 100644 --- a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java +++ b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java @@ -27,7 +27,7 @@ import java.net.InetSocketAddress; import static java.util.concurrent.TimeUnit.SECONDS; import static net.openhft.chronicle.map.Builder.getPersistenceFile; -import static net.openhft.chronicle.map.TcpReplicationReplicatorConfig.of; +import static net.openhft.chronicle.map.TcpReplicationConfig.of; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -43,7 +43,7 @@ public class TwoMapOnDifferentServers { @Before public void setup() throws IOException { - final TcpReplicationReplicatorConfig tcpConfig = + final TcpReplicationConfig tcpConfig = of(8076, new InetSocketAddress("localhost", 8077)) .heartBeatInterval(1, SECONDS); diff --git a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java index b47ee20de..8a2550b74 100644 --- a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java @@ -37,7 +37,7 @@ public class UDPSocketReplicationTest { final int identifier, final int udpPort) throws IOException { - UdpReplicationReplicatorConfig udpConfig = UdpReplicationReplicatorConfig + UdpReplicationConfig udpConfig = UdpReplicationConfig .simple(Inet4Address.getByName("255.255.255.255"), udpPort); return ChronicleMapBuilder.of(Integer.class, CharSequence.class) diff --git a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java index 702d68152..57576a192 100644 --- a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java +++ b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java @@ -27,7 +27,7 @@ import java.net.InetSocketAddress; import static java.util.concurrent.TimeUnit.SECONDS; import static net.openhft.chronicle.map.Builder.getPersistenceFile; -import static net.openhft.chronicle.map.TcpReplicationReplicatorConfig.of; +import static net.openhft.chronicle.map.TcpReplicationConfig.of; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; From 4f2f2592dd0f31311b86a30f7d7797c64230b0da Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 10:54:06 +0100 Subject: [PATCH 20/64] HCOLL-163 updated readme --- README.md | 175 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 103 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index c5f86244d..a4bba415b 100644 --- a/README.md +++ b/README.md @@ -666,53 +666,69 @@ Lets assume that we had two server, lets call them server1 and server2, if we wi between them, this is how we could set it up ``` java -Map map1; -Map map2; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class YourClass { + + + @Test + public void test() throws IOException, InterruptedException { + + Map map1; + Map map2; // ---------- SERVER1 1 ---------- -{ + { - // we connect the maps via a TCP socket connection on port 8077 + // we connect the maps via a TCP socket connection on port 8077 - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8076, new InetSocketAddress("localhost", 8077)) - .heartBeatInterval(1L, SECONDS); - ChronicleMapBuilder map1Builder = - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(20000L) - .addReplicator((byte) 1, tcpConfig); + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8076, new InetSocketAddress("localhost", 8077)) + .heartBeatInterval(1L, TimeUnit.SECONDS); + ChronicleMapBuilder map1Builder = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000L) + .replicators((byte) 1, tcpConfig); - map1 = map1Builder.create(); -} + map1 = map1Builder.create(); + } // ---------- SERVER2 on the same server as ---------- -{ - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8077) - .heartBeatInterval(1L, SECONDS); - ChronicleMapBuilder map2Builder = - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(20000L) - .addReplicator(tcp((byte) 2, tcpConfig)); - map2 = map2Builder.create(); + { + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8077) + .heartBeatInterval(1L, TimeUnit.SECONDS); + ChronicleMapBuilder map2Builder = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000L) + .replicators((byte) 2, tcpConfig); + map2 = map2Builder.create(); - // we will stores some data into one map here - map2.put(5, "EXAMPLE"); -} + // we will stores some data into one map here + map2.put(5, "EXAMPLE"); + } // ---------- CHECK ---------- // we are now going to check that the two maps contain the same data // allow time for the recompilation to resolve -int t = 0; -for (; t < 5000; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); -} + int t = 0; + for (; t < 5000; t++) { + if (map1.equals(map2)) + break; + Thread.sleep(1); + } + + Assert.assertEquals(map1, map2); + Assert.assertTrue(!map1.isEmpty()); + } -Assert.assertEquals(map1, map2); -assertTrue(!map1.isEmpty()); } ``` @@ -723,71 +739,86 @@ This example is the same as the one above, but it uses a slow throttled TCP/IP c ``` java -Map map1; -Map map2; +import org.junit.Assert; +import org.junit.Test; -int udpPort = 1234; +import java.io.IOException; +import java.net.Inet4Address; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class YourClass { + + + @Test + public void test() throws IOException, InterruptedException { + + Map map1; + Map map2; + + int udpPort = 1234; // ---------- SERVER1 1 ---------- -{ + { - // we connect the maps via a TCP socket connection on port 8077 + // we connect the maps via a TCP socket connection on port 8077 - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8076, new InetSocketAddress("localhost", 8077)) - .heartBeatInterval(1L, SECONDS) + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8076, new InetSocketAddress("localhost", 8077)) + .heartBeatInterval(1L, TimeUnit.SECONDS) - // a maximum of 1024 bits per millisecond - .throttlingConfig(ThrottlingConfig.throttle(1024, TimeUnit.MILLISECONDS)); + // a maximum of 1024 bits per millisecond + .throttlingConfig(ThrottlingConfig.throttle(1024, TimeUnit.MILLISECONDS)); - UdpReplicationConfig udpConfig = UdpReplicationConfig - .simple(Inet4Address.getByName("255.255.255.255"), udpPort); + UdpReplicationConfig udpConfig = UdpReplicationConfig + .simple(Inet4Address.getByName("255.255.255.255"), udpPort); - ChronicleMapBuilder map1Builder = - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(20000L) - .addReplicator(tcp((byte) 1, tcpConfig)) - .addReplicator(udp((byte) 1, udpConfig)); + ChronicleMapBuilder map1Builder = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000L) + .replicators((byte) 1, tcpConfig, udpConfig); - map1 = map1Builder.create(); -} + map1 = map1Builder.create(); + } // ---------- SERVER2 2 on the same server as ---------- -{ - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8077) - .heartBeatInterval(1L, SECONDS) - .throttlingConfig(ThrottlingConfig.throttle(1024, TimeUnit.MILLISECONDS)); + { + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8077) + .heartBeatInterval(1L, TimeUnit.SECONDS) + .throttlingConfig(ThrottlingConfig.throttle(1024, TimeUnit.MILLISECONDS)); - UdpReplicationConfig udpConfig = UdpReplicationConfig - .simple(Inet4Address.getByName("255.255.255.255"), udpPort); + UdpReplicationConfig udpConfig = UdpReplicationConfig + .simple(Inet4Address.getByName("255.255.255.255"), udpPort); - ChronicleMapBuilder map2Builder = - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(20000L) - .addReplicator(tcp((byte) 2, tcpConfig)) - .addReplicator(udp((byte) 1, udpConfig)); + ChronicleMapBuilder map2Builder = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000L) + .replicators((byte) 2, tcpConfig, udpConfig); - map2 = map2Builder.create(); + map2 = map2Builder.create(); - // we will stores some data into one map here - map2.put(5, "EXAMPLE"); -} + // we will stores some data into one map here + map2.put(5, "EXAMPLE"); + } // ---------- CHECK ---------- // we are now going to check that the two maps contain the same data // allow time for the recompilation to resolve -int t = 0; -for (; t < 5000; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); -} + int t = 0; + for (; t < 5000; t++) { + if (map1.equals(map2)) + break; + Thread.sleep(1); + } -Assert.assertEquals(map1, map2); -assertTrue(!map1.isEmpty()); + Assert.assertEquals(map1, map2); + Assert.assertTrue(!map1.isEmpty()); + + } } ``` From a5bf4f8448da8b151bc63385450db922e876e185 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 14 Oct 2014 13:58:28 +0400 Subject: [PATCH 21/64] HCOLL-136 HCOLL-137 HCOLL-164 Extracted ChronicleHash abstraction and updated ChronicleMap docs --- .../net/openhft/chronicle/ChronicleHash.java | 64 +++++++++++ .../chronicle/ChronicleHashBuilder.java | 5 +- .../net/openhft/chronicle/TimeProvider.java | 2 +- .../openhft/chronicle/map/ChronicleMap.java | 102 +++++++++++++++--- .../openhft/chronicle/map/ChronicleSet.java | 8 -- .../openhft/chronicle/set/ChronicleSet.java | 10 ++ .../net/openhft/chronicle/set/SetFromMap.java | 7 +- .../SetForTCPSocketReplicationTest3way.java | 1 - 8 files changed, 172 insertions(+), 27 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/ChronicleHash.java delete mode 100644 src/main/java/net/openhft/chronicle/map/ChronicleSet.java create mode 100644 src/main/java/net/openhft/chronicle/set/ChronicleSet.java diff --git a/src/main/java/net/openhft/chronicle/ChronicleHash.java b/src/main/java/net/openhft/chronicle/ChronicleHash.java new file mode 100644 index 000000000..996213937 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/ChronicleHash.java @@ -0,0 +1,64 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.set.ChronicleSet; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; + +/** + * This interface defines common {@link ChronicleMap} and {@link ChronicleSet}, related to off-heap + * memory management and file-mapping. Not usable by itself. + */ +public interface ChronicleHash extends Closeable { + /** + * Returns the file this hash container mapped to, i. e. when it is created by + * {@link ChronicleHashBuilder#create()} call, or {@code null} if it is purely in-memory, + * i. e. if it is created by {@link ChronicleHashBuilder#create()} call. + * + * @return the file this {@link ChronicleMap} or {@link ChronicleSet} is mapped to, + * or {@code null} if it is not mapped to any file + * @see ChronicleHashBuilder#create(File) + * @see ChronicleHashBuilder#create() + */ + File file(); + + /** + * Releases the off-heap memory, used by this hash container and resources, used by replication, + * if any. However, if hash container (hence off-heap memory, used by it) is mapped to the file + * and there are other instances mapping the same data on the server across JVMs, the memory + * won't be actually freed on operation system level. I. e. this method call doesn't affect + * other {@link ChronicleMap} or {@link ChronicleSet} instances mapping the same data. + * + *

If you won't call this method, memory would be held at least until next garbage + * collection. This could be a problem if, for example, you target rare garbage collections, + * but load and drop {@code ChronicleHash}es regularly. + * + *

+ * TODO what about commit guarantees, when ChronicleMap is used with memory-mapped files, if + * {@code ChronicleMap}/{@code ChronicleSet} closed/not? + * + *

After this method call behaviour of all methods of {@code ChronicleMap} + * or {@code ChronicleSet} is undefined. Any method call on the map might throw + * any exception, or even JVM crash. Shortly speaking, don't call use map closing. + */ + @Override + void close() throws IOException; +} diff --git a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java index c37ea2cda..505d9614b 100644 --- a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java +++ b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java @@ -1,6 +1,7 @@ package net.openhft.chronicle; import net.openhft.chronicle.map.*; +import net.openhft.chronicle.set.ChronicleSet; import net.openhft.chronicle.set.ChronicleSetBuilder; import org.jetbrains.annotations.NotNull; @@ -26,8 +27,8 @@ import java.util.concurrent.TimeUnit; * @param the concrete builder type, i. e. {@link ChronicleMapBuilder} * or {@link ChronicleSetBuilder} */ -public interface ChronicleHashBuilder> - extends Cloneable { +public interface ChronicleHashBuilder> extends Cloneable { B clone(); diff --git a/src/main/java/net/openhft/chronicle/TimeProvider.java b/src/main/java/net/openhft/chronicle/TimeProvider.java index 5f48dcb51..8f5885286 100644 --- a/src/main/java/net/openhft/chronicle/TimeProvider.java +++ b/src/main/java/net/openhft/chronicle/TimeProvider.java @@ -17,7 +17,7 @@ package net.openhft.chronicle; import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleSet; +import net.openhft.chronicle.set.ChronicleSet; import java.io.ObjectStreamException; import java.io.Serializable; diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java index e6858b4b8..57385aa98 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java @@ -16,24 +16,103 @@ package net.openhft.chronicle.map; +import net.openhft.chronicle.ChronicleHash; +import net.openhft.chronicle.map.serialization.BytesReader; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshaller; +import net.openhft.lang.io.serialization.ObjectFactory; +import net.openhft.lang.model.Byteable; + import java.io.Closeable; import java.io.File; +import java.io.IOException; +import java.util.Map; import java.util.concurrent.ConcurrentMap; -public interface ChronicleMap extends ConcurrentMap, Closeable { +/** + * Extension of {@link ConcurrentMap} interface, stores the data off-heap. + * + *

For information on + *

    + *
  • how to construct a {@code ChronicleMap}
    • + *
  • {@code ChronicleMap} flavors and properties
    • + *
  • available configurations
    • + *
+ * see {@link ChronicleMapBuilder} documentation. + * + *

Functionally this interface defines some methods supporting garbage-free off-heap programming: + * {@link #getUsing(Object, Object)}, {@link #acquireUsing(Object, Object)}. + * + *

Note that {@code ChronicleMap} extends {@link Closeable}, don't forget + * to {@linkplain #close() close} map when it is no longer needed. + * + * @param the map key type + * @param the map value type + */ +public interface ChronicleMap extends ConcurrentMap, ChronicleHash { /** - * @return size as a long. - * @see java.util.Map size() + * Returns the number of entries in this map. + * + * @return number of entries in this map + * @see Map#size() */ long longSize(); /** - * Get a value for a key if available. If the value is Byteable, it will be assigned to - * reference the value, instead of copying the data for zero copy access to the collection. + * Returns the value to which the specified key is mapped, or {@code null} if this map contains + * no mapping for the key. * - * @param key to lookup. - * @param value to reuse if possible. If null, a new object will be created. - * @return value found or null if not. + *

If the specified key if absent in the map, {@linkplain + * ChronicleMapBuilder#defaultValue(Object) default value} is taken or {@linkplain + * ChronicleMapBuilder#defaultValueProvider(DefaultValueProvider) default value provider} + * is called (depending on what was configured last in {@code ChronicleMapBuilder}). + * If it results to something different from {@code null}, this object is put to this map for + * the specified key, and then returned from this {@code get()} call. Otherwise, if default + * value is {@code null} and default value provider returns {@code null}, the map remains + * unchanged and {@code null} is returned. + * + *

If the value class allows reusing, particularly if it is a {@link Byteable} subclass, + * consider {@link #getUsing(Object, Object)} method instead of this to reduce garbage creation. + * + * @param key the key whose associated value is to be returned + * @return the value to which the specified key is mapped after this method call, + * or {@code null} if no value is mapped + * @see #getUsing(Object, Object) + */ + @Override + V get(Object key); + + /** + * Returns the value to which the specified key is mapped, read to the provided {@code value} + * object, if possible, or returns {@code null}, if this map contains no mapping for the key. + * + *

If the specified key is present in the map, the value data is read to the provided + * {@code value} object via value marshaller's {@link BytesMarshaller#read(Bytes, Object) + * read(Bytes, value)} or value reader's {@link BytesReader#read(Bytes, long, Object) + * read(Bytes, size, value)} method, depending on what deserialization strategy is configured + * on this map builder. If the value deserializer is able to reuse the given {@code value} + * object, calling this method instead of {@link #get(Object)} could help to reduce garbage + * creation. + * + *

The provided {@code value} object is allowed to be {@code null}, in this case + * {@code map.getUsing(key, null)} call is semantically equivalent to simple + * {@code map.get(key)} call. + * + *

If the specified key if absent in the map, {@linkplain + * ChronicleMapBuilder#defaultValue(Object) default value} is taken or {@linkplain + * ChronicleMapBuilder#defaultValueProvider(DefaultValueProvider) default value provider} + * is called. If it results to something different from {@code null}, this object is put + * to this map for the specified key, and then a new value is returned from this + * {@code getUsing()} call. Otherwise, if default value is {@code null} and default value + * provider, if specified, returns {@code null}, the map remains unchanged and {@code null} + * is returned. The provided {@code value} is untouched anyway in this case. + * + * @param key the key whose associated value is to be returned + * @param value the object to read value data in, if possible + * @return the value mapped to the specified key after this method call, or {@code null} if no + * value is mapped + * @see #get(Object) + * @see ChronicleMapBuilder#valueMarshallerAndFactory(BytesMarshaller, ObjectFactory) */ V getUsing(K key, V value); @@ -41,15 +120,10 @@ public interface ChronicleMap extends ConcurrentMap, Closeable { * Acquire a value for a key, creating if absent. If the value is Byteable, it will be assigned * to reference the value, instead of copying the data. * - * @param key to lookup. + * @param key the key whose associated value is to be returned * @param value to reuse if possible. If null, a new object will be created. * @return value created or found. */ V acquireUsing(K key, V value); - /** - * @return The file or directory for this ChronicleMap - */ - File file(); - } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleSet.java b/src/main/java/net/openhft/chronicle/map/ChronicleSet.java deleted file mode 100644 index 5d3b92abf..000000000 --- a/src/main/java/net/openhft/chronicle/map/ChronicleSet.java +++ /dev/null @@ -1,8 +0,0 @@ -package net.openhft.chronicle.map; - -import java.io.Closeable; -import java.util.Set; - -public interface ChronicleSet extends Set, Closeable { - public long longSize(); -} diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSet.java b/src/main/java/net/openhft/chronicle/set/ChronicleSet.java new file mode 100644 index 000000000..c167caad1 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSet.java @@ -0,0 +1,10 @@ +package net.openhft.chronicle.set; + +import net.openhft.chronicle.ChronicleHash; + +import java.io.Closeable; +import java.util.Set; + +public interface ChronicleSet extends Set, ChronicleHash { + public long longSize(); +} diff --git a/src/main/java/net/openhft/chronicle/set/SetFromMap.java b/src/main/java/net/openhft/chronicle/set/SetFromMap.java index 47fd84883..cb513154c 100644 --- a/src/main/java/net/openhft/chronicle/set/SetFromMap.java +++ b/src/main/java/net/openhft/chronicle/set/SetFromMap.java @@ -17,8 +17,8 @@ package net.openhft.chronicle.set; import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleSet; +import java.io.File; import java.io.IOException; import java.io.Serializable; import java.util.AbstractSet; @@ -115,6 +115,11 @@ class SetFromMap extends AbstractSet return m.longSize(); } + @Override + public File file() { + return m.file(); + } + @Override public void close() throws IOException { m.close(); diff --git a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java index 57576a192..057f70161 100644 --- a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java +++ b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java @@ -16,7 +16,6 @@ package net.openhft.chronicle.set; -import net.openhft.chronicle.map.ChronicleSet; import org.junit.After; import org.junit.Before; import org.junit.Test; From b5b3d2f0594860d35759cda4e9702e4aff0efe3a Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 14 Oct 2014 14:06:19 +0400 Subject: [PATCH 22/64] Fixed compilation issue in SerializationBuilder --- .../java/net/openhft/chronicle/map/SerializationBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index 282da7327..e95967aa9 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -80,7 +80,7 @@ final class SerializationBuilder implements Cloneable { if (role == Role.VALUE) { factory = concreteClass(eClass) && marshallerUseFactory(eClass) ? new NewInstanceObjectFactory(eClass) : - (ObjectFactory) NullObjectFactory.of(); + (ObjectFactory) NullObjectFactory.of(); } if (concreteClass(eClass) && Byteable.class.isAssignableFrom(eClass)) { From 97ef5bbf85dc52f840eb7bb9523b386d89cd29a1 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:16:40 +0100 Subject: [PATCH 23/64] added readme - ##### Don't forget to se the EntrySize --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/README.md b/README.md index a4bba415b..79d8b1216 100644 --- a/README.md +++ b/README.md @@ -620,6 +620,50 @@ will run out of space between 1 and 2 million entries. You should set the .entries(..) and .entrySize(..) to those you require. +##### Don't forget to se the EntrySize + +If you put() and entry that is much larger than the max entry size set via entrySize();, see example below + +```java + ChronicleMap map = + ChronicleMapBuilder.of(Integer.class, String.class) + .entrySize(10).create(); + +``` + +it could error : + +this code will produce a java.lang.IllegalArgumentException + +```java +@Test +public void test() throws IOException, InterruptedException { + ChronicleMap map = + ChronicleMapBuilder.of(Integer.class, String.class) + .entrySize(10).create(); + + String value = new String(new char[2000]); + map.put(1, value); + + Assert.assertEquals(value, map.get(1)); +} + +``` + +and this will cause a malloc_error_break +```java +@Test +public void test() throws IOException, InterruptedException { + ChronicleMap map = + ChronicleMapBuilder.of(Integer.class, String.class) + .entrySize(10).create(); + + String value = new String(new char[20000000]); + map.put(1, value); + + Assert.assertEquals(value, map.get(1)); +} +``` # Example : Simple Hello World @@ -634,6 +678,9 @@ System.out.println(map.get(1)); ``` + + + # Example : Sharing the map on two ( or more ) processes on the same machine Lets assume that we had two server, lets call them server1 and server2, if we wished to share a map From df46290cbaf54792f8e30e2b98f6eaa9ab93a51f Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:17:13 +0100 Subject: [PATCH 24/64] HCOLL-163 updated readme --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 79d8b1216..21ef37884 100644 --- a/README.md +++ b/README.md @@ -625,9 +625,9 @@ You should set the .entries(..) and .entrySize(..) to those you require. If you put() and entry that is much larger than the max entry size set via entrySize();, see example below ```java - ChronicleMap map = - ChronicleMapBuilder.of(Integer.class, String.class) - .entrySize(10).create(); +ChronicleMap map = + ChronicleMapBuilder.of(Integer.class, String.class) + .entrySize(10).create(); ``` @@ -678,9 +678,6 @@ System.out.println(map.get(1)); ``` - - - # Example : Sharing the map on two ( or more ) processes on the same machine Lets assume that we had two server, lets call them server1 and server2, if we wished to share a map From 930a460626e71fcd5864bded503d0009d9d487b5 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:25:27 +0100 Subject: [PATCH 25/64] added - Don't forget to se the EntrySize --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 21ef37884..f47149b69 100644 --- a/README.md +++ b/README.md @@ -622,7 +622,9 @@ You should set the .entries(..) and .entrySize(..) to those you require. ##### Don't forget to se the EntrySize -If you put() and entry that is much larger than the max entry size set via entrySize();, see example below +If you put() and entry that is much larger than the max entry size set via entrySize(), the code will error, +see below the example, to set the entry size to 10, you should pick a number that is the size in bytes of +your entries : ```java ChronicleMap map = @@ -631,9 +633,7 @@ ChronicleMap map = ``` -it could error : - -this code will produce a java.lang.IllegalArgumentException +This example will throw an java.lang.IllegalArgumentException because the entrySize is too small. ```java @Test @@ -650,7 +650,9 @@ public void test() throws IOException, InterruptedException { ``` -and this will cause a malloc_error_break +If the entry size is dramatically too small ( like in the example below ), +you will get a *malloc_error_break* : + ```java @Test public void test() throws IOException, InterruptedException { From fca3ac5486c9eb7038e32f706594741a205ebf06 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:26:48 +0100 Subject: [PATCH 26/64] HCOLL-163 updated readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f47149b69..f2f699c6f 100644 --- a/README.md +++ b/README.md @@ -622,9 +622,9 @@ You should set the .entries(..) and .entrySize(..) to those you require. ##### Don't forget to se the EntrySize -If you put() and entry that is much larger than the max entry size set via entrySize(), the code will error, -see below the example, to set the entry size to 10, you should pick a number that is the size in bytes of -your entries : +If you put() and entry that is much larger than the max entry size set via entrySize(), +the code will error. To see how to set the entry size the example below sets the entry size to 10, +you should pick a size that is the size in bytes of your entries : ```java ChronicleMap map = From 39786e9e09e7efcba45835c719c3a4c67888746b Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:28:29 +0100 Subject: [PATCH 27/64] Update README.md https://github.com/OpenHFT/Chronicle-Map#known-issues --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f2f699c6f..3dc297c78 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Replicate your Key Value Store across your network, with consistency, durability ``` Click here to get the [Latest Version Number](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22net.openhft%22%20AND%20a%3A%22chronicle-map%22) + #### Contents * [Should I use Chronicle Queue or Chronicle Map](https://github.com/OpenHFT/Chronicle-Map#should-i-use-chronicle-queue-or-chronicle-map) From e706a717f35ca51e35ab5c1c392be467d5e20a50 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:28:55 +0100 Subject: [PATCH 28/64] HCOLL-163 updated readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2f699c6f..566d22938 100644 --- a/README.md +++ b/README.md @@ -620,7 +620,7 @@ will run out of space between 1 and 2 million entries. You should set the .entries(..) and .entrySize(..) to those you require. -##### Don't forget to se the EntrySize +##### Don't forget to set the EntrySize If you put() and entry that is much larger than the max entry size set via entrySize(), the code will error. To see how to set the entry size the example below sets the entry size to 10, From 9e7e474a0922120cf923fd7bdbb4c419a36feee2 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:41:07 +0100 Subject: [PATCH 29/64] updated Oversized Entries Support --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3884a0b81..ccd99db80 100644 --- a/README.md +++ b/README.md @@ -304,8 +304,9 @@ added and the value return will we the same value which you provided. ## Oversized Entries Support -It is possible for an entry to be twice as large as the maximum entry, we refer to this type of -entry as an oversized entry. Oversized entries are there to cater for the case where only a small +It is possible for the size of your entry to be twice as large as the maximum entry size, +we refer to this type of entry as an oversized entry. Oversized entries are there to cater for the case +where only a small percentage of your entries are twise as large as the others, in this case your large entry will span across two entries. The alternative would be to increase your maximum entry size to be similar to the size of the largest entry, but this approach is wasteful of memory, especially when most From 19574fd790e3bee1c6fe2abb600b403728711efb Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 11:45:31 +0100 Subject: [PATCH 30/64] HCOLL-163 added public ChronicleSetBuilder channel(ChannelProvider.ChronicleChannel chronicleChannel) { --- .../chronicle/ChronicleHashBuilder.java | 193 ++++++++---------- .../chronicle/set/ChronicleSetBuilder.java | 6 + 2 files changed, 93 insertions(+), 106 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java index 505d9614b..e4a21799a 100644 --- a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java +++ b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java @@ -1,12 +1,16 @@ package net.openhft.chronicle; -import net.openhft.chronicle.map.*; +import net.openhft.chronicle.map.ChannelProvider; +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import net.openhft.chronicle.map.ReplicationConfig; import net.openhft.chronicle.set.ChronicleSet; import net.openhft.chronicle.set.ChronicleSetBuilder; -import org.jetbrains.annotations.NotNull; - -import net.openhft.lang.io.serialization.*; +import net.openhft.lang.io.serialization.BytesMarshaller; +import net.openhft.lang.io.serialization.BytesMarshallerFactory; +import net.openhft.lang.io.serialization.ObjectSerializer; import net.openhft.lang.model.Byteable; +import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; @@ -18,14 +22,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; /** - * This interface defines the meaning of configurations, common to {@link ChronicleMapBuilder} - * and {@link ChronicleSetBuilder}, i. e. Chronicle hash container configurations. + * This interface defines the meaning of configurations, common to {@link ChronicleMapBuilder} and {@link + * ChronicleSetBuilder}, i. e. Chronicle hash container configurations. * * @param the type of keys in hash containers, created by this builder - * @param the container type, created by this builder, i. e. {@link ChronicleMap} - * or {@link ChronicleSet} - * @param the concrete builder type, i. e. {@link ChronicleMapBuilder} - * or {@link ChronicleSetBuilder} + * @param the container type, created by this builder, i. e. {@link ChronicleMap} or {@link ChronicleSet} + * @param the concrete builder type, i. e. {@link ChronicleMapBuilder} or {@link ChronicleSetBuilder} */ public interface ChronicleHashBuilder> extends Cloneable { @@ -41,14 +43,13 @@ public interface ChronicleHashBuilderIf key size varies moderately, specify the size higher than average, but lower than - * the maximum possible, to minimize average memory overuse. If key size varies in a wide range, - * it's better to use {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it - * directly. + *

If key size varies moderately, specify the size higher than average, but lower than the maximum + * possible, to minimize average memory overuse. If key size varies in a wide range, it's better to use + * {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it directly. * * @param keySize number of bytes, taken by serialized form of keys * @return this builder back @@ -58,15 +59,15 @@ public interface ChronicleHashBuilderIf keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size - * is known statically, it is automatically accounted and this method shouldn't be called. + *

If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size is known + * statically, it is automatically accounted and this method shouldn't be called. * - *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be - * called instead of this one. + *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be called instead + * of this one. * * @param sampleKey the sample key * @return this builder back @@ -75,55 +76,43 @@ public interface ChronicleHashBuilder{@link ChronicleMap} and {@link ChronicleSet} store their data off-heap, so it is required - * to serialize key (and values, in {@code ChronicleMap} case) (unless they are direct - * {@link Byteable} instances). Serialized key bytes (+ serialized value bytes, - * in {@code ChronicleMap} case) + some metadata bytes comprise "entry space", which - * {@code ChronicleMap} or {@code ChronicleSet} should allocate. So entry size - * is a minimum allocation portion in the hash containers, created by this builder. - * E. g. if entry size is 100, the created container could only allocate 100, 200, 300... bytes - * for an entry. If say 150 bytes of entry space are required by the entry, 200 bytes - * will be allocated, 150 used and 50 wasted. To minimize memory overuse and improve speed, you - * should pay decent attention to this configuration. + *

{@link ChronicleMap} and {@link ChronicleSet} store their data off-heap, so it is required to + * serialize key (and values, in {@code ChronicleMap} case) (unless they are direct {@link Byteable} + * instances). Serialized key bytes (+ serialized value bytes, in {@code ChronicleMap} case) + some + * metadata bytes comprise "entry space", which {@code ChronicleMap} or {@code ChronicleSet} should + * allocate. So entry size is a minimum allocation portion in the hash containers, created by this + * builder. E. g. if entry size is 100, the created container could only allocate 100, 200, 300... bytes + * for an entry. If say 150 bytes of entry space are required by the entry, 200 bytes will be allocated, + * 150 used and 50 wasted. To minimize memory overuse and improve speed, you should pay decent attention + * to this configuration. * - *

There are three major patterns of this configuration usage: - *

    - *
  1. Key (and value, in {@code ChronicleMap} case) sizes are constant. Configure them - * via {@link #constantKeySizeBySample(Object)} and - * {@link ChronicleMapBuilder#constantValueSizeBySample(Object)} methods, and you will - * experience no memory waste at all.
    2. - *
  2. Key (and/or value size, in {@code ChronicleMap} case) varies moderately. Specify them - * using corresponding methods, or specify entry size directly by calling this method, - * by sizes somewhere between average and maximum possible. The idea is to have most - * (90% or more) entries to fit a single "entry size" with moderate memory waste - * (10-20% on average), rest 10% or less of entries should take 2 "entry sizes", thus with - * ~50% memory overuse.
    3. - *
  3. Key (and/or value size, in {@code ChronicleMap} case) varies in a wide range. - * Then it's best to use entry size configuration in chunk mode. Specify entry size - * so that most entries should take from 5 to several dozens of "chunks". With this - * approach, average memory waste should be very low. + *

    There are three major patterns of this configuration usage:

    1. Key (and value, in {@code + * ChronicleMap} case) sizes are constant. Configure them via {@link #constantKeySizeBySample(Object)} and + * {@link ChronicleMapBuilder#constantValueSizeBySample(Object)} methods, and you will experience no + * memory waste at all.
    2. Key (and/or value size, in {@code ChronicleMap} case) varies moderately. + * Specify them using corresponding methods, or specify entry size directly by calling this method, by + * sizes somewhere between average and maximum possible. The idea is to have most (90% or more) entries to + * fit a single "entry size" with moderate memory waste (10-20% on average), rest 10% or less of entries + * should take 2 "entry sizes", thus with ~50% memory overuse.
    3. Key (and/or value size, in {@code + * ChronicleMap} case) varies in a wide range. Then it's best to use entry size configuration in chunk + * mode. Specify entry size so that most entries should take from 5 to several dozens of "chunks". + * With this approach, average memory waste should be very low. * - *

      However, remember that - *

        - *
      • Operations with entries that span several "entry sizes" are a bit slower, than - * entries which take a single "entry size". That is why "chunk" approach - * is not recommended, when key (and/or value size) varies moderately.
        • - *
      • The maximum number of chunks could be taken by an entry is 64. - * {@link IllegalArgumentException} is thrown on attempt to insert too large entry, - * compared to the configured or computed entry size.
        • - *
      • The number of "entries" {@linkplain #entries(long) in the whole map} and - * {@linkplain #actualEntriesPerSegment(int) per segment} is actually the number - * "chunks".
        • - *
      + *

      However, remember that

      • Operations with entries that span several "entry sizes" are a bit + * slower, than entries which take a single "entry size". That is why "chunk" approach is not recommended, + * when key (and/or value size) varies moderately.
      • The maximum number of chunks could be taken by + * an entry is 64. {@link IllegalArgumentException} is thrown on attempt to insert too large entry, + * compared to the configured or computed entry size.
      • The number of "entries" {@linkplain + * #entries(long) in the whole map} and {@linkplain #actualEntriesPerSegment(int) per segment} is actually + * the number "chunks".
      * - *

      Example: if values in your {@code ChronicleMap} are adjacency lists of some social - * graph, where nodes are represented as {@code long} ids, and adjacency lists - * are serialized in efficient manner, for example as {@code long[]} arrays. Typical number - * of connections is 100-300, maximum is 3000. In this case entry size of - * 50 * (8 bytes for each id) = 400 bytes would be a good choice: 

      {@code
+     * 
      Example: if values in your {@code ChronicleMap} are adjacency lists of some social graph, where
+     * nodes are represented as {@code long} ids, and adjacency lists are serialized in efficient manner, for
+     * example as {@code long[]} arrays. Typical number of connections is 100-300, maximum is 3000. In this
+     * case entry size of
+     * 50 * (8 bytes for each id) = 400 bytes would be a good choice: 
      {@code
      * Map socialGraph = ChronicleMapBuilder
      *     .of(Long.class, long[].class)
      *     // given that graph should have of 1 billion nodes, and 150 average adjacency list size
@@ -131,11 +120,8 @@ public interface ChronicleHashBuilder
-     *     It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max
-     *     chunks by single entry" limit, and ensures moderate average memory overuse (not more
-     *     than 20%).
-     *
      4. - *
    + * It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max chunks by single + * entry" limit, and ensures moderate average memory overuse (not more than 20%).
* * @param entrySize the "chunk size" in bytes * @return this builder back @@ -144,35 +130,28 @@ public interface ChronicleHashBuilder - *
  • If key size (and value size, in {@code ChronicleMap} case) is constant, this number - * is equal to the maximum number of entries (because each entry takes exactly one - * "entry size" memory unit).
    • - *
  • If key (and/or value, in {@code ChronicleMap} case) size varies moderately, you - * should pass to this method the maximum number of entries + 5-25%, depending on your data - * properties and configured {@linkplain #keySize(int) key}/{@linkplain - * ChronicleMapBuilder#valueSize(int) value}/{@linkplain #entrySize(int) entry} sizes.
    • - *
  • If your data size varies in a wide range, pass the maximum number of entries - * multiplied by average data size and divided by the configured "entry size" (i. e. chunk - * size). See an example in the documentation to {@link #entrySize(int)} method.
    • - * + *
    1. If key size (and value size, in {@code ChronicleMap} case) is constant, this number is equal + * to the maximum number of entries (because each entry takes exactly one "entry size" memory unit).
      2. + *
    2. If key (and/or value, in {@code ChronicleMap} case) size varies moderately, you should pass to this + * method the maximum number of entries + 5-25%, depending on your data properties and configured + * {@linkplain #keySize(int) key}/{@linkplain ChronicleMapBuilder#valueSize(int) value}/{@linkplain + * #entrySize(int) entry} sizes.
    3. If your data size varies in a wide range, pass the maximum + * number of entries multiplied by average data size and divided by the configured "entry size" (i. e. + * chunk size). See an example in the documentation to {@link #entrySize(int)} method.
    * - *

    You shouldn't put additional margin over the number, computed according the rules above. - * This bad practice was popularized by {@link HashMap#HashMap(int)} - * and {@link HashSet#HashSet(int)} constructors, which accept "capacity", that should - * be multiplied by "load factor" to obtain actual maximum expected number of entries. - * {@code ChronicleMap} and {@code ChronicleSet} don't have a notion of load factor. + *

    You shouldn't put additional margin over the number, computed according the rules above. This bad + * practice was popularized by {@link HashMap#HashMap(int)} and {@link HashSet#HashSet(int)} constructors, + * which accept "capacity", that should be multiplied by "load factor" to obtain actual maximum expected + * number of entries. {@code ChronicleMap} and {@code ChronicleSet} don't have a notion of load factor. * *

    Default value is 2^20 (~ 1 million). * - * @param entries maximum size of the created maps, in memory allocation units, - * so-called "entry size" + * @param entries maximum size of the created maps, in memory allocation units, so-called "entry size" * @return this builder back * @see #entrySize(int) */ @@ -192,6 +171,8 @@ public interface ChronicleHashBuilder keyMarshaller); /** - * Specifies that key objects, queried with the hash containers, created by this builder, - * are inherently immutable. Keys in {@link ChronicleMap} or {@link ChronicleSet} - * are not required to be immutable, as in ordinary {@link Map} or {@link Set} implementations, - * because they are serialized off-heap. However, {@code ChronicleMap} and {@code ChronicleSet} - * implementations can benefit from the knowledge that keys are not mutated between queries. + * Specifies that key objects, queried with the hash containers, created by this builder, are inherently + * immutable. Keys in {@link ChronicleMap} or {@link ChronicleSet} are not required to be immutable, as in + * ordinary {@link Map} or {@link Set} implementations, because they are serialized off-heap. However, + * {@code ChronicleMap} and {@code ChronicleSet} implementations can benefit from the knowledge that keys + * are not mutated between queries. * - *

    By default, {@code ChronicleHashBuilder}s detects immutability automatically - * only for very few standard JDK types (for example, for {@link String}), it is not recommended - * to rely on {@code ChronicleHashBuilder} to be smart enough about this. + *

    By default, {@code ChronicleHashBuilder}s detects immutability automatically only for very few + * standard JDK types (for example, for {@link String}), it is not recommended to rely on {@code + * ChronicleHashBuilder} to be smart enough about this. * * @return this builder back */ diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index 1e5eb3310..1b2571609 100644 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -169,6 +169,12 @@ public class ChronicleSetBuilder return this; } + @Override + public ChronicleSetBuilder channel(ChannelProvider.ChronicleChannel chronicleChannel) { + chronicleMapBuilder.channel(chronicleChannel); + return this; + } + @Override public String toString() { From 1c82c327e307e688c8e0d827ee2f5cc79f0e80f0 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 17:20:51 +0100 Subject: [PATCH 31/64] HCOLL-167 Created a stateless map - very much in draft at the moment --- .../chronicle/map/StatelessMapClient.java | 626 ++++++++++++++++++ 1 file changed, 626 insertions(+) create mode 100644 src/test/java/net/openhft/chronicle/map/StatelessMapClient.java diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClient.java new file mode 100644 index 000000000..73cae54ba --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -0,0 +1,626 @@ +package net.openhft.chronicle.map; + +import net.openhft.chronicle.map.serialization.*; +import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.BytesStore; +import net.openhft.lang.io.serialization.ObjectFactory; +import net.openhft.lang.model.Byteable; +import org.jetbrains.annotations.NotNull; + +import javax.swing.text.Segment; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; + +import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; + +/** + * + * **** THIS IS VERY MUCH IN DRAFT *** + * + * @author Rob Austin. + */ +public class StatelessMapClient, + V, VW, MVW extends MetaBytesWriter> extends AbstractMap + implements ChronicleMap, Serializable { + + final Class kClass; + private ThreadLocal sourceBuffer = new ThreadLocal(); + + final BytesReader originalKeyReader; + transient Provider> keyReaderProvider; + final KI originalKeyInterop; + transient Provider keyInteropProvider; + final MKI originalMetaKeyInterop; + final MetaProvider metaKeyInteropProvider; + + final Class vClass; + final SizeMarshaller valueSizeMarshaller; + final BytesReader originalValueReader; + transient Provider> valueReaderProvider; + final VW originalValueWriter; + transient Provider valueWriterProvider; + final MVW originalMetaValueWriter; + final MetaProvider metaValueWriterProvider; + final ObjectFactory valueFactory; + final DefaultValueProvider defaultValueProvider; + + Map transactionIDs = new HashMap(); + + + AtomicLong transactionID = new AtomicLong(); + + private final long lockTimeOutNS; + + transient Segment[] segments; // non-final for close() + // non-final for close() and because it is initialized out of constructor + transient BytesStore ms; + transient long headerSize; + transient Set> entrySet; + + private int bits; + private int mask; + private long timeoutMs = TimeUnit.SECONDS.toMillis(20); + + + @Override + public File file() { + return null; + } + + @Override + public void close() throws IOException { + throw new UnsupportedOperationException("This is not supported in the " + this.getClass() + .getSimpleName()); + } + + enum EventId { + LONG_SIZE, SIZE, + IS_EMPTY, + CONTAINS_KEY, + CONTAINS_VALUE, + GET, PUT, + REMOVE, + PUT_ALL, + CLEAR, + KEY_SET, + VALUES, + ENTRY_SET, + REPLACE, + REPLACE_WITH_OLD_AND_NEW_VALUE, + PUT_IF_ABSENT, + REMOVE_WITH_VALUE + } + + + long nextUniqueTransaction() { + long time = System.currentTimeMillis(); + + long l = transactionID.get(); + if (time > l) { + boolean b = transactionID.compareAndSet(l, time); + if (b) return time; + } + + return transactionID.incrementAndGet(); + } + + + private final ByteBufferBytes target; + private final SizeMarshaller keySizeMarshaller; + + public StatelessMapClient(ByteBufferBytes target, ChronicleMapBuilder builder) { + this.target = target; + + SerializationBuilder keyBuilder = builder.keyBuilder; + kClass = keyBuilder.eClass; + keySizeMarshaller = keyBuilder.sizeMarshaller(); + originalKeyReader = keyBuilder.reader(); + + originalKeyInterop = (KI) keyBuilder.interop(); + + originalMetaKeyInterop = (MKI) keyBuilder.metaInterop(); + metaKeyInteropProvider = (MetaProvider) keyBuilder.metaInteropProvider(); + + SerializationBuilder valueBuilder = builder.valueBuilder; + vClass = valueBuilder.eClass; + valueSizeMarshaller = valueBuilder.sizeMarshaller(); + originalValueReader = valueBuilder.reader(); + + originalValueWriter = (VW) valueBuilder.interop(); + originalMetaValueWriter = (MVW) valueBuilder.metaInterop(); + metaValueWriterProvider = (MetaProvider) valueBuilder.metaInteropProvider(); + valueFactory = valueBuilder.factory(); + defaultValueProvider = builder.defaultValueProvider(); + + lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); + + + } + + @Override + public V putIfAbsent(K key, V value) { + target.writeByte(PUT_IF_ABSENT.ordinal()); + + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + + writeKey(key, copies); + writeValue(value, copies); + + return null; + } + + + @Override + public boolean remove(Object key, Object value) { + target.writeByte(REMOVE_WITH_VALUE.ordinal()); + + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + writeKey((K) key, copies); + writeValue((V) value, copies); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return blockingFetch(transactionId).readBoolean(); + + } + + @Override + public boolean replace(K key, V oldValue, V newValue) { + target.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + writeKey(key, copies); + writeValue(oldValue, copies); + writeValue(newValue, copies); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return blockingFetch(transactionId).readBoolean(); + + } + + @Override + public V replace(K key, V value) { + target.writeByte(REPLACE.ordinal()); + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + writeKey(key, copies); + writeValue(value, copies); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return readValue(blockingFetch(transactionId)); + } + + + @Override + public int size() { + + target.writeByte(SIZE.ordinal()); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return blockingFetch(transactionId).readInt(); + } + + + @Override + public boolean isEmpty() { + target.writeByte(IS_EMPTY.ordinal()); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return blockingFetch(transactionId).readBoolean(); + } + + @Override + public boolean containsKey(Object key) { + target.writeByte(CONTAINS_KEY.ordinal()); + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + writeKey((K) key, copies); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return blockingFetch(transactionId).readBoolean(); + + } + + @Override + public boolean containsValue(Object value) { + target.writeByte(CONTAINS_VALUE.ordinal()); + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + writeValue((V) value, copies); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return blockingFetch(transactionId).readBoolean(); + } + + @Override + public long longSize() { + target.writeByte(LONG_SIZE.ordinal()); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return blockingFetch(transactionId).readLong(); + } + + @Override + public V get(Object key) { + target.writeByte(GET.ordinal()); + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + writeKey((K) key, copies); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return readValue(blockingFetch(transactionId)); + + } + + private V readValue(ByteBufferBytes byteBufferBytes) { + throw new UnsupportedOperationException("todo"); + } + + @Override + public V getUsing(K key, V value) { + throw new UnsupportedOperationException("getUsing is not supported for stateless clients"); + } + + @Override + public V acquireUsing(K key, V value) { + throw new UnsupportedOperationException("getUsing is not supported for stateless clients"); + } + + @Override + public V put(K key, V value) { + target.writeByte(PUT.ordinal()); + + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + writeKey((K) key, copies); + writeValue((V) value, copies); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return readValue(blockingFetch(transactionId)); + + } + + @Override + public V remove(Object key) { + target.writeByte(REMOVE.ordinal()); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return readValue(blockingFetch(transactionId)); + + } + + @Override + public void putAll(Map map) { + target.writeByte(PUT_ALL.ordinal()); + + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + + HashMap safeCopy = new HashMap(map); + + writeEntries(copies, safeCopy); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + blockingFetch(transactionId); + + return; + + + } + + private void writeEntries(ThreadLocalCopies copies, HashMap safeCopy) { + target.writeLong(safeCopy.size()); + + Set entries = (Set) safeCopy.entrySet(); + + for (Entry e : entries) { + writeKey(e.getKey(), copies); + writeValue(e.getValue(), copies); + } + } + + @Override + public void clear() { + target.writeByte(CLEAR.ordinal()); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + blockingFetch(transactionId); + + return; + } + + @NotNull + @Override + public Set keySet() { + target.writeByte(KEY_SET.ordinal()); + + // send the transaction id + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return readKeySet(blockingFetch(transactionId)); + } + + private Set readKeySet(ByteBufferBytes bufferBytes) { + int size = bufferBytes.readInt(); + + HashSet result = new HashSet<>(); + + for (int i = 0; i < size; i++) { + result.add(readKey(bufferBytes)); + } + return result; + } + + private K readKey(ByteBufferBytes bufferBytes) { + long transactionId = nextUniqueTransaction(); + target.writeLong(transactionId); + + // get the data back from the server + return readKey(blockingFetch(transactionId)); + + } + + @NotNull + @Override + public Collection values() { + target.writeByte(VALUES.ordinal()); + long transactionId = nextUniqueTransaction(); + + target.writeLong(transactionId); + + // get the data back from the server + ByteBufferBytes bufferBytes = blockingFetch(transactionId); + + int size = bufferBytes.readInt(); + + ArrayList result = new ArrayList(); + + for (int i = 0; i < size; i++) { + result.add(readValue(bufferBytes)); + } + return result; + } + + @NotNull + @Override + public Set> entrySet() { + target.writeByte(ENTRY_SET.ordinal()); + + long transactionId = nextUniqueTransaction(); + + // get the data back from the server + ByteBufferBytes bufferBytes = blockingFetch(transactionId); + + int size = bufferBytes.readInt(); + + Set> result = new HashSet>(); + + for (int i = 0; i < size; i++) { + K k = readKey(bufferBytes); + V v = readValue(bufferBytes); + result.add(new Entry(k, v)); + + } + + + return result; + } + + + private void writeValue(V value, ThreadLocalCopies copies) { + + VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); + copies = valueWriterProvider.getCopies(copies); + long valueSize; + MetaBytesWriter metaValueWriter = null; + Byteable valueAsByteable = null; + + if (value instanceof Byteable) { + valueAsByteable = (Byteable) value; + valueSize = valueAsByteable.maxSize(); + + target.writeLong(valueSize); + target.write(valueAsByteable.bytes()); + } else { + copies = valueWriterProvider.getCopies(copies); + valueWriter = valueWriterProvider.get(copies, originalValueWriter); + copies = metaValueWriterProvider.getCopies(copies); + metaValueWriter = metaValueWriterProvider.get( + copies, originalMetaValueWriter, valueWriter, value); + valueSize = metaValueWriter.size(valueWriter, value); + target.writeLong(valueSize); + metaValueWriter.write(valueWriter, target, value); + } + + } + + /** + * write the keysize and the key to the the {@code target} buffer + * + * @param key the key of the map + * @param copies + */ + private void writeKey(K key, final ThreadLocalCopies copies) { + KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); + + MKI metaKeyWriter = + metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); + + long keySize = metaKeyWriter.size(keyWriter, key); + + keySizeMarshaller.writeSize(target, keySize); + metaKeyWriter.write(keyWriter, target, key); + } + + + public Buffer buffer() { + Buffer result = sourceBuffer.get(); + + + if (result != null) { + return result; + } + + result = new Buffer() { + + volatile ByteBufferBytes buffer = null; + + @Override + public void set(ByteBufferBytes source) { + buffer = source; + } + + @Override + public ByteBufferBytes get() { + return buffer; + } + }; + + sourceBuffer.set(result); + return result; + } + + + /** + * blocks until a message is received from the server or a timeout is reached + * + * @param transactionId + * @return + */ + private ByteBufferBytes blockingFetch(long transactionId) { + final Buffer buffer = buffer(); + buffer.set(null); + + synchronized (this) { + transactionIDs.put(transactionId, buffer); + try { + buffer.wait(timeoutMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + final ByteBufferBytes bufferBytes = buffer.get(); + + if (bufferBytes == null) + throw new RuntimeException("Timed-out", new TimeoutException()); + + return bufferBytes; + } + + + class Entry implements Map.Entry { + + final K key; + final V value; + + /** + * Creates new entry. + */ + Entry(K k1, V v) { + value = v; + key = k1; + } + + public final K getKey() { + return key; + } + + public final V getValue() { + return value; + } + + public final V setValue(V newValue) { + V oldValue = value; + StatelessMapClient.this.put((K) getKey(), (V) newValue); + return oldValue; + } + + public final boolean equals(Object o) { + if (!(o instanceof Map.Entry)) + return false; + Map.Entry e = (Map.Entry) o; + Object k1 = getKey(); + Object k2 = e.getKey(); + if (k1 == k2 || (k1 != null && k1.equals(k2))) { + Object v1 = getValue(); + Object v2 = e.getValue(); + if (v1 == v2 || (v1 != null && v1.equals(v2))) + return true; + } + return false; + } + + public final int hashCode() { + return (key == null ? 0 : key.hashCode()) ^ + (value == null ? 0 : value.hashCode()); + } + + public final String toString() { + return getKey() + "=" + getValue(); + } + + + } +} + + +interface Buffer { + + void set(ByteBufferBytes source); + + ByteBufferBytes get(); +} From 3f615d03e0d8bb98191efc63c093289cdb78d602 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 17:22:09 +0100 Subject: [PATCH 32/64] HCOLL-167 Created a stateless map - very much in draft at the moment --- .../java/net/openhft/chronicle/map/StatelessMapClient.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{test => main}/java/net/openhft/chronicle/map/StatelessMapClient.java (100%) diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java similarity index 100% rename from src/test/java/net/openhft/chronicle/map/StatelessMapClient.java rename to src/main/java/net/openhft/chronicle/map/StatelessMapClient.java From 5fb61e909b62c056d96795fd3a60a406c1bee699 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 17:37:19 +0100 Subject: [PATCH 33/64] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ccd99db80..f4860e6dc 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@ requirements. - [Contact Us](sales@higherfrequencytrading.com)* Replicate your Key Value Store across your network, with consistency, durability and performance. ![Chronicle Map](http://openhft.net/wp-content/uploads/2014/07/ChronicleMap_200px.png) + +![](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) + #### Maven Artifact Download ```xml From 17b68d3ff5001e1fcc4aeda9cf81dcc773b9b784 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 17:38:50 +0100 Subject: [PATCH 34/64] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4860e6dc..991cfba56 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ requirements. - [Contact Us](sales@higherfrequencytrading.com)* Replicate your Key Value Store across your network, with consistency, durability and performance. ![Chronicle Map](http://openhft.net/wp-content/uploads/2014/07/ChronicleMap_200px.png) -![](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) +[![ScreenShot](https://raw.github.com/GabLeRoux/WebMole/master/ressources/WebMole_Youtube_Video.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) #### Maven Artifact Download ```xml From 8ae25bf72b01c3797842552e3055a1337809b96e Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 17:43:25 +0100 Subject: [PATCH 35/64] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 991cfba56..5bf3e1982 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ requirements. - [Contact Us](sales@higherfrequencytrading.com)* Replicate your Key Value Store across your network, with consistency, durability and performance. ![Chronicle Map](http://openhft.net/wp-content/uploads/2014/07/ChronicleMap_200px.png) -[![ScreenShot](https://raw.github.com/GabLeRoux/WebMole/master/ressources/WebMole_Youtube_Video.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.41.49.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) #### Maven Artifact Download ```xml From 12dc633a8820c6954d062627576ef4c830f9d069 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 17:51:08 +0100 Subject: [PATCH 36/64] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5bf3e1982..9f6528ac0 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ requirements. - [Contact Us](sales@higherfrequencytrading.com)* Replicate your Key Value Store across your network, with consistency, durability and performance. ![Chronicle Map](http://openhft.net/wp-content/uploads/2014/07/ChronicleMap_200px.png) -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.41.49.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) #### Maven Artifact Download ```xml From 0d7704435b0b2652a1011c1ff5b93577bfbc2798 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Tue, 14 Oct 2014 17:52:53 +0100 Subject: [PATCH 37/64] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f6528ac0..4ef414560 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ requirements. - [Contact Us](sales@higherfrequencytrading.com)* Replicate your Key Value Store across your network, with consistency, durability and performance. ![Chronicle Map](http://openhft.net/wp-content/uploads/2014/07/ChronicleMap_200px.png) -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) #### Maven Artifact Download ```xml @@ -119,6 +118,9 @@ product suite. In addition, The original Chronicle has been renamed to Chronicle ## Getting Started +#### Tutorial 1 - Creating an instance of Chronicle Map +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) + ### Simple Construction To download the JAR which contains Chronicle Map, we recommend you use maven, which will download it From 331bdf832863531d749ea772cdc2039b54289036 Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 15 Oct 2014 11:10:23 +0400 Subject: [PATCH 38/64] #3 Fixed a number of issue related to bit sets and blocks allocation in VanillaChronicleMap --- .../chronicle/map/VanillaChronicleMap.java | 30 ++++++++++++++----- .../net/openhft/chronicle/map/Issue3.java | 28 +++++++++++++++++ 2 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 src/test/java/net/openhft/chronicle/map/Issue3.java diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 1a2189a09..d34852d43 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -133,7 +133,8 @@ class VanillaChronicleMap, this.removeReturnsNull = builder.removeReturnsNull(); this.actualSegments = builder.actualSegments(); - this.entriesPerSegment = builder.actualEntriesPerSegment(); + // align by 8 because otherwise sizeOfBitSets() might address less blocks than specified + this.entriesPerSegment = align8(builder.actualEntriesPerSegment()); this.metaDataBytes = builder.metaDataBytes(); this.eventListener = builder.eventListener(); @@ -158,6 +159,10 @@ class VanillaChronicleMap, return (l + 63) & ~63; } + static int align8(int n) { + return (n + 7) & ~7; + } + void initTransients() { keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); @@ -555,8 +560,8 @@ class VanillaChronicleMap, long start = bytes.startAddr() + SEGMENT_HEADER; createHashLookups(start); start += align64(sizeOfMultiMap() + sizeOfMultiMapBitSet()) * multiMapsPerSegment(); - final NativeBytes bsBytes = new NativeBytes( - tmpBytes.objectSerializer(), start, start + sizeOfBitSets(), null); + final NativeBytes bsBytes = new NativeBytes(tmpBytes.objectSerializer(), + start, start + ((entriesPerSegment + 7) / 8), null); freeList = new SingleThreadedDirectBitSet(bsBytes); start += numberOfBitSets() * sizeOfBitSets(); entriesOffset = start - bytes.startAddr(); @@ -914,15 +919,23 @@ class VanillaChronicleMap, ); } } + updateNextPosToSearchFrom(ret, blocks); + } else { + // if bit at nextPosToSearchFrom is clear, it was skipped because + // more than 1 block was requested. Don't move nextPosToSearchFrom + // in this case. blocks == 1 clause is just a fast path. + if (blocks == 1 || freeList.isSet(nextPosToSearchFrom)) { + updateNextPosToSearchFrom(ret, blocks); + } } - // if bit at nextPosToSearchFrom is clear, it was skipped because - // more than 1 block was requested. Don't move nextPosToSearchFrom - // in this case. blocks == 1 clause is just a fast path. - if (blocks == 1 || freeList.isSet(nextPosToSearchFrom)) - nextPosToSearchFrom = ret + blocks; return ret; } + private void updateNextPosToSearchFrom(int allocated, int blocks) { + if ((nextPosToSearchFrom = allocated + blocks) >= freeList.size()) + nextPosToSearchFrom = 0; + } + private boolean realloc(int fromPos, int oldBlocks, int newBlocks) { if (freeList.allClear(fromPos + oldBlocks, fromPos + newBlocks)) { freeList.set(fromPos + oldBlocks, fromPos + newBlocks); @@ -1188,6 +1201,7 @@ class VanillaChronicleMap, try { hashLookup.clear(); freeList.clear(); + nextPosToSearchFrom = 0; resetSize(); } finally { unlock(); diff --git a/src/test/java/net/openhft/chronicle/map/Issue3.java b/src/test/java/net/openhft/chronicle/map/Issue3.java new file mode 100644 index 000000000..faf02b435 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/Issue3.java @@ -0,0 +1,28 @@ +package net.openhft.chronicle.map; + +import net.openhft.chronicle.set.ChronicleSet; +import net.openhft.chronicle.set.ChronicleSetBuilder; +import org.junit.Test; + +import java.io.IOException; +import java.util.Random; +import java.util.Set; + +public class Issue3 { + + @Test + public void test() throws IOException { + Set set = ChronicleSetBuilder.of(Long.class) + .actualSegments(1) + .actualEntriesPerSegment(1000) + .create(); + + Random r = new Random(); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 300; j++) { + set.add(r.nextLong()); + } + set.clear(); + } + } +} From 174b11a0752fa73211f7f9589564e149b9cac683 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 10:28:17 +0100 Subject: [PATCH 39/64] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ef414560..fbcf3dba8 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ product suite. In addition, The original Chronicle has been renamed to Chronicle ## Getting Started #### Tutorial 1 - Creating an instance of Chronicle Map -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://www.youtube.com/watch?v=pFVv8r8ZTZ8) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](http://vimeo.com/10893210) ### Simple Construction From 4efa5e2df860925828b575716415c67031d88555 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 10:30:04 +0100 Subject: [PATCH 40/64] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbcf3dba8..1a62e8954 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ product suite. In addition, The original Chronicle has been renamed to Chronicle ## Getting Started #### Tutorial 1 - Creating an instance of Chronicle Map -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](http://vimeo.com/10893210) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://vimeo.com/108932100) ### Simple Construction From d7de8d773c34e5a217f1099f9db7ef658565ec16 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 10:50:53 +0100 Subject: [PATCH 41/64] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a62e8954..4f30fa7ca 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ product suite. In addition, The original Chronicle has been renamed to Chronicle ## Getting Started #### Tutorial 1 - Creating an instance of Chronicle Map -[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](https://vimeo.com/108932100) +[![ScreenShot](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-14-at-17.49.36.png)](http://openhft.net/chronicle-map-video-tutorial-1/) ### Simple Construction From 5c3efb2000dfcef9cb683c298bf1d8b29a2df8ee Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 12:47:35 +0100 Subject: [PATCH 42/64] HCOLL-167 Created a stateless map - added read and write client tests --- .../chronicle/map/StatelessMapClientTest.java | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java new file mode 100644 index 000000000..e03a2d14f --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -0,0 +1,120 @@ +package net.openhft.chronicle.map; + +import junit.framework.TestCase; +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.model.constraints.NotNull; +import org.junit.Assert; +import org.junit.Test; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.Collections; + +public class StatelessMapClientTest extends TestCase { + + + @Test + public void testReadValueWriteIntegerValue() throws Exception { + testReadValueWriteValue("Test"); + testReadValueWriteValue(1); + testReadValueWriteValue(1L); + testReadValueWriteValue(1.0); + testReadValueWriteValue(1.0f); + testReadValueWriteValue(Collections.singleton("Test")); + testReadValueWriteValue(Collections.EMPTY_MAP); + testReadValueWriteValue(new MyTestClass(3)); + testReadValueWriteValue(new MyTestClassMarshallable(3)); + + } + + public void testReadValueWriteValue(Object value) throws Exception { + + Class valueClass = value.getClass(); + + final ByteBufferBytes buffer = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + ByteBufferBytes slice = buffer.slice(); + + ChronicleMapBuilder builder = ChronicleMapBuilder.of(Integer.class, valueClass); + builder.preMapConstruction(); + + StatelessMapClient statelessMap = new StatelessMapClient(buffer, builder); + + statelessMap.writeValue(value); + long position = buffer.position(); + slice.limit(position); + + Object actual = statelessMap.readValue(slice); + Assert.assertEquals(actual, value); + } + + + public static class MyTestClass implements Serializable { + int a; + + MyTestClass(int a) { + this.a = a; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClass myTestClass = (MyTestClass) o; + + if (a != myTestClass.a) return false; + + return true; + } + + @Override + public int hashCode() { + return a; + } + } + + public static class MyTestClassMarshallable implements BytesMarshallable { + int a; + + public MyTestClassMarshallable() { + } + + public MyTestClassMarshallable(int a) { + this.a = a; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClassMarshallable that = (MyTestClassMarshallable) o; + + if (a != that.a) return false; + + return true; + } + + @Override + public int hashCode() { + return a; + } + + public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { + a = in.readInt(); + } + + /** + * write an object to bytes + * + * @param out to write to + */ + public void writeMarshallable(@NotNull Bytes out) { + out.writeInt(a); + } + } + + +} \ No newline at end of file From 120454fce9f6b006404bfd609e3b829abfef481b Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:15:39 +0100 Subject: [PATCH 43/64] added Key Value Object Types --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 4f30fa7ca..154edc2b4 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,29 @@ and define the snapshot version in your pom.xml, for example: 1.0.1-SNAPSHOT ``` + +#### Key Value Object Types + +Unlike HashMap which will support any heap object, Chronicle Map only works with objects that it +can store off heap, so the objects have to be : (one of the following ) + +- AutoBoxed primitives - for good performance. +- Strings - for good performance. +- implements Serializable +- implements Externalizable ( with a public default constructor ) +- Implements our custom interface BytesMarshallable ( with a public default constructor ) - use this for best performance. + +or value objects that are created through, a directClass interface, for example : +``` java + ChronicleMap chm = ChronicleMapBuilder + .of(String.class, directClassFor(BondVOInterface.class)) + .create(file); + +``` + +Object graphs can also be included as long as the outer object supports Serializable, Externalizable or BytesMarshallable. + + #### Java Class Construction Creating an instance of Chronicle Map is a little more complexed than just calling a constructor. From 65a5477f55fbf340fd7de6543cd67c85f7ca14c2 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:21:14 +0100 Subject: [PATCH 44/64] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 154edc2b4..6445ff3d2 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,10 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [JavaDoc](http://openhft.github.io/Chronicle-Map/apidocs) * [Getting Started Guide](https://github.com/OpenHFT/Chronicle-Map#getting-started) * [Simple Construction](https://github.com/OpenHFT/Chronicle-Map#simple-construction) - * [Sharing Data Between Two or More Maps](https://github.com/OpenHFT/Chronicle-Map#sharing-data-between-two-or-more-maps) + * [Maven Download](https://github.com/OpenHFT/Chronicle-Map#maven-artifact-download-1) + * [Snapshot Download](https://github.com/OpenHFT/Chronicle-Map#maven-snapshot-download) + * [Key Value Object Types](https://github.com/OpenHFT/Chronicle-Map#key-value-object-types) + * [Sharing Data Between Two or More Maps](https://github.com/OpenHFT/Chronicle-Map#sharing-data-between-two-or-more-maps) * [Entries](https://github.com/OpenHFT/Chronicle-Map#entries) * [Size of Space Reserved on Disk](https://github.com/OpenHFT/Chronicle-Map#size-of-space-reserved-on-disk) * [Chronicle Map Interface](https://github.com/OpenHFT/Chronicle-Map#chronicle-map-interface) From 1fb48ade341beb09bd59c4e2a6a8dd90c49c06c9 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:22:29 +0100 Subject: [PATCH 45/64] formatting only --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6445ff3d2..8fcdf6bcd 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,8 @@ can store off heap, so the objects have to be : (one of the following ) - Strings - for good performance. - implements Serializable - implements Externalizable ( with a public default constructor ) -- Implements our custom interface BytesMarshallable ( with a public default constructor ) - use this for best performance. +- implements our custom interface BytesMarshallable ( with a public default constructor ) - use +this for best performance. or value objects that are created through, a directClass interface, for example : ``` java From 7719a05aaa344a3d00721f457c9c6d399f6de9d9 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:24:04 +0100 Subject: [PATCH 46/64] added MyTestClassObjectGraph test --- .../chronicle/map/StatelessMapClientTest.java | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java index e03a2d14f..19f5c39a5 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -8,7 +8,7 @@ import net.openhft.lang.model.constraints.NotNull; import org.junit.Assert; import org.junit.Test; -import java.io.Serializable; +import java.io.*; import java.nio.ByteBuffer; import java.util.Collections; @@ -27,6 +27,10 @@ public class StatelessMapClientTest extends TestCase { testReadValueWriteValue(new MyTestClass(3)); testReadValueWriteValue(new MyTestClassMarshallable(3)); + testReadValueWriteValue(new MyTestClassExternalizable(3)); + testReadValueWriteValue(new MyTestClassObjectGraph(3)); + + } public void testReadValueWriteValue(Object value) throws Exception { @@ -50,6 +54,45 @@ public class StatelessMapClientTest extends TestCase { } + public static class MyTestClassExternalizable implements Externalizable { + int a; + + public MyTestClassExternalizable() { + } + + + MyTestClassExternalizable(int a) { + this.a = a; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClassExternalizable that = (MyTestClassExternalizable) o; + + if (a != that.a) return false; + + return true; + } + + @Override + public int hashCode() { + return a; + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.write(a); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + a = in.readInt(); + } + } + public static class MyTestClass implements Serializable { int a; @@ -85,6 +128,10 @@ public class StatelessMapClientTest extends TestCase { this.a = a; } + public void setA(int a) { + this.a = a; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -116,5 +163,36 @@ public class StatelessMapClientTest extends TestCase { } } + public static class MyTestClassObjectGraph implements Serializable { + + MyTestClassMarshallable delegate; + + public MyTestClassObjectGraph() { + + } + + MyTestClassObjectGraph(int a) { + delegate = new MyTestClassMarshallable(a); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MyTestClassObjectGraph that = (MyTestClassObjectGraph) o; + + if (delegate != null ? !delegate.equals(that.delegate) : that.delegate != null) + return false; + + return true; + } + + @Override + public int hashCode() { + return delegate != null ? delegate.hashCode() : 0; + } + } + } \ No newline at end of file From a7b213f3add2cf44a3492653c2095ed1a443431d Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:51:36 +0100 Subject: [PATCH 47/64] fixed typo around "processes" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8fcdf6bcd..d0c77e809 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ the instances of Chronicle Map on the same server. The name and location of the up to you. For the best performance on many unix systems we recommend using [tmpfs](http://en.wikipedia.org/wiki/Tmpfs). -If instead, you do not wish to replicate between process on the same server or if you are only +If instead, you do not wish to replicate between processes on the same server or if you are only using TCP replication to replicate between servers, you do not have to provide the "file", so you can call `create()` method on ChronicleMapBuilder without file parameter: ``` From 2e0c5d0c86fc680e5045aa484f2f851f8fdc1adb Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:52:55 +0100 Subject: [PATCH 48/64] fixed typo around --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d0c77e809..bfb51bbc2 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,8 @@ up to you. For the best performance on many unix systems we recommend using If instead, you do not wish to replicate between processes on the same server or if you are only using TCP replication to replicate between servers, you do not have to provide the "file", -so you can call `create()` method on ChronicleMapBuilder without file parameter: +so you can call the `create()` method on you ChronicleMapBuilder without providing the file +parameter: ``` ConcurrentMap map = builder.create(); ``` From 0240485c910e5f98d88980707bb7c9dc4fda2f4f Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:53:16 +0100 Subject: [PATCH 49/64] HCOLL-167 Created a stateless map - added tests --- .../chronicle/map/ChronicleMapBuilder.java | 2 +- .../chronicle/map/StatelessMapClient.java | 104 ++++++++++++++---- .../chronicle/map/StatelessMapClientTest.java | 6 +- 3 files changed, 85 insertions(+), 27 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 725815757..504d08b58 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -786,7 +786,7 @@ public class ChronicleMapBuilder implements Cloneable, return identifier != -1; } - private void preMapConstruction() { + void preMapConstruction() { keyBuilder.objectSerializer(objectSerializer()); valueBuilder.objectSerializer(objectSerializer()); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 73cae54ba..31a84a946 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -21,7 +21,6 @@ import java.util.concurrent.atomic.AtomicLong; import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; /** - * * **** THIS IS VERY MUCH IN DRAFT *** * * @author Rob Austin. @@ -141,9 +140,43 @@ public class StatelessMapClient, lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); + keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); + keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); + valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); + valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); + + if (defaultValueProvider instanceof ConstantValueProvider) { + ConstantValueProvider constantValueProvider = + (ConstantValueProvider) defaultValueProvider; + if (constantValueProvider.wasDeserialized()) { + ThreadLocalCopies copies = valueReaderProvider.getCopies(null); + BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); + constantValueProvider.initTransients(valueReader); + } + } } + + void initTransients() { + keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); + keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); + + valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); + valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); + + if (defaultValueProvider instanceof ConstantValueProvider) { + ConstantValueProvider constantValueProvider = + (ConstantValueProvider) defaultValueProvider; + if (constantValueProvider.wasDeserialized()) { + ThreadLocalCopies copies = valueReaderProvider.getCopies(null); + BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); + constantValueProvider.initTransients(valueReader); + } + } + } + + @Override public V putIfAbsent(K key, V value) { target.writeByte(PUT_IF_ABSENT.ordinal()); @@ -151,7 +184,7 @@ public class StatelessMapClient, ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey(key, copies); - writeValue(value, copies); + writeValue(value); return null; } @@ -163,7 +196,7 @@ public class StatelessMapClient, ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey((K) key, copies); - writeValue((V) value, copies); + writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -179,8 +212,8 @@ public class StatelessMapClient, target.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey(key, copies); - writeValue(oldValue, copies); - writeValue(newValue, copies); + writeValue(oldValue); + writeValue(newValue); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -191,12 +224,13 @@ public class StatelessMapClient, } + @Override public V replace(K key, V value) { target.writeByte(REPLACE.ordinal()); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey(key, copies); - writeValue(value, copies); + writeValue(value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -251,8 +285,7 @@ public class StatelessMapClient, @Override public boolean containsValue(Object value) { target.writeByte(CONTAINS_VALUE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeValue((V) value, copies); + writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -289,8 +322,25 @@ public class StatelessMapClient, } - private V readValue(ByteBufferBytes byteBufferBytes) { - throw new UnsupportedOperationException("todo"); + V readValue(ByteBufferBytes source) { + + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + + copies = metaKeyInteropProvider.getCopies(copies); + + copies = valueReaderProvider.getCopies(copies); + BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); + + try { + final V v = valueFactory.create(); + + + long valueSize = valueSizeMarshaller.readSize(source); + return valueReader.read(source, valueSize, v); + } catch (Exception e) { + throw new RuntimeException(e); + } + } @Override @@ -309,7 +359,7 @@ public class StatelessMapClient, ThreadLocalCopies copies = keyInteropProvider.getCopies(null); writeKey((K) key, copies); - writeValue((V) value, copies); + writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); @@ -362,7 +412,7 @@ public class StatelessMapClient, for (Entry e : entries) { writeKey(e.getKey(), copies); - writeValue(e.getValue(), copies); + writeValue(e.getValue()); } } @@ -460,31 +510,39 @@ public class StatelessMapClient, } - private void writeValue(V value, ThreadLocalCopies copies) { - + void writeValue(V value) { + boolean byteableValue = value instanceof Byteable; + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); copies = valueWriterProvider.getCopies(copies); long valueSize; MetaBytesWriter metaValueWriter = null; Byteable valueAsByteable = null; - if (value instanceof Byteable) { - valueAsByteable = (Byteable) value; - valueSize = valueAsByteable.maxSize(); - - target.writeLong(valueSize); - target.write(valueAsByteable.bytes()); - } else { + if (!byteableValue) { copies = valueWriterProvider.getCopies(copies); valueWriter = valueWriterProvider.get(copies, originalValueWriter); copies = metaValueWriterProvider.getCopies(copies); metaValueWriter = metaValueWriterProvider.get( copies, originalMetaValueWriter, valueWriter, value); valueSize = metaValueWriter.size(valueWriter, value); - target.writeLong(valueSize); - metaValueWriter.write(valueWriter, target, value); + } else { + valueAsByteable = (Byteable) value; + valueSize = valueAsByteable.maxSize(); } + valueSizeMarshaller.writeSize(target, valueSize); + + + if (metaValueWriter != null) { + metaValueWriter.write(valueWriter, target, value); + } else { + throw new UnsupportedOperationException(""); + /* assert valueAsByteable != null; + long valueOffset = entry.positionAddr() - bytes.address(); + target.zeroOut(valueOffset, valueOffset + valueSize); + valueAsByteable.bytes(target, valueOffset);*/ + } } /** diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java index 19f5c39a5..4fb5c6bdf 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -25,10 +25,10 @@ public class StatelessMapClientTest extends TestCase { testReadValueWriteValue(Collections.singleton("Test")); testReadValueWriteValue(Collections.EMPTY_MAP); testReadValueWriteValue(new MyTestClass(3)); - testReadValueWriteValue(new MyTestClassMarshallable(3)); + testReadValueWriteValue(new MyTestClassMarshallable(3)); - testReadValueWriteValue(new MyTestClassExternalizable(3)); - testReadValueWriteValue(new MyTestClassObjectGraph(3)); + // testReadValueWriteValue(new MyTestClassExternalizable(3)); + // testReadValueWriteValue(new MyTestClassObjectGraph(3)); } From 078a22a1417a04203767f5f886d76bbad71a8452 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 13:59:43 +0100 Subject: [PATCH 50/64] HCOLL-167 Created a stateless map - added tests --- .../net/openhft/chronicle/map/StatelessMapClientTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java index 4fb5c6bdf..4b9e271f0 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java @@ -27,8 +27,8 @@ public class StatelessMapClientTest extends TestCase { testReadValueWriteValue(new MyTestClass(3)); testReadValueWriteValue(new MyTestClassMarshallable(3)); - // testReadValueWriteValue(new MyTestClassExternalizable(3)); - // testReadValueWriteValue(new MyTestClassObjectGraph(3)); + testReadValueWriteValue(new MyTestClassExternalizable(3)); + testReadValueWriteValue(new MyTestClassObjectGraph(3)); } @@ -165,14 +165,14 @@ public class StatelessMapClientTest extends TestCase { public static class MyTestClassObjectGraph implements Serializable { - MyTestClassMarshallable delegate; + MyTestClass delegate; public MyTestClassObjectGraph() { } MyTestClassObjectGraph(int a) { - delegate = new MyTestClassMarshallable(a); + delegate = new MyTestClass(a); } @Override From 3d228b591b00ec7ab23a75aabe9bdd01237c131f Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 15:45:34 +0100 Subject: [PATCH 51/64] HCOLL-167 Created a stateless map - added tests --- .../net/openhft/chronicle/map/Serializer.java | 114 ++++++++++++++++++ ...MapClientTest.java => SerializerTest.java} | 96 +++++++++++---- 2 files changed, 186 insertions(+), 24 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/map/Serializer.java rename src/test/java/net/openhft/chronicle/map/{StatelessMapClientTest.java => SerializerTest.java} (61%) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java new file mode 100644 index 000000000..ec112db76 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -0,0 +1,114 @@ +package net.openhft.chronicle.map; + +import net.openhft.chronicle.map.serialization.BytesReader; +import net.openhft.chronicle.map.serialization.MetaBytesWriter; +import net.openhft.chronicle.map.serialization.MetaProvider; +import net.openhft.chronicle.map.serialization.SizeMarshaller; +import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.io.serialization.ObjectFactory; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.constraints.NotNull; + +/** + * uses a {@code SerializationBuilder} to readMarshallable and writeMarshallable objects + * @author Rob Austin. + */ +public class Serializer> + implements BytesMarshallable { + + final Class clazz; + final SizeMarshaller sizeMarshaller; + final BytesReader originalReader; + transient Provider> readerProvider; + final VW originalWriter; + transient Provider writerProvider; + final MVW originalMetaValueWriter; + final MetaProvider metaValueWriterProvider; + final ObjectFactory objectFactory; + + final ThreadLocal value = new ThreadLocal(); + + public Serializer(final SerializationBuilder serializationBuilder) { + + clazz = serializationBuilder.eClass; + sizeMarshaller = serializationBuilder.sizeMarshaller(); + originalReader = serializationBuilder.reader(); + + originalWriter = (VW) serializationBuilder.interop(); + originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); + metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); + objectFactory = serializationBuilder.factory(); + + readerProvider = Provider.of((Class) originalReader.getClass()); + writerProvider = Provider.of((Class) originalWriter.getClass()); + } + + + public O getObject() { + return value.get(); + } + + public void setObject(O value) { + this.value.set(value); + } + + @Override + public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { + O o = readObject(in); + setObject(o); + } + + @Override + public void writeMarshallable(@NotNull Bytes out) { + O value1 = getObject(); + boolean byteableValue = value1 instanceof Byteable; + ThreadLocalCopies copies; + copies = readerProvider.getCopies(null); + VW valueWriter = writerProvider.get(copies, originalWriter); + copies = writerProvider.getCopies(copies); + long valueSize; + MetaBytesWriter metaValueWriter = null; + Byteable valueAsByteable = null; + + if (!byteableValue) { + copies = writerProvider.getCopies(copies); + valueWriter = writerProvider.get(copies, originalWriter); + copies = metaValueWriterProvider.getCopies(copies); + metaValueWriter = metaValueWriterProvider.get( + copies, originalMetaValueWriter, valueWriter, value1); + valueSize = metaValueWriter.size(valueWriter, value1); + } else { + valueAsByteable = (Byteable) value1; + valueSize = valueAsByteable.maxSize(); + } + + sizeMarshaller.writeSize(out, valueSize); + + + if (metaValueWriter != null) { + metaValueWriter.write(valueWriter, out, value1); + } else { + throw new UnsupportedOperationException(""); + } + } + + private O readObject(Bytes in) { + + ThreadLocalCopies copies; + copies = readerProvider.getCopies(null); + BytesReader valueReader = readerProvider.get(copies, originalReader); + + try { + final O v = objectFactory.create(); + long valueSize = sizeMarshaller.readSize(in); + return valueReader.read(in, valueSize, v); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + +} diff --git a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java similarity index 61% rename from src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java rename to src/test/java/net/openhft/chronicle/map/SerializerTest.java index 4b9e271f0..11bcfa046 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessMapClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -6,54 +6,102 @@ import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.BytesMarshallable; import net.openhft.lang.model.constraints.NotNull; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.io.*; import java.nio.ByteBuffer; import java.util.Collections; -public class StatelessMapClientTest extends TestCase { - +public class SerializerTest extends TestCase { @Test - public void testReadValueWriteIntegerValue() throws Exception { - testReadValueWriteValue("Test"); - testReadValueWriteValue(1); - testReadValueWriteValue(1L); - testReadValueWriteValue(1.0); - testReadValueWriteValue(1.0f); - testReadValueWriteValue(Collections.singleton("Test")); - testReadValueWriteValue(Collections.EMPTY_MAP); - testReadValueWriteValue(new MyTestClass(3)); - testReadValueWriteValue(new MyTestClassMarshallable(3)); - - testReadValueWriteValue(new MyTestClassExternalizable(3)); - testReadValueWriteValue(new MyTestClassObjectGraph(3)); - + public void testValueMarshallable() throws Exception { + testReadWriteValue("Test"); + testReadWriteValue(1); + testReadWriteValue(1L); + testReadWriteValue(1.0); + testReadWriteValue(1.0f); + testReadWriteValue(Collections.singleton("Test")); + testReadWriteValue(Collections.EMPTY_MAP); + testReadWriteValue(new MyTestClass(3)); + testReadWriteValue(new MyTestClassMarshallable(3)); + testReadWriteValue(new MyTestClassExternalizable(3)); + testReadWriteValue(new MyTestClassObjectGraph(3)); } - public void testReadValueWriteValue(Object value) throws Exception { + @Ignore + @Test + public void testKeyMarshallable() throws Exception { + testReadWriteKey("Test"); + testReadWriteKey(1); + testReadWriteKey(1L); + testReadWriteKey(1.0); + testReadWriteKey(1.0f); + testReadWriteKey(Collections.singleton("Test")); + testReadWriteKey(Collections.EMPTY_MAP); + testReadWriteKey(new MyTestClass(3)); + testReadWriteKey(new MyTestClassMarshallable(3)); + + testReadWriteKey(new MyTestClassExternalizable(3)); + testReadWriteKey(new MyTestClassObjectGraph(3)); + } + + + public void testReadWriteValue(Object value) throws Exception { Class valueClass = value.getClass(); - final ByteBufferBytes buffer = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); - ByteBufferBytes slice = buffer.slice(); + final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + ByteBufferBytes in = out.slice(); ChronicleMapBuilder builder = ChronicleMapBuilder.of(Integer.class, valueClass); builder.preMapConstruction(); - StatelessMapClient statelessMap = new StatelessMapClient(buffer, builder); + Serializer v = new Serializer(builder.valueBuilder); - statelessMap.writeValue(value); - long position = buffer.position(); - slice.limit(position); + v.setObject(value); + v.writeMarshallable(out); - Object actual = statelessMap.readValue(slice); + long position = out.position(); + in.limit(position); + + // let just blank the value so that we can test if it works + v.setObject(null); + + v.readMarshallable(in); + Object actual = v.getObject(); Assert.assertEquals(actual, value); } + public void testReadWriteKey(Object key) throws Exception { + + Class clazz = key.getClass(); + + final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + ByteBufferBytes in = out.slice(); + + ChronicleMapBuilder builder = ChronicleMapBuilder.of(clazz, Integer.class); + builder.preMapConstruction(); + + Serializer v = new Serializer(builder.keyBuilder); + + v.setObject(key); + v.writeMarshallable(out); + + long position = out.position(); + in.limit(position); + + // let just blank the value so that we can test if it works + v.setObject(null); + + v.readMarshallable(in); + Object actual = v.getObject(); + Assert.assertEquals(actual, key); + } + public static class MyTestClassExternalizable implements Externalizable { int a; From 3bc0be04c65161610309c8772507bd9b8367170d Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 15:48:31 +0100 Subject: [PATCH 52/64] HCOLL-167 Created a stateless map - woops missed this file --- src/main/java/net/openhft/chronicle/map/StatelessMapClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 31a84a946..8d6108386 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -79,7 +79,7 @@ public class StatelessMapClient, .getSimpleName()); } - enum EventId { + public static enum EventId { LONG_SIZE, SIZE, IS_EMPTY, CONTAINS_KEY, From 2e83944837688d424adf11409cd51c9abaac8a48 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 15:58:04 +0100 Subject: [PATCH 53/64] HCOLL-167 Created a stateless map - corrected @Ignore --- src/test/java/net/openhft/chronicle/map/SerializerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index 11bcfa046..dd6360f28 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -1,6 +1,5 @@ package net.openhft.chronicle.map; -import junit.framework.TestCase; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.BytesMarshallable; @@ -13,7 +12,7 @@ import java.io.*; import java.nio.ByteBuffer; import java.util.Collections; -public class SerializerTest extends TestCase { +public class SerializerTest { @Test public void testValueMarshallable() throws Exception { @@ -31,8 +30,9 @@ public class SerializerTest extends TestCase { testReadWriteValue(new MyTestClassObjectGraph(3)); } - @Ignore + @Test + @Ignore public void testKeyMarshallable() throws Exception { testReadWriteKey("Test"); testReadWriteKey(1); From e48a44e6e9cc62c5fe643144b3fdea4de733881d Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 16:23:57 +0100 Subject: [PATCH 54/64] HCOLL-167 Created a stateless map - fixed test case -net.openhft.chronicle.map.SerializerTest.testKeyMarshallable --- .../net/openhft/chronicle/map/Serializer.java | 65 ++--- .../chronicle/map/StatelessMapClient.java | 273 +++++------------- .../openhft/chronicle/map/SerializerTest.java | 20 +- 3 files changed, 94 insertions(+), 264 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index ec112db76..52643c9f2 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -7,19 +7,16 @@ import net.openhft.chronicle.map.serialization.SizeMarshaller; import net.openhft.chronicle.map.threadlocal.Provider; import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; -import net.openhft.lang.io.serialization.BytesMarshallable; -import net.openhft.lang.io.serialization.ObjectFactory; import net.openhft.lang.model.Byteable; import net.openhft.lang.model.constraints.NotNull; /** * uses a {@code SerializationBuilder} to readMarshallable and writeMarshallable objects + * * @author Rob Austin. */ -public class Serializer> - implements BytesMarshallable { +public class Serializer> { - final Class clazz; final SizeMarshaller sizeMarshaller; final BytesReader originalReader; transient Provider> readerProvider; @@ -27,44 +24,38 @@ public class Serializer> transient Provider writerProvider; final MVW originalMetaValueWriter; final MetaProvider metaValueWriterProvider; - final ObjectFactory objectFactory; - - final ThreadLocal value = new ThreadLocal(); public Serializer(final SerializationBuilder serializationBuilder) { - clazz = serializationBuilder.eClass; + sizeMarshaller = serializationBuilder.sizeMarshaller(); originalReader = serializationBuilder.reader(); originalWriter = (VW) serializationBuilder.interop(); originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); - objectFactory = serializationBuilder.factory(); readerProvider = Provider.of((Class) originalReader.getClass()); writerProvider = Provider.of((Class) originalWriter.getClass()); } + public O readMarshallable(@NotNull Bytes in) throws IllegalStateException { - public O getObject() { - return value.get(); + ThreadLocalCopies copies; + copies = readerProvider.getCopies(null); + BytesReader valueReader = readerProvider.get(copies, originalReader); + + try { + long valueSize = sizeMarshaller.readSize(in); + return valueReader.read(in, valueSize, null); + } catch (Exception e) { + throw new RuntimeException(e); + } } - public void setObject(O value) { - this.value.set(value); - } + public void writeMarshallable(@NotNull Bytes out, O value) { - @Override - public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { - O o = readObject(in); - setObject(o); - } - - @Override - public void writeMarshallable(@NotNull Bytes out) { - O value1 = getObject(); - boolean byteableValue = value1 instanceof Byteable; + boolean byteableValue = value instanceof Byteable; ThreadLocalCopies copies; copies = readerProvider.getCopies(null); VW valueWriter = writerProvider.get(copies, originalWriter); @@ -78,10 +69,10 @@ public class Serializer> valueWriter = writerProvider.get(copies, originalWriter); copies = metaValueWriterProvider.getCopies(copies); metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, value1); - valueSize = metaValueWriter.size(valueWriter, value1); + copies, originalMetaValueWriter, valueWriter, value); + valueSize = metaValueWriter.size(valueWriter, value); } else { - valueAsByteable = (Byteable) value1; + valueAsByteable = (Byteable) value; valueSize = valueAsByteable.maxSize(); } @@ -89,26 +80,10 @@ public class Serializer> if (metaValueWriter != null) { - metaValueWriter.write(valueWriter, out, value1); + metaValueWriter.write(valueWriter, out, value); } else { throw new UnsupportedOperationException(""); } } - private O readObject(Bytes in) { - - ThreadLocalCopies copies; - copies = readerProvider.getCopies(null); - BytesReader valueReader = readerProvider.get(copies, originalReader); - - try { - final O v = objectFactory.create(); - long valueSize = sizeMarshaller.readSize(in); - return valueReader.read(in, valueSize, v); - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 8d6108386..3297e5b20 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -1,15 +1,12 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.map.serialization.*; -import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.serialization.MetaBytesInterop; +import net.openhft.chronicle.map.serialization.MetaBytesWriter; import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.ByteBufferBytes; -import net.openhft.lang.io.BytesStore; -import net.openhft.lang.io.serialization.ObjectFactory; -import net.openhft.lang.model.Byteable; +import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; -import javax.swing.text.Segment; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -29,43 +26,18 @@ public class StatelessMapClient, V, VW, MVW extends MetaBytesWriter> extends AbstractMap implements ChronicleMap, Serializable { - final Class kClass; + private ThreadLocal sourceBuffer = new ThreadLocal(); - final BytesReader originalKeyReader; - transient Provider> keyReaderProvider; - final KI originalKeyInterop; - transient Provider keyInteropProvider; - final MKI originalMetaKeyInterop; - final MetaProvider metaKeyInteropProvider; - - final Class vClass; - final SizeMarshaller valueSizeMarshaller; - final BytesReader originalValueReader; - transient Provider> valueReaderProvider; - final VW originalValueWriter; - transient Provider valueWriterProvider; - final MVW originalMetaValueWriter; - final MetaProvider metaValueWriterProvider; - final ObjectFactory valueFactory; - final DefaultValueProvider defaultValueProvider; Map transactionIDs = new HashMap(); AtomicLong transactionID = new AtomicLong(); - private final long lockTimeOutNS; - - transient Segment[] segments; // non-final for close() - // non-final for close() and because it is initialized out of constructor - transient BytesStore ms; - transient long headerSize; - transient Set> entrySet; - - private int bits; - private int mask; private long timeoutMs = TimeUnit.SECONDS.toMillis(20); + private final Serializer valueSerializer; + private final Serializer keySerializer; @Override @@ -111,79 +83,24 @@ public class StatelessMapClient, } - private final ByteBufferBytes target; - private final SizeMarshaller keySizeMarshaller; - - public StatelessMapClient(ByteBufferBytes target, ChronicleMapBuilder builder) { - this.target = target; - - SerializationBuilder keyBuilder = builder.keyBuilder; - kClass = keyBuilder.eClass; - keySizeMarshaller = keyBuilder.sizeMarshaller(); - originalKeyReader = keyBuilder.reader(); - - originalKeyInterop = (KI) keyBuilder.interop(); - - originalMetaKeyInterop = (MKI) keyBuilder.metaInterop(); - metaKeyInteropProvider = (MetaProvider) keyBuilder.metaInteropProvider(); - - SerializationBuilder valueBuilder = builder.valueBuilder; - vClass = valueBuilder.eClass; - valueSizeMarshaller = valueBuilder.sizeMarshaller(); - originalValueReader = valueBuilder.reader(); - - originalValueWriter = (VW) valueBuilder.interop(); - originalMetaValueWriter = (MVW) valueBuilder.metaInterop(); - metaValueWriterProvider = (MetaProvider) valueBuilder.metaInteropProvider(); - valueFactory = valueBuilder.factory(); - defaultValueProvider = builder.defaultValueProvider(); - - lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); - - keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); - keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); - - valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); - valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); - - if (defaultValueProvider instanceof ConstantValueProvider) { - ConstantValueProvider constantValueProvider = - (ConstantValueProvider) defaultValueProvider; - if (constantValueProvider.wasDeserialized()) { - ThreadLocalCopies copies = valueReaderProvider.getCopies(null); - BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); - constantValueProvider.initTransients(valueReader); - } - } - } + private final Bytes out; - void initTransients() { - keyReaderProvider = Provider.of((Class) originalKeyReader.getClass()); - keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); + public StatelessMapClient(Bytes out, ChronicleMapBuilder builder) { - valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); - valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); + valueSerializer = new Serializer(builder.valueBuilder); + keySerializer = new Serializer(builder.keyBuilder); + + this.out = out; - if (defaultValueProvider instanceof ConstantValueProvider) { - ConstantValueProvider constantValueProvider = - (ConstantValueProvider) defaultValueProvider; - if (constantValueProvider.wasDeserialized()) { - ThreadLocalCopies copies = valueReaderProvider.getCopies(null); - BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); - constantValueProvider.initTransients(valueReader); - } - } } @Override public V putIfAbsent(K key, V value) { - target.writeByte(PUT_IF_ABSENT.ordinal()); + out.writeByte(PUT_IF_ABSENT.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - - writeKey(key, copies); + writeKey(key); writeValue(value); return null; @@ -192,15 +109,15 @@ public class StatelessMapClient, @Override public boolean remove(Object key, Object value) { - target.writeByte(REMOVE_WITH_VALUE.ordinal()); + out.writeByte(REMOVE_WITH_VALUE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + + writeKey((K) key); writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -209,15 +126,15 @@ public class StatelessMapClient, @Override public boolean replace(K key, V oldValue, V newValue) { - target.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey(key, copies); + out.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); + + writeKey(key); writeValue(oldValue); writeValue(newValue); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -227,14 +144,14 @@ public class StatelessMapClient, @Override public V replace(K key, V value) { - target.writeByte(REPLACE.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey(key, copies); + out.writeByte(REPLACE.ordinal()); + + writeKey(key); writeValue(value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -244,11 +161,11 @@ public class StatelessMapClient, @Override public int size() { - target.writeByte(SIZE.ordinal()); + out.writeByte(SIZE.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readInt(); @@ -257,11 +174,11 @@ public class StatelessMapClient, @Override public boolean isEmpty() { - target.writeByte(IS_EMPTY.ordinal()); + out.writeByte(IS_EMPTY.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -269,13 +186,13 @@ public class StatelessMapClient, @Override public boolean containsKey(Object key) { - target.writeByte(CONTAINS_KEY.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + out.writeByte(CONTAINS_KEY.ordinal()); + + writeKey((K) key); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -284,12 +201,12 @@ public class StatelessMapClient, @Override public boolean containsValue(Object value) { - target.writeByte(CONTAINS_VALUE.ordinal()); + out.writeByte(CONTAINS_VALUE.ordinal()); writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readBoolean(); @@ -297,11 +214,11 @@ public class StatelessMapClient, @Override public long longSize() { - target.writeByte(LONG_SIZE.ordinal()); + out.writeByte(LONG_SIZE.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return blockingFetch(transactionId).readLong(); @@ -309,13 +226,13 @@ public class StatelessMapClient, @Override public V get(Object key) { - target.writeByte(GET.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + out.writeByte(GET.ordinal()); + + writeKey((K) key); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -324,22 +241,9 @@ public class StatelessMapClient, V readValue(ByteBufferBytes source) { - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - copies = metaKeyInteropProvider.getCopies(copies); + return valueSerializer.readMarshallable(source); - copies = valueReaderProvider.getCopies(copies); - BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); - - try { - final V v = valueFactory.create(); - - - long valueSize = valueSizeMarshaller.readSize(source); - return valueReader.read(source, valueSize, v); - } catch (Exception e) { - throw new RuntimeException(e); - } } @@ -355,15 +259,15 @@ public class StatelessMapClient, @Override public V put(K key, V value) { - target.writeByte(PUT.ordinal()); + out.writeByte(PUT.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - writeKey((K) key, copies); + + writeKey((K) key); writeValue((V) value); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -372,11 +276,11 @@ public class StatelessMapClient, @Override public V remove(Object key) { - target.writeByte(REMOVE.ordinal()); + out.writeByte(REMOVE.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readValue(blockingFetch(transactionId)); @@ -385,9 +289,10 @@ public class StatelessMapClient, @Override public void putAll(Map map) { - target.writeByte(PUT_ALL.ordinal()); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + throw new UnsupportedOperationException("todo"); + /* out.writeByte(PUT_ALL.ordinal()); + HashMap safeCopy = new HashMap(map); @@ -395,34 +300,34 @@ public class StatelessMapClient, // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server blockingFetch(transactionId); - - return; +*/ + //return; } private void writeEntries(ThreadLocalCopies copies, HashMap safeCopy) { - target.writeLong(safeCopy.size()); + out.writeLong(safeCopy.size()); Set entries = (Set) safeCopy.entrySet(); for (Entry e : entries) { - writeKey(e.getKey(), copies); + writeKey(e.getKey()); writeValue(e.getValue()); } } @Override public void clear() { - target.writeByte(CLEAR.ordinal()); + out.writeByte(CLEAR.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server blockingFetch(transactionId); @@ -433,11 +338,11 @@ public class StatelessMapClient, @NotNull @Override public Set keySet() { - target.writeByte(KEY_SET.ordinal()); + out.writeByte(KEY_SET.ordinal()); // send the transaction id long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readKeySet(blockingFetch(transactionId)); @@ -456,7 +361,7 @@ public class StatelessMapClient, private K readKey(ByteBufferBytes bufferBytes) { long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server return readKey(blockingFetch(transactionId)); @@ -466,10 +371,10 @@ public class StatelessMapClient, @NotNull @Override public Collection values() { - target.writeByte(VALUES.ordinal()); + out.writeByte(VALUES.ordinal()); long transactionId = nextUniqueTransaction(); - target.writeLong(transactionId); + out.writeLong(transactionId); // get the data back from the server ByteBufferBytes bufferBytes = blockingFetch(transactionId); @@ -487,7 +392,7 @@ public class StatelessMapClient, @NotNull @Override public Set> entrySet() { - target.writeByte(ENTRY_SET.ordinal()); + out.writeByte(ENTRY_SET.ordinal()); long transactionId = nextUniqueTransaction(); @@ -511,56 +416,16 @@ public class StatelessMapClient, void writeValue(V value) { - boolean byteableValue = value instanceof Byteable; - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = valueWriterProvider.getCopies(copies); - long valueSize; - MetaBytesWriter metaValueWriter = null; - Byteable valueAsByteable = null; - - if (!byteableValue) { - copies = valueWriterProvider.getCopies(copies); - valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = metaValueWriterProvider.getCopies(copies); - metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, value); - valueSize = metaValueWriter.size(valueWriter, value); - } else { - valueAsByteable = (Byteable) value; - valueSize = valueAsByteable.maxSize(); - } - - valueSizeMarshaller.writeSize(target, valueSize); - - - if (metaValueWriter != null) { - metaValueWriter.write(valueWriter, target, value); - } else { - throw new UnsupportedOperationException(""); - /* assert valueAsByteable != null; - long valueOffset = entry.positionAddr() - bytes.address(); - target.zeroOut(valueOffset, valueOffset + valueSize); - valueAsByteable.bytes(target, valueOffset);*/ - } + valueSerializer.writeMarshallable(out, value); } /** * write the keysize and the key to the the {@code target} buffer * - * @param key the key of the map - * @param copies + * @param key the key of the map */ - private void writeKey(K key, final ThreadLocalCopies copies) { - KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); - - MKI metaKeyWriter = - metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); - - long keySize = metaKeyWriter.size(keyWriter, key); - - keySizeMarshaller.writeSize(target, keySize); - metaKeyWriter.write(keyWriter, target, key); + private void writeKey(K key) { + keySerializer.writeMarshallable(out, key); } diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index dd6360f28..eeea8b527 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -5,7 +5,6 @@ import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.BytesMarshallable; import net.openhft.lang.model.constraints.NotNull; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import java.io.*; @@ -32,7 +31,6 @@ public class SerializerTest { @Test - @Ignore public void testKeyMarshallable() throws Exception { testReadWriteKey("Test"); testReadWriteKey(1); @@ -61,17 +59,13 @@ public class SerializerTest { Serializer v = new Serializer(builder.valueBuilder); - v.setObject(value); - v.writeMarshallable(out); + v.writeMarshallable(out, value); long position = out.position(); in.limit(position); - // let just blank the value so that we can test if it works - v.setObject(null); - v.readMarshallable(in); - Object actual = v.getObject(); + Object actual = v.readMarshallable(in); Assert.assertEquals(actual, value); } @@ -88,17 +82,13 @@ public class SerializerTest { Serializer v = new Serializer(builder.keyBuilder); - v.setObject(key); - v.writeMarshallable(out); + + v.writeMarshallable(out, key); long position = out.position(); in.limit(position); - // let just blank the value so that we can test if it works - v.setObject(null); - - v.readMarshallable(in); - Object actual = v.getObject(); + Object actual = v.readMarshallable(in); Assert.assertEquals(actual, key); } From cffa0916ab945f6b2e6e81e3bc41bb16282cc4e4 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 17:03:19 +0100 Subject: [PATCH 55/64] HCOLL-167 improved the code in net.openhft.chronicle.map.StatelessMapClient, but its still in draft. --- .../net/openhft/chronicle/map/Serializer.java | 39 ++- .../chronicle/map/StatelessMapClient.java | 282 +++++++----------- 2 files changed, 118 insertions(+), 203 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index 52643c9f2..d91eec3da 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -11,7 +11,7 @@ import net.openhft.lang.model.Byteable; import net.openhft.lang.model.constraints.NotNull; /** - * uses a {@code SerializationBuilder} to readMarshallable and writeMarshallable objects + * uses a {@code SerializationBuilder} to read and write object(s) as bytes * * @author Rob Austin. */ @@ -27,23 +27,21 @@ public class Serializer> { public Serializer(final SerializationBuilder serializationBuilder) { - sizeMarshaller = serializationBuilder.sizeMarshaller(); - originalReader = serializationBuilder.reader(); - - originalWriter = (VW) serializationBuilder.interop(); originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); + originalReader = serializationBuilder.reader(); + originalWriter = (VW) serializationBuilder.interop(); + readerProvider = Provider.of((Class) originalReader.getClass()); writerProvider = Provider.of((Class) originalWriter.getClass()); } public O readMarshallable(@NotNull Bytes in) throws IllegalStateException { - ThreadLocalCopies copies; - copies = readerProvider.getCopies(null); - BytesReader valueReader = readerProvider.get(copies, originalReader); + final ThreadLocalCopies copies = readerProvider.getCopies(null); + final BytesReader valueReader = readerProvider.get(copies, originalReader); try { long valueSize = sizeMarshaller.readSize(in); @@ -55,35 +53,32 @@ public class Serializer> { public void writeMarshallable(@NotNull Bytes out, O value) { - boolean byteableValue = value instanceof Byteable; - ThreadLocalCopies copies; - copies = readerProvider.getCopies(null); + ThreadLocalCopies copies = readerProvider.getCopies(null); VW valueWriter = writerProvider.get(copies, originalWriter); copies = writerProvider.getCopies(copies); - long valueSize; - MetaBytesWriter metaValueWriter = null; - Byteable valueAsByteable = null; - if (!byteableValue) { + final long valueSize; + + MetaBytesWriter metaValueWriter = null; + + if ((value instanceof Byteable)) { + valueSize = ((Byteable) value).maxSize(); + } else { copies = writerProvider.getCopies(copies); valueWriter = writerProvider.get(copies, originalWriter); copies = metaValueWriterProvider.getCopies(copies); metaValueWriter = metaValueWriterProvider.get( copies, originalMetaValueWriter, valueWriter, value); valueSize = metaValueWriter.size(valueWriter, value); - } else { - valueAsByteable = (Byteable) value; - valueSize = valueAsByteable.maxSize(); } sizeMarshaller.writeSize(out, valueSize); - - if (metaValueWriter != null) { + if (metaValueWriter != null) metaValueWriter.write(valueWriter, out, value); - } else { + else throw new UnsupportedOperationException(""); - } + } } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 3297e5b20..5693cc691 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -1,15 +1,11 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.map.serialization.MetaBytesInterop; -import net.openhft.chronicle.map.serialization.MetaBytesWriter; -import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; -import java.io.Serializable; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -22,34 +18,7 @@ import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; * * @author Rob Austin. */ -public class StatelessMapClient, - V, VW, MVW extends MetaBytesWriter> extends AbstractMap - implements ChronicleMap, Serializable { - - - private ThreadLocal sourceBuffer = new ThreadLocal(); - - - Map transactionIDs = new HashMap(); - - - AtomicLong transactionID = new AtomicLong(); - - private long timeoutMs = TimeUnit.SECONDS.toMillis(20); - private final Serializer valueSerializer; - private final Serializer keySerializer; - - - @Override - public File file() { - return null; - } - - @Override - public void close() throws IOException { - throw new UnsupportedOperationException("This is not supported in the " + this.getClass() - .getSimpleName()); - } +public class StatelessMapClient implements ChronicleMap { public static enum EventId { LONG_SIZE, SIZE, @@ -69,6 +38,27 @@ public class StatelessMapClient, REMOVE_WITH_VALUE } + private final ThreadLocal sourceBuffer = new ThreadLocal(); + private final Map transactionIDs = new HashMap(); + + private final AtomicLong transactionID = new AtomicLong(); + + private long timeoutMs = TimeUnit.SECONDS.toMillis(20); + private final Serializer valueSerializer; + private final Serializer keySerializer; + + + @Override + public File file() { + return null; + } + + @Override + public void close() throws IOException { + throw new UnsupportedOperationException("This is not supported in the " + this.getClass() + .getSimpleName()); + } + long nextUniqueTransaction() { long time = System.currentTimeMillis(); @@ -86,165 +76,127 @@ public class StatelessMapClient, private final Bytes out; - public StatelessMapClient(Bytes out, ChronicleMapBuilder builder) { + public StatelessMapClient(Bytes out, Class kClass, Class vClass) { - valueSerializer = new Serializer(builder.valueBuilder); - keySerializer = new Serializer(builder.keyBuilder); + final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, + SerializationBuilder.Role.KEY); + final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, + SerializationBuilder.Role.VALUE); + + keySerializer = new Serializer(keyBuilder); + valueSerializer = new Serializer(valueBuilder); this.out = out; - } @Override public V putIfAbsent(K key, V value) { - out.writeByte(PUT_IF_ABSENT.ordinal()); - + writeEvent(PUT_IF_ABSENT); writeKey(key); writeValue(value); - return null; + return readValue(blockingFetch()); + } @Override public boolean remove(Object key, Object value) { - out.writeByte(REMOVE_WITH_VALUE.ordinal()); - - + writeEvent(REMOVE_WITH_VALUE); writeKey((K) key); writeValue((V) value); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } + @Override public boolean replace(K key, V oldValue, V newValue) { - out.writeByte(REPLACE_WITH_OLD_AND_NEW_VALUE.ordinal()); - + writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); writeKey(key); writeValue(oldValue); writeValue(newValue); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); + } + private void writeValue(V newValue) { + valueSerializer.writeMarshallable(out, newValue); } @Override public V replace(K key, V value) { - out.writeByte(REPLACE.ordinal()); + writeEvent(REPLACE); writeKey(key); writeValue(value); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } @Override public int size() { - out.writeByte(SIZE.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(SIZE); // get the data back from the server - return blockingFetch(transactionId).readInt(); + return blockingFetch().readInt(); } @Override public boolean isEmpty() { - out.writeByte(IS_EMPTY.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(IS_EMPTY); // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } @Override public boolean containsKey(Object key) { - out.writeByte(CONTAINS_KEY.ordinal()); - + writeEvent(CONTAINS_KEY); writeKey((K) key); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } @Override public boolean containsValue(Object value) { - out.writeByte(CONTAINS_VALUE.ordinal()); + writeEvent(CONTAINS_VALUE); writeValue((V) value); - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return blockingFetch(transactionId).readBoolean(); + return blockingFetch().readBoolean(); } @Override public long longSize() { - out.writeByte(LONG_SIZE.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(LONG_SIZE); // get the data back from the server - return blockingFetch(transactionId).readLong(); + return blockingFetch().readLong(); } @Override public V get(Object key) { - out.writeByte(GET.ordinal()); - + writeEvent(GET); writeKey((K) key); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } - V readValue(ByteBufferBytes source) { - - + V readValue(Bytes source) { return valueSerializer.readMarshallable(source); - - } @Override @@ -259,61 +211,43 @@ public class StatelessMapClient, @Override public V put(K key, V value) { - out.writeByte(PUT.ordinal()); + writeEvent(PUT); + writeKey(key); + writeValue(value); - writeKey((K) key); - writeValue((V) value); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } @Override public V remove(Object key) { - out.writeByte(REMOVE.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - + writeEvent(REMOVE); + writeKey((K) key); // get the data back from the server - return readValue(blockingFetch(transactionId)); + return readValue(blockingFetch()); } @Override public void putAll(Map map) { - throw new UnsupportedOperationException("todo"); - /* out.writeByte(PUT_ALL.ordinal()); - - - HashMap safeCopy = new HashMap(map); - - writeEntries(copies, safeCopy); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(PUT_ALL); + writeEntries(map); // get the data back from the server - blockingFetch(transactionId); -*/ - //return; - + blockingFetch(); } - private void writeEntries(ThreadLocalCopies copies, HashMap safeCopy) { + private void writeEntries(Map map) { + + final HashMap safeCopy = new HashMap(map); + out.writeLong(safeCopy.size()); - Set entries = (Set) safeCopy.entrySet(); + final Set entries = (Set) safeCopy.entrySet(); for (Entry e : entries) { writeKey(e.getKey()); @@ -323,68 +257,50 @@ public class StatelessMapClient, @Override public void clear() { - out.writeByte(CLEAR.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(CLEAR); // get the data back from the server - blockingFetch(transactionId); + blockingFetch(); - return; } @NotNull @Override public Set keySet() { - out.writeByte(KEY_SET.ordinal()); - - // send the transaction id - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); + writeEvent(KEY_SET); // get the data back from the server - return readKeySet(blockingFetch(transactionId)); + return readKeySet(blockingFetch()); } - private Set readKeySet(ByteBufferBytes bufferBytes) { - int size = bufferBytes.readInt(); - - HashSet result = new HashSet<>(); + private Set readKeySet(Bytes in) { + int size = in.readInt(); + final HashSet result = new HashSet<>(); for (int i = 0; i < size; i++) { - result.add(readKey(bufferBytes)); + result.add(readKey(out)); } return result; } - private K readKey(ByteBufferBytes bufferBytes) { - long transactionId = nextUniqueTransaction(); - out.writeLong(transactionId); - - // get the data back from the server - return readKey(blockingFetch(transactionId)); - + private K readKey(Bytes bufferBytes) { + return keySerializer.readMarshallable(bufferBytes); } @NotNull @Override public Collection values() { - out.writeByte(VALUES.ordinal()); - long transactionId = nextUniqueTransaction(); - - out.writeLong(transactionId); + writeEvent(VALUES); // get the data back from the server - ByteBufferBytes bufferBytes = blockingFetch(transactionId); + Bytes in = blockingFetch(); - int size = bufferBytes.readInt(); + int size = in.readInt(); ArrayList result = new ArrayList(); for (int i = 0; i < size; i++) { - result.add(readValue(bufferBytes)); + result.add(readValue(in)); } return result; } @@ -392,33 +308,27 @@ public class StatelessMapClient, @NotNull @Override public Set> entrySet() { - out.writeByte(ENTRY_SET.ordinal()); + writeEvent(ENTRY_SET); - long transactionId = nextUniqueTransaction(); // get the data back from the server - ByteBufferBytes bufferBytes = blockingFetch(transactionId); + Bytes bytes = blockingFetch(); - int size = bufferBytes.readInt(); + int size = bytes.readInt(); Set> result = new HashSet>(); for (int i = 0; i < size; i++) { - K k = readKey(bufferBytes); - V v = readValue(bufferBytes); + K k = keySerializer.readMarshallable(bytes); + V v = valueSerializer.readMarshallable(bytes); result.add(new Entry(k, v)); } - return result; } - void writeValue(V value) { - valueSerializer.writeMarshallable(out, value); - } - /** * write the keysize and the key to the the {@code target} buffer * @@ -428,6 +338,9 @@ public class StatelessMapClient, keySerializer.writeMarshallable(out, key); } + private void writeEvent(EventId event) { + out.write((byte) event.ordinal()); + } public Buffer buffer() { Buffer result = sourceBuffer.get(); @@ -463,7 +376,7 @@ public class StatelessMapClient, * @param transactionId * @return */ - private ByteBufferBytes blockingFetch(long transactionId) { + private Bytes blockingFetch(long transactionId) { final Buffer buffer = buffer(); buffer.set(null); @@ -538,6 +451,13 @@ public class StatelessMapClient, } + + + private Bytes blockingFetch() { + long transactionId = nextUniqueTransaction(); + out.writeLong(transactionId); + return blockingFetch(transactionId); + } } From 3c47a33777073d48a0ab5c67fb09c52ada5dd2a8 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 17:33:39 +0100 Subject: [PATCH 56/64] HCOLL-167 updated to - net.openhft lang 6.4.10 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bb883ecb1..5c1b49bfe 100755 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ net.openhft lang - 6.4.10-SNAPSHOT + 6.4.10 From fe574a33b10e20b22c1b24a8f81808faecad73d4 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 18:29:35 +0100 Subject: [PATCH 57/64] HCOLL-167 created StatelessServerConnector, but its still in draft. --- .../net/openhft/chronicle/map/Serializer.java | 2 +- .../chronicle/map/StatelessMapClient.java | 48 ++-- .../map/StatelessServerConnector.java | 265 ++++++++++++++++++ .../openhft/chronicle/map/SerializerTest.java | 4 +- 4 files changed, 292 insertions(+), 27 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index d91eec3da..00bcdacb6 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -51,7 +51,7 @@ public class Serializer> { } } - public void writeMarshallable(@NotNull Bytes out, O value) { + public void writeMarshallable(O value, @NotNull Bytes out) { ThreadLocalCopies copies = readerProvider.getCopies(null); VW valueWriter = writerProvider.get(copies, originalWriter); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 5693cc691..967a000a8 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -47,6 +47,18 @@ public class StatelessMapClient implements ChronicleMap { private final Serializer valueSerializer; private final Serializer keySerializer; + public StatelessMapClient(Bytes out, Class kClass, Class vClass) { + + final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, + SerializationBuilder.Role.KEY); + final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, + SerializationBuilder.Role.VALUE); + + keySerializer = new Serializer(keyBuilder); + valueSerializer = new Serializer(valueBuilder); + + this.out = out; + } @Override public File file() { @@ -76,20 +88,6 @@ public class StatelessMapClient implements ChronicleMap { private final Bytes out; - public StatelessMapClient(Bytes out, Class kClass, Class vClass) { - - final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, - SerializationBuilder.Role.KEY); - final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, - SerializationBuilder.Role.VALUE); - - keySerializer = new Serializer(keyBuilder); - valueSerializer = new Serializer(valueBuilder); - - this.out = out; - } - - @Override public V putIfAbsent(K key, V value) { writeEvent(PUT_IF_ABSENT); @@ -125,7 +123,7 @@ public class StatelessMapClient implements ChronicleMap { } private void writeValue(V newValue) { - valueSerializer.writeMarshallable(out, newValue); + valueSerializer.writeMarshallable(newValue, out); } @@ -245,7 +243,7 @@ public class StatelessMapClient implements ChronicleMap { final HashMap safeCopy = new HashMap(map); - out.writeLong(safeCopy.size()); + out.writeStopBit(safeCopy.size()); final Set entries = (Set) safeCopy.entrySet(); @@ -274,10 +272,10 @@ public class StatelessMapClient implements ChronicleMap { } private Set readKeySet(Bytes in) { - int size = in.readInt(); + long size = in.readStopBit(); final HashSet result = new HashSet<>(); - for (int i = 0; i < size; i++) { + for (long i = 0; i < size; i++) { result.add(readKey(out)); } return result; @@ -293,11 +291,14 @@ public class StatelessMapClient implements ChronicleMap { writeEvent(VALUES); // get the data back from the server - Bytes in = blockingFetch(); + final Bytes in = blockingFetch(); - int size = in.readInt(); + final long size = in.readStopBit(); - ArrayList result = new ArrayList(); + if (size > Integer.MAX_VALUE) + throw new IllegalStateException("size=" + size + " is too large."); + + final ArrayList result = new ArrayList((int) size); for (int i = 0; i < size; i++) { result.add(readValue(in)); @@ -314,7 +315,7 @@ public class StatelessMapClient implements ChronicleMap { // get the data back from the server Bytes bytes = blockingFetch(); - int size = bytes.readInt(); + long size = bytes.readStopBit(); Set> result = new HashSet>(); @@ -322,7 +323,6 @@ public class StatelessMapClient implements ChronicleMap { K k = keySerializer.readMarshallable(bytes); V v = valueSerializer.readMarshallable(bytes); result.add(new Entry(k, v)); - } return result; @@ -335,7 +335,7 @@ public class StatelessMapClient implements ChronicleMap { * @param key the key of the map */ private void writeKey(K key) { - keySerializer.writeMarshallable(out, key); + keySerializer.writeMarshallable(key, out); } private void writeEvent(EventId event) { diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java new file mode 100644 index 000000000..f40669bbf --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -0,0 +1,265 @@ +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.model.constraints.NotNull; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author Rob Austin. + */ +public class StatelessServerConnector { + + + private final ChronicleMap map; + + + private final Serializer valueSerializer; + private final Serializer keySerializer; + + public StatelessServerConnector(Class kClass, Class vClass, ChronicleMap map) { + + this.map = map; + + final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, + SerializationBuilder.Role.KEY); + final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, + SerializationBuilder.Role.VALUE); + + keySerializer = new Serializer(keyBuilder); + valueSerializer = new Serializer(valueBuilder); + + } + + + public void marshall(@NotNull Bytes in, @NotNull Bytes out) { + + byte b = in.readByte(); + + StatelessMapClient.EventId eventId = StatelessMapClient.EventId.values()[b]; + + switch (eventId) { + + case LONG_SIZE: + longSize(in, out); + break; + + case IS_EMPTY: + isEmpty(in, out); + break; + + case CONTAINS_KEY: + containsKey(in, out); + break; + case CONTAINS_VALUE: + containsValue(in, out); + break; + case GET: + get(in, out); + break; + case PUT: + put(in, out); + break; + case REMOVE: + remove(in, out); + break; + case PUT_ALL: + putAll(in, out); + break; + case CLEAR: + clear(in, out); + break; + case KEY_SET: + keySet(in, out); + break; + case VALUES: + values(in, out); + break; + case ENTRY_SET: + entrySet(in, out); + break; + case REPLACE: + replace(in, out); + break; + case REPLACE_WITH_OLD_AND_NEW_VALUE: + replaceWithOldAndNew(in, out); + break; + case PUT_IF_ABSENT: + putIfAbsent(in, out); + break; + case REMOVE_WITH_VALUE: + removeWithValue(in, out); + break; + + } + + } + + private void removeWithValue(Bytes in, Bytes out) { + boolean result = map.remove(readKey(in), readValue(in)); + reflectTransactionId(in, out); + out.writeBoolean(result); + } + + private void replaceWithOldAndNew(Bytes in, Bytes out) { + final K key = readKey(in); + V oldValue = readValue(in); + V newValue = readValue(in); + reflectTransactionId(in, out); + map.replace(key, oldValue, newValue); + } + + + public void longSize(Bytes in, Bytes out) { + reflectTransactionId(in, out); + out.writeLong(map.longSize()); + } + + + public void size(Bytes in, Bytes out) { + reflectTransactionId(in, out); + out.writeInt(map.size()); + } + + + public void isEmpty(Bytes in, Bytes out) { + reflectTransactionId(in, out); + out.writeBoolean(map.isEmpty()); + } + + + public void containsKey(Bytes in, Bytes out) { + K k = readKey(in); + reflectTransactionId(in, out); + out.writeBoolean(map.containsKey(k)); + } + + + public void containsValue(Bytes in, Bytes out) { + V v = readValue(in); + reflectTransactionId(in, out); + out.writeBoolean(map.containsValue(v)); + } + + + public void get(Bytes in, Bytes out) { + K k = readKey(in); + reflectTransactionId(in, out); + writeValue(map.get(k), out); + } + + + public void put(Bytes in, Bytes out) { + K k = readKey(in); + V v = readValue(in); + reflectTransactionId(in, out); + writeValue(map.put(k, v), out); + } + + + public void remove(Bytes in, Bytes out) { + final V value = map.remove(readKey(in)); + reflectTransactionId(in, out); + writeValue(value, out); + } + + + public void putAll(Bytes in, Bytes out) { + map.putAll(readEntries(in)); + reflectTransactionId(in, out); + } + + private Map readEntries(Bytes in) { + + long size = in.readStopBit(); + final HashMap result = new HashMap(); + + for (long i = 0; i < size; i++) { + result.put(readKey(in), readValue(in)); + } + return result; + + } + + public void clear(Bytes in, Bytes out) { + map.clear(); + reflectTransactionId(in, out); + } + + + public void keySet(Bytes in, Bytes out) { + reflectTransactionId(in, out); + + final Set ks = map.keySet(); + out.writeStopBit(ks.size()); + for (K key : ks) { + writeKey(key, out); + } + } + + public void values(Bytes in, Bytes out) { + reflectTransactionId(in, out); + final Collection values = map.values(); + out.writeStopBit(values.size()); + for (final V value : values) { + writeValue(value, out); + } + } + + public void entrySet(Bytes in, Bytes out) { + reflectTransactionId(in, out); + + final Set> entries = map.entrySet(); + out.writeStopBit(entries.size()); + for (Map.Entry e : entries) { + writeKey(e.getKey(), out); + writeValue(e.getValue(), out); + } + } + + public void putIfAbsent(Bytes in, Bytes out) { + K key = readKey(in); + V v = readValue(in); + reflectTransactionId(in, out); + writeValue(map.putIfAbsent(key, v), out); + } + + + public void replace(Bytes in, Bytes out) { + K k = readKey(in); + V v = readValue(in); + + reflectTransactionId(in, out); + map.replace(k, v); + } + + + /** + * write the keysize and the key to the the {@code target} buffer + * + * @param key the key of the map + */ + private void writeKey(K key, Bytes out) { + keySerializer.writeMarshallable(key, out); + } + + private void reflectTransactionId(Bytes in, Bytes out) { + out.writeLong(in.readLong()); + } + + private void writeValue(V value, final Bytes out) { + valueSerializer.writeMarshallable(value, out); + } + + private K readKey(Bytes in) { + return keySerializer.readMarshallable(in); + } + + private V readValue(Bytes in) { + return valueSerializer.readMarshallable(in); + } + +} diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index eeea8b527..c5d52aa3e 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -59,7 +59,7 @@ public class SerializerTest { Serializer v = new Serializer(builder.valueBuilder); - v.writeMarshallable(out, value); + v.writeMarshallable(value, out); long position = out.position(); in.limit(position); @@ -83,7 +83,7 @@ public class SerializerTest { Serializer v = new Serializer(builder.keyBuilder); - v.writeMarshallable(out, key); + v.writeMarshallable(key, out); long position = out.position(); in.limit(position); From 808e14cf8d7d548ef6db75d47fda1bf9df0a3100 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:30:27 +0100 Subject: [PATCH 58/64] [maven-release-plugin] prepare release chronicle-map-2.0.3a --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5c1b49bfe..381ca0212 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.3a-SNAPSHOT + 2.0.3a OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - master + chronicle-map-2.0.3a From cb5f2e06931a161172b7008fdedf832df5cd5783 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:30:32 +0100 Subject: [PATCH 59/64] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 381ca0212..b5acc57f7 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.3a + 2.0.4a-SNAPSHOT OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - chronicle-map-2.0.3a + master From b7a7d5a3852d6f840e8e317b2ee96c7e7e4b5826 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:41:14 +0100 Subject: [PATCH 60/64] [maven-release-plugin] prepare release chronicle-map-2.0.4a --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b5acc57f7..8288d4918 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.4a-SNAPSHOT + 2.0.4a OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - master + chronicle-map-2.0.4a From 15db660e42d5143f94941f033103566c65578e89 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 15 Oct 2014 18:41:18 +0100 Subject: [PATCH 61/64] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8288d4918..608f7d39a 100755 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 chronicle-map - 2.0.4a + 2.0.5a-SNAPSHOT OpenHFT/Chronicle-Map Chronicle-Map @@ -139,7 +139,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - chronicle-map-2.0.4a + master From 61c32a0490ca63012a9a26c87f8a17eeed66d2a7 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 15 Oct 2014 18:43:39 +0100 Subject: [PATCH 62/64] updated to dependency> net.openhft affinity 2.1.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5c1b49bfe..badceb380 100755 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ net.openhft affinity - 2.1.0 + 2.1.2 From 6030b6338cd772603ee9d359fdac8d4b2561bf3c Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Thu, 16 Oct 2014 10:35:55 +0100 Subject: [PATCH 63/64] HCOLL-167 improved StatelessServerConnector, but its still in draft. --- .../net/openhft/chronicle/map/Serializer.java | 27 +++---- .../chronicle/map/StatelessMapClient.java | 78 +++++++----------- .../map/StatelessServerConnector.java | 79 ++++++++++++------- .../openhft/chronicle/map/SerializerTest.java | 18 +++-- 4 files changed, 104 insertions(+), 98 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Serializer.java b/src/main/java/net/openhft/chronicle/map/Serializer.java index 00bcdacb6..d30bb388a 100644 --- a/src/main/java/net/openhft/chronicle/map/Serializer.java +++ b/src/main/java/net/openhft/chronicle/map/Serializer.java @@ -1,9 +1,6 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.map.serialization.BytesReader; -import net.openhft.chronicle.map.serialization.MetaBytesWriter; -import net.openhft.chronicle.map.serialization.MetaProvider; -import net.openhft.chronicle.map.serialization.SizeMarshaller; +import net.openhft.chronicle.map.serialization.*; import net.openhft.chronicle.map.threadlocal.Provider; import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; @@ -15,24 +12,24 @@ import net.openhft.lang.model.constraints.NotNull; * * @author Rob Austin. */ -public class Serializer> { +class Serializer { final SizeMarshaller sizeMarshaller; final BytesReader originalReader; transient Provider> readerProvider; - final VW originalWriter; - transient Provider writerProvider; - final MVW originalMetaValueWriter; - final MetaProvider metaValueWriterProvider; + final Object originalWriter; + transient Provider writerProvider; + final MetaBytesInterop originalMetaValueWriter; + final MetaProvider metaValueWriterProvider; - public Serializer(final SerializationBuilder serializationBuilder) { + Serializer(final SerializationBuilder serializationBuilder) { sizeMarshaller = serializationBuilder.sizeMarshaller(); - originalMetaValueWriter = (MVW) serializationBuilder.metaInterop(); - metaValueWriterProvider = (MetaProvider) serializationBuilder.metaInteropProvider(); + originalMetaValueWriter = serializationBuilder.metaInterop(); + metaValueWriterProvider = serializationBuilder.metaInteropProvider(); originalReader = serializationBuilder.reader(); - originalWriter = (VW) serializationBuilder.interop(); + originalWriter = serializationBuilder.interop(); readerProvider = Provider.of((Class) originalReader.getClass()); writerProvider = Provider.of((Class) originalWriter.getClass()); @@ -54,12 +51,12 @@ public class Serializer> { public void writeMarshallable(O value, @NotNull Bytes out) { ThreadLocalCopies copies = readerProvider.getCopies(null); - VW valueWriter = writerProvider.get(copies, originalWriter); + Object valueWriter = writerProvider.get(copies, originalWriter); copies = writerProvider.getCopies(copies); final long valueSize; - MetaBytesWriter metaValueWriter = null; + MetaBytesWriter metaValueWriter = null; if ((value instanceof Byteable)) { valueSize = ((Byteable) value).maxSize(); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 967a000a8..6e2fd77b4 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -18,7 +18,7 @@ import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; * * @author Rob Austin. */ -public class StatelessMapClient implements ChronicleMap { +class StatelessMapClient implements ChronicleMap { public static enum EventId { LONG_SIZE, SIZE, @@ -44,20 +44,11 @@ public class StatelessMapClient implements ChronicleMap { private final AtomicLong transactionID = new AtomicLong(); private long timeoutMs = TimeUnit.SECONDS.toMillis(20); - private final Serializer valueSerializer; - private final Serializer keySerializer; - public StatelessMapClient(Bytes out, Class kClass, Class vClass) { - - final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, - SerializationBuilder.Role.KEY); - final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, - SerializationBuilder.Role.VALUE); - - keySerializer = new Serializer(keyBuilder); - valueSerializer = new Serializer(valueBuilder); + public StatelessMapClient(Bytes out, KeyValueSerializer keyValueSerializer) { this.out = out; + this.keyValueSerializer = keyValueSerializer; } @Override @@ -86,6 +77,7 @@ public class StatelessMapClient implements ChronicleMap { private final Bytes out; + private final KeyValueSerializer keyValueSerializer; @Override @@ -94,8 +86,7 @@ public class StatelessMapClient implements ChronicleMap { writeKey(key); writeValue(value); - return readValue(blockingFetch()); - + return readKey(); } @@ -110,7 +101,6 @@ public class StatelessMapClient implements ChronicleMap { } - @Override public boolean replace(K key, V oldValue, V newValue) { writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); @@ -122,20 +112,19 @@ public class StatelessMapClient implements ChronicleMap { return blockingFetch().readBoolean(); } - private void writeValue(V newValue) { - valueSerializer.writeMarshallable(newValue, out); + private void writeKey(K key) { + keyValueSerializer.writeKey(key, out); } @Override public V replace(K key, V value) { writeEvent(REPLACE); - writeKey(key); writeValue(value); // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } @@ -189,13 +178,10 @@ public class StatelessMapClient implements ChronicleMap { writeEvent(GET); writeKey((K) key); // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } - V readValue(Bytes source) { - return valueSerializer.readMarshallable(source); - } @Override public V getUsing(K key, V value) { @@ -209,22 +195,30 @@ public class StatelessMapClient implements ChronicleMap { @Override public V put(K key, V value) { + writeEvent(PUT); writeKey(key); writeValue(value); - // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } + private V readKey() { + return keyValueSerializer.readValue(blockingFetch()); + } + + private void writeValue(V value) { + keyValueSerializer.writeValue(value, out); + } + @Override public V remove(Object key) { writeEvent(REMOVE); writeKey((K) key); // get the data back from the server - return readValue(blockingFetch()); + return readKey(); } @@ -276,14 +270,11 @@ public class StatelessMapClient implements ChronicleMap { final HashSet result = new HashSet<>(); for (long i = 0; i < size; i++) { - result.add(readKey(out)); + result.add(keyValueSerializer.readKey(in)); } return result; } - private K readKey(Bytes bufferBytes) { - return keySerializer.readMarshallable(bufferBytes); - } @NotNull @Override @@ -291,21 +282,22 @@ public class StatelessMapClient implements ChronicleMap { writeEvent(VALUES); // get the data back from the server - final Bytes in = blockingFetch(); + final Bytes in = blockingFetch(); - final long size = in.readStopBit(); + final long size = in.readStopBit(); if (size > Integer.MAX_VALUE) throw new IllegalStateException("size=" + size + " is too large."); - final ArrayList result = new ArrayList((int) size); + final ArrayList result = new ArrayList((int) size); for (int i = 0; i < size; i++) { - result.add(readValue(in)); + result.add(keyValueSerializer.readValue(in)); } return result; } + @NotNull @Override public Set> entrySet() { @@ -313,15 +305,15 @@ public class StatelessMapClient implements ChronicleMap { // get the data back from the server - Bytes bytes = blockingFetch(); + Bytes in = blockingFetch(); - long size = bytes.readStopBit(); + long size = in.readStopBit(); Set> result = new HashSet>(); for (int i = 0; i < size; i++) { - K k = keySerializer.readMarshallable(bytes); - V v = valueSerializer.readMarshallable(bytes); + K k = keyValueSerializer.readKey(in); + V v = keyValueSerializer.readValue(in); result.add(new Entry(k, v)); } @@ -329,15 +321,6 @@ public class StatelessMapClient implements ChronicleMap { } - /** - * write the keysize and the key to the the {@code target} buffer - * - * @param key the key of the map - */ - private void writeKey(K key) { - keySerializer.writeMarshallable(key, out); - } - private void writeEvent(EventId event) { out.write((byte) event.ordinal()); } @@ -449,7 +432,6 @@ public class StatelessMapClient implements ChronicleMap { return getKey() + "=" + getValue(); } - } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java index f40669bbf..0069e56c7 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -11,27 +11,16 @@ import java.util.Set; /** * @author Rob Austin. */ -public class StatelessServerConnector { +class StatelessServerConnector { + private final KeyValueSerializer keyValueSerializer; private final ChronicleMap map; - private final Serializer valueSerializer; - private final Serializer keySerializer; - - public StatelessServerConnector(Class kClass, Class vClass, ChronicleMap map) { - + public StatelessServerConnector(KeyValueSerializer keyValueSerializer, ChronicleMap map) { + this.keyValueSerializer = keyValueSerializer; this.map = map; - - final SerializationBuilder keyBuilder = new SerializationBuilder(kClass, - SerializationBuilder.Role.KEY); - final SerializationBuilder valueBuilder = new SerializationBuilder(vClass, - SerializationBuilder.Role.VALUE); - - keySerializer = new Serializer(keyBuilder); - valueSerializer = new Serializer(valueBuilder); - } @@ -112,46 +101,39 @@ public class StatelessServerConnector { map.replace(key, oldValue, newValue); } - public void longSize(Bytes in, Bytes out) { reflectTransactionId(in, out); out.writeLong(map.longSize()); } - public void size(Bytes in, Bytes out) { reflectTransactionId(in, out); out.writeInt(map.size()); } - public void isEmpty(Bytes in, Bytes out) { reflectTransactionId(in, out); out.writeBoolean(map.isEmpty()); } - public void containsKey(Bytes in, Bytes out) { K k = readKey(in); reflectTransactionId(in, out); out.writeBoolean(map.containsKey(k)); } - public void containsValue(Bytes in, Bytes out) { V v = readValue(in); reflectTransactionId(in, out); out.writeBoolean(map.containsValue(v)); } - public void get(Bytes in, Bytes out) { K k = readKey(in); reflectTransactionId(in, out); writeValue(map.get(k), out); } - public void put(Bytes in, Bytes out) { K k = readKey(in); V v = readValue(in); @@ -159,14 +141,12 @@ public class StatelessServerConnector { writeValue(map.put(k, v), out); } - public void remove(Bytes in, Bytes out) { final V value = map.remove(readKey(in)); reflectTransactionId(in, out); writeValue(value, out); } - public void putAll(Bytes in, Bytes out) { map.putAll(readEntries(in)); reflectTransactionId(in, out); @@ -236,14 +216,13 @@ public class StatelessServerConnector { map.replace(k, v); } - /** * write the keysize and the key to the the {@code target} buffer * * @param key the key of the map */ private void writeKey(K key, Bytes out) { - keySerializer.writeMarshallable(key, out); + keyValueSerializer.writeKey(key, out); } private void reflectTransactionId(Bytes in, Bytes out) { @@ -251,15 +230,59 @@ public class StatelessServerConnector { } private void writeValue(V value, final Bytes out) { - valueSerializer.writeMarshallable(value, out); + keyValueSerializer.writeValue(value, out); } private K readKey(Bytes in) { - return keySerializer.readMarshallable(in); + return keyValueSerializer.readKey(in); } private V readValue(Bytes in) { + return keyValueSerializer.readValue(in); + } + +} + +class KeyValueSerializer { + + private final Serializer valueSerializer; + private final Serializer keySerializer; + + KeyValueSerializer(SerializationBuilder keySerializer, + SerializationBuilder valueSerializer) { + this.keySerializer = new Serializer(keySerializer); + this.valueSerializer = new Serializer(valueSerializer); + } + + V readValue(Bytes in) { + if (in.readBoolean()) + return null; return valueSerializer.readMarshallable(in); + } + + K readKey(Bytes in) { + if (in.readBoolean()) + return null; + + return keySerializer.readMarshallable(in); + } + + /** + * write the keysize and the key to the the {@code target} buffer + * + * @param key the key of the map + */ + void writeKey(K key, Bytes out) { + + out.writeBoolean(key == null); + if (key != null) + keySerializer.writeMarshallable(key, out); + } + + void writeValue(V value, Bytes out) { + out.writeBoolean(value == null); + if (value != null) + valueSerializer.writeMarshallable(value, out); } } diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index c5d52aa3e..f6f1a00df 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -79,17 +79,21 @@ public class SerializerTest { ChronicleMapBuilder builder = ChronicleMapBuilder.of(clazz, Integer.class); builder.preMapConstruction(); - - Serializer v = new Serializer(builder.keyBuilder); + { + Serializer v = new Serializer(builder.keyBuilder); - v.writeMarshallable(key, out); + v.writeMarshallable(key, out); + + long position = out.position(); + in.limit(position); + + Object actual = v.readMarshallable(in); + Assert.assertEquals(actual, key); + + } - long position = out.position(); - in.limit(position); - Object actual = v.readMarshallable(in); - Assert.assertEquals(actual, key); } public static class MyTestClassExternalizable implements Externalizable { From e6868ea233666f5ceb5488c6b5dc9f40c3f38a00 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Thu, 16 Oct 2014 20:45:05 +0100 Subject: [PATCH 64/64] HCOLL-167 first working draft of StatelessServerConnector --- .../map/AbstractChannelReplicator.java | 22 +- .../chronicle/map/ChannelProvider.java | 2 +- .../chronicle/map/ChannelProviderBuilder.java | 2 +- .../chronicle/map/ChronicleMapBuilder.java | 2 +- .../net/openhft/chronicle/map/Replicator.java | 3 +- .../openhft/chronicle/map/Replicators.java | 16 +- .../chronicle/map/StatelessMapClient.java | 308 +++++++++++------- .../map/StatelessServerConnector.java | 83 +++-- .../openhft/chronicle/map/TcpReplicator.java | 157 ++++++--- .../net/openhft/chronicle/map/Builder.java | 5 +- .../chronicle/map/StatelessClientTest.java | 49 +++ 11 files changed, 451 insertions(+), 198 deletions(-) create mode 100644 src/test/java/net/openhft/chronicle/map/StatelessClientTest.java diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java index 89e741a0b..1f9cfba3e 100644 --- a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java @@ -197,8 +197,9 @@ abstract class AbstractChannelReplicator implements Closeable { /** - * checks the number of bytes written in this interval, if this number of bytes exceeds a threshold, - * the selected will de-register the socket that is being written to, until the interval is finished. + * checks the number of bytes written in this interval, if this number of bytes exceeds a + * threshold, the selected will de-register the socket that is being written to, until the + * interval is finished. * * @throws ClosedChannelException */ @@ -261,12 +262,22 @@ abstract class AbstractChannelReplicator implements Closeable { @Override public boolean onEntry(final Bytes entry, final int chronicleId) { + + long pos0 = in.position(); + + // used to denote that this is not a stateless map event + in.writeByte(StatelessMapClient.EventId.STATEFUL_UPDATE.ordinal()); + + long sizeLocation = in.position(); + + // this is where we will store the size of the entry in.skip(SIZE_OF_SHORT); + final long start = in.position(); externalizable.writeExternalEntry(entry, in, chronicleId); if (in.position() == start) { - in.position(in.position() - SIZE_OF_SHORT); + in.position(pos0); return false; } @@ -276,7 +287,10 @@ abstract class AbstractChannelReplicator implements Closeable { if (entrySize > MAX_UNSIGNED_SHORT) throw new IllegalStateException("entry too large, the entry size=" + entrySize + ", " + "entries are limited to a size of " + MAX_UNSIGNED_SHORT); - in.writeUnsignedShort(start - SIZE_OF_SHORT, entrySize); + + if (LOG.isDebugEnabled()) + LOG.debug("sending entry of entrySize=" + entrySize); + in.writeUnsignedShort(sizeLocation, entrySize); return true; } diff --git a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java index c51a56e8c..a59519c97 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java @@ -407,7 +407,7 @@ public final class ChannelProvider implements Closeable { @Override protected Closeable applyTo(ChronicleMapBuilder builder, - Replica map, EntryExternalizable entryExternalizable) { + Replica map, EntryExternalizable entryExternalizable, final ChronicleMap chronicleMap) { if (builder.entrySize() > maxEntrySize) { throw new IllegalArgumentException("During ReplicatingChannelBuilder setup, " + "maxEntrySize=" + maxEntrySize + " was specified, but map with " + diff --git a/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java b/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java index 0bb05dab4..38726cbff 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProviderBuilder.java @@ -68,7 +68,7 @@ public final class ChannelProviderBuilder { replicatingCluster.asReplica, replicatingCluster.asEntryExternalizable, tcpReplicationConfig, - maxEntrySize); + maxEntrySize, null); replicatingCluster.add(tcpReplicator); } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 504d08b58..5dc965fa1 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -805,7 +805,7 @@ public class ChronicleMapBuilder implements Cloneable, if (map instanceof ReplicatedChronicleMap) { ReplicatedChronicleMap result = (ReplicatedChronicleMap) map; for (Replicator replicator : replicators.values()) { - Closeable token = replicator.applyTo(this, result, result); + Closeable token = replicator.applyTo(this, result, result, map); if (replicators.size() == 1 && token.getClass() == UdpReplicator.class) { LOG.warn( "MISSING TCP REPLICATION : The UdpReplicator only attempts to read data " + diff --git a/src/main/java/net/openhft/chronicle/map/Replicator.java b/src/main/java/net/openhft/chronicle/map/Replicator.java index 86ef2ed82..3dede414d 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicator.java +++ b/src/main/java/net/openhft/chronicle/map/Replicator.java @@ -41,6 +41,7 @@ public abstract class Replicator { * map configurations, not accessible via the map instance. * @param map a replicated map instance. Provides basic tools for replication implementation. * @param entryExternalizable the callback for ser/deser implementation in the replicator + * @param chronicleMap * @return a {@code Closeable} token to control replication resources. * It should be closed on closing the replicated map. * @throws java.io.IOException if an io error occurred during the replicator setup @@ -49,6 +50,6 @@ public abstract class Replicator { * applied to a map (or the specified number of maps) */ protected abstract Closeable applyTo(ChronicleMapBuilder builder, - Replica map, Replica.EntryExternalizable entryExternalizable) + Replica map, Replica.EntryExternalizable entryExternalizable, final ChronicleMap chronicleMap) throws IOException; } diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java index 023e8a7b2..aeb5118d3 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicators.java +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -30,10 +30,18 @@ final class Replicators { @Override protected Closeable applyTo(ChronicleMapBuilder builder, - Replica map, Replica.EntryExternalizable entryExternalizable) + Replica replica, Replica.EntryExternalizable entryExternalizable, + final ChronicleMap chronicleMap) throws IOException { - return new TcpReplicator(map, entryExternalizable, replicationConfig, - builder.entrySize()); + + final KeyValueSerializer keyValueSerializer = new KeyValueSerializer(builder + .keyBuilder, builder.valueBuilder); + + StatelessServerConnector statelessServerConnector = new StatelessServerConnector + (keyValueSerializer, chronicleMap); + + return new TcpReplicator(replica, entryExternalizable, replicationConfig, + builder.entrySize(), statelessServerConnector); } }; } @@ -44,7 +52,7 @@ final class Replicators { @Override protected Closeable applyTo(ChronicleMapBuilder builder, - Replica map, Replica.EntryExternalizable entryExternalizable) + Replica map, Replica.EntryExternalizable entryExternalizable, final ChronicleMap chronicleMap) throws IOException { return new UdpReplicator(map, entryExternalizable, replicationConfig, builder.entrySize()); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java index 6e2fd77b4..397a69ee3 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapClient.java @@ -3,9 +3,14 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -18,16 +23,23 @@ import static net.openhft.chronicle.map.StatelessMapClient.EventId.*; * * @author Rob Austin. */ -class StatelessMapClient implements ChronicleMap { +class StatelessMapClient { + + private static final Logger LOG = LoggerFactory.getLogger(StatelessMapClient.class); + + + private final SocketChannel clientChannel; + public static final ByteBuffer STATELESS_CLIENT_IDENTIFER = ByteBuffer.wrap(new byte[]{-127}); public static enum EventId { + HEARTBEAT, + STATEFUL_UPDATE, LONG_SIZE, SIZE, IS_EMPTY, CONTAINS_KEY, CONTAINS_VALUE, GET, PUT, REMOVE, - PUT_ALL, CLEAR, KEY_SET, VALUES, @@ -46,23 +58,48 @@ class StatelessMapClient implements ChronicleMap { private long timeoutMs = TimeUnit.SECONDS.toMillis(20); - public StatelessMapClient(Bytes out, KeyValueSerializer keyValueSerializer) { - this.out = out; + public StatelessMapClient(final KeyValueSerializer keyValueSerializer, final InetSocketAddress remote) throws IOException { + this.keyValueSerializer = keyValueSerializer; + + clientChannel = SocketChannel.open(); + clientChannel.connect(remote); + + doHandShaking(); } - @Override + private void doHandShaking() throws IOException { + + + while (STATELESS_CLIENT_IDENTIFER.remaining() > 0) { + clientChannel.write(STATELESS_CLIENT_IDENTIFER); + } + + in.buffer().clear(); + + while (in.buffer().position() <= 0) { + clientChannel.read(in.buffer()); + } + + in.limit(in.buffer().position()); + in.position(0); + byte remoteIdentifier = in.readByte(); + + LOG.info("Attached to a map with a remote identifier=" + remoteIdentifier); + + } + + public File file() { return null; } - @Override public void close() throws IOException { throw new UnsupportedOperationException("This is not supported in the " + this.getClass() .getSimpleName()); } - long nextUniqueTransaction() { long time = System.currentTimeMillis(); @@ -76,160 +113,142 @@ class StatelessMapClient implements ChronicleMap { } - private final Bytes out; + private final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocate(1024)); + private final ByteBufferBytes in = new ByteBufferBytes(ByteBuffer.allocate(1024)); private final KeyValueSerializer keyValueSerializer; - @Override - public V putIfAbsent(K key, V value) { - writeEvent(PUT_IF_ABSENT); + public synchronized V putIfAbsent(K key, V value) throws IOException { + + long sizeLocation = writeEvent(PUT_IF_ABSENT); writeKey(key); writeValue(value); - - return readKey(); + return readKey(sizeLocation); } - @Override - public boolean remove(Object key, Object value) { - writeEvent(REMOVE_WITH_VALUE); + public synchronized boolean remove(Object key, Object value) throws IOException { + long sizeLocation = writeEvent(REMOVE_WITH_VALUE); writeKey((K) key); writeValue((V) value); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public boolean replace(K key, V oldValue, V newValue) { - writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); + + public synchronized boolean replace(K key, V oldValue, V newValue) throws IOException { + long sizeLocation = writeEvent(REPLACE_WITH_OLD_AND_NEW_VALUE); writeKey(key); writeValue(oldValue); writeValue(newValue); // get the data back from the server - return blockingFetch().readBoolean(); - } - - private void writeKey(K key) { - keyValueSerializer.writeKey(key, out); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public V replace(K key, V value) { - writeEvent(REPLACE); + public synchronized V replace(K key, V value) throws IOException { + long sizeLocation = writeEvent(REPLACE); writeKey(key); writeValue(value); // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - @Override - public int size() { + public synchronized int size() throws IOException { - writeEvent(SIZE); + long sizeLocation = writeEvent(SIZE); // get the data back from the server - return blockingFetch().readInt(); + return blockingFetch(sizeLocation).readInt(); } - @Override - public boolean isEmpty() { - writeEvent(IS_EMPTY); + public synchronized boolean isEmpty() throws IOException { + long sizeLocation = writeEvent(IS_EMPTY); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public boolean containsKey(Object key) { - writeEvent(CONTAINS_KEY); + + public synchronized boolean containsKey(Object key) throws IOException { + long sizeLocation = writeEvent(CONTAINS_KEY); writeKey((K) key); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public boolean containsValue(Object value) { - writeEvent(CONTAINS_VALUE); + + public synchronized boolean containsValue(Object value) throws IOException { + long sizeLocation = writeEvent(CONTAINS_VALUE); writeValue((V) value); // get the data back from the server - return blockingFetch().readBoolean(); + return blockingFetch(sizeLocation).readBoolean(); } - @Override - public long longSize() { - writeEvent(LONG_SIZE); + public synchronized long longSize() throws IOException { + long sizeLocation = writeEvent(LONG_SIZE); // get the data back from the server - return blockingFetch().readLong(); + return blockingFetch(sizeLocation).readLong(); } - @Override - public V get(Object key) { - writeEvent(GET); + + public synchronized V get(Object key) throws IOException { + long sizeLocation = writeEvent(GET); writeKey((K) key); + // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - @Override - public V getUsing(K key, V value) { + public synchronized V getUsing(K key, V value) { throw new UnsupportedOperationException("getUsing is not supported for stateless clients"); } - @Override - public V acquireUsing(K key, V value) { + + public synchronized V acquireUsing(K key, V value) { throw new UnsupportedOperationException("getUsing is not supported for stateless clients"); } - @Override - public V put(K key, V value) { - writeEvent(PUT); + public synchronized V put(K key, V value) throws IOException { + + long sizeLocation = writeEvent(PUT); writeKey(key); writeValue(value); // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - private V readKey() { - return keyValueSerializer.readValue(blockingFetch()); - } - private void writeValue(V value) { - keyValueSerializer.writeValue(value, out); - } - - @Override - public V remove(Object key) { - writeEvent(REMOVE); + public synchronized V remove(Object key) throws IOException { + long sizeLocation = writeEvent(REMOVE); writeKey((K) key); + // get the data back from the server - return readKey(); + return readKey(sizeLocation); } - @Override - public void putAll(Map map) { - writeEvent(PUT_ALL); - writeEntries(map); + public synchronized void putAll(Map map) throws IOException { - // get the data back from the server - blockingFetch(); + for (Map.Entry e : map.entrySet()) { + put(e.getKey(), e.getValue()); + } } @@ -247,42 +266,32 @@ class StatelessMapClient implements ChronicleMap { } } - @Override - public void clear() { - writeEvent(CLEAR); + + public synchronized void clear() throws IOException { + long sizeLocation = writeEvent(CLEAR); // get the data back from the server - blockingFetch(); + blockingFetch(sizeLocation); } @NotNull - @Override - public Set keySet() { - writeEvent(KEY_SET); + + public synchronized Set keySet() throws IOException { + long sizeLocation = writeEvent(KEY_SET); // get the data back from the server - return readKeySet(blockingFetch()); - } - - private Set readKeySet(Bytes in) { - long size = in.readStopBit(); - final HashSet result = new HashSet<>(); - - for (long i = 0; i < size; i++) { - result.add(keyValueSerializer.readKey(in)); - } - return result; + return readKeySet(blockingFetch(sizeLocation)); } @NotNull - @Override - public Collection values() { - writeEvent(VALUES); + + public synchronized Collection values() throws IOException { + long sizeLocation = writeEvent(VALUES); // get the data back from the server - final Bytes in = blockingFetch(); + final Bytes in = blockingFetch(sizeLocation); final long size = in.readStopBit(); @@ -299,13 +308,12 @@ class StatelessMapClient implements ChronicleMap { @NotNull - @Override - public Set> entrySet() { - writeEvent(ENTRY_SET); + public synchronized Set> entrySet() throws IOException { + long sizeLocation = writeEvent(ENTRY_SET); // get the data back from the server - Bytes in = blockingFetch(); + Bytes in = blockingFetch(sizeLocation); long size = in.readStopBit(); @@ -321,8 +329,13 @@ class StatelessMapClient implements ChronicleMap { } - private void writeEvent(EventId event) { + private long writeEvent(EventId event) { + out.clear(); + out.buffer().clear(); + out.write((byte) event.ordinal()); + long sizeLocation = markSizeLocation(); + return sizeLocation; } public Buffer buffer() { @@ -337,12 +350,12 @@ class StatelessMapClient implements ChronicleMap { volatile ByteBufferBytes buffer = null; - @Override + public void set(ByteBufferBytes source) { buffer = source; } - @Override + public ByteBufferBytes get() { return buffer; } @@ -359,7 +372,7 @@ class StatelessMapClient implements ChronicleMap { * @param transactionId * @return */ - private Bytes blockingFetch(long transactionId) { + private Bytes blockingFetchWithTransaction(long transactionId) { final Buffer buffer = buffer(); buffer.set(null); @@ -404,7 +417,11 @@ class StatelessMapClient implements ChronicleMap { public final V setValue(V newValue) { V oldValue = value; - StatelessMapClient.this.put((K) getKey(), (V) newValue); + try { + StatelessMapClient.this.put((K) getKey(), (V) newValue); + } catch (IOException e) { + LOG.error("", e); + } return oldValue; } @@ -431,15 +448,80 @@ class StatelessMapClient implements ChronicleMap { public final String toString() { return getKey() + "=" + getValue(); } - } - - private Bytes blockingFetch() { + private Bytes blockingFetch(long sizeLocation) throws IOException { long transactionId = nextUniqueTransaction(); + + LOG.info("sending data with transactionId=" + transactionId); out.writeLong(transactionId); - return blockingFetch(transactionId); + writeSizeAt(sizeLocation); + + + out.buffer().limit((int) out.position()); + while (out.buffer().remaining() > 0) { + int write = clientChannel.write(out.buffer()); + System.out.println(write); + } + + in.clear(); + in.buffer().clear(); + + // read size + while (in.buffer().position() < 2) { + clientChannel.read(in.buffer()); + } + + int size = in.readUnsignedShort(); + + while (in.buffer().position() < size + 2) { + clientChannel.read(in.buffer()); + } + + long inTransactionId = in.readLong(); + + if (inTransactionId != transactionId) { + throw new IllegalStateException("the received transaction-id=" + inTransactionId + + ", does not match the expected transaction-id=" + transactionId); + } + + return in; } + + private void writeSizeAt(long locationOfSize) { + long size = out.position() - locationOfSize; + out.writeUnsignedShort(locationOfSize, (int) size - 2); + } + + private long markSizeLocation() { + long position = out.position(); + out.skip(2); + return position; + } + + private void writeKey(K key) { + keyValueSerializer.writeKey(key, out); + } + + private V readKey(final long sizeLocation) throws IOException { + return keyValueSerializer.readValue(blockingFetch(sizeLocation)); + } + + private void writeValue(V value) { + keyValueSerializer.writeValue(value, out); + } + + + private Set readKeySet(Bytes in) { + long size = in.readStopBit(); + final HashSet result = new HashSet<>(); + + for (long i = 0; i < size; i++) { + result.add(keyValueSerializer.readKey(in)); + } + return result; + } + } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java index 0069e56c7..ca3f949e4 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessServerConnector.java @@ -24,13 +24,11 @@ class StatelessServerConnector { } - public void marshall(@NotNull Bytes in, @NotNull Bytes out) { + public void processStatelessEvent(byte eventId, @NotNull Bytes in, @NotNull Bytes out) { - byte b = in.readByte(); - StatelessMapClient.EventId eventId = StatelessMapClient.EventId.values()[b]; - - switch (eventId) { + StatelessMapClient.EventId event = StatelessMapClient.EventId.values()[eventId]; + switch (event) { case LONG_SIZE: longSize(in, out); @@ -55,9 +53,6 @@ class StatelessServerConnector { case REMOVE: remove(in, out); break; - case PUT_ALL: - putAll(in, out); - break; case CLEAR: clear(in, out); break; @@ -83,73 +78,93 @@ class StatelessServerConnector { removeWithValue(in, out); break; + case SIZE: + size(in, out); + break; + + default: + throw new IllegalStateException("unsupported event=" + event); + } } private void removeWithValue(Bytes in, Bytes out) { boolean result = map.remove(readKey(in), readValue(in)); - reflectTransactionId(in, out); out.writeBoolean(result); } private void replaceWithOldAndNew(Bytes in, Bytes out) { + final K key = readKey(in); V oldValue = readValue(in); V newValue = readValue(in); - reflectTransactionId(in, out); + + long sizeLocation = reflectTransactionId(in, out); map.replace(key, oldValue, newValue); + + writeSizeAt(sizeLocation, out); } + public void longSize(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeLong(map.longSize()); + writeSizeAt(sizeLocation, out); } public void size(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeInt(map.size()); + writeSizeAt(sizeLocation, out); } public void isEmpty(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.isEmpty()); + writeSizeAt(sizeLocation, out); } public void containsKey(Bytes in, Bytes out) { K k = readKey(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.containsKey(k)); + writeSizeAt(sizeLocation, out); } public void containsValue(Bytes in, Bytes out) { V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); out.writeBoolean(map.containsValue(v)); + writeSizeAt(sizeLocation, out); } public void get(Bytes in, Bytes out) { K k = readKey(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(map.get(k), out); + writeSizeAt(sizeLocation, out); } public void put(Bytes in, Bytes out) { K k = readKey(in); V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(map.put(k, v), out); + writeSizeAt(sizeLocation, out); } public void remove(Bytes in, Bytes out) { final V value = map.remove(readKey(in)); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(value, out); + writeSizeAt(sizeLocation, out); } public void putAll(Bytes in, Bytes out) { map.putAll(readEntries(in)); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); + writeSizeAt(sizeLocation, out); } private Map readEntries(Bytes in) { @@ -166,31 +181,34 @@ class StatelessServerConnector { public void clear(Bytes in, Bytes out) { map.clear(); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); + writeSizeAt(sizeLocation, out); } public void keySet(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); final Set ks = map.keySet(); out.writeStopBit(ks.size()); for (K key : ks) { writeKey(key, out); } + writeSizeAt(sizeLocation, out); } public void values(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); final Collection values = map.values(); out.writeStopBit(values.size()); for (final V value : values) { writeValue(value, out); } + writeSizeAt(sizeLocation, out); } public void entrySet(Bytes in, Bytes out) { - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); final Set> entries = map.entrySet(); out.writeStopBit(entries.size()); @@ -203,8 +221,9 @@ class StatelessServerConnector { public void putIfAbsent(Bytes in, Bytes out) { K key = readKey(in); V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); writeValue(map.putIfAbsent(key, v), out); + writeSizeAt(sizeLocation, out); } @@ -212,8 +231,9 @@ class StatelessServerConnector { K k = readKey(in); V v = readValue(in); - reflectTransactionId(in, out); + long sizeLocation = reflectTransactionId(in, out); map.replace(k, v); + writeSizeAt(sizeLocation, out); } /** @@ -225,8 +245,12 @@ class StatelessServerConnector { keyValueSerializer.writeKey(key, out); } - private void reflectTransactionId(Bytes in, Bytes out) { - out.writeLong(in.readLong()); + private long reflectTransactionId(Bytes in, Bytes out) { + long sizeLocation = out.position(); + out.skip(2); + long transactionId = in.readLong(); + out.writeLong(transactionId); + return sizeLocation; } private void writeValue(V value, final Bytes out) { @@ -241,6 +265,10 @@ class StatelessServerConnector { return keyValueSerializer.readValue(in); } + private void writeSizeAt(long locationOfSize, Bytes out) { + long size = out.position() - locationOfSize; + out.writeUnsignedShort((int) size); + } } class KeyValueSerializer { @@ -285,4 +313,5 @@ class KeyValueSerializer { valueSerializer.writeMarshallable(value, out); } + } diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 61a013ce3..be8fd7b70 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -17,6 +17,7 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,10 +33,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static java.nio.channels.SelectionKey.*; +import static net.openhft.chronicle.map.StatelessMapClient.EventId.HEARTBEAT; /** - * Used with a {@see net.openhft.map.ReplicatedSharedHashMap} to send data between the maps using a socket - * connection

    {@see net.openhft.map.OutSocketReplicator} + * Used with a {@see net.openhft.map.ReplicatedSharedHashMap} to send data between the maps using a + * socket connection

    {@see net.openhft.map.OutSocketReplicator} * * @author Rob Austin. */ @@ -43,6 +45,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName()); private static final int BUFFER_SIZE = 0x100000; // 1MB + public static final int STATELESS_CLIENT = -127; + public static final byte NOT_SET = (byte) HEARTBEAT.ordinal(); private final SelectionKey[] selectionKeysStore = new SelectionKey[Byte.MAX_VALUE + 1]; // used to instruct the selector thread to set OP_WRITE on a key correlated by the bit index in the @@ -57,22 +61,28 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private final Replica.EntryExternalizable externalizable; private final TcpReplicationConfig replicationConfig; + private final StatelessServerConnector statelessServerConnector; + private long selectorTimeout; /** - * @param maxEntrySizeBytes used to check that the last entry will fit into the buffer, it can not be - * smaller than the size of and entry, if it is set smaller the buffer will over - * flow, it can be larger then the entry, but setting it too large reduces the - * workable space in the buffer. + * @param maxEntrySizeBytes used to check that the last entry will fit into the buffer, + * it can not be smaller than the size of and entry, if it is + * set smaller the buffer will over flow, it can be larger then + * the entry, but setting it too large reduces the workable + * space in the buffer. + * @param statelessServerConnector * @throws IOException */ TcpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable externalizable, @NotNull final TcpReplicationConfig replicationConfig, - final int maxEntrySizeBytes) throws IOException { + final int maxEntrySizeBytes, + StatelessServerConnector statelessServerConnector) throws IOException { super("TcpSocketReplicator-" + replica.identifier(), replicationConfig.throttlingConfig(), maxEntrySizeBytes); + this.statelessServerConnector = statelessServerConnector; final ThrottlingConfig throttlingConfig = replicationConfig.throttlingConfig(); long throttleBucketInterval = TimeUnit.MILLISECONDS.convert(throttlingConfig @@ -251,7 +261,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * check to see if we have lost connection with the remote node and if we have attempts a reconnect. + * check to see if we have lost connection with the remote node and if we have attempts a + * reconnect. * * @param key the key relating to the heartbeat that we are checking * @param approxTimeOutTime the approximate time in milliseconds @@ -374,8 +385,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear all the - * old bootstrap information + * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear + * all the old bootstrap information * * @param key the nio SelectionKey */ @@ -387,7 +398,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * used to exchange identifiers and timestamps and heartbeat intervals between the server and client + * used to exchange identifiers and timestamps and heartbeat intervals between the server and + * client * * @param key the SelectionKey relating to the this cha * @param socketChannel @@ -404,6 +416,11 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { final byte remoteIdentifier = reader.identifierFromBuffer(); + if (remoteIdentifier == STATELESS_CLIENT) { + attached.handShakingComplete = true; + return; + } + if (remoteIdentifier == Byte.MIN_VALUE) return; @@ -425,7 +442,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { final SocketAddress remoteAddress = socketChannel.getRemoteAddress(); - if ( (identifierListener != null && !identifierListener.isIdentifierUnique(remoteIdentifier, + if ((identifierListener != null && !identifierListener.isIdentifierUnique(remoteIdentifier, remoteAddress)) || remoteIdentifier == localIdentifier) // throwing this exception will cause us to disconnect, both the client and server @@ -477,7 +494,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { // now we're finished we can get on with reading the entries attached.handShakingComplete = true; attached.remoteModificationIterator.dirtyEntries(attached.remoteBootstrapTimestamp); - reader.entriesFromBuffer(); + reader.entriesFromBuffer(attached); } } @@ -531,17 +548,17 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { attached.entryReader.lastHeartBeatReceived = approxTime; if (attached.isHandShakingComplete()) { - attached.entryReader.entriesFromBuffer(); + attached.entryReader.entriesFromBuffer(attached); } else { doHandShaking(key, socketChannel); } } /** - * sets interestOps to "selector keys",The change to interestOps much be on the same thread as the - * selector. This class, allows via {@link AbstractChannelReplicator .KeyInterestUpdater#set(int)} to - * holds a pending change in interestOps ( via a bitset ), this change is processed later on the same - * thread as the selector + * sets interestOps to "selector keys",The change to interestOps much be on the same thread as + * the selector. This class, allows via {@link AbstractChannelReplicator + * .KeyInterestUpdater#set(int)} to holds a pending change in interestOps ( via a bitset ), + * this change is processed later on the same thread as the selector */ private static class KeyInterestUpdater { @@ -572,8 +589,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * @param keyIndex the index of the key that has changed, the list of keys is provided by the - * constructor {@link KeyInterestUpdater(int, SelectionKey[])} + * @param keyIndex the index of the key that has changed, the list of keys is provided by + * the constructor {@link KeyInterestUpdater(int, SelectionKey[])} */ public void set(int keyIndex) { changeOfOpWriteRequired.set(keyIndex); @@ -773,12 +790,19 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } + /** + * @return the buffer messages can be written to + */ + Bytes buffer() { + return in; + } + /** * writes the timestamp into the buffer * * @param localIdentifier the current nodes identifier */ - void identifierToBuffer(final int localIdentifier) { + void identifierToBuffer(final byte localIdentifier) { in.writeByte(localIdentifier); } @@ -792,7 +816,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * writes all the entries that have changed, to the buffer which will later be written to TCP/IP + * writes all the entries that have changed, to the buffer which will later be written to + * TCP/IP * * @param modificationIterator a record of which entries have modification * @param selectionKey @@ -804,7 +829,7 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { final SocketChannel socketChannel = (SocketChannel) selectionKey.channel(); final Attached attached = (Attached) selectionKey.attachment(); - // this can occur when new CHM's are added to a cluster + // this can occur when new map are added to a channel final boolean handShakingComplete = attached.isHandShakingComplete(); for (; ; ) { @@ -867,9 +892,15 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { } /** - * used to send an single zero byte if we have not send any data for up to the localHeartbeatInterval + * used to send an single zero byte if we have not send any data for up to the + * localHeartbeatInterval */ private void writeHeartbeatToBuffer() { + + // denotes the state - 0 for a heartbeat + in.writeByte(HEARTBEAT.ordinal()); + + // denotes the size in bytes in.writeUnsignedShort(0); } @@ -879,8 +910,8 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { /** - * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will get - * added back in as soon as we have data to write + * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE + * will get added back in as soon as we have data to write * * @param socketChannel the socketChannel we wish to stop writing to * @param attached data associated with the socketChannels key @@ -914,8 +945,9 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { private final ByteBuffer in; private final ByteBufferBytes out; public long lastHeartBeatReceived = System.currentTimeMillis(); - // we use Integer.MIN_VALUE as N/A - private int sizeOfNextEntry = Integer.MIN_VALUE; + + private int sizeInBytes; + private byte state; private TcpSocketChannelEntryReader() { in = ByteBuffer.allocateDirect(replicationConfig.packetSize() + maxEntrySizeBytes); @@ -943,45 +975,79 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { /** * reads entries from the buffer till empty * + * @param attached * @throws InterruptedException */ - private void entriesFromBuffer() throws InterruptedException, IOException { - + private void entriesFromBuffer(Attached attached) throws InterruptedException, IOException { for (; ; ) { out.limit(in.position()); // its set to MIN_VALUE when it should be read again - if (sizeOfNextEntry == Integer.MIN_VALUE) { - if (out.remaining() < SIZE_OF_SHORT) { + if (state == NOT_SET) { + if (out.remaining() < SIZE_OF_SHORT + 1 ) { return; } - int value = out.readUnsignedShort(); + // state is used for both heartbeat and stateless + state = out.readByte(); - // this is the heartbeat - if (value == 0) + + + sizeInBytes = out.readUnsignedShort(); + + if (state == 0 && sizeInBytes != 0) { + int i = 1; + } + + // this is the : + // -- heartbeat if its 0 + // -- stateful update if its 1 + // -- the id of the stateful event + if (state == NOT_SET) continue; - - sizeOfNextEntry = value; } - if (out.remaining() < sizeOfNextEntry) { + + if (out.remaining() < sizeInBytes) { return; } - final long nextEntryPos = out.position() + sizeOfNextEntry; + if (sizeInBytes == 0) { + int j = 1; + } + + + final long nextEntryPos = out.position() + sizeInBytes; final long limit = out.limit(); out.limit(nextEntryPos); - externalizable.readExternalEntry(out); + + boolean isStateless = (state != 1); + + if (isStateless) { + if (statelessServerConnector == null) { + + LOG.error("", new IllegalArgumentException("received an event " + + "from a stateless map, stateless maps are not " + + "currently supported when using Chronicle Channels")); + } else + + statelessServerConnector.processStatelessEvent(state, + out, attached.entryWriter.buffer()); + // to allow the sizeInBytes to be read the next time around + + + } else + externalizable.readExternalEntry(out); out.limit(limit); // skip onto the next entry out.position(nextEntryPos); - // to allow the sizeOfNextEntry to be read the next time around - sizeOfNextEntry = Integer.MIN_VALUE; + + state = NOT_SET; + sizeInBytes = 0; } } @@ -1016,7 +1082,12 @@ class TcpReplicator extends AbstractChannelReplicator implements Closeable { * @return the timestamp or -1 if unsuccessful */ long remoteBootstrapTimestamp() { - return (out.remaining() >= 8) ? out.readLong() : Long.MIN_VALUE; + if (out.remaining() >= 8) + return out.readLong(); + else + return Long.MIN_VALUE; + + // return (out.remaining() >= 8) ? out.readLong() : Long.MIN_VALUE; } public long remoteHeartbeatIntervalFromBuffer() { diff --git a/src/test/java/net/openhft/chronicle/map/Builder.java b/src/test/java/net/openhft/chronicle/map/Builder.java index 4c1abddc9..ae60b1915 100644 --- a/src/test/java/net/openhft/chronicle/map/Builder.java +++ b/src/test/java/net/openhft/chronicle/map/Builder.java @@ -21,8 +21,7 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.net.InetSocketAddress; - -import static java.util.concurrent.TimeUnit.SECONDS; +import java.util.concurrent.TimeUnit; /** * @author Rob Austin. @@ -69,7 +68,7 @@ public class Builder { final int serverPort, final InetSocketAddress... endpoints) throws IOException { TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints) - .heartBeatInterval(1L, SECONDS); + .heartBeatInterval(1L, TimeUnit.SECONDS); return ChronicleMapBuilder.of(kClass, vClass) .entries(20000L) .replicators(identifier, tcpConfig); diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java new file mode 100644 index 000000000..da229954a --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -0,0 +1,49 @@ +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import org.junit.Test; +import org.testng.Assert; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static net.openhft.chronicle.map.TcpReplicationConfig.of; + +/** + * @author Rob Austin. + */ +public class StatelessClientTest { + + + @Test + public void test() throws IOException, InterruptedException { + + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000L) + .replicators((byte) 2, of(8076).heartBeatInterval(1L, SECONDS)); + { + final ChronicleMap map2a = + builder.create(getPersistenceFile()); + map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 + } + + KeyValueSerializer keyValueSerializer = new KeyValueSerializer(builder.keyBuilder, builder.valueBuilder); + + final ByteBufferBytes out = new ByteBufferBytes(ByteBuffer.allocateDirect(1024)); + // ByteBufferBytes in = out.slice(); + + InetSocketAddress remote = new InetSocketAddress("localhost", 8076); + StatelessMapClient map = new StatelessMapClient(keyValueSerializer, remote); + + + Assert.assertEquals("EXAMPLE-10", map.get(10)); + Assert.assertEquals("EXAMPLE-10", map.get(10)); + + Assert.assertEquals(1, map.size()); + } +} +