diff --git a/README.md b/README.md index e92fade03..0dd09991e 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Maven Download](https://github.com/OpenHFT/Chronicle-Map#maven-artifact-download-1) * [Snapshot Download](https://github.com/OpenHFT/Chronicle-Map#maven-snapshot-download) * [Key Value Object Types](https://github.com/OpenHFT/Chronicle-Map#key-value-object-types) + * [Off Heap and How to improve performance]() * [Sharing Data Between Two or More Maps](https://github.com/OpenHFT/Chronicle-Map#sharing-data-between-two-or-more-maps) * [Entries](https://github.com/OpenHFT/Chronicle-Map#entries) * [Size of Space Reserved on Disk](https://github.com/OpenHFT/Chronicle-Map#size-of-space-reserved-on-disk) @@ -45,15 +46,16 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [TCP / UDP Background.](https://github.com/OpenHFT/Chronicle-Map#tcp--udp-background) * [How to setup UDP Replication](https://github.com/OpenHFT/Chronicle-Map#how-to-setup-udp-replication) * [TCP/IP Throttling](https://github.com/OpenHFT/Chronicle-Map#tcpip--throttling) - * [Replication How it works](https://github.com/OpenHFT/Chronicle-Map#replication-how-it-works) - * [Multiple Chronicle Maps on a the same server with Replication](https://github.com/OpenHFT/Chronicle-Map#multiple-chronicle-maps-on-a-the-same-server-with-replication) - * [Identifier for Replication](https://github.com/OpenHFT/Chronicle-Map#identifier-for-replication) - * [Bootstrapping](https://github.com/OpenHFT/Chronicle-Map#bootstrapping) - * [Identifier](https://github.com/OpenHFT/Chronicle-Map#identifier) + * [Replication How it works](https://github.com/OpenHFT/Chronicle-Map#replication-how-it-works) + * [Multiple Processes on the same server with Replication](https://github.com/OpenHFT/Chronicle-Map#multiple-processes-on-the-same-server-with-replication) + * [Identifier for Replication](https://github.com/OpenHFT/Chronicle-Map#identifier-for-replication) + * [Bootstrapping](https://github.com/OpenHFT/Chronicle-Map#bootstrapping) + * [Identifier](https://github.com/OpenHFT/Chronicle-Map#identifier) * [Port](https://github.com/OpenHFT/Chronicle-Map#port) * [Heart Beat Interval](https://github.com/OpenHFT/Chronicle-Map#heart-beat-interval) -* [Channels and the Channel Provider](https://github.com/OpenHFT/Chronicle-Map#channels-and-channelprovider) +* [Multi Chronicle Maps - Network Distributed](https://github.com/OpenHFT/Chronicle-Map#multiple-chronicle-maps---network-distributed) * [Stateless Client](https://github.com/OpenHFT/Chronicle-Map#stateless-client) +* [Questions/Answers](https://github.com/OpenHFT/Chronicle-Map#questions-and-answers) #### Miscellaneous @@ -61,7 +63,7 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Stackoverflow](http://stackoverflow.com/tags/chronicle/info) * [Development Tasks - JIRA] (https://higherfrequencytrading.atlassian.net/browse/HCOLL) * [Use Case Which include Chronicle Map] (http://openhft.net/products/chronicle-engine/) - + #### Examples * [Hello World - A map which stores data off heap](https://github.com/OpenHFT/Chronicle-Map/blob/master/README.md#example--simple-hello-world) @@ -87,7 +89,7 @@ java map, ChronicleMap is able to share your entries accross processes: #### When to use HashMap If you compare `HashMap`, `ConcurrentHashMap` and `ChronicleMap`, most of the maps in your system are likely to be HashMap. This is because `HashMap` is lightweight and synchronized HashMap works -well for lightly contended use cases. By contention I mean, how many threads on average are trying +well for lightly contended use cases. By contention we mean, how many threads on average are trying to use a Map. One reason you can't have many contended resources, is that you only have so many CPUs and they can only be accessing so many resources at once (ideally no more than one or two per thread at a time). @@ -196,6 +198,8 @@ or value objects that are created through, a directClass interface, for example Object graphs can also be included as long as the outer object supports Serializable, Externalizable or BytesMarshallable. + + #### Java Class Construction Creating an instance of Chronicle Map is a little more complexed than just calling a constructor. @@ -214,7 +218,7 @@ try { ChronicleMapBuilder builder = ChronicleMapBuilder.of(Integer.class, CharSequence.class); - ConcurrentMap map = builder.file(file).create(); + ConcurrentMap map = builder.createPersistedTo(file); } catch (IOException e) { e.printStackTrace(); @@ -227,13 +231,8 @@ the instances of Chronicle Map on the same server. The name and location of the up to you. For the best performance on many unix systems we recommend using [tmpfs](http://en.wikipedia.org/wiki/Tmpfs). -If instead, you do not wish to replicate between processes on the same server or if you are only -using TCP replication to replicate between servers, you do not have to provide the "file", -so you can call the `create()` method on you ChronicleMapBuilder without providing the file -parameter: -``` -ConcurrentMap map = builder.create(); -``` + + ### Sharing Data Between Two or More Maps Since this file is memory mapped, if you were to create another instance of the Chronicle Map, @@ -248,10 +247,10 @@ between processes by just using memory and in around 40 nanoseconds. ConcurrentMap map1, map2; // this could could be on one process -map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class).file(file).create(); +map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class).createPersistedTo(file); // this could be on the other process -map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class).file(file).create(); +map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class).createPersistedTo(file); ``` Note: In order to share data between map1 and map2, the file has to point to the same file location on your server. @@ -325,53 +324,216 @@ These methods let you provide the object which the data will be written to, even immutable. For example ``` java -StringBuilder myString = new StringBuilder(); -StringBuilder myResult = map.getUsing("key", myString); +String myString = ""; +String myResult = map.getUsing("key", myString); // at this point the myString and myResult will both point to the same object ``` -The `map.getUsing()` method is similar to `get()`, but because Chronicle Map stores its data off +The `map.getUsing()` method is similar to `map.get()`, but because Chronicle Map stores its data off heap, if you were to call get("key"), a new object would be created each time, map.getUsing() works by reusing the heap memory which was used by the original Object "myString". This technique provides you with better control over your object creation. -Exactly like `map.getUsing()`, `acquireUsing()` will give you back a reference to an value based on -a key, but unlike `getUsing()` if there is not an entry in the map for this key the entry will be -added and the value return will we the same value which you provided. +Exactly like `map.getUsing()`, `map.acquireUsing()` will give you back a reference to an value +based on +a key, but unlike `map.getUsing()` if there is not an entry in the map for this key the entry +will be added and the value return will we the same value which you provided. ( The section below + covers `map.getUsing()`, `map.acquireUsing()` in greater details ) +#### Off Heap and How to Improve Performance +Chronicle Map stores its data off heap. There are some distinct advantages in using off heap data storage -#### Use getUsing(..) or acquireUsing(..) to avoid Object creation + * When the off heap data is backed by a memory mapped file, the entire map and its contents are automatically persisted. So if you have to restart your system, you won’t lose the content of your map. + * Off heap data structures are not visited by the garbage collector, with on heap maps like HashMap the garbage collector has to scan your entire object graph ( in fact just the live objects ) to remove garbage, if you are able to keep your on heap footprint low, the garbage collector has a lot less work to do, this in turn improves your performance. + * Your Chronicle map and your data contained within it is serialised so it can be stored in off +heap memory. When this memory is shared between processes on the same server, +the chronicle map is able to distribute entries between processes with extremely low overhead, +as both processes are sharing the same off heap memory space. Since your objects are already stored off heap ( as a series of bytes ), replication you entries over the network add relatively low over head. -The point of these methods is to avoid creating any objects. +One of the downsides of an Off Heap Map is that whenever you wish to get a value ( on heap ) from an entry + ``` java +Value v= get(key) +``` +that entry has to be deserialised onto the java heap so that you can use its value just like any other java object. So if you were to call get(key) ten times, + ``` java +for(int i=1;i<=10;i++) { + Value v= get(key) +} +``` +this would create 10 separate copies of the value. As each time get() is called the map has to +first create a Object to store the result in and then deserialise the value stored in the off +heap entry. If you want to get the value back on heap so that you can use it like a normal java +value, There is nothing we can do about the deserialisation as this has to occur every time, +( since the value may have been changed by another thread ), but we don’t have to create the +object each time. This is why we create getUsing(key,using). By reducing the number of objects +you create, you reduce the amount of work that the garbage collector has to carry out, +this in turn may, improve your overall performance. So back to getUsing(key,using), If you wish to reuse and existing object ( in this case the ‘using’ value ), you can instead call + + ``` java +Value using = new Value(); + +for(int i=1;i<=10;i++) { + Value v= map.getUsing(key,using); // this won’t create a new value each time. +} +``` +We can get a further performance improvement if we don't deserialize the whole object, +but only deserialize only the bytes that we are actually interested in, + +Lets assume that we had the following interface : -Pattern 1 ``` java -// get or create/initialise add needed -// assumption: I need this key value to exist or be initialised. -map.acquireUsing(key, value); +public interface LongValue { + long getValue(); -// use value + void setValue(long value); +} +``` +It is possible to use chronicle as an off heap proxy that can go directly to the memory location off heap and just get back the long that you require. This is especially use full if the value object has a lot of fields in it like the BondVOInterface +``` java +public interface BondVOInterface { + ... -if (changed) - map.put(key, value); + long getIssueDate(); + void setIssueDate(long issueDate); /* time in millis */ + + + long getMaturityDate(); + void setMaturityDate(long maturityDate); /* time in millis */ + + + double getCoupon(); + void setCoupon(double coupon); + + + String getSymbol(); + void setSymbol(@MaxSize(20) String symbol); + ... +} +``` +If for example you want to just do the following + +``` java +BondVOInterface bond = map.get(key); +bond.getCoupon() ``` - -Pattern 2 +if its only the coupon that you are interested in then it would be better if we did not have to +deserialize the whole object that implements the BondVOInterface. This is why we created the +OffHeapUpdatableChronicleMapBuilder ``` java -// get or don't do anything else. -// assumption: I only need the value if it exists, otherwise I will do something else, or nothing. -if (map.getUsing(key, value) != null) { - // use value + ChronicleMap chm = OffHeapUpdatableChronicleMapBuilder + .of(String.class, BondVOInterface.class) + .keySize(10) + .create(); +``` - if (changed) - map.put(key, value); -} else { - // don't use value +notice that the +``` java +.of(String.class, BondVOInterface.class) +``` + +value class is an interface, now like before, we can use the getUsing(key,using) method, +but this time we have to create the ‘using’ instance slightly differently + +``` java +BondVOInterface using = DataValueClasses.newDirectReference(BondVOInterface.class); +``` + +now we can call + +``` java +BondVOInterface bond = map.getUsing(key,value); +``` + +this won’t create any on heap objects, and it won’t deserialize the ‘bond’ from off heap to on +heap, all it does is sets the bond as a proxy object created by DataValueClasses +.newDirectReference(BondVOInterface.class) to point to the off heap storage, +under the covers, moves a pointer to point at the start of the value in off heap memory, so +calling : + +``` java +bond.getCoupon() +``` + +its only the coupon which gets deserialized. + +###### Locking + +Just like any other concurrent map, chronicle map uses segment locking, if you wish to hold onto the segment log when calling getUsing(key,using) you can do this : + +``` java +try (ReadContext context = map.getUsingLocked(key,using)) { + BondVOInterface bond = context.value().getValue(); + + long issueDate = bond.getIssueDate(); + String symbol = bond.getSymbol(); + + // add your logic here ( the lock will ensure this bond can not be changed by another thread ) + +} // the lock is released here + +To ensure that you can read the 'issueDate' and 'symbol' can be read atomically, these values +must be read while the segment lock is in place. + + +``` +when you call map.getUsingLocked(key,using) we return a ReadContext, the ReadContext extends +AutoCloseable so will automatically unlock the segment when the try block is exited. + +If you wish not to use a try block you must manually release the segment lock by calling +``` java +context.close() +``` +when you wish to unlock the segment. + +#### what is acquireUsing() and acquireUsingLocked() + +just like getUsing(), acquireUsing() will also recycle the value you pass it, the following +code is a pattern that you will often come across, acquireUsing(key,value) offers this +functionality with a single method call + ``` java +V acquireUsing(key,value) { + Lock l = ...; // the segement lock of the map + l.lock(); + try { + V v = map.getUsing(key,value) + if (v == null) { + map.put(key,value) + return v; + } else + return v + } finally { + l.unlock(); + } } ``` - + +if you are only accessing ChronicleMap from a single thread and you are not doing replication +and don't are about atomic reads then its simpler ( and faster ) to use acquireUsing() otherwise we +recommend acquireUsingLocked(key,value) + +you use + +because acquireUsing can end up creating an entry the acquireUsingLocked(key,value) method holds +a segment write lock, this is unlike the getUsing(key,using) method that holds a segment read lock. + +``` java +try (WriteContext context = map.acquireUsingLocked("one", value)) { + + BondVOInterface bond = context.value(); + long issueDate = bond.getIssueDate(); + String symbol = bond.getSymbol(); + + if (condition) + context.removeEntry(); +} +``` + +if after you have read the 'issueDate' and 'symbol' you wish to remove the entry based on some +business logic, it more efficient to use the 'context' to remove the entry, as the contents is +already aware when the entry is in memory. + ## Oversized Entries Support @@ -489,14 +651,12 @@ to the other. This dilemma is resolved by using a node identifier, the node iden identifier is smaller than the local nodes identifier, this update will be accepted otherwise it will be ignored. -### Multiple Chronicle Maps on a the same server with Replication -If two or more Chronicle Maps are on the same server, they exchange data via shared memory rather -than TCP or UDP replication. So if a Chronicle Map which is not performing TCP Replication is -updated, this update can be picked up by another Chronicle Map, this other Chronicle Hash Map could -be a TCP replicated Chronicle Map, In this example the TCP replicated Chronicle Map would then push -the update to the remote nodes. +### Multiple Processes on the same server with Replication + +On a server if you have a number of java processes and then within each java process you create a instance of the same Chronicle map which binds to the same underline 'file', they exchange data via shared memory rather than TCP or UDP replication. So if a Chronicle Map which is not performing TCP Replication is updated, this update can be picked up by another Chronicle Map, this other Chronicle Hash Map could be a TCP replicated Chronicle Map, In this example the TCP replicated Chronicle Map would then push the update to the remote nodes. Likewise, If the TCP replicated Chronicle Map was to received an update from a remote node, then + this update would be immediately available to all the Chronicle Maps on the server. ### Identifier for Replication @@ -522,10 +682,10 @@ information on identifiers and how they are used in replication. ) The identifier is setup on the builder as follows. ```java -TcpReplicationConfig tcpConfig = ... +TcpConfig tcpConfig = ... map = ChronicleMapBuilder .of(Integer.class, CharSequence.class) - .replicators(identifier, tcpReplicationConfig) + .replication(identifier, tcpConfig) .create(); ``` @@ -548,17 +708,17 @@ Each map is allocated a unique identifier Server 1 has: ``` -.replicators((byte) 1, tcpReplicationConfigServer1) +.replication((byte) 1, tcpConfigServer1) ``` Server 2 has: ``` -.replicators((byte) 2, tcpReplicationConfigServer2) +.replication((byte) 2, tcpConfigServer2) ``` Server 3 has: ``` -.replicators((byte) 3, tcpReplicationConfigServer3) +.replication((byte) 3, tcpConfigServer3) ``` If you fail to allocate a unique identifier replication will not work correctly. @@ -574,12 +734,12 @@ require 6 connections. In our case we are only using 2 maps, this is how we connected map1 to map 2. ``` -TcpReplicationConfig.of(8076, new InetSocketAddress("localhost", 8077)) +TcpConfig.of(8076, new InetSocketAddress("localhost", 8077)) .heartBeatInterval(1, SECONDS); ``` you could have put this instruction on map2 instead, like this ``` -TcpReplicationConfig.of(8077, new InetSocketAddress("localhost", 8076)) +TcpConfig.of(8077, new InetSocketAddress("localhost", 8076)) .heartBeatInterval(1, SECONDS); ``` even though data flows from map1 to map2 and map2 to map1 it doesn't matter which way you connected @@ -590,7 +750,7 @@ this, in other words its a bidirectional connection. ![TCP/IP Replication 3Way](http://openhft.net/wp-content/uploads/2014/09/Screen-Shot-2014-10-27-at-18.19.05.png) -Below is example how to set up tcpReplicationConfig for 3 host +Below is example how to set up tcpConfig for 3 host ```java String hostServer1 = "localhost"; // change this to your host @@ -607,14 +767,14 @@ InetSocketAddress inetSocketAddress3 = new InetSocketAddress(hostServer3, server // this is to go on server 1 -TcpReplicationConfig tcpReplicationConfigServer1 = TcpReplicationConfig.of(serverPort1); +TcpConfig tcpConfigServer1 = TcpConfig.of(serverPort1); // this is to go on server 2 -TcpReplicationConfig tcpReplicationConfigServer2 = TcpReplicationConfig.of(serverPort2, +TcpConfig tcpConfigServer2 = TcpConfig.of(serverPort2, inetSocketAddress1); // this is to go on server 3 -TcpReplicationConfig tcpReplicationConfigServer3 = TcpReplicationConfig.of(serverPort3, +TcpConfig tcpConfigServer3 = TcpConfig.of(serverPort3, inetSocketAddress1,inetSocketAddress2); ``` @@ -627,7 +787,11 @@ A heartbeat will only be send if no data is transmitted, if the maps are constan no heartbeat message is sent. If a map does not receive either data of a heartbeat the connection is dropped and re-established. -# Channels and ChannelProvider +# Multiple Chronicle Maps - Network Distributed + +![Chronicle Maps Network Distributed](http://openhft.net/wp-content/uploads/2014/07/Chronicle-Map_channels_diagram_02.jpg) + + Chronicle Map TCP Replication lets you distribute a single Chronicle Map, to a number of servers across your network. Replication is point to point and the data transfer is bidirectional, so in the @@ -647,9 +811,10 @@ Each map must be given a unique Channel. ``` java int maxEntrySize = 1024; byte identifier= 2; -ChannelProvider channelProvider = new ChannelProviderBuilder() +ReplicationHub replicationHub = ReplicationHub.builder() .maxEntrySize(maxEntrySize) - .replicators(identifier, tcpReplicationConfig).create(); + .tcpTransportAndNetwork(tcpConfig) + .create(identifier); ``` In this example above the channel is given the identifier of 2 @@ -664,9 +829,10 @@ When creating the ChannelProvider you should attach your tcp or udp configuratio ``` java int maxEntrySize = 1024; byte identifier = 1; -ChannelProvider channelProvider = new ChannelProviderBuilder() +ReplicationHub replicationHub = ReplicationHub.builder() .maxEntrySize(maxEntrySize) - .replicators(identifier, tcpReplicationConfig).create(); + .tcpTransportAndNetwork(tcpConfig) + .create(identifier); ``` Attaching ChannelProvider replication to the map: @@ -674,8 +840,9 @@ Attaching ChannelProvider replication to the map: ``` java short channel = (short) 2; ChronicleMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .channel(channelProvider.createChannel(channel) + .entries(1000) + .instance() + .replicatedViaChannel(replicationHub.createChannel(channel)) .create(); ``` @@ -696,71 +863,71 @@ unique for each map you have. ### Channels and ChannelProvider - Example ``` java - // server 1 with identifier = 1 + // server 1 with identifier = 1 { + ChronicleMapBuilder smallStringToStringMapBuilder = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(1000); + byte identifier = (byte) 1; - TcpReplicationConfig tcpConfig = TcpReplicationConfig + TcpConfig tcpConfig = TcpConfig .of(8086, new InetSocketAddress("localhost", 8087)) - .heartBeatInterval(1, java.util.concurrent.TimeUnit.SECONDS); + .heartBeatInterval(1, SECONDS); - channelProviderServer1 = new ChannelProviderBuilder() - .replicators(identifier, tcpConfig) - .create(); + hubOnServer1 = ReplicationHub.builder() + .tcpTransportAndNetwork(tcpConfig) + .create(identifier); // this demotes favoriteColour short channel1 = (short) 1; - favoriteColourServer1 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer1.createChannel(channel1)).create(); + favoriteColourServer1 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer1.createChannel(channel1)).create(); + favoriteColourServer1.put("peter", "green"); // this demotes favoriteComputer short channel2 = (short) 2; - favoriteComputerServer1 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer1.createChannel(channel2)).create(); - - - favoriteColourServer1.put("peter", "green"); + favoriteComputerServer1 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer1.createChannel(channel2)).create(); favoriteComputerServer1.put("peter", "dell"); } - // server 2 with identifier = 2 + // server 2 with identifier = 2 { + ChronicleMapBuilder smallStringToStringMapBuilder = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(1000); byte identifier = (byte) 2; - TcpReplicationConfig tcpConfig = - TcpReplicationConfig.of(8087) - .heartBeatInterval(1, java.util.concurrent.TimeUnit.SECONDS); + TcpConfig tcpConfig = + TcpConfig.of(8087).heartBeatInterval(1, SECONDS); - channelProviderServer2 = new ChannelProviderBuilder() - .replicators(identifier, tcpConfig) - .create(); + hubOnServer2 = ReplicationHub.builder() + .tcpTransportAndNetwork(tcpConfig) + .create(identifier); // this demotes favoriteColour short channel1 = (short) 1; - favoriteColourServer2 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer2.createChannel(channel1)).create(); + favoriteColourServer2 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer2.createChannel(channel1)).create(); + + + favoriteColourServer2.put("rob", "blue"); // this demotes favoriteComputer short channel2 = (short) 2; - favoriteComputerServer2 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer2.createChannel(channel2)).create(); - - - favoriteColourServer2.put("rob", "blue"); + favoriteComputerServer2 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer2.createChannel(channel2)).create(); favoriteComputerServer2.put("rob", "mac"); favoriteComputerServer2.put("daniel", "mac"); @@ -782,11 +949,12 @@ unique for each map you have. assertEquals(favoriteColourServer1, favoriteColourServer2); Assert.assertEquals(2, favoriteColourServer1.size()); - - favoriteComputerServer1.close(); - favoriteComputerServer2.close(); + + favoriteColourServer1.close(); + favoriteComputerServer2.close(); favoriteColourServer2.close(); + favoriteColourServer1.close(); } ``` @@ -815,7 +983,7 @@ final ChronicleMap statelessMap; { ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8076)) + .replication((byte) 2, TcpConfig.of(8076)) .create(); serverMap.put(10, "EXAMPLE-10"); @@ -824,7 +992,7 @@ final ChronicleMap statelessMap; // stateless client { statelessMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .stateless(StatelessBuilder.remoteAddress(new InetSocketAddress("localhost", 8076))) + .statelessClient(new InetSocketAddress("localhost", 8076)) .create(); Assert.assertEquals("EXAMPLE-10", statelessMap.get(10)); @@ -842,25 +1010,27 @@ being used by another application. In this example we choose the port 8076 ``` java -.replicators((byte) 2, TcpReplicationConfig.of(8076)) // sets the server to run on localhost:8076 +.replication((byte) 2, TcpConfig.of(8076)) // sets the server to run on localhost:8076 ``` On the "stateless client" we connect to the server via TCP/IP on localhost:8076 : ``` java -.stateless(remoteAddress(new InetSocketAddress("localhost", 8076))) +.statelessClient(new InetSocketAddress("localhost", 8076)) ``` but in your example you should choose the host of the state-full server and the port you allocated it. ``` java -.stateless(StatelessBuilder.remoteAddress(new InetSocketAddress(, -))) +.statelessClient(new InetSocketAddress(, )) ``` -the ".stateless(..)" method tells `ChronicleMap` that its going to build a stateless client. If you -don’t add this line a normal state-full `ChronicleMap` will be created. For this example we ran both +the ".statelessClient(..)" returns an instance of `StatelessClientConfig`, which has only a few +own configurations and `create()` method to create a new stateless client. +method tells `ChronicleMap` that its going to build a stateless client. +If you don’t add this line a normal state-full `ChronicleMap` will be created. For this example +we ran both the client an the server on the same host ( hence the “localhost" setting ), but in a real life example the stateless client will typically be on a different server than the state-full host. If you are aiming to create a stateless client and server on the same host, its @@ -896,9 +1066,9 @@ and also for the `remove()` method ``` java statelessClientMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .stateless(StatelessBuilder.remoteAddress(new InetSocketAddress("localhost", 8076))) .putReturnsNull(true) .removeReturnsNull(true) + .statelessClient(new InetSocketAddress("localhost", 8076)) .create(); ``` @@ -907,7 +1077,7 @@ For the very best performance you should also set these properties on the server ``` java ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8076)) + .replication((byte) 2, TcpConfig.of(8076)) .putReturnsNull(true) .removeReturnsNull(true) .create(); @@ -1035,16 +1205,16 @@ between them, this is how we could set it up // --- RUN ON ONE JAVA PROCESS ( BUT ON THE SAME SERVER ) { File file = new File("a-new-file-on-your-sever"); - Map map1 = ChronicleMapBuilder.of(Integer.class, - CharSequence.class).create(file); // this has to be the same file as used by map 2 + Map map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .createPersistedTo(file); // this has to be the same file as used by map 2 map1.put(1, "hello world"); } // --- RUN ON THE OTHER JAVA PROCESS ( BUT ON THE SAME SERVER ) { File file = new File("a-new-file-on-your-sever"); // this has to be the same file as used by map 1 - Map map1 = ChronicleMapBuilder.of(Integer.class, - CharSequence.class).create(file); + Map map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .createPersistedTo(file); System.out.println(map1.get(1)); } @@ -1079,12 +1249,12 @@ public class YourClass { // we connect the maps via a TCP/IP socket connection on port 8077 - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8076, new InetSocketAddress("localhost", 8077)) + TcpConfig tcpConfig = TcpConfig.of(8076, new InetSocketAddress("localhost", 8077)) .heartBeatInterval(1L, TimeUnit.SECONDS); ChronicleMapBuilder map1Builder = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000L) - .replicators((byte) 1, tcpConfig); + .replication((byte) 1, tcpConfig); map1 = map1Builder.create(); @@ -1092,12 +1262,12 @@ public class YourClass { // ---------- SERVER2 on the same server as ---------- { - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8077) + TcpConfig tcpConfig = TcpConfig.of(8077) .heartBeatInterval(1L, TimeUnit.SECONDS); ChronicleMapBuilder map2Builder = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000L) - .replicators((byte) 2, tcpConfig); + .replication((byte) 2, tcpConfig); map2 = map2Builder.create(); // we will stores some data into one map here @@ -1159,20 +1329,23 @@ public class YourClass { // we connect the maps via a TCP socket connection on port 8077 - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8076, new InetSocketAddress("localhost", 8077)) + TcpConfig tcpConfig = TcpConfig.of(8076, new InetSocketAddress("localhost", 8077)) .heartBeatInterval(1L, TimeUnit.SECONDS) // a maximum of 1024 bits per millisecond .throttlingConfig(ThrottlingConfig.throttle(1024, TimeUnit.MILLISECONDS)); - UdpReplicationConfig udpConfig = UdpReplicationConfig + UdpConfig udpConfig = UdpConfig .simple(Inet4Address.getByName("255.255.255.255"), udpPort); ChronicleMapBuilder map1Builder = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000L) - .replicators((byte) 1, tcpConfig, udpConfig); + .replication(SimpleReplication.builder() + .tcpTransportAndNetwork(tcpConfig) + .udpTransport(udpConfig) + .create((byte) 1)); map1 = map1Builder.create(); @@ -1180,17 +1353,20 @@ public class YourClass { // ---------- SERVER2 2 on the same server as ---------- { - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(8077) + TcpConfig tcpConfig = TcpConfig.of(8077) .heartBeatInterval(1L, TimeUnit.SECONDS) .throttlingConfig(ThrottlingConfig.throttle(1024, TimeUnit.MILLISECONDS)); - UdpReplicationConfig udpConfig = UdpReplicationConfig + UdpConfig udpConfig = UdpConfig .simple(Inet4Address.getByName("255.255.255.255"), udpPort); ChronicleMapBuilder map2Builder = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000L) - .replicators((byte) 2, tcpConfig, udpConfig); + .replication(SimpleReplication.builder() + .tcpTransportAndNetwork(tcpConfig) + .udpTransport(udpConfig) + .create((byte) 2)); map2 = map2Builder.create(); @@ -1409,3 +1585,84 @@ to be performed in memory as possible. vm.dirty_expire_centisecs = 30000 vm.dirty_ratio = 90 vm.dirty_writeback_centisecs = 5000 + +# Questions and Answers + +#### Question +I'm searching for a Map implementation that is backed by either direct off-heap or a memory +mapped file. I want to use it as write-once, read-n-times kind of cache. +#### Answer +The latest version of chronicle map has been optimised for this use case. Ie for heavy read to write ratios. +Note: heavy writers will see about the same performance. + +--- + +#### Question +The backing of to off-heap is only needed to prevent OOM-situations. Otherwise the process can use a big part of the available memory for the map. + +#### Answer +Chronicle Map can be larger than main memory. In fact you might want to reduce the heap size to give chronicle map more memory in extreme cases. Ie because the map can't utilise on heap memory much. + +--- + +#### Question +I stumbled over chronicle-map and it looks like it could do most of those things. Correct me if i'm wrong but +``` java +// would create the off-heap version +ChronicleMapBuilder.of(A.class, B.class).create(); +``` +#### Answer +Yes. + +--- + +#### Question +``` java +// the memory-mapped file version ? + ChronicleMapBuilder.of(A.class, B.class).file(mapFile).create(); +``` +#### Answer +Yes. + +--- + +#### Question +But both versions write the complete map at every single moment to off-heap, right ? + +#### Answer +Every key/value is off heap. There is no cache of objects on heap. + +--- + +#### Question +Is there something in chronicle map which can have part of the map-content still in on-heap memory. + +#### Answer +There are data structures which are on heap to manahe the off heap data. This is typically less than 100 kB. + +--- + +#### Question +Either serialized or deserialized. Performance wise do you think that would even matter ? +#### Answer +It matter a lot but you can get the performance close to on heap objects without the long GC pause times. Ie only the bit on the heap makes any difference. + +--- + +#### Question +Last but not least... do you think chronicle is suitable for this simple purpose ? + +#### Answer +Actually this is all it does really. We try to make it fast by keeping it simple. +Eg it is not a cache and doesn't have expiries etc. There is no background thread managing it. +In fact it is so simple that many methods are completely in lined. +Ie if you look at a crash dump the chronicle methods have been completely in lined and don't appear in the stack trace any more. (The still appear in java stack traces) + +--- + +#### Question +Or is there a lot of overhead for other functionality which i do not intend to use (yet) !? + +#### Answer +You can share the data between processes on the same machine but this doesn't add overhead. +You can add replication between machines. But this uses extra classes which are not used in the simple case ie it is as if they were not there, only an option. diff --git a/src/main/java/net/openhft/chronicle/hash/ChannelProvider.java b/src/main/java/net/openhft/chronicle/hash/ChannelProvider.java deleted file mode 100644 index 54a621cf2..000000000 --- a/src/main/java/net/openhft/chronicle/hash/ChannelProvider.java +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; - - -import net.openhft.chronicle.map.*; -import net.openhft.lang.collection.DirectBitSet; -import net.openhft.lang.collection.SingleThreadedDirectBitSet; -import net.openhft.lang.io.ByteBufferBytes; -import net.openhft.lang.io.Bytes; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.atomic.AtomicReferenceArray; - -import static java.lang.Math.min; -import static java.nio.ByteBuffer.wrap; -import static net.openhft.chronicle.map.Replica.EntryExternalizable; -import static net.openhft.chronicle.map.Replica.ModificationIterator; -import static net.openhft.chronicle.map.Replica.ModificationNotifier.NOP; - -/** - * @author Rob Austin. - */ -public final class ChannelProvider implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(ChannelProvider.class.getName()); - - private static final byte BOOTSTRAP_MESSAGE = 'B'; - final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { - /** - * writes the entry to the chronicle channel provided - * - * @param entry the byte location of the entry to be stored - * @param destination a buffer the entry will be written to, the segment may reject - * this operation and add zeroBytes, if the identifier in the entry - * did not match the maps local - * @param chronicleChannel used in cluster into identify the canonical map or queue - */ - @Override - public void writeExternalEntry(@NotNull Bytes entry, @NotNull Bytes destination, - int chronicleChannel) { - destination.writeStopBit(chronicleChannel); - channelEntryExternalizables[chronicleChannel] - .writeExternalEntry(entry, destination, chronicleChannel); - } - - @Override - public void readExternalEntry(@NotNull Bytes source) { - final int chronicleId = (int) source.readStopBit(); - if (chronicleId < chronicleChannels.length) - channelEntryExternalizables[chronicleId].readExternalEntry(source); - else - LOG.info("skipped entry with chronicleId=" + chronicleId + ", "); - } - }; - private final byte localIdentifier; - final Replica asReplica = new Replica() { - - @Override - public byte identifier() { - return localIdentifier; - } - - @Override - public ModificationIterator acquireModificationIterator( - final short remoteIdentifier, final ModificationNotifier notifier) { - final ModificationIterator result = modificationIterator.get(remoteIdentifier); - if (result != null) - return result; - - final ModificationIterator result0 = new ModificationIterator() { - - @Override - public boolean hasNext() { - // old-style iteration to avoid 1) iterator object creation - // 2) ConcurrentModificationException - for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - final ModificationIterator modificationIterator = - chronicleChannelList.get(i) - .acquireModificationIterator(remoteIdentifier, notifier); - if (modificationIterator.hasNext()) - return true; - } - return false; - } - - @Override - public boolean nextEntry(@NotNull EntryCallback callback, - final int na) { - for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - Replica chronicleChannel = chronicleChannelList.get(i); - final ModificationIterator modificationIterator = chronicleChannel - .acquireModificationIterator(remoteIdentifier, notifier); - if (modificationIterator.nextEntry(callback, chronicleChannelIds.get(i))) - return true; - } - return false; - } - - @Override - public void dirtyEntries(long fromTimeStamp) { - for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - chronicleChannelList.get(i) - .acquireModificationIterator(remoteIdentifier, notifier) - .dirtyEntries(fromTimeStamp); - notifier.onChange(); - } - } - }; - - modificationIterator.set((int) remoteIdentifier, result0); - return result0; - - } - - /** - * gets the earliest modification time for all of the chronicles - */ - @Override - public long lastModificationTime(byte remoteIdentifier) { - long t = 0; - // not including the SystemQueue at index 0 - for (int i = 1, len = chronicleChannelList.size(); i < len; i++) { - t = (t == 0) ? chronicleChannels[i].lastModificationTime(remoteIdentifier) : - min(t, chronicleChannels[i].lastModificationTime(remoteIdentifier)); - } - return t; - } - - @Override - public void close() throws IOException { - ChannelProvider.this.close(); - } - }; - private final int maxEntrySize; - private final Replica[] chronicleChannels; - private final List chronicleChannelList; - private final List chronicleChannelIds; - private final EntryExternalizable[] channelEntryExternalizables; - private final AtomicReferenceArray systemModificationIterator = - new AtomicReferenceArray(128); - private final DirectBitSet systemModificationIteratorBitSet = - newBitSet(systemModificationIterator.length()); - private final AtomicReferenceArray modificationIterator = - new AtomicReferenceArray(128); - private final Set replicators = new CopyOnWriteArraySet(); - - ChannelProvider(ChannelProviderBuilder builder) { - localIdentifier = builder.identifier; - maxEntrySize = builder.maxEntrySize; - chronicleChannels = new Replica[builder.maxNumberOfChronicles]; - channelEntryExternalizables = new EntryExternalizable[builder.maxNumberOfChronicles]; - chronicleChannelList = new ArrayList(); - chronicleChannelIds = new ArrayList(); - MessageHandler systemMessageHandler = new MessageHandler() { - @Override - public void onMessage(Bytes bytes) { - final byte type = bytes.readByte(); - if (type == BOOTSTRAP_MESSAGE) { - onBootstrapMessage(bytes); - } else { - LOG.info("message of type=" + type + " was ignored."); - } - } - }; - SystemQueue systemMessageQueue = new SystemQueue( - systemModificationIteratorBitSet, systemModificationIterator, systemMessageHandler); - add((short) 0, systemMessageQueue.asReplica, systemMessageQueue.asEntryExternalizable); - } - - public int maxEntrySize() { - return maxEntrySize; - } - - /** - * creates a bit set based on a number of bits - * - * @param numberOfBits the number of bits the bit set should include - * @return a new DirectBitSet backed by a byteBuffer - */ - private static DirectBitSet newBitSet(int numberOfBits) { - final ByteBufferBytes bytes = new ByteBufferBytes(wrap(new byte[(numberOfBits + 7) / 8])); - return new SingleThreadedDirectBitSet(bytes); - } - - /** - * Returns a replicator, dedicated to the specified channel. Channel is basically just a number, that - * should correspond on different servers for instances of the same replicated map. Only one replicator - * per channel could be obtained from a single {@code ReplicatingChannel}. The returned replicator could - * be applied to a map at most once. - * - * @param channel number to create - * @return a replicator, dedicated to the specified channel - */ - public ChronicleChannel createChannel(short channel) { - return new ChronicleChannel(channel); - } - - /** - * called whenever we receive a bootstrap message - */ - private void onBootstrapMessage(Bytes bytes) { - final short remoteIdentifier = bytes.readByte(); - final int chronicleChannel = bytes.readUnsignedShort(); - final long lastModificationTime = bytes.readLong(); - - // this could be null if one node has a chronicle channel before the other - if (chronicleChannels[chronicleChannel] != null) { - chronicleChannels[chronicleChannel].acquireModificationIterator(remoteIdentifier, NOP) - .dirtyEntries(lastModificationTime); - } - } - - private ByteBufferBytes toBootstrapMessage(short chronicleChannel, final long lastModificationTime) { - final ByteBufferBytes writeBuffer = new ByteBufferBytes(ByteBuffer.allocate(1 + 1 + 2 + 8)); - writeBuffer.writeByte(BOOTSTRAP_MESSAGE); - writeBuffer.writeByte(localIdentifier); - writeBuffer.writeUnsignedShort(chronicleChannel); - writeBuffer.writeLong(lastModificationTime); - writeBuffer.flip(); - return writeBuffer; - } - - private void add(short chronicleChannel, - Replica replica, EntryExternalizable entryExternalizable) { - if (chronicleChannels[chronicleChannel] != null) { - throw new IllegalStateException("chronicleId=" + chronicleChannel + - " is already in use."); - } - chronicleChannels[chronicleChannel] = replica; - chronicleChannelList.add(replica); - chronicleChannelIds.add((int) chronicleChannel); - channelEntryExternalizables[chronicleChannel] = entryExternalizable; - - if (chronicleChannel == 0) - return; - - for (int i = (int) systemModificationIteratorBitSet.nextSetBit(0); i > 0; - i = (int) systemModificationIteratorBitSet.nextSetBit(i + 1)) { - byte remoteIdentifier = (byte) i; - final long lastModificationTime = replica.lastModificationTime(remoteIdentifier); - final ByteBufferBytes message = - toBootstrapMessage(chronicleChannel, lastModificationTime); - systemModificationIterator.get(remoteIdentifier).addPayload(message); - } - - } - - @Override - public void close() throws IOException { - for (Closeable replicator : replicators) { - replicator.close(); - } - // indexed reverse loop to avoid ConcurrentModificationException and remove elements from - // the end of the loop for correct iteration - for (int i = chronicleChannelList.size() - 1; i >= 0; i--) { - Replica chronicleChannel = chronicleChannelList.get(i); - chronicleChannel.close(); - } - } - - void add(Closeable replicator) { - replicators.add(replicator); - } - - private interface MessageHandler { - void onMessage(Bytes bytes); - } - - private interface PayloadProvider extends ModificationIterator { - void addPayload(final Bytes bytes); - } - - /** - * used to send system messages such as bootstrap from one remote node to another, it also can be used in - * a broadcast context - */ - static class SystemQueue { - - - final Replica asReplica = new Replica() { - - @Override - public byte identifier() { - return 0; - } - - @Override - public ModificationIterator acquireModificationIterator( - final short remoteIdentifier, final ModificationNotifier modificationNotifier) { - - final ModificationIterator result = systemModificationIterator.get(remoteIdentifier); - - if (result != null) - return result; - - final PayloadProvider iterator = new PayloadProvider() { - - - // todo it maybe possibel to replace this with a single atomic refernce, - // todo this way no aditional objects will be created - final Queue payloads = new LinkedTransferQueue(); - - @Override - public boolean hasNext() { - return payloads.peek() != null; - } - - @Override - public boolean nextEntry(@NotNull EntryCallback callback, int na) { - final Bytes bytes = payloads.poll(); - if (bytes == null) - return false; - callback.onEntry(bytes, 0); - return true; - } - - @Override - public void dirtyEntries(long fromTimeStamp) { - // do nothing - } - - @Override - public void addPayload(Bytes bytes) { - if (bytes.remaining() == 0) - return; - payloads.add(bytes); - // notifies that a change has been made, this will nudge the OP_WRITE - // selector to push this update out over the nio socket - modificationNotifier.onChange(); - } - }; - - systemModificationIterator.set(remoteIdentifier, iterator); - systemModificationIteratorBitSet.set(remoteIdentifier); - - return iterator; - - } - - @Override - public long lastModificationTime(byte remoteIdentifier) { - return 0; - } - - @Override - public void close() throws IOException { - // do nothing - } - }; - final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { - @Override - public void writeExternalEntry(@NotNull Bytes entry, @NotNull Bytes destination, - int na) { - destination.write(entry); - } - - @Override - public void readExternalEntry(@NotNull Bytes source) { - messageHandler.onMessage(source); - } - }; - private final DirectBitSet systemModificationIteratorBitSet; - private final AtomicReferenceArray systemModificationIterator; - private final MessageHandler messageHandler; - - SystemQueue(DirectBitSet systemModificationIteratorBitSet, - AtomicReferenceArray systemModificationIterator, - MessageHandler messageHandler) { - - this.systemModificationIteratorBitSet = systemModificationIteratorBitSet; - this.systemModificationIterator = systemModificationIterator; - this.messageHandler = messageHandler; - } - } - - public class ChronicleChannel extends Replicator implements Closeable { - - private final short chronicleChannel; - - private ChronicleChannel(short chronicleChannel) { - this.chronicleChannel = chronicleChannel; - } - - public byte identifier() { - return localIdentifier; - } - - @Override - protected Closeable applyTo(AbstractChronicleMapBuilder builder, - Replica map, EntryExternalizable entryExternalizable, - final ChronicleMap chronicleMap) { - add(chronicleChannel, map, entryExternalizable); - return this; - } - - public ChannelProvider provider() { - return ChannelProvider.this; - } - - @Override - public void close() throws IOException { - chronicleChannelList.remove(chronicleChannels[chronicleChannel]); - chronicleChannelIds.remove(chronicleChannel); - chronicleChannels[chronicleChannel] = null; - channelEntryExternalizables[chronicleChannel] = null; - } - - } -} \ No newline at end of file diff --git a/src/main/java/net/openhft/chronicle/hash/ChannelProviderBuilder.java b/src/main/java/net/openhft/chronicle/hash/ChannelProviderBuilder.java deleted file mode 100644 index d9a26b346..000000000 --- a/src/main/java/net/openhft/chronicle/hash/ChannelProviderBuilder.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; - -import net.openhft.chronicle.map.TcpReplicator; -import net.openhft.chronicle.map.UdpReplicator; - -import java.io.IOException; - -/** - * @author Rob Austin. - */ -public final class ChannelProviderBuilder { - - byte identifier; - int maxEntrySize = 1024; - int maxNumberOfChronicles = 128; - private UdpReplicationConfig udpReplicationConfig = null; - private TcpReplicationConfig tcpReplicationConfig = null; - - - - public int maxEntrySize() { - return maxEntrySize; - } - - public ChannelProviderBuilder maxEntrySize(int maxEntrySize) { - this.maxEntrySize = maxEntrySize; - return this; - } - - public ChannelProviderBuilder replicators(byte identifier, ReplicationConfig... replicationConfigs) { - this.identifier = identifier; - - - for (ReplicationConfig replicationConfig : replicationConfigs) { - - if (replicationConfig instanceof TcpReplicationConfig) { - this.tcpReplicationConfig = (TcpReplicationConfig) replicationConfig; - } else if (replicationConfig instanceof UdpReplicationConfig) { - this.udpReplicationConfig = (UdpReplicationConfig) replicationConfig; - } else - throw new UnsupportedOperationException(); - } - return this; - } - - public ChannelProviderBuilder maxNumberOfChronicles(int maxNumberOfChronicles) { - this.maxNumberOfChronicles = maxNumberOfChronicles; - return this; - } - - public ChannelProvider create() throws IOException { - final ChannelProvider replicatingCluster = new ChannelProvider(this); - if (tcpReplicationConfig != null) { - final TcpReplicator tcpReplicator = new TcpReplicator( - replicatingCluster.asReplica, - replicatingCluster.asEntryExternalizable, - tcpReplicationConfig, - maxEntrySize, null); - replicatingCluster.add(tcpReplicator); - } - - if (udpReplicationConfig != null) { - final UdpReplicator udpReplicator = - new UdpReplicator(replicatingCluster.asReplica, - replicatingCluster.asEntryExternalizable, - udpReplicationConfig, - maxEntrySize); - replicatingCluster.add(udpReplicator); - } - return replicatingCluster; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java index ab5624b35..4f86d0980 100644 --- a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java +++ b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java @@ -18,6 +18,9 @@ package net.openhft.chronicle.hash; +import net.openhft.chronicle.hash.replication.SimpleReplication; +import net.openhft.chronicle.hash.replication.TcpConfig; +import net.openhft.chronicle.hash.replication.TimeProvider; import net.openhft.chronicle.map.*; import net.openhft.chronicle.set.ChronicleSet; import net.openhft.chronicle.set.ChronicleSetBuilder; @@ -32,6 +35,7 @@ import org.jetbrains.annotations.NotNull; import java.io.Externalizable; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -224,67 +228,6 @@ public interface ChronicleHashBuilder - * the section about replication in ChronicleMap manual for more information. - * - *

Another way to establish replication is {@link #channel(ChannelProvider.ChronicleChannel)} - * method. - * - *

By default, hash containers, created by this builder doesn't replicate their data. - * - *

This method call overrides all previous replication configurations of this builder, made - * either by this method or {@link #channel(ChannelProvider.ChronicleChannel)} method. - * - * @param identifier the new {@linkplain Replica#identifier() identifier} of the containers, - * created by this builder - * @param replicators several replicators, used to sync the data across replicating nodes - * independently - * @return this builder back - * @see #channel(ChannelProvider.ChronicleChannel) - * @see #disableReplication() - */ - B replicators(byte identifier, ReplicationConfig... replicators); - - /** - * Configures replication of the hash containers, created by this builder, via so called - * "channels". See the - * section about Channels and ChannelProvider in ChronicleMap manual for more information. - * - *

Another way to establish replication if {@link #replicators(byte, ReplicationConfig[])} - * method. - * - *

By default, hash containers, created by this builder doesn't replicate their data. - * - *

This method call overrides all previous replication configurations of this builder, made - * either by means of this method or {@link #replicators(byte, ReplicationConfig[])} method - * calls. - * - * @param chronicleChannel the channel responsible for gathering updates of hash containers, - * created by this builder, and replicating them over network - * @return this builder object back - * @see #replicators(byte, ReplicationConfig[]) - * @see #disableReplication() - */ - B channel(ChannelProvider.ChronicleChannel chronicleChannel); - - /** - * Specifies that hash containers, created by this builder, shouldn't replicate their data over - * network. This method call clears any replication configurations, made via {@link - * #replicators(byte, ReplicationConfig[])} or {@link #channel(ChannelProvider.ChronicleChannel)} - * method call. - * - *

By default replication is not configured, so this method is useful only to ensure - * that hash containers, created by some inherited or passed builder, wouldn't replicate. I. e. - * in rare cases. - * - * @return this builder back - * @see #replicators(byte, ReplicationConfig[]) - * @see #channel(ChannelProvider.ChronicleChannel) - */ - B disableReplication(); - /** * Configures a time provider, used by hash containers, created by this builder, for needs of * replication consensus protocol (conflicting data updates resolution). @@ -293,8 +236,7 @@ public interface ChronicleHashBuilder + * the section about replication in ChronicleMap manual for more information. * - * @param statelessBuilder Stateless Client Config, {@code null} demotes no statelessBuilder + *

By default, hash containers, created by this builder doesn't replicate their data. + * + *

This method call overrides all previous replication configurations of this builder, made + * either by this method or {@link #replication(byte, TcpConfig)} shortcut method. + * + * @param replication the replication config * @return this builder back + * @see ChronicleHashInstanceConfig#replicated(SimpleReplication) + * @see #replication(byte, TcpConfig) */ - B stateless(StatelessBuilder statelessBuilder); - + B replication(SimpleReplication replication); /** - * Specifies that hash containers us the specified file, if create is called and this is not set - * the map will be created in off-heap memory, and changes will not be persisted to disk. If a - * file is provided all changes to the map are persisted to disk (this is an operating system - * guarantee) independently from JVM process lifecycle. + * Shortcut for {@code replication(SimpleReplication.builder() + * .tcpTransportAndNetwork(tcpTransportAndNetwork).create(identifier))}. * - * @param file the file with existing hash container or a desired location of a new off-heap - * persisted hash container + * @param identifier the network-wide identifier of the containers, created by this builder + * @param tcpTransportAndNetwork configuration of tcp connection and network * @return this builder back - * @throws IOException If the file cannot be created and read. - * @see ChronicleHash#file() - * @see ChronicleHash#close() + * @see #replication(SimpleReplication) + * @see ChronicleHashInstanceConfig#replicated(byte, TcpConfig) */ - B file(File file) throws IOException; + B replication(byte identifier, TcpConfig tcpTransportAndNetwork); + + StatelessClientConfig statelessClient(InetSocketAddress remoteAddress); + + ChronicleHashInstanceConfig instance(); /** * Creates a new hash container, storing it's data in off-heap memory, not mapped to any file. @@ -426,15 +367,42 @@ public interface ChronicleHashBuilderThis method is a shortcut for {@code instance().create()}. + * * @return a new off-heap hash container * @throws IOException if any IO error relates to off-heap memory allocation occurs - * @see #create() + * @see #createPersistedTo(File) + * @see #instance() */ C create() throws IOException; - - - - + /** + * Opens a hash container residing the specified file, or creates a new one if the file not yet + * exists and maps its off-heap memory to the file. All changes to the map are persisted to disk + * (this is an operating system guarantee) independently from JVM process lifecycle. + * + *

Multiple containers could give access to the same data simultaneously, either inside a + * single JVM or across processes. Access is synchronized correctly across all instances, i. e. + * hash container mapping the data from the first JVM isn't able to modify the data, + * concurrently accessed from the second JVM by another hash container instance, mapping the + * same data. + * + *

On container's {@link ChronicleHash#close() close()} the data isn't removed, it remains on + * disk and available to be opened again (given the same file name) or during different JVM + * run. + * + *

This method is shortcut for {@code instance().persistedTo(file).create()}. + * + * @param file the file with existing hash container or a desired location of a new off-heap + * persisted hash container + * @return a hash container mapped to the given file + * @throws IOException if any IO error, related to off-heap memory allocation or file mapping + * occurs + * @see ChronicleHash#file() + * @see ChronicleHash#close() + * @see #create() + * @see ChronicleHashInstanceConfig#persistedTo(File) + */ + C createPersistedTo(File file) throws IOException; } diff --git a/src/main/java/net/openhft/chronicle/hash/ReplicationConfig.java b/src/main/java/net/openhft/chronicle/hash/ReplicationConfig.java deleted file mode 100644 index edb7e9269..000000000 --- a/src/main/java/net/openhft/chronicle/hash/ReplicationConfig.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; - -/** - * @author Rob Austin. - */ -public interface ReplicationConfig { - - -} diff --git a/src/main/java/net/openhft/chronicle/hash/StatelessBuilder.java b/src/main/java/net/openhft/chronicle/hash/StatelessBuilder.java deleted file mode 100644 index 5434eb768..000000000 --- a/src/main/java/net/openhft/chronicle/hash/StatelessBuilder.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; - -import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; - -/** - * @author Rob Austin. - */ -public class StatelessBuilder { - - private long timeoutMs = TimeUnit.SECONDS.toMillis(10); - - private InetSocketAddress remoteAddress; - - public long timeoutMs() { - return timeoutMs; - } - - public void timeout(long timeout, TimeUnit units) { - this.timeoutMs = units.toMillis(timeout); - } - - private StatelessBuilder(InetSocketAddress remoteAddress) { - this.remoteAddress = remoteAddress; - } - - public static StatelessBuilder remoteAddress(InetSocketAddress inetSocketAddress) { - return new StatelessBuilder(inetSocketAddress); - } - - public InetSocketAddress remoteAddress() { - return remoteAddress; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/TcpReplicationConfig.java b/src/main/java/net/openhft/chronicle/hash/TcpReplicationConfig.java deleted file mode 100644 index 22e4aa6de..000000000 --- a/src/main/java/net/openhft/chronicle/hash/TcpReplicationConfig.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; - -import net.openhft.chronicle.map.IdentifierListener; - -import org.jetbrains.annotations.NotNull; - -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static java.util.Collections.unmodifiableSet; -import static java.util.concurrent.TimeUnit.SECONDS; - - -public class TcpReplicationConfig implements ReplicationConfig { - - private static final int DEFAULT_PACKET_SIZE = 1024 * 8; - private static final long DEFAULT_HEART_BEAT_INTERVAL = 20; - private static final TimeUnit DEFAULT_HEART_BEAT_INTERVAL_UNIT = SECONDS; - - private int serverPort = DEFAULT_PACKET_SIZE; - private Set endpoints; - private int packetSize = DEFAULT_PACKET_SIZE; - private boolean autoReconnectedUponDroppedConnection = true; - private ThrottlingConfig throttlingConfig = ThrottlingConfig.noThrottling(); - private long heartBeatInterval = DEFAULT_HEART_BEAT_INTERVAL; - private TimeUnit heartBeatIntervalUnit = DEFAULT_HEART_BEAT_INTERVAL_UNIT; - public IdentifierListener identifierListener; - - TcpReplicationConfig() { - - } - - public IdentifierListener identifierListener() { - return identifierListener; - } - - public TcpReplicationConfig nonUniqueIdentifierListener(@NotNull final IdentifierListener - identifierListener) { - this.identifierListener = identifierListener; - return this; - } - - public boolean autoReconnectedUponDroppedConnection() { - return autoReconnectedUponDroppedConnection; - } - - public TcpReplicationConfig autoReconnectedUponDroppedConnection(boolean autoReconnectedUponDroppedConnection) { - this.autoReconnectedUponDroppedConnection = autoReconnectedUponDroppedConnection; - return this; - } - - public ThrottlingConfig throttlingConfig() { - return throttlingConfig; - } - - public TcpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { - this.throttlingConfig = throttlingConfig; - return this; - } - - public long heartBeatInterval(TimeUnit unit) { - return unit.convert(heartBeatInterval, heartBeatIntervalUnit); - } - - public int serverPort() { - return serverPort; - } - - public TcpReplicationConfig serverPort(int serverPort) { - this.serverPort = serverPort; - return this; - } - - public Set endpoints() { - return endpoints; - } - - public TcpReplicationConfig endpoints(Set endpoints) { - - for (final InetSocketAddress endpoint : endpoints) { - if (endpoint.getPort() == serverPort && "localhost".equals(endpoint.getHostName())) - throw new IllegalArgumentException("endpoint=" + endpoint - + " can not point to the same port as the server"); - } - - this.endpoints = endpoints; - return this; - } - - public int packetSize() { - return packetSize; - } - - public static TcpReplicationConfig of(int serverPort, InetSocketAddress... endpoints) { - return of(serverPort, Arrays.asList(endpoints)); - } - - public static TcpReplicationConfig of(int serverPort, Collection endpoints) { - // at least in tests, we configure "receive-only" replication without endpoints. - // TODO decide what to do with this case -// if (endpoints.isEmpty()) -// throw new IllegalArgumentException("There should be some endpoints"); - for (final InetSocketAddress endpoint : endpoints) { - if (endpoint.getPort() == serverPort && "localhost".equals(endpoint.getHostName())) - throw new IllegalArgumentException("endpoint=" + endpoint - + " can not point to the same port as the server"); - } - - TcpReplicationConfig tcpReplicationConfig = new TcpReplicationConfig(); - - tcpReplicationConfig.serverPort(serverPort); - tcpReplicationConfig.endpoints = unmodifiableSet(new HashSet(endpoints)); - - return tcpReplicationConfig; - - - } - - - public TcpReplicationConfig packetSize(int packetSize) { - if (packetSize <= 0) - throw new IllegalArgumentException(); - this.packetSize = packetSize; - return this; - } - - public TcpReplicationConfig throttlingConfig(long heartBeatInterval, TimeUnit heartBeatIntervalUnit) { - this.heartBeatInterval = heartBeatInterval; - this.heartBeatIntervalUnit = heartBeatIntervalUnit; - return this; - } - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TcpReplicationConfig that = (TcpReplicationConfig) o; - - if (autoReconnectedUponDroppedConnection != that.autoReconnectedUponDroppedConnection) return false; - if (heartBeatInterval != that.heartBeatInterval) return false; - if (packetSize != that.packetSize) return false; - if (serverPort != that.serverPort) return false; - if (endpoints != null ? !endpoints.equals(that.endpoints) : that.endpoints != null) return false; - if (heartBeatIntervalUnit != that.heartBeatIntervalUnit) return false; - if (throttlingConfig != null ? !throttlingConfig.equals(that.throttlingConfig) : that.throttlingConfig != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = serverPort; - result = 31 * result + (endpoints != null ? endpoints.hashCode() : 0); - result = 31 * result + packetSize; - result = 31 * result + (autoReconnectedUponDroppedConnection ? 1 : 0); - result = 31 * result + (throttlingConfig != null ? throttlingConfig.hashCode() : 0); - result = 31 * result + (int) (heartBeatInterval ^ (heartBeatInterval >>> 32)); - result = 31 * result + (heartBeatIntervalUnit != null ? heartBeatIntervalUnit.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "TcpReplicationConfig{" + - "serverPort=" + serverPort + - ", endpoints=" + endpoints + - ", packetSize=" + packetSize + - ", autoReconnectedUponDroppedConnection=" + autoReconnectedUponDroppedConnection + - ", throttlingConfig=" + throttlingConfig + - ", heartBeatInterval=" + heartBeatInterval + - ", heartBeatIntervalUnit=" + heartBeatIntervalUnit + - '}'; - } - - public TcpReplicationConfig heartBeatInterval(long time, TimeUnit unit) { - this.heartBeatInterval = time; - this.heartBeatIntervalUnit = unit; - return this; - } - - -} diff --git a/src/main/java/net/openhft/chronicle/hash/ThrottlingConfig.java b/src/main/java/net/openhft/chronicle/hash/ThrottlingConfig.java deleted file mode 100644 index dbb466e2d..000000000 --- a/src/main/java/net/openhft/chronicle/hash/ThrottlingConfig.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; - - -import java.util.concurrent.TimeUnit; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; - - -public class ThrottlingConfig { - - private static final long DEFAULT_BUCKET_INTERVAL = 100L; - private static final TimeUnit DEFAULT_BUCKET_INTERVAL_UNIT = MILLISECONDS; - - private static final ThrottlingConfig NO_THROTTLING = create(0L, MILLISECONDS, - DEFAULT_BUCKET_INTERVAL, DEFAULT_BUCKET_INTERVAL_UNIT); - - private final long throttle; - private final TimeUnit throttlePerUnit; - private final long bucketInterval; - private final TimeUnit bucketIntervalUnit; - - - ThrottlingConfig( - long throttle, - TimeUnit throttlePerUnit, - long bucketInterval, - TimeUnit bucketIntervalUnit) { - this.throttle = throttle; - if (throttlePerUnit == null) { - throw new NullPointerException("Null throttlePerUnit"); - } - this.throttlePerUnit = throttlePerUnit; - this.bucketInterval = bucketInterval; - if (bucketIntervalUnit == null) { - throw new NullPointerException("Null bucketIntervalUnit"); - } - this.bucketIntervalUnit = bucketIntervalUnit; - } - - /** - * Returns a config which prescribe that a replicator shouldn't throttle. - * - * @return a config which prescribe that a replicator shouldn't throttle - */ - public static ThrottlingConfig noThrottling() { - return NO_THROTTLING; - } - - /** - * Returns a config which throttle the specified max bits per the given time unit with default bucketing - * interval. - * - * @param maxBits the preferred maximum bits - * @param perUnit the time unit per which maximum bits specified - * @return a config which throttle the specified max bits per the given time unit - * @throws IllegalArgumentException if the specified {@code maxBits} is non-positive value - */ - public static ThrottlingConfig throttle(long maxBits, TimeUnit perUnit) { - if (maxBits <= 0L) - throw new IllegalArgumentException(); - return create(maxBits, perUnit, DEFAULT_BUCKET_INTERVAL, DEFAULT_BUCKET_INTERVAL_UNIT); - } - - public static void checkMillisecondBucketInterval(ThrottlingConfig config, String replicator) { - if (config.bucketInterval(MILLISECONDS) < 1) { - throw new IllegalArgumentException( - "Minimum throttle bucketing interval of 1 millisecond is supported by " + - replicator + " replicator"); - } - } - - static ThrottlingConfig create( - long throttle, TimeUnit throttlePerUnit, - long throttleBucketInterval, TimeUnit throttleBucketIntervalUnit) { - return new ThrottlingConfig(throttle, throttlePerUnit, - throttleBucketInterval, throttleBucketIntervalUnit); - } - - - /** - * Returns maximum bits per the given time unit, i. e. the throttling. {@code 0} (zero) designates there - * is no throttling. - * - * @param perUnit maximum bits is returned per this time unit - * @return maximum bits per the given time unit, or {@code 0} if there is no throttling - */ - public final long throttling(TimeUnit perUnit) { - return throttlePerUnit.convert(throttle, perUnit); - } - - - /** - * Returns the throttle bucketing interval in the given time units.

Default throttle bucketing - * interval is 100 millis. - * - * @param unit the time unit of the interval - * @return the bucketing interval for throttling in the given time units - */ - public long bucketInterval(TimeUnit unit) { - return unit.convert(bucketInterval, bucketIntervalUnit); - } - - /** - * Returns a copy of this config with the specified bucketing interval. - * - * @param throttleBucketInterval the bucketing interval for throttling - * @param unit the time unit of the interval - * @return a copy of this config with the specified bucketing interval - * @throws IllegalArgumentException if the given bucketing interval is non-positive - */ - public ThrottlingConfig bucketInterval(long throttleBucketInterval, TimeUnit unit) { - if (throttleBucketInterval <= 0L) - throw new IllegalArgumentException(); - return create(throttle, throttlePerUnit, throttleBucketInterval, unit); - } - - @Override - public String toString() { - return "ThrottlingConfig{" - + "throttle=" + throttle - + ", throttlePerUnit=" + throttlePerUnit - + ", bucketInterval=" + bucketInterval - + ", bucketIntervalUnit=" + bucketIntervalUnit - + "}"; - } - - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - if (o instanceof ThrottlingConfig) { - ThrottlingConfig that = (ThrottlingConfig) o; - return (this.throttle == that.throttle) - && (this.throttlePerUnit.equals(that.throttlePerUnit)) - && (this.bucketInterval == that.bucketInterval) - && (this.bucketIntervalUnit.equals(that.bucketIntervalUnit)); - } - return false; - } - - @Override - public int hashCode() { - int h = 1; - h *= 1000003; - h ^= (throttle >>> 32) ^ throttle; - h *= 1000003; - h ^= throttlePerUnit.hashCode(); - h *= 1000003; - h ^= (bucketInterval >>> 32) ^ bucketInterval; - h *= 1000003; - h ^= bucketIntervalUnit.hashCode(); - return h; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/TimeProvider.java b/src/main/java/net/openhft/chronicle/hash/TimeProvider.java deleted file mode 100644 index bdf1f565c..000000000 --- a/src/main/java/net/openhft/chronicle/hash/TimeProvider.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; - -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.set.ChronicleSet; - -import java.io.ObjectStreamException; -import java.io.Serializable; - -/** - * TimeProvider was aims to possibly later provide an optimization to - * {@link System#currentTimeMillis()} on every call to replicated {@link ChronicleMap} - * or {@link ChronicleSet}. - * - * Subclasses should be immutable, because {@link ChronicleHashBuilder} doesn't make defensive - * copies. - * - * @author Rob Austin. - * @see ChronicleHashBuilder#timeProvider(TimeProvider) - */ -public abstract class TimeProvider implements Serializable { - /** - * Delegates {@link #currentTimeMillis()} to {@link System#currentTimeMillis()}. - */ - public static final TimeProvider SYSTEM = new System(); - private static final long serialVersionUID = 0L; - - public abstract long currentTimeMillis(); - - private static class System extends TimeProvider { - private static final long serialVersionUID = 0L; - - public long currentTimeMillis() { - return java.lang.System.currentTimeMillis(); - } - - @Override - public boolean equals(Object o) { - return o != null && o.getClass() == getClass(); - } - - @Override - public int hashCode() { - return getClass().hashCode(); - } - - @Override - public String toString() { - return getClass().getSimpleName(); - } - - private Object readResolve() throws ObjectStreamException { - return SYSTEM; - } - } - -} diff --git a/src/main/java/net/openhft/chronicle/hash/UdpReplicationConfig.java b/src/main/java/net/openhft/chronicle/hash/UdpReplicationConfig.java deleted file mode 100644 index 85076edcb..000000000 --- a/src/main/java/net/openhft/chronicle/hash/UdpReplicationConfig.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.net.InetAddress; -import java.net.NetworkInterface; - - -public abstract class UdpReplicationConfig implements ReplicationConfig { - - /** - * Package-private constructor forbids subclassing from outside of the package - */ - UdpReplicationConfig() { - // nothing to do - } - - public static UdpReplicationConfig simple(@NotNull InetAddress address, int port) { - if (address.isMulticastAddress()) - throw new IllegalArgumentException(); - - - return create(address, port, null, ThrottlingConfig.noThrottling()); - } - - public static UdpReplicationConfig multiCast(@NotNull InetAddress address, int port, - @NotNull NetworkInterface networkInterface) { - if (!address.isMulticastAddress() || networkInterface == null) - throw new IllegalArgumentException(); - return create(address, port, networkInterface, ThrottlingConfig.noThrottling()); - } - - static UdpReplicationConfig create(InetAddress address, int port, - NetworkInterface networkInterface, - ThrottlingConfig throttlingConfig) { - return new UdpReplicationConfigBean(address, port, networkInterface, - throttlingConfig); - } - - public abstract InetAddress address(); - - public abstract int port(); - - @Nullable - public abstract NetworkInterface networkInterface(); - - public abstract ThrottlingConfig throttlingConfig(); - - public UdpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { - ThrottlingConfig.checkMillisecondBucketInterval(throttlingConfig, "UDP"); - return create(address(), port(), networkInterface(), throttlingConfig); - } -} - - - -final class UdpReplicationConfigBean extends UdpReplicationConfig { - private final InetAddress address; - private final int port; - private final NetworkInterface networkInterface; - private final ThrottlingConfig throttlingConfig; - - UdpReplicationConfigBean( - InetAddress address, - int port, - NetworkInterface networkInterface, - ThrottlingConfig throttlingConfig) { - if (address == null) { - throw new NullPointerException("Null address"); - } - this.address = address; - this.port = port; - - this.networkInterface = networkInterface; - if (throttlingConfig == null) { - throw new NullPointerException("Null throttlingConfig"); - } - this.throttlingConfig = throttlingConfig; - } - - @Override - public InetAddress address() { - return address; - } - - @Override - public int port() { - return port; - } - - @Override - public NetworkInterface networkInterface() { - return networkInterface; - } - - @Override - public ThrottlingConfig throttlingConfig() { - return throttlingConfig; - } - - @Override - public String toString() { - return "UdpReplicationConfig{" - + "address=" + address - + ", port=" + port - + ", networkInterface=" + networkInterface - + ", throttlingConfig=" + throttlingConfig - + "}"; - } - - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - if (o instanceof UdpReplicationConfig) { - UdpReplicationConfig that = (UdpReplicationConfig) o; - return (this.address.equals(that.address())) - && (this.port == that.port()) - && (this.networkInterface.equals(that.networkInterface())) - && (this.throttlingConfig.equals(that.throttlingConfig())); - } - return false; - } - - @Override - public int hashCode() { - int h = 1; - h *= 1000003; - h ^= address.hashCode(); - h *= 1000003; - h ^= port; - h *= 1000003; - h ^= networkInterface.hashCode(); - h *= 1000003; - h ^= throttlingConfig.hashCode(); - return h; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/AgileBytesMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/AgileBytesMarshaller.java deleted file mode 100644 index 6ab1f7b15..000000000 --- a/src/main/java/net/openhft/chronicle/hash/serialization/AgileBytesMarshaller.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash.serialization; - -public interface AgileBytesMarshaller extends BytesReader, BytesInterop, SizeMarshaller { -} diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/BytesInterop.java b/src/main/java/net/openhft/chronicle/hash/serialization/BytesInterop.java index 33231635c..204a49f69 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/BytesInterop.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/BytesInterop.java @@ -22,15 +22,30 @@ import net.openhft.lang.io.Bytes; import net.openhft.lang.model.Byteable; /** - * Marshaller for byte sequences, which are copied to off-heap {@link Bytes} - * in a very straightforward manner, e. g. {@link Byteable Byteables}, {@code byte[]} arrays, - * {@code Bytes} themselves. The criterion of this interface applicability -- - * {@link Object#equals(Object)} implementation shouldn't require deserialization and any garbage creation. + * Writer for objects, which themselves are bytes sequence in some sense: + * {@link Byteable Byteables}, {@code byte[]} arrays, other primitive arrays, "flat" objects + * (with primitives fields only), particularly boxed primitive types. + * + *

This interface is called "BytesInterop" because it allows to work with object as they + * are already marshalled to {@code Bytes}: {@linkplain #startsWith(Bytes, Object) compare} + * with other {@code Bytes}, i. e. interoperate objects and {@code Bytes}. * * @param type of marshalled objects */ public interface BytesInterop extends BytesWriter { + /** + * Checks if the given {@code bytes} starts (from the {@code bytes}' {@linkplain + * Bytes#position() position}) with the byte sequence the given object + * {@linkplain #write(Bytes, Object) is serialized} to, without actual serialization. + * + * @param bytes the bytes to check if starts with the serialized form of the given object. + * {@code bytes}' is positioned at the first byte to compare. {@code bytes}' limit + * is unspecified. {@code bytes}' position and limit shouldn't be altered during + * this call. + * @param e the object to serialize virtually and compare with the given {@code bytes} + * @return if the given {@code bytes} starts with the given {@code e} object's serialized form + */ boolean startsWith(Bytes bytes, E e); long hash(E e); diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java b/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java index d2d62e49e..b43ba57aa 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java @@ -18,13 +18,86 @@ package net.openhft.chronicle.hash.serialization; +import net.openhft.chronicle.hash.ChronicleHash; import net.openhft.lang.io.Bytes; +import org.jetbrains.annotations.Nullable; import java.io.Serializable; +/** + * Deserializer (object from {@link Bytes}, mirroring the {@link BytesWriter}, i. e. assuming the + * length of the serialized form isn't written in the beginning of the serialized form itself, + * but managed by {@link ChronicleHash} implementation and passed to the reading methods. + * + *

Implementation example:


+ * class LongPair { long first, second; }
+ * 
+ * enum LongPairArrayReader implements BytesReader<LongPair[]> {
+ *     INSTANCE;
+ *
+ *     @Override
+ *     public LongPair[] read(Bytes bytes, long size) {
+ *         return read(bytes, size, null);
+ *     }
+ *
+ *     @Override
+ *     public LongPair[] read(Bytes bytes, long size, LongPair[] toReuse) {
+ *         int resLen = (int) (size / 16L);
+ *         LongPair[] res;
+ *         if (toReuse != null) {
+ *             if (toReuse.length == resLen) {
+ *                 res = toReuse;
+ *             } else {
+ *                 res = Arrays.copyOf(toReuse, resLen);
+ *             }
+ *         } else {
+ *             res = new LongPair[resLen];
+ *         }
+ *         for (int i = 0; i < resLen; i++) {
+ *             LongPair pair = res[i];
+ *             if (pair == null)
+ *                 res[i] = pair = new LongPair();
+ *             pair.first = bytes.readLong();
+ *             pair.second = bytes.readLong();
+ *         }
+ *         return res;
+ *     }
+ * }
+ * + * @param the type of the object demarshalled + * @see BytesWriter + */ public interface BytesReader extends Serializable { + /** + * Reads and returns the object from {@code [position(), position() + size]} bytes of the given + * {@link Bytes}. + * + *

Implementation of this method should increment the {@code bytes}' position by the given + * size, i. e. "consume" the bytes of the deserialized object. It must not alter the + * {@code bytes}' {@link Bytes#limit() limit} and contents. + * + * @param bytes the {@code Bytes} to read the object from + * @param size the size of the serialized form of the returned object + * @return the object read from the bytes + * @see BytesWriter#write(Bytes, Object) + * @see #read(Bytes, long, Object) + */ E read(Bytes bytes, long size); - E read(Bytes bytes, long size, E e); + /** + * Similar to {@link #read(Bytes, long)}, but should attempt to reuse the given object, i. e. + * to read the deserialized data into the given instance. If it is possible, this objects than + * returned from this method. If it isn't possible for any reason, a new object should be + * created and returned, just like in {@link #read(Bytes, long)} method. The given object could + * be {@code null}, in this case this method should behave exactly the same as + * {@link #read(Bytes, long)} method does. + * + * @param bytes the {@code Bytes} to read the object from + * @param size the size of the serialized form of the returned object + * @param toReuse the object to read the deserialized data into + * @return the object read from the bytes, either reused or newly created + * @see #read(Bytes, long) + */ + E read(Bytes bytes, long size, @Nullable E toReuse); } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/BytesReaders.java b/src/main/java/net/openhft/chronicle/hash/serialization/BytesReaders.java deleted file mode 100644 index 6d6f89202..000000000 --- a/src/main/java/net/openhft/chronicle/hash/serialization/BytesReaders.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash.serialization; - -import net.openhft.lang.io.Bytes; -import net.openhft.lang.io.serialization.BytesMarshaller; - -public final class BytesReaders { - - public static BytesReader fromBytesMarshaller(BytesMarshaller marshaller) { - return new SimpleBytesReader(marshaller); - } - - private static class SimpleBytesReader implements BytesReader { - private static final long serialVersionUID = 0L; - - private final BytesMarshaller marshaller; - - public SimpleBytesReader(BytesMarshaller marshaller) { - this.marshaller = marshaller; - } - - @Override - public E read(Bytes bytes, long size) { - return marshaller.read(bytes); - } - - @Override - public E read(Bytes bytes, long size, E e) { - return marshaller.read(bytes, e); - } - } - - private BytesReaders() {} -} diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java b/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java index f14c31fa2..957c2a3dc 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java @@ -18,13 +18,66 @@ package net.openhft.chronicle.hash.serialization; +import net.openhft.chronicle.hash.ChronicleHash; import net.openhft.lang.io.Bytes; import java.io.Serializable; +/** + * Serializer (object to {@link Bytes} writer) which 1) knows the length of serialized form + * of any object before actual serialization 2) doesn't include that length in the serialized + * form itself, assuming it will be passed by the {@link ChronicleHash} into + * {@link BytesReader#read(Bytes, long)} deserialization method. + * + *

Implementation example: 


+ * class LongPair { long first, second; }
+ *
+ * enum LongPairArrayWriter implements BytesWriter<LongPair[]> {
+ *     INSTANCE;
+ *
+ *     @Override
+ *     public long size(LongPair[] longPairs) {
+ *         return longPairs.length * 16L;
+ *     }
+ *
+ *     @Override
+ *     public void write(Bytes bytes, LongPair[] longPairs) {
+ *         for (LongPair pair : longPairs) {
+ *             bytes.writeLong(pair.first);
+ *             bytes.writeLong(pair.second);
+ *         }
+ *     }
+ * }
+ * + * @param the type of the object marshalled + * @see BytesReader + * @see BytesInterop + */ public interface BytesWriter extends Serializable { + /** + * Returns the length (in bytes) of the serialized form of the given object. Serialization form + * in terms of this interface, i. e. how much bytes are written to {@code bytes} on + * {@link #write(Bytes, Object) write(bytes, e)} call. + * + * @param e the object which serialized form length should be returned + * @return the length (in bytes) of the serialized form of the given object + */ long size(E e); + /** + * Serializes the given object to the given {@code bytes}, without writing the length of the + * serialized form itself. + * + *

Implementation of this method should increment the {@code bytes}' {@linkplain + * Bytes#position() position} by {@link #size(Object) size(e)}. The given object should + * be written into these range between the initial {@code bytes}' position and the position + * after this method call returns. Implementation of this method must not alter the + * {@code bytes}' {@linkplain Bytes#limit() limit} and contents outside of the specified range. + * + * @param bytes the {@code Bytes} to write the given object to + * @param e the object to serialize to the given bytes + * @see BytesReader#read(Bytes, long) + */ void write(Bytes bytes, E e); } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayMarshaller.java index 8ea01472e..43f11cdc5 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayMarshaller.java @@ -59,10 +59,10 @@ public enum ByteArrayMarshaller implements BytesInterop, BytesReader implements AgileBytesMarshaller { +public class ByteableMarshaller + implements BytesInterop, BytesReader, SizeMarshaller { private static final long serialVersionUID = 0L; public static ByteableMarshaller of(@NotNull Class eClass) { @@ -114,13 +117,13 @@ public class ByteableMarshaller implements AgileBytesMarshal } @Override - public E read(Bytes bytes, long size, E e) { + public E read(Bytes bytes, long size, E toReuse) { try { - if (e == null) - e = getInstance(); - setBytesAndOffset(e, bytes); + if (toReuse == null) + toReuse = getInstance(); + setBytesAndOffset(toReuse, bytes); bytes.skip(size); - return e; + return toReuse; } catch (Exception ex) { throw new IllegalStateException(ex); } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/CharArrayMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/CharArrayMarshaller.java index 1cc1da171..1f285d8ae 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/CharArrayMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/CharArrayMarshaller.java @@ -49,11 +49,11 @@ public enum CharArrayMarshaller implements BytesInterop, BytesReader } @Override - public S read(Bytes bytes, long size, S s) { + public S read(Bytes bytes, long size, S toReuse) { Appendable appendable; - if (s instanceof Appendable) { - appendable = (Appendable) s; - if (s instanceof StringBuilder) { - ((StringBuilder) s).setLength(0); - } else if (s instanceof StringBuffer) { - ((StringBuffer) s).setLength(0); + if (toReuse instanceof Appendable) { + appendable = (Appendable) toReuse; + if (toReuse instanceof StringBuilder) { + ((StringBuilder) toReuse).setLength(0); + } else if (toReuse instanceof StringBuffer) { + ((StringBuffer) toReuse).setLength(0); } } else { sb.setLength(0); @@ -96,8 +96,8 @@ public final class CharSequenceReader } catch (IOException e) { throw new IllegalStateException(e); } - if (appendable == s) - return s; + if (appendable == toReuse) + return toReuse; return interner.intern(sb); } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/DoubleMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/DoubleMarshaller.java index 1f663ca21..92a99b534 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/DoubleMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/DoubleMarshaller.java @@ -18,13 +18,15 @@ package net.openhft.chronicle.hash.serialization.impl; -import net.openhft.chronicle.hash.serialization.AgileBytesMarshaller; +import net.openhft.chronicle.hash.serialization.BytesInterop; +import net.openhft.chronicle.hash.serialization.BytesReader; import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.lang.io.Bytes; import static java.lang.Double.doubleToLongBits; -public enum DoubleMarshaller implements AgileBytesMarshaller { +public enum DoubleMarshaller implements BytesInterop, BytesReader, SizeMarshaller { INSTANCE; @Override @@ -68,7 +70,7 @@ public enum DoubleMarshaller implements AgileBytesMarshaller { } @Override - public Double read(Bytes bytes, long size, Double e) { + public Double read(Bytes bytes, long size, Double toReuse) { return read(bytes, size); } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/IntegerMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/IntegerMarshaller.java index 407ec7968..898bb71b3 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/IntegerMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/IntegerMarshaller.java @@ -18,11 +18,11 @@ package net.openhft.chronicle.hash.serialization.impl; -import net.openhft.chronicle.hash.serialization.AgileBytesMarshaller; -import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.chronicle.hash.serialization.*; import net.openhft.lang.io.Bytes; -public enum IntegerMarshaller implements AgileBytesMarshaller { +public enum IntegerMarshaller + implements BytesInterop, BytesReader, SizeMarshaller { INSTANCE; @Override @@ -66,7 +66,7 @@ public enum IntegerMarshaller implements AgileBytesMarshaller { } @Override - public Integer read(Bytes bytes, long size, Integer e) { + public Integer read(Bytes bytes, long size, Integer toReuse) { return bytes.readInt(); } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/LongMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/LongMarshaller.java index 822d063a7..1e4f8db01 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/LongMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/LongMarshaller.java @@ -18,11 +18,14 @@ package net.openhft.chronicle.hash.serialization.impl; -import net.openhft.chronicle.hash.serialization.AgileBytesMarshaller; +import net.openhft.chronicle.hash.serialization.BytesInterop; +import net.openhft.chronicle.hash.serialization.BytesReader; import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.lang.io.Bytes; -public enum LongMarshaller implements AgileBytesMarshaller { +public enum LongMarshaller + implements BytesInterop, BytesReader, SizeMarshaller { INSTANCE; @Override @@ -66,7 +69,7 @@ public enum LongMarshaller implements AgileBytesMarshaller { } @Override - public Long read(Bytes bytes, long size, Long e) { + public Long read(Bytes bytes, long size, Long toReuse) { return bytes.readLong(); } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/VoidMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/VoidMarshaller.java index 81f2a5ec5..e743414e7 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/VoidMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/VoidMarshaller.java @@ -18,10 +18,13 @@ package net.openhft.chronicle.hash.serialization.impl; -import net.openhft.chronicle.hash.serialization.AgileBytesMarshaller; +import net.openhft.chronicle.hash.serialization.BytesInterop; +import net.openhft.chronicle.hash.serialization.BytesReader; +import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.lang.io.Bytes; -public enum VoidMarshaller implements AgileBytesMarshaller { +public enum VoidMarshaller + implements BytesInterop, BytesReader, SizeMarshaller { INSTANCE; @Override @@ -66,7 +69,7 @@ public enum VoidMarshaller implements AgileBytesMarshaller { } @Override - public Void read(Bytes bytes, long size, Void aVoid) { + public Void read(Bytes bytes, long size, Void toReuse) { return null; } diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java index e63616205..3c88d5c7e 100644 --- a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.ThrottlingConfig; +import net.openhft.chronicle.hash.replication.ThrottlingConfig; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import net.openhft.lang.thread.NamedThreadFactory; diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java index ca7d43f3d..1fd906ff7 100644 --- a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java @@ -19,6 +19,7 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.hash.*; +import net.openhft.chronicle.hash.replication.*; import net.openhft.chronicle.hash.serialization.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.MetaBytesWriter; import net.openhft.chronicle.hash.serialization.MetaProvider; @@ -40,8 +41,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -103,8 +107,6 @@ public abstract class AbstractChronicleMapBuilder keyBuilder; SerializationBuilder valueBuilder; - private Map, Replicator> replicators = new HashMap, Replicator>(); - // used when configuring the number of segments. private int minSegments = -1; private int actualSegments = -1; @@ -133,10 +135,8 @@ public abstract class AbstractChronicleMapBuilder defaultValueProvider = NullValueProvider.INSTANCE; private PrepareValueBytes prepareValueBytes = null; - private byte identifier = -1; - private StatelessBuilder statelessBuilder; - private File file; + private SimpleReplication simpleReplication = null; AbstractChronicleMapBuilder(Class keyClass, Class valueClass) { keyBuilder = new SerializationBuilder(keyClass, SerializationBuilder.Role.KEY); @@ -283,14 +283,14 @@ public abstract class AbstractChronicleMapBuilder 0) return entryAndValueAlignment().alignSize(entrySize); int size = metaDataBytes; int keySize = keySize(); size += keyBuilder.sizeMarshaller().sizeEncodingSize(keySize); size += keySize; - if (useReplication()) + if (replicated) size += ReplicatedChronicleMap.ADDITIONAL_ENTRY_BYTES; int valueSize = valueSize(); size += valueBuilder.sizeMarshaller().sizeEncodingSize(valueSize); @@ -559,43 +559,6 @@ public abstract class AbstractChronicleMapBuilder> statelessClient( + InetSocketAddress remoteAddress) { + return new StatelessMapConfig<>(this.clone(), remoteAddress); } /** @@ -748,7 +705,7 @@ public abstract class AbstractChronicleMapBuilder defaultValueProvider() { if (defaultValueProvider != null) @@ -766,7 +723,43 @@ public abstract class AbstractChronicleMapBuilder(defaultValue, metaValueWriter, valueWriter); } - public ChronicleMap createWithFile(File file) throws IOException { + @Override + public B replication(SimpleReplication replication) { + this.simpleReplication = replication; + return self(); + } + + @Override + public B replication(byte identifier, TcpConfig tcpTransportAndNetwork) { + return replication(SimpleReplication.builder() + .tcpTransportAndNetwork(tcpTransportAndNetwork).create(identifier)); + } + + @Override + public ChronicleHashInstanceConfig> instance() { + return new InstanceConfig<>(this.clone(), simpleReplication, null, null); + } + + @Override + public ChronicleMap createPersistedTo(File file) throws IOException { + return createWithFile(file, simpleReplication, null); + } + + @Override + public ChronicleMap create() throws IOException { + return createWithoutFile(simpleReplication, null); + } + + ChronicleMap create(InstanceConfig ib) throws IOException { + if (ib.file != null) { + return createWithFile(ib.file, ib.simpleReplication, ib.channel); + } else { + return createWithoutFile(ib.simpleReplication, ib.channel); + } + } + + ChronicleMap createWithFile(File file, SimpleReplication simpleReplication, + ReplicationChannel channel) throws IOException { for (int i = 0; i < 10; i++) { if (file.exists() && file.length() > 0) { FileInputStream fis = new FileInputStream(file); @@ -776,7 +769,7 @@ public abstract class AbstractChronicleMapBuilder) ois.readObject(); map.headerSize = roundUpMapHeaderSize(fis.getChannel().position()); map.createMappedStoreAndSegments(file); - return establishReplication(map); + return establishReplication(map, simpleReplication, channel); } catch (ClassNotFoundException e) { throw new IOException(e); } finally { @@ -796,7 +789,7 @@ public abstract class AbstractChronicleMapBuilder map = newMap(); + VanillaChronicleMap map = newMap(simpleReplication, channel); FileOutputStream fos = new FileOutputStream(file); ObjectOutputStream oos = new ObjectOutputStream(fos); @@ -809,54 +802,66 @@ public abstract class AbstractChronicleMapBuilder create() throws IOException { - - if (statelessBuilder != null) - return createStatelessMap(statelessBuilder); - - return (file != null) ? createWithFile(file) : createWithoutFile(); - } - - private ChronicleMap createWithoutFile() throws IOException { - VanillaChronicleMap map = newMap(); + /** + * Only for testing. + * + * @param identifier id + * @return map + */ + ChronicleMap createReplicated(byte identifier) throws IOException { + VanillaChronicleMap map = + new ReplicatedChronicleMap, + V, Object, MetaBytesWriter>(this, identifier); BytesStore bytesStore = new DirectStore(JDKObjectSerializer.INSTANCE, map.sizeInBytes(), true); map.createMappedStoreAndSegments(bytesStore); - return establishReplication(map); + return map; } - private ChronicleMap createStatelessMap(StatelessBuilder statelessBuilder) throws IOException { - preMapConstruction(); + private ChronicleMap createWithoutFile( + SimpleReplication simpleReplication, ReplicationChannel channel) throws IOException { + VanillaChronicleMap map = newMap(simpleReplication, channel); + BytesStore bytesStore = new DirectStore(JDKObjectSerializer.INSTANCE, + map.sizeInBytes(), true); + map.createMappedStoreAndSegments(bytesStore); + return establishReplication(map, simpleReplication, channel); + } + + ChronicleMap createStatelessMap(StatelessMapConfig statelessBuilder) + throws IOException { + preMapConstruction(false); return new StatelessChronicleMap( statelessBuilder, this); } - private VanillaChronicleMap newMap() throws IOException { - preMapConstruction(); - if (useReplication()) { + private VanillaChronicleMap newMap( + SimpleReplication simpleReplication, ReplicationChannel channel) throws IOException { + boolean replicated = simpleReplication != null || channel != null; + preMapConstruction(replicated); + if (replicated) { + byte identifier; + if (simpleReplication != null) { + identifier = simpleReplication.identifier(); + } else { + identifier = channel.hub().identifier(); + } return new ReplicatedChronicleMap, - V, Object, MetaBytesWriter>(this); + V, Object, MetaBytesWriter>(this, identifier); } else { return new VanillaChronicleMap, V, Object, MetaBytesWriter>(this); } } - private boolean useReplication() { - return identifier != -1; - } - - void preMapConstruction() { + void preMapConstruction(boolean replicated) { keyBuilder.objectSerializer(objectSerializer()); valueBuilder.objectSerializer(objectSerializer()); - long maxSize = (long) entrySize() * figureBufferAllocationFactor(); + long maxSize = (long) entrySize(replicated) * figureBufferAllocationFactor(); keyBuilder.maxSize(maxSize); valueBuilder.maxSize(maxSize); @@ -866,32 +871,37 @@ public abstract class AbstractChronicleMapBuilder establishReplication(VanillaChronicleMap map) - throws IOException { + private ChronicleMap establishReplication( + VanillaChronicleMap map, SimpleReplication simpleReplication, + ReplicationChannel channel) throws IOException { if (map instanceof ReplicatedChronicleMap) { + if (simpleReplication != null && channel != null) { + throw new AssertionError("Only one non-null replication should be passed"); + } ReplicatedChronicleMap result = (ReplicatedChronicleMap) map; - for (Replicator replicator : replicators.values()) { - if (replicator instanceof ChannelProvider.ChronicleChannel) { - ChannelProvider.ChronicleChannel channel = - (ChannelProvider.ChronicleChannel) replicator; - if (entrySize() > channel.provider().maxEntrySize()) { - throw new IllegalArgumentException("During ChannelProviderBuilder setup, " + - "maxEntrySize=" + channel.provider().maxEntrySize() + - " was specified, but map with " + - "entrySize=" + entrySize() + " is attempted to apply" + - "to the replicator"); - } + List replicators = new ArrayList<>(2); + if (simpleReplication != null) { + if (simpleReplication.tcpTransportAndNetwork() != null) + replicators.add(Replicators.tcp(simpleReplication.tcpTransportAndNetwork())); + if (simpleReplication.udpTransport() != null) + replicators.add(Replicators.udp(simpleReplication.udpTransport())); + } else { + ReplicationHub hub = channel.hub(); + int entrySize = entrySize(true); + if (entrySize > hub.maxEntrySize()) { + throw new IllegalArgumentException("During ChannelProviderBuilder setup, " + + "maxEntrySize=" + hub.maxEntrySize() + + " was specified, but map with " + + "entrySize=" + entrySize + " is attempted to apply to the replicator"); } + ChannelProvider provider = ChannelProvider.getProvider(hub); + ChannelProvider.ChronicleChannel ch = provider.createChannel(channel.channelId()); + replicators.add(ch); + } + for (Replicator replicator : replicators) { Closeable token = replicator.applyTo(this, result, result, map); if (replicators.size() == 1 && token.getClass() == UdpReplicator.class) { - LOG.warn( - "MISSING TCP REPLICATION : The UdpReplicator only attempts to read data " + - "(it does not enforce or guarantee delivery), you should use" + - "the UdpReplicator if you have a large number of nodes, and you wish" + - "to receive the data before it becomes available on TCP/IP. Since data" + - "delivery is not guaranteed, it is recommended that you only use" + - "the UDP Replicator in conjunction with a TCP Replicator" - ); + LOG.warn(Replicators.ONLY_UDP_WARN_MESSAGE); } result.addCloseable(token); } @@ -906,11 +916,7 @@ public abstract class AbstractChronicleMapBuilder return this; } - public ChronicleMapBuilder valueMarshaller( - @NotNull AgileBytesMarshaller valueMarshaller) { - valueBuilder.agileMarshaller(valueMarshaller, null); + public ChronicleMapBuilder valueMarshallers( + BytesWriter valueWriter, BytesReader valueReader) { + valueBuilder.writer(valueWriter); + valueBuilder.reader(valueReader, valueBuilder.factory()); return this; } } diff --git a/src/main/java/net/openhft/chronicle/map/IdentifierListener.java b/src/main/java/net/openhft/chronicle/map/IdentifierListener.java deleted file mode 100644 index d1b12f369..000000000 --- a/src/main/java/net/openhft/chronicle/map/IdentifierListener.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.map; - -import java.net.SocketAddress; - -/** - * @author Rob Austin. - */ -public interface IdentifierListener { - - /** - * checks the identifier that is unique and we haven't seen it before, unless it comes from the same port - * and host. - * - * @param remoteIdentifier remoteIdentifier - * @param remoteAddress remoteAddress - * @return true if unique - */ - boolean isIdentifierUnique(byte remoteIdentifier, SocketAddress remoteAddress); - -} diff --git a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java index 16bb25b95..97eca2983 100644 --- a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java +++ b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java @@ -18,8 +18,9 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; -import net.openhft.chronicle.hash.UdpReplicationConfig; +import net.openhft.chronicle.hash.replication.IdentifierListener; +import net.openhft.chronicle.hash.replication.TcpConfig; +import net.openhft.chronicle.hash.replication.UdpConfig; import net.openhft.lang.collection.ATSDirectBitSet; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.io.ByteBufferBytes; @@ -41,7 +42,7 @@ import java.util.concurrent.atomic.AtomicReference; import static java.nio.ByteBuffer.allocateDirect; import static java.util.concurrent.TimeUnit.SECONDS; -import static net.openhft.chronicle.hash.UdpReplicationConfig.simple; +import static net.openhft.chronicle.hash.replication.UdpConfig.simple; import static net.openhft.chronicle.map.ConcurrentExpiryMap.getDefaultAddress; import static net.openhft.chronicle.map.DiscoveryNodeBytesMarshallable.ProposedNodes; import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.BOOTSTRAP_BYTES; @@ -55,7 +56,7 @@ import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.LOG; public class NodeDiscovery { private final AddressAndPort ourAddressAndPort; - private final UdpReplicationConfig udpConfig; + private final UdpConfig udpConfig; private final DiscoveryNodeBytesMarshallable discoveryNodeBytesMarshallable; private final AtomicReference nodeDiscoveryEventListenerAtomicReference = new AtomicReference(); @@ -220,9 +221,11 @@ public class NodeDiscovery { // add our identifier and host:port to the list of known identifiers knownNodes.add(ourAddressAndPort, identifier); - final TcpReplicationConfig tcpConfig = TcpReplicationConfig - .of(ourAddressAndPort.getPort(), toInetSocketArray(knownHostPorts)) - .heartBeatInterval(1, SECONDS).nonUniqueIdentifierListener(identifierListener); + final TcpConfig tcpConfig = TcpConfig + .unknownTopology(ourAddressAndPort.getPort(), + toInetSocketCollection(knownHostPorts)) + .heartBeatInterval(1, SECONDS) + .nonUniqueIdentifierListener(identifierListener); LOG.info("Using Remote identifier=" + identifier); @@ -231,28 +234,26 @@ public class NodeDiscovery { return ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000L) - .replicators(identifier, tcpConfig).create(); + .replication(identifier, tcpConfig).create(); } - private InetSocketAddress[] toInetSocketArray(Set source) throws - UnknownHostException { + private static Collection toInetSocketCollection(Set source) + throws UnknownHostException { // make a safe copy final HashSet addressAndPorts = new HashSet(source); if (addressAndPorts.isEmpty()) - return new InetSocketAddress[0]; + return Collections.emptyList(); - final InetSocketAddress[] addresses = new InetSocketAddress[addressAndPorts.size()]; - - int i = 0; + ArrayList addresses = + new ArrayList(addressAndPorts.size()); for (final AddressAndPort addressAndPort : addressAndPorts) { - addresses[i++] = new InetSocketAddress(InetAddress.getByAddress(addressAndPort.address()) - .getHostAddress(), addressAndPort.port()); + addresses.add(new InetSocketAddress(InetAddress.getByAddress(addressAndPort.address()) + .getHostAddress(), addressAndPort.port())); } - return addresses; } @@ -359,7 +360,7 @@ class NodeDiscoveryBroadcaster extends UdpChannelReplicator { * @throws java.io.IOException */ NodeDiscoveryBroadcaster( - final UdpReplicationConfig replicationConfig, + final UdpConfig replicationConfig, final int serializedEntrySize, final BytesMarshallable externalizable) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/map/OffHeapUpdatableChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/OffHeapUpdatableChronicleMapBuilder.java index 0c7880724..2f906a84c 100644 --- a/src/main/java/net/openhft/chronicle/map/OffHeapUpdatableChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/OffHeapUpdatableChronicleMapBuilder.java @@ -25,6 +25,8 @@ public final class OffHeapUpdatableChronicleMapBuilder public static OffHeapUpdatableChronicleMapBuilder of( @NotNull Class keyClass, @NotNull Class valueClass) { + if (keyClass != CharSequence.class && keyClass.isInterface()) + keyClass = DataValueClasses.directClassFor(keyClass); if (!offHeapReference(valueClass)) { if (!valueClass.isInterface()) { throw new IllegalArgumentException( diff --git a/src/main/java/net/openhft/chronicle/map/Replica.java b/src/main/java/net/openhft/chronicle/map/Replica.java index b4664a1c5..1041f7322 100644 --- a/src/main/java/net/openhft/chronicle/map/Replica.java +++ b/src/main/java/net/openhft/chronicle/map/Replica.java @@ -26,7 +26,7 @@ import java.io.Closeable; /** * @author Rob Austin. */ -public interface Replica extends Closeable { +interface Replica extends Closeable { /** * Provides the unique Identifier associated with this map instance.

An identifier is used to diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 88abf77df..c865ed036 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -19,7 +19,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TimeProvider; +import net.openhft.chronicle.hash.replication.TimeProvider; import net.openhft.chronicle.hash.serialization.BytesReader; import net.openhft.chronicle.hash.serialization.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.MetaBytesWriter; @@ -82,7 +82,7 @@ import static net.openhft.lang.collection.DirectBitSet.NOT_FOUND; * @param the entries key type * @param the entries value type */ -class ReplicatedChronicleMap, +final class ReplicatedChronicleMap, V, VW, MVW extends MetaBytesWriter> extends VanillaChronicleMap implements ChronicleMap, @@ -102,12 +102,13 @@ class ReplicatedChronicleMap, private transient ModificationDelegator modificationDelegator; private transient long startOfModificationIterators; - public ReplicatedChronicleMap(@NotNull AbstractChronicleMapBuilder builder) + public ReplicatedChronicleMap(@NotNull AbstractChronicleMapBuilder builder, + byte identifier) throws IOException { super(builder); this.timeProvider = builder.timeProvider(); - this.localIdentifier = builder.identifier(); + this.localIdentifier = identifier; if (localIdentifier == -1) { throw new IllegalStateException("localIdentifier should not be -1"); @@ -632,8 +633,6 @@ class ReplicatedChronicleMap, V acquireWithoutLock(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V usingValue, long hash2, boolean create, long timestamp) { - - long keySize = metaKeyInterop.size(keyInterop, key); MultiStoreBytes entry = acquireTmpBytes(); long offset = searchKey(keyInterop, metaKeyInterop, key, keySize, hash2, entry, diff --git a/src/main/java/net/openhft/chronicle/map/Replicator.java b/src/main/java/net/openhft/chronicle/map/Replicator.java index 7d84ae262..7580d519d 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicator.java +++ b/src/main/java/net/openhft/chronicle/map/Replicator.java @@ -19,7 +19,7 @@ package net.openhft.chronicle.map; import java.io.Closeable; import java.io.IOException; -public abstract class Replicator { +abstract class Replicator { /** * A constructor for use in subclasses. diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java index 6b97d0ee6..b62683c44 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicators.java +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -16,8 +16,8 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; -import net.openhft.chronicle.hash.UdpReplicationConfig; +import net.openhft.chronicle.hash.replication.TcpConfig; +import net.openhft.chronicle.hash.replication.UdpConfig; import org.jetbrains.annotations.NotNull; import java.io.Closeable; @@ -25,10 +25,18 @@ import java.io.IOException; final class Replicators { + static final String ONLY_UDP_WARN_MESSAGE = + "MISSING TCP REPLICATION : The UdpReplicator only attempts to read data " + + "(it does not enforce or guarantee delivery), you should use" + + "the UdpReplicator if you have a large number of nodes, and you wish" + + "to receive the data before it becomes available on TCP/IP. Since data" + + "delivery is not guaranteed, it is recommended that you only use" + + "the UDP Replicator in conjunction with a TCP Replicator"; + private Replicators() { } - static Replicator tcp(final TcpReplicationConfig replicationConfig) { + static Replicator tcp(final TcpConfig replicationConfig) { return new Replicator() { @Override @@ -42,16 +50,17 @@ final class Replicators { .keyBuilder, builder.valueBuilder); StatelessServerConnector statelessServer = new StatelessServerConnector - (keyValueSerializer, (VanillaChronicleMap) chronicleMap, builder.entrySize()); + (keyValueSerializer, (VanillaChronicleMap) chronicleMap, + builder.entrySize(true)); return new TcpReplicator(replica, entryExternalizable, replicationConfig, - builder.entrySize(), statelessServer); + builder.entrySize(true), statelessServer); } }; } static Replicator udp( - final UdpReplicationConfig replicationConfig) { + final UdpConfig replicationConfig) { return new Replicator() { @Override @@ -61,7 +70,7 @@ final class Replicators { final ChronicleMap chronicleMap) throws IOException { return new UdpReplicator(map, entryExternalizable, replicationConfig, - builder.entrySize()); + builder.entrySize(true)); } }; } diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index 9d8b0a740..f69682bcf 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -83,18 +83,29 @@ final class SerializationBuilder implements Cloneable { NullObjectFactory.of(); if (concreteClass(eClass) && Byteable.class.isAssignableFrom(eClass)) { - agileMarshaller(ByteableMarshaller.of((Class) eClass), factory); + ByteableMarshaller agileBytesMarshaller = ByteableMarshaller.of((Class) eClass); + sizeMarshaller(agileBytesMarshaller) + .reader(agileBytesMarshaller, factory) + .interop(agileBytesMarshaller); } else if (eClass == CharSequence.class || eClass == String.class) { reader((BytesReader) CharSequenceReader.of(), factory); writer((BytesWriter) CharSequenceWriter.instance()); } else if (eClass == Void.class) { - agileMarshaller((AgileBytesMarshaller) VoidMarshaller.INSTANCE, factory); + sizeMarshaller(VoidMarshaller.INSTANCE) + .reader((BytesReader) VoidMarshaller.INSTANCE, factory) + .interop((BytesInterop) VoidMarshaller.INSTANCE); } else if (eClass == Long.class) { - agileMarshaller((AgileBytesMarshaller) LongMarshaller.INSTANCE, factory); + sizeMarshaller(LongMarshaller.INSTANCE) + .reader((BytesReader) LongMarshaller.INSTANCE, factory) + .interop((BytesInterop) LongMarshaller.INSTANCE); } else if (eClass == Double.class) { - agileMarshaller((AgileBytesMarshaller) DoubleMarshaller.INSTANCE, factory); + sizeMarshaller(DoubleMarshaller.INSTANCE) + .reader((BytesReader) DoubleMarshaller.INSTANCE, factory) + .interop((BytesInterop) DoubleMarshaller.INSTANCE); } else if (eClass == Integer.class) { - agileMarshaller((AgileBytesMarshaller) IntegerMarshaller.INSTANCE, factory); + sizeMarshaller(IntegerMarshaller.INSTANCE) + .reader((BytesReader) IntegerMarshaller.INSTANCE, factory) + .interop((BytesInterop) IntegerMarshaller.INSTANCE); } else if (eClass == byte[].class) { reader((BytesReader) ByteArrayMarshaller.INSTANCE, factory); interop((BytesInterop) ByteArrayMarshaller.INSTANCE); @@ -125,14 +136,6 @@ final class SerializationBuilder implements Cloneable { return this; } - public SerializationBuilder agileMarshaller(AgileBytesMarshaller agileBytesMarshaller, - ObjectFactory factory) { - if (factory == null) factory = this.factory; - return sizeMarshaller(agileBytesMarshaller) - .reader(agileBytesMarshaller, factory) - .interop(agileBytesMarshaller); - } - public SerializationBuilder interop(BytesInterop interop) { return copyingInterop(null) .setInterop(interop) @@ -142,6 +145,8 @@ final class SerializationBuilder implements Cloneable { } public SerializationBuilder writer(BytesWriter writer) { + if (writer instanceof BytesInterop) + return interop((BytesInterop) writer); return copyingInterop(CopyingInterop.FROM_WRITER) .setInterop(writer) .metaInterop(CopyingMetaBytesInterop @@ -287,10 +292,19 @@ final class SerializationBuilder implements Cloneable { " value don't use object factory"); } else if (interop instanceof ByteableMarshaller) { if (factory instanceof AllocateInstanceObjectFactory) { - agileMarshaller(ByteableMarshaller.of((Class) eClass), factory); + ByteableMarshaller agileBytesMarshaller = + ByteableMarshaller.of((Class) eClass); + sizeMarshaller(agileBytesMarshaller) + .reader(agileBytesMarshaller, factory) + .interop(agileBytesMarshaller); } else { - agileMarshaller(ByteableMarshaller.of((Class) eClass, (ObjectFactory) factory), - factory); + ByteableMarshaller agileBytesMarshaller = + ByteableMarshaller.of((Class) eClass, (ObjectFactory) factory); + ObjectFactory factory1 = factory; + if (factory1 == null) factory1 = this.factory; + sizeMarshaller(agileBytesMarshaller) + .reader(agileBytesMarshaller, factory1) + .interop(agileBytesMarshaller); } } else if (interop instanceof BytesMarshallableMarshaller) { if (factory instanceof AllocateInstanceObjectFactory) { diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index 531e118ac..c74f1bc1a 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -19,7 +19,6 @@ package net.openhft.chronicle.map; import com.sun.jdi.connect.spi.ClosedConnectionException; -import net.openhft.chronicle.hash.StatelessBuilder; import net.openhft.chronicle.hash.exceptions.IORuntimeException; import net.openhft.chronicle.hash.exceptions.TimeoutRuntimeException; import net.openhft.lang.io.ByteBufferBytes; @@ -61,14 +60,13 @@ class StatelessChronicleMap implements ChronicleMap, Closeable { private volatile SocketChannel clientChannel; private CloseablesManager closeables; - private final StatelessBuilder builder; + private final StatelessMapConfig builder; private int maxEntrySize; private final Class kClass; private final Class vClass; private final boolean putReturnsNull; private final boolean removeReturnsNull; - static enum EventId { HEARTBEAT, STATEFUL_UPDATE, @@ -92,20 +90,20 @@ class StatelessChronicleMap implements ChronicleMap, Closeable { TO_STRING, PUT_ALL, PUT_ALL_WITHOUT_ACC, - HASH_CODE + HASH_CODE, + ACQUIRE_USING } private long transactionID; - StatelessChronicleMap(final StatelessBuilder builder, + StatelessChronicleMap(final StatelessMapConfig builder, final AbstractChronicleMapBuilder chronicleMapBuilder) throws IOException { - this.builder = builder; this.keyValueSerializer = new KeyValueSerializer(chronicleMapBuilder.keyBuilder, chronicleMapBuilder.valueBuilder); this.putReturnsNull = chronicleMapBuilder.putReturnsNull(); this.removeReturnsNull = chronicleMapBuilder.removeReturnsNull(); - this.maxEntrySize = chronicleMapBuilder.entrySize(); + this.maxEntrySize = chronicleMapBuilder.entrySize(false); this.vClass = chronicleMapBuilder.valueBuilder.eClass; this.kClass = chronicleMapBuilder.keyBuilder.eClass; // this.putReturnsNull = putReturnsNull; @@ -395,6 +393,11 @@ class StatelessChronicleMap implements ChronicleMap, Closeable { } + /** + * @param key the key whose associated value is to be returned + * @param usingValue the object to read value data in, if possible + * @return + */ public synchronized V acquireUsing(K key, V usingValue) { throw new UnsupportedOperationException("acquireUsing() is not supported for stateless " + "clients"); diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 087e44180..d08f361d3 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -18,8 +18,9 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; -import net.openhft.chronicle.hash.ThrottlingConfig; +import net.openhft.chronicle.hash.replication.IdentifierListener; +import net.openhft.chronicle.hash.replication.TcpConfig; +import net.openhft.chronicle.hash.replication.ThrottlingConfig; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import net.openhft.lang.threadlocal.ThreadLocalCopies; @@ -51,12 +52,12 @@ interface Work { } /** - * Used with a {@link net.openhft.chronicle.map.ReplicatedChronicleMap} to send data between the maps using a socket - * connection {@link net.openhft.chronicle.map.TcpReplicator} + * Used with a {@link net.openhft.chronicle.map.ReplicatedChronicleMap} to send data between the + * maps using a socket connection {@link net.openhft.chronicle.map.TcpReplicator} * * @author Rob Austin. */ -public class TcpReplicator extends AbstractChannelReplicator implements Closeable { +final class TcpReplicator extends AbstractChannelReplicator implements Closeable { public static final int STATELESS_CLIENT = -127; public static final byte NOT_SET = (byte) HEARTBEAT.ordinal(); @@ -73,7 +74,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl private final byte localIdentifier; private final int maxEntrySizeBytes; private final Replica.EntryExternalizable externalizable; - private final TcpReplicationConfig replicationConfig; + private final TcpConfig replicationConfig; private final StatelessServerConnector statelessServerConnector; private long selectorTimeout; @@ -85,11 +86,11 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl * the entry, but setting it too large reduces the workable * space in the buffer. * @param statelessServerConnector set to NULL if not required - * @throws IOException on an io error. + * @throws IOException on an io error. */ public TcpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable externalizable, - @NotNull final TcpReplicationConfig replicationConfig, + @NotNull final TcpConfig replicationConfig, final int maxEntrySizeBytes, @Nullable StatelessServerConnector statelessServerConnector) throws IOException { @@ -278,7 +279,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl } /** - * check to see if we have lost connection with the remote node and if we have attempts a reconnect. + * check to see if we have lost connection with the remote node and if we have attempts a + * reconnect. * * @param key the key relating to the heartbeat that we are checking * @param approxTimeOutTime the approximate time in milliseconds @@ -425,8 +427,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl } /** - * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear all the old - * bootstrap information + * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear + * all the old bootstrap information * * @param key the nio SelectionKey */ @@ -438,7 +440,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl } /** - * used to exchange identifiers and timestamps and heartbeat intervals between the server and client + * used to exchange identifiers and timestamps and heartbeat intervals between the server and + * client * * @param key the SelectionKey relating to the this cha * @param socketChannel @@ -478,7 +481,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl // this can occur sometimes when if 2 or more remote node attempt to use node discovery at the // same time - final IdentifierListener identifierListener = replicationConfig.identifierListener; + final IdentifierListener identifierListener = + replicationConfig.nonUniqueIdentifierListener(); final SocketAddress remoteAddress = socketChannel.getRemoteAddress(); @@ -618,9 +622,10 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl } /** - * sets interestOps to "selector keys",The change to interestOps much be on the same thread as the selector. This - * class, allows via {@link AbstractChannelReplicator .KeyInterestUpdater#set(int)} to holds a pending change in - * interestOps ( via a bitset ), this change is processed later on the same thread as the selector + * sets interestOps to "selector keys",The change to interestOps much be on the same thread as + * the selector. This class, allows via {@link AbstractChannelReplicator + * .KeyInterestUpdater#set(int)} to holds a pending change in interestOps ( via a bitset ), + * this change is processed later on the same thread as the selector */ private static class KeyInterestUpdater { @@ -651,8 +656,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl } /** - * @param keyIndex the index of the key that has changed, the list of keys is provided by the constructor {@link - * KeyInterestUpdater(int, SelectionKey[])} + * @param keyIndex the index of the key that has changed, the list of keys is provided by + * the constructor {@link KeyInterestUpdater(int, SelectionKey[])} */ public void set(int keyIndex) { changeOfOpWriteRequired.set(keyIndex); @@ -893,7 +898,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl } /** - * writes all the entries that have changed, to the buffer which will later be written to TCP/IP + * writes all the entries that have changed, to the buffer which will later be written to + * TCP/IP * * @param modificationIterator a record of which entries have modification * @param selectionKey @@ -971,7 +977,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl } /** - * used to send an single zero byte if we have not send any data for up to the localHeartbeatInterval + * used to send an single zero byte if we have not send any data for up to the + * localHeartbeatInterval */ private void writeHeartbeatToBuffer() { @@ -988,8 +995,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl /** - * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will get added back - * in as soon as we have data to write + * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE + * will get added back in as soon as we have data to write * * @param socketChannel the socketChannel we wish to stop writing to * @param attached data associated with the socketChannels key @@ -1277,6 +1284,7 @@ class StatelessServerConnector { switch (event) { + case LONG_SIZE: return longSize(reader, writer, sizeLocation); @@ -1292,6 +1300,10 @@ class StatelessServerConnector { case GET: return get(reader, writer, sizeLocation); + case ACQUIRE_USING: + return acquireUsing(reader, writer, sizeLocation); + + case PUT: return put(reader, writer, sizeLocation); @@ -1476,6 +1488,26 @@ class StatelessServerConnector { return null; } + + private Work acquireUsing(Bytes reader, Bytes writer, final long sizeLocation) { + if (MAP_SUPPORTS_BYTES) { + map.acquireUsing(reader, writer); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K k = readKey(reader, local); + final V v = readValue(reader, local); + + try { + writeValue(map.acquireUsing(k, v), writer, local); + } catch (RuntimeException e) { + return sendException(writer, sizeLocation, e); + } + } + writeSizeAndFlags(sizeLocation, false, writer); + return null; + + } + private Work get(Bytes reader, Bytes writer, final long sizeLocation) { if (MAP_SUPPORTS_BYTES) { map.get(reader, writer); diff --git a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java index 337e48336..2896832d2 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.UdpReplicationConfig; +import net.openhft.chronicle.hash.replication.UdpConfig; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,7 +69,7 @@ class UdpChannelReplicator extends AbstractChannelReplicator implements Replica. * @param localIdentifier just used for logging * @throws IOException */ - UdpChannelReplicator(@NotNull final UdpReplicationConfig replicationConfig, + UdpChannelReplicator(@NotNull final UdpConfig replicationConfig, final int maxPayloadSize, final byte localIdentifier) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java index ed43075dd..504804982 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.UdpReplicationConfig; +import net.openhft.chronicle.hash.replication.UdpConfig; import net.openhft.lang.io.ByteBufferBytes; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -36,14 +36,14 @@ import java.nio.channels.DatagramChannel; * it becomes available on TCP/IP. In order to not miss data, UdpReplicator should be used in conjunction with * the TCP Replicator. */ -public class UdpReplicator extends UdpChannelReplicator implements Replica.ModificationNotifier, Closeable { +final class UdpReplicator extends UdpChannelReplicator implements Replica.ModificationNotifier, Closeable { private static final Logger LOG = LoggerFactory.getLogger(UdpReplicator.class.getName()); public UdpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable entryExternalizable, - @NotNull final UdpReplicationConfig replicationConfig, + @NotNull final UdpConfig replicationConfig, final int serializedEntrySize) throws IOException { diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index f827a5e8f..8bfbea60e 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -41,7 +41,6 @@ import java.nio.channels.FileChannel; import java.util.*; import java.util.concurrent.TimeUnit; -import static java.lang.Long.numberOfTrailingZeros; import static java.lang.Math.max; import static java.lang.Thread.currentThread; import static net.openhft.lang.MemoryUnit.*; @@ -129,8 +128,8 @@ class VanillaChronicleMap, lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); - // this.replicas = builder.replicas(); - this.entrySize = builder.entrySize(); + boolean replicated = getClass() == ReplicatedChronicleMap.class; + this.entrySize = builder.entrySize(replicated); this.alignment = builder.entryAndValueAlignment(); this.errorListener = builder.errorListener(); @@ -276,19 +275,7 @@ class VanillaChronicleMap, + sizeOfEntriesInSegment(); if ((ss & 63L) != 0) throw new AssertionError(); - return breakL1CacheAssociativityContention(ss); - } - - private long breakL1CacheAssociativityContention(long segmentSize) { - // Conventional alignment to break is 4096 (given Intel's 32KB 8-way L1 cache), - // for any case break 2 times smaller alignment - int alignmentToBreak = 2048; - int eachNthSegmentFallIntoTheSameSet = - max(1, alignmentToBreak >> numberOfTrailingZeros(segmentSize)); - if (eachNthSegmentFallIntoTheSameSet < actualSegments) { - segmentSize |= 64L; // make segment size "odd" (in cache lines) - } - return segmentSize; + return ss; } int multiMapsPerSegment() { @@ -597,6 +584,10 @@ class VanillaChronicleMap, throw new UnsupportedOperationException("todo"); } + public V acquireUsing(Bytes reader, Bytes writer) { + throw new UnsupportedOperationException("todo"); + } + void get(Bytes reader, Bytes writer) { throw new UnsupportedOperationException("todo"); } @@ -1415,8 +1406,7 @@ class VanillaChronicleMap, @Override public void removeEntry() { - if (copies != null) - removeWithoutLock(copies, metaKeyInterop, keyInterop, key(), null, segmentHash); + removeWithoutLock(copies, metaKeyInterop, keyInterop, key(), null, segmentHash); } } @@ -1430,9 +1420,7 @@ class VanillaChronicleMap, putWithoutLock(copies, metaKeyInterop, keyInterop, key(), value(), segmentHash, true); - copies = null; putOnClose = true; - Segment.this.writeUnlock(); } diff --git a/src/main/java/net/openhft/chronicle/map/WriteContext.java b/src/main/java/net/openhft/chronicle/map/WriteContext.java index 26cc84baf..ba5471903 100644 --- a/src/main/java/net/openhft/chronicle/map/WriteContext.java +++ b/src/main/java/net/openhft/chronicle/map/WriteContext.java @@ -21,10 +21,9 @@ package net.openhft.chronicle.map; * @author Rob Austin. */ public interface WriteContext extends Context { + void dontPutOnClose(); void removeEntry(); - // todo I have no idea why we have this, as this will always be done on when calling close() - // void update(); } diff --git a/src/main/java/net/openhft/chronicle/map/ZeroOutValueBytes.java b/src/main/java/net/openhft/chronicle/map/ZeroOutValueBytes.java index a6e2ce845..668e976a6 100644 --- a/src/main/java/net/openhft/chronicle/map/ZeroOutValueBytes.java +++ b/src/main/java/net/openhft/chronicle/map/ZeroOutValueBytes.java @@ -29,7 +29,7 @@ final class ZeroOutValueBytes implements PrepareValueBytes { @Override public void prepare(Bytes bytes, K key) { long pos = bytes.position(); - bytes.zeroOut(pos, pos + valueSize); + bytes.zeroOut(pos, pos + valueSize, true); bytes.skip(valueSize); } } diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index e4c6bf44e..fd607424c 100644 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -19,7 +19,9 @@ package net.openhft.chronicle.set; import net.openhft.chronicle.hash.*; -import net.openhft.chronicle.map.Alignment; +import net.openhft.chronicle.hash.replication.SimpleReplication; +import net.openhft.chronicle.hash.replication.TcpConfig; +import net.openhft.chronicle.hash.replication.TimeProvider; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.lang.io.serialization.BytesMarshaller; @@ -30,6 +32,7 @@ import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.concurrent.TimeUnit; /** @@ -50,7 +53,7 @@ public class ChronicleSetBuilder ChronicleSetBuilder(Class keyClass) { chronicleMapBuilder = ChronicleMapBuilder.of(keyClass, DummyValue.class) - .valueMarshaller(DummyValueMarshaller.INSTANCE); + .valueMarshallers(DummyValueMarshaller.INSTANCE, DummyValueMarshaller.INSTANCE); } public static ChronicleSetBuilder of(Class keyClass) { @@ -167,25 +170,6 @@ public class ChronicleSetBuilder return this; } - @Override - public ChronicleSetBuilder replicators(byte identifier, ReplicationConfig... replicators) { - chronicleMapBuilder.replicators(identifier, replicators); - return this; - } - - @Override - public ChronicleSetBuilder channel(ChannelProvider.ChronicleChannel chronicleChannel) { - chronicleMapBuilder.channel(chronicleChannel); - return this; - } - - @Override - public ChronicleSetBuilder disableReplication() { - chronicleMapBuilder.disableReplication(); - return this; - } - - @Override public String toString() { return " ChronicleSetBuilder{" + @@ -265,23 +249,39 @@ public class ChronicleSetBuilder } @Override - public ChronicleSetBuilder stateless(StatelessBuilder statelessBuilder) { - chronicleMapBuilder.stateless(statelessBuilder); + public ChronicleSetBuilder replication(SimpleReplication replication) { + chronicleMapBuilder.replication(replication); return this; } @Override - public ChronicleSetBuilder file(File file) throws IOException { - chronicleMapBuilder.file(file); + public ChronicleSetBuilder replication(byte identifier, TcpConfig tcpTransportAndNetwork) { + chronicleMapBuilder.replication(identifier, tcpTransportAndNetwork); return this; } + @Override + public StatelessClientConfig> statelessClient( + InetSocketAddress remoteAddress) { + return new StatelessSetConfig<>(chronicleMapBuilder.statelessClient(remoteAddress)); + } + + @Override + public ChronicleHashInstanceConfig> instance() { + return new InstanceConfig<>(chronicleMapBuilder.instance()); + } + @Override public ChronicleSet create() throws IOException { final ChronicleMap map = chronicleMapBuilder.create(); return new SetFromMap(map); } + @Override + public ChronicleSet createPersistedTo(File file) throws IOException { + ChronicleMap map = chronicleMapBuilder.createPersistedTo(file); + return new SetFromMap(map); + } } diff --git a/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java b/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java index 3ec1e79bb..ddb3956f7 100644 --- a/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java +++ b/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java @@ -18,12 +18,15 @@ package net.openhft.chronicle.set; -import net.openhft.chronicle.hash.serialization.AgileBytesMarshaller; +import net.openhft.chronicle.hash.serialization.BytesInterop; +import net.openhft.chronicle.hash.serialization.BytesReader; +import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.lang.io.Bytes; import static net.openhft.chronicle.set.DummyValue.DUMMY_VALUE; -enum DummyValueMarshaller implements AgileBytesMarshaller { +enum DummyValueMarshaller + implements BytesInterop, BytesReader, SizeMarshaller { INSTANCE; @Override @@ -42,7 +45,7 @@ enum DummyValueMarshaller implements AgileBytesMarshaller { } @Override - public DummyValue read(Bytes bytes, long size, DummyValue dummyValue) { + public DummyValue read(Bytes bytes, long size, DummyValue toReuse) { return DUMMY_VALUE; } diff --git a/src/test/java/eg/BigData.java b/src/test/java/eg/BigData.java index 745998a78..177b1059f 100644 --- a/src/test/java/eg/BigData.java +++ b/src/test/java/eg/BigData.java @@ -52,7 +52,7 @@ public class BigData { String chmPath = dir + "/testmap-" + Long.toString(System.nanoTime(), 36); new File(chmPath).deleteOnExit(); try { - theMap = builder.file(new File(chmPath)).create(); + theMap = builder.createPersistedTo(new File(chmPath)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); diff --git a/src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java b/src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java index befa4148a..fe6b10497 100644 --- a/src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java +++ b/src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java @@ -29,7 +29,9 @@ import static org.junit.Assert.fail; public class AcquireGetUsingMain { public static void main(String[] args) throws Exception { File file = new File(System.getProperty("java.io.tmpdir") + "/test1"); - ChronicleMap theSharedMap = OffHeapUpdatableChronicleMapBuilder.of(String.class, Data.class).file(file).create(); + ChronicleMap theSharedMap = + OffHeapUpdatableChronicleMapBuilder.of(String.class, Data.class) + .createPersistedTo(file); Data data = DataValueClasses.newDirectReference(Data.class); String processType = "testkey"; if (theSharedMap.getUsing(processType, data) == null) { diff --git a/src/test/java/net/openhft/chronicle/map/Builder.java b/src/test/java/net/openhft/chronicle/map/Builder.java index 1225a0fc9..41db8cced 100644 --- a/src/test/java/net/openhft/chronicle/map/Builder.java +++ b/src/test/java/net/openhft/chronicle/map/Builder.java @@ -19,12 +19,13 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.TcpConfig; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.net.InetSocketAddress; +import java.util.Arrays; import java.util.concurrent.TimeUnit; /** @@ -71,10 +72,10 @@ public class Builder { final byte identifier, final int serverPort, final InetSocketAddress... endpoints) throws IOException { - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints) + TcpConfig tcpConfig = TcpConfig.unknownTopology(serverPort, Arrays.asList(endpoints)) .heartBeatInterval(1L, TimeUnit.SECONDS); return ChronicleMapBuilder.of(kClass, vClass) - .replicators(identifier, tcpConfig); + .replication(identifier, tcpConfig); } diff --git a/src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java b/src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java index e6711b4ca..6649c1d86 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java +++ b/src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java @@ -79,7 +79,7 @@ public class CHMLatencyTestMain { OffHeapUpdatableChronicleMapBuilder.of(LongValue.class, LongValue.class) .entries(KEYS) .entrySize(16) - .createWithFile(file); + .createPersistedTo(file); // add keys LongValue key = DataValueClasses.newInstance(LongValue.class); diff --git a/src/test/java/net/openhft/chronicle/map/CHMTest5.java b/src/test/java/net/openhft/chronicle/map/CHMTest5.java index e631c45be..6979bc473 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMTest5.java +++ b/src/test/java/net/openhft/chronicle/map/CHMTest5.java @@ -44,7 +44,8 @@ public class CHMTest5 { String chmPath = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "CHMTest5"; - ChronicleMap theSharedMap = builder.file(new File(chmPath)).create(); + ChronicleMap theSharedMap = + builder.createPersistedTo(new File(chmPath)); //Now get the shared data object, auto-creating if it's not there CHMTest5Data data = DataValueClasses.newDirectReference(CHMTest5Data.class); diff --git a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java index c726da6eb..ed0b5cb2f 100644 --- a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java @@ -19,9 +19,8 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.ChannelProvider; -import net.openhft.chronicle.hash.ChannelProviderBuilder; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.ReplicationHub; +import net.openhft.chronicle.hash.replication.TcpConfig; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -47,44 +46,44 @@ public class ChannelReplicationTest { private ChronicleMap map1b; private ChronicleMap map2b; - private ChannelProvider channelProviderA; - private ChannelProvider channelProviderB; + private ReplicationHub hubA; + private ReplicationHub hubB; @Before public void setup() throws IOException { { - TcpReplicationConfig tcpConfig = TcpReplicationConfig - .of(8086, new InetSocketAddress("localhost", 8087)) + TcpConfig tcpConfig = TcpConfig + .forSendingNode(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - channelProviderA = new ChannelProviderBuilder() - .replicators((byte) 1, tcpConfig) - .create(); + hubA = ReplicationHub.builder() + .tcpTransportAndNetwork(tcpConfig) + .create((byte) 1); map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderA.createChannel((short) 1)).create(); + .instance().replicatedViaChannel(hubA.createChannel((short) 1)).create(); } { - TcpReplicationConfig tcpConfig = - TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); + TcpConfig tcpConfig = + TcpConfig.forReceivingOnlyNode(8087).heartBeatInterval(1, SECONDS); - channelProviderB = new ChannelProviderBuilder().replicators((byte) 2, tcpConfig) - .create(); + hubB = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig) + .create((byte) 2); map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderB.createChannel((short) 1)).create(); + .instance().replicatedViaChannel(hubB.createChannel((short) 1)).create(); } } @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{channelProviderA, channelProviderB}) { + for (final Closeable closeable : new Closeable[]{map1a, map1b, map2a, map2b}) { try { closeable.close(); } catch (IOException e) { @@ -99,12 +98,12 @@ public class ChannelReplicationTest { map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderB.createChannel((short) 2)).create(); + .instance().replicatedViaChannel(hubB.createChannel((short) 2)).create(); map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderA.createChannel((short) 2)).create(); + .instance().replicatedViaChannel(hubA.createChannel((short) 2)).create(); map2a.put(1, "EXAMPLE-2"); map1a.put(1, "EXAMPLE-1"); diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java index 091b097f8..294304c45 100644 --- a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java @@ -529,6 +529,79 @@ public class ChronicleMapTest { map.close(); } + @Test + @Ignore + public void testLeanEntrySizeInt() throws IOException { + File file = File.createTempFile("testLeanEntrySize-Int", ".deleteme"); + file.deleteOnExit(); + int entries = 1000000; + int entrySize = 8; + ChronicleMap map = OffHeapUpdatableChronicleMapBuilder + .of(IntValue.class, IntValue.class) + .entrySize(entrySize) + .entries(entries) + .create(); + IntValue key = DataValueClasses.newInstance(IntValue.class); + IntValue value = DataValueClasses.newDirectReference(IntValue.class); + for (int i = 0; i < entries; i++) { + key.setValue(i); + try (WriteContext wc = map.acquireUsingLocked(key, value)) { + value.setValue(i); + } + } + for (int i = 0; i < entries; i++) { + key.setValue(i); + try (ReadContext rc = map.getUsingLocked(key, value)) { + assertTrue(rc.present()); + assertEquals(i, value.getValue()); + } + } + map.close(); + + // check the extents of the map. + int hashSize = 8; // assume short-short hashing + int sizePerEntry = hashSize * 2 + entrySize * 5 / 4 /* + 25% */; + long fileSizeExpected = entries * sizePerEntry; + long length = file.length(); + assertTrue(length > fileSizeExpected); + assertTrue(length < fileSizeExpected * 12 / 10); + } + + @Test + @Ignore + public void testLeanEntrySizeLong() throws IOException { + File file = File.createTempFile("testLeanEntrySize-Long", ".deleteme"); + file.deleteOnExit(); + int entries = 1000000; + int entrySize = 16; + ChronicleMap map = OffHeapUpdatableChronicleMapBuilder + .of(LongValue.class, LongValue.class) + .entrySize(entrySize) + .entries(entries) + .create(); + LongValue key = DataValueClasses.newInstance(LongValue.class); + LongValue value = DataValueClasses.newDirectReference(LongValue.class); + for (long i = 0; i < entries; i++) { + key.setValue(i); + map.acquireUsing(key, value); + value.setValue(i); + } + for (long i = 0; i < entries; i++) { + key.setValue(i); + assertNotNull(map.getUsing(key, value)); + assertEquals(i, value.getValue()); + } + map.close(); + + // check the extents of the map. + int hashSize = 8; // assume short-short hashing + int sizePerEntry = hashSize * 2 + entrySize * 5 / 4 /* + 25% */; + long fileSizeExpected = entries * sizePerEntry; + long length = file.length(); + assertTrue(length > fileSizeExpected); + assertTrue(length < fileSizeExpected * 12 / 10); + } + @Test public void testAcquireFromMultipleThreads() throws Exception { int entries = 1000 * 1000; @@ -578,19 +651,19 @@ public class ChronicleMapTest { int procs = Runtime.getRuntime().availableProcessors(); int threads = procs * 2; // runs > 100 ? procs / 2 : procs; ExecutorService es = Executors.newFixedThreadPool(procs); - for (int runs : new int[]{10, 50, 250, 500, 1000, 2500}) { + for (int runs : new int[]{10, 25, 50, 250, 500, 1000, 2500}) { // JAVA 8 produces more garbage than previous versions for internal work. // System.gc(); final long entries = runs * 1000 * 1000L; - OffHeapUpdatableChronicleMapBuilder builder = OffHeapUpdatableChronicleMapBuilder - .of(CharSequence.class, LongValue.class) + OffHeapUpdatableChronicleMapBuilder builder = OffHeapUpdatableChronicleMapBuilder + .of(LongValue.class, LongValue.class) .entries(entries) .entryAndValueAlignment(OF_8_BYTES) - .entrySize(24); + .entrySize(16); File tmpFile = File.createTempFile("testAcquirePerf", ".deleteme"); tmpFile.deleteOnExit(); - final ChronicleMap map = builder.createWithFile(tmpFile); + final ChronicleMap map = builder.createPersistedTo(tmpFile); int count = runs > 500 ? runs > 1200 ? 3 : 5 : 5; final int independence = Math.min(procs, runs > 500 ? 8 : 4); @@ -603,16 +676,14 @@ public class ChronicleMapTest { futures.add(es.submit(new Runnable() { @Override public void run() { + LongValue key = DataValueClasses.newDirectInstance(LongValue.class); LongValue value = nativeLongValue(); - StringBuilder sb = new StringBuilder(); long next = 50 * 1000 * 1000; // use a factor to give up to 10 digit numbers. int factor = Math.max(1, (int) ((10 * 1000 * 1000 * 1000L - 1) / entries)); for (long j = t % independence; j < entries + independence - 1; j += independence) { - sb.setLength(0); - sb.append("us:"); - sb.append(j * factor); - map.acquireUsing(sb, value); + key.setValue(j * factor); + map.acquireUsing(key, value); long n = value.addAtomicValue(1); assert n > 0 && n < 1000 : "Counter corrupted " + n; if (t == 0 && j >= next) { @@ -639,6 +710,7 @@ public class ChronicleMapTest { es.shutdown(); es.awaitTermination(1, TimeUnit.MINUTES); } + @Test @Ignore public void testAcquireLockedPerf() @@ -660,7 +732,7 @@ public class ChronicleMapTest { File tmpFile = File.createTempFile("testAcquirePerf", ".deleteme"); tmpFile.deleteOnExit(); - final ChronicleMap map = builder.createWithFile(tmpFile); + final ChronicleMap map = builder.createPersistedTo(tmpFile); int count = runs > 500 ? runs > 1200 ? 3 : 5 : 5; final int independence = Math.min(procs, runs > 500 ? 8 : 4); @@ -683,7 +755,7 @@ public class ChronicleMapTest { sb.append("us:"); sb.append(j * factor); long n; - try(WriteContext wc = map.acquireUsingLocked(sb, value)) { + try (WriteContext wc = map.acquireUsingLocked(sb, value)) { n = value.addValue(1); } assert n > 0 && n < 1000 : "Counter corrupted " + n; @@ -1229,10 +1301,9 @@ public class ChronicleMapTest { .entries(10) .minSegments(1) .entrySize(10); - builder.file(getPersistenceFile()); testOversizeEntriesPutRemoveReplace( (VanillaChronicleMap) - builder.create() + builder.createPersistedTo(getPersistenceFile()) ); } @@ -1334,7 +1405,7 @@ public class ChronicleMapTest { .entrySize(16); File tmpFile = File.createTempFile("testAcquireUsingLocked", ".deleteme"); tmpFile.deleteOnExit(); - final ChronicleMap map = builder.createWithFile(tmpFile); + final ChronicleMap map = builder.createPersistedTo(tmpFile); LongValue value = nativeLongValue(); @@ -1391,7 +1462,7 @@ public class ChronicleMapTest { .entrySize(40); File tmpFile = File.createTempFile("testAcquireUsingLocked", ".deleteme"); tmpFile.deleteOnExit(); - final ChronicleMap map = builder.createWithFile(tmpFile); + final ChronicleMap map = builder.createPersistedTo(tmpFile); LongValue value = DataValueClasses.newInstance(LongValue.class); diff --git a/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java b/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java index 249b017ad..96180ebc8 100644 --- a/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java +++ b/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java @@ -39,7 +39,7 @@ public class DefaultValueTest { defaultValue.add(42); try (ChronicleMap> map = ChronicleMapBuilder .of(String.class, (Class>) ((Class) List.class)) - .defaultValue(defaultValue).file(file).create()) { + .defaultValue(defaultValue).createPersistedTo(file)) { assertEquals(defaultValue, map.acquireUsing("a", null)); assertEquals(1, map.size()); @@ -49,8 +49,8 @@ public class DefaultValueTest { } try (ChronicleMap> map = ChronicleMapBuilder - .of(String.class, (Class>) ((Class) List.class)).file(file). - create()) { + .of(String.class, (Class>) ((Class) List.class)) + .createPersistedTo(file)) { assertEquals(defaultValue, map.acquireUsing("c", null)); } } finally { diff --git a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java index c15ff7d53..d3cad018a 100644 --- a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java @@ -55,9 +55,8 @@ public class EntryCountMapTest { .entries(entries) .minSegments(segments) .entrySize(entrySize) - .entryAndValueAlignment(alignment) - .file(getPersistenceFile()); - return mapBuilder.create(); + .entryAndValueAlignment(alignment); + return mapBuilder.createPersistedTo(getPersistenceFile()); } @Test diff --git a/src/test/java/net/openhft/chronicle/map/OSResizesMain.java b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java index da15a2848..4298bd5cc 100644 --- a/src/test/java/net/openhft/chronicle/map/OSResizesMain.java +++ b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java @@ -29,15 +29,17 @@ import java.util.Arrays; /** * This example shows that the OS resizes the usage of a CHM as needed. It is not as critical to worry about this. *

- * System memory: 7.7 GB, Extents of map: 2199.0 GB, disk used: 13MB, addressRange: 7d380b7bd000-7f380c000000 + * System memory: 7.7 GB, Extents of map: 100000.0 GB, disk used: 104MB, addressRange: 24e02a7fc000-7fd33c000000 *

*/ public class OSResizesMain { public static void main(String[] args) throws IOException, InterruptedException { File file = File.createTempFile("over-sized", "deleteme"); ChronicleMap map = ChronicleMapBuilder.of(String.class, String.class) - .entrySize(100 * 1024 * 1024) - .entries(1000000).create(); + .entrySize(100 * 1000 * 1000) + .entries(1000 * 1000) + .actualSegments(1) + .createPersistedTo(file); for (int i = 0; i < 10000; i++) { char[] chars = new char[i]; Arrays.fill(chars, '+'); diff --git a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java index dcd9890bc..2b5f61eff 100644 --- a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java @@ -19,9 +19,8 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.ChannelProvider; -import net.openhft.chronicle.hash.ChannelProviderBuilder; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.ReplicationHub; +import net.openhft.chronicle.hash.replication.TcpConfig; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -48,8 +47,8 @@ public class PostChannelCreationTest { private ChronicleMap map1b; private ChronicleMap map2b; - private ChannelProvider channelProviderA; - private ChannelProvider channelProviderB; + private ReplicationHub hubA; + private ReplicationHub hubB; public static File getPersistenceFile() { String TMP = System.getProperty("java.io.tmpdir"); @@ -64,40 +63,40 @@ public class PostChannelCreationTest { public void setup() throws IOException { { - final TcpReplicationConfig tcpConfig = TcpReplicationConfig - .of(8086, new InetSocketAddress("localhost", 8087)) + final TcpConfig tcpConfig = TcpConfig + .forSendingNode(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); - channelProviderA = new ChannelProviderBuilder() - .replicators((byte) 1, tcpConfig).create(); + hubA = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig) + .create((byte) 1); // this is how you add maps after the custer is created map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderA.createChannel((short) 1)).create(); + .instance().replicatedViaChannel(hubA.createChannel((short) 1)).create(); map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderA.createChannel((short) 2)).create(); + .instance().replicatedViaChannel(hubA.createChannel((short) 2)).create(); } { - final TcpReplicationConfig tcpConfig = TcpReplicationConfig - .of(8087, new InetSocketAddress("localhost", 8086)) + final TcpConfig tcpConfig = TcpConfig + .forSendingNode(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); - channelProviderB = new ChannelProviderBuilder() - .replicators((byte) 2, tcpConfig).create(); + hubB = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig) + .create((byte) 2); // this is how you add maps after the custer is created map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderB.createChannel((short) 1)).create(); + .instance().replicatedViaChannel(hubB.createChannel((short) 1)).create(); map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderB.createChannel((short) 2)).create(); + .instance().replicatedViaChannel(hubB.createChannel((short) 2)).create(); } @@ -106,7 +105,7 @@ public class PostChannelCreationTest { @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{channelProviderA, channelProviderB}) { + for (final Closeable closeable : new Closeable[]{map1a, map1b, map2a, map2b}) { try { closeable.close(); } catch (IOException e) { diff --git a/src/test/java/net/openhft/chronicle/map/SerializerTest.java b/src/test/java/net/openhft/chronicle/map/SerializerTest.java index 5808d89b4..4bf4a05e7 100644 --- a/src/test/java/net/openhft/chronicle/map/SerializerTest.java +++ b/src/test/java/net/openhft/chronicle/map/SerializerTest.java @@ -73,7 +73,7 @@ public class SerializerTest { ByteBufferBytes in = out.slice(); ChronicleMapBuilder builder = ChronicleMapBuilder.of(Integer.class, valueClass); - builder.preMapConstruction(); + builder.preMapConstruction(false); Serializer v = new Serializer(builder.valueBuilder); @@ -96,7 +96,7 @@ public class SerializerTest { ByteBufferBytes in = out.slice(); ChronicleMapBuilder builder = ChronicleMapBuilder.of(clazz, Integer.class); - builder.preMapConstruction(); + builder.preMapConstruction(false); { Serializer v = new Serializer(builder.keyBuilder); diff --git a/src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java b/src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java index 8cbd1fe0b..1d48e9579 100644 --- a/src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java +++ b/src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java @@ -19,9 +19,8 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.ChannelProvider; -import net.openhft.chronicle.hash.ChannelProviderBuilder; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.ReplicationHub; +import net.openhft.chronicle.hash.replication.TcpConfig; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -46,8 +45,8 @@ public class SingleMapChannelTest { private ChronicleMap map1b; - private ChannelProvider channelProviderA; - private ChannelProvider channelProviderB; + private ReplicationHub hubA; + private ReplicationHub hubB; public static File getPersistenceFile() { String TMP = System.getProperty("java.io.tmpdir"); @@ -61,40 +60,41 @@ public class SingleMapChannelTest { public void setup() throws IOException { { - final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig - .of(8086, new InetSocketAddress("localhost", 8087)) + final TcpConfig tcpConfig = TcpConfig + .forSendingNode(8086, new InetSocketAddress("localhost", 8087)) .heartBeatInterval(1, SECONDS); int maxEntrySize = 1024; byte identifier= 1; - channelProviderA = new ChannelProviderBuilder() + hubA = ReplicationHub.builder() .maxEntrySize(maxEntrySize) - .replicators(identifier, tcpReplicationConfig).create(); + .tcpTransportAndNetwork(tcpConfig) + .create(identifier); // this is how you add maps after the custer is created map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .channel(channelProviderA.createChannel((short) 1)).create(); + .instance().replicatedViaChannel(hubA.createChannel((short) 1)).create(); } { - final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig - .of(8087, new InetSocketAddress("localhost", 8086)) + final TcpConfig tcpConfig = TcpConfig + .forSendingNode(8087, new InetSocketAddress("localhost", 8086)) .heartBeatInterval(1, SECONDS); - channelProviderB = new ChannelProviderBuilder() - .replicators((byte) 2, tcpReplicationConfig).create(); + hubB = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig).create((byte) 2); // this is how you add maps after the custer is created map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .channel(channelProviderB.createChannel((short) 1)) - .entries(1000).create(); + .entries(1000) + .instance().replicatedViaChannel(hubB.createChannel((short) 1)) + .create(); } } @After public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{channelProviderA, channelProviderB}) { + for (final Closeable closeable : new Closeable[]{map1a, map1b}) { try { closeable.close(); } catch (IOException e) { diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index e3495a017..35225b056 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.TcpConfig; import org.junit.Assert; import org.junit.Test; @@ -29,8 +29,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import static net.openhft.chronicle.hash.StatelessBuilder.remoteAddress; - /** * @author Rob Austin. @@ -41,7 +39,7 @@ public class StatelessClientTest { public static final int SIZE = 100000; - @Test + @Test(timeout = 5000) public void testBufferOverFlowPutAllAndEntrySet() throws IOException, InterruptedException { final ChronicleMap serverMap; @@ -52,13 +50,13 @@ public class StatelessClientTest { { statelessMap = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)).create(); } @@ -84,7 +82,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void testBufferOverFlowPutAllAndValues() throws IOException, InterruptedException { final ChronicleMap serverMap; @@ -95,13 +93,13 @@ public class StatelessClientTest { { statelessMap = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)).create(); } @@ -125,7 +123,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void testBufferOverFlowPutAllAndKeySet() throws IOException, InterruptedException { final ChronicleMap serverMap; @@ -136,13 +134,13 @@ public class StatelessClientTest { { statelessMap = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)).create(); } @@ -166,7 +164,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void test() throws IOException, InterruptedException { final ChronicleMap serverMap; @@ -175,7 +173,7 @@ public class StatelessClientTest { // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)).create(); serverMap.put(10, "EXAMPLE-10"); } @@ -185,7 +183,7 @@ public class StatelessClientTest { { statelessMap = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); Assert.assertEquals("EXAMPLE-10", statelessMap.get(10)); Assert.assertEquals(1, statelessMap.size()); @@ -197,7 +195,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void testClientCreatedBeforeServer() throws IOException, InterruptedException { final ChronicleMap serverMap; @@ -208,13 +206,13 @@ public class StatelessClientTest { { statelessMap = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)).create(); serverMap.put(10, "EXAMPLE-10"); } @@ -229,7 +227,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void testBufferOverFlowPutAll() throws IOException, InterruptedException { final ChronicleMap serverMap; @@ -240,13 +238,13 @@ public class StatelessClientTest { { statelessMap = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)).create(); } @@ -270,7 +268,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void testBufferOverFlowPutAllWherePutReturnsNull() throws IOException, InterruptedException { @@ -283,13 +281,13 @@ public class StatelessClientTest { statelessMap = ChronicleMapBuilder.of(Integer .class, CharSequence.class) .putReturnsNull(true) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)) + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)) .putReturnsNull(true) .create(); } @@ -315,7 +313,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void testPutWherePutReturnsNull() throws IOException, InterruptedException { @@ -329,13 +327,13 @@ public class StatelessClientTest { .class, CharSequence.class) .putReturnsNull(true) .removeReturnsNull(true) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)) + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)) .putReturnsNull(true) .removeReturnsNull(true) .create(); @@ -355,7 +353,7 @@ public class StatelessClientTest { - @Test + @Test(timeout = 5000) public void testRemoveWhereRemoveReturnsNull() throws IOException, InterruptedException { @@ -369,13 +367,13 @@ public class StatelessClientTest { .class, CharSequence.class) .putReturnsNull(true) .removeReturnsNull(true) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); } // server { serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)) + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)) .putReturnsNull(true) .removeReturnsNull(true) .create(); @@ -397,7 +395,7 @@ public class StatelessClientTest { } - @Test + @Test(timeout = 5000) public void testEquals() throws IOException, InterruptedException { final ChronicleMap serverMap1; @@ -409,18 +407,18 @@ public class StatelessClientTest { { statelessMap1 = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8086))).create(); + .statelessClient(new InetSocketAddress("localhost", 8086)).create(); statelessMap2 = ChronicleMapBuilder.of(Integer .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", 8077))).create(); + .statelessClient(new InetSocketAddress("localhost", 8077)).create(); } // server { serverMap1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8086)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8086)).create(); serverMap2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(8077)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8077)).create(); } diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java index 2eec55000..f59a03c59 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java @@ -53,18 +53,6 @@ public class TCPSocketReplication4WayMapTest { } - - static ChronicleMap newTcpSocketShmIntValueString( - final byte identifier, - final int serverPort, - final InetSocketAddress... endpoints) throws IOException { - return newTcpSocketShmBuilder(IntValue.class, CharSequence.class, - identifier, serverPort, endpoints) - .entries(20000L) - .keyMarshaller(ByteableIntValueMarshaller.INSTANCE).create(); - } - - @Before public void setup() throws IOException { diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java index 7eb7f7d7c..5dae68d57 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java @@ -51,7 +51,7 @@ public class TCPSocketReplicationBootStrapTests { newTcpSocketShmBuilder(Integer.class, CharSequence.class, (byte) 2, 8076, new InetSocketAddress("localhost", 8079)); final ChronicleMap map2a = - map2aBuilder.file(getPersistenceFile()).create(); + map2aBuilder.createPersistedTo(getPersistenceFile()); map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 long lastModificationTime; @@ -69,16 +69,14 @@ public class TCPSocketReplicationBootStrapTests { // restart map 2 but don't doConnect it to map one final ChronicleMap map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .file(map2File) - .create(); + .createPersistedTo(map2File); // add data into it map2b.put(11, "ADDED WHEN DISCONNECTED TO MAP1"); map2b.close(); } // now restart map2a and doConnect it to map1, map1 should bootstrap the missing entry - map2aBuilder.file(map2File); - map2 = map2aBuilder.create(); + map2 = map2aBuilder.createPersistedTo(map2File); // add data into it waitTillEqual(5000); @@ -96,7 +94,7 @@ public class TCPSocketReplicationBootStrapTests { final ReplicatedChronicleMap map2a = (ReplicatedChronicleMap) - map2aBuilder.file(getPersistenceFile()).create(); + map2aBuilder.createPersistedTo(getPersistenceFile()); map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 @@ -113,18 +111,16 @@ public class TCPSocketReplicationBootStrapTests { { // restart map 2 but don't doConnect it to map one - ChronicleMapBuilder.of(Integer.class, CharSequence.class).file(map2File); final ChronicleMap map2b = - ChronicleMapBuilder.of(Integer.class, CharSequence.class).file(map2File). - create(); + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .createPersistedTo(map2File); // add data into it map2b.put(11, "ADDED WHEN DISCONNECTED TO MAP1"); map2b.close(); } // now restart map2a and doConnect it to map1, map1 should bootstrap the missing entry - map2aBuilder.file(map2File); - map2 = map2aBuilder.create(); + map2 = map2aBuilder.createPersistedTo(map2File); // add data into it waitTillEqual(20000); diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java index 982d5ca78..edd1aa3c4 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java @@ -34,7 +34,6 @@ import java.util.Random; import static net.openhft.chronicle.map.Builder.getPersistenceFile; import static net.openhft.chronicle.map.Builder.newTcpSocketShmBuilder; -import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmIntValueString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -57,9 +56,14 @@ public class TCPSocketReplicationIntValueTest { ((Byteable) value).bytes(new ByteBufferBytes(ByteBuffer.allocateDirect(4)), 0); map1Builder = newTcpSocketShmBuilder(IntValue.class, CharSequence.class, (byte) 1, 8076, new InetSocketAddress("localhost", 8077)); - map1Builder.keyMarshaller(ByteableIntValueMarshaller.INSTANCE).file(getPersistenceFile()); - map1 = map1Builder.keyMarshaller(ByteableIntValueMarshaller.INSTANCE).create(); - map2 = newTcpSocketShmIntValueString((byte) 2, 8077); + map1Builder.keyMarshaller(ByteableIntValueMarshaller.INSTANCE); + map1 = map1Builder.keyMarshaller(ByteableIntValueMarshaller.INSTANCE) + .createPersistedTo(getPersistenceFile()); + map2 = newTcpSocketShmBuilder(IntValue.class, CharSequence.class, + (byte) 2, 8077) + .entries(20000L) + .keyMarshaller(ByteableIntValueMarshaller.INSTANCE) + .create(); } diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java index 7ba19246e..fac6062dd 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java @@ -50,8 +50,7 @@ public class TCPSocketReplicationTest { ChronicleMapBuilder map1Builder = newTcpSocketShmBuilder(Integer.class, CharSequence.class, (byte) 1, 8076, new InetSocketAddress("localhost", 8077)); - map1Builder.file(getPersistenceFile()); - map1 = map1Builder.create(); + map1 = map1Builder.createPersistedTo(getPersistenceFile()); map2 = newTcpSocketShmIntString((byte) 2, 8077); } diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java index 4a2f2a52e..23703da79 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.TcpConfig; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -26,6 +26,7 @@ import org.junit.Test; import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Arrays; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -47,11 +48,11 @@ public class TCPSocketReplicationTest3wayPutReturnsNull { final byte identifier, final int serverPort, final InetSocketAddress... endpoints) throws IOException { - TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints); + TcpConfig tcpConfig = TcpConfig.unknownTopology(serverPort, Arrays.asList(endpoints)); return (T) ChronicleMapBuilder.of(Integer.class, CharSequence.class) .putReturnsNull(true) - .replicators(identifier, tcpConfig).create(); + .replication(identifier, tcpConfig).create(); } diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java index c2656ddd5..4fcaf22c9 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java @@ -79,9 +79,8 @@ public class TCPSocketReplicationTestPostConnection { ChronicleMapBuilder map2aBuilder = newTcpSocketShmBuilder(Integer.class, CharSequence.class, (byte) 2, 8077, new InetSocketAddress("localhost", 8076)); - map2aBuilder.file(Builder.getPersistenceFile()); final ChronicleMap map2a = - map2aBuilder.create(); + map2aBuilder.createPersistedTo(Builder.getPersistenceFile()); map1 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 1, 8076); Thread.sleep(1); @@ -94,8 +93,7 @@ public class TCPSocketReplicationTestPostConnection { map1.put(6, "EXAMPLE-1"); // recreate map2 with new unique file - map2aBuilder.file(Builder.getPersistenceFile()); - map2 = map2aBuilder.create(); + map2 = map2aBuilder.createPersistedTo(Builder.getPersistenceFile()); // allow time for the recompilation to resolve diff --git a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java index f35862472..e26b077bd 100644 --- a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java +++ b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TimeProvider; +import net.openhft.chronicle.hash.replication.TimeProvider; import net.openhft.chronicle.map.jrs166.JSR166TestCase; import org.junit.Test; import org.mockito.Mockito; @@ -49,12 +49,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testIgnoreALatePut() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(10).replicators((byte) 1); ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -80,11 +78,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testIgnoreALatePutIfAbsent() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(10).replicators((byte) 1); ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -111,11 +108,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testIgnoreALateReplace() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(10).replicators((byte) 1); ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -145,14 +141,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testIgnoreALateReplaceWithValue() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(10).replicators((byte) 1); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).file(getPersistenceFile()); ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -179,13 +171,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testIgnoreALateRemoveWithValue() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class).replicators((byte) 1); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).file(getPersistenceFile()); ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -211,13 +200,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testIgnoreALateRemove() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class).replicators((byte) 1); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).file(getPersistenceFile()); ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -243,13 +229,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testIgnoreWithRemoteRemove() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class).replicators((byte) 1); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).file(getPersistenceFile()); ReplicatedChronicleMap map = (ReplicatedChronicleMap) ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -276,13 +259,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class).replicators((byte) 1); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).file(getPersistenceFile()); ReplicatedChronicleMap map = (ReplicatedChronicleMap) ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 1) - .timeProvider(timeProvider).create(); + .entries(10) + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -309,16 +289,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testRemoveFollowedByLatePut() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(10) - .replicators((byte) 1) - .entries(10) - .timeProvider(timeProvider).file(getPersistenceFile()); ReplicatedChronicleMap map = (ReplicatedChronicleMap) ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) .entries(10) - .replicators((byte) 1) - .entries(10) - .timeProvider(timeProvider).create(); + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); @@ -343,16 +317,10 @@ public class TimeBasedReplicationTests extends JSR166TestCase { public void testPutRemovePut() throws IOException { final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(10).replicators((byte) 1); - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(10) - .entries(10) - .timeProvider(timeProvider).file(getPersistenceFile()); ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) .entries(10) - .entries(10) - .timeProvider(timeProvider).create(); + .timeProvider(timeProvider) + .createReplicated((byte) 1); current(timeProvider); diff --git a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java index 6d64e8ae2..8bb712717 100644 --- a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java +++ b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.TcpConfig; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -30,7 +30,6 @@ import java.net.InetSocketAddress; import static java.util.concurrent.TimeUnit.SECONDS; import static net.openhft.chronicle.map.Builder.getPersistenceFile; -import static net.openhft.chronicle.hash.TcpReplicationConfig.of; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -46,19 +45,20 @@ public class TwoMapOnDifferentServers { @Before public void setup() throws IOException { - final TcpReplicationConfig tcpConfig = - of(8076, new InetSocketAddress("localhost", 8077)) + final TcpConfig tcpConfig = + TcpConfig.forSendingNode(8076, new InetSocketAddress("localhost", 8077)) .heartBeatInterval(1, SECONDS); File persistenceFile = getPersistenceFile(); map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000) - .replicators((byte) 1, tcpConfig).create(); + .replication((byte) 1, tcpConfig).create(); map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(20000) - .replicators((byte) 2, of(8077).heartBeatInterval(1, SECONDS)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(8077) + .heartBeatInterval(1, SECONDS)).create(); } @After diff --git a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java index 22887c0d8..1dc54598a 100644 --- a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java @@ -18,7 +18,8 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.UdpReplicationConfig; +import net.openhft.chronicle.hash.replication.SimpleReplication; +import net.openhft.chronicle.hash.replication.UdpConfig; import org.junit.After; import org.junit.Before; @@ -40,12 +41,13 @@ public class UDPSocketReplicationTest { final int identifier, final int udpPort) throws IOException { - UdpReplicationConfig udpConfig = UdpReplicationConfig + UdpConfig udpConfig = UdpConfig .simple(Inet4Address.getByName("255.255.255.255"), udpPort); return ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(1000) - .replicators((byte) identifier, udpConfig).create(); + .replication(SimpleReplication.builder().udpTransport(udpConfig) + .create((byte) identifier)).create(); } @Before diff --git a/src/test/java/net/openhft/chronicle/map/example/ChannelExample.java b/src/test/java/net/openhft/chronicle/map/example/ChannelExample.java index 404c74741..dbff0417d 100644 --- a/src/test/java/net/openhft/chronicle/map/example/ChannelExample.java +++ b/src/test/java/net/openhft/chronicle/map/example/ChannelExample.java @@ -1,8 +1,7 @@ package net.openhft.chronicle.map.example; -import net.openhft.chronicle.hash.ChannelProvider; -import net.openhft.chronicle.hash.ChannelProviderBuilder; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.ReplicationHub; +import net.openhft.chronicle.hash.replication.TcpConfig; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; import org.junit.Assert; @@ -11,6 +10,7 @@ import org.junit.Test; import java.io.IOException; import java.net.InetSocketAddress; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; /** @@ -54,8 +54,8 @@ public class ChannelExample { private ChronicleMap favoriteColourServer1; - private ChannelProvider channelProviderServer1; - private ChannelProvider channelProviderServer2; + private ReplicationHub hubOnServer1; + private ReplicationHub hubOnServer2; @Test @@ -63,32 +63,34 @@ public class ChannelExample { // server 1 with identifier = 1 { + ChronicleMapBuilder smallStringToStringMapBuilder = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(1000); + byte identifier = (byte) 1; - TcpReplicationConfig tcpConfig = TcpReplicationConfig - .of(8086, new InetSocketAddress("localhost", 8087)) - .heartBeatInterval(1, java.util.concurrent.TimeUnit.SECONDS); + TcpConfig tcpConfig = TcpConfig + .forSendingNode(8086, new InetSocketAddress("localhost", 8087)) + .heartBeatInterval(1, SECONDS); - channelProviderServer1 = new ChannelProviderBuilder() - .replicators(identifier, tcpConfig) - .create(); + hubOnServer1 = ReplicationHub.builder() + .tcpTransportAndNetwork(tcpConfig) + .create(identifier); // this demotes favoriteColour short channel1 = (short) 1; - favoriteColourServer1 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer1.createChannel(channel1)).create(); + favoriteColourServer1 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer1.createChannel(channel1)).create(); favoriteColourServer1.put("peter", "green"); // this demotes favoriteComputer short channel2 = (short) 2; - favoriteComputerServer1 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer1.createChannel(channel2)).create(); + favoriteComputerServer1 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer1.createChannel(channel2)).create(); favoriteComputerServer1.put("peter", "dell"); @@ -96,23 +98,25 @@ public class ChannelExample { // server 2 with identifier = 2 { + ChronicleMapBuilder smallStringToStringMapBuilder = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(1000); byte identifier = (byte) 2; - TcpReplicationConfig tcpConfig = - TcpReplicationConfig.of(8087).heartBeatInterval(1, java.util.concurrent.TimeUnit.SECONDS); + TcpConfig tcpConfig = + TcpConfig.forReceivingOnlyNode(8087).heartBeatInterval(1, SECONDS); - channelProviderServer2 = new ChannelProviderBuilder() - .replicators(identifier, tcpConfig) - .create(); + hubOnServer2 = ReplicationHub.builder() + .tcpTransportAndNetwork(tcpConfig) + .create(identifier); // this demotes favoriteColour short channel1 = (short) 1; - favoriteColourServer2 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer2.createChannel(channel1)).create(); + favoriteColourServer2 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer2.createChannel(channel1)).create(); favoriteColourServer2.put("rob", "blue"); @@ -120,9 +124,8 @@ public class ChannelExample { // this demotes favoriteComputer short channel2 = (short) 2; - favoriteComputerServer2 = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(1000) - .channel(channelProviderServer2.createChannel(channel2)).create(); + favoriteComputerServer2 = smallStringToStringMapBuilder.instance() + .replicatedViaChannel(hubOnServer2.createChannel(channel2)).create(); favoriteComputerServer2.put("rob", "mac"); favoriteComputerServer2.put("daniel", "mac"); diff --git a/src/test/java/net/openhft/chronicle/map/example/DistributedSequenceMain.java b/src/test/java/net/openhft/chronicle/map/example/DistributedSequenceMain.java index 562ad1a71..1a8d3b140 100644 --- a/src/test/java/net/openhft/chronicle/map/example/DistributedSequenceMain.java +++ b/src/test/java/net/openhft/chronicle/map/example/DistributedSequenceMain.java @@ -30,9 +30,6 @@ public class DistributedSequenceMain { public static void main(String... ignored) throws IOException { Class longValueClass = DataValueClasses.directClassFor(LongValue.class); - ChronicleMapBuilder.of(String.class, longValueClass) - .entries(128) - .actualSegments(1).file(new File("/tmp/counters")); ChronicleMap map = ChronicleMapBuilder.of(String.class, longValueClass) .entries(128) diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java index 35dc2aa58..e33723d2d 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java @@ -59,8 +59,7 @@ public class OpenJDKAndHashMapExamplesTest { ChronicleMap chm = OffHeapUpdatableChronicleMapBuilder .of(String.class, BondVOInterface.class) .keySize(10) - .file(file) - .create(); + .createPersistedTo(file); BondVOInterface bondVO = DataValueClasses.newDirectReference(BondVOInterface.class); @@ -80,8 +79,7 @@ public class OpenJDKAndHashMapExamplesTest { ChronicleMap chmB = OffHeapUpdatableChronicleMapBuilder .of(String.class, BondVOInterface.class) .keySize(10) - .file(file) - .create(); + .createPersistedTo(file); // ZERO Copy but creates a new off heap reference each time diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java index df78c1860..0d31aaade 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java @@ -90,14 +90,7 @@ public class PingPongCASLeft { static ChronicleMap acquireCHM() throws IOException { // ensure thread ids are globally unique. AffinitySupport.setThreadId(); - - String TMP = System.getProperty("java.io.tmpdir"); - ChronicleMapBuilder.of(String.class, BondVOInterface.class) - - .entries(16) - .entrySize(64).file(new File(TMP + "/BondPortfolioCHM")); return ChronicleMapBuilder.of(String.class, BondVOInterface.class) - .entries(16) .entrySize(64).create(); } diff --git a/src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java index 9abbd9dd4..7cc518e78 100644 --- a/src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java +++ b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java @@ -40,12 +40,7 @@ public class StateMachineTutorial { try { File dataFile = new File(System.getProperty("java.io.tmpdir"), "hft-state-machine"); - - ChronicleMapBuilder.of(Integer.class, StateMachineData.class) - - .entries(8).file(dataFile); map = ChronicleMapBuilder.of(Integer.class, StateMachineData.class) - .entries(8).create(); if (args.length > 0) { diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java index 7620fb4a3..c3665fa8c 100644 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java @@ -49,57 +49,33 @@ public class ChronicleMapTest extends JSR166TestCase { } static ChronicleMap newShmIntString(int size) throws IOException { - - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(size).file(getPersistenceFile()); return ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(size).create(); } static ChronicleMap newShmListBoolean(int size) throws IOException { - - ChronicleMapBuilder.of(ArrayList.class, CharSequence.class) - - .entries(size).file(getPersistenceFile()); return ChronicleMapBuilder.of(ArrayList.class, CharSequence.class) - .entries(size).create(); - } static ChronicleMap newShmListBoolean() throws IOException { - - ChronicleMapBuilder.of(ArrayList.class, CharSequence.class).file(getPersistenceFile()); return ChronicleMapBuilder.of(ArrayList.class, CharSequence.class).create(); - } static ChronicleMap newShmStringString(int size) throws IOException { - - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - - .entries(size).file(getPersistenceFile()); return ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .entries(size).create(); - } static ChronicleMap newShmIntString() throws IOException { - - ChronicleMapBuilder.of(Integer.class, CharSequence.class).file(getPersistenceFile()); return ChronicleMapBuilder.of(Integer.class, CharSequence.class).create(); - } static ChronicleMap newShmBiBoolean() throws IOException { - - ChronicleMapBuilder.of(BI.class, Boolean.class).file(getPersistenceFile()); return ChronicleMapBuilder.of(BI.class, Boolean.class).create(); - } /** diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java index 261d4d796..b076f8e07 100644 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java @@ -123,9 +123,6 @@ public class MapCheck { static Map newMap() { try { - -// return new ConcurrentHashMap(); - ChronicleMapBuilder.of(Object.class, Object.class).file(getPersistenceFile()); return ChronicleMapBuilder.of(Object.class, Object.class).create(); } catch (Exception e) { diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java index e66eef94e..1f19d83e3 100644 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map.jrs166.map; -import net.openhft.chronicle.hash.TcpReplicationConfig; +import net.openhft.chronicle.hash.replication.TcpConfig; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.chronicle.map.ReadContext; @@ -33,7 +33,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.*; -import static net.openhft.chronicle.hash.StatelessBuilder.remoteAddress; import static org.junit.Assert.*; /* @@ -184,12 +183,13 @@ public class StatelessChronicleMapTest extends JSR166TestCase { static ChronicleMap newShmIntString(int port) throws IOException { - final ChronicleMap serverMap = ChronicleMapBuilder.of(Integer.class, String.class) - .replicators((byte) 1, TcpReplicationConfig.of(port)).create(); + final ChronicleMap serverMap = + ChronicleMapBuilder.of(Integer.class, String.class) + .replication((byte) 1, TcpConfig.forReceivingOnlyNode(port)).create(); final ChronicleMap statelessMap = ChronicleMapBuilder.of(Integer .class, String.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", port))).create(); + .statelessClient(new InetSocketAddress("localhost", port)).create(); return new SingleCloseMap(statelessMap, statelessMap, serverMap); @@ -199,12 +199,12 @@ public class StatelessChronicleMapTest extends JSR166TestCase { IOException { final ChronicleMap serverMap = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .replicators((byte) 2, TcpReplicationConfig.of(port)).create(); + .replication((byte) 2, TcpConfig.forReceivingOnlyNode(port)).create(); final ChronicleMap statelessMap = ChronicleMapBuilder.of(CharSequence .class, CharSequence.class) - .stateless(remoteAddress(new InetSocketAddress("localhost", port))).create(); + .statelessClient(new InetSocketAddress("localhost", port)).create(); return new SingleCloseMap(statelessMap, statelessMap, serverMap); } diff --git a/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java b/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java index 4997e3c36..cecae189c 100644 --- a/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java +++ b/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java @@ -138,8 +138,7 @@ public class ProcessInstanceLimiter implements Runnable { ChronicleMapBuilder.of(String.class, Data.class); builder.entries(1000); builder.entrySize(1024); - builder.file(new File(sharedMapPath)); - this.theSharedMap = builder.create(); + this.theSharedMap = builder.createPersistedTo(new File(sharedMapPath)); Thread t = new Thread(this, "ProcessInstanceLimiter updater"); t.setDaemon(true); t.start(); diff --git a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java index efb1d7868..88709f59f 100644 --- a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java +++ b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplicationTest3way.java @@ -18,6 +18,7 @@ package net.openhft.chronicle.set; +import net.openhft.chronicle.hash.replication.TcpConfig; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -25,10 +26,11 @@ import org.junit.Test; import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Arrays; +import static java.util.Arrays.asList; import static java.util.concurrent.TimeUnit.SECONDS; import static net.openhft.chronicle.map.Builder.getPersistenceFile; -import static net.openhft.chronicle.hash.TcpReplicationConfig.of; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -51,10 +53,9 @@ public class SetForTCPSocketReplicationTest3way { final int serverPort, final InetSocketAddress... endpoints) throws IOException { return (T) ChronicleSetBuilder.of(Integer.class) - .file(getPersistenceFile()) - .replicators(identifier, of(serverPort, endpoints) + .replication(identifier, TcpConfig.unknownTopology(serverPort, asList(endpoints)) .heartBeatInterval(1L, SECONDS)) - .create(); + .createPersistedTo(getPersistenceFile()); }