From dd4c2f67d89152bdc6f14579ea22bc53a73a683b Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 15 Jan 2016 22:19:52 +0300 Subject: [PATCH] Remove replication frontend implementation in favor of Chronicle Engine --- README.md | 485 +----- .../chronicle/hash/ChronicleHashBuilder.java | 66 - .../hash/ChronicleHashInstanceBuilder.java | 148 -- .../ChronicleHashRecoveryFailedException.java | 6 +- .../hash/impl/VanillaChronicleHash.java | 1 - .../hash/replication/AbstractReplication.java | 205 --- .../hash/replication/ConnectionListener.java | 40 - .../hash/replication/RemoteNodeValidator.java | 36 - .../hash/replication/ReplicationChannel.java | 65 - .../hash/replication/ReplicationHub.java | 157 -- .../SingleChronicleHashReplication.java | 48 - .../replication/TcpInetSocketAddress.java | 35 - .../TcpTransportAndNetworkConfig.java | 198 --- .../hash/replication/ThrottlingConfig.java | 166 -- .../hash/replication/UdpTransportConfig.java | 161 -- .../map/AbstractChannelReplicator.java | 579 ------- .../chronicle/map/ChannelProvider.java | 620 ------- .../chronicle/map/ChronicleMapBuilder.java | 142 +- .../chronicle/map/MapInstanceBuilder.java | 109 -- .../net/openhft/chronicle/map/Replica.java | 45 +- .../chronicle/map/ReplicatedChronicleMap.java | 85 +- .../net/openhft/chronicle/map/Replicator.java | 70 - .../openhft/chronicle/map/TcpReplicator.java | 1437 ----------------- .../chronicle/map/UdpChannelReplicator.java | 335 ---- .../openhft/chronicle/map/UdpReplicator.java | 183 --- .../chronicle/set/ChronicleSetBuilder.java | 21 - .../chronicle/set/SetInstanceBuilder.java | 80 - .../java/eg/TestNesterInterfaceMarshal.java | 147 -- .../openhft/chronicle/TestReplication.java | 86 - .../chronicle/map/CHMUseCasesTest.java | 93 +- .../chronicle/map/ChannelReplicationTest.java | 182 --- ...dMapWithoutConfiguringReplicatorsTest.java | 116 -- ...tListenerWithTCPSocketReplicationTest.java | 309 ---- .../openhft/chronicle/map/Issue37Test.java | 68 - .../chronicle/map/Issue62ChronicleClient.java | 5 +- .../chronicle/map/Issue62ChronicleServer.java | 7 +- .../map/OldDeletedEntriesCleanupTest.java | 198 --- .../map/PostChannelCreationTest.java | 156 -- ...ateProxyMapUpdatedViaSharedMemoryTest.java | 80 - .../chronicle/map/SingleMapChannelTest.java | 127 -- ...cketReplication3wayPutReturnsNullTest.java | 167 -- .../map/TCPSocketReplication3wayTest.java | 166 -- .../map/TCPSocketReplication4WayMapTest.java | 164 -- ...SocketReplicationAfterNodeFailureTest.java | 185 --- .../TCPSocketReplicationBootStrapTest.java | 207 --- .../map/TCPSocketReplicationIntValueTest.java | 192 --- .../map/TCPSocketReplicationNativeTest.java | 142 -- ...CPSocketReplicationPostConnectionTest.java | 149 -- .../map/TCPSocketReplicationTest.java | 236 --- .../map/TcpReplicationLargeEntryTest.java | 168 -- .../map/TcpReplicationSoakCrdtTest.java | 299 ---- .../chronicle/map/TcpReplicationSoakTest.java | 207 --- .../TcpReplicationWithAcquireContextTest.java | 200 --- .../map/TcpTimeBasedReplicationSoakTest.java | 149 -- ...esSpanningSeveralChunksReplicationHub.java | 408 ----- .../chronicle/map/TestReplication.java | 127 -- .../map/TimeBasedReplicationTest.java | 233 --- .../map/TwoMapOnDifferentServersTest.java | 115 -- .../map/UDPSocketReplicationTest.java | 89 - .../map/example/ChannelReplicationTest.java | 348 ---- .../chronicle/map/example/ServersMapMain.java | 96 -- .../chronicle/map/example/StringsMapMain.java | 100 -- .../net/openhft/chronicle/set/Builder.java | 33 - .../SetForTCPSocketReplication3wayTest.java | 148 -- .../set/SetRemoteOperationsTest.java | 239 --- .../TCPSocketReplication3VoidValueTest.java | 145 -- 66 files changed, 99 insertions(+), 11710 deletions(-) delete mode 100644 src/main/java/net/openhft/chronicle/hash/ChronicleHashInstanceBuilder.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/AbstractReplication.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/ConnectionListener.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/RemoteNodeValidator.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/ReplicationChannel.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/ReplicationHub.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/SingleChronicleHashReplication.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/TcpInetSocketAddress.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/ThrottlingConfig.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/replication/UdpTransportConfig.java delete mode 100644 src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java delete mode 100644 src/main/java/net/openhft/chronicle/map/ChannelProvider.java delete mode 100644 src/main/java/net/openhft/chronicle/map/MapInstanceBuilder.java delete mode 100644 src/main/java/net/openhft/chronicle/map/Replicator.java delete mode 100644 src/main/java/net/openhft/chronicle/map/TcpReplicator.java delete mode 100644 src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java delete mode 100644 src/main/java/net/openhft/chronicle/map/UdpReplicator.java delete mode 100644 src/main/java/net/openhft/chronicle/set/SetInstanceBuilder.java delete mode 100644 src/test/java/eg/TestNesterInterfaceMarshal.java delete mode 100755 src/test/java/net/openhft/chronicle/TestReplication.java delete mode 100644 src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/DeserializeReplicatedMapWithoutConfiguringReplicatorsTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/Issue37Test.java delete mode 100644 src/test/java/net/openhft/chronicle/map/OldDeletedEntriesCleanupTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/ReplicateProxyMapUpdatedViaSharedMemoryTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java delete mode 100755 src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayPutReturnsNullTest.java delete mode 100755 src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationAfterNodeFailureTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationNativeTest.java delete mode 100755 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationPostConnectionTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TcpReplicationLargeEntryTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TcpReplicationSoakCrdtTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TcpReplicationSoakTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireContextTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TcpTimeBasedReplicationSoakTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TestEntriesSpanningSeveralChunksReplicationHub.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TestReplication.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServersTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/example/ChannelReplicationTest.java delete mode 100755 src/test/java/net/openhft/chronicle/map/example/ServersMapMain.java delete mode 100755 src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java delete mode 100644 src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplication3wayTest.java delete mode 100644 src/test/java/net/openhft/chronicle/set/SetRemoteOperationsTest.java delete mode 100644 src/test/java/net/openhft/chronicle/set/TCPSocketReplication3VoidValueTest.java diff --git a/README.md b/README.md index 74cd01efc..f64f01adc 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,16 @@ Chronicle Map doesn't support sorted. - LRU entry eviction -Not supported out of the box in open-source version, but could be added in ad-hoc manner: - - Partially-redundant replication, i. e. distributed/cluster solution on top of Chronicle Map - - Synchronous replication - - Entry expiration timeouts - -[We could help to implement such things](http://chronicle.software/consultancy/). +### Feature matrix +| | Chronicle Map | [Chronicle Engine](https://github.com/OpenHFT/Chronicle-Engine) | [On-demand](http://chronicle.software/consultancy/) | +| --- |:---:| :---:| :---: | +| In-memory off-heap Map | ✔ | | | +| Persistence to disk | ✔ | | | +| Remote calls | | ✔ | | +| Eventually-consistent replication (100% redundancy) | | ✔ | | +| Partially-redundant replication | | | ✔ | +| Synchronous replication | | | ✔ | +| Entry expiration timeouts | | | ✔ | ### Peer projects - [Chronicle Engine](https://github.com/OpenHFT/Chronicle-Engine) - reactive processing framework @@ -116,16 +120,9 @@ Not supported out of the box in open-source version, but could be added in ad-ho - [Single-key queries](#single-key-queries) - [Multi-key queries](#multi-key-queries) - [Close `ChronicleMap`](#close-chroniclemap) - - [TCP / UDP Replication](#tcp--udp-replication) - - [Multiple Processes on the same server with Replication]( - #multiple-processes-on-the-same-server-with-replication) - - [Configuring Three Way TCP/IP Replication](#configuring-three-way-tcp--ip-replication) - - [Multi Chronicle Maps - Network Distributed](#multiple-chronicle-maps---network-distributed) - [Behaviour Customization](#behaviour-customization) - [Example - Simple logging](#example---simple-logging) - [Example - BiMap](#example---bimap) - - [Example - CRDT values for replicated Chronicle Maps - Grow-only set]( - #example---crdt-values-for-replicated-chronicle-maps---grow-only-set) - [Example - Monitor Chronicle Map statistics](#example---monitor-chronicle-map-statistics) ### Difference between Chronicle Map 2 and 3 @@ -137,8 +134,9 @@ Functional changes in Chronicle Map 3: section. This has a number of important consequences, most notable is: - Possibility to define replication eventual-consistency strategy, different from "last write wins", e. g. any state-based CRDT. - - "Stateless clients" removed, access Chronicle Maps from remote servers via [Chronicle Engine]( + - "Stateless clients" functionality (i. e. remote calls) is moved to [Chronicle Engine]( https://github.com/OpenHFT/Chronicle-Engine). + - Replication is done via [Chronicle Engine](https://github.com/OpenHFT/Chronicle-Engine). - Chronicle Map 2 has hard creation-time limit on the number of entries storable in the Chronicle Map instance. If the size exceeds this limit, an exception is thrown. In Chronicle Map 3, this limitation is removed, though the number of entries still has to be configured on the Chronicle Map @@ -641,386 +639,6 @@ its not something that you must do, it's just something that we recommend you sh If you call close() too early before you have finished working with the map, this can cause your JVM to crash. Close MUST BE the last thing that you do with the map. - -### TCP / UDP Replication - -ChronicleMap supports both TCP and UDP replication - -![TCP/IP Replication](http://chronicle.software/wp-content/uploads/2014/07/Chronicle-Map-TCP-Replication_simple_02.jpg) - -#### TCP / UDP Background. -TCP/IP is a reliable protocol, what this means is that unless you have a network failure or hardware -outage the data is guaranteed to arrive. TCP/IP provides point to point connectivity. So in effect -( over simplified ), if the message was sent to 100 hosts, the message would have to be sent -100 times. With UDP, the message is only sent once. This is ideal if you have a large number of -hosts and you wish to broadcast the same data to each of them. However, one of the big drawbacks -with UDP is that it's not a reliable protocol. This means, if the UDP message is Broadcast onto -the network, the hosts are not guaranteed to receive it, so they can miss data. Some solutions -attempt to build resilience into UDP, but arguably, this is in effect reinventing TCP/IP. - -#### How to setup UDP Replication -In reality on a good quality wired LAN, when using UDP, you will rarely miss messages. Nevertheless this is -a risk that we suggest you don't take. We suggest that whenever you use UDP replication you use it -in conjunction with a throttled TCP replication, therefore if a host misses a message over UDP, they -will later pick it up via TCP/IP. - -#### TCP/IP Throttling -We are careful not to swamp your network with too much TCP/IP traffic, We do this by providing -a throttled version of TCP replication. This works because ChronicleMap only broadcasts the latest -update of each entry. - -#### Replication How it works - -ChronicleMap provides multi master hash map replication. What this means, is that each remote -map, mirrors its changes over to another remote map, neither map is considered -the master store of data. Each map uses timestamps to reconcile changes. -We refer to in instance of a remote map as a node. -A node can be connected to up to 128 other nodes. -The data that is stored locally in each node becomes eventually consistent. So changes made to one -node, for example by calling put(), will be replicated over to the other node. To achieve a high -level of performance and throughput, the call to put() won’t block, -With ConcurrentHashMap, It is typical to check the return code of some methods to obtain the old -value, for example remove(). Due to the loose coupling and lock free nature of this multi master -implementation, this return value is only the old value on the nodes local data store. In other -words the nodes are only concurrent locally. Its worth realising that another node performing -exactly the same operation may return a different value. However reconciliation will ensure the maps -themselves become eventually consistent. - -#### Reconciliation -If two ( or more nodes ) receive a change to their maps for the same key but different values, say -by a user of the maps, calling the put(key,value), then, initially each node will update its local -store and each local store will hold a different value. The aim of multi master replication is -to provide eventual consistency across the nodes. So, with multi master whenever a node is changed -it will notify the other nodes of its change. We will refer to this notification as an event. -The event will hold a timestamp indicating the time the change occurred, it will also hold the state -transition, in this case it was a put with a key and value. -Eventual consistency is achieved by looking at the timestamp from the remote node, if for a given -key, the remote nodes timestamp is newer than the local nodes timestamp, then the event from -the remote node will be applied to the local node, otherwise the event will be ignored. Since -none of the nodes is a primary, each node holds information about the other nodes. For this node its -own identifier is referred to as its 'localIdentifier', the identifiers of other nodes are the -'remoteIdentifiers'. On an update or insert of a key/value, this node pushes the information of -the change to the remote nodes. The nodes use non-blocking java NIO I/O and all replication is done -on a single thread. However there is an edge case. If two nodes update their map at the same time -with different values, we have to deterministically resolve which update wins. This is because eventual -consistency mandates that both nodes should end up locally holding the same data. Although it is rare that two remote -nodes receive an update to their maps at exactly the same time for the same key, we have -to handle this edge case. We can not therefore rely on timestamps alone to reconcile -the updates. Typically the update with the newest timestamp should win, but in this example both -timestamps are the same, and the decision made to one node should be identical to the decision made -to the other. This dilemma is resolved by using a node identifier, the node identifier is a unique -'byte' value that is assigned to each node. When the time stamps are the same the remote node with the -smaller identifier will be preferred. - -#### Multiple Processes on the same server with Replication - -On a server if you have a number of Java processes and then within each Java process you create an instance of a ChronicleMap which binds to the same underline 'file', they exchange data via shared memory rather than TCP or UDP replication. So if a ChronicleMap which is not performing TCP Replication is updated, this update can be picked up by another ChronicleMap. This other ChronicleMap could be a TCP replicated ChronicleMap. In such an example the TCP replicated ChronicleMap would then push the update to the remote nodes. - -Likewise, if the TCP replicated ChronicleMap was to received an update from a remote node, then this update would be immediately available to all the ChronicleMaps on the server. - -### Identifier for Replication -If all you are doing is replicating your ChronicleMaps on the same server you don't have to set up -TCP and UDP replication. You also don't have to set the identifiers - as explained earlier this identifier is only for the resolution of conflicts amongst remote servers. - -If however you wish to replicate data between 2 or more servers, then ALL of the ChronicleMaps -including those not actively participating in TCP or UDP replication must have the identifier set. -The identifier must be unique to each server. Each ChronicleMap on the same server must have -the same identifier. The reason that all ChronicleMaps must have the identifier set, is because -the memory is laid out slightly differently when using replication, so even if a Map is not actively -performing TCP or UDP replication itself, if it wishes to replicate with one that is, it must have -its memory laid out the same way to be compatible. - -If the identifiers are not set up uniquely then the updates will be ignored, as for example -a ChronicleMap set up with the identifiers equals '1', will ignore all events which contain -the remote identifier of '1', in other words Chronicle Map replication is set up to ignore updates -which have originated from itself. This is to avoid the circularity of events. - -When setting up the identifier you can use values from 1 to 127. ( see the section above for more -information on identifiers and how they are used in replication. ) - -The identifier is setup on the builder as follows. - -```java -TcpTransportAndNetworkConfig tcpConfig = ... -map = ChronicleMapBuilder - .of(IntValue.class, CharSequence.class) - .averageValue(averageValue) - .entries(1_000_000) - .replication(identifier, tcpConfig) - .create(); -``` - - -#### Bootstrapping -When a node is connected over the network to an active grid of nodes. It must first receive any data -that it does not have from the other nodes. Eventually, all the nodes in the grid have to hold a -copy of exactly the same data. We refer to this initial data load phase as bootstrapping. -Bootstrapping by its very nature is point to point, so it is only performed over TCP replication. -For architectures that wish to use UDP replication it is advised you use TCP Replication as well. A -grid which only uses UDP replication will miss out on the bootstrapping, possibly leaving the nodes -in an inconsistent state. To avoid this, if you would rather reduce the amount of TCP traffic on -your network, we suggest you consider using a throttle TCP replication along with UDP replication. -Bootstrapping is not used when the nodes are on the same server, so for this case, TCP replication -is not required. - -#### Identifier -Each map is allocated a unique identifier - -Server 1 has: -```java -.replication((byte) 1, tcpConfigServer1) -``` - -Server 2 has: -```java -.replication((byte) 2, tcpConfigServer2) -``` - -Server 3 has: -```java -.replication((byte) 3, tcpConfigServer3) -``` - -If you fail to allocate a unique identifier replication will not work correctly. - -#### Port -Each map must be allocated a unique port, the port has to be unique per server, if the maps are -running on different hosts they could be allocated the same port, but in our example we allocated -them different ports, we allocated map1 port 8076 and map2 port 8077. Currently we don't support -data forwarding, so it important to connect every remote map, to every other remote map, in other -words you can't have a hub configuration where all the data passes through a single map which every -other map is connected to. So currently, if you had 4 servers each with a Chronicle Map, you would -require 6 connections. - -In our case we are only using 2 maps, this is how we connected map1 to map 2. -```java -TcpTransportAndNetworkConfig.of(8076, new InetSocketAddress("localhost", 8077)) - .heartBeatInterval(1, SECONDS); -``` -you could have put this instruction on map2 instead, like this -```java -TcpTransportAndNetworkConfig.of(8077, new InetSocketAddress("localhost", 8076)) - .heartBeatInterval(1, SECONDS); -``` -even though data flows from map1 to map2 and map2 to map1 it doesn't matter which way you connected -this, in other words its a bidirectional connection. - -#### Configuring Three Way TCP/IP Replication - -![TCP/IP Replication 3Way](http://chronicle.software/wp-content/uploads/2014/09/Screen-Shot-2014-10-27-at-18.19.05.png) - -Below is example how to set up tcpConfig for 3 host - -```java -String hostServer1 = "localhost"; // change this to your host -int serverPort1 = 8076; // change this to your port -InetSocketAddress inetSocketAddress1 = new InetSocketAddress(hostServer1, serverPort1); - -String hostServer2 = "localhost"; // change this to your host -int serverPort2= 8077; // change this to your port -InetSocketAddress inetSocketAddress2 = new InetSocketAddress(hostServer2, serverPort2); - -String hostServer3 = "localhost"; // change this to your host -int serverPort3 = 8078; // change this to your port -InetSocketAddress inetSocketAddress3 = new InetSocketAddress(hostServer3, serverPort3); - -// this is to go on server 1 -TcpTransportAndNetworkConfig tcpConfigServer1 = - TcpTransportAndNetworkConfig.of(serverPort1); - -// this is to go on server 2 -TcpTransportAndNetworkConfig tcpConfigServer2 = TcpTransportAndNetworkConfig - .of(serverPort2, inetSocketAddress1); - -// this is to go on server 3 -TcpTransportAndNetworkConfig tcpConfigServer3 = TcpTransportAndNetworkConfig - .of(serverPort3, inetSocketAddress1, inetSocketAddress2); -``` - -#### Heart Beat Interval -We set a heartBeatInterval, in our example to 1 second -``` java - heartBeatInterval(1, SECONDS) -``` -A heartbeat will only be send if no data is transmitted, if the maps are constantly exchanging data -no heartbeat message is sent. If a map does not receive either data of a heartbeat the connection -is dropped and re-established. - -### Multiple Chronicle Maps - Network Distributed - -![Chronicle Maps Network Distributed](http://chronicle.software/wp-content/uploads/2014/07/Chronicle-Map_channels_diagram_02.jpg) - -ChronicleMap TCP Replication lets you distribute a single ChronicleMap, to a number of servers -across your network. Replication is point to point and the data transfer is bidirectional, so in the -example of just two servers, they only have to be connected via a single TCP socket connection and -the data is transferred both ways. This is great, but what if you wanted to replicate more than -just one ChronicleMap, what if you were going to replicate two ChronicleMaps across your network, -unfortunately with just TCP replication you would have to have two tcp socket connections, which is -not ideal. This is why we created the `ReplicationHub`. The `ReplicationHub` lets you replicate numerous -ChronicleMaps via a single point to point socket connection. - -The `ReplicationHub` encompasses TCP replication, where each map has to be given a -unique identifier, but when using the `ReplicationHub` we use a channel to identify the map, -rather than the identifier. The identifier is used to identify the host/server which broadcasts the -update. Put simply: - -* Each host must be given a unique identifier. -* Each map must be given a unique channel. - -```java -byte identifier= 2; -ReplicationHub replicationHub = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); -``` - -In this example above the `ReplicationHub` is given the identifier of 2. - -With channels you are able to attach additional maps to a `ReplicationChannel` once its up and -running. - -When creating the `ReplicationChannel` you should attach your tcp or udp configuration : -```java -byte identifier = 1; -ReplicationHub replicationHub = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); -``` - -Attaching a `ReplicationChannel` to the map : - -```java -short channel = (short) 2; -ChronicleMap map = ChronicleMap - .of(IntValue.class, CharSequence.class) - .averageValue(averageValue) - .entries(1000) - .instance().replicatedViaChannel(replicationHub.createChannel(channel)) - .create(); -``` - -The Chronicle channel is use to identify which map is to be replicated to which other map on -the remote node. In the example above this is assigned to '(short) 1', so for example if you have -two maps, lets call them map1 and map2, you could assign them with chronicle -channels 1 and 2 respectively. Map1 would have the chronicle channel of 1 on both servers. You -should not confuse the Chronicle Channels with the identifiers, the identifiers are unique per -replicating node ( in this case which host, the reason we say replicating node rather than host as it is -possible to have more than one replicating node per host if each of them had a different TCP/IP port ), where as the chronicle channels are used to identify which map you are referring. No additional socket - connection is made per chronicle channel that -you use, so we allow up to 32767 chronicle channels. - -If you inadvertently got the chronicle channels around the wrong way, then chronicle would attempt -to replicate the wrong maps data. The chronicle channels don't have to be in order but they must be -unique for each map you have. - -#### Channels and ReplicationChannel - Example - -```java - -import net.openhft.chronicle.hash.replication.ReplicationChannel; -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; - -... - -ChronicleMap favoriteColourServer1, favoriteColourServer2; -ChronicleMap favoriteComputerServer1, favoriteComputerServer2; - - -// server 1 with identifier = 1 -{ - ChronicleMapBuilder builder = ChronicleMapBuilder - .of(CharSequence.class, CharSequence.class) - .averageKeySize(10).averageValueSize(10) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8086, new InetSocketAddress("localhost", 8087)) - .heartBeatInterval(1, TimeUnit.SECONDS); - - ReplicationHub hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - ReplicationChannel channel = hubOnServer1.createChannel(channel1); - favoriteColourServer1 = builder.instance() - .replicatedViaChannel(channel).create(); - - favoriteColourServer1.put("peter", "green"); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer1 = builder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel2)).create(); - - favoriteComputerServer1.put("peter", "dell"); -} - -// server 2 with identifier = 2 -{ - ChronicleMapBuilder builder = ChronicleMapBuilder - .of(CharSequence.class, CharSequence.class) - .averageKeySize(10).averageValueSize(10) - .entries(1000); - - byte identifier = (byte) 2; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8087).heartBeatInterval(1, TimeUnit.SECONDS); - - ReplicationHub hubOnServer2 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer2 = builder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel1)).create(); - - favoriteColourServer2.put("rob", "blue"); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer2 = builder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel2)).create(); - - favoriteComputerServer2.put("rob", "mac"); - favoriteComputerServer2.put("daniel", "mac"); -} - -// allow time for the recompilation to resolve -for (int t = 0; t < 2500; t++) { - if (favoriteComputerServer2.equals(favoriteComputerServer1) && - favoriteColourServer2.equals(favoriteColourServer1)) - break; - Thread.sleep(1); -} - -assertEquals(favoriteComputerServer1, favoriteComputerServer2); -assertEquals(3, favoriteComputerServer2.size()); - -assertEquals(favoriteColourServer1, favoriteColourServer2); -assertEquals(2, favoriteColourServer1.size()); - -favoriteColourServer1.close(); -favoriteComputerServer2.close(); -favoriteColourServer2.close(); -favoriteColourServer1.close(); - -``` - ### Behaviour Customization You could customize `ChronicleMap` behaviour on several levels: @@ -1334,85 +952,6 @@ map1.put(1, "1"); System.out.println(map2.get("1")); ``` -#### Example - CRDT values for replicated Chronicle Maps - Grow-only set - -`Set` values won't replace each other on replication, but will converge to a single, common set, -the union of all elements added to all sets on all replicated nodes. - -```java -class GrowOnlySetValuedMapEntryOperations - implements MapEntryOperations, Void> { - - private static final GrowOnlySetValuedMapEntryOperations INSTANCE = - new GrowOnlySetValuedMapEntryOperations(); - - public static - MapEntryOperations, Void> growOnlySetValuedMapEntryOperations() { - return GrowOnlySetValuedMapEntryOperations.INSTANCE; - } - - private GrowOnlySetValuedMapEntryOperations() {} - - @Override - public Void remove(@NotNull MapEntry> entry) { - throw new UnsupportedOperationException("Map with grow-only set values " + - "doesn't support map value removals"); - } -} -``` - -```java -class GrowOnlySetValuedMapRemoteOperations - implements MapRemoteOperations, Void> { - - private static final GrowOnlySetValuedMapRemoteOperations INSTANCE = - new GrowOnlySetValuedMapRemoteOperations(); - - public static - MapRemoteOperations, Void> growOnlySetValuedMapRemoteOperations() { - return GrowOnlySetValuedMapRemoteOperations.INSTANCE; - } - - private GrowOnlySetValuedMapRemoteOperations() {} - - @Override - public void put(MapRemoteQueryContext, Void> q, Data, ?> newValue) { - MapReplicableEntry> entry = q.entry(); - if (entry != null) { - Set merged = new HashSet<>(entry.value().get()); - merged.addAll(newValue.get()); - q.replaceValue(entry, q.wrapValueAsValue(merged)); - } else { - q.insert(q.absentEntry(), newValue); - q.entry().updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - } - } - - @Override - public void remove(MapRemoteQueryContext, Void> q) { - throw new UnsupportedOperationException(); - } -} -``` - -Usage: -```java -HashSet averageValue = new HashSet<>(); -for (int i = 0; i < AVERAGE_SET_SIZE; i++) { - averageValue.add(i); -} -ChronicleMap> map1 = ChronicleMapBuilder - .of(Integer.class, (Class>) (Class) Set.class) - .entries(100) - .averageValue(averageValue) - .entryOperations(growOnlySetValuedMapEntryOperations()) - .remoteOperations(growOnlySetValuedMapRemoteOperations()) - .replication((byte) 1, /* ... replicated nodes */) - .instance() - .name("map1") - .create(); -``` - #### Example - Monitor Chronicle Map Statistics ```java diff --git a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java index 82a360eaa..d11592e68 100755 --- a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java +++ b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java @@ -19,8 +19,6 @@ package net.openhft.chronicle.hash; import net.openhft.chronicle.bytes.Byteable; import net.openhft.chronicle.bytes.Bytes; import net.openhft.chronicle.hash.replication.ReplicableEntry; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; import net.openhft.chronicle.hash.serialization.*; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; @@ -45,13 +43,6 @@ import java.util.concurrent.TimeUnit; * the corresponding configuration changed. To make an independent configuration, {@linkplain * #clone} the builder. * - *

{@code ChronicleHashBuilder} focuses on configurations, it could be used to create several - * containers of the same key-value domain. In this case {@code ChronicleHashBuilder} play like a - * factory (design pattern), rather than builder. The {@code ChronicleHashBuilder} could be - * "converted" to an {@link ChronicleHashInstanceBuilder} using {@link #instance()} method. - * {@code ChronicleHashInstanceBuilder} is a classic disposable builder, allowing to configure - * parameters, supposed to be unique for each created Chronicle Hash. - * *

{@code ChronicleHashBuilder} instances are not safe for concurrent use from multiple threads, * if at least one of the threads mutates the {@code ChronicleHashBuilder}'s state. * @@ -520,64 +511,8 @@ public interface ChronicleHashBuilder, */ B checksumEntries(boolean checksumEntries); - /** - * Configures replication of the hash containers, created by this builder. See the section about - * replication in ChronicleMap manual for more information. - * - *

By default, hash containers, created by this builder doesn't replicate their data. - * - *

This method call overrides all previous replication configurations of this builder, made - * either by this method or {@link #replication(byte, TcpTransportAndNetworkConfig)}, or - * {@link #replication(byte)} shortcut methods. - * - * @param replication the replication config - * @return this builder back - * @see ChronicleHashInstanceBuilder#replicated(SingleChronicleHashReplication) - * @see #replication(byte, TcpTransportAndNetworkConfig) - */ - B replication(SingleChronicleHashReplication replication); - - /** - * Shortcut for {@code replication(SingleChronicleHashReplication.builder() - * .tcpTransportAndNetwork(tcpTransportAndNetwork).createWithId(identifier))}. - * - * @param identifier the network-wide identifier of the containers, created by this - * builder - * @param tcpTransportAndNetwork configuration of tcp connection and network - * @return this builder back - * @see #replication(SingleChronicleHashReplication) - * @see ChronicleHashInstanceBuilder#replicated(byte, TcpTransportAndNetworkConfig) - */ - B replication(byte identifier, TcpTransportAndNetworkConfig tcpTransportAndNetwork); - - /** - * Shortcut for {@link #replication(SingleChronicleHashReplication) - * replication(SingleChronicleHashReplication.builder().createWithId(identifier)}. - * - *

This method is used to configure a replicated Chronicle Hash, that shouldn't communicate - * to other nodes. For example, if several processes access the same persisted Chronicle Hash, - * only one of them should communicate to other nodes and replicate entries. - * - * @param identifier the network-wide identifier for containers, created by this builder. This - * identifier is used when calling e. g. put() of the created container, to write the entry - * origin identifier (along with timestamp) into the entry memory. This change may be replicated - * to other nodes by a concurrent (or later) process, mapping to the same file. - * @return this builder back - */ B replication(byte identifier); - /** - * Returns a new {@code ChronicleHashInstanceBuilder}, which inherits configuration from this - * builder (in it's current state) and could be used to specify individual configurations - * for a Chronicle Hash container. Changes to this {@code ChronicleHashBuilder}, made after - * an instance is create, doesn't affect that instance, i. e. the returned instance is - * independent from the {@code ChronicleHashBuilder}, used to create it. - * - * @return a new Chronicle Hash instance builder - */ - ChronicleHashInstanceBuilder instance(); - /** * Creates a new hash container from this builder, storing it's data in off-heap memory, not * mapped to any file. On {@link ChronicleHash#close()} called on the returned container, or @@ -615,7 +550,6 @@ public interface ChronicleHashBuilder, * @see ChronicleHash#file() * @see ChronicleHash#close() * @see #create() - * @see ChronicleHashInstanceBuilder#persistedTo(File) * @see #recoverPersistedTo(File, boolean) */ H createPersistedTo(File file) throws IOException; diff --git a/src/main/java/net/openhft/chronicle/hash/ChronicleHashInstanceBuilder.java b/src/main/java/net/openhft/chronicle/hash/ChronicleHashInstanceBuilder.java deleted file mode 100644 index 33a91c503..000000000 --- a/src/main/java/net/openhft/chronicle/hash/ChronicleHashInstanceBuilder.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash; - -import net.openhft.chronicle.hash.replication.ReplicationChannel; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.set.ChronicleSet; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; - -/** - * A disposable builder of {@link ChronicleHash} instances, allowing to set unique configurations - * using "chaining" pattern. - * - *

Usage example: 

{@code
- * ChronicleMap = ChronicleMap
- *     .of(LongValue.class, Order.class)
- *     .entries(10_000_000)
- *     .instance()      // a ChronicleHashInstanceBuilder is returned from this method
- *                      // continue "chaining"
- *     .replicated(...)
- *     .name(...)
- *     .persistedTo(...)
- *     .create();
- * }
- * - * @param the container type, created by this builder, {@link ChronicleMap} or - * {@link ChronicleSet} - * @see ChronicleHashBuilder#instance() - */ -public interface ChronicleHashInstanceBuilder { - - ChronicleHashInstanceBuilder replicated( - byte identifier, TcpTransportAndNetworkConfig tcpTransportAndNetwork); - - ChronicleHashInstanceBuilder replicated(SingleChronicleHashReplication replication); - - /** - * Configures replication of the hash container, which is going to be created by this builder, - * via so called "channels". See the section - * about Channels and ChannelProvider in ChronicleMap manual for more information. - * - *

Another way to establish replication is {@link #replicated(SingleChronicleHashReplication) - * } method or it's shortcut: {@link #replicated(byte, TcpTransportAndNetworkConfig)}. - * - *

By default, hash containers, created by this builder doesn't replicate their data. - * - *

This method call overrides all previous replication configurations of this builder, made - * either by means of this method, {@link #replicated(SingleChronicleHashReplication)} - * or {@link #replicated(byte, TcpTransportAndNetworkConfig)} method calls. - * - * @param channel the channel responsible for gathering updates of hash containers, - * created by this builder, and replicating them over network - * @return this builder object back - * @see #replicated(SingleChronicleHashReplication) - */ - ChronicleHashInstanceBuilder replicatedViaChannel(ReplicationChannel channel); - - /** - * Configures the name for the Chronicle Hash, which is going to be created by this builder. It - * is used to name background threads, run by replicated Chronicle Hash instances. - * - * @param name the name for the Chronicle Hash is going to be created by this builder - * @return this builder back - */ - ChronicleHashInstanceBuilder name(String name); - - /** - * Configures the file, the Chronicle Hash, which is going to be created by this builder, should - * be persisted to. See {@link ChronicleHashBuilder#createPersistedTo(File)} for more - * information on Chronicle Hash persistence. If the given file is {@code null}, in-memory - * Chronicle Hash is created, like by {@link ChronicleHashBuilder#create()}. - * - *

By default, persistence file is configured to {@code null}, i. e. the in-memory Chronicle - * Hash instance is created. - * - * @param file the file the Chronicle Hash, which is going to be created by this builder, should - * be persisted to, to {@code null}, if the Chronicle Hash should be purely - * in-memory - * @return this builder back - */ - ChronicleHashInstanceBuilder persistedTo(File file); - - /** - * Creates (or opens existing, if {@link #persistedTo(File)} is configured with the existing - * file) and returns a Chronicle Hash from this builder. After this method is called once the - * builder couldn't be used to create new Chronicle Hash instances, a new {@code - * ChronicleHashInstanceBuilder} should be created via {@link ChronicleHashBuilder#instance()}. - * - * @return a new in-memory or persisted Chronicle Hash instance - * @throws IOException if any IO error, related to off-heap memory allocation or file mapping, - * or establishing replication connections, occurs - * @throws IllegalStateException if {@code create()} or {@link #recover(boolean)} method has - * already been called on this - * {@code ChronicleHashInstanceBuilder} - * @see ChronicleHashBuilder#create() - * @see ChronicleHashBuilder#createPersistedTo(File) - */ - H create() throws IOException; - - /** - * Recovers and returns a Chronicle Hash instance, persisted to the file, configured via {@link - * #persistedTo(File)} method. See documentation to {@link - * ChronicleHashBuilder#recoverPersistedTo(File, boolean)} for more information of Chronicle - * Hash instance recovery. - * - * @param sameBuilderConfig if the {@link ChronicleHashBuilder}, from which this {@code - * ChronicleHashInstanceBuilder} inherits (i. e. this {@code ChronicleHashInstanceBuilder} is - * created via calling {@link ChronicleHashBuilder#instance()} on that {@link - * ChronicleHashBuilder}) is configured with the same configurations, as the builder, which - * created the persistence file (the persisted Chronicle Hash instance) for the first time, and - * with the same version of the Chronicle Map library. In this case, the header of the file is - * overridden (with presumably the same configurations), protecting from {@link - * ChronicleHashRecoveryFailedException}, if the header is corrupted. - * @return a recovered Chronicle Hash instance, mapped to the specified file - * @throws IllegalStateException if persistence file is not configured (or explicitly configured - * to {@code null}) for this {@code ChronicleHashInstanceBuilder}, using {@link - * #persistedTo(File)} method, or if {@link #create()} or {@code recover()} method has already - * been called on this builder - * @throws FileNotFoundException if the specified persistence file doesn't exist - * @throws IOException if any IO error occurs on reading data from the file, or related to - * off-heap memory allocation or file mapping, or establishing replication connections. Probably - * the file is corrupted on OS level, and should be recovered on that level first, before - * calling this procedure. - * @throws ChronicleHashRecoveryFailedException if recovery is impossible - * @see ChronicleHashBuilder#recoverPersistedTo(File, boolean) - */ - H recover(boolean sameBuilderConfig) throws IOException; -} diff --git a/src/main/java/net/openhft/chronicle/hash/ChronicleHashRecoveryFailedException.java b/src/main/java/net/openhft/chronicle/hash/ChronicleHashRecoveryFailedException.java index e0a70840c..116d0c314 100644 --- a/src/main/java/net/openhft/chronicle/hash/ChronicleHashRecoveryFailedException.java +++ b/src/main/java/net/openhft/chronicle/hash/ChronicleHashRecoveryFailedException.java @@ -20,12 +20,10 @@ import java.io.File; /** * This exception is thrown, when a Chronicle Hash recovery using {@link - * ChronicleHashBuilder#recoverPersistedTo(File, boolean)} or {@link - * ChronicleHashInstanceBuilder#recover(boolean)} method is impossible, for example, if the - * persistence file is corrupted too much. + * ChronicleHashBuilder#recoverPersistedTo(File, boolean)} method is impossible, for example, + * if the persistence file is corrupted too much. * * @see ChronicleHashBuilder#recoverPersistedTo(File, boolean) - * @see ChronicleHashInstanceBuilder#recover(boolean) */ public final class ChronicleHashRecoveryFailedException extends RuntimeException { private static final long serialVersionUID = 0L; diff --git a/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java b/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java index 35e3d667a..8aec28bd7 100644 --- a/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java +++ b/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java @@ -39,7 +39,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.io.ObjectInputStream; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; diff --git a/src/main/java/net/openhft/chronicle/hash/replication/AbstractReplication.java b/src/main/java/net/openhft/chronicle/hash/replication/AbstractReplication.java deleted file mode 100644 index 0b74a50e7..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/AbstractReplication.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -/** - * Common configurations of {@link SingleChronicleHashReplication} and {@link ReplicationHub}. - */ -public abstract class AbstractReplication { - private final byte localIdentifier; - - @Nullable - private final TcpTransportAndNetworkConfig tcpConfig; - - @Nullable - private final UdpTransportConfig udpConfig; - - @Nullable - private final transient RemoteNodeValidator remoteNodeValidator; - - private final boolean bootstrapOnlyLocalEntries; - private final String name; - - @Nullable - private final transient ConnectionListener connectionListener; - - // package-private to forbid subclassing from outside of the package - AbstractReplication(byte localIdentifier, Builder builder) { - this.localIdentifier = localIdentifier; - tcpConfig = builder.tcpConfig; - udpConfig = builder.udpConfig; - remoteNodeValidator = builder.remoteNodeValidator; - bootstrapOnlyLocalEntries = builder.bootstrapOnlyLocalEntries; - name = builder.name; - connectionListener = builder.connectionListener; - } - - @Override - public String toString() { - return "localIdentifier=" + localIdentifier + - ", tcpConfig=" + tcpConfig + - ", udpConfig=" + udpConfig + - ", remoteNodeValidator=" + remoteNodeValidator + - ", bootstrapOnlyLocalEntries=" + bootstrapOnlyLocalEntries + - ", name=" + name; - } - - public byte identifier() { - return localIdentifier; - } - - @Nullable - public TcpTransportAndNetworkConfig tcpTransportAndNetwork() { - return tcpConfig; - } - - @Nullable - public UdpTransportConfig udpTransport() { - return udpConfig; - } - - @Nullable - public RemoteNodeValidator remoteNodeValidator() { - return remoteNodeValidator; - } - - public boolean bootstrapOnlyLocalEntries() { - return bootstrapOnlyLocalEntries; - } - - public String name() { - return name; - } - - public ConnectionListener connectionListener() { - return this.connectionListener; - } - - /** - * Builder of {@link AbstractReplication} configurations. This class and it's subclasses are - * mutable, configuration methods mutate the builder and return it back for convenient - * chaining. - * - * @param the concrete {@link AbstractReplication} subclass: {@link - * SingleChronicleHashReplication} or {@link ReplicationHub} - * @param the concrete builder subclass: {@link SingleChronicleHashReplication.Builder} or - * {@link ReplicationHub.Builder} - */ - public static abstract class Builder> { - private TcpTransportAndNetworkConfig tcpConfig = null; - private UdpTransportConfig udpConfig = null; - private RemoteNodeValidator remoteNodeValidator = null; - private ConnectionListener connectionListener = null; - - private boolean bootstrapOnlyLocalEntries = false; - private String name = "(unknown)"; - - // package-private to forbid subclassing from outside of the package - Builder() { - } - - @NotNull - public B tcpTransportAndNetwork(@Nullable TcpTransportAndNetworkConfig tcpConfig) { - this.tcpConfig = tcpConfig; - return (B) this; - } - - /** - * Configures UDP transport settings, used by Replications, created by this builder. {@code - * null} means that UDP transport shouldn't being used. - * - * @param udpConfig the new UDP transport config for replications, created by this builder. - * @return this builder back, for chaining - * @see AbstractReplication#udpTransport() - */ - @NotNull - public B udpTransport(@Nullable UdpTransportConfig udpConfig) { - this.udpConfig = udpConfig; - return (B) this; - } - - @NotNull - public B remoteNodeValidator(@Nullable RemoteNodeValidator remoteNodeValidator) { - this.remoteNodeValidator = remoteNodeValidator; - return (B) this; - } - - - @NotNull - public B connectionListener(@Nullable ConnectionListener connectionListener) { - this.connectionListener = connectionListener; - return (B) this; - } - - - /** - * Configures if the node, provided with replication, created by this builder, should - * replicate only local data, last updated with own identifier, or all data currently - * present on the node, when a new node joins the replication grid. - * - *

Default configuration is {@code false}, potentially swamping new nodes with - * duplicates. However, this does guarantee that all the data is replicated over to the new - * node and is useful especially in the case that the originating node of some data is not - * currently running. - * - * @param bootstrapOnlyLocalEntries if a node provided with this replication should - * replicate only local data to the new nodes joining the - * replication network - * @return this builder back - */ - @NotNull - public B bootstrapOnlyLocalEntries(boolean bootstrapOnlyLocalEntries) { - this.bootstrapOnlyLocalEntries = bootstrapOnlyLocalEntries; - return (B) this; - } - - /** - * Configures replication "name", which could appear in log messages and replication - * thread names. Default is {@code "(unknown)"}. - * - * @param name human-readable replication name - * @return this builder back - */ - public B name(String name) { - this.name = name; - return (B) this; - } - - /** - * Creates a Replication instance with the given node (server) identifier. - * - * @param identifier the node (server) identifier of the returned replication - * @return a new Replication instance with the specified node (server) identifier - * @throws IllegalArgumentException if the given identifier is non-positive - * @throws IllegalStateException if neither {@link #tcpTransportAndNetwork( - * TcpTransportAndNetworkConfig)} nor {@link #udpTransport(UdpTransportConfig)} are - * configured to non-{@code null}. At least one of the transport-level configs - * should be specified. - */ - @NotNull - public abstract R createWithId(byte identifier); - - void check(byte identifier) { - if (identifier <= 0) - throw new IllegalArgumentException("Identifier must be positive, " + identifier + - " given"); - } - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/ConnectionListener.java b/src/main/java/net/openhft/chronicle/hash/replication/ConnectionListener.java deleted file mode 100644 index 7293d830d..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/ConnectionListener.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import java.net.InetAddress; - -public interface ConnectionListener { - - /** - * @param address the address that we are connected to - * @param identifier the remote identifier that we are not connected to. - * @param isServer if this host accepted the connection {@code isServer} is {@code true}, if - * we made the connection to a remote host then {@code isServer} is {@code - * false} - */ - void onConnect(InetAddress address, byte identifier, boolean isServer); - - - /** - * @param address the address that we have been disconnected from - * @param identifier the identifer the address that we have been disconnected from or Byte - */ - void onDisconnect(InetAddress address, byte identifier); - - -} \ No newline at end of file diff --git a/src/main/java/net/openhft/chronicle/hash/replication/RemoteNodeValidator.java b/src/main/java/net/openhft/chronicle/hash/replication/RemoteNodeValidator.java deleted file mode 100644 index 580d04757..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/RemoteNodeValidator.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import java.net.SocketAddress; - -/** - * @author Rob Austin. - */ -public interface RemoteNodeValidator { - - /** - * checks the identifier that is unique and we haven't seen it before, unless it comes from the same port - * and host. - * - * @param remoteIdentifier remoteIdentifier - * @param remoteAddress remoteAddress - * @return true if unique - */ - boolean validate(byte remoteIdentifier, SocketAddress remoteAddress); - -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/ReplicationChannel.java b/src/main/java/net/openhft/chronicle/hash/replication/ReplicationChannel.java deleted file mode 100644 index 6745890a6..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/ReplicationChannel.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import net.openhft.chronicle.hash.ChronicleHash; -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; -import org.jetbrains.annotations.NotNull; - -/** - * A token which should be created via {@link ReplicationHub#createChannel(int)} call and passed to {@link - * ChronicleHashInstanceBuilder#replicatedViaChannel(ReplicationChannel)} method, to establish a channel - * replication. See - * the corresponding section in ChronicleMap manual for more information. - * - *

A {@code ReplicationChannel} could be used to replicate only one {@link ChronicleHash} instance. - * - * @see ReplicationHub - * @see ReplicationHub#createChannel(int) - */ -public final class ReplicationChannel { - private ReplicationHub hub; - private final int channelId; - - ReplicationChannel(ReplicationHub hub, int channelId) { - this.hub = hub; - this.channelId = channelId; - } - - /** - * Returns the {@link ReplicationHub} on which this {@code ReplicationChannel} was {@linkplain - * ReplicationHub#createChannel(int) created}. - * - * @return the {@code ReplicationHub} to which this {@code ReplicationChannel} belongs - * @see ReplicationHub#createChannel(int) - */ - @NotNull - public ReplicationHub hub() { - return hub; - } - - /** - * Returns the identifier of this channel, with which was {@linkplain ReplicationHub#createChannel(int) - * created}. - * - * @return the identifier of this channel - * @see ReplicationHub#createChannel(int) - */ - public int channelId() { - return channelId; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/ReplicationHub.java b/src/main/java/net/openhft/chronicle/hash/replication/ReplicationHub.java deleted file mode 100644 index a537501ef..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/ReplicationHub.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import net.openhft.chronicle.hash.ChronicleHash; -import net.openhft.chronicle.hash.ChronicleHashBuilder; -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.set.ChronicleSet; -import org.jetbrains.annotations.NotNull; - -import java.util.Arrays; - -/** - * This class holds all configurations of - * multicontainer replication, which is usable, when you want to replicate several {@linkplain - * ChronicleHash ChronicleHashes} ({@linkplain ChronicleMap maps}, {@linkplain ChronicleSet sets}, - * etc.) between same servers. {@code ReplicationHub} allows to share TCP/UDP connection, sockets, - * buffers, worker threads for that, considerably reducing resources usage and increasing cumulative - * (for all replicated containers) replication throughput. See - * the corresponding section in ChronicleMap manual for more information. - * - *

Create instances of this class using this pattern: 

{@code
- * ReplicationHub hub = ReplicationHub.builder()
- *     .tcpTransportAndNetwork(tcpConfig)
- *     // more configurations...
- *     .createWithId(identifierOfThisServerWithinTheChronicleReplicationNetwork);}
- * Then, given you prepared a {@link ChronicleHashBuilder builder} to create a {@link ChronicleHash - * }, configure it's replication via channel of this hub like this: 
{@code
- * ChronicleMap myMap = builder.instance()
- *     .replicatedViaChannel(hub.createChannel(myMapChannelId))
- *     .persistedTo(myMapFile) // optional, for this example
- *     .create();}
- * - * @see ChronicleHashInstanceBuilder#replicatedViaChannel(ReplicationChannel) - * @see ReplicationChannel - */ -public final class ReplicationHub extends AbstractReplication { - - /** - * Creates and returns a new {@link ReplicationHub.Builder}. - * - * @return a new {@link ReplicationHub.Builder} - */ - @NotNull - public static Builder builder() { - return new Builder(); - } - - private final ReplicationChannel[] channels; - - private ReplicationHub(byte localIdentifier, Builder builder) { - super(localIdentifier, builder); - - channels = new ReplicationChannel[builder.maxNumberOfChannels]; - } - - @Override - public String toString() { - return "ReplicationHub{" + super.toString() + - ", channels=" + Arrays.toString(channels) + '}'; - } - - - /** - * Returns the maximum number of channels could be {@linkplain #createChannel(int) created} for - * this {@code ReplicationHub}. - * - * @return the maximum number of channels - * @see Builder#maxNumberOfChannels() - * @see #createChannel(int) - */ - public int maxNumberOfChannels() { - return channels.length; - } - - /** - * Creates a new {@link ReplicationChannel} in this {@code ReplicationHub} with the given - * identifier. Identifier shouldn't be lesser than zero and greater or equal to {@code - * maxNumberOfChannels() - 1}. On a {@code ReplicationHub} instance, {@code ReplicationChannel} - * could be created only once for each possible {@code channelId} value. - * - * @param channelId the identifier of the channel. Should be equal for replicated containers on - * different nodes (servers) - * @return a new {@code ReplicationChannel} instance, in fact just incapsulating the given - * {@code channelId} for this {@code ReplicationHub} - * @throws IllegalArgumentException if the specified {@code channelId} is out of
[0, - * {@link #maxNumberOfChannels()}) range - * @throws IllegalStateException if {@code ReplicationChannel} with the specified {@code - * channelId} has already been acquired on this {@code - * ReplicationHub} - * @see ReplicationChannel - */ - public synchronized ReplicationChannel createChannel(int channelId) { - if (channelId < 0) - throw new IllegalArgumentException("channelId should be positive"); - if (channelId >= maxNumberOfChannels()) - throw new IllegalArgumentException("maxNumberOfChannels is configured (or defaulted) " + - "to " + maxNumberOfChannels() + ", channelId=" + channelId + " is requested"); - if (channels[channelId] != null) - throw new IllegalStateException("The requested channelId=" + channelId + - " is already in use"); - ReplicationChannel channel = new ReplicationChannel(this, channelId); - channels[channelId] = channel; - return channel; - } - - /** - * Builder of {@link ReplicationHub}s. - */ - public static final class Builder extends AbstractReplication.Builder { - - private int maxNumberOfChannels = 128; - - private Builder() { - } - - /** - * Configures the maximum number of channels could be {@linkplain #createChannel(int)} - * created for {@code ReplicationHub}s, created by this builder. - * - *

Default value is {@code 128}. - * - * @param maxNumberOfChannels {@link ReplicationHub#maxNumberOfChannels()} of {@link - * ReplicationHub}s, created by this builder - * @return this builder object back, for chaining - * @see ReplicationHub#maxNumberOfChannels() - */ - @NotNull - public Builder maxNumberOfChannels(int maxNumberOfChannels) { - this.maxNumberOfChannels = maxNumberOfChannels; - return this; - } - - @NotNull - @Override - public ReplicationHub createWithId(byte identifier) { - check(identifier); - return new ReplicationHub(identifier, this); - } - } - -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/SingleChronicleHashReplication.java b/src/main/java/net/openhft/chronicle/hash/replication/SingleChronicleHashReplication.java deleted file mode 100644 index 80da6d59a..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/SingleChronicleHashReplication.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import org.jetbrains.annotations.NotNull; - -public final class SingleChronicleHashReplication extends AbstractReplication { - - @NotNull - public static Builder builder() { - return new Builder(); - } - - SingleChronicleHashReplication(byte localIdentifier, Builder builder) { - super(localIdentifier, builder); - } - - @Override - public String toString() { - return "SingleChronicleHashReplication{" + super.toString() + "}"; - } - - public static final class Builder - extends AbstractReplication.Builder { - private Builder() {} - - @NotNull - @Override - public SingleChronicleHashReplication createWithId(byte identifier) { - check(identifier); - return new SingleChronicleHashReplication(identifier, this); - } - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/TcpInetSocketAddress.java b/src/main/java/net/openhft/chronicle/hash/replication/TcpInetSocketAddress.java deleted file mode 100644 index 4d38ccd5e..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/TcpInetSocketAddress.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import java.net.InetSocketAddress; - -class TcpInetSocketAddress extends InetSocketAddress { - private static final long serialVersionUID = 2747559307347096475L; - - private final String toString; - - public TcpInetSocketAddress(String hostname, int port) { - super(hostname, port); - toString = super.toString(); - } - - @Override - public String toString() { - return toString; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java b/src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java deleted file mode 100644 index 2389b96fa..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static java.util.Collections.unmodifiableSet; -import static java.util.concurrent.TimeUnit.SECONDS; - -public final class TcpTransportAndNetworkConfig { - - private static final int DEFAULT_TCP_BUFFER_SIZE = 1024 * 64; - private static final long DEFAULT_HEART_BEAT_INTERVAL = 20; - private static final TimeUnit DEFAULT_HEART_BEAT_INTERVAL_UNIT = SECONDS; - - private final int serverPort; - private final Set endpoints; - private final int tcpBufferSize; - private final boolean autoReconnectedUponDroppedConnection; - private final ThrottlingConfig throttlingConfig; - private final long heartBeatInterval; - private final TimeUnit heartBeatIntervalUnit; - - private TcpTransportAndNetworkConfig(int serverPort, Set endpoints, - int tcpBufferSize, - boolean autoReconnectedUponDroppedConnection, - ThrottlingConfig throttlingConfig, long heartBeatInterval, - TimeUnit heartBeatIntervalUnit) { - this.serverPort = serverPort; - this.endpoints = endpoints; - this.tcpBufferSize = tcpBufferSize; - this.autoReconnectedUponDroppedConnection = autoReconnectedUponDroppedConnection; - this.throttlingConfig = throttlingConfig; - this.heartBeatInterval = heartBeatInterval; - this.heartBeatIntervalUnit = heartBeatIntervalUnit; - } - - public static TcpTransportAndNetworkConfig of(int serverPort, - InetSocketAddress... endpoints) { - return of(serverPort, Arrays.asList(endpoints)); - } - - public static TcpTransportAndNetworkConfig of(int serverPort, - Collection endpoints) { - for (final InetSocketAddress endpoint : endpoints) { - if (endpoint.getPort() == serverPort && "localhost".equals(endpoint.getHostName())) - throw new IllegalArgumentException("endpoint=" + endpoint - + " can not point to the same port as the server"); - } - Set s = new HashSet<>(); - for (InetSocketAddress endpoint : endpoints) { - s.add(new TcpInetSocketAddress(endpoint.getHostName(), endpoint.getPort())); - } - return new TcpTransportAndNetworkConfig( - serverPort, unmodifiableSet(s), - DEFAULT_TCP_BUFFER_SIZE, - true, // autoReconnectedUponDroppedConnection - ThrottlingConfig.noThrottling(), - DEFAULT_HEART_BEAT_INTERVAL, - DEFAULT_HEART_BEAT_INTERVAL_UNIT); - } - - public boolean autoReconnectedUponDroppedConnection() { - return autoReconnectedUponDroppedConnection; - } - - public TcpTransportAndNetworkConfig autoReconnectedUponDroppedConnection( - boolean autoReconnectedUponDroppedConnection) { - return new TcpTransportAndNetworkConfig(serverPort, endpoints, tcpBufferSize, - autoReconnectedUponDroppedConnection, throttlingConfig, heartBeatInterval, - heartBeatIntervalUnit); - } - - public ThrottlingConfig throttlingConfig() { - return throttlingConfig; - } - - public TcpTransportAndNetworkConfig throttlingConfig(ThrottlingConfig throttlingConfig) { - ThrottlingConfig.checkMillisecondBucketInterval(throttlingConfig, "TCP"); - return new TcpTransportAndNetworkConfig(serverPort, endpoints, tcpBufferSize, - autoReconnectedUponDroppedConnection, throttlingConfig, heartBeatInterval, - heartBeatIntervalUnit); - } - - public long heartBeatInterval(TimeUnit unit) { - return unit.convert(heartBeatInterval, heartBeatIntervalUnit); - } - - public int serverPort() { - return serverPort; - } - - public TcpTransportAndNetworkConfig serverPort(int serverPort) { - return new TcpTransportAndNetworkConfig(serverPort, endpoints, tcpBufferSize, - autoReconnectedUponDroppedConnection, throttlingConfig, heartBeatInterval, - heartBeatIntervalUnit); - } - - public Set endpoints() { - return endpoints; - } - - public TcpTransportAndNetworkConfig endpoints(Set endpoints) { - for (final InetSocketAddress endpoint : endpoints) { - if (endpoint.getPort() == serverPort && "localhost".equals(endpoint.getHostName())) - throw new IllegalArgumentException("endpoint=" + endpoint - + " can not point to the same port as the server"); - } - return new TcpTransportAndNetworkConfig(serverPort, endpoints, tcpBufferSize, - autoReconnectedUponDroppedConnection, throttlingConfig, heartBeatInterval, - heartBeatIntervalUnit); - } - - public int tcpBufferSize() { - return tcpBufferSize; - } - - public TcpTransportAndNetworkConfig tcpBufferSize(int tcpBufferSize) { - if (tcpBufferSize <= 0) - throw new IllegalArgumentException(); - return new TcpTransportAndNetworkConfig(serverPort, endpoints, tcpBufferSize, - autoReconnectedUponDroppedConnection, throttlingConfig, heartBeatInterval, - heartBeatIntervalUnit); - } - - public TcpTransportAndNetworkConfig heartBeatInterval(long heartBeatInterval, - TimeUnit heartBeatIntervalUnit) { - return new TcpTransportAndNetworkConfig(serverPort, endpoints, tcpBufferSize, - autoReconnectedUponDroppedConnection, throttlingConfig, heartBeatInterval, - heartBeatIntervalUnit); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TcpTransportAndNetworkConfig that = (TcpTransportAndNetworkConfig) o; - - if (autoReconnectedUponDroppedConnection != that.autoReconnectedUponDroppedConnection) - return false; - if (heartBeatInterval != that.heartBeatInterval) return false; - if (tcpBufferSize != that.tcpBufferSize) return false; - if (serverPort != that.serverPort) return false; - if (endpoints != null ? !endpoints.equals(that.endpoints) : that.endpoints != null) - return false; - if (heartBeatIntervalUnit != that.heartBeatIntervalUnit) return false; - if (throttlingConfig != null ? !throttlingConfig.equals(that.throttlingConfig) : - that.throttlingConfig != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = serverPort; - result = 31 * result + (endpoints != null ? endpoints.hashCode() : 0); - result = 31 * result + tcpBufferSize; - result = 31 * result + (autoReconnectedUponDroppedConnection ? 1 : 0); - result = 31 * result + (throttlingConfig != null ? throttlingConfig.hashCode() : 0); - result = 31 * result + (int) (heartBeatInterval ^ (heartBeatInterval >>> 32)); - result = 31 * result + (heartBeatIntervalUnit != null ? heartBeatIntervalUnit.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "TcpReplicationConfig{" + - "serverPort=" + serverPort + - ", endpoints=" + endpoints + - ", tcpBufferSize=" + tcpBufferSize + - ", autoReconnectedUponDroppedConnection=" + autoReconnectedUponDroppedConnection + - ", throttlingConfig=" + throttlingConfig + - ", heartBeatInterval=" + heartBeatInterval + - ", heartBeatIntervalUnit=" + heartBeatIntervalUnit + - '}'; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/ThrottlingConfig.java b/src/main/java/net/openhft/chronicle/hash/replication/ThrottlingConfig.java deleted file mode 100644 index 284cb2ab1..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/ThrottlingConfig.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import java.util.concurrent.TimeUnit; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public final class ThrottlingConfig { - - private static final long DEFAULT_BUCKET_INTERVAL = 100L; - private static final TimeUnit DEFAULT_BUCKET_INTERVAL_UNIT = MILLISECONDS; - - private static final ThrottlingConfig NO_THROTTLING = create(0L, MILLISECONDS, - DEFAULT_BUCKET_INTERVAL, DEFAULT_BUCKET_INTERVAL_UNIT); - - private final long throttle; - private final TimeUnit throttlePerUnit; - private final long bucketInterval; - private final TimeUnit bucketIntervalUnit; - - ThrottlingConfig( - long throttle, - TimeUnit throttlePerUnit, - long bucketInterval, - TimeUnit bucketIntervalUnit) { - this.throttle = throttle; - if (throttlePerUnit == null) { - throw new NullPointerException("Null throttlePerUnit"); - } - this.throttlePerUnit = throttlePerUnit; - this.bucketInterval = bucketInterval; - if (bucketIntervalUnit == null) { - throw new NullPointerException("Null bucketIntervalUnit"); - } - this.bucketIntervalUnit = bucketIntervalUnit; - } - - /** - * Returns a config which prescribe that a replicator shouldn't throttle. - * - * @return a config which prescribe that a replicator shouldn't throttle - */ - public static ThrottlingConfig noThrottling() { - return NO_THROTTLING; - } - - /** - * Returns a config which throttle the specified max bits per the given time unit with default bucketing - * interval. - * - * @param maxBits the preferred maximum bits - * @param perUnit the time unit per which maximum bits specified - * @return a config which throttle the specified max bits per the given time unit - * @throws IllegalArgumentException if the specified {@code maxBits} is non-positive value - */ - public static ThrottlingConfig throttle(long maxBits, TimeUnit perUnit) { - if (maxBits <= 0L) - throw new IllegalArgumentException(); - return create(maxBits, perUnit, DEFAULT_BUCKET_INTERVAL, DEFAULT_BUCKET_INTERVAL_UNIT); - } - - static void checkMillisecondBucketInterval(ThrottlingConfig config, String replicator) { - if (config.bucketInterval(MILLISECONDS) < 1) { - throw new IllegalArgumentException( - "Minimum throttle bucketing interval of 1 millisecond is supported by " + - replicator + " replicator"); - } - } - - static ThrottlingConfig create( - long throttle, TimeUnit throttlePerUnit, - long throttleBucketInterval, TimeUnit throttleBucketIntervalUnit) { - return new ThrottlingConfig(throttle, throttlePerUnit, - throttleBucketInterval, throttleBucketIntervalUnit); - } - - /** - * Returns maximum bits per the given time unit, i. e. the throttling. {@code 0} (zero) designates there - * is no throttling. - * - * @param perUnit maximum bits is returned per this time unit - * @return maximum bits per the given time unit, or {@code 0} if there is no throttling - */ - public final long throttling(TimeUnit perUnit) { - return throttlePerUnit.convert(throttle, perUnit); - } - - /** - * Returns the throttle bucketing interval in the given time units.

Default throttle bucketing - * interval is 100 millis. - * - * @param unit the time unit of the interval - * @return the bucketing interval for throttling in the given time units - */ - public long bucketInterval(TimeUnit unit) { - return unit.convert(bucketInterval, bucketIntervalUnit); - } - - /** - * Returns a copy of this config with the specified bucketing interval. - * - * @param throttleBucketInterval the bucketing interval for throttling - * @param unit the time unit of the interval - * @return a copy of this config with the specified bucketing interval - * @throws IllegalArgumentException if the given bucketing interval is non-positive - */ - public ThrottlingConfig bucketInterval(long throttleBucketInterval, TimeUnit unit) { - if (throttleBucketInterval <= 0L) - throw new IllegalArgumentException(); - return create(throttle, throttlePerUnit, throttleBucketInterval, unit); - } - - @Override - public String toString() { - return "ThrottlingConfig{" - + "throttle=" + throttle - + ", throttlePerUnit=" + throttlePerUnit - + ", bucketInterval=" + bucketInterval - + ", bucketIntervalUnit=" + bucketIntervalUnit - + "}"; - } - - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - if (o instanceof ThrottlingConfig) { - ThrottlingConfig that = (ThrottlingConfig) o; - return (this.throttle == that.throttle) - && (this.throttlePerUnit.equals(that.throttlePerUnit)) - && (this.bucketInterval == that.bucketInterval) - && (this.bucketIntervalUnit.equals(that.bucketIntervalUnit)); - } - return false; - } - - @Override - public int hashCode() { - int h = 1; - h *= 1000003; - h ^= (throttle >>> 32) ^ throttle; - h *= 1000003; - h ^= throttlePerUnit.hashCode(); - h *= 1000003; - h ^= (bucketInterval >>> 32) ^ bucketInterval; - h *= 1000003; - h ^= bucketIntervalUnit.hashCode(); - return h; - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/replication/UdpTransportConfig.java b/src/main/java/net/openhft/chronicle/hash/replication/UdpTransportConfig.java deleted file mode 100644 index dc708bc9e..000000000 --- a/src/main/java/net/openhft/chronicle/hash/replication/UdpTransportConfig.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.hash.replication; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.net.InetAddress; -import java.net.NetworkInterface; - -public final class UdpTransportConfig { - public static final int DEFAULT_UDP_BUFFER_SIZE = 64 * 1024; - private final InetAddress address; - private final int port; - - @Nullable - private final - NetworkInterface networkInterface; - - @NotNull - private final ThrottlingConfig throttlingConfig; - - private final int udpBufferSize; - private String name; - - private UdpTransportConfig( - InetAddress address, - int port, - @Nullable NetworkInterface networkInterface, - ThrottlingConfig throttlingConfig, - int udpBufferSize) { - this.udpBufferSize = udpBufferSize; - if (address == null) { - throw new NullPointerException("Null address"); - } - this.address = address; - this.port = port; - - this.networkInterface = networkInterface; - if (throttlingConfig == null) { - throw new NullPointerException("Null throttlingConfig"); - } - this.throttlingConfig = throttlingConfig; - } - - public static UdpTransportConfig multiCast(@NotNull InetAddress address, int port, - @Nullable NetworkInterface networkInterface) { - if (!address.isMulticastAddress() || networkInterface == null) - throw new IllegalArgumentException(); - return create(address, port, networkInterface, ThrottlingConfig.noThrottling(), DEFAULT_UDP_BUFFER_SIZE); - } - - public static UdpTransportConfig of(@NotNull InetAddress address, int port) { - if (address.isMulticastAddress()) - throw new IllegalArgumentException(); - return create(address, port, null, ThrottlingConfig.noThrottling(), DEFAULT_UDP_BUFFER_SIZE); - } - - static UdpTransportConfig create(InetAddress address, int port, - @Nullable NetworkInterface networkInterface, - ThrottlingConfig throttlingConfig, - final int udpBufferSize1) { - return new UdpTransportConfig(address, port, networkInterface, throttlingConfig, udpBufferSize1); - } - - @NotNull - public InetAddress address() { - return address; - } - - public int port() { - return port; - } - - @Nullable - public NetworkInterface networkInterface() { - return networkInterface; - } - - @NotNull - public ThrottlingConfig throttlingConfig() { - return throttlingConfig; - } - - @Override - public String toString() { - return "UdpReplicationConfig{" - + "address=" + address - + ", port=" + port - + ", networkInterface=" + networkInterface - + ", throttlingConfig=" + throttlingConfig - + "}"; - } - - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - if (o instanceof UdpTransportConfig) { - UdpTransportConfig that = (UdpTransportConfig) o; - return (this.address.equals(that.address())) - && (this.port == that.port()) - && (this.networkInterface.equals(that.networkInterface())) - && (this.throttlingConfig.equals(that.throttlingConfig())); - } - return false; - } - - @Override - public int hashCode() { - int h = 1; - h *= 1000003; - h ^= address.hashCode(); - h *= 1000003; - h ^= port; - h *= 1000003; - h ^= networkInterface.hashCode(); - h *= 1000003; - h ^= throttlingConfig.hashCode(); - return h; - } - - public UdpTransportConfig networkInterfaceForMulticast( - @Nullable NetworkInterface networkInterface) { - if (!address().isMulticastAddress()) - throw new IllegalArgumentException(); - return create(address(), port(), networkInterface, throttlingConfig(), udpBufferSize()); - } - - public UdpTransportConfig throttlingConfig(@NotNull ThrottlingConfig throttlingConfig) { - ThrottlingConfig.checkMillisecondBucketInterval(throttlingConfig, "UDP"); - return create(address(), port(), networkInterface(), throttlingConfig, udpBufferSize()); - } - - public int udpBufferSize() { - return udpBufferSize; - } - - public UdpTransportConfig udpBufferSize(int udpBufferSize) { - return create(address(), port(), networkInterface(), throttlingConfig(), udpBufferSize); - } - - public String name() { - return this.name; - } -} diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java deleted file mode 100644 index 65d2b14dc..000000000 --- a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java +++ /dev/null @@ -1,579 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.bytes.Bytes; -import net.openhft.chronicle.core.io.CloseablesManager; -import net.openhft.chronicle.hash.replication.ReplicableEntry; -import net.openhft.chronicle.hash.replication.ThrottlingConfig; -import net.openhft.chronicle.threads.NamedThreadFactory; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.*; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.*; - -import static java.lang.Math.round; -import static java.nio.channels.SelectionKey.OP_WRITE; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -/** - * @author Rob Austin. - */ -abstract class AbstractChannelReplicator implements Closeable { - - public static final int BITS_IN_A_BYTE = 8; - - static final int SIZE_OF_SIZE = 4; - public static final int SIZE_OF_TRANSACTION_ID = 8; - - static final byte STATEFUL_UPDATE = 1; - - private static final Logger LOG = LoggerFactory.getLogger(AbstractChannelReplicator.class); - - // this was not safe to use as it wont work with the stateless client - static boolean useJavaNIOSelectionKeys = - Boolean.valueOf(System.getProperty("useJavaNIOSelectionKeys")); - - final SelectedSelectionKeySet selectedKeys = new SelectedSelectionKeySet(); - - final CloseablesManager closeables = new CloseablesManager(); - final Selector selector = openSelector(closeables); - private final ExecutorService executorService; - - private final ExecutorService connectExecutorService = Executors.newCachedThreadPool( - new NamedThreadFactory("TCP-connect", true) { - @NotNull - @Override - public Thread newThread(@NotNull Runnable r) { - return lastThread = super.newThread(r); - } - }); - - private volatile Thread lastThread; - private Throwable startedHere; - private Future future; - private final Queue pendingRegistrations = new ConcurrentLinkedQueue<>(); - @Nullable - private final Throttler throttler; - - volatile boolean isClosed = false; - - AbstractChannelReplicator(String name, ThrottlingConfig throttlingConfig) - throws IOException { - executorService = Executors.newSingleThreadExecutor( - new NamedThreadFactory(name, true) { - @NotNull - @Override - public Thread newThread(@NotNull Runnable r) { - return lastThread = super.newThread(r); - } - }); - - throttler = throttlingConfig.throttling(DAYS) > 0 ? - new Throttler(selector, - throttlingConfig.bucketInterval(MILLISECONDS), - throttlingConfig.throttling(DAYS)) : null; - - startedHere = new Throwable("Started here"); - } - - Selector openSelector(final CloseablesManager closeables) throws IOException { - - Selector result = Selector.open(); - closeables.add(result); - - if (!useJavaNIOSelectionKeys) { - closeables.add(new Closeable() { - @Override - public void close() throws IOException { - { - SelectionKey[] keys = selectedKeys.flip(); - for (int i = 0; i < keys.length && keys[i] != null; i++) { - keys[i] = null; - } - } - { - SelectionKey[] keys = selectedKeys.flip(); - for (int i = 0; i < keys.length && keys[i] != null; i++) { - keys[i] = null; - } - } - } - } - ); - return openSelector(result, selectedKeys); - } - - return result; - } - - static SocketChannel openSocketChannel(final CloseablesManager closeables) throws IOException { - SocketChannel result = null; - - try { - result = SocketChannel.open(); - result.socket().setTcpNoDelay(true); - } finally { - if (result != null) - try { - closeables.add(result); - } catch (IllegalStateException e) { - // already closed - } - } - return result; - } - - /** - * this is similar to the code in Netty - * - * @param selector - * @return - */ - private static Selector openSelector(@NotNull final Selector selector, - @NotNull final SelectedSelectionKeySet selectedKeySet) { - try { - - Class selectorImplClass = - Class.forName("sun.nio.ch.SelectorImpl", false, getSystemClassLoader()); - - // Ensure the current selector implementation is what we can instrument. - if (!selectorImplClass.isAssignableFrom(selector.getClass())) { - return selector; - } - - Field selectedKeysField = selectorImplClass.getDeclaredField("selectedKeys"); - Field publicSelectedKeysField = selectorImplClass.getDeclaredField("publicSelectedKeys"); - - selectedKeysField.setAccessible(true); - publicSelectedKeysField.setAccessible(true); - - selectedKeysField.set(selector, selectedKeySet); - publicSelectedKeysField.set(selector, selectedKeySet); - - // logger.trace("Instrumented an optimized java.util.Set into: {}", selector); - } catch (Exception e) { - LOG.error("", e); - // logger.trace("Failed to instrument an optimized java.util.Set into: {}", selector, t); - } - - return selector; - } - - static ClassLoader getSystemClassLoader() { - if (System.getSecurityManager() == null) { - return ClassLoader.getSystemClassLoader(); - } else { - return AccessController.doPrivileged(new PrivilegedAction() { - @Override - public ClassLoader run() { - return ClassLoader.getSystemClassLoader(); - } - }); - } - } - - void addPendingRegistration(Runnable registration) { - pendingRegistrations.add(registration); - } - - void registerPendingRegistrations() throws ClosedChannelException { - for (Runnable runnable = pendingRegistrations.poll(); runnable != null; - runnable = pendingRegistrations.poll()) { - try { - runnable.run(); - } catch (Exception e) { - LOG.info("", e); - } - } - } - - abstract void processEvent() throws IOException; - - final void start() { - future = executorService.submit( - new Runnable() { - @Override - public void run() { - try { - processEvent(); - } catch (Exception e) { - LOG.error("", e); - } - } - } - ); - } - - @Override - public void close() { - - if (Thread.interrupted()) - LOG.warn("Already interrupted"); - long start = System.currentTimeMillis(); - isClosed = true; - connectExecutorService.shutdown(); - try { - if (!connectExecutorService.awaitTermination(5, TimeUnit.MILLISECONDS)) { - connectExecutorService.shutdownNow(); - connectExecutorService.awaitTermination(10, TimeUnit.SECONDS); - } - } catch (InterruptedException e) { - LOG.error("", e); - } - - try { - closeResources(); - if (!executorService.awaitTermination(5, TimeUnit.MILLISECONDS)) { - executorService.shutdownNow(); - executorService.awaitTermination(10, TimeUnit.SECONDS); - } - } catch (InterruptedException e) { - LOG.error("", e); - } - - try { - if (future != null) - future.cancel(true); - - // we HAVE to be sure we have terminated before calling close() on the ReplicatedChronicleMap - // if this thread is still running and you close() the ReplicatedChronicleMap without - // fulling terminating this, you can cause the ReplicatedChronicleMap to attempt to - // process data on a map that has been closed, which would result in a core dump. - - Thread thread = lastThread; - if (thread != null && Thread.currentThread() != lastThread) - for (int i = 0; i < 10; i++) { - if (thread.isAlive()) { - thread.join(1000); - dumpThreadStackTrace(start); - } - } - } catch (InterruptedException e) { - dumpThreadStackTrace(start); - LOG.error("", e); - LOG.error("", startedHere); - } - } - - public void closeResources() { - isClosed = true; - executorService.shutdown(); - closeables.closeQuietly(); - - } - - private void dumpThreadStackTrace(long start) { - if (lastThread != null && lastThread.isAlive()) { - StringBuilder sb = new StringBuilder(); - sb.append("Replicator thread still running after "); - sb.append((System.currentTimeMillis() - start) / 100 / 10.0); - sb.append(" secs "); - sb.append(lastThread); - sb.append(" isAlive= "); - sb.append(lastThread.isAlive()); - for (StackTraceElement ste : lastThread.getStackTrace()) { - sb.append("\n\t").append(ste); - } - LOG.warn(sb.toString()); - } - } - - void closeEarlyAndQuietly(SelectableChannel channel) { - if (throttler != null) - throttler.remove(channel); - closeables.closeQuietly(channel); - } - - void checkThrottleInterval() throws ClosedChannelException { - if (throttler != null) - throttler.checkThrottleInterval(); - } - - void contemplateThrottleWrites(int bytesJustWritten) throws ClosedChannelException { - if (throttler != null) - throttler.contemplateThrottleWrites(bytesJustWritten); - } - - void throttle(SelectableChannel channel) { - if (throttler != null) - throttler.add(channel); - } - - /** - * throttles 'writes' to ensure the network is not swamped, this is achieved by periodically - * de-registering the write selector during periods of high volume. - */ - static class Throttler { - - private final Selector selector; - private final Set channels = new CopyOnWriteArraySet(); - private final long throttleInterval; - private final long maxBytesInInterval; - - private long lastTime = System.currentTimeMillis(); - private long bytesWritten; - - Throttler(@NotNull Selector selector, - long throttleIntervalInMillis, - long bitsPerDay) { - this.selector = selector; - this.throttleInterval = throttleIntervalInMillis; - double bytesPerMs = ((double) bitsPerDay) / DAYS.toMillis(1) / BITS_IN_A_BYTE; - this.maxBytesInInterval = round(bytesPerMs * throttleInterval); - } - - public void add(SelectableChannel selectableChannel) { - channels.add(selectableChannel); - } - - public void remove(SelectableChannel socketChannel) { - channels.remove(socketChannel); - } - - /** - * re register the 'write' on the selector if the throttleInterval has passed - * - * @throws java.nio.channels.ClosedChannelException - */ - public void checkThrottleInterval() throws ClosedChannelException { - final long time = System.currentTimeMillis(); - - if (lastTime + throttleInterval >= time) - return; - - lastTime = time; - bytesWritten = 0; - - if (LOG.isDebugEnabled()) - LOG.debug("Restoring OP_WRITE on all channels"); - - for (SelectableChannel selectableChannel : channels) { - final SelectionKey selectionKey = selectableChannel.keyFor(selector); - if (selectionKey != null) - selectionKey.interestOps(selectionKey.interestOps() | OP_WRITE); - } - } - - /** - * checks the number of bytes written in this interval, if this number of bytes exceeds a - * threshold, the selected will de-register the socket that is being written to, until the - * interval is finished. - * - * @throws ClosedChannelException - */ - public void contemplateThrottleWrites(int bytesJustWritten) - throws ClosedChannelException { - bytesWritten += bytesJustWritten; - if (bytesWritten > maxBytesInInterval) { - for (SelectableChannel channel : channels) { - final SelectionKey selectionKey = channel.keyFor(selector); - if (selectionKey != null) { - selectionKey.interestOps(selectionKey.interestOps() & ~OP_WRITE); - } - if (LOG.isDebugEnabled()) - LOG.debug("Throttling UDP writes"); - } - } - } - } - - /** - * details about the socket connection - */ - static class Details { - - private final InetSocketAddress address; - private final byte localIdentifier; - - Details(@NotNull final InetSocketAddress address, final byte localIdentifier) { - this.address = address; - this.localIdentifier = localIdentifier; - } - - public InetSocketAddress address() { - return address; - } - - public byte localIdentifier() { - return localIdentifier; - } - - @Override - public String toString() { - return "Details{" + - "address=" + address + - ", localIdentifier=" + localIdentifier + - '}'; - } - } - - static class EntryCallback implements Replica.EntryCallback { - - private final Replica.EntryExternalizable externalizable; - - /** For writing entries for replication from Ch Map memory */ - private Bytes entryIn; - - EntryCallback(@NotNull final Replica.EntryExternalizable externalizable, - final int tcpBufferSize) { - this.externalizable = externalizable; - entryIn = Bytes.elasticByteBuffer(tcpBufferSize); - } - - @NotNull - public Bytes entryIn() { - return entryIn; - } - - @NotNull - public ByteBuffer socketOut() { - return entryIn.underlyingObject(); - } - - @Override - public void onEntry(ReplicableEntry entry, Bytes payload, int chronicleId) { - // used to denote that this is not a heartbeat - entryIn.writeByte(STATEFUL_UPDATE); - - long sizeLocation = entryIn.writePosition(); - - // this is where we will store the size of the entry - entryIn.writeSkip(SIZE_OF_SIZE); - - long start = entryIn.writePosition(); - - externalizable.writeExternalEntry(entry, payload, entryIn, chronicleId); - - // write the length of the entry, just before the start, so when we read it back - // we read the length of the entry first and hence know how many preceding writer - // to read - final long bytesWritten = (int) (entryIn.writePosition() - start); - - if (bytesWritten > Integer.MAX_VALUE) - throw new IllegalStateException("entry size " + bytesWritten + " too large, " + - "entries are limited to a size of " + Integer.MAX_VALUE); - - if (LOG.isDebugEnabled()) - LOG.debug("sending entry of entrySize=" + (int) bytesWritten); - - entryIn.writeInt(sizeLocation, (int) bytesWritten); - } - } - - abstract class AbstractConnector { - - private final String name; - private int connectionAttempts = 0; - private volatile SelectableChannel socketChannel; - - public AbstractConnector(String name) { - this.name = name; - } - - abstract SelectableChannel doConnect() throws IOException, InterruptedException; - - /** - * connects or reconnects, but first waits a period of time proportional to the {@code - * connectionAttempts} - */ - public final void connectLater() { - if (socketChannel != null) { - closeables.closeQuietly(socketChannel); - socketChannel = null; - } - - final long reconnectionInterval = connectionAttempts * 100; - if (connectionAttempts < 5) - connectionAttempts++; - doConnect(reconnectionInterval); - } - - /** - * connects or reconnects immediately - */ - public void connect() { - doConnect(0); - } - - /** - * @param reconnectionInterval the period to wait before connecting - */ - private void doConnect(final long reconnectionInterval) { - try { - connectExecutorService.submit(new Runnable() { - public void run() { - SelectableChannel socketChannel = null; - try { - if (reconnectionInterval > 0) - Thread.sleep(reconnectionInterval); - - socketChannel = doConnect(); - - try { - closeables.add(socketChannel); - } catch (IllegalStateException e) { - // close could have be called from another thread, while we were in Thread.sleep() - // which would cause a IllegalStateException - closeQuietly(socketChannel); - return; - } - - AbstractConnector.this.socketChannel = socketChannel; - } catch (Exception e) { - closeQuietly(socketChannel); - LOG.debug("", e); - } - } - }); - } catch (Exception e) { - if (!isClosed) - LOG.error("", e); - } - } - - private void closeQuietly(SelectableChannel socketChannel) { - if (socketChannel == null) - return; - try { - socketChannel.close(); - } catch (Exception e1) { - // do nothing - } - } - - - public void setSuccessfullyConnected() { - connectionAttempts = 0; - } - - } - -} diff --git a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java deleted file mode 100644 index d3b2f2590..000000000 --- a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java +++ /dev/null @@ -1,620 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.algo.bitset.BitSet; -import net.openhft.chronicle.algo.bitset.ReusableBitSet; -import net.openhft.chronicle.algo.bitset.SingleThreadedFlatBitSetFrame; -import net.openhft.chronicle.algo.bytes.Access; -import net.openhft.chronicle.bytes.Bytes; -import net.openhft.chronicle.bytes.BytesStore; -import net.openhft.chronicle.bytes.HeapBytesStore; -import net.openhft.chronicle.hash.replication.ReplicableEntry; -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.hash.replication.UdpTransportConfig; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.atomic.AtomicReferenceArray; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static java.lang.Math.min; -import static java.nio.ByteBuffer.wrap; -import static net.openhft.chronicle.map.Replica.EntryExternalizable; -import static net.openhft.chronicle.map.Replica.ModificationIterator; - -/** - * @author Rob Austin. - */ -final class ChannelProvider implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(ChannelProvider.class.getName()); - - static final Map implMapping = new IdentityHashMap<>(); - - public static synchronized ChannelProvider getProvider(ReplicationHub hub) throws IOException { - ChannelProvider channelProvider = implMapping.get(hub); - if (channelProvider != null) - return channelProvider; - channelProvider = new ChannelProvider(hub); - TcpTransportAndNetworkConfig tcpConfig = hub.tcpTransportAndNetwork(); - - - if (tcpConfig != null) { - - final TcpReplicator tcpReplicator = new TcpReplicator( - channelProvider.asReplica, - channelProvider.asEntryExternalizable, - tcpConfig, hub.remoteNodeValidator(), hub.name(), - hub.connectionListener()); - channelProvider.add(tcpReplicator); - } - - UdpTransportConfig udpConfig = hub.udpTransport(); - if (udpConfig != null) { - final UdpReplicator udpReplicator = - new UdpReplicator( - channelProvider.asReplica, - channelProvider.asEntryExternalizable, - udpConfig); - channelProvider.add(udpReplicator); - if (tcpConfig == null) - LOG.warn(Replicator.ONLY_UDP_WARN_MESSAGE); - } - implMapping.put(hub, channelProvider); - return channelProvider; - } - - private static final byte BOOTSTRAP_MESSAGE = 'B'; - final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { - - @Override - public boolean identifierCheck(@NotNull ReplicableEntry entry, int chronicleChannel) { - channelDataReadLock(); - try { - - return channelEntryExternalizables[chronicleChannel] - .identifierCheck(entry, chronicleChannel); - } finally { - channelDataLock.readLock().unlock(); - } - } - - /** - * writes the entry to the chronicle channel provided - * @param entry the byte location of the entry to be stored - * @param destination a buffer the entry will be written to, the segment may reject - * this operation and add zeroBytes, if the identifier in the entry - * did not match the maps local - * @param chronicleChannel used in cluster into identify the canonical map or queue - */ - @Override - public void writeExternalEntry(ReplicableEntry entry, Bytes payload, - @NotNull Bytes destination, - int chronicleChannel) { - channelDataReadLock(); - try { - destination.writeStopBit(chronicleChannel); - channelEntryExternalizables[chronicleChannel].writeExternalEntry( - entry, payload, destination, chronicleChannel); - } finally { - channelDataLock.readLock().unlock(); - } - } - - @Override - public void readExternalEntry(@NotNull Bytes source, byte remoteNodeIdentifier) { - channelDataReadLock(); - try { - final int chronicleId = (int) source.readStopBit(); - if (chronicleId < chronicleChannels.length) { - // this channel is has not currently been created so it updates will be ignored - if (channelEntryExternalizables[chronicleId] != null) - channelEntryExternalizables[chronicleId] - .readExternalEntry(source, remoteNodeIdentifier); - } else - LOG.info("skipped entry with chronicleId=" + chronicleId + ", "); - } finally { - channelDataLock.readLock().unlock(); - } - } - }; - - - private final byte localIdentifier; - final Replica asReplica = new Replica() { - @Override - public byte identifier() { - return localIdentifier; - } - - @Override - public ModificationIterator acquireModificationIterator( - final byte remoteIdentifier) { - - channelDataLock.writeLock().lock(); - try { - final ModificationIterator result = modificationIterator.get(remoteIdentifier); - if (result != null) - return result; - - final ModificationIterator result0 = new ModificationIterator() { - - volatile Replica.ModificationNotifier notifier0; - - @Override - public boolean hasNext() { - channelDataReadLock(); - try { - // old-style iteration to avoid 1) iterator object creation - // 2) ConcurrentModificationException - for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - final ModificationIterator modificationIterator = - chronicleChannelList.get(i).acquireModificationIterator( - remoteIdentifier); - if (modificationIterator.hasNext()) - return true; - } - return false; - } finally { - channelDataLock.readLock().unlock(); - } - } - - @Override - public boolean nextEntry(@NotNull EntryCallback callback, - final int na) { - channelDataReadLock(); - try { - for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - - final ModificationIterator modificationIterator = - chronicleChannelList.get(i) - .acquireModificationIterator(remoteIdentifier); - if (modificationIterator - .nextEntry(callback, chronicleChannelIds.get(i))) - return true; - } - return false; - } finally { - channelDataLock.readLock().unlock(); - } - } - - @Override - public void dirtyEntries(long fromTimeStamp) { - channelDataReadLock(); - try { - for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - - chronicleChannelList.get(i) - .acquireModificationIterator(remoteIdentifier) - .dirtyEntries(fromTimeStamp); - notifier0.onChange(); - } - } finally { - channelDataLock.readLock().unlock(); - } - } - - @Override - public void setModificationNotifier( - @NotNull ModificationNotifier modificationNotifier) { - for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - chronicleChannelList.get(i) - .acquireModificationIterator(remoteIdentifier) - .setModificationNotifier(modificationNotifier); - } - notifier0 = modificationNotifier; - } - }; - - modificationIterator.set((int) remoteIdentifier, result0); - return result0; - } finally { - channelDataLock.writeLock().unlock(); - } - } - - /** - * gets the earliest modification time for all of the chronicles - */ - @Override - public long remoteNodeCouldBootstrapFrom(byte remoteIdentifier) { - channelDataReadLock(); - try { - long t = 0; - // not including the SystemQueue at index 0 - for (int i = 1, len = chronicleChannelList.size(); i < len; i++) { - Replica channel = chronicleChannelList.get(i); - t = (t == 0) ? channel.remoteNodeCouldBootstrapFrom(remoteIdentifier) : - min(t, channel.remoteNodeCouldBootstrapFrom(remoteIdentifier)); - } - return t; - } finally { - channelDataLock.readLock().unlock(); - } - } - - @Override - public void setRemoteNodeCouldBootstrapFrom(byte remoteIdentifier, long bootstrapTimestamp) { - channelDataReadLock(); - try { - // not including the SystemQueue at index 0 - for (int i = 1, len = chronicleChannelList.size(); i < len; i++) { - chronicleChannelList.get(i) - .setRemoteNodeCouldBootstrapFrom(remoteIdentifier, bootstrapTimestamp); - } - - } finally { - channelDataLock.readLock().unlock(); - } - } - - @Override - public void close() throws IOException { - ChannelProvider.this.close(); - } - }; - - private void channelDataReadLock() { - try { - channelDataLock.readLock().lockInterruptibly(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - private final ReplicationHub hub; - - - private final ReadWriteLock channelDataLock = new ReentrantReadWriteLock(); - - // start of channel data - private final Replica[] chronicleChannels; - private final int[] chronicleChannelPositionsInList; - private final List chronicleChannelList; - private final List chronicleChannelIds; - - private final EntryExternalizable[] channelEntryExternalizables; - private final AtomicReferenceArray systemModificationIterator = - new AtomicReferenceArray(128); - private final BitSet systemModificationIteratorBitSet = - newBitSet(systemModificationIterator.length()); - private final AtomicReferenceArray modificationIterator = - new AtomicReferenceArray(128); - - // end of channel data - private final Set replicators = new CopyOnWriteArraySet(); - - private volatile boolean isClosed = false; - private final SystemQueue systemMessageQueue; - - private ChannelProvider(ReplicationHub hub) { - localIdentifier = hub.identifier(); - this.hub = hub; - - chronicleChannels = new Replica[hub.maxNumberOfChannels()]; - channelEntryExternalizables = new EntryExternalizable[hub.maxNumberOfChannels()]; - chronicleChannelPositionsInList = new int[hub.maxNumberOfChannels()]; - Arrays.fill(chronicleChannelPositionsInList, -1); - chronicleChannelList = new ArrayList<>(); - chronicleChannelIds = new ArrayList<>(); - MessageHandler systemMessageHandler = new MessageHandler() { - @Override - public void onMessage(Bytes bytes) { - final byte type = bytes.readByte(); - if (type == BOOTSTRAP_MESSAGE) { - onBootstrapMessage(bytes); - } else { - LOG.info("message of type=" + type + " was ignored."); - } - } - }; - systemMessageQueue = new SystemQueue( - systemModificationIteratorBitSet, systemModificationIterator, systemMessageHandler); - add((short) 0, systemMessageQueue.asReplica, systemMessageQueue.asEntryExternalizable); - } - - - /** - * creates a bit set based on a number of bits - * - * @param numberOfBits the number of bits the bit set should include - * @return a new DirectBitSet backed by a byteBuffer - */ - private static BitSet newBitSet(int numberOfBits) { - HeapBytesStore bs = BytesStore.wrap(new byte[(numberOfBits + 7) / 8]); - SingleThreadedFlatBitSetFrame frame = new SingleThreadedFlatBitSetFrame(numberOfBits); - return new ReusableBitSet(frame, Access.checkedBytesStoreAccess(), bs, 0); - } - - public ChronicleChannel createChannel(int channel) { - return new ChronicleChannel(channel); - } - - /** - * called whenever we receive a bootstrap message - */ - private void onBootstrapMessage(Bytes bytes) { - final byte remoteIdentifier = bytes.readByte(); - final int chronicleChannel = bytes.readUnsignedShort(); - final long bootstrapFrom = bytes.readLong(); - - if (LOG.isDebugEnabled()) - LOG.debug("received bootstrap message received for localIdentifier=" + this.localIdentifier + ", " + - "remoteIdentifier=" + remoteIdentifier + ",chronicleChannel=" + chronicleChannel + "," + - "remoteNodeCouldBootstrapFrom=" + bootstrapFrom); - - // this could be null if one node has a chronicle channel before the other - if (chronicleChannels[chronicleChannel] != null) { - chronicleChannels[chronicleChannel].acquireModificationIterator(remoteIdentifier) - .dirtyEntries(bootstrapFrom); - } - } - - private static Bytes toBootstrapMessage( - int chronicleChannel, final long bootstrapFrom, final byte localIdentifier) { - final Bytes writeBuffer = Bytes.wrapForWrite(new byte[1 + 1 + 2 + 8]); - writeBuffer.writeByte(BOOTSTRAP_MESSAGE); - writeBuffer.writeByte(localIdentifier); - writeBuffer.writeUnsignedShort(chronicleChannel); - writeBuffer.writeLong(bootstrapFrom); - return writeBuffer; - } - - private void add(int chronicleChannel, - Replica replica, - @NotNull EntryExternalizable entryExternalizable) { - if (LOG.isDebugEnabled()) - LOG.debug("adding chronicleChannel=" + chronicleChannel + ",entryExternalizable=" + - entryExternalizable); - channelDataLock.writeLock().lock(); - try { - if (chronicleChannels[chronicleChannel] != null) { - throw new IllegalStateException("chronicleId=" + chronicleChannel + - " is already in use."); - } - chronicleChannels[chronicleChannel] = replica; - chronicleChannelList.add(replica); - chronicleChannelIds.add(chronicleChannel); - chronicleChannelPositionsInList[chronicleChannel] = chronicleChannelList.size() - 1; - channelEntryExternalizables[chronicleChannel] = entryExternalizable; - - if (chronicleChannel == 0) - return; - - // send bootstrap message - - for (int i = (int) systemModificationIteratorBitSet.nextSetBit(0); i > 0; - i = (int) systemModificationIteratorBitSet.nextSetBit(i + 1)) { - byte remoteIdentifier = (byte) i; - final long bootstrapFrom = replica.remoteNodeCouldBootstrapFrom(remoteIdentifier); - final Bytes message = - toBootstrapMessage(chronicleChannel, bootstrapFrom, localIdentifier); - systemModificationIterator.get(remoteIdentifier).addPayload(message); - - if (LOG.isDebugEnabled()) - LOG.debug("sending bootstrap message received for localIdentifier=" + localIdentifier + ", " + - "remoteIdentifier=" + remoteIdentifier + ",chronicleChannel=" + chronicleChannel + "," + - "remoteNodeCouldBootstrapFrom=" + bootstrapFrom); - } - } finally { - channelDataLock.writeLock().unlock(); - } - } - - @Override - public void close() throws IOException { - if (isClosed) - return; - // to be synchronized with #getProvider() - synchronized (ChannelProvider.class) { - isClosed = true; - for (Closeable replicator : replicators) { - try { - replicator.close(); - } catch (IOException e) { - LOG.error("", e); - } - } - - implMapping.remove(hub); - } - } - - void add(Closeable replicator) { - replicators.add(replicator); - } - - private interface MessageHandler { - void onMessage(Bytes bytes); - } - - private interface PayloadProvider extends ModificationIterator { - void addPayload(final Bytes bytes); - } - - /** - * used to send system messages such as bootstrap from one remote node to another, it also can - * be used in a broadcast context - */ - class SystemQueue { - - final Replica asReplica = new Replica() { - @Override - public byte identifier() { - throw new UnsupportedOperationException(); - } - - @Override - public ModificationIterator acquireModificationIterator( - final byte remoteIdentifier) { - ModificationIterator result = systemModificationIterator.get(remoteIdentifier); - - if (result != null) - return result; - - final PayloadProvider iterator = new PayloadProvider() { - - final Queue payloads = new LinkedTransferQueue<>(); - ModificationNotifier modificationNotifier0; - - @Override - public boolean hasNext() { - return payloads.peek() != null; - } - - @Override - public boolean nextEntry(@NotNull EntryCallback callback, int na) { - final Bytes payload = payloads.poll(); - if (payload == null) - return false; - callback.onEntry(null, payload, 0); - return true; - } - - @Override - public void dirtyEntries(long fromTimeStamp) { - // do nothing - } - - @Override - public void setModificationNotifier( - @NotNull ModificationNotifier modificationNotifier) { - modificationNotifier0 = modificationNotifier; - } - - @Override - public void addPayload(Bytes bytes) { - if (bytes.readRemaining() == 0) - return; - payloads.add(bytes); - // notifies that a change has been made, this will nudge the OP_WRITE - // selector to push this update out over the nio socket - modificationNotifier0.onChange(); - } - }; - - systemModificationIterator.set(remoteIdentifier, iterator); - systemModificationIteratorBitSet.set(remoteIdentifier); - - return iterator; - } - - @Override - public long remoteNodeCouldBootstrapFrom(byte remoteIdentifier) { - return 0; - } - - @Override - public void setRemoteNodeCouldBootstrapFrom(byte remoteIdentifier, long bootstrapTimestamp) { - // do nothing - } - - @Override - public void close() throws IOException { - // do nothing - } - }; - final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { - - @Override - public boolean identifierCheck(@NotNull ReplicableEntry entry, int chronicleId) { - return true; - } - - @Override - public void writeExternalEntry( - ReplicableEntry entry, @NotNull Bytes payload, @NotNull Bytes destination, - int na) { - destination.write(payload, payload.readPosition(), payload.readRemaining()); - } - - @Override - public void readExternalEntry( - @NotNull Bytes source, byte remoteNodeIdentifier) { - messageHandler.onMessage(source); - } - }; - private final BitSet systemModificationIteratorBitSet; - private final AtomicReferenceArray systemModificationIterator; - private final MessageHandler messageHandler; - - SystemQueue(BitSet systemModificationIteratorBitSet, - AtomicReferenceArray systemModificationIterator, - MessageHandler messageHandler) { - this.systemModificationIteratorBitSet = systemModificationIteratorBitSet; - this.systemModificationIterator = systemModificationIterator; - this.messageHandler = messageHandler; - } - } - - public class ChronicleChannel implements Replicator, Closeable { - - private final int chronicleChannel; - - private ChronicleChannel(int chronicleChannel) { - this.chronicleChannel = chronicleChannel; - } - - public byte identifier() { - return localIdentifier; - } - - @Override - public Closeable applyTo(ChronicleMapBuilder builder, - Replica map, EntryExternalizable entryExternalizable, - final ReplicatedChronicleMap replicatedMap) { - add(chronicleChannel, map, entryExternalizable); - return this; - } - - @Override - public void close() throws IOException { - channelDataLock.writeLock().lock(); - try { - int removedPos = chronicleChannelPositionsInList[chronicleChannel]; - if (removedPos == -1) - return; - chronicleChannelPositionsInList[chronicleChannel] = -1; - chronicleChannelList.remove(removedPos); - chronicleChannelIds.remove(removedPos); - for (int i = 0, len = chronicleChannelIds.size(); i < len; i++) { - int channelId = chronicleChannelIds.get(i); - int pos = chronicleChannelPositionsInList[channelId]; - if (pos > removedPos) - chronicleChannelPositionsInList[channelId] = pos - 1; - } - chronicleChannels[chronicleChannel] = null; - channelEntryExternalizables[chronicleChannel] = null; - - if (chronicleChannelList.size() == 1) // i. e. only SystemQueue is left - ChannelProvider.this.close(); - } finally { - channelDataLock.writeLock().unlock(); - } - } - } -} \ No newline at end of file diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index f6de2f219..1af652829 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -23,12 +23,10 @@ import net.openhft.chronicle.bytes.BytesStore; import net.openhft.chronicle.core.OS; import net.openhft.chronicle.hash.ChronicleHashBuilder; import net.openhft.chronicle.hash.ChronicleHashBuilderPrivateAPI; -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; import net.openhft.chronicle.hash.ChronicleHashRecoveryFailedException; import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable; import net.openhft.chronicle.hash.impl.stage.entry.ChecksumStrategy; import net.openhft.chronicle.hash.impl.util.math.PoissonDistribution; -import net.openhft.chronicle.hash.replication.*; import net.openhft.chronicle.hash.serialization.*; import net.openhft.chronicle.hash.serialization.impl.SerializationBuilder; import net.openhft.chronicle.map.replication.MapRemoteOperations; @@ -41,10 +39,15 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -104,8 +107,6 @@ import static net.openhft.chronicle.map.VanillaChronicleMap.alignAddr; public final class ChronicleMapBuilder implements ChronicleHashBuilder, ChronicleMapBuilder> { - static final byte UDP_REPLICATION_MODIFICATION_ITERATOR_ID = (byte) 127; - private static final int UNDEFINED_ALIGNMENT_CONFIG = -1; private static final int NO_ALIGNMENT = 1; @@ -193,7 +194,7 @@ public final class ChronicleMapBuilder implements DefaultValueProvider defaultValueProvider = DefaultSpi.defaultValueProvider(); - private SingleChronicleHashReplication singleHashReplication = null; + byte replicationIdentifier = -1; MapMethods methods = DefaultSpi.mapMethods(); MapEntryOperations entryOperations = mapEntryOperations(); @@ -1341,27 +1342,13 @@ public final class ChronicleMapBuilder implements return this; } - @Override - public ChronicleMapBuilder replication(SingleChronicleHashReplication replication) { - this.singleHashReplication = replication; - return this; - } - @Override public ChronicleMapBuilder replication(byte identifier) { - return replication(SingleChronicleHashReplication.builder().createWithId(identifier)); - } - - @Override - public ChronicleMapBuilder replication( - byte identifier, TcpTransportAndNetworkConfig tcpTransportAndNetwork) { - return replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpTransportAndNetwork).createWithId(identifier)); - } - - @Override - public ChronicleHashInstanceBuilder> instance() { - return new MapInstanceBuilder<>(this.clone(), singleHashReplication); + if (identifier <= 0) + throw new IllegalArgumentException("Identifier must be positive, " + identifier + + " given"); + this.replicationIdentifier = identifier; + return this; } @Override @@ -1369,13 +1356,13 @@ public final class ChronicleMapBuilder implements // clone() to make this builder instance thread-safe, because createWithFile() method // computes some state based on configurations, but doesn't synchronize on configuration // changes. - return clone().createWithFile(file, singleHashReplication, null, false, false); + return clone().createWithFile(file, false, false); } @Override public ChronicleMap recoverPersistedTo(File file, boolean sameBuilderConfig) throws IOException { - return clone().createWithFile(file, singleHashReplication, null, true, sameBuilderConfig); + return clone().createWithFile(file, true, sameBuilderConfig); } @Override @@ -1383,27 +1370,14 @@ public final class ChronicleMapBuilder implements // clone() to make this builder instance thread-safe, because createWithoutFile() method // computes some state based on configurations, but doesn't synchronize on configuration // changes. - return clone().createWithoutFile(singleHashReplication, null); - } - - ChronicleMap create( - MapInstanceBuilder ib, boolean recover, boolean sameBuilderConfig) - throws IOException { - if (ib.file != null) { - return createWithFile( - ib.file, ib.singleHashReplication, ib.channel, recover, sameBuilderConfig); - } else { - return createWithoutFile(ib.singleHashReplication, ib.channel); - } + return clone().createWithoutFile(); } ChronicleMap createWithFile( - File file, SingleChronicleHashReplication singleHashReplication, - ReplicationChannel channel, - boolean recover, boolean overrideBuilderConfig) throws IOException { + File file, boolean recover, boolean overrideBuilderConfig) throws IOException { if (overrideBuilderConfig && !recover) throw new AssertionError("recover -> overrideBuilderConfig"); - replicated = singleHashReplication != null || channel != null; + replicated = replicationIdentifier != -1; persisted = true; for (int i = 0; i < 10; i++) { @@ -1414,7 +1388,7 @@ public final class ChronicleMapBuilder implements int headerSize = 0; if (overrideBuilderConfig) { VanillaChronicleMap mapObjectForHeaderOverwrite = - newMap(singleHashReplication, channel); + newMap(); headerSize = writeHeader(fileChannel, mapObjectForHeaderOverwrite); } ByteBuffer headerBuffer = ByteBuffer.allocate(MAX_BOOTSTRAPPING_HEADER_SIZE); @@ -1442,7 +1416,6 @@ public final class ChronicleMapBuilder implements } map.initTransientsFromBuilder(this); - initTransientsFromReplication(map, singleHashReplication, channel); if (!recover) { map.createMappedStoreAndSegments(file, raf); @@ -1450,7 +1423,7 @@ public final class ChronicleMapBuilder implements map.recover(file, raf); } - establishReplication(map, singleHashReplication, channel); + establishReplication(map); fileChannel.force(true); // TODO according to Self Boostrapping Data spec, should write "init complete" @@ -1477,7 +1450,7 @@ public final class ChronicleMapBuilder implements if (!file.exists()) throw new FileNotFoundException("Unable to create " + file); - VanillaChronicleMap map = newMap(singleHashReplication, channel); + VanillaChronicleMap map = newMap(); try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) { FileChannel fileChannel = raf.getChannel(); @@ -1485,7 +1458,7 @@ public final class ChronicleMapBuilder implements map.initBeforeMapping(file, fileChannel, headerSize); map.createMappedStoreAndSegments(file, raf); - establishReplication(map, singleHashReplication, channel); + establishReplication(map); return map; } } @@ -1505,30 +1478,12 @@ public final class ChronicleMapBuilder implements return headerSize; } - private static void initTransientsFromReplication( - VanillaChronicleMap map, - SingleChronicleHashReplication singleHashReplication, ReplicationChannel channel) { - if (map instanceof ReplicatedChronicleMap) { - AbstractReplication replication; - if (singleHashReplication != null) { - replication = singleHashReplication; - } else if (channel != null) { - replication = channel.hub(); - } else { - replication = null; - } - if (replication != null) - ((ReplicatedChronicleMap) map).initTransientsFromReplication(replication); - } - } - - ChronicleMap createWithoutFile( - SingleChronicleHashReplication singleHashReplication, ReplicationChannel channel) { - replicated = singleHashReplication != null || channel != null; + ChronicleMap createWithoutFile() { + replicated = replicationIdentifier != -1; persisted = false; try { - VanillaChronicleMap map = newMap(singleHashReplication, channel); + VanillaChronicleMap map = newMap(); // TODO this method had been moved // if(OS.warnOnWindows(map.sizeInBytesWithoutTiers())){ // throw new IllegalStateException("Windows cannot support this configuration"); @@ -1537,7 +1492,7 @@ public final class ChronicleMapBuilder implements BytesStore bytesStore = lazyNativeBytesStoreWithFixedCapacity(map.sizeInBytesWithoutTiers()); map.createMappedStoreAndSegments(bytesStore); - establishReplication(map, singleHashReplication, channel); + establishReplication(map); return map; } catch (IOException e) { // file-less version should never trigger an IOException. @@ -1545,18 +1500,11 @@ public final class ChronicleMapBuilder implements } } - private VanillaChronicleMap newMap( - SingleChronicleHashReplication singleHashReplication, ReplicationChannel channel) + private VanillaChronicleMap newMap() throws IOException { preMapConstruction(); if (replicated) { - AbstractReplication replication; - if (singleHashReplication != null) { - replication = singleHashReplication; - } else { - replication = channel.hub(); - } - return new ReplicatedChronicleMap<>(this, replication); + return new ReplicatedChronicleMap<>(this); } else { return new VanillaChronicleMap<>(this); } @@ -1599,43 +1547,11 @@ public final class ChronicleMapBuilder implements } private void establishReplication( - VanillaChronicleMap map, - SingleChronicleHashReplication singleHashReplication, - ReplicationChannel channel) throws IOException { + VanillaChronicleMap map) throws IOException { if (map instanceof ReplicatedChronicleMap) { - if (singleHashReplication != null && channel != null) - throw new AssertionError("Only one non-null replication should be passed"); - ReplicatedChronicleMap result = (ReplicatedChronicleMap) map; if (cleanupRemovedEntries) establishCleanupThread(result); - - List replicators = new ArrayList<>(2); - if (singleHashReplication != null) { - if (singleHashReplication.tcpTransportAndNetwork() != null) - replicators.add(Replicator.tcp(singleHashReplication)); - if (singleHashReplication.udpTransport() != null) - replicators.add(Replicator.udp(singleHashReplication.udpTransport())); - } else if (channel != null) { - ReplicationHub hub = channel.hub(); - - ChannelProvider provider = ChannelProvider.getProvider(hub); - ChannelProvider.ChronicleChannel ch = provider.createChannel(channel.channelId()); - replicators.add(ch); - } else { - assert persisted && !((ReplicatedChronicleMap) map).createdOrInMemory : - "No Replicators for replicated ChronicleMap could be only on " + - "deserialization/access of existing replicated ChronicleMap, " + - "when replication in not needed in this JVM/run"; - } - for (Replicator replicator : replicators) { - Closeable token = replicator.applyTo(this, result, result, - (ReplicatedChronicleMap)map); - if (replicators.size() == 1 && token.getClass() == UdpReplicator.class) { - LOG.warn(Replicator.ONLY_UDP_WARN_MESSAGE); - } - result.addCloseable(token); - } } } diff --git a/src/main/java/net/openhft/chronicle/map/MapInstanceBuilder.java b/src/main/java/net/openhft/chronicle/map/MapInstanceBuilder.java deleted file mode 100644 index 4646a577f..000000000 --- a/src/main/java/net/openhft/chronicle/map/MapInstanceBuilder.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; -import net.openhft.chronicle.hash.replication.ReplicationChannel; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; - -import java.io.File; -import java.io.IOException; - -final class MapInstanceBuilder implements ChronicleHashInstanceBuilder> { - - final ChronicleMapBuilder mapBuilder; - transient SingleChronicleHashReplication singleHashReplication; - transient ReplicationChannel channel; - transient File file; - String name; - - volatile boolean used = false; - - MapInstanceBuilder(ChronicleMapBuilder mapBuilder, - SingleChronicleHashReplication singleHashReplication) { - this.mapBuilder = mapBuilder; - this.singleHashReplication = singleHashReplication; - } - - @Override - public MapInstanceBuilder replicated( - byte identifier, TcpTransportAndNetworkConfig tcpTransportAndNetwork) { - return replicated(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpTransportAndNetwork).createWithId(identifier)); - } - - @Override - public MapInstanceBuilder replicated(SingleChronicleHashReplication replication) { - singleHashReplication = replication; - channel = null; - return this; - } - - @Override - public MapInstanceBuilder replicatedViaChannel(ReplicationChannel channel) { - singleHashReplication = null; - this.channel = channel; - return this; - } - - @Override - public MapInstanceBuilder persistedTo(File file) { - this.file = file; - return this; - } - - @Override - public MapInstanceBuilder name(String name) { - this.name = name; - return this; - } - - @Override - public synchronized ChronicleMap create() throws IOException { - if (!used) { - ChronicleMap map = mapBuilder.create(this, false, false); - // Set used = true, if map created successfully. - // If exception is thrown in mapBuilder.create(), the next line is not executed. - used = true; - return map; - } else { - throw alreadyCreated(); - } - } - - @Override - public synchronized ChronicleMap recover(boolean sameBuilderConfig) throws IOException { - if (file == null) - throw new IllegalStateException("persistedTo(file) must be configured for " + - "ChronicleHashInstanceBuilder, for recover()"); - if (!used) { - ChronicleMap map = mapBuilder.create(this, true, sameBuilderConfig); - used = true; - return map; - } else { - throw alreadyCreated(); - } - } - - private static IllegalStateException alreadyCreated() { - return new IllegalStateException( - "A ChronicleHash has already been created using this " + - "ChronicleHashInstanceBuilder. Create a new instance builder " + - "(chronicleHashBuilder.instance()) to create another Chronicle Hash"); - } -} diff --git a/src/main/java/net/openhft/chronicle/map/Replica.java b/src/main/java/net/openhft/chronicle/map/Replica.java index 57acf47b8..7bd40f739 100644 --- a/src/main/java/net/openhft/chronicle/map/Replica.java +++ b/src/main/java/net/openhft/chronicle/map/Replica.java @@ -113,7 +113,7 @@ public interface Replica extends Closeable { /** * @return {@code true} if the is another entry to be received via {@link - * #nextEntry(Replica.EntryCallback, int chronicleId)} + * #nextEntry(Callback, int chronicleId)} */ boolean hasNext(); @@ -126,13 +126,14 @@ public interface Replica extends Closeable { * @return {@code true} if the entry was accepted by the {@code callback.onEntry()} method, * {@code false} if the entry was not accepted or was not available */ - boolean nextEntry(@NotNull final EntryCallback callback, final int chronicleId); + boolean nextEntry(@NotNull final Callback callback, final int chronicleId); /** - * Dirties all entries with a modification time equal to {@code fromTimeStamp} or newer. It - * means all these entries will be considered as "new" by this ModificationIterator and - * iterated once again no matter if they have already been.

This functionality is used - * to publish recently modified entries to a new remote node as it connects. + * Dirties all entries with a modification time equal to {@code fromTimeStamp} or newer and + * origin identifier equal to the current node identifier. It means all these entries will + * be considered as "new" by this ModificationIterator and iterated once again no matter if + * they have already been.

This functionality is used to publish recently modified + * entries to a new remote node as it connects. * * @param fromTimeStamp the timestamp from which all entries should be dirty */ @@ -145,6 +146,23 @@ public interface Replica extends Closeable { * @param modificationNotifier gets notified when a change occurs */ void setModificationNotifier(@NotNull final ModificationNotifier modificationNotifier); + + /** + * Implemented typically by a replicator, This interface provides the event, which will get + * called whenever a put() or remove() has occurred to the map + */ + interface Callback { + + /** + * Called whenever a put() or remove() has occurred to a replicating map. + * @param entry the entry you will receive, this does not have to be locked, as + * locking is already provided from the caller. + * @param chronicleId only assigned when clustering + */ + void onEntry(ReplicableEntry entry, int chronicleId); + + void onBootstrapTime(long bootstrapTime, int chronicleId); + } } /** @@ -185,20 +203,5 @@ public interface Replica extends Closeable { void readExternalEntry(@NotNull Bytes source, byte remoteNodeIdentifier); } - /** - * Implemented typically by a replicator, This interface provides the event, which will get - * called whenever a put() or remove() has occurred to the map - */ - interface EntryCallback { - - /** - * Called whenever a put() or remove() has occurred to a replicating map. - * - * @param entry the entry you will receive, this does not have to be locked, as - * locking is already provided from the caller. - * @param chronicleId only assigned when clustering - */ - void onEntry(ReplicableEntry entry, Bytes payload, int chronicleId); - } } diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 3230390d6..52b7a65d3 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -22,7 +22,6 @@ import net.openhft.chronicle.hash.Data; import net.openhft.chronicle.hash.VanillaGlobalMutableState; import net.openhft.chronicle.hash.impl.TierCountersArea; import net.openhft.chronicle.hash.impl.stage.hash.ChainingInterface; -import net.openhft.chronicle.hash.replication.AbstractReplication; import net.openhft.chronicle.hash.replication.ReplicableEntry; import net.openhft.chronicle.map.impl.CompiledReplicatedMapIterationContext; import net.openhft.chronicle.map.impl.CompiledReplicatedMapQueryContext; @@ -116,7 +115,6 @@ public class ReplicatedChronicleMap extends VanillaChronicleMap modificationIterators; private transient long startOfModificationIterators; - private transient boolean bootstrapOnlyLocalEntries; public transient boolean cleanupRemovedEntries; public transient long cleanupTimeout; @@ -128,8 +126,7 @@ public class ReplicatedChronicleMap extends VanillaChronicleMap builder, - AbstractReplication replication) throws IOException { + public ReplicatedChronicleMap(@NotNull ChronicleMapBuilder builder) throws IOException { super(builder); tierModIterBitSetSizeInBits = computeTierModIterBitSetSizeInBits(); tierModIterBitSetOuterSize = computeTierModIterBitSetOuterSize(); @@ -137,8 +134,6 @@ public class ReplicatedChronicleMap extends VanillaChronicleMap extends VanillaChronicleMap builder) { super.initTransientsFromBuilder(builder); + this.localIdentifier = builder.replicationIdentifier; + if (localIdentifier == -1) + throw new IllegalStateException("localIdentifier should not be -1"); //noinspection unchecked this.remoteOperations = (MapRemoteOperations) builder.remoteOperations; cleanupRemovedEntries = builder.cleanupRemovedEntries; @@ -200,13 +198,6 @@ public class ReplicatedChronicleMap extends VanillaChronicleMap extends VanillaChronicleMap extends VanillaChronicleMap extends VanillaChronicleMap extends VanillaChronicleMap= 0) { allBitSetsScannedFromTheStart = segmentIndex == 0 && entryPos == -1; if (allBitSetsScannedFromTheStart) { - setBootstrapTimeAfterNextReplicationIterationComplete(currentTime()); + bootstrapTimeAfterNextIterationComplete = currentTime(); somethingSentOnThisIteration = false; } while (segmentIndex < actualSegments) { // This is needed to ensure, that any entry update with the timestamp, - // smaller than assigned for - // bootstrapTimeAfterNextReplicationIterationComplete, is visible during - // the iteration of the current segment. Bits are raised during the update - // via non-volatile bit set (performance concerns), hence to guarantee - // visibility during the iteration, we build a happens-before between bit - // raise and bit reading: + // smaller than assigned for bootstrapTimeAfterNextIterationComplete, is + // visible during the iteration of the current segment. Bits are raised + // during the update via non-volatile bit set (performance concerns), hence + // to guarantee visibility during the iteration, we build a happens-before + // between bit raise and bit reading: // bit raised -> // lock released (end of update operation) -> // lock acquired (the following acquireAndReleaseUpdateLock() call) -> @@ -815,32 +796,29 @@ public class ReplicatedChronicleMap extends VanillaChronicleMap c = - iterationContext()) { + try (CompiledReplicatedMapIterationContext c = iterationContext()) { c.initSegmentIndex(segmentIndex); c.updateLock().lock(); } } /** - * @param entryCallback call this to get an entry, this class will take care of the locking + * @param callback call this to get an entry, this class will take care of the locking * @return true if an entry was processed */ @Override - public boolean nextEntry(@NotNull EntryCallback entryCallback, int chronicleId) { + public boolean nextEntry(@NotNull Callback callback, int chronicleId) { while (true) { - long nextEntryPos = nextEntryPos(entryCallback, chronicleId); + long nextEntryPos = nextEntryPos(callback, chronicleId); if (nextEntryPos == NOT_FOUND) return false; entryPos = nextEntryPos; @@ -866,8 +844,8 @@ public class ReplicatedChronicleMap extends VanillaChronicleMap extends VanillaChronicleMap= fromTimeStamp) { - raiseChange0(c.tierIndex(), c.pos()); - } - } else { - // TODO currently, all entries, originating not from the current node, - // are bootstrapped. This could be optimized, but requires to generate - // unique connection id, it identify two ChronicleMap instances - // reconnecting vs. different Map start-up - if (e.originIdentifier() != localIdentifier || - e.originTimestamp() >= fromTimeStamp) { - raiseChange0(c.tierIndex(), c.pos()); - } + // TODO currently, all entries, originating not from the current node, + // are bootstrapped. This could be optimized, but requires to generate + // unique connection id, it identify two ChronicleMap instances + // reconnecting vs. different Map start-up + if (e.originIdentifier() != localIdentifier || + e.originTimestamp() >= fromTimeStamp) { + raiseChange0(c.tierIndex(), c.pos()); } }); } diff --git a/src/main/java/net/openhft/chronicle/map/Replicator.java b/src/main/java/net/openhft/chronicle/map/Replicator.java deleted file mode 100644 index 45b49c32b..000000000 --- a/src/main/java/net/openhft/chronicle/map/Replicator.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.AbstractReplication; -import net.openhft.chronicle.hash.replication.UdpTransportConfig; - -import java.io.Closeable; -import java.io.IOException; - -interface Replicator { - - String ONLY_UDP_WARN_MESSAGE = - "MISSING TCP REPLICATION : The UdpReplicator only attempts to read data " + - "(it does not enforce or guarantee delivery), you should use" + - "the UdpReplicator if you have a large number of nodes, and you wish" + - "to receive the data before it becomes available on TCP/IP. Since data" + - "delivery is not guaranteed, it is recommended that you only use" + - "the UDP Replicator in conjunction with a TCP Replicator"; - - static Replicator tcp(final AbstractReplication replication) { - return (builder, replica, entryExternalizable, replicatedMap) -> - new TcpReplicator(replica, entryExternalizable, - replication.tcpTransportAndNetwork(), replication.remoteNodeValidator(), - replication.name(), replication.connectionListener()); - } - - static Replicator udp(final UdpTransportConfig replicationConfig) { - return (builder, map, entryExternalizable, replicatedMap) -> - new UdpReplicator(map, entryExternalizable, replicationConfig); - } - - /** - * Applies the replicator to the map instance and returns a Closeable token to manage resources, - * associated with the replication.

This method isn't intended to be called from the client - * code. - * - * @param builder the builder from which the map was constructed. The replicator may - * obtain some map configurations, not accessible via the map - * instance. - * @param map a replicated map instance. Provides basic tools for replication - * implementation. - * @param entryExternalizable the callback for ser/deser implementation in the replicator - * @return a {@code Closeable} token to control replication resources. It should be closed on - * closing the replicated map. - * @throws java.io.IOException if an io error occurred during the replicator setup - * @throws IllegalStateException if this replicator doesn't support application to more than one - * map (or the specified number of maps), and this replicator has - * already been applied to a map (or the specified number of - * maps) - */ - Closeable applyTo( - ChronicleMapBuilder builder, Replica map, - Replica.EntryExternalizable entryExternalizable, ReplicatedChronicleMap replicatedMap) - throws IOException; -} diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java deleted file mode 100644 index 2e8f29d3c..000000000 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ /dev/null @@ -1,1437 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.algo.bitset.ConcurrentFlatBitSetFrame; -import net.openhft.chronicle.algo.bitset.ReusableBitSet; -import net.openhft.chronicle.bytes.Bytes; -import net.openhft.chronicle.bytes.BytesStore; -import net.openhft.chronicle.hash.impl.util.BuildVersion; -import net.openhft.chronicle.hash.replication.ConnectionListener; -import net.openhft.chronicle.hash.replication.RemoteNodeValidator; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.hash.replication.ThrottlingConfig; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.net.*; -import java.nio.BufferUnderflowException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.CharBuffer; -import java.nio.channels.*; -import java.nio.charset.StandardCharsets; -import java.util.BitSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -import static java.nio.channels.SelectionKey.*; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static net.openhft.chronicle.algo.MemoryUnit.*; -import static net.openhft.chronicle.algo.bytes.Access.checkedBytesStoreAccess; -import static net.openhft.chronicle.hash.impl.util.BuildVersion.version; - -/** - * Used with a {@link net.openhft.chronicle.map.ReplicatedChronicleMap} to send data between the - * maps using a socket connection {@link net.openhft.chronicle.map.TcpReplicator} - * - * @author Rob Austin. - */ -final class TcpReplicator extends AbstractChannelReplicator implements Closeable { - - private static final byte HEARTBEAT = 0; - private static final byte NOT_SET = 0;//(byte) HEARTBEAT.ordinal(); - private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName()); - private static final int BUFFER_SIZE = 0x100000; // 1MB - - public static final long SPIN_LOOP_TIME_IN_NONOSECONDS = TimeUnit.MICROSECONDS.toNanos(500); - private final SelectionKey[] selectionKeysStore = new SelectionKey[Byte.MAX_VALUE + 1]; - // used to instruct the selector thread to set OP_WRITE on a key correlated by the bit index - // in the bitset - private final KeyInterestUpdater opWriteUpdater = - new KeyInterestUpdater(OP_WRITE, selectionKeysStore); - private final BitSet activeKeys = new BitSet(selectionKeysStore.length); - private final long heartBeatIntervalMillis; - private final ConnectionListener connectionListener; - private long largestEntrySoFar = 128; - - @NotNull - private final Replica replica; - private final byte localIdentifier; - - @NotNull - private final Replica.EntryExternalizable externalizable; - @NotNull - private final TcpTransportAndNetworkConfig replicationConfig; - - - private final - @Nullable - RemoteNodeValidator remoteNodeValidator; - private final String name; - - private long selectorTimeout; - - - enum State { - CONNECTED, DISCONNECTED; - } - - /** - * @throws IOException on an io error. - */ - public TcpReplicator(@NotNull final Replica replica, - @NotNull final Replica.EntryExternalizable externalizable, - @NotNull final TcpTransportAndNetworkConfig replicationConfig, - @Nullable final RemoteNodeValidator remoteNodeValidator, - String name, - @Nullable final ConnectionListener connectionListener) - throws IOException { - - super("TcpSocketReplicator-" + replica.identifier(), replicationConfig.throttlingConfig()); - - final ThrottlingConfig throttlingConfig = replicationConfig.throttlingConfig(); - long throttleBucketInterval = throttlingConfig.bucketInterval(MILLISECONDS); - - heartBeatIntervalMillis = replicationConfig.heartBeatInterval(MILLISECONDS); - - selectorTimeout = Math.min(heartBeatIntervalMillis / 4, throttleBucketInterval); - - this.replica = replica; - this.localIdentifier = replica.identifier(); - - this.externalizable = externalizable; - this.replicationConfig = replicationConfig; - - this.remoteNodeValidator = remoteNodeValidator; - this.name = name; - - this.connectionListener = (connectionListener == null) ? null : new ConnectionListener() { - - private final Map listenerStateMap = new ConcurrentHashMap<>(); - - @Override - public void onConnect(InetAddress address, byte identifier, boolean isServer) { - - if (listenerStateMap.put(address, State.CONNECTED) == State.CONNECTED) - return; - - connectionListener.onConnect(address, identifier, isServer); - } - - @Override - public void onDisconnect(InetAddress address, byte identifier) { - if (listenerStateMap.put(address, State.DISCONNECTED) == State.DISCONNECTED) - return; - - connectionListener.onDisconnect(address, identifier); - } - }; - - start(); - } - - @Override - void processEvent() throws IOException { - try { - final InetSocketAddress serverInetSocketAddress = - new InetSocketAddress(replicationConfig.serverPort()); - final Details serverDetails = new Details(serverInetSocketAddress, localIdentifier); - new ServerConnector(serverDetails).connect(); - - for (InetSocketAddress client : replicationConfig.endpoints()) { - final Details clientDetails = new Details(client, localIdentifier); - new ClientConnector(clientDetails).connect(); - } - - while (selector.isOpen()) { - registerPendingRegistrations(); - - final int nSelectedKeys = select(); - - // its less resource intensive to set this less frequently and use an approximation - final long approxTime = System.currentTimeMillis(); - - checkThrottleInterval(); - - // check that we have sent and received heartbeats - heartBeatMonitor(approxTime); - - // set the OP_WRITE when data is ready to send - opWriteUpdater.applyUpdates(); - - if (useJavaNIOSelectionKeys) { - // use the standard java nio selector - - if (nSelectedKeys == 0) - continue; // go back and check pendingRegistrations - - final Set selectionKeys = selector.selectedKeys(); - for (final SelectionKey key : selectionKeys) { - processKey(approxTime, key); - } - selectionKeys.clear(); - } else { - // use the netty like selector - - final SelectionKey[] keys = selectedKeys.flip(); - - try { - for (int i = 0; i < keys.length && keys[i] != null; i++) { - final SelectionKey key = keys[i]; - - try { - processKey(approxTime, key); - } catch (BufferUnderflowException e) { - if (!isClosed) - LOG.error("", e); - } - } - } finally { - for (int i = 0; i < keys.length && keys[i] != null; i++) { - keys[i] = null; - } - } - } - } - } catch (CancelledKeyException | ConnectException | ClosedChannelException | - ClosedSelectorException e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } catch (Exception e) { - LOG.error("", e); - } catch (Throwable e) { - LOG.error("", e); - throw e; - } finally { - - if (LOG.isDebugEnabled()) - LOG.debug("closing name=" + name); - if (!isClosed) { - closeResources(); - } - } - } - - private void processKey(long approxTime, @NotNull SelectionKey key) { - try { - - if (!key.isValid()) - return; - - if (key.isAcceptable()) { - if (LOG.isDebugEnabled()) - LOG.debug("onAccept - " + name); - onAccept(key); - } - - if (key.isConnectable()) { - if (LOG.isDebugEnabled()) - LOG.debug("onConnect - " + name); - onConnect(key); - } - - if (key.isReadable()) { - if (LOG.isDebugEnabled()) - LOG.debug("onRead - " + name); - onRead(key, approxTime); - } - if (key.isWritable()) { - if (LOG.isDebugEnabled()) - LOG.debug("onWrite - " + name); - onWrite(key, approxTime); - } - } catch (InterruptedException e) { - quietClose(key, e); - } catch (BufferUnderflowException | IOException | - ClosedSelectorException | CancelledKeyException e) { - if (!isClosed) - quietClose(key, e); - } catch (Exception e) { - LOG.info("", e); - if (!isClosed) - closeEarlyAndQuietly(key.channel()); - } - } - - /** - * spin loops 100000 times first before calling the selector with timeout - * - * @return The number of keys, possibly zero, whose ready-operation sets were updated - * @throws IOException - */ - private int select() throws IOException { - - long start = System.nanoTime(); - - while (System.nanoTime() < start + SPIN_LOOP_TIME_IN_NONOSECONDS) { - final int keys = selector.selectNow(); - if (keys != 0) - return keys; - } - - return selector.select(selectorTimeout); - } - - /** - * checks that we receive heartbeats and send out heart beats. - * - * @param approxTime the approximate time in milliseconds - */ - - void heartBeatMonitor(long approxTime) { - for (int i = activeKeys.nextSetBit(0); i >= 0; i = activeKeys.nextSetBit(i + 1)) { - try { - final SelectionKey key = selectionKeysStore[i]; - if (!key.isValid() || !key.channel().isOpen()) { - activeKeys.clear(i); - continue; - } - - final Attached attachment = (Attached) key.attachment(); - - if (attachment == null) - continue; - - if (!attachment.hasRemoteHeartbeatInterval) - continue; - - try { - sendHeartbeatIfRequired(approxTime, key); - } catch (Exception e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } - - try { - heartbeatCheckHasReceived(key, approxTime); - } catch (Exception e) { - - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } - } catch (Exception e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } - } - } - - - /** - * check to see if its time to send a heartbeat, and send one if required - * - * @param approxTime the current time ( approximately ) - * @param key nio selection key - */ - private void sendHeartbeatIfRequired(final long approxTime, - @NotNull final SelectionKey key) { - final Attached attachment = (Attached) key.attachment(); - - if (attachment.isHandShakingComplete() && attachment.entryWriter.lastSentTime + - heartBeatIntervalMillis < approxTime) { - attachment.entryWriter.lastSentTime = approxTime; - attachment.entryWriter.writeHeartbeatToBuffer(); - - enableOpWrite(key); - - if (LOG.isDebugEnabled()) - LOG.debug("sending heartbeat"); - } - } - - private void enableOpWrite(@NotNull SelectionKey key) { - int ops = key.interestOps(); - if ((ops & (OP_CONNECT | OP_ACCEPT)) == 0) - key.interestOps(ops | OP_WRITE); - } - - /** - * check to see if we have lost connection with the remote node and if we have attempts a - * reconnect. - * - * @param key the key relating to the heartbeat that we are checking - * @param approxTimeOutTime the approximate time in milliseconds - * @throws ConnectException - */ - private void heartbeatCheckHasReceived(@NotNull final SelectionKey key, - final long approxTimeOutTime) { - - final Attached attached = (Attached) key.attachment(); - - if (attached.isServer && !attached.isHandShakingComplete()) - return; - - final SocketChannel channel = (SocketChannel) key.channel(); - - if (approxTimeOutTime > - attached.entryReader.lastHeartBeatReceived + attached.remoteHeartbeatInterval) { - if (LOG.isDebugEnabled()) - LOG.debug("lost connection, attempting to reconnect. " + - "missed heartbeat from identifier=" + attached.remoteIdentifier); - activeKeys.clear(attached.remoteIdentifier); - quietClose(key, null); - - closeables.closeQuietly(channel.socket()); - - // when node discovery is used ( by nodes broadcasting out their host:port over UDP ), - // when new or restarted nodes are started up. they attempt to find the nodes - // on the grid by listening to the host and ports of the other nodes, so these nodes - // will establish the connection when they come back up, hence under these - // circumstances, polling a dropped node to attempt to reconnect is no-longer - // required as the remote node will establish the connection its self on startup. - if (replicationConfig.autoReconnectedUponDroppedConnection()) - attached.connector.connectLater(); - } - } - - /** - * closes and only logs the exception at debug - * - * @param key the SelectionKey - * @param e the Exception that caused the issue - */ - private void quietClose(@NotNull final SelectionKey key, @Nullable final Exception e) { - if (LOG.isDebugEnabled() && e != null) - LOG.debug("", e); - - if (key.channel() != null && key.attachment() != null && connectionListener != null) - connectionListener.onDisconnect(((SocketChannel) key.channel()).socket().getInetAddress(), - ((Attached) key.attachment()).remoteIdentifier); - - - closeEarlyAndQuietly(key.channel()); - } - - /** - * called when the selector receives a OP_CONNECT message - */ - private void onConnect(@NotNull final SelectionKey key) - throws IOException { - - SocketChannel channel = null; - - try { - channel = (SocketChannel) key.channel(); - } finally { - closeables.add(channel); - } - - final Attached attached = (Attached) key.attachment(); - - try { - if (!channel.finishConnect()) { - return; - } - - channel.configureBlocking(false); - channel.socket().setTcpNoDelay(true); - channel.socket().setSoTimeout(0); - channel.socket().setSoLinger(false, 0); - - } catch (IOException e) { - quietClose(key, e); - - // when node discovery is used ( by nodes broadcasting out their host:port over UDP ), - // when new or restarted nodes are started up. they attempt to find the nodes - // on the grid by listening to the host and ports of the other nodes, - // so these nodes will establish the connection when they come back up, - // hence under these circumstances, polling a dropped node to attempt to reconnect - // is no-longer required as the remote node will establish the connection its self - // on startup. - - attached.connector.connectLater(); - - throw e; - } - - attached.connector.setSuccessfullyConnected(); - - if (LOG.isDebugEnabled()) - LOG.debug("successfully connected to {}, local-id={}", - channel.socket().getInetAddress(), localIdentifier); - - attached.entryReader = new TcpSocketChannelEntryReader(); - attached.entryWriter = new TcpSocketChannelEntryWriter(); - - key.interestOps(OP_WRITE | OP_READ); - - throttle(channel); - - // register it with the selector and store the ModificationIterator for this key - attached.entryWriter.identifierToBuffer(localIdentifier); - } - - /** - * called when the selector receives a OP_ACCEPT message - */ - private void onAccept(@NotNull final SelectionKey key) throws IOException { - ServerSocketChannel server = null; - - try { - server = (ServerSocketChannel) key.channel(); - } finally { - if (server != null) - closeables.add(server); - } - - SocketChannel channel = null; - - try { - channel = server.accept(); - } finally { - if (channel != null) - closeables.add(channel); - } - - channel.configureBlocking(false); - channel.socket().setReuseAddress(true); - channel.socket().setTcpNoDelay(true); - channel.socket().setSoTimeout(0); - channel.socket().setSoLinger(false, 0); - - final Attached attached = new Attached(); - - final InetSocketAddress serverInetSocketAddress = - new InetSocketAddress(replicationConfig.serverPort()); - final Details serverDetails = new Details(serverInetSocketAddress, localIdentifier); - - attached.connector = new ServerConnector(serverDetails); - attached.entryReader = new TcpSocketChannelEntryReader(); - attached.entryWriter = new TcpSocketChannelEntryWriter(); - - attached.entryWriter.identifierToBuffer(localIdentifier); - attached.isServer = true; - - channel.register(selector, OP_READ, attached); - - throttle(channel); - } - - - /** - * check that the version number is valid text, - * - * @param versionNumber the version number to check - * @return true, if the version number looks correct - */ - boolean isValidVersionNumber(String versionNumber) { - - if (versionNumber.length() <= 2) - return false; - - for (char c : versionNumber.toCharArray()) { - - if (c >= '0' && c <= '9') - continue; - - if (c == '.' || c == '-' || c == '_') - continue; - - if (c >= 'A' && c <= 'Z') - continue; - - if (c >= 'a' && c <= 'z') - continue; - - return false; - - } - - return true; - } - - private void checkVersions(final Attached attached) { - - final String localVersion = BuildVersion.version(); - final String remoteVersion = attached.serverVersion; - - - if (!remoteVersion.equals(localVersion)) { - byte remoteIdentifier = attached.remoteIdentifier; - LOG.warn("DIFFERENT CHRONICLE-MAP VERSIONS : " + - "local-map=" + localVersion + - ", remote-map-id-" + remoteIdentifier + "=" + - remoteVersion + - - ", The Remote Chronicle Map with " + - "identifier=" + remoteIdentifier + - " and this Chronicle Map are on different " + - "versions, we suggest that you use the same version." - - ); - } - } - - /** - * used to exchange identifiers and timestamps and heartbeat intervals between the server and - * client - * - * @param key the SelectionKey relating to the this cha - * @param socketChannel - * @throws java.io.IOException - * @throws InterruptedException - */ - private void doHandShaking(@NotNull final SelectionKey key, - @NotNull SocketChannel socketChannel) - throws IOException { - final Attached attached = (Attached) key.attachment(); - final TcpSocketChannelEntryWriter writer = attached.entryWriter; - final TcpSocketChannelEntryReader reader = attached.entryReader; - - socketChannel.register(selector, OP_READ | OP_WRITE, attached); - - if (attached.remoteIdentifier == Byte.MIN_VALUE) { - final byte remoteIdentifier = reader.identifierFromBuffer(); - - if (remoteIdentifier == Byte.MIN_VALUE) - return; - - attached.remoteIdentifier = remoteIdentifier; - - final SocketChannel channel = (SocketChannel) key.channel(); - if (channel != null && channel.socket() != null && connectionListener != null) { - connectionListener.onConnect( - channel.socket().getInetAddress(), - attached.remoteIdentifier, - attached.isServer); - } - - // we use the as iterating the activeKeys via the bitset wont create and Objects - // but if we use the selector.keys() this will. - selectionKeysStore[remoteIdentifier] = key; - activeKeys.set(remoteIdentifier); - - if (LOG.isDebugEnabled()) { - LOG.debug("server-connection id={}, remoteIdentifier={}", localIdentifier, - remoteIdentifier); - } - - // this can occur sometimes when if 2 or more remote node attempt to use node discovery - // at the same time - - final SocketAddress remoteAddress = socketChannel.getRemoteAddress(); - - if ((remoteNodeValidator != null && - !remoteNodeValidator.validate(remoteIdentifier, remoteAddress)) - || remoteIdentifier == localIdentifier) - - // throwing this exception will cause us to disconnect, both the client and server - // will be able to detect the the remote and local identifiers are the same, - // as the identifier is send early on in the hand shaking via the connect(() - // and accept() methods - throw new IllegalStateException("dropping connection, " + - "as the remote-identifier is already being used, identifier=" + - remoteIdentifier); - if (LOG.isDebugEnabled()) - LOG.debug("handshaking for localIdentifier=" + localIdentifier + "," + - "remoteIdentifier=" + remoteIdentifier); - - attached.remoteModificationIterator = - replica.acquireModificationIterator(remoteIdentifier); - - attached.remoteModificationIterator.setModificationNotifier(attached); - - writer.writeRemoteBootstrapTimestamp( - replica.remoteNodeCouldBootstrapFrom(remoteIdentifier)); - - writer.writeServerVersion(); - - // tell the remote node, what are heartbeat interval is - writer.writeRemoteHeartbeatInterval(heartBeatIntervalMillis); - } - - if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) { - attached.remoteBootstrapTimestamp = reader.remoteBootstrapTimestamp(); - if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) - return; - } - - if (attached.serverVersion == null) { - try { - attached.serverVersion = reader.readRemoteServerVersion(); - } catch (IllegalStateException e1) { - socketChannel.close(); - return; - } - - if (attached.serverVersion == null) - return; - - if (!isValidVersionNumber(attached.serverVersion)) { - LOG.warn("Closing the remote connection : Please check that you don't have " + - "a third party system incorrectly connecting to ChronicleMap, " + - "remoteAddress=" + socketChannel.getRemoteAddress() + ", " + - "so closing the remote connection as Chronicle can not make sense " + - "of the remote version number received from the external connection, " + - "version=" + attached.serverVersion + ", " + - "Chronicle is expecting the version number to only contain " + - "'.','-', ,A-Z,a-z,0-9"); - socketChannel.close(); - return; - } - - checkVersions(attached); - } - - if (!attached.hasRemoteHeartbeatInterval) { - final long value = reader.readRemoteHeartbeatIntervalFromBuffer(); - - if (value == Long.MIN_VALUE) - return; - - if (value < 0) { - LOG.error("value=" + value); - } - - // we add a 10% safety margin to the timeout time due to latency fluctuations - // on the network, in other words we wont consider a connection to have - // timed out, unless the heartbeat interval has exceeded 25% of the expected time. - attached.remoteHeartbeatInterval = (long) (value * 1.25); - - // we have to make our selector poll interval at least as short as the minimum selector - // timeout - selectorTimeout = Math.min(selectorTimeout, value); - - if (selectorTimeout < 0) - LOG.info(""); - - attached.hasRemoteHeartbeatInterval = true; - - // now we're finished we can get on with reading the entries - attached.remoteModificationIterator.dirtyEntries(attached.remoteBootstrapTimestamp); - try { - reader.entriesFromBuffer(attached, key); - } finally { - attached.handShakingComplete = true; - } - } - } - - /** - * called when the selector receives a OP_WRITE message - */ - private void onWrite(@NotNull final SelectionKey key, - final long approxTime) throws IOException { - final SocketChannel socketChannel = (SocketChannel) key.channel(); - final Attached attached = (Attached) key.attachment(); - if (attached == null) { - LOG.info("Closing connection " + socketChannel + ", nothing attached"); - socketChannel.close(); - return; - } - - TcpSocketChannelEntryWriter entryWriter = attached.entryWriter; - - if (entryWriter == null) throw - new NullPointerException("No entryWriter"); - - if (entryWriter.isWorkIncomplete()) { - final boolean completed = entryWriter.doWork(); - - if (completed) - entryWriter.workCompleted(); - - } else if (attached.remoteModificationIterator != null) - entryWriter.entriesToBuffer(attached.remoteModificationIterator); - - try { - final int len = entryWriter.writeBufferToSocket(socketChannel, approxTime); - - if (len == -1) - socketChannel.close(); - - if (len > 0) - contemplateThrottleWrites(len); - - if (!entryWriter.hasBytesToWrite() - && !entryWriter.isWorkIncomplete() - && !hasNext(attached) - && attached.isHandShakingComplete()) { - // if we have no more data to write to the socket then we will - // un-register OP_WRITE on the selector, until more data becomes available - // The OP_WRITE will get added back in as soon as we have data to write - - if (LOG.isDebugEnabled()) - LOG.debug("Disabling OP_WRITE to remoteIdentifier=" + - attached.remoteIdentifier + - ", localIdentifier=" + localIdentifier); - key.interestOps(key.interestOps() & ~OP_WRITE); - } - } catch (IOException e) { - quietClose(key, e); - if (!attached.isServer) - attached.connector.connectLater(); - throw e; - } - } - - private boolean hasNext(Attached attached) { - return attached.remoteModificationIterator != null - && attached.remoteModificationIterator.hasNext(); - } - - /** - * called when the selector receives a OP_READ message - */ - - private void onRead(@NotNull final SelectionKey key, - final long approxTime) throws IOException, InterruptedException { - - final SocketChannel socketChannel = (SocketChannel) key.channel(); - final Attached attached = (Attached) key.attachment(); - - if (attached == null) { - LOG.info("Closing connection " + socketChannel + ", nothing attached"); - socketChannel.close(); - return; - } - - try { - - int len = attached.entryReader.readSocketToBuffer(socketChannel); - if (len == -1) { - socketChannel.register(selector, 0, attached); - if (replicationConfig.autoReconnectedUponDroppedConnection()) { - // TODO why != null check here, absent everywhere else? on connectLater() calls? - // TODO shouldn't check attached.isServer, same as in catch (IOE) clause below? - AbstractConnector connector = attached.connector; - if (connector != null) - connector.connectLater(); - } else - socketChannel.close(); - return; - } - - if (len == 0) - return; - - if (attached.entryWriter.isWorkIncomplete()) - return; - } catch (IOException e) { - if (!attached.isServer) - attached.connector.connectLater(); - throw e; - } - - if (LOG.isDebugEnabled()) - LOG.debug("heartbeat or data received."); - - attached.entryReader.lastHeartBeatReceived = approxTime; - - if (attached.isHandShakingComplete()) { - attached.entryReader.entriesFromBuffer(attached, key); - } else { - doHandShaking(key, socketChannel); - } - } - - @Nullable - private ServerSocketChannel openServerSocketChannel() throws IOException { - ServerSocketChannel result = null; - - try { - result = ServerSocketChannel.open(); - } finally { - if (result != null) - closeables.add(result); - } - return result; - } - - /** - * sets interestOps to "selector keys",The change to interestOps much be on the same thread as - * the selector. This class, allows via {@link AbstractChannelReplicator - * .KeyInterestUpdater#set(int)} to holds a pending change in interestOps ( via a bitset ), - * this change is processed later on the same thread as the selector - */ - private static class KeyInterestUpdater { - - @NotNull - private final net.openhft.chronicle.algo.bitset.BitSet changeOfOpWriteRequired; - @NotNull - private final SelectionKey[] selectionKeys; - private final int op; - - KeyInterestUpdater(int op, @NotNull final SelectionKey[] selectionKeys) { - this.op = op; - this.selectionKeys = selectionKeys; - int bitSetSizeInBytes = - (int) LONGS.align(BYTES.alignAndConvert(selectionKeys.length, BITS), BYTES); - ConcurrentFlatBitSetFrame frame = new ConcurrentFlatBitSetFrame(selectionKeys.length); - changeOfOpWriteRequired = new ReusableBitSet(frame, checkedBytesStoreAccess(), - BytesStore.wrap(new byte[bitSetSizeInBytes]), 0); - } - - public void applyUpdates() { - for (long i = changeOfOpWriteRequired.clearNextSetBit(0L); i >= 0; - i = changeOfOpWriteRequired.clearNextSetBit(i + 1)) { - final SelectionKey key = selectionKeys[((int) i)]; - try { - key.interestOps(key.interestOps() | op); - } catch (Exception e) { - LOG.debug("", e); - } - } - } - - /** - * @param keyIndex the index of the key that has changed, the list of keys is provided by - * the constructor {@link KeyInterestUpdater(int, SelectionKey[])} - */ - public void set(int keyIndex) { - changeOfOpWriteRequired.setIfClear(keyIndex); - } - } - - private class ServerConnector extends AbstractConnector { - - @NotNull - private final Details details; - - private ServerConnector(@NotNull Details details) { - super("TCP-ServerConnector-" + localIdentifier); - this.details = details; - } - - @NotNull - @Override - public String toString() { - return "ServerConnector{" + - "" + details + - '}'; - } - - @Nullable - SelectableChannel doConnect() throws IOException, InterruptedException { - - final ServerSocketChannel serverChannel = openServerSocketChannel(); - - if (serverChannel == null) - return null; - - serverChannel.socket().setReceiveBufferSize(BUFFER_SIZE); - serverChannel.configureBlocking(false); - serverChannel.register(TcpReplicator.this.selector, 0); - ServerSocket serverSocket = null; - - try { - serverSocket = serverChannel.socket(); - } finally { - if (serverSocket != null) - closeables.add(serverSocket); - } - - if (serverSocket == null) - return null; - - serverSocket.setReuseAddress(true); - - serverSocket.bind(details.address()); - - // these can be run on this thread - addPendingRegistration(() -> { - final Attached attached = new Attached(); - attached.connector = ServerConnector.this; - try { - serverChannel.register(TcpReplicator.this.selector, OP_ACCEPT, attached); - } catch (ClosedChannelException e) { - LOG.debug("", e); - } - }); - - selector.wakeup(); - - return serverChannel; - } - } - - private class ClientConnector extends AbstractConnector { - - @NotNull - private final Details details; - - private ClientConnector(@NotNull Details details) { - super("TCP-ClientConnector-" + details.localIdentifier()); - this.details = details; - } - - @NotNull - @Override - public String toString() { - return "ClientConnector{" + details + '}'; - } - - /** - * blocks until connected - */ - @Override - SelectableChannel doConnect() throws IOException, InterruptedException { - boolean success = false; - - final SocketChannel socketChannel = openSocketChannel(TcpReplicator.this.closeables); - - try { - socketChannel.configureBlocking(false); - socketChannel.socket().setReuseAddress(true); - socketChannel.socket().setSoLinger(false, 0); - socketChannel.socket().setSoTimeout(0); - - socketChannel.connect(details.address()); - - // Under experiment, the concoction was found to be more successful if we - // paused before registering the OP_CONNECT - Thread.sleep(10); - - // the registration has be be run on the same thread as the selector - addPendingRegistration(new Runnable() { - @Override - public void run() { - final Attached attached = new Attached(); - attached.connector = ClientConnector.this; - - try { - socketChannel.register(selector, OP_CONNECT, attached); - } catch (ClosedChannelException e) { - if (socketChannel.isOpen()) - LOG.error("", e); - onFail(socketChannel); - } - } - }); - - selector.wakeup(); - success = true; - return socketChannel; - } finally { - if (!success) { - onFail(socketChannel); - } - } - } - - private void onFail(SocketChannel socketChannel) { - try { - try { - socketChannel.socket().close(); - } catch (Exception e) { - LOG.error("", e); - } - socketChannel.close(); - } catch (IOException e) { - LOG.error("", e); - } finally { - this.connectLater(); - } - } - } - - /** - * Attached to the NIO selection key via methods such as {@link SelectionKey#attach(Object)} - */ - class Attached implements Replica.ModificationNotifier { - - public TcpSocketChannelEntryReader entryReader; - public TcpSocketChannelEntryWriter entryWriter; - - @Nullable - public Replica.ModificationIterator remoteModificationIterator; - // used for connect later - public AbstractConnector connector; - public long remoteBootstrapTimestamp = Long.MIN_VALUE; - public byte remoteIdentifier = Byte.MIN_VALUE; - public boolean hasRemoteHeartbeatInterval; - // true if its socket is a ServerSocket - public boolean isServer; - public boolean handShakingComplete; - public String serverVersion; - public long remoteHeartbeatInterval = heartBeatIntervalMillis; - - boolean isHandShakingComplete() { - return handShakingComplete; - } - - /** - * called whenever there is a change to the modification iterator - */ - @Override - public void onChange() { - if (remoteIdentifier != Byte.MIN_VALUE) - TcpReplicator.this.opWriteUpdater.set(remoteIdentifier); - } - } - - /** - * @author Rob Austin. - */ - public class TcpSocketChannelEntryWriter { - - @NotNull - private final EntryCallback entryCallback; - // if uncompletedWork is set ( not null ) , this must be completed before any further work - // is carried out - @Nullable - public Work uncompletedWork; - private long lastSentTime; - - private TcpSocketChannelEntryWriter() { - entryCallback = new EntryCallback(externalizable, replicationConfig.tcpBufferSize()); - } - - public boolean isWorkIncomplete() { - return uncompletedWork != null; - } - - public void workCompleted() { - uncompletedWork = null; - } - - /** - * writes the timestamp into the buffer - * - * @param localIdentifier the current nodes identifier - */ - void identifierToBuffer(final byte localIdentifier) { - entryIn().writeByte(localIdentifier); - } - - public Bytes entryIn() { - return entryCallback.entryIn(); - } - - private ByteBuffer socketOut() { - return entryCallback.socketOut(); - } - - /** - * sends the identity and timestamp of this node to a remote node - * - * @param timeStampOfLastMessage the last timestamp we received a message from that node - */ - void writeRemoteBootstrapTimestamp(final long timeStampOfLastMessage) { - entryIn().writeLong(timeStampOfLastMessage); - } - - void writeServerVersion() { - String version = String.format("%1$" + 64 + "s", version()); - ByteBuffer codedVersion = StandardCharsets.US_ASCII.encode(version); - if (codedVersion.remaining() != 64) - throw new AssertionError(); - entryIn().write(codedVersion); - } - - /** - * writes all the entries that have changed, to the buffer which will later be written to - * TCP/IP - * - * @param modificationIterator a record of which entries have modification - */ - void entriesToBuffer(@NotNull final Replica.ModificationIterator modificationIterator) { - - int entriesWritten = 0; - try { - for (; ; entriesWritten++) { - - long start = entryIn().writePosition(); - - boolean success = modificationIterator.nextEntry(entryCallback, 0); - - // if not success this is most likely due to the next entry not fitting into - // the buffer and the buffer can not be re-sized above Integer.max_value, in - // this case success will return false, so we return so that we can send to - // the socket what we have. - if (!success) - return; - - long entrySize = entryIn().writePosition() - start; - - if (entrySize > largestEntrySoFar) - largestEntrySoFar = entrySize; - - // we've filled up the buffer lets give another channel a chance to send - // some data - if (entryIn().writeRemaining() <= largestEntrySoFar || - entryIn().writePosition() > replicationConfig.tcpBufferSize()) - return; - - // if we have space in the buffer to write more data and we just wrote data - // into the buffer then let try and write some more - } - } finally { - if (LOG.isDebugEnabled()) - LOG.debug("Entries written: {}", entriesWritten); - } - } - - /** - * writes the contents of the buffer to the socket - * - * @param socketChannel the socket to publish the buffer to - * @param approxTime an approximation of the current time in millis - * @throws IOException - */ - private int writeBufferToSocket(@NotNull final SocketChannel socketChannel, - final long approxTime) throws IOException { - - final Bytes in = entryIn(); - final ByteBuffer out = socketOut(); - - int bytesToWrite = (int) in.readRemaining(); - if (bytesToWrite == 0) - return 0; - - // if we still have some unwritten writer from last time - lastSentTime = approxTime; - - out.position(0); - out.limit(bytesToWrite); - - final int bytesWritten = socketChannel.write(out); - - if (LOG.isDebugEnabled()) - LOG.debug("bytes-written=" + bytesWritten); - - if (bytesWritten == bytesToWrite) { - out.clear(); - in.clear(); - } else { - out.compact(); - in.writePosition(out.position()); - out.clear(); - } - - return bytesWritten; - } - - - /** - * used to send an single zero byte if we have not send any data for up to the - * localHeartbeatInterval - */ - private void writeHeartbeatToBuffer() { - // denotes the state - 0 for a heartbeat - entryIn().writeByte(HEARTBEAT); - - // denotes the size in bytes - entryIn().writeInt(0); - } - - private void writeRemoteHeartbeatInterval(long localHeartbeatInterval) { - entryIn().writeLong(localHeartbeatInterval); - } - - public boolean doWork() { - return uncompletedWork != null && uncompletedWork.doWork(entryIn()); - } - - public boolean hasBytesToWrite() { - return entryIn().readRemaining() > 0; - } - } - - /** - * Reads map entries from a socket, this could be a client or server socket - */ - class TcpSocketChannelEntryReader { - public static final int HEADROOM = 1024; - public long lastHeartBeatReceived = System.currentTimeMillis(); - ByteBuffer socketIn; - Bytes entryOut; - private long sizeInBytes; - private byte state; - - private TcpSocketChannelEntryReader() { - socketIn = ByteBuffer.allocateDirect(replicationConfig.tcpBufferSize()); - // TODO why here (and in UDP) bytes created through slice? - entryOut = Bytes.wrapForRead(socketIn.slice()); - entryOut.readLimit(0); - socketIn.clear(); - } - - void resizeBuffer(long size) { - assert size < Integer.MAX_VALUE; - - if (size < socketIn.capacity()) - throw new IllegalStateException("it not possible to resize the buffer smaller"); - - final ByteBuffer buffer = ByteBuffer.allocateDirect((int) size) - .order(ByteOrder.nativeOrder()); - - final int inPosition = socketIn.position(); - - long outPosition = entryOut.readPosition(); - long outLimit = entryOut.readLimit(); - - entryOut = Bytes.wrapForRead(buffer.slice()); - - socketIn.flip(); - buffer.put(socketIn); - buffer.clear(); - buffer.position(inPosition); - socketIn = buffer; - - entryOut.readLimit(outLimit); - entryOut.readPosition(outPosition); - } - - /** - * reads from the socket and writes them to the buffer - * - * @param socketChannel the socketChannel to read from - * @return the number of bytes read - * @throws IOException - */ - private int readSocketToBuffer(@NotNull final SocketChannel socketChannel) - throws IOException { - - compactBuffer(); - final int bytesRead = socketChannel.read(socketIn); - entryOut.readLimit(socketIn.position()); - return bytesRead; - } - - /** - * reads entries from the buffer till empty - */ - void entriesFromBuffer(@NotNull Attached attached, @NotNull SelectionKey key) { - int entriesRead = 0; - try { - for (; ; entriesRead++) { - entryOut.readLimit(socketIn.position()); - - // its set to MIN_VALUE when it should be read again - if (state == NOT_SET) { - if (entryOut.readRemaining() < SIZE_OF_SIZE + 1) - return; - - // state is used for heartbeat - state = entryOut.readByte(); - sizeInBytes = entryOut.readInt(); - - assert sizeInBytes >= 0; - - // if the buffer is too small to read this payload we will have to grow the - // size of the buffer - long requiredSize = sizeInBytes + SIZE_OF_SIZE + 1; - if (entryOut.capacity() < requiredSize) { - attached.entryReader.resizeBuffer(requiredSize + HEADROOM); - } - - // means this is heartbeat - if (state == NOT_SET) { - entriesRead--; - continue; - } - } - - if (entryOut.readRemaining() < sizeInBytes) { - return; - } - - final long nextEntryPos = entryOut.readPosition() + sizeInBytes; - assert nextEntryPos > 0; - final long limit = entryOut.readLimit(); - entryOut.readLimit(nextEntryPos); - - externalizable.readExternalEntry(entryOut, attached.remoteIdentifier); - - entryOut.readLimit(limit); - - // skip onto the next entry - entryOut.readPosition(nextEntryPos); - - state = NOT_SET; - sizeInBytes = 0; - } - } finally { - if (LOG.isDebugEnabled()) - LOG.debug("Entries read: {}", entriesRead); - } - } - - /** - * compacts the buffer and updates the {@code socketIn} and {@code entryOut} accordingly - */ - private void compactBuffer() { - // the maxEntrySizeBytes used here may not be the maximum size of the entry in its - // serialized form however, its only use as an indication that the buffer is becoming - // full and should be compacted the buffer can be compacted at any time - if (socketIn.position() == 0 || socketIn.remaining() > largestEntrySoFar) - return; - - socketIn.limit(socketIn.position()); - assert entryOut.readPosition() < Integer.MAX_VALUE; - socketIn.position((int) entryOut.readPosition()); - - socketIn.compact(); - entryOut.readPosition(0); - } - - /** - * @return the identifier or -1 if unsuccessful - */ - byte identifierFromBuffer() { - return (entryOut.readRemaining() >= 1) ? entryOut.readByte() : Byte.MIN_VALUE; - } - - /** - * @return the timestamp or -1 if unsuccessful - */ - long remoteBootstrapTimestamp() { - if (entryOut.readRemaining() >= 8) - return entryOut.readLong(); - else - return Long.MIN_VALUE; - } - - /** - * @return the timestamp or -1 if unsuccessful - */ - String readRemoteServerVersion() { - if (entryOut.readRemaining() >= 64) { - ByteBuffer bytes = ByteBuffer.allocate(64); - entryOut.read(bytes); - if (bytes.flip().remaining() != 64) - throw new AssertionError(); - CharBuffer version = StandardCharsets.US_ASCII.decode(bytes); - return version.toString().trim(); - } else - return null; - } - - - public long readRemoteHeartbeatIntervalFromBuffer() { - return (entryOut.readRemaining() >= 8) ? entryOut.readLong() : Long.MIN_VALUE; - } - } - - interface Work { - - /** - * @param in the buffer that we will fill up - * @return true when all the work is complete - */ - boolean doWork(@NotNull Bytes in); - } -} - diff --git a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java deleted file mode 100644 index fdd4ea163..000000000 --- a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.UdpTransportConfig; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.net.*; -import java.nio.BufferUnderflowException; -import java.nio.channels.*; -import java.util.Set; - -import static java.net.StandardProtocolFamily.INET; -import static java.net.StandardProtocolFamily.INET6; -import static java.net.StandardSocketOptions.IP_MULTICAST_IF; -import static java.net.StandardSocketOptions.SO_REUSEADDR; -import static java.nio.channels.SelectionKey.OP_READ; -import static java.nio.channels.SelectionKey.OP_WRITE; - -/** - * The UdpReplicator attempts to read the data ( but it does not enforce or grantee delivery ), - * typically, you should use the UdpReplicator if you have a large number of nodes, and you wish to - * receive the data before it becomes available on TCP/IP. In order to not miss data, UdpReplicator - * should be used in conjunction with the TCP Replicator. - */ -class UdpChannelReplicator extends AbstractChannelReplicator - implements Replica.ModificationNotifier, Closeable { - - private static final Logger LOG = - LoggerFactory.getLogger(UdpChannelReplicator.class.getName()); - - private final byte localIdentifier; - private EntryWriter writer; - private EntryReader reader; - - private final InetAddress address; - private final int port; - private final NetworkInterface networkInterface; - private final ServerConnector serverConnector; - - private SelectableChannel writeChannel; - private volatile boolean shouldEnableOpWrite; - private final String name; - - /** - * @param replicationConfig - * @param localIdentifier just used for logging - * @throws IOException - */ - UdpChannelReplicator(@NotNull final UdpTransportConfig replicationConfig, - final byte localIdentifier) - throws IOException { - - super("UdpReplicator-" + localIdentifier, replicationConfig.throttlingConfig() - ); - - this.localIdentifier = localIdentifier; - - address = replicationConfig.address(); - port = replicationConfig.port(); - networkInterface = replicationConfig.networkInterface(); - name = replicationConfig.name(); - serverConnector = new ServerConnector(); - } - - void setWriter(EntryWriter writer) { - this.writer = writer; - } - - void setReader(EntryReader reader) { - this.reader = reader; - } - - @Override - public void close() { - super.close(); - writeChannel = null; - } - - /** - * binds to the server socket and process data This method will block until interrupted - */ - - - @Override - void processEvent() throws IOException { - try { - - connectClient().register(selector, OP_READ); - serverConnector.connectLater(); - - while (selector.isOpen()) { - registerPendingRegistrations(); - - // this may block for a long time, upon return the - // selected set contains keys of the ready channels - final int n = selector.select(100); - - if (shouldEnableOpWrite) - enableWrites(); - - checkThrottleInterval(); - - if (n == 0) { - continue; // nothing to do - } - - if (useJavaNIOSelectionKeys) { - // use the standard java nio selector - - final Set selectionKeys = selector.selectedKeys(); - for (final SelectionKey key : selectionKeys) { - processKey(key); - } - selectionKeys.clear(); - } else { - // use the netty like selector - - final SelectionKey[] keys = selectedKeys.flip(); - - try { - for (int i = 0; i < keys.length && keys[i] != null; i++) { - final SelectionKey key = keys[i]; - - try { - processKey(key); - } catch (BufferUnderflowException e) { - if (!isClosed) - LOG.error("", e); - } - } - } finally { - for (int i = 0; i < keys.length && keys[i] != null; i++) { - keys[i] = null; - } - } - } - } - } catch (CancelledKeyException | ConnectException | ClosedChannelException | - ClosedSelectorException e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } catch (Exception e) { - LOG.error("", e); - } catch (Throwable e) { - LOG.error("", e); - throw e; - } finally { - - if (LOG.isDebugEnabled()) - LOG.debug("closing name=" + this.name); - if (!isClosed) { - closeResources(); - } - } - } - - private void processKey(SelectionKey key) { - - try { - - if (key.isReadable()) { - final DatagramChannel socketChannel = (DatagramChannel) key.channel(); - reader.readAll(socketChannel); - } - - if (key.isWritable()) { - final DatagramChannel socketChannel = (DatagramChannel) key.channel(); - try { - int bytesJustWritten = writer.writeAll(socketChannel); - contemplateThrottleWrites(bytesJustWritten); - } catch (NotYetConnectedException e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - serverConnector.connectLater(); - } catch (IOException e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - serverConnector.connectLater(); - } - } - } catch (Exception e) { - LOG.error("", e); - if (!isClosed) - closeEarlyAndQuietly(key.channel()); - } - - } - - private DatagramChannel connectClient() throws IOException { - final DatagramChannel client = address.isMulticastAddress() ? - DatagramChannel.open(address.getAddress().length == 4 ? INET : INET6) : - DatagramChannel.open(); - - final InetSocketAddress hostAddress = new InetSocketAddress(port); - client.configureBlocking(false); - if (address.isMulticastAddress()) { - client.setOption(SO_REUSEADDR, true); - client.bind(hostAddress); - if (networkInterface != null) { - // This is probably not needed, because client socket doesn't send datagrams, - // but since EVERYBODY on the internet configures this for any channels, and - // I don't see any harm this config could make, I leave it here - client.setOption(IP_MULTICAST_IF, networkInterface); - client.join(address, networkInterface); - } else { - client.join(address, NetworkInterface.getByInetAddress(hostAddress.getAddress())); - } - if (LOG.isDebugEnabled()) - LOG.debug("Connecting via multicast, group=" + address); - } else { - client.bind(hostAddress); - } - - if (LOG.isDebugEnabled()) - LOG.debug("Listening on port " + port); - closeables.add(client); - return client; - } - - /** - * called whenever there is a change to the modification iterator - */ - @Override - public void onChange() { - // the write have to be enabled on the same thread as the selector - shouldEnableOpWrite = true; - } - - private void enableWrites() { - if (writeChannel == null) - return; - - try { - final SelectionKey selectionKey = writeChannel.keyFor(this.selector); - if (selectionKey != null) - selectionKey.interestOps(selectionKey.interestOps() | OP_WRITE); - // we have just enabled it, so don't have to do this again. - shouldEnableOpWrite = false; - } catch (Exception e) { - LOG.error("", e); - } - } - - void disableWrites() { - if (writeChannel == null) - return; - - try { - final SelectionKey selectionKey = writeChannel.keyFor(this.selector); - selectionKey.interestOps(selectionKey.interestOps() & ~OP_WRITE); - } catch (Exception e) { - LOG.error("", e); - } - } - - private class ServerConnector extends AbstractConnector { - private final InetSocketAddress socketAddress; - - private ServerConnector() { - super("UDP-Connector"); - this.socketAddress = new InetSocketAddress(address, port); - } - - SelectableChannel doConnect() throws - IOException, InterruptedException { - - final DatagramChannel server = DatagramChannel.open(); - server.configureBlocking(false); - - // Kick off connection establishment - try { - // Create a non-blocking socket channel - server.socket().setBroadcast(true); - server.connect(socketAddress); - } catch (IOException e) { - if (LOG.isDebugEnabled()) - LOG.debug("details=" + new Details(socketAddress, localIdentifier), e); - connectLater(); - return null; - } - - server.setOption(SO_REUSEADDR, true) - .setOption(StandardSocketOptions.IP_MULTICAST_LOOP, false) - .setOption(StandardSocketOptions.SO_BROADCAST, true) - .setOption(SO_REUSEADDR, true); - if (networkInterface != null) - server.setOption(StandardSocketOptions.IP_MULTICAST_IF, networkInterface); - - // the registration has be be run on the same thread as the selector - addPendingRegistration(new Runnable() { - @Override - public void run() { - try { - server.register(selector, OP_WRITE); - writeChannel = server; - throttle(server); - } catch (ClosedChannelException e) { - LOG.error("", e); - } - } - }); - - return server; - } - } - -} - -interface EntryReader { - void readAll(@NotNull final DatagramChannel socketChannel) throws IOException, InterruptedException; -} - -interface EntryWriter { - int writeAll(@NotNull final DatagramChannel socketChannel) throws InterruptedException, IOException; -} - diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java deleted file mode 100644 index c46dca206..000000000 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.bytes.Bytes; -import net.openhft.chronicle.hash.replication.UdpTransportConfig; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.DatagramChannel; - -import static net.openhft.chronicle.map.ChronicleMapBuilder.UDP_REPLICATION_MODIFICATION_ITERATOR_ID; - -/** - * The UdpReplicator attempts to read the data ( but it does not enforce or grantee delivery ), - * typically, you should use the UdpReplicator if you have a large number of nodes, and you wish to - * receive the data before it becomes available on TCP/IP. In order to not miss data, UdpReplicator - * should be used in conjunction with the TCP Replicator. - */ -final class UdpReplicator extends UdpChannelReplicator - implements Replica.ModificationNotifier, Closeable { - - private static final Logger LOG = - LoggerFactory.getLogger(UdpReplicator.class.getName()); - public static final int UPD_BUFFER_SIZE = 64 * 1204; - - public UdpReplicator(@NotNull final Replica replica, - @NotNull final Replica.EntryExternalizable entryExternalizable, - @NotNull final UdpTransportConfig replicationConfig) - throws IOException { - - super(replicationConfig, replica.identifier()); - - Replica.ModificationIterator modificationIterator = replica.acquireModificationIterator( - UDP_REPLICATION_MODIFICATION_ITERATOR_ID); - modificationIterator.setModificationNotifier(this); - setReader(new UdpSocketChannelEntryReader(replicationConfig.udpBufferSize(), entryExternalizable)); - - setWriter(new UdpSocketChannelEntryWriter(replicationConfig.udpBufferSize(), - entryExternalizable, modificationIterator, this, UPD_BUFFER_SIZE)); - - start(); - } - - private class UdpSocketChannelEntryReader implements EntryReader { - - private final Replica.EntryExternalizable externalizable; - private final ByteBuffer socketIn; - private final Bytes entryOut; - - /** - * @param serializedEntrySize the maximum size of an entry include the meta data - * @param externalizable supports reading and writing serialize entries - */ - UdpSocketChannelEntryReader(final int serializedEntrySize, - @NotNull final Replica.EntryExternalizable externalizable) { - // we make the buffer twice as large just to give ourselves headroom - socketIn = ByteBuffer.allocateDirect(serializedEntrySize * 2); - this.externalizable = externalizable; - entryOut = Bytes.wrapForRead(socketIn.slice()); - entryOut.readLimit(0); - socketIn.clear(); - } - - /** - * reads entries from the socket till it is empty - * - * @param socketChannel the socketChannel that we will read from - * @throws IOException - * @throws InterruptedException - */ - public void readAll(@NotNull final DatagramChannel socketChannel) throws IOException, - InterruptedException { - - entryOut.clear(); - socketIn.clear(); - - socketChannel.receive(socketIn); - - final int bytesRead = socketIn.position(); - - if (bytesRead < SIZE_OF_SIZE + SIZE_OF_SIZE) - return; - - entryOut.readLimit(socketIn.position()); - - final int invertedSize = entryOut.readInt(); - final int size = entryOut.readInt(); - - // check the the first 4 bytes are the inverted len followed by the len - // we do this to check that this is a valid start of entry, otherwise we throw it away - if (~size != invertedSize) - return; - - if (entryOut.readRemaining() != size) - return; - - externalizable.readExternalEntry(entryOut, UDP_REPLICATION_MODIFICATION_ITERATOR_ID); - } - } - - private static class UdpSocketChannelEntryWriter implements EntryWriter { - - - private final EntryCallback entryCallback; - private final UdpChannelReplicator udpReplicator; - private final Bytes entryIn; - private Replica.ModificationIterator modificationIterator; - - UdpSocketChannelEntryWriter(final int updBufferSize, - @NotNull final Replica.EntryExternalizable externalizable, - @NotNull final Replica.ModificationIterator modificationIterator, - UdpChannelReplicator udpReplicator, int udpBufferSize) { - this.udpReplicator = udpReplicator; - - this.entryIn = Bytes.elasticByteBuffer(udpBufferSize); - // we make the buffer twice as large just to give ourselves headroom - - entryCallback = new EntryCallback(externalizable, updBufferSize); - this.modificationIterator = modificationIterator; - } - - /** - * writes all the entries that have changed, to the tcp socket - * - * @param socketChannel - * @param modificationIterator - * @throws InterruptedException - * @throws java.io.IOException - */ - - /** - * update that are throttled are rejected. - * - * @param socketChannel the socketChannel that we will write to - * @throws InterruptedException - * @throws IOException - */ - public int writeAll(@NotNull final DatagramChannel socketChannel) - throws InterruptedException, IOException { - - final Bytes in = entryIn; - final ByteBuffer out = entryIn.underlyingObject(); - - out.clear(); - in.clear(); - in.writeSkip(SIZE_OF_SIZE); - - final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0); - - if (!wasDataRead) { - udpReplicator.disableWrites(); - return 0; - } - - // we'll write the size inverted at the start - in.writeShort(0, (short) ~(in.readUnsignedShort(SIZE_OF_SIZE))); - out.limit((int) in.writePosition()); - - return socketChannel.write(out); - } - } - -} - diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index be956da46..f528bd4e0 100755 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -18,10 +18,7 @@ package net.openhft.chronicle.set; import net.openhft.chronicle.hash.ChronicleHashBuilder; import net.openhft.chronicle.hash.ChronicleHashBuilderPrivateAPI; -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; import net.openhft.chronicle.hash.Data; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; import net.openhft.chronicle.hash.serialization.*; import net.openhft.chronicle.map.*; import net.openhft.chronicle.map.replication.MapRemoteOperations; @@ -265,19 +262,6 @@ public final class ChronicleSetBuilder return this; } - @Override - public ChronicleSetBuilder replication(SingleChronicleHashReplication replication) { - chronicleMapBuilder.replication(replication); - return this; - } - - @Override - public ChronicleSetBuilder replication( - byte identifier, TcpTransportAndNetworkConfig tcpTransportAndNetwork) { - chronicleMapBuilder.replication(identifier, tcpTransportAndNetwork); - return this; - } - @Override public ChronicleSetBuilder replication(byte identifier) { chronicleMapBuilder.replication(identifier); @@ -316,11 +300,6 @@ public final class ChronicleSetBuilder return this; } - @Override - public ChronicleHashInstanceBuilder> instance() { - return new SetInstanceBuilder<>(chronicleMapBuilder.instance()); - } - @Override public ChronicleSet create() { final ChronicleMap map = chronicleMapBuilder.create(); diff --git a/src/main/java/net/openhft/chronicle/set/SetInstanceBuilder.java b/src/main/java/net/openhft/chronicle/set/SetInstanceBuilder.java deleted file mode 100644 index 6869f86e4..000000000 --- a/src/main/java/net/openhft/chronicle/set/SetInstanceBuilder.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.set; - -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; -import net.openhft.chronicle.hash.replication.ReplicationChannel; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.VanillaChronicleMap; - -import java.io.File; -import java.io.IOException; - -final class SetInstanceBuilder implements ChronicleHashInstanceBuilder> { - - private final ChronicleHashInstanceBuilder> mapBuilder; - - SetInstanceBuilder(ChronicleHashInstanceBuilder> mapBuilder) { - this.mapBuilder = mapBuilder; - } - - @Override - public ChronicleHashInstanceBuilder> replicated( - byte identifier, TcpTransportAndNetworkConfig tcpTransportAndNetwork) { - mapBuilder.replicated(identifier, tcpTransportAndNetwork); - return this; - } - - @Override - public ChronicleHashInstanceBuilder> replicated( - SingleChronicleHashReplication replication) { - mapBuilder.replicated(replication); - return this; - } - - @Override - public ChronicleHashInstanceBuilder> replicatedViaChannel( - ReplicationChannel channel) { - mapBuilder.replicatedViaChannel(channel); - return this; - } - - @Override - public ChronicleHashInstanceBuilder> persistedTo(File file) { - mapBuilder.persistedTo(file); - return this; - } - - @Override - public ChronicleHashInstanceBuilder> name(String name) { - mapBuilder.name(name); - return this; - } - - @Override - public synchronized ChronicleSet create() throws IOException { - return new SetFromMap<>((VanillaChronicleMap) mapBuilder.create()); - } - - @Override - public synchronized ChronicleSet recover(boolean sameBuilderConfig) throws IOException { - ChronicleMap map = mapBuilder.recover(sameBuilderConfig); - return new SetFromMap<>((VanillaChronicleMap) map); - } -} diff --git a/src/test/java/eg/TestNesterInterfaceMarshal.java b/src/test/java/eg/TestNesterInterfaceMarshal.java deleted file mode 100644 index 96000e587..000000000 --- a/src/test/java/eg/TestNesterInterfaceMarshal.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package eg; - -import net.openhft.chronicle.core.io.Closeable; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.set.Builder; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleMapBuilder; -import net.openhft.chronicle.values.Values; -import org.junit.Assert; -import org.junit.Test; - -import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; - -/** - * Created by Vanitha on 12/5/2014. - */ -public class TestNesterInterfaceMarshal { - - - /** - * Test that both maps are created first and then populated the first map so that second will - * get the contents - * - * @throws Exception - */ - @Test - public void testReplicatedMap() throws Exception { - - TcpTransportAndNetworkConfig tcpTransportAndNetworkConfig = - TcpTransportAndNetworkConfig.of(8076).heartBeatInterval(1L, TimeUnit.SECONDS); - - TcpTransportAndNetworkConfig tcpTransportAndNetworkConfig1 = - TcpTransportAndNetworkConfig.of(8077, new InetSocketAddress("127.0.0.1", 8076)) - .heartBeatInterval(1L, TimeUnit.SECONDS); - - try (ChronicleMap map = - ChronicleMapBuilder - .of(CharSequence.class, TestInstrumentVOInterface.class) - .entries(5000L) - .averageKeySize(10) - .replication((byte) 1, tcpTransportAndNetworkConfig).create(); - - ChronicleMap replicatedMap = - ChronicleMapBuilder - .of(CharSequence.class, TestInstrumentVOInterface.class) - .putReturnsNull(true) - .removeReturnsNull(true) - .entries(5000L).averageKeySize(10) - .replication((byte) 2, tcpTransportAndNetworkConfig1).create()) { - //Store some data into MAP1 - TestInstrumentVOInterface intrumentVOInterface = - Values.newNativeReference(TestInstrumentVOInterface.class); - - - try (Closeable c = map.acquireContext("KEY1", intrumentVOInterface)) { - intrumentVOInterface.setSymbol("Flyer"); - intrumentVOInterface.setCurrencyCode("USA"); - intrumentVOInterface.setSizeOfInstrumentIDArray(2); - - // TestIntrumentVOInterface.TestInstrumentIDVOInterface instrumentIDVOInterface = - intrumentVOInterface.getInstrumentIDAt(0); - // instrumentIDVOInterface.setIdSource("CUSIP"); - // instrumentIDVOInterface.setSecurityId("TEST"); - - // intrumentVOInterface.setInstrumentIDAt(0, instrumentIDVOInterface); - - - // Assert.assertNotNull(map); - // Assert.assertNotNull(replicatedMap); - } - - Builder.waitTillEqual(map, replicatedMap, 5000); - - Assert.assertEquals(map, replicatedMap); - Assert.assertTrue(!replicatedMap.isEmpty()); - - System.out.println(replicatedMap.get("KEY1")); - } - } - - protected ChronicleMap createReplicatedMap1() { - - ChronicleMap chronicleMap = null; - TcpTransportAndNetworkConfig tcpTransportAndNetworkConfig = - TcpTransportAndNetworkConfig.of(8076).heartBeatInterval(1L, TimeUnit.SECONDS); - try { - ChronicleMapBuilder builder = ChronicleMapBuilder - .of(CharSequence.class, TestInstrumentVOInterface.class) - .entries(5000L) - .averageKeySize(10) - .replication((byte) 1, tcpTransportAndNetworkConfig); - - - chronicleMap = builder.create(); - - } catch (Exception e) { - System.out.println("Error(s) creating instrument cache: " + e); - } - return chronicleMap; - } - - protected ChronicleMap createReplicatedMap2() { - - ChronicleMap chronicleMap = null; - TcpTransportAndNetworkConfig tcpTransportAndNetworkConfig = - TcpTransportAndNetworkConfig.of(8077, new InetSocketAddress("127.0.0.1", 8076)) - .heartBeatInterval(1L, TimeUnit.SECONDS); - - try { - ChronicleMapBuilder builder = ChronicleMapBuilder - .of(CharSequence.class, TestInstrumentVOInterface.class) - .putReturnsNull(true) - .removeReturnsNull(true) - .entries(5000L) - .averageKeySize(10) - .replication((byte) 2, tcpTransportAndNetworkConfig); - - - chronicleMap = builder.create(); - - } catch (Exception e) { - System.out.println("*********************Error(s) creating launcher " + - "instrument cache: " + e); - } - return chronicleMap; - } - -} - - diff --git a/src/test/java/net/openhft/chronicle/TestReplication.java b/src/test/java/net/openhft/chronicle/TestReplication.java deleted file mode 100755 index d7d1b00f4..000000000 --- a/src/test/java/net/openhft/chronicle/TestReplication.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle; - -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleMapBuilder; -import org.junit.Assert; -import org.junit.Test; - -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; - -/** - * Created by Rob Austin - */ -public class TestReplication { - - - @Test - public void testAllDataGetsReplicated() throws InterruptedException { - - - TcpTransportAndNetworkConfig tcpConfigServer1 = - TcpTransportAndNetworkConfig.of(8082); - - TcpTransportAndNetworkConfig tcpConfigServer2 = - TcpTransportAndNetworkConfig.of(8083, new InetSocketAddress("localhost", - 8082)); - - final ChronicleMap map2 = ChronicleMapBuilder.of(Integer.class, - Integer.class) - .replication((byte) 2, tcpConfigServer2) - .entries(70000) - .create(); - - - final ChronicleMap map1 = ChronicleMapBuilder.of(Integer.class, - Integer.class) - .replication((byte) 3, tcpConfigServer1) - .entries(70000) - .create(); - - - for (int i = 0; i < 70000; i++) { - map1.put(i, i); - } - - - for (int i = 0; i < 10; i++) { - Thread.sleep(100); - System.out.println(map2.size()); - } - - - Assert.assertEquals(map1.size(), map2.size()); - - } - - - public static final int SIZE = 1000; - private Map> maps = new HashMap<>(); - - - private ReplicationHub hubOnServer1; - - - -} - diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index 1ba58a3c9..67dd1fd68 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -21,8 +21,6 @@ import net.openhft.chronicle.bytes.Bytes; import net.openhft.chronicle.bytes.BytesMarshallable; import net.openhft.chronicle.core.util.SerializableFunction; import net.openhft.chronicle.core.values.*; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; import net.openhft.chronicle.hash.serialization.DataAccess; import net.openhft.chronicle.hash.serialization.ListMarshaller; import net.openhft.chronicle.hash.serialization.MapMarshaller; @@ -44,11 +42,9 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.Serializable; -import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import static java.util.Arrays.asList; @@ -62,7 +58,7 @@ import static org.junit.Assert.*; public class CHMUseCasesTest { - enum TypeOfMap {SIMPLE, SIMPLE_PERSISTED, REPLICATED} + enum TypeOfMap {SIMPLE, SIMPLE_PERSISTED} private final TypeOfMap typeOfMap; @@ -97,10 +93,6 @@ public class CHMUseCasesTest { TypeOfMap.SIMPLE }, - { - TypeOfMap.REPLICATED - }, - { TypeOfMap.SIMPLE_PERSISTED } @@ -129,43 +121,8 @@ public class CHMUseCasesTest { private void mapChecks() { - if (typeOfMap == TypeOfMap.REPLICATED) { - - // see HCOLL-265 Chronicle Maps with Identical char[] values are not equal1 - if (map1.valueClass() == char[].class || - map1.valueClass() == byte[].class || - map1.valueClass() == byte[][].class) { - - waitTillEqual(5000); - - assertArrayValueEquals(map1, map2); - if (typeOfMap == TypeOfMap.SIMPLE) - checkJsonSerilization(); - - return; - } - - - // see HCOLL-265 Chronicle Maps with Identical char[] values are not equal1 - if (map1.keyClass() == char[].class || - map1.keyClass() == byte[][].class) { - - return; - } - - - if (ByteBuffer.class.isAssignableFrom(map1.valueClass()) || - ByteBuffer.class.isAssignableFrom(map1.keyClass())) - return; - - - waitTillEqual(5000); - assertEquals(map1, map2); - } - if (typeOfMap == TypeOfMap.SIMPLE) checkJsonSerilization(); - } private void assertArrayValueEquals(ChronicleMap map1, ChronicleMap map2) { @@ -267,56 +224,10 @@ public class CHMUseCasesTest { final File file = file0; map1 = builder.createPersistedTo(file); closeables.add(map1); - closeables.add(new Closeable() { - @Override - public void close() throws IOException { - file.delete(); - } - }); + closeables.add(() -> file.delete()); return map1; - case REPLICATED: { - - map2 = null; - { - final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig - .of(8086) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - - map2 = builder - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig1).name("server") - .createWithId((byte) 1)) - .instance() - .name("server") - .create(); - closeables.add(map2); - - } - { - final TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of - (8087, new InetSocketAddress("localhost", 8086)) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - map1 = builder - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig2).name("map2") - .createWithId((byte) 2)) - .instance() - .name("map2") - .create(); - closeables.add(map1); - return map1; - - } - - - } - default: throw new IllegalStateException(); } diff --git a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java deleted file mode 100644 index 75a6b1426..000000000 --- a/src/test/java/net/openhft/chronicle/map/ChannelReplicationTest.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.*; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Map; -import java.util.Set; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes - * - * @author Rob Austin. - */ -public class ChannelReplicationTest { - - private ChronicleMap map1a; - private ChronicleMap map2a; - - private ChronicleMap map1b; - private ChronicleMap map2b; - - private ReplicationHub hubA; - private ReplicationHub hubB; - - public static void checkThreadsShutdown(Set threads) { - // give them a change to stop if there were killed. - try { - Thread.sleep(100); - } catch (InterruptedException e) { - e.printStackTrace(); - } - Map threadMap = Thread.getAllStackTraces(); - threadMap.keySet().removeAll(threads); - if (!threadMap.isEmpty()) { - System.out.println("### threads still running after the test ###"); - for (Map.Entry entry : threadMap.entrySet()) { - System.out.println(entry.getKey()); - for (StackTraceElement ste : entry.getValue()) { - System.out.println("\t" + ste); - } - } - try { - for (Thread thread : threadMap.keySet()) { - if (thread.isAlive()) { - System.out.println("Waiting for " + thread); - thread.join(1000); - if (thread.isAlive()) { - System.out.println("Forcing " + thread + " to die"); - thread.stop(); - } - } - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - - @Before - public void setup() throws IOException { - { - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8086, new InetSocketAddress("localhost", 8087)) - .heartBeatInterval(1, SECONDS).autoReconnectedUponDroppedConnection(true); - - hubA = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId((byte) 1); - - map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(2) - .constantValueSizeBySample("EXAMPLE-1") - .instance().replicatedViaChannel(hubA.createChannel((short) 1)).create(); - } - - { - TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(8087).heartBeatInterval(1, SECONDS) - .autoReconnectedUponDroppedConnection(true); - - hubB = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig) - .createWithId((byte) 2); - - map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(2) - .constantValueSizeBySample("EXAMPLE-1") - .instance().replicatedViaChannel(hubB.createChannel((short) 1)).create(); - } - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1a, map1b, map2a, map2b}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - checkThreadsShutdown(threads); - } - - @Test - public void test() throws IOException, InterruptedException { - - map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(2) - .constantValueSizeBySample("EXAMPLE-1") - .instance().replicatedViaChannel(hubB.createChannel((short) 2)).create(); - - map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(2) - .constantValueSizeBySample("EXAMPLE-1") - .instance().replicatedViaChannel(hubA.createChannel((short) 2)).create(); - - map2a.put(1, "EXAMPLE-2"); - map1a.put(1, "EXAMPLE-1"); - - // allow time for the recompilation to resolve - waitTillEqual(2500); - - Assert.assertEquals("map1a=map1b", map1a, map1b); - Assert.assertEquals("map2a=map2b", map2a, map2b); - - assertTrue("map1a.empty", !map1a.isEmpty()); - assertTrue("map2a.empty", !map2a.isEmpty()); - } - - /** - * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - for (int t = 0; t < timeOutMs; t++) { - if (map1a.equals(map1b) && - map2a.equals(map2b)) - break; - Thread.sleep(1); - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/DeserializeReplicatedMapWithoutConfiguringReplicatorsTest.java b/src/test/java/net/openhft/chronicle/map/DeserializeReplicatedMapWithoutConfiguringReplicatorsTest.java deleted file mode 100644 index dd44c6d2e..000000000 --- a/src/test/java/net/openhft/chronicle/map/DeserializeReplicatedMapWithoutConfiguringReplicatorsTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import com.google.common.collect.ImmutableMap; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.set.Builder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -public class DeserializeReplicatedMapWithoutConfiguringReplicatorsTest { - - @Test - public void deserializeReplicatedMapWithoutConfiguringReplicatorsTest() - throws IOException, InterruptedException { - - File file1 = File.createTempFile("test", "cron"); - file1.deleteOnExit(); - try (ChronicleMap map1 = ChronicleMap.of(Integer.class, Integer.class) - .entries(100) - .replication((byte) 1) - .createPersistedTo(file1)) { - map1.put(1, 1); - } - - try (ChronicleMap map1 = ChronicleMap.of(Integer.class, Integer.class) - .entries(100) -// .replication((byte) 1) - .createPersistedTo(file1)) { - assertEquals((Integer) 1, map1.get(1)); - try { - map1.put(2, 2); - throw new AssertionError("replicated map update when id is not configured " + - "should throw IllegalStateException"); - } catch (IllegalStateException expected) { - } - } - - try (ChronicleMap map1 = ChronicleMap.of(Integer.class, Integer.class) - .entries(100) - .replication((byte) 1) - .createPersistedTo(file1)) { - map1.put(2, 2); - } - - File file2 = File.createTempFile("test", "cron"); - file2.deleteOnExit(); - try (ChronicleMap map2 = ChronicleMap.of(Integer.class, Integer.class) - .entries(100) - .replication((byte) 2) - .createPersistedTo(file2)) { - map2.put(3, 3); - } - - // Let the subsequent remove from map2 be later than put in map1 - Thread.sleep(1); - - try (ChronicleMap map2 = ChronicleMap.of(Integer.class, Integer.class) - .entries(100) - .replication((byte) 2) - .createPersistedTo(file2)) { - map2.put(2, 0); - map2.remove(2); - } - - try (ChronicleMap map1 = ChronicleMap.of(Integer.class, Integer.class) - .entries(100) - .replication((byte) 1, TcpTransportAndNetworkConfig - .of(10000, new InetSocketAddress("localhost", 10001))) - .createPersistedTo(file1); - ChronicleMap map2 = ChronicleMap.of(Integer.class, Integer.class) - .entries(100) - .replication((byte) 2, TcpTransportAndNetworkConfig - .of(10001, new InetSocketAddress("localhost", 10000))) - .createPersistedTo(file2)) { - Builder.waitTillEqual(map1, map2, 3000); - assertEquals(map1, map2); - assertEquals(map1, ImmutableMap.of(1, 1, 3, 3)); - assertEquals(map2, ImmutableMap.of(1, 1, 3, 3)); - } - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } -} diff --git a/src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java deleted file mode 100644 index 55010bcee..000000000 --- a/src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.Data; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.replication.MapRemoteOperations; -import net.openhft.chronicle.map.replication.MapRemoteQueryContext; -import net.openhft.chronicle.map.replication.MapReplicableEntry; -import org.jetbrains.annotations.NotNull; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Tests ReplicatedChronicleMap when the Replication is over a TCP Socket - * - * @author Rob Austin. - */ -public class EventListenerWithTCPSocketReplicationTest { - - ChronicleMap map1; - ChronicleMap map2; - - final AtomicReference keyRef = new AtomicReference(); - - final AtomicReference replacedValueRef = new AtomicReference(); - final AtomicReference valueRef = new AtomicReference(); - final AtomicBoolean putWasCalled = new AtomicBoolean(false); - - final AtomicBoolean wasRemoved = new AtomicBoolean(false); - final AtomicReference valueRemoved = new AtomicReference(); - - final MapEntryOperations entryOperations = new - MapEntryOperations() { - - @Override - public Void remove(@NotNull MapEntry entry) { - wasRemoved.set(true); - valueRemoved.set(entry.value().get()); - entry.doRemove(); - return null; - } - - @Override - public Void replaceValue(@NotNull MapEntry entry, - Data newValue) { - putWasCalled.getAndSet(true); - keyRef.set(entry.key().get()); - valueRef.set(newValue.get()); - replacedValueRef.set(entry.value().get()); - entry.doReplaceValue(newValue); - return null; - } - - @Override - public Void insert(@NotNull MapAbsentEntry absentEntry, - Data value) { - putWasCalled.getAndSet(true); - keyRef.set(absentEntry.absentKey().get()); - valueRef.set(value.get()); - replacedValueRef.set(null); - absentEntry.doInsert(value); - return null; - } - }; - - final MapRemoteOperations remoteOperations = - new MapRemoteOperations() { - @Override - public void remove(MapRemoteQueryContext q) { - MapReplicableEntry entry = q.entry(); - if (entry != null) { - wasRemoved.set(true); - valueRemoved.set(entry.value().get()); - } - MapRemoteOperations.super.remove(q); - } - - @Override - public void put(MapRemoteQueryContext q, - Data newValue) { - MapReplicableEntry entry = q.entry(); - CharSequence oldValue = entry != null ? entry.value().get() : null; - putWasCalled.getAndSet(true); - keyRef.set(q.queriedKey().get()); - valueRef.set(newValue.get()); - replacedValueRef.set(oldValue); - MapRemoteOperations.super.put(q, newValue); - } - }; - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - wasRemoved.set(false); - valueRemoved.set(null); - putWasCalled.set(false); - valueRef.set(null); - replacedValueRef.set(null); - keyRef.set(null); - System.gc(); - } - - @Test - public void testAdded() throws IOException, InterruptedException { - - // ---------- SERVER1 1 ---------- - { - - // we connect the maps via a TCP socket connection on port 8077 - - ChronicleMapBuilder map1Builder = serverBuilder(); - - map1 = map1Builder.create(); - } - // ---------- SERVER2 2 on the same server as ---------- - - { - ChronicleMapBuilder map2Builder = clientBuilder(); - - map2 = map2Builder.create(); - } - - // we will stores some data into one map here - map1.put(5, "EXAMPLE"); - waitTillReplicated(5000); - - Assert.assertEquals("Maps should be equal", map1, map2); - Assert.assertTrue("Map 1 should not be empty", !map1.isEmpty()); - Assert.assertTrue("Map 2 should not be empty", !map2.isEmpty()); - - Assert.assertTrue("The eventListener.onPut should have been called", putWasCalled.get()); - Assert.assertEquals(Integer.valueOf(5), keyRef.get()); - Assert.assertEquals("EXAMPLE", valueRef.get().toString()); - } - - @Test - public void testRemoteNotifyPutLargerValue() throws IOException, InterruptedException { - - // ---------- SERVER1 1 ---------- - { - - // we connect the maps via a TCP socket connection on port 8077 - - ChronicleMapBuilder map1Builder = serverBuilder(); - - map1 = map1Builder.create(); - } - // ---------- SERVER2 2 on the same server as ---------- - - { - - map2 = clientBuilder().create(); - map2.put(5, "WILL_GET_OVER-WRITTEN"); - } - - // we will stores some data into one map here - map1.put(5, "EXAMPLE"); - waitTillReplicated(5000); - - Assert.assertTrue("The eventListener.onPut should have been called", putWasCalled.get()); - Assert.assertEquals(Integer.valueOf(5), keyRef.get()); - Assert.assertEquals("EXAMPLE", valueRef.get().toString()); - Assert.assertEquals("WILL_GET_OVER-WRITTEN", replacedValueRef.get().toString()); - } - - @Test - public void testRemoteNotifyPutSmallerValue() throws IOException, InterruptedException { - - // ---------- SERVER1 1 ---------- - { - - // we connect the maps via a TCP socket connection on port 8077 - - map1 = serverBuilder().create(); - } - // ---------- SERVER2 2 on the same server as ---------- - - { - - map2 = clientBuilder().create(); - map2.put(5, "small"); - } - - // we will stores some data into one map here - map1.put(5, "EXAMPLE"); - - waitTillReplicated(5000); - - Assert.assertTrue("The eventListener.onPut should have been called", putWasCalled.get()); - Assert.assertEquals(Integer.valueOf(5), keyRef.get()); - Assert.assertEquals("small", replacedValueRef.get().toString()); - Assert.assertEquals("EXAMPLE", valueRef.get().toString()); - } - - @Test - public void testRemoteNotifyRemove() throws IOException, InterruptedException { - - // ---------- SERVER1 1 ---------- - { - - // we connect the maps via a TCP socket connection on port 8077 - - ChronicleMapBuilder map1Builder = serverBuilder(); - - map1 = map1Builder.create(); - } - - // ---------- SERVER2 2 on the same server as ---------- - { - map2 = clientBuilder().create(); - map2.put(5, "WILL_GET_REMOVED"); - } - - // its we have to wait here for this entry to be removed - as removing an entry that you dont have - // on map 1 will have no effect - waitTillReplicated(5000); - - // no more puts after this point - putWasCalled.set(false); - - // we will stores some data into one map here - map1.remove(5); - - waitTillReplicated(5000); - - Assert.assertTrue(!putWasCalled.get()); - Assert.assertTrue(wasRemoved.get()); - Assert.assertEquals("WILL_GET_REMOVED", valueRemoved.get().toString()); - Assert.assertEquals(null, map2.get(5)); - } - - private ChronicleMapBuilder clientBuilder() { - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig.of(8077) - .autoReconnectedUponDroppedConnection(true); - return ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValueSize(100) - .entries(20000L) - .replication((byte) 1, tcpConfig) - .entryOperations(entryOperations) - .remoteOperations(remoteOperations); - } - - private void waitTillReplicated(final int timeOutMs) throws InterruptedException { - - Map map1UnChanged = new HashMap(); - Map map2UnChanged = new HashMap(); - - int numberOfTimesTheSame = 0; - for (int t = 0; t < timeOutMs + 100; t++) { - if (map1.equals(map2)) { - if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) { - numberOfTimesTheSame++; - } else { - numberOfTimesTheSame = 0; - map1UnChanged = new HashMap(map1); - map2UnChanged = new HashMap(map2); - } - Thread.sleep(1); - if (numberOfTimesTheSame == 100) { - break; - } - - } - Thread.sleep(1); - } - } - - private ChronicleMapBuilder serverBuilder() { - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8076, new InetSocketAddress("localhost", 8077)) - .autoReconnectedUponDroppedConnection(true); - return ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValueSize(100) - .entries(20000L) - .replication((byte) 2, tcpConfig); - } - -} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/Issue37Test.java b/src/test/java/net/openhft/chronicle/map/Issue37Test.java deleted file mode 100644 index 23b0611cc..000000000 --- a/src/test/java/net/openhft/chronicle/map/Issue37Test.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.set.Builder; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; - -public class Issue37Test { - - @Test - public void issue37Test() throws IOException { - int port = 1000; - String remoteHost = "localhost"; - int remotePort = 2000; - - TcpTransportAndNetworkConfig tcpReplicationConfig = TcpTransportAndNetworkConfig - .of(port, new InetSocketAddress(remoteHost, remotePort)) - .heartBeatInterval(1, TimeUnit.SECONDS) - .autoReconnectedUponDroppedConnection(true); - - ReplicationHub replicationHub = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpReplicationConfig) - .maxNumberOfChannels(1024) - .createWithId((byte)1); - - File file1 = Builder.getPersistenceFile(); - ChronicleMap map1 = ChronicleMapBuilder.of(String.class, String.class) - .entries(1) - .averageKeySize(10).averageValueSize(10) - .instance() - .replicatedViaChannel(replicationHub.createChannel(2)) - .persistedTo(file1) - .create(); - - File file2 = Builder.getPersistenceFile(); - ChronicleMap map2 = ChronicleMapBuilder.of(String.class, String.class) - .entries(1) - .averageKeySize(10).averageValueSize(10) - .instance() - .replicatedViaChannel(replicationHub.createChannel(3)) - .persistedTo(file2) - .create(); - - map2.close(); - map1.close(); - } -} diff --git a/src/test/java/net/openhft/chronicle/map/Issue62ChronicleClient.java b/src/test/java/net/openhft/chronicle/map/Issue62ChronicleClient.java index e438f0268..ad8c0b30e 100644 --- a/src/test/java/net/openhft/chronicle/map/Issue62ChronicleClient.java +++ b/src/test/java/net/openhft/chronicle/map/Issue62ChronicleClient.java @@ -16,7 +16,6 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,9 +37,7 @@ public class Issue62ChronicleClient { ChronicleMapBuilder.of(String.class, Long.class) //.averageKeySize(100) .averageKey(Issue62ChronicleServer.STR) - .replication((byte) 2, TcpTransportAndNetworkConfig - .of(8077, new InetSocketAddress("localhost", 8076)) - .heartBeatInterval(1, TimeUnit.SECONDS)) + .replication((byte) 2) .entries(50_000); try (ChronicleMap map = diff --git a/src/test/java/net/openhft/chronicle/map/Issue62ChronicleServer.java b/src/test/java/net/openhft/chronicle/map/Issue62ChronicleServer.java index dc0dfa96c..f2b1faf42 100644 --- a/src/test/java/net/openhft/chronicle/map/Issue62ChronicleServer.java +++ b/src/test/java/net/openhft/chronicle/map/Issue62ChronicleServer.java @@ -16,14 +16,11 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.net.InetSocketAddress; import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; public class Issue62ChronicleServer { @@ -52,9 +49,7 @@ public class Issue62ChronicleServer { ChronicleMapBuilder.of(String.class, Long.class) //.averageKeySize(100) .averageKey(STR) - .replication((byte) 1, TcpTransportAndNetworkConfig - .of(8076, new InetSocketAddress("localhost", 8077)) - .heartBeatInterval(1, TimeUnit.SECONDS)) + .replication((byte) 1) .entries(50_000); ChronicleMap cityPostalCodes = diff --git a/src/test/java/net/openhft/chronicle/map/OldDeletedEntriesCleanupTest.java b/src/test/java/net/openhft/chronicle/map/OldDeletedEntriesCleanupTest.java deleted file mode 100644 index 8a4b00549..000000000 --- a/src/test/java/net/openhft/chronicle/map/OldDeletedEntriesCleanupTest.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import com.google.common.collect.Maps; -import com.google.common.collect.SortedMapDifference; -import net.openhft.chronicle.bytes.Byteable; -import net.openhft.chronicle.bytes.BytesStore; -import net.openhft.chronicle.core.values.IntValue; -import net.openhft.chronicle.hash.replication.ReplicableEntry; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.impl.CompiledReplicatedMapQueryContext; -import net.openhft.chronicle.map.impl.QueryContextInterface; -import net.openhft.chronicle.set.Builder; -import net.openhft.chronicle.values.Values; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.TimeUnit; - -public class OldDeletedEntriesCleanupTest { - - private ChronicleMap map1; - private ChronicleMap map2; - static int s_port = 8093; - - @Before - public void setup() throws IOException { - IntValue value = Values.newNativeReference(IntValue.class); - ((Byteable) value).bytesStore(BytesStore.wrap(ByteBuffer.allocateDirect(4)), 0, 4); - - final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1); - - { - final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(s_port, - endpoint).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - - map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(Builder.SIZE + Builder.SIZE) - .removedEntryCleanupTimeout(1, TimeUnit.MILLISECONDS) - .actualSegments(1) - .averageValue("test" + 1000) - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig1) - .name("map1") - .createWithId((byte) 1)) - .instance() - .name("map1") - .create(); - } - { - final TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of - (s_port + 1).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(Builder.SIZE + Builder.SIZE) - .removedEntryCleanupTimeout(1, TimeUnit.MILLISECONDS) - .averageValue("test" + 1000) - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig2) - .name("map2") - .createWithId((byte) 2)) - .instance() - .name("map2") - .create(); - - } - s_port += 2; - } - - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - - @Test - public void testFloodReplicatedMapWithDeletedEntries() throws InterruptedException { - try { - Random r = new Random(42); - NavigableSet put = new TreeSet<>(); - for (int i = 0; i < Builder.SIZE * 10; i++) { - if (r.nextBoolean() || put.isEmpty()) { - int key = r.nextInt(); - map1.put(key, "test"); - put.add(key); - } else { - Integer key = put.pollFirst(); - map1.remove(key); - } -// Thread.sleep(0, 1000); - } - System.out.println("\nwaiting till equal"); - - waitTillEqual(15000); - if (!map1.equals(map2)) { - - SortedMapDifference diff = - Maps.difference(new TreeMap<>(map1), new TreeMap<>(map2)); - diff.entriesOnlyOnLeft().forEach((k, v) -> { - try (ExternalMapQueryContext c = - map1.queryContext(k)) { - c.entry(); - System.out.println(c.entry().key() + " " + c.entry().value() + " " + - ((ReplicableEntry) c.entry()).originTimestamp() + " " + - ((CompiledReplicatedMapQueryContext) c).pos); - int x = 0; - c.entry(); - } - }); - Assert.assertEquals( - diff.toString(), - new TreeMap<>(map1), new TreeMap<>(map2)); - } - } finally { - map1.close(); - map2.close(); - } - } - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - - Map map1UnChanged = new HashMap(); - Map map2UnChanged = new HashMap(); - - int numberOfTimesTheSame = 0; - long startTime = System.currentTimeMillis(); - for (int t = 0; t < timeOutMs + 100; t++) { - if (map1.equals(map2)) { - if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) { - numberOfTimesTheSame++; - } else { - numberOfTimesTheSame = 0; - map1UnChanged = new HashMap(map1); - map2UnChanged = new HashMap(map2); - } - Thread.sleep(1); - if (numberOfTimesTheSame == 10) { - System.out.println("same"); - break; - } - - } - Thread.sleep(1); - if (System.currentTimeMillis() - startTime > timeOutMs) - break; - } - } -} diff --git a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java deleted file mode 100644 index 3c33501c1..000000000 --- a/src/test/java/net/openhft/chronicle/map/PostChannelCreationTest.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.*; - -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes - * - * @author Rob Austin. - */ -public class PostChannelCreationTest { - - private ChronicleMap map1a; - private ChronicleMap map2a; - - private ChronicleMap map1b; - private ChronicleMap map2b; - - private ReplicationHub hubA; - private ReplicationHub hubB; - - public static File getPersistenceFile() { - File file = null; - try { - file = File.createTempFile("chm-test", "map"); - } catch (IOException e) { - e.printStackTrace(); - } - - file.deleteOnExit(); - return file; - } - - @Before - public void setup() throws IOException { - - { - final TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8086, new InetSocketAddress("localhost", 8087)) - .heartBeatInterval(1, SECONDS); - - hubA = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig) - .createWithId((byte) 1); - - // this is how you add maps after the custer is created - map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-1") - .entries(1000) - .instance().replicatedViaChannel(hubA.createChannel((short) 1)).create(); - - map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-1") - .entries(1000) - .instance().replicatedViaChannel(hubA.createChannel((short) 2)).create(); - } - - { - final TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8087, new InetSocketAddress("localhost", 8086)) - .heartBeatInterval(1, SECONDS); - - hubB = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig) - .createWithId((byte) 2); - - // this is how you add maps after the custer is created - map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-1") - .entries(1000) - .instance().replicatedViaChannel(hubB.createChannel((short) 1)).create(); - - map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-1") - .entries(1000) - .instance().replicatedViaChannel(hubB.createChannel((short) 2)).create(); - } - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1a, map1b, map2a, map2b}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - /** - * sets up a 2 clusters ( notionally one for each host ), and replicates 2 Chronicle Maps - * between them - * - * @throws IOException - * @throws InterruptedException - */ - @Test - public void test() throws IOException, InterruptedException { - - map2a.put(1, "EXAMPLE-2"); - map1a.put(1, "EXAMPLE-1"); - - // allow time for the recompilation to resolve - waitTillEqual(2500); - - Assert.assertEquals("map1a=map1b", map1a, map1b); - Assert.assertEquals("map2a=map2b", map2a, map2b); - - assertTrue("map1a.empty", !map1a.isEmpty()); - assertTrue("map2a.empty", !map2a.isEmpty()); - } - - /** - * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - for (int t = 0; t < timeOutMs; t++) { - if (map1a.equals(map1b) && - map2a.equals(map2b)) - break; - Thread.sleep(1); - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/ReplicateProxyMapUpdatedViaSharedMemoryTest.java b/src/test/java/net/openhft/chronicle/map/ReplicateProxyMapUpdatedViaSharedMemoryTest.java deleted file mode 100644 index 29babd1f0..000000000 --- a/src/test/java/net/openhft/chronicle/map/ReplicateProxyMapUpdatedViaSharedMemoryTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; - -/** - * Created by khoxsey on 5/22/15. - */ -public class ReplicateProxyMapUpdatedViaSharedMemoryTest { - - @Test(timeout = 10000) - public void testReplicateProxyMapUpdatedViaSharedMemory() throws IOException, InterruptedException { - - ChronicleMap server1 = null; - ChronicleMap server2 = null; - ChronicleMap replicationMap = null; - - { - final File tempFile = File.createTempFile("temp", "chron"); - - server1 = ChronicleMapBuilder.of(String.class, String.class) - .averageKey("key").averageValue("value") - .entries(1) - .replication((byte) 1).createPersistedTo(tempFile); - - TcpTransportAndNetworkConfig serverConfig = TcpTransportAndNetworkConfig.of(7077) - .heartBeatInterval(500, TimeUnit.MILLISECONDS); - replicationMap = ChronicleMapBuilder.of(String.class, String.class) - .replication((byte) 1, serverConfig).createPersistedTo(tempFile); - } - - { - TcpTransportAndNetworkConfig server2Config = - TcpTransportAndNetworkConfig.of(6161, new InetSocketAddress("localhost", 7077)); - - server2 = ChronicleMapBuilder.of(String.class, String.class) - .averageKey("key").averageValue("value") - .entries(1) - .replication((byte) 2, server2Config).create(); - } - - final String expected = "value"; - Thread.sleep(1000); - server1.put("key", expected); - - String actual; - do { - actual = server2.get("key"); - } while (actual == null); - - Assert.assertEquals(expected, actual); - - server1.close(); - server2.close(); - replicationMap.close(); - } -} diff --git a/src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java b/src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java deleted file mode 100644 index 7fcc98133..000000000 --- a/src/test/java/net/openhft/chronicle/map/SingleMapChannelTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.*; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes - * - * @author Rob Austin. - */ -public class SingleMapChannelTest { - - private ChronicleMap map1a; - - private ChronicleMap map1b; - - private ReplicationHub hubA; - private ReplicationHub hubB; - - - - @Before - public void setup() throws IOException { - - { - final TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8086, new InetSocketAddress("localhost", 8087)) - .autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, SECONDS); - - byte identifier = 1; - hubA = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - // this is how you add maps after the custer is created - map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-1") - .entries(1000) - .instance().replicatedViaChannel(hubA.createChannel((short) 1)).create(); - } - - { - final TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8087, new InetSocketAddress("localhost", 8086)) - .autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, SECONDS); - - hubB = ReplicationHub.builder().tcpTransportAndNetwork(tcpConfig).createWithId( - (byte) 2); - - // this is how you add maps after the custer is created - map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-1") - .entries(1000) - .instance().replicatedViaChannel(hubB.createChannel((short) 1)) - .create(); - } - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1a, map1b}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - @Test - public void test() throws IOException, InterruptedException { - - map1a.put(1, "EXAMPLE-1"); - - // allow time for the recompilation to resolve - waitTillEqual(2500); - - Assert.assertEquals("map1a=map1b", map1a, map1b); - - assertTrue("map1a.empty", !map1a.isEmpty()); - } - - /** - * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - for (int t = 0; t < timeOutMs; t++) { - if (map1a.equals(map1b)) - - break; - Thread.sleep(1); - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayPutReturnsNullTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayPutReturnsNullTest.java deleted file mode 100755 index af495f122..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayPutReturnsNullTest.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplication3wayPutReturnsNullTest { - - private ChronicleMap map1; - private ChronicleMap map2; - private ChronicleMap map3; - - static > T newTcpSocketShmIntString( - final byte identifier, - final int serverPort, - final InetSocketAddress... endpoints) throws IOException { - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig.of(serverPort, Arrays.asList(endpoints)) - .autoReconnectedUponDroppedConnection(true); - - return (T) ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(10) - .averageValue("EXAMPLE-1") - .putReturnsNull(true) - .replication(identifier, tcpConfig).create(); - } - - int portOffset = 0; - - @Before - public void setup() throws IOException { - map1 = newTcpSocketShmIntString((byte) 1, portOffset + 8026, new InetSocketAddress - ("localhost", portOffset + 8027), - new InetSocketAddress("localhost", portOffset + 8028)); - map2 = newTcpSocketShmIntString((byte) 2, portOffset + 8027, new InetSocketAddress - ("localhost", portOffset + 8028)); - map3 = newTcpSocketShmIntString((byte) 3, portOffset + 8028); - portOffset += 10; - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2, map3}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test3() throws IOException, InterruptedException { - - map3.put(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals(map1, map2); - assertEquals(map3, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void test() throws IOException, InterruptedException { - - assertEquals(null, map1.put(1, "EXAMPLE-1")); - assertEquals(null, map1.put(2, "EXAMPLE-2")); - assertEquals(null, map1.put(2, "EXAMPLE-1")); - - assertEquals(null, map2.put(5, "EXAMPLE-2")); - assertEquals(null, map2.put(6, "EXAMPLE-2")); - - map1.remove(2); - map2.remove(3); - map1.remove(3); - map2.put(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals(map1, map2); - assertEquals(map3, map3); - assertTrue(!map1.isEmpty()); - } - - @Test - public void testPutIfAbsent() throws IOException, InterruptedException { - - assertEquals(null, map1.putIfAbsent(1, "EXAMPLE-1")); - assertEquals("EXAMPLE-1", map1.putIfAbsent(1, "EXAMPLE-2").toString()); - assertEquals(null, map1.putIfAbsent(2, "EXAMPLE-2")); - assertEquals(null, map1.putIfAbsent(3, "EXAMPLE-1")); - - assertEquals(null, map2.putIfAbsent(5, "EXAMPLE-2")); - assertEquals(null, map2.putIfAbsent(6, "EXAMPLE-2")); - - map1.remove(2); - map2.remove(3); - map1.remove(3); - map2.putIfAbsent(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals(map1, map2); - assertEquals(map3, map3); - assertTrue(!map1.isEmpty()); - } - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2) && - map1.equals(map3) - && map2.equals(map3)) - break; - Thread.sleep(1); - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayTest.java deleted file mode 100755 index eb091a328..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication3wayTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; - -import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmIntString; -import static org.junit.Assert.*; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplication3wayTest { - - private ChronicleMap map1; - private ChronicleMap map2; - private ChronicleMap map3; - - @Before - public void setup() throws IOException { - map1 = newTcpSocketShmIntString((byte) 1, 8026, new InetSocketAddress("localhost", 8027), - new InetSocketAddress("localhost", 8029)); - map2 = newTcpSocketShmIntString((byte) 2, 8027, new InetSocketAddress("localhost", 8029)); - map3 = newTcpSocketShmIntString((byte) 3, 8029); - } - - @After - public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{map1, map2, map3}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test3() throws IOException, InterruptedException { - - assertEquals(null, map3.put(5, "EXAMPLE-2")); - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals(map1, map2); - assertEquals(map3, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void test() throws IOException, InterruptedException { - - assertEquals(null, map1.put(1, "EXAMPLE-1")); - assertEquals(null, map1.put(2, "EXAMPLE-2")); - assertNotEquals(null, map1.put(2, "EXAMPLE-1")); - - assertEquals(null, map2.put(5, "EXAMPLE-2")); - assertEquals(null, map2.put(6, "EXAMPLE-2")); - - map1.remove(2); - map2.remove(3); - map1.remove(3); - map2.put(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertEquals(map3, map3); - assertTrue(!map1.isEmpty()); - } - - @Test - public void testClear() throws IOException, InterruptedException { - - assertEquals(null, map1.put(1, "EXAMPLE-1")); - assertEquals(null, map1.put(2, "EXAMPLE-2")); - assertNotEquals(null, map1.put(2, "EXAMPLE-1")); - - assertEquals(null, map2.put(5, "EXAMPLE-2")); - assertEquals(null, map2.put(6, "EXAMPLE-2")); - - map1.clear(); - - map2.put(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertEquals(map3, map3); - assertTrue(!map1.isEmpty()); - } - - @Test - public void testPutIfAbsent() throws IOException, InterruptedException { - - assertEquals(null, map1.putIfAbsent(1, "EXAMPLE-1")); - assertNotEquals(null, map1.putIfAbsent(1, "EXAMPLE-2")); - assertEquals(null, map1.putIfAbsent(2, "EXAMPLE-2")); - assertEquals(null, map1.putIfAbsent(3, "EXAMPLE-1")); - - assertEquals(null, map2.putIfAbsent(5, "EXAMPLE-2")); - assertEquals(null, map2.putIfAbsent(6, "EXAMPLE-2")); - - map1.remove(2); - map2.remove(3); - map1.remove(3); - map2.putIfAbsent(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertEquals(map3, map3); - assertTrue(!map1.isEmpty()); - } - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2) && - map1.equals(map3) - && map2.equals(map3)) - break; - Thread.sleep(1); - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java deleted file mode 100644 index e43da3fa5..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; - -import static net.openhft.chronicle.set.Builder.newTcpSocketShmBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes - * - * @author Rob Austin. - */ -public class TCPSocketReplication4WayMapTest { - - private ChronicleMap map1; - private ChronicleMap map2; - private ChronicleMap map3; - private ChronicleMap map4; - - public static > T newTcpSocketShmIntString( - final byte identifier, - final int serverPort, - final InetSocketAddress... endpoints) throws IOException { - return (T) newTcpSocketShmBuilder(Integer.class, CharSequence.class, - identifier, serverPort, endpoints).averageValueSize(10).create(); - } - - @Before - public void setup() throws IOException { - - map1 = newTcpSocketShmIntString((byte) 1, 8086, new InetSocketAddress("localhost", 8087), - new InetSocketAddress("localhost", 8088), new InetSocketAddress("localhost", 8089)); - map2 = newTcpSocketShmIntString((byte) 2, 8087, new InetSocketAddress("localhost", 8088), - new InetSocketAddress("localhost", 8089)); - map3 = newTcpSocketShmIntString((byte) 3, 8088, new InetSocketAddress("localhost", 8089)); - map4 = newTcpSocketShmIntString((byte) 4, 8089); - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2, map3, map4}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test() throws IOException, InterruptedException { - - map1.put(1, "EXAMPLE-1"); - map2.put(2, "EXAMPLE-1"); - map3.put(3, "EXAMPLE-1"); - map4.put(3, "EXAMPLE-1"); - map4.remove(3); - - // allow time for the recompilation to resolve - waitTillEqual(2500); - - assertEquals("map2", map1, map2); - assertEquals("map3", map1, map3); - assertEquals("map4", map1, map4); - assertTrue("map2.empty", !map2.isEmpty()); - } - - @Test - public void testBufferOverflow() throws IOException, InterruptedException { - - for (int i = 0; i < 50; i++) { - map1.put(i, "EXAMPLE-1"); - } - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals("map2", map1, map2); - assertEquals("map3", map1, map3); - assertEquals("map4", map1, map4); - assertTrue("map2.empty", !map2.isEmpty()); - } - - - @Test - public void testBufferOverflowPutIfAbsent() throws IOException, InterruptedException { - for (int i = 0; i < 1024; i++) { - map1.putIfAbsent(i, "EXAMPLE-1"); - } - - for (int i = 0; i < 1024; i++) { - map1.putIfAbsent(i, ""); - } - - // allow time for the recompilation to resolve - waitTillEqual(10000); - - assertEquals("map2", map1, map2); - assertEquals("map3", map1, map3); - assertEquals("map4", map1, map4); - assertTrue("map2.empty", !map2.isEmpty()); - } - - /** - * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if ( map1.equals(map2) && - map1.equals(map3) && - map1.equals(map4) && - map2.equals(map3) && - map2.equals(map4) && - map3.equals(map4)) - break; - Thread.sleep(1); - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationAfterNodeFailureTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationAfterNodeFailureTest.java deleted file mode 100644 index 3a97331f3..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationAfterNodeFailureTest.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.ReplicationChannel; -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; - -/** - * Test ReplicatedChronicleMap where a cluster is formed with two servers. Then when when a server - * in the cluster is down, replication to a new server that is bootstrapping is tested. - * - * @author Ozan Ozen. - */ - -public class TCPSocketReplicationAfterNodeFailureTest { - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - - @Test - public void testTCPSocketReplicationAfterNodeFailure() - throws IOException, InterruptedException { - - ChronicleMap favoriteComputerServer1, favoriteComputerServer2, favoriteComputerServer3; - ReplicationHub hubOnServer1, hubOnServer2, hubOnServer3; - - // server 1 with identifier = 1 - { - ChronicleMapBuilder builder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(10) - .averageValueSize(100) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8121) - .heartBeatInterval(1, TimeUnit.SECONDS); - - hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteComputer - short channel1 = (short) 1; - - ReplicationChannel channel = hubOnServer1.createChannel(channel1); - favoriteComputerServer1 = builder.instance() - .replicatedViaChannel(channel).create(); - - favoriteComputerServer1.put("peter", "dell"); - } - - // Checkpoint for all the threads working before replicationHub on server 2 is created. - // We would like to go back to this point in time, replication is finished between server 1 - // and server 2 - - Set threadsBeforeServer2Bootstrap = Thread.getAllStackTraces().keySet(); - - // server 2 with identifier = 2 - { - ChronicleMapBuilder builder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(10) - .averageValueSize(100) - .entries(1000); - - byte identifier = (byte) 2; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8122, new InetSocketAddress("localhost", 8121)) - .heartBeatInterval(1, TimeUnit.SECONDS); - - hubOnServer2 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteComputer - short channel1 = (short) 1; - - favoriteComputerServer2 = builder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel1)).create(); - - favoriteComputerServer2.put("rob", "mac"); - } - - waitTillEqual(2500, favoriteComputerServer1, favoriteComputerServer2); - - - assertEquals(favoriteComputerServer1, favoriteComputerServer2); - assertEquals(2, favoriteComputerServer1.size()); - - // Close map on server2 - favoriteComputerServer2.close(); - // Close everything on server2 as well. - ChannelReplicationTest.checkThreadsShutdown(threadsBeforeServer2Bootstrap); - - // server 3 with identifier = 3 - { - ChronicleMapBuilder builder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(10) - .averageValueSize(100) - .entries(1000); - - byte identifier = (byte) 3; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8123, - new InetSocketAddress("localhost", 8121), - new InetSocketAddress("localhost", 8122)) // this wont be available but just added for completeness - .heartBeatInterval(1, TimeUnit.SECONDS); - - hubOnServer3 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteComputer - short channel1 = (short) 1; - - favoriteComputerServer3 = builder.instance() - .replicatedViaChannel(hubOnServer3.createChannel(channel1)).create(); - - favoriteComputerServer3.put("ozan", "mac"); - } - - - waitTillEqual(2500, favoriteComputerServer1, favoriteComputerServer3); - - - assertEquals(favoriteComputerServer1, favoriteComputerServer3); - assertEquals(3, favoriteComputerServer1.size()); - - favoriteComputerServer1.close(); - favoriteComputerServer3.close(); - } - - - private void waitTillEqual(final int timeOutMs, Map map1, Map map2) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTest.java deleted file mode 100644 index 61abd5878..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTest.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static net.openhft.chronicle.set.Builder.getPersistenceFile; -import static net.openhft.chronicle.set.Builder.newTcpSocketShmBuilder; -import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmIntString; -import static org.junit.Assert.assertEquals; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplicationBootStrapTest { - - private ReplicatedChronicleMap map1; - private ChronicleMap map2; - - @Test - public void testBootstrap() throws IOException, InterruptedException { - - map1 = newTcpSocketShmIntString((byte) 1, 8091); - - ChronicleMapBuilder map2aBuilder = - newTcpSocketShmBuilder(Integer.class, CharSequence.class, - (byte) 2, 8092, new InetSocketAddress("localhost", 8091)) - .averageValue("EXAMPLE-10"); - File persistenceFile = getPersistenceFile(); - final ChronicleMap map2a = - map2aBuilder.createPersistedTo(persistenceFile); - map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 - - long bootstrapFrom; - - // lets make sure that the message has got to map 1 - do { - bootstrapFrom = map1.remoteNodeCouldBootstrapFrom((byte) 2); - Thread.yield(); - } while (bootstrapFrom == 0); - - final File map2File = map2a.file(); - map2a.close(); - - { - // restart map 2 but don't doConnect it to map one - final ChronicleMap map2b = - map2aBuilder.createPersistedTo(map2File); - // add data into it - map2b.put(11, "ADDED WHEN DISCONNECTED TO MAP1"); - map2b.close(); - } - - // now restart map2a and doConnect it to map1, map1 should bootstrap the missing entry - map2 = map2aBuilder.createPersistedTo(map2File); - - // add data into it - waitTillEqual(5000); - assertEquals("ADDED WHEN DISCONNECTED TO MAP1", map1.get(11).toString()); - - map2.file().delete(); - persistenceFile.delete(); - } - - @Test - public void testBootstrapAndHeartbeat() throws IOException, InterruptedException { - - TcpTransportAndNetworkConfig map1Config = TcpTransportAndNetworkConfig - .of(8068, Arrays.asList(new InetSocketAddress("localhost", 8067))) - .heartBeatInterval(1L, TimeUnit.SECONDS) - .autoReconnectedUponDroppedConnection(true); - - map1 = (ReplicatedChronicleMap) ChronicleMap - .of(Integer.class, CharSequence.class) - .entries(10) - .averageValueSize(100) - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(map1Config) - .name("map1") - .createWithId((byte) 1)) - .create(); - - File persistenceFile = getPersistenceFile(); - - TcpTransportAndNetworkConfig map2Config = TcpTransportAndNetworkConfig.of(8067) - .heartBeatInterval(1L, TimeUnit.SECONDS); - - final ReplicatedChronicleMap map2a = - (ReplicatedChronicleMap) - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(10) - .averageValueSize(100) - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(map2Config) - .name("map2") - .createWithId((byte) 2)) - .createPersistedTo(persistenceFile); - - map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 - - long bootstrapFrom; - - // lets make sure that the message has got to map 1 - do { - bootstrapFrom = map1.remoteNodeCouldBootstrapFrom((byte) 2); - Thread.yield(); - } while (bootstrapFrom == 0); - - map2a.close(); - - { - // restart map 2 but does not connect it to map1 - final ChronicleMap map2b = ChronicleMap - .of(Integer.class, CharSequence.class) - .replication((byte) 2).createPersistedTo(persistenceFile); - // add data into it - map2b.put(11, "ADDED WHEN DISCONNECTED TO MAP1"); - map2b.close(); - } - - // now restart map2a and doConnect it to map1, map1 should bootstrap the missing entry - TcpTransportAndNetworkConfig tcpConfigNewMap2 = TcpTransportAndNetworkConfig.of(8067) - .heartBeatInterval(1L, TimeUnit.SECONDS); - map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfigNewMap2) - .name("newMap2") - .createWithId((byte) 2)) - .createPersistedTo(persistenceFile); - - // add data into it - waitTillEqual(5000); - assertEquals("ADDED WHEN DISCONNECTED TO MAP1", map1.get(11).toString()); - } - - @After - public void tearDown() { - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - /** - * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - long startTime = System.currentTimeMillis(); - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - if (System.currentTimeMillis() - startTime > timeOutMs) - break; - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java deleted file mode 100644 index 5607f8938..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.core.values.IntValue; -import net.openhft.chronicle.set.Builder; -import net.openhft.chronicle.values.Values; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Random; -import java.util.Set; - -import static net.openhft.chronicle.set.Builder.newTcpSocketShmBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplicationIntValueTest { - - static int s_port = 8010; - private ChronicleMapBuilder map1Builder; - private ChronicleMap map1; - private ChronicleMap map2; - private IntValue value; - - @Before - public void setup() throws IOException { - value = Values.newHeapInstance(IntValue.class); - map1Builder = newTcpSocketShmBuilder(IntValue.class, CharSequence.class, - (byte) 1, s_port, new InetSocketAddress("localhost", s_port + 1)) - .averageValue("EXAMPLE-1"); - map1 = map1Builder.create(); - map2 = newTcpSocketShmBuilder(IntValue.class, CharSequence.class, - (byte) 2, s_port + 1) - .averageValue("EXAMPLE-1") - .entries(Builder.SIZE) - .create(); - s_port += 2; - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - - closeable.close(); - - } catch (Exception e) { - e.printStackTrace(); - } - } - - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test3() throws IOException, InterruptedException { - - map1.put(set(5), "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - private IntValue set(int x) { - value.setValue(x); - return value; - } - - // see https://higherfrequencytrading.atlassian.net/browse/HCOLL-148 - @Test - public void test() throws IOException, InterruptedException { - - map1.put(set(1), "EXAMPLE-1"); - map1.put(set(2), "EXAMPLE-2"); - map1.put(set(3), "EXAMPLE-1"); - - map2.put(set(5), "EXAMPLE-2"); - map2.put(set(6), "EXAMPLE-2"); - - map1.remove(set(2)); - map2.remove(set(3)); - map1.remove(set(3)); - map2.put(set(5), "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void testBufferOverflow() throws IOException, InterruptedException { - - for (int i = 0; i < 1000; i++) { - map1.put(set(i), "EXAMPLE-1"); - } - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map2.isEmpty()); - } - - /** - * * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - assertEquals(map1, map2); - } - - - @Test(timeout = 12000) - public void testSoakTestWithRandomData() throws IOException, InterruptedException { - final long start = System.currentTimeMillis(); - System.out.print("SoakTesting "); - for (int j = 1; j < 2 * Builder.SIZE; j++) { - if (j % 1000 == 0) - System.out.print("."); - Random rnd = new Random(j); - for (int i = 1; i < 10; i++) { - final int select = rnd.nextInt(2); - final ChronicleMap map = select > 0 ? map1 : map2; - - if (rnd.nextBoolean()) { - map.put(set(rnd.nextInt(Builder.SIZE)), "test"); - } else { - map.remove(set(rnd.nextInt(Builder.SIZE))); - } - } - } - - waitTillEqual(10000); - - final long time = System.currentTimeMillis() - start; - - //assertTrue("timeTaken="+time, time < 2200); - System.out.println("\ntime taken millis=" + time); - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationNativeTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationNativeTest.java deleted file mode 100644 index 1561ef123..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationNativeTest.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.core.values.LongValue; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.values.Values; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplicationNativeTest { - - private ChronicleMap map1; - private ChronicleMap map2; - - @Before - public void setup() throws IOException { - InetSocketAddress endpoint = new InetSocketAddress("localhost", 8077); - - TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(8076, endpoint) - .heartBeatInterval(1L, TimeUnit.SECONDS).autoReconnectedUponDroppedConnection(true); - - map1 = ChronicleMap.of(Integer.class, LongValue.class) - .entries(10).replication((byte) 1, tcpConfig1).create(); - - TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of(8077) - .heartBeatInterval(1L, TimeUnit.SECONDS).autoReconnectedUponDroppedConnection(true); - - map2 = ChronicleMap.of(Integer.class, LongValue.class) - .entries(10).replication((byte) 2, tcpConfig2).create(); - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test() throws IOException, InterruptedException { - - LongValue value = Values.newNativeReference(LongValue.class); - - try (net.openhft.chronicle.core.io.Closeable c = - map1.acquireContext(1, value)) { - value.setValue(10); - } - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void test2Puts() throws IOException, InterruptedException { - - LongValue value = Values.newNativeReference(LongValue.class); - - try (net.openhft.chronicle.core.io.Closeable c = - map1.acquireContext(1, value)) { - value.setValue(10); - } - - try (net.openhft.chronicle.core.io.Closeable c = - map1.acquireContext(1, value)) { - value.setValue(20); - } - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - /** - * * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationPostConnectionTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationPostConnectionTest.java deleted file mode 100755 index 973ce23c8..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationPostConnectionTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.set.Builder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; - -import static net.openhft.chronicle.set.Builder.newTcpSocketShmBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplicationPostConnectionTest { - - private static int s_port = 13050; - private ChronicleMap map1; - private ChronicleMap map2; - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void testPostConnection() throws IOException, InterruptedException { - int port = s_port; - s_port += 2; - map1 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 1, port); - map1.put(5, "EXAMPLE-2"); - Thread.sleep(10); - map2 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 2, port + 1, - new InetSocketAddress("localhost", port)); - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void testPostConnectionNoSleep() throws IOException, InterruptedException { - int port = s_port; - s_port += 2; - map1 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 1, port); - map1.put(5, "EXAMPLE-2"); - map2 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 2, port + 1, - new InetSocketAddress("localhost", port)); - - // allow time for the recompilation to resolve - waitTillEqual(1000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void testBootStrapIntoNewMapWithNewFile() throws IOException, InterruptedException { - int port = s_port; - s_port += 2; - - ChronicleMapBuilder map2aBuilder = - newTcpSocketShmBuilder(Integer.class, CharSequence.class, - (byte) 2, port + 1, new InetSocketAddress("localhost", port)) - .averageValue("EXAMPLE-1"); - try (final ChronicleMap map2a = - map2aBuilder.createPersistedTo(Builder.getPersistenceFile())) { - map1 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 1, port); - - Thread.sleep(1); - map1.put(5, "EXAMPLE-2"); - } - - Thread.sleep(1); - map1.put(6, "EXAMPLE-1"); - - // recreate map2 with new unique file - map2 = map2aBuilder.createPersistedTo(Builder.getPersistenceFile()); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - assertEquals("EXAMPLE-1", map2.get(6).toString()); - } - - @After - public void tearDown() { - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - if (closeable != null) - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - /** - * * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java deleted file mode 100644 index cc49bbddd..000000000 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.set.Builder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.Set; - -import static net.openhft.chronicle.set.Builder.newTcpSocketShmBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplicationTest { - - static int s_port = 12050; - private ChronicleMap map1; - private ChronicleMap map2; - - @Before - public void setup() throws IOException { - int port = s_port; - ChronicleMapBuilder map1Builder = - newTcpSocketShmBuilder(Integer.class, CharSequence.class, - (byte) 1, port, new InetSocketAddress("localhost", port + 1)); - map1 = map1Builder.entries(Builder.SIZE).averageValueSize(10).create(); - - ChronicleMapBuilder map2Builder = - newTcpSocketShmBuilder(Integer.class, CharSequence.class, - (byte) 2, port + 1); - map2 = map2Builder.entries(Builder.SIZE).averageValueSize(10).create(); - s_port += 2; - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test3() throws IOException, InterruptedException { - - map1.put(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - - } - - @Test - public void test() throws IOException, InterruptedException { - - map1.put(1, "EXAMPLE-1"); - map1.put(2, "EXAMPLE-2"); - map1.put(3, "EXAMPLE-1"); - - map2.put(5, "EXAMPLE-2"); - map2.put(6, "EXAMPLE-2"); - - map1.remove(2); - map2.remove(3); - map1.remove(3); - map2.put(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void test2() throws IOException, InterruptedException { - - map2.put(1, "EXAMPLE-1"); - map2.remove(1); - - map1.put(1, "EXAMPLE-2"); - map1.remove(1); - - map2.put(1, "EXAMPLE-1"); - - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - } - - @Test - public void test4() throws IOException, InterruptedException { - - map2.put(1, "EXAMPLE-1"); - map2.remove(1); - - map1.put(1, "EXAMPLE-2"); - map1.remove(1); - - map2.put(1, "EXAMPLE-1"); - map2.remove(1); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map1, map2); - assertTrue(map1.isEmpty()); - } - - - @Test - public void testBufferOverflow() throws IOException, InterruptedException { - - for (int i = 0; i < 1000; i++) { - map1.put(i, "EXAMPLE-1"); - } - - // allow time for the recompilation to resolve - waitTillEqual(9000); - - assertEquals(map1, map2); - assertTrue(!map2.isEmpty()); - } - - /** - * * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - - Map map1UnChanged = new HashMap(); - Map map2UnChanged = new HashMap(); - - long startTime = System.currentTimeMillis(); - int numberOfTimesTheSame = 0; - for (int t = 0; t < timeOutMs + 100; t++) { - if (map1.equals(map2)) { - if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) { - numberOfTimesTheSame++; - } else { - numberOfTimesTheSame = 0; - map1UnChanged = new HashMap(map1); - map2UnChanged = new HashMap(map2); - } - Thread.sleep(1); - if (numberOfTimesTheSame == 10) { - System.out.println("same"); - break; - } - - } - Thread.sleep(1); - if (System.currentTimeMillis() - startTime > timeOutMs) - break; - } - } - - // TODO test this with larger sizes. - @Test - public void testSoakTestWithRandomData() throws IOException, InterruptedException { - System.out.print("SoakTesting "); - for (int j = 1; j < Builder.SIZE * 2; j++) { - if (j % 1000 == 0) - System.out.print("."); - Random rnd = new Random(j); - for (int i = 1; i < 10; i++) { - final int select = rnd.nextInt(2); - final ChronicleMap map = select > 0 ? map1 : map2; - - switch (rnd.nextInt(2)) { - case 0: - map.put(rnd.nextInt(Builder.SIZE), "test"); - break; - case 1: - map.remove(rnd.nextInt(Builder.SIZE)); - break; - } - } - } - System.out.println(""); - waitTillEqual(1000); - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TcpReplicationLargeEntryTest.java b/src/test/java/net/openhft/chronicle/map/TcpReplicationLargeEntryTest.java deleted file mode 100644 index 4fadb8909..000000000 --- a/src/test/java/net/openhft/chronicle/map/TcpReplicationLargeEntryTest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.algo.MemoryUnit; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.*; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - * @author Rob Austin. - */ -public class TcpReplicationLargeEntryTest { - - private ChronicleMap map1; - private ChronicleMap map2; - - static int s_port = 8010; - int valueSize = (int) MemoryUnit.MEGABYTES.toBytes(1); - - char[] valueX = new char[valueSize - 100]; - char[] valueY = new char[valueSize - 100]; - - @Before - public void setup() throws IOException { - Arrays.fill(valueX, 'X'); - Arrays.fill(valueX, 'Y'); - - String sampleValue = new String(valueX); - final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1); - - { - final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(s_port, - endpoint).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - - map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .constantValueSizeBySample(sampleValue) - .entries(2) - .actualSegments(1) - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig1) - .name("map1") - .createWithId((byte) 1)) - .instance() - .name("map1") - .create(); - } - { - final TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of - (s_port + 1).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .constantValueSizeBySample(sampleValue) - .entries(2) - .replication(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig2) - .name("map2") - .createWithId((byte) 2)) - .instance() - .name("map2") - .create(); - - } - s_port += 2; - } - - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - - @Test - public void testLargeValues() throws IOException, InterruptedException { - - String xString = new String(valueX); - String yString = new String(valueY); - - map1.put(1, xString); - map1.put(2, yString); - - System.out.println("\nwaiting till equal"); - - waitTillEqual(10000); - - Assert.assertEquals(xString, map2.get(1).toString()); - Assert.assertEquals(yString, map2.get(2).toString()); - - } - - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - - Map map1UnChanged = new HashMap(); - Map map2UnChanged = new HashMap(); - - int numberOfTimesTheSame = 0; - for (int t = 0; t < timeOutMs + 100; t++) { - if (map1.equals(map2)) { - if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) { - numberOfTimesTheSame++; - } else { - numberOfTimesTheSame = 0; - map1UnChanged = new HashMap(map1); - map2UnChanged = new HashMap(map2); - } - Thread.sleep(1); - if (numberOfTimesTheSame == 100) { - System.out.println("same"); - break; - } - - } - Thread.sleep(1); - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/TcpReplicationSoakCrdtTest.java b/src/test/java/net/openhft/chronicle/map/TcpReplicationSoakCrdtTest.java deleted file mode 100644 index 75cd044c2..000000000 --- a/src/test/java/net/openhft/chronicle/map/TcpReplicationSoakCrdtTest.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import com.google.common.collect.MapDifference; -import net.openhft.chronicle.algo.MemoryUnit; -import net.openhft.chronicle.hash.Data; -import net.openhft.chronicle.hash.replication.ReplicableEntry; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.replication.MapRemoteOperations; -import net.openhft.chronicle.map.replication.MapRemoteQueryContext; -import net.openhft.chronicle.map.replication.MapReplicableEntry; -import net.openhft.chronicle.set.Builder; -import org.jetbrains.annotations.NotNull; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.*; -import java.util.concurrent.TimeUnit; - -import static com.google.common.collect.Maps.difference; -import static net.openhft.chronicle.algo.MemoryUnit.MEGABYTES; -import static net.openhft.chronicle.map.TcpReplicationSoakCrdtTest.GrowOnlySetValuedMapEntryOperations.growOnlySetValuedMapEntryOperations; -import static net.openhft.chronicle.map.TcpReplicationSoakCrdtTest.GrowOnlySetValuedMapRemoteOperations.growOnlySetValuedMapRemoteOperations; -import static org.junit.Assert.assertEquals; - -public class TcpReplicationSoakCrdtTest { - - static class GrowOnlySetValuedMapEntryOperations - implements MapEntryOperations, Void> { - - private static final GrowOnlySetValuedMapEntryOperations INSTANCE = - new GrowOnlySetValuedMapEntryOperations(); - - public static - MapEntryOperations, Void> growOnlySetValuedMapEntryOperations() { - return GrowOnlySetValuedMapEntryOperations.INSTANCE; - } - - private GrowOnlySetValuedMapEntryOperations() {} - - @Override - public Void remove(@NotNull MapEntry> entry) { - throw new UnsupportedOperationException("Map with grow-only set values " + - "doesn't support map value removals"); - } - } - - static class GrowOnlySetValuedMapRemoteOperations - implements MapRemoteOperations, Void> { - - private static final GrowOnlySetValuedMapRemoteOperations INSTANCE = - new GrowOnlySetValuedMapRemoteOperations(); - - public static - MapRemoteOperations, Void> growOnlySetValuedMapRemoteOperations() { - return GrowOnlySetValuedMapRemoteOperations.INSTANCE; - } - - private GrowOnlySetValuedMapRemoteOperations() {} - - @Override - public void put(MapRemoteQueryContext, Void> q, Data> newValue) { - MapReplicableEntry> entry = q.entry(); - if (entry != null) { - Set merged = new HashSet<>(entry.value().get()); - merged.addAll(newValue.get()); - q.replaceValue(entry, q.wrapValueAsData(merged)); - } else { - q.insert(q.absentEntry(), newValue); - q.entry().updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - } - } - - @Override - public void remove(MapRemoteQueryContext, Void> q) { - throw new UnsupportedOperationException(); - } - } - - private ChronicleMap> map1; - private ChronicleMap> map2; - static int s_port = 8095; - - @Before - public void setup() throws IOException { - final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1); - - HashSet averageValue = new HashSet<>(); - for (int i = 0; i < 10; i++) { - averageValue.add(i); - } - ChronicleMapBuilder> builder = ChronicleMapBuilder - .of(Integer.class, (Class>) (Class) Set.class) - .entries(Builder.SIZE) - .averageValue(averageValue) - .entryOperations(growOnlySetValuedMapEntryOperations()) - .remoteOperations(growOnlySetValuedMapRemoteOperations()); - - { - final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(s_port, - endpoint).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize((int) MEGABYTES.toBytes(64)); - - map1 = builder - .instance() - .replicated(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig1) - .name("map1") - .createWithId((byte) 1)) - .name("map1") - .create(); - } - { - final TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of - (s_port + 1).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize((int) MEGABYTES.toBytes(64)); - - map2 = builder - .instance() - .replicated(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig2) - .name("map2") - .createWithId((byte) 2)) - .name("map2") - .create(); - - } - s_port += 2; - } - - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - - @Test - public void testReplicatedValuesMerged() throws InterruptedException { - HashSet v1 = new HashSet<>(); - v1.add(1); - map1.put(1, v1); - HashSet v2 = new HashSet<>(); - v2.add(2); - map2.put(1, v2); - waitTillEqual(5000); - // Map1 not equal to Map2, because it use byte-to-byte comparison of serialized HashSet - // values, that are different for some reason - assertEquals(map1.entrySet(), map2.entrySet()); - } - - @Test - public void testSoakTestWithRandomData() throws IOException, InterruptedException { - try { - System.out.print("SoakTesting "); - for (int j = 1; j < 2 * Builder.SIZE; j++) { - if (j % 1000 == 0) - System.out.print("."); - Random rnd = new Random(j); - for (int i = 1; i < 10; i++) { - final ChronicleMap> map = rnd.nextBoolean() ? map1 : map2; - int key = rnd.nextInt(Builder.SIZE); - try (ExternalMapQueryContext, ?> q = - map.queryContext(key)) { - q.updateLock().lock(); - MapEntry> entry = q.entry(); - if (entry != null) { - Set value = entry.value().get(); - value.add(rnd.nextInt(20)); - q.replaceValue(entry, q.wrapValueAsData(value)); - } else { - HashSet value = new HashSet<>(); - q.insert(q.absentEntry(), q.wrapValueAsData(value)); - } - } - } - } - - System.out.println("\nwaiting till equal"); - - waitTillEqual(15000); - - MapDifference> difference = difference(map1, map2); - if (difference.areEqual()) { - System.out.println("same"); - return; - } - - Map> onlyOnMap1 = difference.entriesOnlyOnLeft(); - System.out.println("only on map1:"); - for (Integer k : onlyOnMap1.keySet()) { - printState(map1, k); - } - Map> onlyOnMap2 = difference.entriesOnlyOnRight(); - System.out.println("only on map2:"); - for (Integer k : onlyOnMap2.keySet()) { - printState(map2, k); - } - Set entryDifferingKeys = difference.entriesDiffering().keySet(); - System.out.println("value difference:"); - for (Integer k : entryDifferingKeys) { - printState(map1, k); - printState(map2, k); - } - throw new AssertionError(difference); - } finally { - map1.close(); - map2.close(); - } - - } - - - private void printState(ChronicleMap> m, Integer k) { - try (ExternalMapQueryContext, ?> q = m.queryContext(k)) { - MapEntry> entry = q.entry(); - ReplicableEntry re = (ReplicableEntry) entry; - System.out.println(k + "=" + entry.value() + ", isChanged=" + - re.isChanged() + " oTs: " + re.originTimestamp() + - " oId: " + re.originIdentifier()); - } - } - - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - - Map map1UnChanged = new HashMap(); - Map map2UnChanged = new HashMap(); - - int numberOfTimesTheSame = 0; - long startTime = System.currentTimeMillis(); - for (int t = 0; t < timeOutMs + 100; t++) { - // not map1.equals(map2), the reason is described above - if (difference(map1, map2).areEqual()) { - if (difference(map1, map1UnChanged).areEqual() && - difference(map2, map2UnChanged).areEqual()) { - numberOfTimesTheSame++; - } else { - numberOfTimesTheSame = 0; - map1UnChanged = new HashMap(map1); - map2UnChanged = new HashMap(map2); - } - Thread.sleep(1); - if (numberOfTimesTheSame == 10) { - System.out.println("same"); - break; - } - - } - Thread.sleep(1); - if (System.currentTimeMillis() - startTime > timeOutMs) - break; - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/TcpReplicationSoakTest.java b/src/test/java/net/openhft/chronicle/map/TcpReplicationSoakTest.java deleted file mode 100644 index 6bd54d45b..000000000 --- a/src/test/java/net/openhft/chronicle/map/TcpReplicationSoakTest.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import com.google.common.collect.MapDifference; -import com.google.common.collect.Maps; -import net.openhft.chronicle.hash.replication.ReplicableEntry; -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.set.Builder; -import org.junit.*; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - * @author Rob Austin. - */ -public class TcpReplicationSoakTest { - - private ChronicleMap map1; - private ChronicleMap map2; - static int s_port = 8093; - - @Before - public void setup() throws IOException { - - final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1); - - ChronicleMapBuilder builder = ChronicleMapBuilder - .of(Integer.class, CharSequence.class) - .entries(Builder.SIZE + Builder.SIZE) - .actualSegments(1) - .averageValue("test" + 1000); - - { - final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(s_port, - endpoint).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - - map1 = builder - .instance() - .replicated(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig1) - .name("map1") - .createWithId((byte) 1)) - .name("map1") - .create(); - } - { - final TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of - (s_port + 1).autoReconnectedUponDroppedConnection(true) - .heartBeatInterval(1, TimeUnit.SECONDS) - .tcpBufferSize(1024 * 64); - - map2 = builder - .instance() - .replicated(SingleChronicleHashReplication.builder() - .tcpTransportAndNetwork(tcpConfig2) - .name("map2") - .createWithId((byte) 2)) - .name("map2") - .create(); - } - s_port += 2; - } - - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Ignore - @Test - public void testSoakTestWithRandomData() throws IOException, InterruptedException { - try { - System.out.print("SoakTesting "); - for (int j = 1; j < 2 * Builder.SIZE; j++) { - if (j % 1000 == 0) - System.out.print("."); - Random rnd = new Random(j); - for (int i = 1; i < 10; i++) { - final int select = rnd.nextInt(2); - final ChronicleMap map = select > 0 ? map1 : map2; - - if (rnd.nextBoolean()) { - map.put(rnd.nextInt(Builder.SIZE), "test" + j); - } else { - map.remove(rnd.nextInt(Builder.SIZE)); - } - } - } - - System.out.println("\nwaiting till equal"); - - waitTillEqual(15000); - - if (map1.equals(map2)) { - System.out.println("same"); - return; - } - MapDifference difference = Maps.difference(map1, map2); - Map onlyOnMap1 = difference.entriesOnlyOnLeft(); - System.out.println("only on map1:"); - for (Integer k : onlyOnMap1.keySet()) { - printState(map1, k); - } - Map onlyOnMap2 = difference.entriesOnlyOnRight(); - System.out.println("only on map2:"); - for (Integer k : onlyOnMap2.keySet()) { - printState(map2, k); - } - if (!map1.equals(map2)) { - Assert.assertEquals(difference.toString(), map1, map2); - } - } finally { - map1.close(); - map2.close(); - } - - } - - private void printState(ChronicleMap m, Integer k) { - try (ExternalMapQueryContext q = m.queryContext(k)) { - MapEntry entry = q.entry(); - ReplicableEntry re = (ReplicableEntry) entry; - System.out.println(k + "=" + entry.value() + ", isChanged=" + - re.isChanged() + " oTs: " + re.originTimestamp() + - " oId: " + re.originIdentifier()); - } - } - - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - - Map map1UnChanged = new HashMap(); - Map map2UnChanged = new HashMap(); - - int numberOfTimesTheSame = 0; - long startTime = System.currentTimeMillis(); - for (int t = 0; t < timeOutMs + 100; t++) { - if (map1.equals(map2)) { - if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) { - numberOfTimesTheSame++; - } else { - numberOfTimesTheSame = 0; - map1UnChanged = new HashMap(map1); - map2UnChanged = new HashMap(map2); - } - Thread.sleep(1); - if (numberOfTimesTheSame == 10) { - System.out.println("same"); - break; - } - - } - Thread.sleep(1); - if (System.currentTimeMillis() - startTime > timeOutMs) - break; - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireContextTest.java b/src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireContextTest.java deleted file mode 100644 index f7c0170d1..000000000 --- a/src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireContextTest.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import eg.TestInstrumentVOInterface; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.values.Values; -import org.junit.Assert; -import org.junit.Test; - -import java.net.InetSocketAddress; - -/** - * @author Rob Austin. - */ -public class TcpReplicationWithAcquireContextTest { - - - @Test - public void testReplicationWithAcquireUsingLocked() throws InterruptedException { - - - TcpTransportAndNetworkConfig config1 = TcpTransportAndNetworkConfig.of(8076); - - ChronicleMap map2a = null; - ChronicleMap map1a = null; - - try (ChronicleMap map1 = ChronicleMap - .of(CharSequence.class, TestInstrumentVOInterface.class) - .entries(1) - .averageKey("hello") - .replication((byte) 1, config1) - .create()) { - - map1a = map1; - - TcpTransportAndNetworkConfig config2 = - TcpTransportAndNetworkConfig.of(8077, new InetSocketAddress("127.0.0.1", 8076)); - - - try (ChronicleMap map2 = - ChronicleMap.of(CharSequence.class, TestInstrumentVOInterface.class) - .entries(1) - .averageKey("hello") - .replication((byte) 2, config2) - .create()) { - - map2a = map2; - - //Store some data into MAP1 - TestInstrumentVOInterface instrumentVOInterface = - Values.newNativeReference(TestInstrumentVOInterface.class); - - - try (net.openhft.chronicle.core.io.Closeable c = - map1.acquireContext("KEY1", instrumentVOInterface)) { - instrumentVOInterface.setSymbol("Flyer"); - instrumentVOInterface.setCurrencyCode("USA"); - } - - - int t = 0; - for (; t < 5000; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - - - Assert.assertEquals(map1a, map2a); - } - } - - - } - - - @Test - public void testReplicationWithEmptyOffHeapObject() throws InterruptedException { - - - TcpTransportAndNetworkConfig config1 = TcpTransportAndNetworkConfig.of(8076); - - ChronicleMap map2a = null; - ChronicleMap map1a = null; - - try (ChronicleMap map1 = ChronicleMapBuilder.of - (CharSequence.class, - TestInstrumentVOInterface - .class) - .entries(5000L).averageKeySize("hello".length()) - .replication((byte) 1, config1).create()) { - - map1a = map1; - - TcpTransportAndNetworkConfig config2 = TcpTransportAndNetworkConfig - .of(8077, new InetSocketAddress("127.0.0.1", 8076)); - - - try (ChronicleMap map2 = - ChronicleMapBuilder.of(CharSequence.class, - TestInstrumentVOInterface.class) - .putReturnsNull(true) - .removeReturnsNull(true) - .replication((byte) 2, config2) - .entries(5000L).averageKeySize("hello".length()).create()) { - - map2a = map2; - - //Store some data into MAP1 - TestInstrumentVOInterface instrumentVOInterface = - Values.newHeapInstance(TestInstrumentVOInterface.class); - - map2a.put("hello", instrumentVOInterface); - - - int t = 0; - for (; t < 5000; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - - Assert.assertEquals(map1a, map2a); - } - } - - - } - - - @Test - public void testReplicationWithOffHeapObject() throws InterruptedException { - - - TcpTransportAndNetworkConfig config1 = TcpTransportAndNetworkConfig.of(8076); - - ChronicleMap map2a = null; - ChronicleMap map1a = null; - - try (ChronicleMap map1 = ChronicleMapBuilder.of - (CharSequence.class, - TestInstrumentVOInterface - .class) - .entries(5000L).averageKeySize("hello".length()) - .replication((byte) 1, config1).create()) { - - map1a = map1; - - TcpTransportAndNetworkConfig config2 = - TcpTransportAndNetworkConfig.of(8077, new InetSocketAddress("127.0.0.1", 8076)); - - - try (ChronicleMap map2 = - ChronicleMapBuilder.of(CharSequence.class, - TestInstrumentVOInterface.class) - .putReturnsNull(true) - .removeReturnsNull(true) - .replication((byte) 2, config2) - .entries(5000L).averageKeySize("hello".length()).create()) { - - map2a = map2; - - //Store some data into MAP1 - TestInstrumentVOInterface instrumentVOInterface = - Values.newHeapInstance(TestInstrumentVOInterface.class); - instrumentVOInterface.setSymbol("Flyer"); - - map2a.put("hello", instrumentVOInterface); - - - int t = 0; - for (; t < 5000; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - - Assert.assertEquals(map1a, map2a); - } - } - - - } - -} diff --git a/src/test/java/net/openhft/chronicle/map/TcpTimeBasedReplicationSoakTest.java b/src/test/java/net/openhft/chronicle/map/TcpTimeBasedReplicationSoakTest.java deleted file mode 100644 index a00b3165e..000000000 --- a/src/test/java/net/openhft/chronicle/map/TcpTimeBasedReplicationSoakTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.set.Builder; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.*; - -/** - * @author Rob Austin. - */ -public class TcpTimeBasedReplicationSoakTest { - - - private ChronicleMap map1; - private ChronicleMap map2; - static int s_port = 8010; - int t = 0; - - @Before - public void setup() throws IOException { - - final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1); - - ChronicleMapBuilder builder = ChronicleMapBuilder - .of(Integer.class, String.class) - .averageValue("test-1000000") - .entries(Builder.SIZE); - { - TcpTransportAndNetworkConfig tcpConfig1 = - TcpTransportAndNetworkConfig.of(s_port, endpoint); - - map1 = builder.replication((byte) 1, tcpConfig1).create(); - } - { - final TcpTransportAndNetworkConfig tcpConfig2 = - TcpTransportAndNetworkConfig.of(s_port + 1); - - map2 = builder.replication((byte) 2, tcpConfig2).create(); - } - s_port += 2; - } - - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - - @Test - public void testSoakTestWithRandomData() throws IOException, InterruptedException { - - // for (int i = 30; i < 50; i++) { - System.out.print("SoakTesting "); - int max = 1000000; - for (int j = 1; j < max; j++) { - if (j % 100 == 0) - System.out.print("."); - Random rnd = new Random(System.currentTimeMillis()); - - final ChronicleMap map = rnd.nextBoolean() ? map1 : map2; - - if (rnd.nextBoolean()) { - map.put(rnd.nextInt(100), "test" + j); - } else { - map.remove(rnd.nextInt(100)); - } - - } - - System.out.println("\nwaiting till equal"); - - waitTillUnchanged(1000); - System.out.println("time t=" + t); - Assert.assertEquals(new TreeMap(map1), new TreeMap(map2)); - - } - - - private void waitTillUnchanged(final int timeOutMs) throws InterruptedException { - - Map map1UnChanged = new HashMap(); - Map map2UnChanged = new HashMap(); - - int numberOfTimesTheSame = 0; - for (int t = 0; t < timeOutMs + 100; t++) { - - if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) { - numberOfTimesTheSame++; - } else { - numberOfTimesTheSame = 0; - map1UnChanged = new HashMap(map1); - map2UnChanged = new HashMap(map2); - } - Thread.sleep(50); - if (numberOfTimesTheSame == 100) { - - break; - } - - - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/TestEntriesSpanningSeveralChunksReplicationHub.java b/src/test/java/net/openhft/chronicle/map/TestEntriesSpanningSeveralChunksReplicationHub.java deleted file mode 100644 index 0251c8c3e..000000000 --- a/src/test/java/net/openhft/chronicle/map/TestEntriesSpanningSeveralChunksReplicationHub.java +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.ReplicationChannel; -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.HashMap; -import java.util.concurrent.TimeUnit; - -import static net.openhft.chronicle.map.ChronicleMapBuilder.of; -import static org.junit.Assert.assertEquals; - -/** - * @author Rob Austin. - */ -public class TestEntriesSpanningSeveralChunksReplicationHub { - - - @Ignore("performacne test") - @Test - public void testReplicationHubHandlesSpanningSeveralChunksEntries() - throws IOException, InterruptedException { - - ChronicleMap favoriteColourServer1, favoriteColourServer2; - ChronicleMap favoriteComputerServer1, favoriteComputerServer2; - - int chunksPerEntry = 65; - char[] largeEntry = new char[100 * chunksPerEntry]; - - Arrays.fill(largeEntry, 'X'); - String largeString = new String(largeEntry); - - // server 1 with identifier = 1 - { - ChronicleMapBuilder builder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(10) - .averageValueSize(10) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8021, new InetSocketAddress("localhost", 8022)) - .heartBeatInterval(1, TimeUnit.SECONDS); - - ReplicationHub hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - ReplicationChannel channel = hubOnServer1.createChannel(channel1); - favoriteColourServer1 = builder.instance() - .replicatedViaChannel(channel).create(); - - favoriteColourServer1.put("peter", largeString); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer1 = builder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel2)).create(); - - favoriteComputerServer1.put("peter", "dell"); - } - - // server 2 with identifier = 2 - { - ChronicleMapBuilder builder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(10) - .averageValueSize(10) - .entries(1000); - - byte identifier = (byte) 2; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8022).heartBeatInterval(1, TimeUnit.SECONDS); - - ReplicationHub hubOnServer2 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer2 = builder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel1)).create(); - - favoriteColourServer2.put("rob", "blue"); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer2 = builder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel2)).create(); - - favoriteComputerServer2.put("rob", "mac"); - favoriteComputerServer2.put("daniel", "mac"); - } - - // allow time for the recompilation to resolve - for (int t = 0; t < 2500; t++) { - if (favoriteComputerServer2.equals(favoriteComputerServer1) && - favoriteColourServer2.equals(favoriteColourServer1)) - break; - Thread.sleep(1); - } - - assertEquals(favoriteComputerServer1, favoriteComputerServer2); - assertEquals(3, favoriteComputerServer2.size()); - - assertEquals(favoriteColourServer1, favoriteColourServer2); - assertEquals(2, favoriteColourServer1.size()); - - favoriteColourServer1.close(); - favoriteComputerServer2.close(); - favoriteColourServer2.close(); - favoriteColourServer1.close(); - } - - - public static final int NUMBER_OF_CHANNELS = 1000; - - @Ignore("TODO performance test? invalid test? a test revealing a bug?") - @Test(timeout = 100000) - public void test1000Channels() - throws IOException, InterruptedException { - - - final HashMap map1 = new HashMap<>(); - final HashMap map2 = new HashMap<>(); - - ReplicationHub hubOnServer1; - ReplicationHub hubOnServer2; - - // server 1 with identifier = 1 - { - ChronicleMapBuilder builder = - of(CharSequence.class, Short.class) - .averageKeySize(10) - .averageValueSize(100) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8059); - - hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .maxNumberOfChannels(NUMBER_OF_CHANNELS) - .createWithId(identifier); - - for (short channelId = 1; channelId < NUMBER_OF_CHANNELS; channelId++) { - ReplicationChannel channel = hubOnServer1.createChannel(channelId); - ChronicleMap map = builder.instance().replicatedViaChannel(channel).create(); - - map.put("key", channelId); - map1.put(channelId, map); - - } - } - - - { - ChronicleMapBuilder builder = - of(CharSequence.class, Short.class) - .averageKeySize(10) - .averageValueSize(100) - .entries(1000); - - byte identifier = (byte) 2; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8054, new InetSocketAddress("localhost", 8059)); - - hubOnServer2 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .maxNumberOfChannels(NUMBER_OF_CHANNELS) - .createWithId(identifier); - - for (short channelId = 1; channelId < NUMBER_OF_CHANNELS; channelId++) { - ReplicationChannel channel = hubOnServer2.createChannel(channelId); - ChronicleMap map = builder.instance().replicatedViaChannel(channel).create(); - - map.put("key", channelId); - map2.put(channelId, map); - - } - } - - // check that the maps are the same, if the are not the same the test will timeout - // we have to give a few seconds for the replication to propagate - for (short channelId = 1; channelId < NUMBER_OF_CHANNELS; channelId++) { - // allow time for the recompilation to resolve - - final ChronicleMap v1 = map1.get(channelId); - final ChronicleMap v2 = map2.get(channelId); - - for (; ; ) { - - - if (v1 == null || v2 == null) { - Thread.sleep(10); - continue; - } - - if (v1.isEmpty() || v2.isEmpty()) { - Thread.sleep(10); - continue; - } - - if (!v1.equals(v2)) { - Thread.sleep(10); - continue; - } - - - Assert.assertEquals(channelId, v1.get("key")); - Assert.assertEquals(channelId, v2.get("key")); - - v1.put("key2", (short) 1); - v2.put("key2", (short) 1); - break; - } - - } - - - // check that the maps are the same, if the are not the same the test will timeout - // we have to give a few seconds for the replication to propagate - for (short channelId = 1; channelId < NUMBER_OF_CHANNELS; channelId++) { - // allow time for the recompilation to resolve - - final ChronicleMap v1 = map1.get(channelId); - final ChronicleMap v2 = map2.get(channelId); - - for (; ; ) { - - - if (v1 == null || v2 == null) { - Thread.sleep(10); - continue; - } - - if (v1.isEmpty() || v2.isEmpty()) { - Thread.sleep(10); - continue; - } - - if (!v1.equals(v2)) { - Thread.sleep(10); - continue; - } - - - Assert.assertEquals((short)1, v1.get("key2")); - Assert.assertEquals((short)1, v2.get("key2")); - - break; - } - - } - - - // close - - for (short channelId = 1; channelId < NUMBER_OF_CHANNELS; channelId++) { - - final ChronicleMap v1 = map1.get(channelId); - if (v1 != null) - v1.close(); - - final ChronicleMap v2 = map2.get(channelId); - if (v2 != null) - v2.close(); - } - } - - @Test - public void testChannelWithDifferentTypes() throws IOException, InterruptedException { - - - // server 1 - in this example only server 1 write any data - ChronicleMap charMap1; - ChronicleMap intMap1; - { - ChronicleMapBuilder intMapBuilder = - of(Integer.class, Integer.class) - .entries(1000); - - ChronicleMapBuilder charMapBuilder = - of(CharSequence.class, CharSequence.class) - .averageKeySize(10) - .averageValueSize(100) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig.of(8025); - - ReplicationHub hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .maxNumberOfChannels(NUMBER_OF_CHANNELS) - .createWithId(identifier); - - - ReplicationChannel channel1 = hubOnServer1.createChannel(1); - - intMap1 = intMapBuilder.instance() - .replicatedViaChannel(channel1) - .create(); - - ReplicationChannel channel2 = hubOnServer1.createChannel(2); - - charMap1 = charMapBuilder.instance() - .replicatedViaChannel - (channel2) - .create(); - - intMap1.put(1, 1); - charMap1.put("Hello", "World"); - - } - - - // server 2 - ChronicleMap charMap2; - ChronicleMap intMap2; - - { - ChronicleMapBuilder intMapBuilder = - of(Integer.class, Integer.class) - .entries(1000); - - ChronicleMapBuilder charMapBuilder = - of(CharSequence.class, CharSequence.class) - .averageKeySize(10) - .averageValueSize(100) - .entries(1000); - - byte identifier = (byte) 2; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig.of(8026, new - InetSocketAddress("localhost", 8025)); // change the localhost to your server - - ReplicationHub hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .maxNumberOfChannels(NUMBER_OF_CHANNELS) - .createWithId(identifier); - - - ReplicationChannel channel1 = hubOnServer1.createChannel(1); - - intMap2 = intMapBuilder.instance() - .replicatedViaChannel(channel1) - .create(); - - ReplicationChannel channel2 = hubOnServer1.createChannel(2); - - charMap2 = charMapBuilder.instance().replicatedViaChannel - (channel2) - .create(); - - } - - // allow time for the recompilation to resolve - for (int t = 0; t < 3500; t++) { - if (intMap1.equals(intMap2) && - charMap1.equals(charMap2)) - break; - Thread.sleep(1); - } - - Assert.assertEquals(intMap1,intMap2); - Assert.assertEquals(charMap1,charMap2); - - } - - - -} - diff --git a/src/test/java/net/openhft/chronicle/map/TestReplication.java b/src/test/java/net/openhft/chronicle/map/TestReplication.java deleted file mode 100644 index 2192dd13d..000000000 --- a/src/test/java/net/openhft/chronicle/map/TestReplication.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.Assert; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; - -/** - * Created by Rob Austin - */ -public class TestReplication { - - @Test - public void testAllDataGetsReplicated() throws InterruptedException { - TcpTransportAndNetworkConfig tcpConfigServer1 = - TcpTransportAndNetworkConfig.of(8097); - - TcpTransportAndNetworkConfig tcpConfigServer2 = - TcpTransportAndNetworkConfig.of(8098, new InetSocketAddress("localhost", - 8097)); - - try (ChronicleMap map2 = ChronicleMap.of(Integer.class, Integer.class) - .entries(70000) - .replication((byte) 2, tcpConfigServer2) - .create(); - ChronicleMap map1 = ChronicleMap.of(Integer.class, Integer.class) - .entries(70000) - .replication((byte) 3, tcpConfigServer1) - .create()) { - - for (int i = 0; i < 70000; i++) { - map1.put(i, i); - } - - for (int i = 0; i < 10; i++) { - Thread.sleep(100); - System.out.println(map2.size()); - } - - Assert.assertEquals(map1.size(), map2.size()); - } - System.gc(); - } - - /** - * tests that replication works when the chronicle map that is getting the update is sharing - * memory with the maps that is doing the update - */ - @Test - public void testReplicationUsingAProxyMapThatsUpdatedViaSharedMemory() throws IOException { - - ChronicleMap server1 = null; - ChronicleMap forServer1Replication = null; - ChronicleMap server2 = null; - - // server 1 - for this test server1 and server 2 are on the same localhost but different - // ports, to make it easier to run in a unit test - { - final File tempFile = File.createTempFile("test", "chron"); - - - server1 = ChronicleMap.of(String.class, String.class) - .entries(1) - .averageKey("key").averageValue("value") - .replication((byte) 1).createPersistedTo(tempFile); - - TcpTransportAndNetworkConfig serverConfig = TcpTransportAndNetworkConfig.of(9000); - - // user for replication only - forServer1Replication = ChronicleMap.of(String.class, String.class) - .replication((byte) 1, serverConfig).createPersistedTo(tempFile); - - } - - // server 2- for this test server1 and server 2 are on the same localhost but different - // ports, to make it easier to run in a unit test - { - TcpTransportAndNetworkConfig server2Config = TcpTransportAndNetworkConfig.of(9001, new - InetSocketAddress("localhost", 9000)); - - server2 = ChronicleMapBuilder.of(String.class, String.class) - .entries(1) - .averageKey("key").averageValue("value") - .replication((byte) 2, server2Config).create(); - - } - - try (ChronicleMap s1 = server1; - ChronicleMap r = forServer1Replication; - ChronicleMap s2 = server2) { - - final String expected = "value"; - server1.put("key", expected); - - String actual; - - - // we have a while loop here as we have to wait a few seconds for the data to replicate - do { - actual = server2.get("key"); - } - while (actual == null); - - Assert.assertEquals(expected, actual); - } - System.gc(); - } -} diff --git a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java deleted file mode 100644 index 48abf0b33..000000000 --- a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.hash.replication.TimeProvider; -import net.openhft.chronicle.map.jsr166.JSR166TestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; - -import static net.openhft.chronicle.set.Builder.waitTillEqual; -import static org.junit.Assert.assertEquals; - -/** - * @author Rob Austin. - */ -@Ignore("issues with running tests") -@RunWith(PowerMockRunner.class) -@PrepareForTest(TimeProvider.class) -public class TimeBasedReplicationTest extends JSR166TestCase { - - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - public void testIgnoreLateAction(BiConsumer, - ChronicleMap> action) - throws IOException, InterruptedException { - - PowerMockito.mockStatic(TimeProvider.class); - ChronicleMapBuilder builder = ChronicleMapBuilder - .of(CharSequence.class, CharSequence.class) - .averageKey("key-1") - .averageValue("value-1") - .entries(10) - .cleanupRemovedEntries(false); - try (ChronicleMap map = builder - .replication((byte) 1, TcpTransportAndNetworkConfig.of(8086)).create()) { - try (ChronicleMap map2 = builder - .replication((byte) 2, TcpTransportAndNetworkConfig - .of(8087, new InetSocketAddress("localhost", 8086))).create()) { - current(); - - // we do a put at the current time - map.put("key-1", "value-1"); - assertEquals(map.size(), 1); - assertEquals(map.get("key-1"), "value-1"); - - // now test assume that we receive a late update to the map, - // the following update should be ignored - late(); - - action.accept(map, map2); - - // we'll now flip the time back to the current in order to do the read the result - current(); - waitTillEqual(map, map2, 5000); - assertEquals(map.size(), 1); - // prove that late update to map2 ignored - assertEquals("value-1", map.get("key-1")); - } - } - } - - @Test - public void testIgnoreALatePut() throws IOException, InterruptedException { - testIgnoreLateAction((m1, m2) -> { - m2.put("key-1", "value-2"); - assertEquals("value-2", m2.get("key-1")); - }); - } - - @Test - public void testIgnoreALatePutIfAbsent() throws IOException { - - PowerMockito.mockStatic(TimeProvider.class); - try (ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKey("key-1").averageValue("value-1") - .entries(10) - .cleanupRemovedEntries(false) - .replication((byte) 1) - .create()) { - current(); - - // we do a put at the current time - map.put("key-1", "value-1"); - assertEquals(map.size(), 1); - assertEquals(map.get("key-1"), "value-1"); - - // now test assume that we receive a late update to the map, the following update - // should be ignored - late(); - - // should NOT throw LateUpdateException because the value is present and it is just - // returned. - assertEquals("value-1", map.putIfAbsent("key-1", "value-2")); - - // we'll now flip the time back to the current in order to do the read the result - current(); - assertEquals(1, map.size()); - assertEquals(map.get("key-1"), "value-1"); - } - } - - @Test - public void testIgnoreALateReplace() throws IOException, InterruptedException { - testIgnoreLateAction((m1, m2) -> { - // in order to be able to call replace() "locally" - m2.put("key-1", "dummy"); - m2.replace("key-1", "value-2"); - assertEquals("value-2", m2.get("key-1")); - }); - } - - @Test - public void testIgnoreALateReplaceWithValue() throws IOException, InterruptedException { - testIgnoreLateAction((m1, m2) -> { - // in order to be able to call replace() "locally" - m2.put("key-1", "dummy"); - m2.replace("key-1", "dummy", "value-2"); - assertEquals("value-2", m2.get("key-1")); - }); - } - - @Test - public void testIgnoreALateRemoveWithValue() throws IOException, InterruptedException { - testIgnoreLateAction((m1, m2) -> { - // in order to be able to call remove() "locally" - m2.put("key-1", "dummy"); - m2.remove("key-1", "dummy"); - assertEquals(null, m2.get("key-1")); - }); - } - - @Test - public void testIgnoreALateRemove() throws IOException, InterruptedException { - testIgnoreLateAction((m1, m2) -> { - // in order to be able to call remove() "locally" - m2.put("key-1", "dummy"); - m2.remove("key-1"); - assertEquals(null, m2.get("key-1")); - }); - } - - @Test - public void testRemoveFollowedByLatePut() throws IOException, InterruptedException { - - PowerMockito.mockStatic(TimeProvider.class); - try (ChronicleMap map = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKey("key-1").averageValue("value-1") - .entries(10) - .cleanupRemovedEntries(false) - .replication((byte) 1, TcpTransportAndNetworkConfig.of(8086)) - .create()) { - try (ChronicleMap map2 = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKey("key-1").averageValue("value-1") - .entries(10) - .cleanupRemovedEntries(false) - .replication((byte) 2, TcpTransportAndNetworkConfig.of(8087, - new InetSocketAddress("localhost", 8086))) - .create()) { - current(); - - // we do a put at the current time - map.put("key-1", "value-1"); - map.remove("key-1"); - assertEquals(map.size(), 0); - assertEquals(null, map.get("key-1")); - - // now test assume that we receive a late update to the map, - // the following update should be ignored - late(); - - map2.put("key-1", "value-2"); - assertEquals("value-2", map2.get("key-1")); - - // we'll now flip the time back to the current in order to do the read the result - current(); - waitTillEqual(map, map2, 5000); - assertEquals("map: " + map + ", map2: " + map2, 0, map.size()); - // prove that late put was ignored - assertEquals(null, map.get("key-1")); - } - } - } - - private void current() { - Mockito.when(TimeProvider.currentTime()).thenReturn(System.currentTimeMillis()); - } - - private void late() { - Mockito.when(TimeProvider.currentTime()) - .thenReturn(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(10)); - } -} diff --git a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServersTest.java b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServersTest.java deleted file mode 100644 index 3e4028f07..000000000 --- a/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServersTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Rob Austin. - */ -public class TwoMapOnDifferentServersTest { - - private ChronicleMap map1; - private ChronicleMap map2; - - @Before - public void setup() throws IOException { - - final TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(8076, new InetSocketAddress("localhost", 8077)) - .heartBeatInterval(1, SECONDS).autoReconnectedUponDroppedConnection(true); - - map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-2") - .entries(20000) - .replication((byte) 1, tcpConfig).create(); - - TcpTransportAndNetworkConfig config2 = TcpTransportAndNetworkConfig.of(8077) - .heartBeatInterval(1, SECONDS).autoReconnectedUponDroppedConnection(true); - - map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("EXAMPLE-2") - .entries(20000) - .replication((byte) 2, config2).create(); - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map1, map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test3() throws IOException, InterruptedException { - - map1.put(5, "EXAMPLE-2"); - - // allow time for the recompilation to resolve - // waitTillEqual(5000); - Thread.sleep(5000); - - assertEquals(map1, map2); - assertTrue(!map1.isEmpty()); - } - - /** - * * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2)) - break; - Thread.sleep(1); - } - } - -} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java deleted file mode 100644 index feb3501f8..000000000 --- a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication; -import net.openhft.chronicle.hash.replication.UdpTransportConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.Inet4Address; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class UDPSocketReplicationTest { - - private ChronicleMap map2; - - static ChronicleMap newUdpSocketShmIntString( - final int identifier, - final int udpPort) throws IOException { - - UdpTransportConfig udpConfig = UdpTransportConfig - .of(Inet4Address.getByName("255.255.255.255"), udpPort); - - return ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .averageValue("E") - .entries(1000) - .replication(SingleChronicleHashReplication.builder().udpTransport(udpConfig) - .createWithId((byte) identifier)) - .create(); - } - - @Before - public void setup() throws IOException { - - map2 = newUdpSocketShmIntString(1, 1234); - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{map2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - @Test - public void testBufferOverflow() throws IOException, InterruptedException { - - for (int i = 0; i < 1024; i++) { - Thread.sleep(5); - map2.put(i * 2, "E"); - System.out.println("" + map2); - } - } - -} - diff --git a/src/test/java/net/openhft/chronicle/map/example/ChannelReplicationTest.java b/src/test/java/net/openhft/chronicle/map/example/ChannelReplicationTest.java deleted file mode 100644 index 45b5c7019..000000000 --- a/src/test/java/net/openhft/chronicle/map/example/ChannelReplicationTest.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map.example; - -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleMapBuilder; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.net.InetSocketAddress; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.assertEquals; - -/** - * @author Rob Austin. - */ -public class ChannelReplicationTest { - - private ChronicleMap favoriteComputerServer1; - private ChronicleMap favoriteComputerServer2; - - private ChronicleMap favoriteColourServer2; - private ChronicleMap favoriteColourServer1; - - private ReplicationHub hubOnServer1; - private ReplicationHub hubOnServer2; - - @Test - public void test() throws IOException, InterruptedException { - - // server 1 with identifier = 1 - { - ChronicleMapBuilder smallStringToStringMapBuilder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(5).averageValueSize(5) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8086, new InetSocketAddress("localhost", 8087)) - .heartBeatInterval(1, SECONDS).autoReconnectedUponDroppedConnection(true); - - hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer1 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel1)).create(); - - favoriteColourServer1.put("peter", "green"); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer1 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel2)).create(); - - favoriteComputerServer1.put("peter", "dell"); - } - - // server 2 with identifier = 2 - { - ChronicleMapBuilder smallStringToStringMapBuilder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(5).averageValueSize(5).entries(1000); - - byte identifier = (byte) 2; - - TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(8087).heartBeatInterval(1, SECONDS) - .autoReconnectedUponDroppedConnection(true); - - hubOnServer2 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer2 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel1)).create(); - - favoriteColourServer2.put("rob", "blue"); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer2 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel2)).create(); - - favoriteComputerServer2.put("rob", "mac"); - favoriteComputerServer2.put("daniel", "mac"); - } - - // allow time for the recompilation to resolve - for (int t = 0; t < 2500; t++) { - if (favoriteComputerServer2.equals(favoriteComputerServer1) && - favoriteColourServer2.equals(favoriteColourServer1)) - break; - Thread.sleep(1); - } - - assertEquals(favoriteComputerServer1, favoriteComputerServer2); - Assert.assertEquals(3, favoriteComputerServer2.size()); - - assertEquals(favoriteColourServer1, favoriteColourServer2); - Assert.assertEquals(2, favoriteColourServer1.size()); - - favoriteColourServer1.close(); - favoriteComputerServer2.close(); - favoriteColourServer2.close(); - favoriteComputerServer1.close(); - } - - @Test - public void testPublishOnOneMapOnlyBootstrapTwice() throws IOException, InterruptedException { - - ChronicleMap favoriteComputerServer3; - - ChronicleMap favoriteColourServer3; - - // server 1 with identifier = 1 - { - ChronicleMapBuilder smallStringToStringMapBuilder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(5).averageValueSize(5) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8086) - .heartBeatInterval(1, SECONDS).autoReconnectedUponDroppedConnection(true); - - hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer1 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel1)).create(); - - favoriteColourServer1.put("peter", "green"); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer1 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel2)).create(); - - favoriteComputerServer1.put("peter", "dell"); - } - - // server 2 with identifier = 2 - { - ChronicleMapBuilder builder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(5).averageValueSize(5) - .entries(1000); - - byte identifier = (byte) 6; - - TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(8087, new InetSocketAddress("localhost", 8086)) - .heartBeatInterval(1, SECONDS) - .autoReconnectedUponDroppedConnection(true); - - ReplicationHub hubOnServer = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - ChronicleHashInstanceBuilder> instance = builder.instance().replicatedViaChannel(hubOnServer.createChannel(channel1)); - - favoriteColourServer2 = instance.create(); - } - - { - ChronicleMapBuilder smallStringToStringMapBuilder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(5).averageValueSize(5) - .entries(1000); - - byte identifier = (byte) 3; - - TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(8088, new InetSocketAddress("localhost", 8086)) - .heartBeatInterval(1, SECONDS) - .autoReconnectedUponDroppedConnection(true); - - ReplicationHub hubOnServer3 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer3 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer3.createChannel(channel1)).create(); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer3 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer3.createChannel(channel2)).create(); - } - - // allow time for the recompilation to resolve - for (int t = 0; t < 250000; t++) { - if (favoriteComputerServer1.equals(favoriteComputerServer3) && - favoriteColourServer1.equals(favoriteColourServer3) && - favoriteColourServer2.equals(favoriteColourServer3)) - break; - Thread.sleep(1); - } - - assertEquals(favoriteComputerServer3, favoriteComputerServer1); - assertEquals(favoriteColourServer2, favoriteColourServer3); - - Assert.assertEquals(1, favoriteColourServer3.size()); - Assert.assertEquals(1, favoriteComputerServer1.size()); - - favoriteComputerServer3.close(); - favoriteColourServer3.close(); - - favoriteComputerServer1.close(); - favoriteColourServer1.close(); - - favoriteColourServer2.close(); - } - - @Test - public void testPublishOnOneMapOnly() throws IOException, InterruptedException { - - // server 1 with identifier = 1 - { - ChronicleMapBuilder smallStringToStringMapBuilder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(5).averageValueSize(5) - .entries(1000); - - byte identifier = (byte) 1; - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(8086, new InetSocketAddress("localhost", 8087)) - .heartBeatInterval(1, SECONDS) - .autoReconnectedUponDroppedConnection(true); - - hubOnServer1 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer1 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel1)).create(); - - favoriteColourServer1.put("peter", "green"); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer1 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer1.createChannel(channel2)).create(); - - favoriteComputerServer1.put("peter", "dell"); - } - - // server 2 with identifier = 2 - { - ChronicleMapBuilder smallStringToStringMapBuilder = - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize(5).averageValueSize(5) - .entries(1000); - - byte identifier = (byte) 2; - - TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(8087).heartBeatInterval(1, SECONDS) - .autoReconnectedUponDroppedConnection(true); - - hubOnServer2 = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .createWithId(identifier); - - // this demotes favoriteColour - short channel1 = (short) 1; - - favoriteColourServer2 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel1)).create(); - - // this demotes favoriteComputer - short channel2 = (short) 2; - - favoriteComputerServer2 = smallStringToStringMapBuilder.instance() - .replicatedViaChannel(hubOnServer2.createChannel(channel2)).create(); - } - - // allow time for the recompilation to resolve - for (int t = 0; t < 2500; t++) { - if (favoriteComputerServer2.equals(favoriteComputerServer1) && - favoriteColourServer2.equals(favoriteColourServer1)) - break; - Thread.sleep(1); - } - - assertEquals(favoriteComputerServer1, favoriteComputerServer2); - Assert.assertEquals(1, favoriteComputerServer2.size()); - - assertEquals(favoriteColourServer1, favoriteColourServer2); - Assert.assertEquals(1, favoriteColourServer1.size()); - - favoriteColourServer1.close(); - favoriteComputerServer2.close(); - favoriteColourServer2.close(); - favoriteComputerServer1.close(); - } -} - diff --git a/src/test/java/net/openhft/chronicle/map/example/ServersMapMain.java b/src/test/java/net/openhft/chronicle/map/example/ServersMapMain.java deleted file mode 100755 index 2bd934690..000000000 --- a/src/test/java/net/openhft/chronicle/map/example/ServersMapMain.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map.example; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleMapBuilder; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; - -/** - * Created by peter.lawrey on 12/11/14. - */ -public class ServersMapMain { - static final int port = Integer.getInteger("port", 8989); - static final int entries = Integer.getInteger("entries", 100000); - static final int runs = Integer.getInteger("runs", 5); - - public static void startServer() throws IOException { - File file = File.createTempFile("testServersMapMain", ".deleteme"); - file.deleteOnExit(); - final ChronicleMap serverMap = - ChronicleMapBuilder.of(byte[].class, byte[].class) - .putReturnsNull(true) - .constantKeySizeBySample(new byte[8]) - .constantValueSizeBySample(new byte[32]) - .replication((byte) 2, TcpTransportAndNetworkConfig.of(port)) - .createPersistedTo(file); - System.out.println("Server started"); - System.in.read(); - serverMap.close(); - } - - public static void startRemoteClient(String hostname) throws IOException { - final ChronicleMap map; - File file = File.createTempFile("testServersMapMain", ".deleteme"); - file.deleteOnExit(); - TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(port, new InetSocketAddress(hostname, port)); - - map = ChronicleMapBuilder - .of(byte[].class, byte[].class) - .putReturnsNull(true) - .replication((byte) 1, tcpConfig) - .createPersistedTo(file); - ByteBuffer key = ByteBuffer.allocate(8); - ByteBuffer value = ByteBuffer.allocate(32); - double lastAveragePut = Double.POSITIVE_INFINITY, lastAverageGet = Double.POSITIVE_INFINITY; - for (int i = 0; i < runs; i++) { - long puts = 0, gets = 0; - for (int j = 0; j < entries; j++) { - key.putLong(0, j); - value.putLong(0, j); - long t1 = System.nanoTime(); - map.put(key.array(), value.array()); - long t2 = System.nanoTime(); - map.get(key.array()); - long t3 = System.nanoTime(); - puts += t2 - t1; - gets += t3 - t2; - if (t2 - t1 > lastAveragePut * 100 || t3 - t2 > lastAverageGet * 100) - System.out.printf("Took put/get took %.1f/%.1f us%n", - (t2 - t1) / 1e3, (t3 - t2) / 1e3); - } - lastAveragePut = puts / entries; - lastAverageGet = gets / entries; - System.out.printf("Average took put/get took %.1f/%.1f us%n", - lastAveragePut / 1e3, lastAverageGet / 1e3); - } - } - - public static void main(String[] args) throws IOException { - if (args.length > 0) - startRemoteClient(args[0]); - else - startServer(); - } - -} diff --git a/src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java b/src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java deleted file mode 100755 index 8f573e3e3..000000000 --- a/src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map.example; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleMapBuilder; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; - -/** - * Created by peter.lawrey on 12/11/14. - */ -public class StringsMapMain { - static final int port = Integer.getInteger("port", 8998); - static final int entries = Integer.getInteger("entries", 100000); - static final int runs = Integer.getInteger("runs", 5); - - public static void startServer() throws IOException { - File file = File.createTempFile("testServersMapMain", ".deleteme"); - file.deleteOnExit(); - int keyAndValueSize = (int) Math.round(Math.log10(entries)); - final ChronicleMap serverMap = ChronicleMapBuilder - .of(CharSequence.class, CharSequence.class) - .entries(entries) - .putReturnsNull(true) - .averageKeySize(keyAndValueSize) - .averageValueSize(keyAndValueSize) - .replication((byte) 2, TcpTransportAndNetworkConfig.of(port)) - .createPersistedTo(file); - System.out.println("Server started"); - System.in.read(); - serverMap.close(); - } - - public static void startRemoteClient(String hostname) throws IOException { - final ChronicleMap map; - File file = File.createTempFile("testServersMapMain", ".deleteme"); - file.deleteOnExit(); - TcpTransportAndNetworkConfig tcpConfig = - TcpTransportAndNetworkConfig.of(port, new InetSocketAddress(hostname, port)); - - map = ChronicleMapBuilder - .of(CharSequence.class, CharSequence.class) - .putReturnsNull(true) - .replication((byte) 1, tcpConfig) - .createPersistedTo(file); - StringBuilder key = new StringBuilder(); - StringBuilder value = new StringBuilder(); - StringBuilder value2 = new StringBuilder(); - double lastAveragePut = Double.POSITIVE_INFINITY, lastAverageGet = Double.POSITIVE_INFINITY; - for (int i = 0; i < runs; i++) { - long puts = 0, gets = 0; - for (int j = 0; j < entries; j++) { - key.setLength(0); - key.append(j); - value.setLength(0); - value.append(j); - long t1 = System.nanoTime(); - map.put(key, value); - long t2 = System.nanoTime(); - map.getUsing(key, value); - long t3 = System.nanoTime(); - puts += t2 - t1; - gets += t3 - t2; -// if (t2 - t1 > lastAveragePut * 100 || t3 - t2 > lastAverageGet * 100) -// System.out.printf("Took put/get took %.1f/%.1f us%n", -// (t2 - t1) / 1e3, (t3 - t2) / 1e3); - } - lastAveragePut = puts / entries; - lastAverageGet = gets / entries; - System.out.printf("Average took put/get took %.1f/%.1f us%n", - lastAveragePut / 1e3, lastAverageGet / 1e3); - } - } - - public static void main(String[] args) throws IOException { - if (args.length > 0) - startRemoteClient(args[0]); - else - startServer(); - } - -} diff --git a/src/test/java/net/openhft/chronicle/set/Builder.java b/src/test/java/net/openhft/chronicle/set/Builder.java index 43b3b63db..dd52aec8d 100644 --- a/src/test/java/net/openhft/chronicle/set/Builder.java +++ b/src/test/java/net/openhft/chronicle/set/Builder.java @@ -16,18 +16,10 @@ package net.openhft.chronicle.set; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.hash.serialization.SizeMarshaller; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleMapBuilder; - import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.net.InetSocketAddress; -import java.util.Arrays; import java.util.Map; -import java.util.concurrent.TimeUnit; /** * @author Rob Austin. @@ -80,29 +72,4 @@ public class Builder { } } - public static > T newMapDummyValue( - final byte identifier, - final int serverPort, - final InetSocketAddress... endpoints) throws IOException { - return (T) newTcpSocketShmBuilder(Integer.class, DummyValue.class, - identifier, serverPort, endpoints) - .valueReaderAndDataAccess( - DummyValueMarshaller.INSTANCE, DummyValueMarshaller.INSTANCE) - .valueSizeMarshaller(SizeMarshaller.constant(0)) - .create(); - } - - public static ChronicleMapBuilder newTcpSocketShmBuilder( - Class kClass, Class vClass, - final byte identifier, - final int serverPort, - final InetSocketAddress... endpoints) throws IOException { - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(serverPort, Arrays.asList(endpoints)) - .heartBeatInterval(1L, TimeUnit.SECONDS).autoReconnectedUponDroppedConnection(true); - return ChronicleMapBuilder.of(kClass, vClass) - .entries(SIZE) - .replication(identifier, tcpConfig); - } - } diff --git a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplication3wayTest.java b/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplication3wayTest.java deleted file mode 100644 index d99518ddc..000000000 --- a/src/test/java/net/openhft/chronicle/set/SetForTCPSocketReplication3wayTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.set; - -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; - -import static java.util.Arrays.asList; -import static java.util.concurrent.TimeUnit.SECONDS; -import static net.openhft.chronicle.set.Builder.getPersistenceFile; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test Replicated Chronicle Set where the replication is over a TCP Socket - * - * @author Rob Austin. - */ -public class SetForTCPSocketReplication3wayTest { - - private ChronicleSet set1; - private ChronicleSet set2; - private ChronicleSet set3; - - private static > T newTcpSocketIntSet( - final byte identifier, - final int serverPort, - final InetSocketAddress... endpoints) throws IOException { - return (T) ChronicleSet.of(Integer.class) - .entries(10) - .replication(identifier, TcpTransportAndNetworkConfig.of(serverPort, asList(endpoints)) - .heartBeatInterval(1L, SECONDS).autoReconnectedUponDroppedConnection(true)) - .createPersistedTo(getPersistenceFile()); - } - - @Before - public void setup() throws IOException { - set1 = newTcpSocketIntSet((byte) 1, 18076, new InetSocketAddress("localhost", 18077), - new InetSocketAddress("localhost", 18079)); - set2 = newTcpSocketIntSet((byte) 2, 18077, new InetSocketAddress("localhost", 18079)); - set3 = newTcpSocketIntSet((byte) 3, 18079); - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{set1, set2, set3}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - @Test - public void test3() throws IOException, InterruptedException { - - set3.add(5); - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals(set1, set2); - assertEquals(set2, set2); - assertTrue(!set1.isEmpty()); - } - - @Test - public void test() throws IOException, InterruptedException { - - set1.add(1); - set1.add(2); - set1.add(2); - - set2.add(5); - set2.add(6); - - set1.remove(2); - set2.remove(3); - set1.remove(3); - set2.add(5); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(set1, set2); - assertEquals(set2, set3); - assertTrue(!set1.isEmpty()); - } - - @Test - public void testClear() throws IOException, InterruptedException { - - set1.add(1); - set1.add(2); - set1.add(2); - - set2.add(5); - set2.add(6); - - set1.clear(); - - set2.add(5); - - Thread.sleep(100); - - // allow time for the recompilation to resolve - waitTillEqual(15000); - - assertEquals(set1, set2); - assertEquals(set2, set3); - assertTrue(!set1.isEmpty()); - } - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (set1.equals(set2) && - set1.equals(set3) && - set2.equals(set3)) - break; - Thread.sleep(1); - } - } -} - diff --git a/src/test/java/net/openhft/chronicle/set/SetRemoteOperationsTest.java b/src/test/java/net/openhft/chronicle/set/SetRemoteOperationsTest.java deleted file mode 100644 index ed71d7231..000000000 --- a/src/test/java/net/openhft/chronicle/set/SetRemoteOperationsTest.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.set; - -import net.openhft.chronicle.hash.replication.ReplicableEntry; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.map.ChannelReplicationTest; -import net.openhft.chronicle.set.replication.SetRemoteOperations; -import net.openhft.chronicle.set.replication.SetRemoteQueryContext; -import net.openhft.chronicle.set.replication.SetReplicableEntry; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static net.openhft.chronicle.hash.replication.DefaultEventualConsistencyStrategy.AcceptanceDecision.ACCEPT; -import static net.openhft.chronicle.hash.replication.DefaultEventualConsistencyStrategy.AcceptanceDecision.DISCARD; -import static net.openhft.chronicle.hash.replication.DefaultEventualConsistencyStrategy.decideOnRemoteModification; -import static org.junit.Assert.assertEquals; - -public class SetRemoteOperationsTest { - - static int s_port = 12050; - private ChronicleSet set1; - private ChronicleSet set2; - private AtomicInteger set1PutCounter; - private AtomicInteger set2PutCounter; - private AtomicInteger set1RemoveCounter; - private AtomicInteger set2RemoveCounter; - - @Before - public void setup() throws IOException { - int port = s_port; - set1PutCounter = new AtomicInteger(); - set1RemoveCounter = new AtomicInteger(); - ChronicleSetBuilder set1Builder = ChronicleSet - .of(Integer.class) - .remoteOperations(new SetRemoteOperations() { - @Override - public void remove(SetRemoteQueryContext q) { - SetReplicableEntry entry = q.entry(); - if (entry != null) { - if (decideOnRemoteModification(entry, q) == ACCEPT) { - q.remove(entry); - ReplicableEntry replicableAbsentEntry = - (ReplicableEntry) q.absentEntry(); - replicableAbsentEntry - .updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - replicableAbsentEntry.dropChanged(); - set1RemoveCounter.addAndGet(1); - } - } else { - SetAbsentEntry absentEntry = q.absentEntry(); - ReplicableEntry replicableAbsentEntry; - if (!(absentEntry instanceof ReplicableEntry)) { - absentEntry.doInsert(); - q.entry().doRemove(); - replicableAbsentEntry = (ReplicableEntry) q.absentEntry(); - } else { - replicableAbsentEntry = (ReplicableEntry) absentEntry; - if (decideOnRemoteModification(replicableAbsentEntry, q) == DISCARD) - return; - } - replicableAbsentEntry - .updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - replicableAbsentEntry.dropChanged(); - set1RemoveCounter.addAndGet(1); - } - } - - @Override - public void put(SetRemoteQueryContext q) { - SetReplicableEntry entry = q.entry(); - if (entry != null) { - if (decideOnRemoteModification(entry, q) == ACCEPT) { - entry.updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - entry.dropChanged(); - set1PutCounter.addAndGet(1); - } - } else { - SetAbsentEntry absentEntry = q.absentEntry(); - assert absentEntry != null; - if (!(absentEntry instanceof ReplicableEntry) || - decideOnRemoteModification((ReplicableEntry) absentEntry, q) == - ACCEPT) { - q.insert(absentEntry); - entry = q.entry(); // q.entry() is not null after insert - assert entry != null; - entry.updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - entry.dropChanged(); - set1PutCounter.addAndGet(1); - } - } - } - }) - .replication((byte) 1, TcpTransportAndNetworkConfig - .of(port, new InetSocketAddress("localhost", port + 1))); - set1 = set1Builder.entries(Builder.SIZE).create(); - - set2PutCounter = new AtomicInteger(); - set2RemoveCounter = new AtomicInteger(); - ChronicleSetBuilder set2Builder = ChronicleSet - .of(Integer.class) - .remoteOperations(new SetRemoteOperations() { - @Override - public void remove(SetRemoteQueryContext q) { - SetReplicableEntry entry = q.entry(); - if (entry != null) { - if (decideOnRemoteModification(entry, q) == ACCEPT) { - q.remove(entry); - ReplicableEntry replicableAbsentEntry = - (ReplicableEntry) q.absentEntry(); - replicableAbsentEntry - .updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - replicableAbsentEntry.dropChanged(); - set2RemoveCounter.addAndGet(1); - } - } else { - SetAbsentEntry absentEntry = q.absentEntry(); - ReplicableEntry replicableAbsentEntry; - if (!(absentEntry instanceof ReplicableEntry)) { - absentEntry.doInsert(); - q.entry().doRemove(); - replicableAbsentEntry = (ReplicableEntry) q.absentEntry(); - } else { - replicableAbsentEntry = (ReplicableEntry) absentEntry; - if (decideOnRemoteModification(replicableAbsentEntry, q) == DISCARD) - return; - } - replicableAbsentEntry - .updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - replicableAbsentEntry.dropChanged(); - set2RemoveCounter.addAndGet(1); - } - } - - @Override - public void put(SetRemoteQueryContext q) { - SetReplicableEntry entry = q.entry(); - if (entry != null) { - if (decideOnRemoteModification(entry, q) == ACCEPT) { - entry.updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - entry.dropChanged(); - set2PutCounter.addAndGet(1); - } - } else { - SetAbsentEntry absentEntry = q.absentEntry(); - assert absentEntry != null; - if (!(absentEntry instanceof ReplicableEntry) || - decideOnRemoteModification((ReplicableEntry) absentEntry, q) == - ACCEPT) { - q.insert(absentEntry); - entry = q.entry(); // q.entry() is not null after insert - assert entry != null; - entry.updateOrigin(q.remoteIdentifier(), q.remoteTimestamp()); - entry.dropChanged(); - set2PutCounter.addAndGet(1); - } - } - } - }) - .replication((byte) 2, TcpTransportAndNetworkConfig.of(port + 1)); - set2 = set2Builder.entries(Builder.SIZE).create(); - s_port += 2; - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{set1, set2}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void setRemoteOperationsTest() throws InterruptedException { - set1.add(1); - set2.add(2); - - waitTillEqual(5000); - assertEquals(set1, set2); - - set1.remove(2); - set2.remove(1); - - waitTillEqual(5000); - assertEquals(set1, set2); - - assertEquals(1, set1PutCounter.get()); - assertEquals(1, set2PutCounter.get()); - assertEquals(1, set1RemoveCounter.get()); - assertEquals(1, set2RemoveCounter.get()); - } - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (set1.equals(set2)) - break; - Thread.sleep(1); - } - } -} diff --git a/src/test/java/net/openhft/chronicle/set/TCPSocketReplication3VoidValueTest.java b/src/test/java/net/openhft/chronicle/set/TCPSocketReplication3VoidValueTest.java deleted file mode 100644 index 89e35eef0..000000000 --- a/src/test/java/net/openhft/chronicle/set/TCPSocketReplication3VoidValueTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.set; - -import net.openhft.chronicle.map.ChannelReplicationTest; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.set.DummyValue; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Set; - -import static net.openhft.chronicle.set.DummyValue.DUMMY_VALUE; -import static net.openhft.chronicle.set.Builder.newMapDummyValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket - * - * @author Rob Austin. - */ -public class TCPSocketReplication3VoidValueTest { - - private ChronicleMap map1; - private ChronicleMap map2; - private ChronicleMap map3; - - @Before - public void setup() throws IOException { - map1 = newMapDummyValue((byte) 1, 8036, new InetSocketAddress("localhost", 8037), - new InetSocketAddress("localhost", 8039)); - map2 = newMapDummyValue((byte) 2, 8037, new InetSocketAddress("localhost", 8039)); - map3 = newMapDummyValue((byte) 3, 8039); - } - - @After - public void tearDown() throws InterruptedException { - for (final Closeable closeable : new Closeable[]{map1, map2, map3}) { - try { - closeable.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - System.gc(); - } - - Set threads; - - @Before - public void sampleThreads() { - threads = Thread.getAllStackTraces().keySet(); - } - - @After - public void checkThreadsShutdown() { - ChannelReplicationTest.checkThreadsShutdown(threads); - } - - @Test - public void test3() throws IOException, InterruptedException { - - assertEquals(null, map3.put(5, DUMMY_VALUE)); - - // allow time for the recompilation to resolve - assertTrue(waitTillEqual(15000)); - - assertEquals(map3, map2); - assertTrue(!map1.isEmpty()); - } - - @Test - public void test() throws IOException, InterruptedException { - - assertEquals(null, map1.put(1, DUMMY_VALUE)); - assertEquals(null, map1.put(2, DUMMY_VALUE)); - - assertEquals(null, map2.put(5, DUMMY_VALUE)); - assertEquals(null, map2.put(6, DUMMY_VALUE)); - - map1.remove(2); - map2.remove(3); - map1.remove(3); - map2.put(5, DUMMY_VALUE); - - // allow time for the recompilation to resolve - waitTillEqual(5000); - - assertEquals(map3, map3); - assertTrue(!map1.isEmpty()); - } - - @Test - public void testClear() throws IOException, InterruptedException { - - assertEquals(null, map1.put(1, DUMMY_VALUE)); - assertEquals(null, map1.put(2, DUMMY_VALUE)); - - assertEquals(null, map2.put(5, DUMMY_VALUE)); - assertEquals(null, map2.put(6, DUMMY_VALUE)); - - map1.clear(); - - map2.put(5, DUMMY_VALUE); - - // allow time for the recompilation to resolve - assertTrue("test timed out", waitTillEqual(15000)); - - assertEquals(map3, map3); - assertTrue(!map1.isEmpty()); - } - - private boolean waitTillEqual(final int timeOutMs) throws InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - if (map1.equals(map2) && - map1.equals(map3) && - map2.equals(map3)) - return true; - Thread.sleep(1); - } - return false; - } - -} -