From ea16356d46f4dc0ba5fd1a95b4cd8d061794e2d0 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 5 Sep 2014 17:39:51 +0400 Subject: [PATCH] New Chronicle-Map --- pom.xml | 61 +- .../map/AbstractChannelReplicator.java | 395 +++++ .../net/openhft/chronicle/map/Alignment.java | 85 ++ .../net/openhft/chronicle/map/BitUnit.java | 80 ++ .../openhft/chronicle/map/ChronicleMap.java | 50 +- .../chronicle/map/ChronicleMapBuilder.java | 905 ++++++++++-- .../openhft/chronicle/map/IntIntMultiMap.java | 81 ++ .../chronicle/map/MapErrorListener.java | 25 + .../chronicle/map/MapErrorListeners.java | 71 + .../chronicle/map/MapEventListener.java | 102 ++ .../chronicle/map/MapEventListeners.java | 134 ++ .../net/openhft/chronicle/map/Objects.java | 41 + .../net/openhft/chronicle/map/Replica.java | 213 +++ .../map/ReplicatedChronicleMap.java} | 975 ++++++------- .../chronicle/map/ReplicatingCluster.java | 416 ++++++ .../map/ReplicatingClusterBuilder.java | 77 + .../net/openhft/chronicle/map/Replicator.java | 62 + .../openhft/chronicle/map/Replicators.java | 63 + .../openhft/chronicle/map/SharedSegment.java | 42 + .../chronicle/map/TcpReplicationConfig.java | 118 ++ .../openhft/chronicle/map/TcpReplicator.java | 1028 +++++++++++++ .../chronicle/map/ThrottlingConfig.java | 124 ++ .../openhft/chronicle/map/TimeProvider.java | 64 + .../chronicle/map/UdpReplicationConfig.java | 69 + .../openhft/chronicle/map/UdpReplicator.java | 400 ++++++ .../map/VanillaChronicleMap.java} | 512 +++---- .../chronicle/map/VanillaIntIntMultiMap.java | 313 ++++ .../map/VanillaShortShortMultiMap.java | 303 ++++ .../collections/ClusterReplicator.java | 412 ------ .../collections/ClusterReplicatorBuilder.java | 136 -- .../openhft/collections/SharedHashMap.java | 57 - .../collections/SharedHashMapBuilder.java | 760 ---------- src/test/java/eg/BigData.java | 226 +++ .../chronicle/map/AcquireGetUsingMain.java | 94 ++ .../openhft/chronicle/map/BitUnitTest.java | 34 + .../net/openhft/chronicle/map/Builder.java | 112 ++ .../map/ByteableIntValueMarshaller.java | 69 + .../map/ByteableLongValueMarshaller.java | 70 + .../chronicle/map/CHMLatencyTestMain.java | 165 +++ .../chronicle/map/CHMMetaDataTest.java | 98 ++ .../chronicle/map/ChronicleMapTest.java | 1280 +++++++++++++++++ .../chronicle/map/ClusterReplicationTest.java | 144 ++ .../chronicle/map/ConcurrentMap8Test.java | 95 +- .../net/openhft/chronicle/map/GuavaTest.java | 126 ++ .../net/openhft/chronicle/map/Histogram.java | 84 ++ .../openhft/chronicle/map/HistogramTest.java | 34 + .../chronicle/map/IntIntMultiMapTest.java | 147 ++ .../chronicle/map/IntValueMapTest.java | 58 + .../openhft/chronicle/map/KeySizesTest.java | 56 + .../map/MultiMapTimeBaseReplicationTest.java | 278 ++++ .../openhft/chronicle/map/OSResizesMain.java | 70 + .../chronicle/map/PageLatencyMain.java | 43 + .../map}/PostClusterMapCreationTest.java | 82 +- .../map}/PreClusterMapCreationTest.java | 87 +- .../map/ProcessInstanceLimiterMain.java | 158 ++ .../chronicle/map/QueueReplicator.java | 258 ++++ .../map/RecursiveRefereneChMapTest.java | 62 + .../map/ReplicatedChronicleMapTest.java | 858 +++++++++++ .../map/ReplicatedChronicleMapTest2.java | 892 ++++++++++++ .../chronicle/map/ReplicationCheckingMap.java | 282 ++++ .../openhft/chronicle/map/SampleValues.java | 83 ++ .../map}/SingleMapClusterTest.java | 81 +- .../map/TCPSocketReplication4WayMapTest.java | 187 +++ .../TCPSocketReplicationBootStrapTests.java | 161 +++ .../map/TCPSocketReplicationIntValueTest.java | 195 +++ .../map/TCPSocketReplicationTest.java | 203 +++ .../map/TCPSocketReplicationTest3way.java | 165 +++ ...cketReplicationTest3wayPutReturnsNull.java | 155 ++ ...CPSocketReplicationTestPostConnection.java | 140 ++ .../map/TimeBasedReplicationTests.java | 359 +++++ .../map}/TwoMapOnDifferentServers.java | 50 +- .../map/UDPSocketReplicationTest.java | 93 ++ .../chronicle/map/blackboard/CHMTest5.java | 256 ++++ .../map/blackboard/CHMTestIterator1.java | 64 + .../map/fromdocs/BondVOInterface.java | 84 ++ .../map/fromdocs/GettingStartedTest.java | 66 + .../OpenJDKAndHashMapExamplesTest.java | 151 ++ .../pingpong_latency/PingPongAddLeft.java | 68 + .../pingpong_latency/PingPongAddRight.java | 33 + .../pingpong_latency/PingPongCASLeft.java | 99 ++ .../pingpong_latency/PingPongCASRight.java | 32 + .../pingpong_latency/PingPongLockLeft.java | 85 ++ .../pingpong_latency/PingPongLockRight.java | 30 + .../chronicle/map/ipc/StateMachineData.java | 173 +++ .../map/ipc/StateMachineProcessor.java | 81 ++ .../chronicle/map/ipc/StateMachineState.java | 60 + .../map/ipc/StateMachineTutorial.java | 97 ++ .../map}/jrs166/JSR166TestCase.java | 14 +- .../map/jrs166/map/ChronicleMapTest.java | 863 +++++++++++ .../chronicle/map/jrs166/map/LoopHelpers.java | 234 +++ .../chronicle/map/jrs166/map/MapCheck.java | 770 ++++++++++ .../map/utility/ProcessInstanceLimiter.java | 596 ++++++++ .../collections/ClusterReplicationTest.java | 174 --- .../openhft/lang/values/IntValue$$Native.java | 124 ++ .../lang/values/LongValue$$Native.java | 116 ++ 95 files changed, 17623 insertions(+), 2655 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java create mode 100644 src/main/java/net/openhft/chronicle/map/Alignment.java create mode 100644 src/main/java/net/openhft/chronicle/map/BitUnit.java create mode 100644 src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java create mode 100644 src/main/java/net/openhft/chronicle/map/MapErrorListener.java create mode 100644 src/main/java/net/openhft/chronicle/map/MapErrorListeners.java create mode 100644 src/main/java/net/openhft/chronicle/map/MapEventListener.java create mode 100644 src/main/java/net/openhft/chronicle/map/MapEventListeners.java create mode 100644 src/main/java/net/openhft/chronicle/map/Objects.java create mode 100644 src/main/java/net/openhft/chronicle/map/Replica.java rename src/main/java/net/openhft/{collections/VanillaSharedReplicatedHashMap.java => chronicle/map/ReplicatedChronicleMap.java} (73%) create mode 100644 src/main/java/net/openhft/chronicle/map/ReplicatingCluster.java create mode 100644 src/main/java/net/openhft/chronicle/map/ReplicatingClusterBuilder.java create mode 100644 src/main/java/net/openhft/chronicle/map/Replicator.java create mode 100644 src/main/java/net/openhft/chronicle/map/Replicators.java create mode 100644 src/main/java/net/openhft/chronicle/map/SharedSegment.java create mode 100644 src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java create mode 100644 src/main/java/net/openhft/chronicle/map/TcpReplicator.java create mode 100644 src/main/java/net/openhft/chronicle/map/ThrottlingConfig.java create mode 100644 src/main/java/net/openhft/chronicle/map/TimeProvider.java create mode 100644 src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java create mode 100644 src/main/java/net/openhft/chronicle/map/UdpReplicator.java rename src/main/java/net/openhft/{collections/VanillaSharedHashMap.java => chronicle/map/VanillaChronicleMap.java} (83%) create mode 100644 src/main/java/net/openhft/chronicle/map/VanillaIntIntMultiMap.java create mode 100644 src/main/java/net/openhft/chronicle/map/VanillaShortShortMultiMap.java delete mode 100644 src/main/java/net/openhft/collections/ClusterReplicator.java delete mode 100644 src/main/java/net/openhft/collections/ClusterReplicatorBuilder.java delete mode 100644 src/main/java/net/openhft/collections/SharedHashMap.java delete mode 100644 src/main/java/net/openhft/collections/SharedHashMapBuilder.java create mode 100644 src/test/java/eg/BigData.java create mode 100644 src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java create mode 100644 src/test/java/net/openhft/chronicle/map/BitUnitTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/Builder.java create mode 100644 src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java create mode 100644 src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java create mode 100644 src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java create mode 100644 src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/ClusterReplicationTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/GuavaTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/Histogram.java create mode 100644 src/test/java/net/openhft/chronicle/map/HistogramTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/IntIntMultiMapTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/IntValueMapTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/KeySizesTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/MultiMapTimeBaseReplicationTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/OSResizesMain.java create mode 100644 src/test/java/net/openhft/chronicle/map/PageLatencyMain.java rename src/test/java/net/openhft/{collections => chronicle/map}/PostClusterMapCreationTest.java (54%) rename src/test/java/net/openhft/{collections => chronicle/map}/PreClusterMapCreationTest.java (52%) create mode 100644 src/test/java/net/openhft/chronicle/map/ProcessInstanceLimiterMain.java create mode 100644 src/test/java/net/openhft/chronicle/map/QueueReplicator.java create mode 100644 src/test/java/net/openhft/chronicle/map/RecursiveRefereneChMapTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest2.java create mode 100644 src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java create mode 100644 src/test/java/net/openhft/chronicle/map/SampleValues.java rename src/test/java/net/openhft/{collections => chronicle/map}/SingleMapClusterTest.java (56%) create mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java create mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3way.java create mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java create mode 100644 src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java create mode 100644 src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java rename src/test/java/net/openhft/{collections => chronicle/map}/TwoMapOnDifferentServers.java (59%) create mode 100644 src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/blackboard/CHMTest5.java create mode 100644 src/test/java/net/openhft/chronicle/map/blackboard/CHMTestIterator1.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/BondVOInterface.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/GettingStartedTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddLeft.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddRight.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASRight.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockLeft.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockRight.java create mode 100644 src/test/java/net/openhft/chronicle/map/ipc/StateMachineData.java create mode 100644 src/test/java/net/openhft/chronicle/map/ipc/StateMachineProcessor.java create mode 100644 src/test/java/net/openhft/chronicle/map/ipc/StateMachineState.java create mode 100644 src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java rename src/test/java/net/openhft/{collections => chronicle/map}/jrs166/JSR166TestCase.java (99%) create mode 100644 src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/jrs166/map/LoopHelpers.java create mode 100644 src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java create mode 100644 src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java delete mode 100644 src/test/java/net/openhft/collections/ClusterReplicationTest.java create mode 100644 src/test/java/net/openhft/lang/values/IntValue$$Native.java create mode 100644 src/test/java/net/openhft/lang/values/LongValue$$Native.java diff --git a/pom.xml b/pom.xml index dd2152448..dfbd1b527 100755 --- a/pom.xml +++ b/pom.xml @@ -1,12 +1,12 @@ - + net.openhft java-parent-pom 1.1.0 - + @@ -32,7 +34,7 @@ 4.0.0 chronicle-map - 1.0.3-SNAPSHOT + 2.0.0-SNAPSHOT OpenHFT/Chronicle-Map Chronicle-Map @@ -43,20 +45,26 @@ net.openhft third-party-bom pom - 3.4.7 + 3.4.8-SNAPSHOT import net.openhft lang - 6.4.6 + 6.4.7-SNAPSHOT net.openhft - collections - 3.2.1 + affinity + 2.1.0 + + + + net.openhft + compiler + 2.2.0 @@ -82,17 +90,17 @@ testng - - net.openhft - collections - - - org.slf4j slf4j-api + + com.google.auto.value + auto-value + provided + + @@ -101,6 +109,23 @@ test + + org.mockito + mockito-core + test + + + + net.openhft + affinity + test + + + + com.google.guava + guava-testlib + test + @@ -115,8 +140,10 @@ maven-compiler-plugin -Xlint:deprecation - 1.8 - 1.8 + 1.6 + 1.6 + 1.8 + 1.8 UTF-8 diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java new file mode 100644 index 000000000..6d3c6e732 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java @@ -0,0 +1,395 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.thread.NamedThreadFactory; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SelectableChannel; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static java.lang.Math.round; +import static java.nio.channels.SelectionKey.OP_WRITE; +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static net.openhft.chronicle.map.ReplicatedChronicleMap.MAX_UNSIGNED_SHORT; + +/** + * @author Rob Austin. + */ +abstract class AbstractChannelReplicator implements Closeable { + + static final int BITS_IN_A_BYTE = 8; + static final int SIZE_OF_SHORT = 2; + private static final Logger LOG = LoggerFactory.getLogger(AbstractChannelReplicator.class); + final Selector selector; + final Set closeables = Collections.synchronizedSet(new LinkedHashSet()); + private final ExecutorService executorService; + private final Queue pendingRegistrations = new ConcurrentLinkedQueue(); + @Nullable + private final Throttler throttler; + + AbstractChannelReplicator(String name, ThrottlingConfig throttlingConfig, + int maxEntrySizeBytes) + throws IOException { + executorService = Executors.newSingleThreadExecutor( + new NamedThreadFactory(name, true)); + selector = Selector.open(); + closeables.add(selector); + + throttler = throttlingConfig.throttling(DAYS) > 0 ? + new Throttler(selector, + throttlingConfig.bucketInterval(MILLISECONDS), + maxEntrySizeBytes, throttlingConfig.throttling(DAYS)) : null; + } + + void addPendingRegistration(Runnable registration) { + pendingRegistrations.add(registration); + } + + void registerPendingRegistrations() throws ClosedChannelException { + for (Runnable runnable = pendingRegistrations.poll(); runnable != null; + runnable = pendingRegistrations.poll()) { + try { + runnable.run(); + } catch (Exception e) { + LOG.info("", e); + } + } + } + + abstract void process() throws IOException; + + final void start() { + executorService.execute( + new Runnable() { + @Override + public void run() { + try { + process(); + } catch (Exception e) { + LOG.error("", e); + } + } + } + ); + } + + + @Override + public void close() { + synchronized (this.closeables) { + for (Closeable closeable : this.closeables) { + try { + closeable.close(); + } catch (IOException e) { + LOG.error("", e); + } + } + closeables.clear(); + } + executorService.shutdownNow(); + } + + void closeEarlyAndQuietly(SelectableChannel channel) { + try { + if (throttler != null) + throttler.remove(channel); + closeables.remove(channel); + channel.close(); + } catch (IOException ex) { + // do nothing + } + } + + /** + * forces the TCP and UDP replicators to re-bootstrap + * This is called whenever a new CHM is added to a custer + */ + public abstract void forceBootstrap(); + + void checkThrottleInterval() throws ClosedChannelException { + if (throttler != null) + throttler.checkThrottleInterval(); + } + + void contemplateThrottleWrites(int bytesJustWritten) throws ClosedChannelException { + if (throttler != null) + throttler.contemplateThrottleWrites(bytesJustWritten); + } + + void throttle(SelectableChannel channel) { + if (throttler != null) + throttler.add(channel); + } + + interface ChannelReplicatorBuilder { + } + + /** + * throttles 'writes' to ensure the network is not swamped, this is achieved by periodically + * de-registering the write selector during periods of high volume. + */ + static class Throttler { + + private final Selector selector; + private final Set channels = new CopyOnWriteArraySet(); + private final long throttleInterval; + private final long maxBytesInInterval; + + private long lastTime = System.currentTimeMillis(); + private long bytesWritten; + + Throttler(@NotNull Selector selector, + long throttleIntervalInMillis, + long serializedEntrySize, + long bitsPerDay) { + + this.selector = selector; + this.throttleInterval = throttleIntervalInMillis; + double bytesPerMs = ((double) bitsPerDay) / DAYS.toMillis(1) / BITS_IN_A_BYTE; + this.maxBytesInInterval = round(bytesPerMs * throttleInterval) - serializedEntrySize; + } + + public void add(SelectableChannel selectableChannel) { + channels.add(selectableChannel); + } + + public void remove(SelectableChannel socketChannel) { + channels.remove(socketChannel); + } + + /** + * re register the 'write' on the selector if the throttleInterval has passed + * + * @throws java.nio.channels.ClosedChannelException + */ + public void checkThrottleInterval() throws ClosedChannelException { + final long time = System.currentTimeMillis(); + + if (lastTime + throttleInterval >= time) + return; + + lastTime = time; + bytesWritten = 0; + + + if (LOG.isDebugEnabled()) + LOG.debug("Restoring OP_WRITE on all channels"); + + for (SelectableChannel selectableChannel : channels) { + + final SelectionKey selectionKey = selectableChannel.keyFor(selector); + if (selectionKey != null) + selectionKey.interestOps(selectionKey.interestOps() | OP_WRITE); + + } + } + + + /** + * checks the number of bytes written in this interval, if this number of bytes exceeds a threshold, + * the selected will de-register the socket that is being written to, until the interval is finished. + * + * @throws ClosedChannelException + */ + public void contemplateThrottleWrites(int bytesJustWritten) + throws ClosedChannelException { + bytesWritten += bytesJustWritten; + if (bytesWritten > maxBytesInInterval) { + for (SelectableChannel channel : channels) { + final SelectionKey selectionKey = channel.keyFor(selector); + if (selectionKey != null) + selectionKey.interestOps(selectionKey.interestOps() & ~OP_WRITE); + + if (LOG.isDebugEnabled()) + LOG.debug("Throttling UDP writes"); + } + } + } + } + + /** + * details about the socket connection + */ + static class Details { + + private final InetSocketAddress address; + private final byte localIdentifier; + + Details(@NotNull final InetSocketAddress address, final byte localIdentifier) { + this.address = address; + this.localIdentifier = localIdentifier; + } + + public InetSocketAddress address() { + return address; + } + + public byte localIdentifier() { + return localIdentifier; + } + + @Override + public String toString() { + return "Details{" + + "address=" + address + + ", localIdentifier=" + localIdentifier + + '}'; + } + } + + static class EntryCallback extends Replica.EntryCallback { + + private final Replica.EntryExternalizable externalizable; + private final ByteBufferBytes in; + + EntryCallback(@NotNull final Replica.EntryExternalizable externalizable, + @NotNull final ByteBufferBytes in) { + this.externalizable = externalizable; + this.in = in; + } + + @Override + public boolean onEntry(final Bytes entry, final int chronicleId) { + in.skip(SIZE_OF_SHORT); + final long start = in.position(); + externalizable.writeExternalEntry(entry, in, chronicleId); + + if (in.position() == start) { + in.position(in.position() - SIZE_OF_SHORT); + return false; + } + + // write the length of the entry, just before the start, so when we read it back + // we read the length of the entry first and hence know how many preceding writer to read + final int entrySize = (int) (in.position() - start); + if (entrySize > MAX_UNSIGNED_SHORT) + throw new IllegalStateException("entry too large, the entry size=" + entrySize + ", " + + "entries are limited to a size of " + MAX_UNSIGNED_SHORT); + in.writeUnsignedShort(start - SIZE_OF_SHORT, entrySize); + + return true; + } + } + + abstract class AbstractConnector { + + private final String name; + private int connectionAttempts = 0; + private volatile SelectableChannel socketChannel; + + public AbstractConnector(String name) { + this.name = name; + } + + abstract SelectableChannel doConnect() throws IOException, InterruptedException; + + /** + * connects or reconnects, but first waits a period of time proportional to the {@code + * connectionAttempts} + */ + public final void connectLater() { + try { + if (socketChannel != null) + socketChannel.close(); + socketChannel = null; + + } catch (IOException e1) { + LOG.error("", e1); + } + + final long reconnectionInterval = connectionAttempts * 100; + if (connectionAttempts < 5) + connectionAttempts++; + doConnect(reconnectionInterval); + + } + + /** + * connects or reconnects immediately + */ + public void connect() { + doConnect(0); + } + + + /** + * @param reconnectionInterval the period to wait before connecting + */ + private void doConnect(final long reconnectionInterval) { + + final Thread thread = new Thread(new Runnable() { + + public void run() { + SelectableChannel socketChannel = null; + try { + if (reconnectionInterval > 0) + Thread.sleep(reconnectionInterval); + + synchronized (closeables) { + socketChannel = doConnect(); + if (socketChannel == null) + return; + + closeables.add(socketChannel); + AbstractConnector.this.socketChannel = socketChannel; + } + + } catch (Exception e) { + if (socketChannel != null) + try { + socketChannel.close(); + AbstractConnector.this.socketChannel = null; + } catch (IOException e1) { + // + } + LOG.error("", e); + } + + } + }); + + thread.setName(name); + thread.setDaemon(true); + thread.start(); + } + + public void setSuccessfullyConnected() { + connectionAttempts = 0; + } + } + +} diff --git a/src/main/java/net/openhft/chronicle/map/Alignment.java b/src/main/java/net/openhft/chronicle/map/Alignment.java new file mode 100644 index 000000000..91f9800d9 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Alignment.java @@ -0,0 +1,85 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; + + +public enum Alignment { + NO_ALIGNMENT { + @Override + void alignPositionAddr(Bytes bytes) { + // no-op + } + + @Override + long alignAddr(long addr) { + return addr; + } + + @Override + int alignSize(int size) { + return size; + } + }, + + OF_4_BYTES { + @Override + void alignPositionAddr(Bytes bytes) { + bytes.alignPositionAddr(4); + } + + @Override + long alignAddr(long addr) { + return (addr + 3) & ~3; + } + + @Override + int alignSize(int size) { + return (size + 3) & ~3; + } + }, + + OF_8_BYTES { + @Override + void alignPositionAddr(Bytes bytes) { + bytes.alignPositionAddr(8); + } + + @Override + long alignAddr(long addr) { + return (addr + 7) & ~7; + } + + @Override + int alignSize(int size) { + return (size + 7) & ~7; + } + }; + + private static final Alignment[] VALUES = values(); + + static Alignment fromOrdinal(int ordinal) { + return VALUES[ordinal]; + } + + abstract void alignPositionAddr(Bytes bytes); + + abstract long alignAddr(long addr); + + abstract int alignSize(int size); +} diff --git a/src/main/java/net/openhft/chronicle/map/BitUnit.java b/src/main/java/net/openhft/chronicle/map/BitUnit.java new file mode 100644 index 000000000..850c5c5f9 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/BitUnit.java @@ -0,0 +1,80 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +/** + * @author Rob Austin. + */ +public enum BitUnit { + + BITS { + public long toBits(long v) { + return v; + } + + public long toMegaBits(long v) { + return v / _MB; + } + + public long toGigaBits(long v) { + return v / _GB; + } + }, + + MEGA_BITS { + public long toBits(long v) { + return v * _MB; + } + + public long toMegaBits(long v) { + return v; + } + + public long toGigaBits(long v) { + return v / (_GB / _MB); + } + }, + + GIGA_BITS { + public long toBits(long v) { + return v * _GB; + } + + public long toMegaBits(long v) { + return v * (_GB / _MB); + } + + public long toGigaBits(long v) { + return v; + } + }; + private static long _KB = 1000; + private static long _MB = 1000 * _KB; + private static long _GB = 1000 * _MB; + + public long toBits(long duration) { + throw new AbstractMethodError(); + } + + long toMegaBits(long value) { + throw new AbstractMethodError(); + } + + public long toGigaBits(long v) { + throw new AbstractMethodError(); + } +} diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java index 6ab43ca93..e6858b4b8 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,10 +16,40 @@ package net.openhft.chronicle.map; -import net.openhft.collections.SharedHashMap; +import java.io.Closeable; +import java.io.File; +import java.util.concurrent.ConcurrentMap; + +public interface ChronicleMap extends ConcurrentMap, Closeable { + /** + * @return size as a long. + * @see java.util.Map size() + */ + long longSize(); + + /** + * Get a value for a key if available. If the value is Byteable, it will be assigned to + * reference the value, instead of copying the data for zero copy access to the collection. + * + * @param key to lookup. + * @param value to reuse if possible. If null, a new object will be created. + * @return value found or null if not. + */ + V getUsing(K key, V value); + + /** + * Acquire a value for a key, creating if absent. If the value is Byteable, it will be assigned + * to reference the value, instead of copying the data. + * + * @param key to lookup. + * @param value to reuse if possible. If null, a new object will be created. + * @return value created or found. + */ + V acquireUsing(K key, V value); + + /** + * @return The file or directory for this ChronicleMap + */ + File file(); -/** - * @author Rob Austin. - */ -public interface ChronicleMap extends SharedHashMap { } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 7b7361963..1decb8a7e 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,50 +16,159 @@ package net.openhft.chronicle.map; -import net.openhft.collections.*; -import net.openhft.lang.io.serialization.BytesMarshallerFactory; -import net.openhft.lang.io.serialization.ObjectSerializer; +import net.openhft.lang.Maths; +import net.openhft.lang.io.*; +import net.openhft.lang.io.serialization.*; +import net.openhft.lang.io.serialization.impl.*; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.DataValueClasses; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; +import java.io.*; +import java.nio.channels.FileChannel; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; -/** - * @author Rob Austin. - */ -public class ChronicleMapBuilder extends SharedHashMapBuilder implements Cloneable { +import static net.openhft.chronicle.map.Objects.builderEquals; +import static net.openhft.chronicle.map.Objects.hash; +public class ChronicleMapBuilder implements Cloneable { + + public static final short UDP_REPLICATION_MODIFICATION_ITERATOR_ID = 128; private static final Logger LOG = LoggerFactory.getLogger(ChronicleMapBuilder.class.getName()); + final Class keyClass; + final Class valueClass; + boolean transactional = false; + Replicator firstReplicator; + Map, Replicator> replicators = + new HashMap, Replicator>(); + boolean forceReplicatedImpl = false; + // used when configuring the number of segments. + private int minSegments = -1; + private int actualSegments = -1; + // used when reading the number of entries per + private int actualEntriesPerSegment = -1; + private int entrySize = 256; + private Alignment alignment = Alignment.OF_4_BYTES; + private long entries = 1 << 20; + private int replicas = 0; + private long lockTimeOut = 2000; + private TimeUnit lockTimeOutUnit = TimeUnit.MILLISECONDS; + private int metaDataBytes = 0; + private MapErrorListener errorListener = MapErrorListeners.logging(); + private boolean putReturnsNull = false; + private boolean removeReturnsNull = false; + private boolean largeSegments = false; + // replication + private TimeProvider timeProvider = TimeProvider.SYSTEM; + private BytesMarshallerFactory bytesMarshallerFactory; + private ObjectSerializer objectSerializer; + @NotNull + private BytesMarshaller keyMarshaller; + @NotNull + private BytesMarshaller valueMarshaller; + @NotNull + private ObjectFactory valueFactory; + private MapEventListener> eventListener = + MapEventListeners.nop(); - public ChronicleMapBuilder(Class kClass, Class vClass) { - super(kClass, vClass); + ChronicleMapBuilder(Class keyClass, Class valueClass) { + this.keyClass = keyClass; + this.valueClass = valueClass; + Class keyClassForMarshaller = + marshallerUseFactory(keyClass) && keyClass.isInterface() ? + DataValueClasses.directClassFor(keyClass) : keyClass; + keyMarshaller = chooseDefaultMarshaller(keyClassForMarshaller); + Class valueClassForMarshaller = + marshallerUseFactory(valueClass) && valueClass.isInterface() ? + DataValueClasses.directClassFor(valueClass) : valueClass; + valueMarshaller = chooseDefaultMarshaller(valueClassForMarshaller); + + valueFactory = marshallerUseFactory(valueClass) ? + new AllocateInstanceObjectFactory(valueClass.isInterface() ? + DataValueClasses.directClassFor(valueClass) : + valueClass) : + NullObjectFactory.INSTANCE; } - public ChronicleMapBuilder() { - super(); + public static ChronicleMapBuilder of(Class keyClass, Class valueClass) { + return new ChronicleMapBuilder(keyClass, valueClass); } + private static boolean marshallerUseFactory(Class c) { + return Byteable.class.isAssignableFrom(c) || + BytesMarshallable.class.isAssignableFrom(c) || + Externalizable.class.isAssignableFrom(c); + } - public static ChronicleMapBuilder of(Class kClass, Class vClass) { - return new ChronicleMapBuilder(kClass, vClass); + @SuppressWarnings("unchecked") + private static BytesMarshaller chooseDefaultMarshaller(@NotNull Class tClass) { + if (Byteable.class.isAssignableFrom(tClass)) + return new ByteableMarshaller(tClass); + if (BytesMarshallable.class.isAssignableFrom(tClass)) + return new BytesMarshallableMarshaller(tClass); + if (Externalizable.class.isAssignableFrom(tClass)) + return new ExternalizableMarshaller(tClass); + if (tClass == CharSequence.class) + return (BytesMarshaller) CharSequenceMarshaller.INSTANCE; + if (tClass == String.class) + return (BytesMarshaller) StringMarshaller.INSTANCE; + if (tClass == Integer.class) + return (BytesMarshaller) IntegerMarshaller.INSTANCE; + if (tClass == Long.class) + return (BytesMarshaller) LongMarshaller.INSTANCE; + if (tClass == Double.class) + return (BytesMarshaller) DoubleMarshaller.INSTANCE; + return SerializableMarshaller.INSTANCE; + } + + private static long roundUpMapHeaderSize(long headerSize) { + long roundUp = (headerSize + 127L) & ~127L; + if (roundUp - headerSize < 64) + roundUp += 128; + return roundUp; } @Override public ChronicleMapBuilder clone() { - return (ChronicleMapBuilder) super.clone(); + + try { + @SuppressWarnings("unchecked") + final ChronicleMapBuilder result = (ChronicleMapBuilder) super.clone(); + return result; + + } catch (CloneNotSupportedException e) { + throw new AssertionError(e); + } } - /** - * Set minimum number of segments. See concurrencyLevel in {@link java.util.concurrent.ConcurrentHashMap}. + * Set minimum number of segments. See concurrencyLevel + * in {@link java.util.concurrent.ConcurrentHashMap}. * + * @param minSegments the minimum number of segments in maps, constructed by this builder * @return this builder object back */ - public ChronicleMapBuilder minSegments(int minSegments) { - super.minSegments(minSegments); + public ChronicleMapBuilder minSegments(int minSegments) { + this.minSegments = minSegments; return this; } + public int minSegments() { + return minSegments < 1 ? tryMinSegments(4, 65536) : minSegments; + } + + private int tryMinSegments(int min, int max) { + for (int i = min; i < max; i <<= 1) { + if (i * i * i >= alignedEntrySize() * 2) + return i; + } + return max; + } /** *

Note that the actual entrySize will be aligned to 4 (default entry alignment). I. e. if you set @@ -69,172 +176,798 @@ public class ChronicleMapBuilder extends SharedHashMapBuilder implem * size to be aligned, set {@code entryAndValueAlignment(Alignment.NO_ALIGNMENT)}. * * @param entrySize the size in bytes - * @return this {@code SharedHashMapBuilder} back - * @see #entryAndValueAlignment(net.openhft.collections.Alignment) + * @return this {@code ChronicleMapBuilder} back + * @see #entryAndValueAlignment(Alignment) * @see #entryAndValueAlignment() */ - public ChronicleMapBuilder entrySize(int entrySize) { - super.entrySize(entrySize); + public ChronicleMapBuilder entrySize(int entrySize) { + this.entrySize = entrySize; return this; } + public int entrySize() { + return entrySize; + } + + int alignedEntrySize() { + return entryAndValueAlignment().alignSize(entrySize()); + } /** - * Specifies alignment of address in memory of entries and independently of address in memory of values - * within entries.

Useful when values of the map are updated intensively, particularly fields with - * volatile access, because it doesn't work well if the value crosses cache lines. Also, on some (nowadays - * rare) architectures any misaligned memory access is more expensive than aligned.

Note that - * specified {@link #entrySize()} will be aligned according to this alignment. I. e. if you set {@code - * entrySize(20)} and {@link net.openhft.collections.Alignment#OF_8_BYTES}, actual entry size will be 24 - * (20 aligned to 8 bytes). + * Specifies alignment of address in memory of entries and independently of address in memory + * of values within entries. + *

+ *

Useful when values of the map are updated intensively, particularly fields with + * volatile access, because it doesn't work well if the value crosses cache lines. Also, on some + * (nowadays rare) architectures any misaligned memory access is more expensive than aligned. + *

+ *

Note that specified {@link #entrySize()} will be aligned according to this alignment. + * I. e. if you set {@code entrySize(20)} and {@link Alignment#OF_8_BYTES}, actual entry size + * will be 24 (20 aligned to 8 bytes). * - * @return this {@code SharedHashMapBuilder} back + * @param alignment the new alignment of the maps constructed by this builder + * @return this {@code ChronicleMapBuilder} back * @see #entryAndValueAlignment() */ public ChronicleMapBuilder entryAndValueAlignment(Alignment alignment) { - super.entryAndValueAlignment(alignment); + this.alignment = alignment; return this; } + /** + * Returns alignment of addresses in memory of entries and independently of values within + * entries. + *

+ *

Default is {@link Alignment#OF_4_BYTES}. + * + * @see #entryAndValueAlignment(Alignment) + */ + public Alignment entryAndValueAlignment() { + return alignment; + } public ChronicleMapBuilder entries(long entries) { - super.entries(entries); + this.entries = entries; return this; } + public long entries() { + return entries; + } + public ChronicleMapBuilder replicas(int replicas) { - super.replicas(replicas); + this.replicas = replicas; return this; } + public int replicas() { + return replicas; + } public ChronicleMapBuilder actualEntriesPerSegment(int actualEntriesPerSegment) { - super.actualEntriesPerSegment(actualEntriesPerSegment); + this.actualEntriesPerSegment = actualEntriesPerSegment; return this; } + public int actualEntriesPerSegment() { + if (actualEntriesPerSegment > 0) + return actualEntriesPerSegment; + int as = actualSegments(); + // round up to the next multiple of 64. + return (int) (Math.max(1, entries * 2L / as) + 63) & ~63; + } public ChronicleMapBuilder actualSegments(int actualSegments) { - super.actualSegments(actualSegments); + this.actualSegments = actualSegments; return this; } + public int actualSegments() { + if (actualSegments > 0) + return actualSegments; + if (!largeSegments && entries > (long) minSegments() << 15) { + long segments = Maths.nextPower2(entries >> 15, 128); + if (segments < 1 << 20) + return (int) segments; + } + // try to keep it 16-bit sizes segments + return (int) Maths.nextPower2(Math.max((entries >> 30) + 1, minSegments()), 1); + } /** * Not supported yet. * * @param transactional if the built map should be transactional - * @return this {@code SharedHashMapBuilder} back + * @return this {@code ChronicleMapBuilder} back */ public ChronicleMapBuilder transactional(boolean transactional) { - super.transactional(transactional); + this.transactional = transactional; return this; } + public boolean transactional() { + return transactional; + } - public ChronicleMapBuilder kClass(Class kClass) { - super.kClass(kClass); + public ChronicleMapBuilder lockTimeOut(long lockTimeOut, TimeUnit unit) { + this.lockTimeOut = lockTimeOut; + lockTimeOutUnit = unit; return this; } + public long lockTimeOut(TimeUnit unit) { + return unit.convert(lockTimeOut, lockTimeOutUnit); + } - public ChronicleMapBuilder file(File file) { - super.file(file); + public ChronicleMapBuilder errorListener(MapErrorListener errorListener) { + this.errorListener = errorListener; return this; } - - public ChronicleMapBuilder lockTimeOutMS(long lockTimeOutMS) { - super.lockTimeOutMS(lockTimeOutMS); - return this; + public MapErrorListener errorListener() { + return errorListener; } - - public ChronicleMapBuilder errorListener(SharedMapErrorListener errorListener) { - super.errorListener(errorListener); - return this; - } - - /** * Map.put() returns the previous value, functionality which is rarely used but fairly cheap for HashMap. * In the case, for an off heap collection, it has to create a new object (or return a recycled one) * Either way it's expensive for something you probably don't use. * - * @param putReturnsNull false if you want SharedHashMap.put() to not return the object that was replaced + * @param putReturnsNull false if you want ChronicleMap.put() to not return the object that was replaced * but instead return null * @return an instance of the map builder */ - public ChronicleMapBuilder putReturnsNull(boolean putReturnsNull) { - super.putReturnsNull(putReturnsNull); + public ChronicleMapBuilder putReturnsNull(boolean putReturnsNull) { + this.putReturnsNull = putReturnsNull; return this; } + /** + * Map.put() returns the previous value, functionality which is rarely used but fairly cheap for HashMap. + * In the case, for an off heap collection, it has to create a new object (or return a recycled one) + * Either way it's expensive for something you probably don't use. + * + * @return true if ChronicleMap.put() is not going to return the object that was replaced but instead + * return null + */ + public boolean putReturnsNull() { + return putReturnsNull; + } /** * Map.remove() returns the previous value, functionality which is rarely used but fairly cheap for * HashMap. In the case, for an off heap collection, it has to create a new object (or return a recycled * one) Either way it's expensive for something you probably don't use. * - * @param removeReturnsNull false if you want SharedHashMap.remove() to not return the object that was + * @param removeReturnsNull false if you want ChronicleMap.remove() to not return the object that was * removed but instead return null * @return an instance of the map builder */ - public ChronicleMapBuilder removeReturnsNull(boolean removeReturnsNull) { - super.removeReturnsNull(removeReturnsNull); + public ChronicleMapBuilder removeReturnsNull(boolean removeReturnsNull) { + this.removeReturnsNull = removeReturnsNull; return this; } + /** + * Map.remove() returns the previous value, functionality which is rarely used but fairly cheap for + * HashMap. In the case, for an off heap collection, it has to create a new object (or return a recycled + * one) Either way it's expensive for something you probably don't use. + * + * @return true if ChronicleMap.remove() is not going to return the object that was removed but instead + * return null + */ + public boolean removeReturnsNull() { + return removeReturnsNull; + } - public ChronicleMapBuilder generatedValueType(boolean generatedValueType) { - super.generatedValueType(generatedValueType); + public boolean largeSegments() { + return entries > 1L << (20 + 15) || largeSegments; + } + + public ChronicleMapBuilder largeSegments(boolean largeSegments) { + this.largeSegments = largeSegments; return this; } - - public ChronicleMapBuilder largeSegments(boolean largeSegments) { - super.largeSegments(largeSegments); + public ChronicleMapBuilder metaDataBytes(int metaDataBytes) { + if ((metaDataBytes & 0xFF) != metaDataBytes) + throw new IllegalArgumentException("MetaDataBytes must be [0..255] was " + metaDataBytes); + this.metaDataBytes = metaDataBytes; return this; } + public int metaDataBytes() { + return metaDataBytes; + } - public ChronicleMapBuilder metaDataBytes(int metaDataBytes) { - super.metaDataBytes(metaDataBytes); + @Override + public String toString() { + return "ChronicleMapBuilder{" + + "actualSegments=" + actualSegments() + + ", minSegments=" + minSegments() + + ", actualEntriesPerSegment=" + actualEntriesPerSegment() + + ", entrySize=" + entrySize() + + ", entryAndValueAlignment=" + entryAndValueAlignment() + + ", entries=" + entries() + + ", replicas=" + replicas() + + ", transactional=" + transactional() + + ", lockTimeOut=" + lockTimeOut + " " + lockTimeOutUnit + + ", metaDataBytes=" + metaDataBytes() + + ", errorListener=" + errorListener() + + ", putReturnsNull=" + putReturnsNull() + + ", removeReturnsNull=" + removeReturnsNull() + + ", largeSegments=" + largeSegments() + + ", timeProvider=" + timeProvider() + + ", bytesMarshallerfactory=" + bytesMarshallerFactory() + + + ", keyClass=" + keyClass + + ", valueClass=" + valueClass + + ", keyMarshaller=" + keyMarshaller + + ", valueMarshaller=" + valueMarshaller + + ", valueFactory=" + valueFactory + + ", eventListener=" + eventListener + + '}'; + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(Object o) { + return builderEquals(this, o); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + + public ChronicleMapBuilder addReplicator(Replicator replicator) { + if (firstReplicator == null) { + firstReplicator = replicator; + replicators.put(replicator.getClass(), replicator); + } else { + if (replicator.identifier() != firstReplicator.identifier()) { + throw new IllegalArgumentException( + "Identifiers of all replicators of the map should be the same"); + } + if (replicators.containsKey(replicator.getClass())) { + throw new IllegalArgumentException("Replicator of " + replicator.getClass() + + " class has already to the map"); + } + replicators.put(replicator.getClass(), replicator); + } return this; } - - public ChronicleMapBuilder canReplicate(boolean canReplicate) { - super.canReplicate(canReplicate); + public ChronicleMapBuilder setReplicators(Replicator... replicators) { + firstReplicator = null; + this.replicators.clear(); + for (Replicator replicator : replicators) { + addReplicator(replicator); + } return this; } - public ChronicleMapBuilder identifier(byte identifier) { - super.identifier(identifier); + public ChronicleMapBuilder timeProvider(TimeProvider timeProvider) { + this.timeProvider = timeProvider; return this; } - public ChronicleMapBuilder tcpReplicatorBuilder(TcpReplicatorBuilder tcpReplicatorBuilder) { - super.tcpReplicatorBuilder(tcpReplicatorBuilder); + public TimeProvider timeProvider() { + return timeProvider; + } + + public BytesMarshallerFactory bytesMarshallerFactory() { + return bytesMarshallerFactory == null ? + bytesMarshallerFactory = new VanillaBytesMarshallerFactory() : + bytesMarshallerFactory; + } + + public ChronicleMapBuilder bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) { + this.bytesMarshallerFactory = bytesMarshallerFactory; return this; } + public ObjectSerializer objectSerializer() { + return objectSerializer == null ? + objectSerializer = BytesMarshallableSerializer.create( + bytesMarshallerFactory(), JDKObjectSerializer.INSTANCE) : + objectSerializer; + } - public ChronicleMapBuilder udpReplicatorBuilder(UdpReplicatorBuilder udpReplicatorBuilder) { - super.udpReplicatorBuilder(udpReplicatorBuilder); + public ChronicleMapBuilder objectSerializer(ObjectSerializer objectSerializer) { + this.objectSerializer = objectSerializer; return this; } - - public ChronicleMapBuilder bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) { - super.bytesMarshallerFactory(bytesMarshallerFactory); + /** + * For testing + */ + ChronicleMapBuilder forceReplicatedImpl() { + this.forceReplicatedImpl = true; return this; } + @NotNull + public BytesMarshaller keyMarshaller() { + return keyMarshaller; + } - public ChronicleMapBuilder objectSerializer(ObjectSerializer objectSerializer) { - super.objectSerializer(objectSerializer); + public ChronicleMapBuilder keyMarshaller( + @NotNull BytesMarshaller keyMarshaller) { + this.keyMarshaller = keyMarshaller; return this; } + @NotNull + public BytesMarshaller valueMarshaller() { + return valueMarshaller; + } + + public ChronicleMapBuilder valueMarshallerAndFactory( + @NotNull BytesMarshaller valueMarshaller, @NotNull ObjectFactory valueFactory) { + this.valueMarshaller = valueMarshaller; + this.valueFactory = valueFactory; + return this; + } + + @NotNull + public ObjectFactory valueFactory() { + return valueFactory; + } + + @SuppressWarnings("unchecked") + public ChronicleMapBuilder valueFactory( + @NotNull ObjectFactory valueFactory) { + if (!marshallerUseFactory(valueClass)) { + throw new IllegalStateException("Default marshaller for " + valueClass + + " value don't use object factory"); + } else if (valueMarshaller instanceof ByteableMarshaller) { + if (valueFactory instanceof AllocateInstanceObjectFactory) { + valueMarshaller = new ByteableMarshaller( + ((AllocateInstanceObjectFactory) valueFactory).allocatedClass()); + } else { + valueMarshaller = new ByteableMarshallerWithCustomFactory( + ((ByteableMarshaller) valueMarshaller).tClass, valueFactory); + } + } else if (valueMarshaller instanceof BytesMarshallableMarshaller) { + if (valueFactory instanceof AllocateInstanceObjectFactory) { + valueMarshaller = new BytesMarshallableMarshaller( + ((AllocateInstanceObjectFactory) valueFactory).allocatedClass()); + } else { + valueMarshaller = new BytesMarshallableMarshallerWithCustomFactory( + ((BytesMarshallableMarshaller) valueMarshaller).marshaledClass(), + valueFactory + ); + } + } else if (valueMarshaller instanceof ExternalizableMarshaller) { + if (valueFactory instanceof AllocateInstanceObjectFactory) { + valueMarshaller = new ExternalizableMarshaller( + ((AllocateInstanceObjectFactory) valueFactory).allocatedClass()); + } else { + valueMarshaller = new ExternalizableMarshallerWithCustomFactory( + ((ExternalizableMarshaller) valueMarshaller).marshaledClass(), + valueFactory + ); + } + } else { + // valueMarshaller is custom, it is user's responsibility to use the same factory inside + // marshaller and standalone + throw new IllegalStateException( + "Change the value factory simultaneously with marshaller " + + "using valueMarshallerAndFactory() method"); + } + this.valueFactory = valueFactory; + return this; + } + + public ChronicleMapBuilder eventListener( + MapEventListener> eventListener) { + this.eventListener = eventListener; + return this; + } + + public MapEventListener> eventListener() { + return eventListener; + } + + public ChronicleMap create(File file) throws IOException { + for (int i = 0; i < 10; i++) { + if (file.exists() && file.length() > 0) { + FileInputStream fis = new FileInputStream(file); + ObjectInputStream ois = new ObjectInputStream(fis); + try { + VanillaChronicleMap map = (VanillaChronicleMap) ois.readObject(); + map.headerSize = roundUpMapHeaderSize(fis.getChannel().position()); + map.createMappedStoreAndSegments(file); + return establishReplication(map); + } catch (ClassNotFoundException e) { + throw new IOException(e); + } finally { + ois.close(); + } + } + if (file.createNewFile() || file.length() == 0) { + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException(e); + } + } + // new file + if (!file.exists()) + throw new FileNotFoundException("Unable to create " + file); + + VanillaChronicleMap map = newMap(); + + FileOutputStream fos = new FileOutputStream(file); + ObjectOutputStream oos = new ObjectOutputStream(fos); + try { + oos.writeObject(map); + oos.flush(); + map.headerSize = roundUpMapHeaderSize(fos.getChannel().position()); + map.createMappedStoreAndSegments(file); + } finally { + oos.close(); + } + + return establishReplication(map); + } + + public ChronicleMap create() throws IOException { + VanillaChronicleMap map = newMap(); + BytesStore bytesStore = new DirectStore(objectSerializer(), map.sizeInBytes(), true); + map.createMappedStoreAndSegments(bytesStore); + return establishReplication(map); + } + + private VanillaChronicleMap newMap() throws IOException { + if (firstReplicator == null && !forceReplicatedImpl) { + return new VanillaChronicleMap(this); + } else { + return new ReplicatedChronicleMap(this); + } + } + + private ChronicleMap establishReplication(ChronicleMap map) + throws IOException { + if (map instanceof ReplicatedChronicleMap) { + ReplicatedChronicleMap result = (ReplicatedChronicleMap) map; + for (Replicator replicator : replicators.values()) { + Closeable token = replicator.applyTo(this, result, result); + if (replicators.size() == 1 && token.getClass() == UdpReplicator.class) { + LOG.warn( + "MISSING TCP REPLICATION : The UdpReplicator only attempts to read data " + + "(it does not enforce or guarantee delivery), you should use" + + "the UdpReplicator if you have a large number of nodes, and you wish" + + "to receive the data before it becomes available on TCP/IP. Since data" + + "delivery is not guaranteed, it is recommended that you only use" + + "the UDP Replicator in conjunction with a TCP Replicator" + ); + } + result.addCloseable(token); + } + } + return map; + } + + private static enum CharSequenceMarshaller implements BytesMarshaller { + INSTANCE; + + @Override + public void write(Bytes bytes, CharSequence s) { + bytes.writeUTFΔ(s); + } + + @Nullable + @Override + public CharSequence read(Bytes bytes) { + return bytes.readUTFΔ(); + } + + @Nullable + @Override + public CharSequence read(Bytes bytes, @Nullable CharSequence s) { + if (s instanceof StringBuilder) { + if (bytes.readUTFΔ((StringBuilder) s)) + return s; + return null; + } + return bytes.readUTFΔ(); + } + } + + private static enum StringMarshaller implements BytesMarshaller { + INSTANCE; + + @Override + public void write(Bytes bytes, String s) { + bytes.writeUTFΔ(s); + } + + @Nullable + @Override + public String read(Bytes bytes) { + return bytes.readUTFΔ(); + } + + @Nullable + @Override + public String read(Bytes bytes, @Nullable String s) { + return bytes.readUTFΔ(); + } + } + + private static enum IntegerMarshaller implements BytesMarshaller { + INSTANCE; + + @Override + public void write(Bytes bytes, Integer v) { + bytes.writeInt(v); + } + + @Nullable + @Override + public Integer read(Bytes bytes) { + return bytes.readInt(); + } + + @Nullable + @Override + public Integer read(Bytes bytes, @Nullable Integer v) { + return bytes.readInt(); + } + } + + private static enum LongMarshaller implements BytesMarshaller { + INSTANCE; + + @Override + public void write(Bytes bytes, Long v) { + bytes.writeLong(v); + } + + @Nullable + @Override + public Long read(Bytes bytes) { + return bytes.readLong(); + } + + @Nullable + @Override + public Long read(Bytes bytes, @Nullable Long v) { + return bytes.readLong(); + } + } + + private static enum DoubleMarshaller implements BytesMarshaller { + INSTANCE; + + @Override + public void write(Bytes bytes, Double v) { + bytes.writeDouble(v); + } + + @Nullable + @Override + public Double read(Bytes bytes) { + return bytes.readDouble(); + } + + @Nullable + @Override + public Double read(Bytes bytes, @Nullable Double v) { + return bytes.readDouble(); + } + } + + + private static enum SerializableMarshaller implements BytesMarshaller { + INSTANCE; + + @Override + public void write(Bytes bytes, Object obj) { + try { + new ObjectOutputStream(bytes.outputStream()).writeObject(obj); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Nullable + @Override + public Object read(Bytes bytes) { + try { + return new ObjectInputStream(bytes.inputStream()).readObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } catch (ClassNotFoundException e) { + throw new IllegalStateException(e); + } + } + + @Nullable + @Override + public Object read(Bytes bytes, @Nullable Object obj) { + return read(bytes); + } + } + + private static class ByteableMarshaller implements BytesMarshaller { + private static final long serialVersionUID = 0L; + + @NotNull + final Class tClass; + + ByteableMarshaller(@NotNull Class tClass) { + this.tClass = tClass; + } + + @Override + public void write(Bytes bytes, T t) { + bytes.write(t.bytes(), t.offset(), t.maxSize()); + } + + @Nullable + @Override + public T read(Bytes bytes) { + return read(bytes, null); + } + + @Nullable + @Override + public T read(Bytes bytes, @Nullable T t) { + try { + if (t == null) + t = getInstance(); + t.bytes(bytes, bytes.position()); + bytes.skip(t.maxSize()); + return t; + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + @SuppressWarnings("unchecked") + @NotNull + T getInstance() throws Exception { + return (T) NativeBytes.UNSAFE.allocateInstance(tClass); + } + + @Override + public boolean equals(Object obj) { + return obj != null && obj.getClass() == getClass() && + ((ByteableMarshaller) obj).tClass == tClass; + } + + @Override + public int hashCode() { + return tClass.hashCode(); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{tClass=" + tClass + "}"; + } + } + + private static class ByteableMarshallerWithCustomFactory + extends ByteableMarshaller { + private static final long serialVersionUID = 0L; + + @NotNull + private final ObjectFactory factory; + + ByteableMarshallerWithCustomFactory(@NotNull Class tClass, + @NotNull ObjectFactory factory) { + super(tClass); + this.factory = factory; + } + + @NotNull + @Override + T getInstance() throws Exception { + return factory.create(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) + return false; + ByteableMarshallerWithCustomFactory that = (ByteableMarshallerWithCustomFactory) obj; + return that.tClass == tClass && that.factory.equals(this.factory); + } + + @Override + public int hashCode() { + return hash(tClass, factory); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{tClass=" + tClass + ",factory=" + factory + "}"; + } + } + + private static class BytesMarshallableMarshallerWithCustomFactory + extends BytesMarshallableMarshaller { + private static final long serialVersionUID = 0L; + + @NotNull + private final ObjectFactory factory; + + BytesMarshallableMarshallerWithCustomFactory(@NotNull Class tClass, + @NotNull ObjectFactory factory) { + super(tClass); + this.factory = factory; + } + + @NotNull + @Override + protected T getInstance() throws Exception { + return factory.create(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) + return false; + BytesMarshallableMarshallerWithCustomFactory that = + (BytesMarshallableMarshallerWithCustomFactory) obj; + return that.marshaledClass() == marshaledClass() && that.factory.equals(this.factory); + } + + @Override + public int hashCode() { + return hash(marshaledClass(), factory); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{marshaledClass=" + marshaledClass() + + ",factory=" + factory + "}"; + } + } + + private static class ExternalizableMarshallerWithCustomFactory + extends ExternalizableMarshaller { + private static final long serialVersionUID = 0L; + + @NotNull + private final ObjectFactory factory; + + ExternalizableMarshallerWithCustomFactory(@NotNull Class tClass, + @NotNull ObjectFactory factory) { + super(tClass); + this.factory = factory; + } + + @NotNull + @Override + protected T getInstance() throws Exception { + return factory.create(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) + return false; + ExternalizableMarshallerWithCustomFactory that = + (ExternalizableMarshallerWithCustomFactory) obj; + return that.marshaledClass() == marshaledClass() && that.factory.equals(this.factory); + } + + @Override + public int hashCode() { + return hash(marshaledClass(), factory); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{marshaledClass=" + marshaledClass() + + ",factory=" + factory + "}"; + } + } } + diff --git a/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java b/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java new file mode 100644 index 000000000..daf585ea1 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java @@ -0,0 +1,81 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.collection.DirectBitSet; + +/** + * This is only used to store keys and positions, but it could store int/int key/values for another purpose. + */ +interface IntIntMultiMap { + + /** + * @return gets all the active positions as a bitset + */ + DirectBitSet getPositions(); + + /** + * Add an entry. Allow duplicate hashes, but not key/position pairs. + * + * @param key to add + * @param value to add + */ + void put(int key, int value); + + /** + * Remove a key/value pair. + * + * @param key to remove + * @param value to remove + * @return whether a match was found. + */ + boolean remove(int key, int value); + + boolean replace(int key, int oldValue, int newValue); + + /** + * Used for start a search for a given key + * + * @return normalized key value, better to be used in subsequent calls + */ + int startSearch(int key); + + int getSearchHash(); + + /** + * Used for getting the next position for a given key + * + * @return the next position for the last search or negative value + */ + int nextPos(); + + void removePrevPos(); + + void replacePrevPos(int newValue); + + void putAfterFailedSearch(int value); + + void clear(); + + void forEach(EntryConsumer action); + + static interface EntryConsumer { + void accept(int key, int value); + } + + +} diff --git a/src/main/java/net/openhft/chronicle/map/MapErrorListener.java b/src/main/java/net/openhft/chronicle/map/MapErrorListener.java new file mode 100644 index 000000000..2517bd5ba --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/MapErrorListener.java @@ -0,0 +1,25 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import java.io.Serializable; + +public interface MapErrorListener extends Serializable { + void onLockTimeout(long threadId) throws IllegalStateException; + + void errorOnUnlock(IllegalMonitorStateException e); +} diff --git a/src/main/java/net/openhft/chronicle/map/MapErrorListeners.java b/src/main/java/net/openhft/chronicle/map/MapErrorListeners.java new file mode 100644 index 000000000..67749d28d --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/MapErrorListeners.java @@ -0,0 +1,71 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class MapErrorListeners { + + public static MapErrorListener logging() { + return LoggingErrorListener.INSTANCE; + } + + public static MapErrorListener error() { + return ThrowingErrorListener.INSTANCE; + } + + /** + * Factories a more flexible than public static instances. + * We can add some configuration-on-the-first-call in the future. + */ + + private static enum LoggingErrorListener implements MapErrorListener { + INSTANCE; + final Logger LOG = LoggerFactory.getLogger(getClass()); + + @Override + public void onLockTimeout(long threadId) throws IllegalStateException { + if (threadId > 1L << 32) { + LOG.warn("Grabbing lock held by processId: {}, threadId: {}", + (threadId >>> 33), (threadId & 0xFFFFFFL)); + } else { + LOG.warn("Grabbing lock held by threadId: {}", threadId); + } + } + + @Override + public void errorOnUnlock(IllegalMonitorStateException e) { + LOG.warn("Failed to unlock as expected", e); + } + } + + private static enum ThrowingErrorListener implements MapErrorListener { + INSTANCE; + + @Override + public void onLockTimeout(long threadId) throws IllegalStateException { + throw new IllegalStateException("Unable to acquire lock held by threadId: " + threadId); + } + + @Override + public void errorOnUnlock(IllegalMonitorStateException e) { + throw e; + } + } +} diff --git a/src/main/java/net/openhft/chronicle/map/MapEventListener.java b/src/main/java/net/openhft/chronicle/map/MapEventListener.java new file mode 100644 index 000000000..4822551e6 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/MapEventListener.java @@ -0,0 +1,102 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; + +import java.io.Serializable; + +/** + * This event listener is called when key events occur. + *

All these calls are synchronous while a lock is held so make them as quick as possible

+ *

+ * TODO specify more clearly in which cases methods are called. + */ +public abstract class MapEventListener> + implements Serializable { + private static final long serialVersionUID = 0L; + + /** + * This is called when there was no existing entry for a key. + * Optionally you can provide a value to add to the map. + * + * @param map accessed + * @param keyBytes bytes of the key looked up + * @param key object used as key + * @param usingValue value provided to reuse, could be null. + * @return null if null should be returned, or a value to put in the map and return. + */ + public V onGetMissing(M map, Bytes keyBytes, K key, V usingValue) { + return null; + } + + /** + * This method is called if a value is found in the map. + * + * @param map accessed + * @param entry bytes of the entry + * @param metaDataBytes length of meta data for this map. + * @param key looked up + * @param value found + */ + public void onGetFound(M map, Bytes entry, int metaDataBytes, K key, V value) { + // do nothing + } + + void onPut(M map, Bytes entry, int metaDataBytes, boolean added, K key, V value, + long pos, SharedSegment segment) { + onPut(map, entry, metaDataBytes, added, key, value); + } + + /** + * This method is called if a key/value is put in the map. + * + * @param map accessed + * @param entry added/modified + * @param metaDataBytes length of the meta data + * @param added if this is a new entry + * @param key looked up + * @param value set for key + */ + public void onPut(M map, Bytes entry, int metaDataBytes, boolean added, K key, V value) { + // do nothing + } + + void onRemove(M map, Bytes entry, int metaDataBytes, K key, V value, + int pos, SharedSegment segment) { + onRemove(map, entry, metaDataBytes, key, value); + } + + /** + * This is called when an entry is removed. Misses are not notified. + * + * @param map accessed + * @param entry removed + * @param metaDataBytes length of meta data + * @param key removed + * @param value removed + */ + public void onRemove(M map, Bytes entry, int metaDataBytes, K key, V value) { + // do nothing + } + + void onRelocation(int pos, SharedSegment segment) { + // do nothing + } + + +} diff --git a/src/main/java/net/openhft/chronicle/map/MapEventListeners.java b/src/main/java/net/openhft/chronicle/map/MapEventListeners.java new file mode 100644 index 000000000..51df6b807 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/MapEventListeners.java @@ -0,0 +1,134 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import net.openhft.lang.io.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class MapEventListeners { + + /** + * Factories a more flexible than public static instances. + * We can add some configuration-on-the-first-call in the future. + */ + + static final MapEventListener NOP = new MapEventListener() { + private static final long serialVersionUID = 0L; + }; + private static final MapEventListener BYTES_LOGGING = new MapEventListener() { + private static final long serialVersionUID = 0L; + public final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + @Override + public Object onGetMissing(ChronicleMap map, Bytes keyBytes, + Object key, Object usingValue) { + StringBuilder sb = new StringBuilder(); + sb.append(map.file()).append(" missed "); + + keyBytes.toString(sb, 0, 0, keyBytes.limit()); + + LOGGER.info(sb.toString()); + return null; + } + + @Override + public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, + Object key, Object value) { + logOperation(map, entry, metaDataBytes, " get "); + } + + private void logOperation(ChronicleMap map, Bytes entry, int metaDataBytes, String oper) { + StringBuilder sb = new StringBuilder(); + sb.append(map.file()).append(oper); + if (metaDataBytes > 0) { + sb.append("Meta: "); + entry.toString(sb, 0L, 0L, metaDataBytes); + sb.append(" | "); + } + Bytes slice = entry.slice(metaDataBytes, entry.limit() - metaDataBytes); + long keyLength = slice.readStopBit(); + slice.toString(sb, slice.position(), 0L, slice.position() + keyLength); + slice.position(slice.position() + keyLength); + long valueLength = slice.readStopBit(); + slice.alignPositionAddr(4); + sb.append(" = "); + slice.toString(sb, slice.position(), 0L, slice.position() + valueLength); + LOGGER.info(sb.toString()); + } + + @Override + public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, boolean added, + Object key, Object value, long pos, SharedSegment segment) { + logOperation(map, entry, metaDataBytes, added ? " +put " : " put "); + } + + @Override + public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, + Object key, Object value, int pos, SharedSegment segment) { + logOperation(map, entry, metaDataBytes, " remove "); + } + }; + private static final MapEventListener KEY_VALUE_LOGGING = new MapEventListener() { + private static final long serialVersionUID = 0L; + public final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + @Override + public Object onGetMissing(ChronicleMap map, Bytes keyBytes, + Object key, Object usingValue) { + LOGGER.info("{} missed {}", map.file(), key); + return null; + } + + @Override + public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, + Object key, Object value) { + LOGGER.info("{} get {} => {}", map.file(), key, value); + } + + @Override + public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, boolean added, + Object key, Object value, long pos, SharedSegment segment) { + LOGGER.info("{} put {} => {}", map.file(), key, value); + } + + @Override + public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, + Object key, Object value, int pos, SharedSegment segment) { + LOGGER.info("{} remove {} was {}", map.file(), key, value); + } + }; + + private MapEventListeners() { + } + + @SuppressWarnings("unchecked") + public static > MapEventListener nop() { + return NOP; + } + + @SuppressWarnings("unchecked") + public static > MapEventListener bytesLogging() { + return BYTES_LOGGING; + } + + @SuppressWarnings("unchecked") + public static > MapEventListener keyValueLogging() { + return KEY_VALUE_LOGGING; + } +} diff --git a/src/main/java/net/openhft/chronicle/map/Objects.java b/src/main/java/net/openhft/chronicle/map/Objects.java new file mode 100644 index 000000000..44b6ee82b --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Objects.java @@ -0,0 +1,41 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Arrays; + +/** + * java.util.Objects since Java 7 + */ +final class Objects { + static int hash(Object... values) { + return Arrays.hashCode(values); + } + + static boolean equal(@Nullable Object a, @Nullable Object b) { + return a != null ? a.equals(b) : b == null; + } + + static boolean builderEquals(@NotNull Object builder, @Nullable Object o) { + return builder == o || + o != null && builder.getClass() == o.getClass() && + builder.toString().equals(o.toString()); + } +} diff --git a/src/main/java/net/openhft/chronicle/map/Replica.java b/src/main/java/net/openhft/chronicle/map/Replica.java new file mode 100644 index 000000000..821d4ab28 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Replica.java @@ -0,0 +1,213 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; +import org.jetbrains.annotations.NotNull; + +import java.io.Closeable; + +/** + * @author Rob Austin. + */ +public interface Replica extends Closeable { + + /** + * Provides the unique Identifier associated with this map instance.

An identifier is used to + * determine which replicating node made the change.

If two nodes update their map at the same time + * with different values, we have to deterministically resolve which update wins, because of eventual + * consistency both nodes should end up locally holding the same data. Although it is rare two remote + * nodes could receive an update to their maps at exactly the same time for the same key, we have to + * handle this edge case, its therefore important not to rely on timestamps alone to reconcile the + * updates. Typically the update with the newest timestamp should win, but in this example both + * timestamps are the same, and the decision made to one node should be identical to the decision made to + * the other. We resolve this simple dilemma by using a node identifier, each node will have a unique + * identifier, the update from the node with the smallest identifier wins. + * + * @return the unique Identifier associated with this map instance + */ + byte identifier(); + + /** + * Gets (if it does not exist, creates) an instance of ModificationIterator associated with a remote node, + * this weak associated is bound using the {@code identifier}. + * + * @param remoteIdentifier the identifier of the remote node + * @param modificationNotifier called when ever there is a change applied to the modification iterator + * @return the ModificationIterator dedicated for replication to the remote node with the given identifier + * @see #identifier() + */ + ModificationIterator acquireModificationIterator(short remoteIdentifier, + ModificationNotifier modificationNotifier); + + /** + * Returns the timestamp of the last change from the specified remote node, already replicated + * to this Replica. + *

+ *

Used in conjunction with replication, to back fill data from a remote node. This node + * may have missed updates while it was not been running or connected via TCP. + * + * @param remoteIdentifier the identifier of the remote node to check last replicated update + * time from + * @return a timestamp of the last modification to an entry, or 0 if there are no entries. + * @see #identifier() + */ + long lastModificationTime(byte remoteIdentifier); + + + /** + * notifies when there is a changed to the modification iterator + */ + interface ModificationNotifier { + public static final ModificationNotifier NOP = new ModificationNotifier() { + @Override + public void onChange() { + } + }; + + /** + * called when ever there is a change applied to the modification iterator + */ + void onChange(); + } + + + /** + * Holds a record of which entries have modification. Each remote map supported will require a + * corresponding ModificationIterator instance + */ + interface ModificationIterator { + + /** + * @return {@code true} if the is another entry to be received via {@link + * #nextEntry(Replica.EntryCallback, int chronicleId)} + */ + boolean hasNext(); + + /** + * A non-blocking call that provides the entry that has changed to {@code callback.onEntry()}. + * + * @param callback a callback which will be called when a new entry becomes available. + * @param chronicleId only assigned when clustering + * @return {@code true} if the entry was accepted by the {@code callback.onEntry()} method, {@code + * false} if the entry was not accepted or was not available + */ + boolean nextEntry(@NotNull final EntryCallback callback, final int chronicleId); + + /** + * Dirties all entries with a modification time equal to {@code fromTimeStamp} or newer. It means all + * these entries will be considered as "new" by this ModificationIterator and iterated once again no + * matter if they have already been. + *

+ *

This functionality is used to publish recently modified entries to a new remote node as it + * connects. + * + * @param fromTimeStamp the timestamp from which all entries should be dirty + */ + void dirtyEntries(long fromTimeStamp); + } + + + /** + * supports reading and writing serialize entries + */ + interface EntryExternalizable { + + /** + * The map implements this method to save its contents. + * + * @param entry the byte location of the entry to be stored + * @param destination a buffer the entry will be written to, the segment may reject this operation and + * add zeroBytes, if the identifier in the entry did not match the maps local + * @param chronicleId used in cluster into identify the canonical map or queue + */ + void writeExternalEntry(@NotNull Bytes entry, @NotNull Bytes destination, int chronicleId); + + /** + * The map implements this method to restore its contents. This method must read the values in the + * same sequence and with the same types as were written by {@code writeExternalEntry()}. This method + * is typically called when we receive a remote replication event, this event could originate from + * either a remote {@code put(K key, V value)} or {@code remove(Object key)} + * + * @param source bytes to read an entry from + */ + void readExternalEntry(@NotNull Bytes source); + + } + + + /** + * provides a key and value from NativeBytes, this can be used in conjunction with the modification + * iterator to get the key and value out of the NativeBytes + */ + interface EntryResolver { + + /** + * gets the key from the entry + * + * @param entry the bytes which the bytes which point to the entry + * @param usingKey the key object to reuse, if possible + * @return the key which is in the entry + */ + K key(@NotNull Bytes entry, K usingKey); + + + /** + * gets the value from the entry + * + * @param entry the bytes which reference to the entry + * @param usingValue the value object to reuse, if possible + * @return the value which is in the entry or null if the value has been remove from the map + */ + V value(@NotNull Bytes entry, V usingValue); + + boolean wasRemoved(@NotNull Bytes entry); + + } + + /** + * Implemented typically by a replicator, This interface provides the event, which will get + * called whenever a put() or remove() has occurred to the map + */ + abstract class EntryCallback { + + /** + * Called whenever a put() or remove() has occurred to a replicating map. + * + * @param entry the entry you will receive, this does not have to be locked, + * as locking is already provided from the caller. + * @param chronicleId only assigned when clustering + * @return {@code false} if this entry should be ignored because the identifier + * of the source node is not from one of our changes, WARNING even though we check + * the identifier in the ModificationIterator the entry may have been updated. + */ + public abstract boolean onEntry(final Bytes entry, final int chronicleId); + + /** + * Called just after {@link #onEntry(Bytes, int)}. No-op by default. + */ + public void onAfterEntry() { + } + + /** + * Called just before {@link #onEntry(Bytes, int)}. No-op by default. + */ + public void onBeforeEntry() { + } + } +} + diff --git a/src/main/java/net/openhft/collections/VanillaSharedReplicatedHashMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java similarity index 73% rename from src/main/java/net/openhft/collections/VanillaSharedReplicatedHashMap.java rename to src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 983d29119..294c054e9 100644 --- a/src/main/java/net/openhft/collections/VanillaSharedReplicatedHashMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,110 +14,91 @@ * limitations under the License. */ -package net.openhft.collections; +package net.openhft.chronicle.map; + -import net.openhft.chronicle.map.ChronicleMap; import net.openhft.lang.Maths; import net.openhft.lang.collection.ATSDirectBitSet; -import net.openhft.lang.io.*; -import net.openhft.lang.io.serialization.ObjectSerializer; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.DirectBytes; +import net.openhft.lang.io.MultiStoreBytes; +import net.openhft.lang.io.NativeBytes; import net.openhft.lang.model.Byteable; -import net.openhft.lang.model.DataValueClasses; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.nio.channels.FileChannel; +import java.io.*; +import java.util.Iterator; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.atomic.AtomicReferenceArray; -import static net.openhft.collections.Replica.EntryResolver; +import static net.openhft.chronicle.map.VanillaChronicleMap.Hasher.hash; import static net.openhft.lang.collection.DirectBitSet.NOT_FOUND; /** * A Replicating Multi Master HashMap - * - *

Each remote hash map, mirrors its changes over to another remote hash map, neither hash map is - * considered the master store of data, each hash map uses timestamps to reconcile changes. We refer to an - * instance of a remote hash-map as a node. A node will be connected to any number of other nodes, for the - * first implementation the maximum number of nodes will be fixed. The data that is stored locally in each - * node will become eventually consistent. So changes made to one node, for example by calling put() will be - * replicated over to the other node. To achieve a high level of performance and throughput, the call to put() - * won’t block, with concurrentHashMap, It is typical to check the return code of some methods to obtain the - * old value for example remove(). Due to the loose coupling and lock free nature of this multi master - * implementation, this return value will only be the old value on the nodes local data store. In other words - * the nodes are only concurrent locally. Its worth realising that another node performing exactly the same - * operation may return a different value. However reconciliation will ensure the maps themselves become + *

+ *

Each remote hash map, mirrors its changes over to another remote hash map, neither hash map is considered the + * master store of data, each hash map uses timestamps to reconcile changes. We refer to an instance of a remote + * hash-map as a node. A node will be connected to any number of other nodes, for the first implementation the maximum + * number of nodes will be fixed. The data that is stored locally in each node will become eventually consistent. So + * changes made to one node, for example by calling put() will be replicated over to the other node. To achieve a high + * level of performance and throughput, the call to put() won’t block, with concurrentHashMap, It is typical to check + * the return code of some methods to obtain the old value for example remove(). Due to the loose coupling and lock free + * nature of this multi master implementation, this return value will only be the old value on the nodes local data + * store. In other words the nodes are only concurrent locally. Its worth realising that another node performing exactly + * the same operation may return a different value. However reconciliation will ensure the maps themselves become * eventually consistent. - * + *

*

Reconciliation - * - *

If two ( or more nodes ) were to receive a change to their maps for the same key but different values, - * say by a user of the maps, calling the put(key, value). Then, initially each node will update its local - * store and each local store will hold a different value, but the aim of multi master replication is to - * provide eventual consistency across the nodes. So, with multi master when ever a node is changed it will - * notify the other nodes of its change. We will refer to this notification as an event. The event will hold a - * timestamp indicating the time the change occurred, it will also hold the state transition, in this case it - * was a put with a key and value. Eventual consistency is achieved by looking at the timestamp from the - * remote node, if for a given key, the remote nodes timestamp is newer than the local nodes timestamp, then - * the event from the remote node will be applied to the local node, otherwise the event will be ignored. - * - *

However there is an edge case that we have to concern ourselves with, If two nodes update their map at - * the same time with different values, we have to deterministically resolve which update wins, because of - * eventual consistency both nodes should end up locally holding the same data. Although it is rare two remote - * nodes could receive an update to their maps at exactly the same time for the same key, we have to handle - * this edge case, its therefore important not to rely on timestamps alone to reconcile the updates. Typically - * the update with the newest timestamp should win, but in this example both timestamps are the same, and the - * decision made to one node should be identical to the decision made to the other. We resolve this simple - * dilemma by using a node identifier, each node will have a unique identifier, the update from the node with - * the smallest identifier wins. + *

+ *

If two ( or more nodes ) were to receive a change to their maps for the same key but different values, say by a + * user of the maps, calling the put(key, value). Then, initially each node will update its local store and each local + * store will hold a different value, but the aim of multi master replication is to provide eventual consistency across + * the nodes. So, with multi master when ever a node is changed it will notify the other nodes of its change. We will + * refer to this notification as an event. The event will hold a timestamp indicating the time the change occurred, it + * will also hold the state transition, in this case it was a put with a key and value. Eventual consistency is achieved + * by looking at the timestamp from the remote node, if for a given key, the remote nodes timestamp is newer than the + * local nodes timestamp, then the event from the remote node will be applied to the local node, otherwise the event + * will be ignored. + *

+ *

However there is an edge case that we have to concern ourselves with, If two nodes update their map at the same + * time with different values, we have to deterministically resolve which update wins, because of eventual consistency + * both nodes should end up locally holding the same data. Although it is rare two remote nodes could receive an update + * to their maps at exactly the same time for the same key, we have to handle this edge case, its therefore important + * not to rely on timestamps alone to reconcile the updates. Typically the update with the newest timestamp should win, + * but in this example both timestamps are the same, and the decision made to one node should be identical to the + * decision made to the other. We resolve this simple dilemma by using a node identifier, each node will have a unique + * identifier, the update from the node with the smallest identifier wins. * * @param the entries key type * @param the entries value type */ -class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap - implements ChronicleMap, ReplicaExternalizable, EntryResolver, - Closeable { - - static final int MAX_UNSIGNED_SHORT = Character.MAX_VALUE; - - private static final Logger LOG = LoggerFactory.getLogger(VanillaSharedReplicatedHashMap.class); - private static final int LAST_UPDATED_HEADER_SIZE = (127 * 8); - +class ReplicatedChronicleMap extends VanillaChronicleMap + implements ChronicleMap, + Replica, Replica.EntryExternalizable, Replica.EntryResolver, Closeable { // for file, jdbc and UDP replication public static final int RESERVED_MOD_ITER = 8; - + static final int MAX_UNSIGNED_SHORT = Character.MAX_VALUE; + private static final long serialVersionUID = 0L; + private static final Logger LOG = LoggerFactory.getLogger(ReplicatedChronicleMap.class); + private static final long LAST_UPDATED_HEADER_SIZE = 128L * 8L; private final TimeProvider timeProvider; private final byte localIdentifier; - private final Set closeables = new CopyOnWriteArraySet(); + transient Set closeables; + private transient Bytes identifierUpdatedBytes; - private Bytes identifierUpdatedBytes; - private Bytes modDelBytes; - - private final ModificationDelegator modificationDelegator; - private int startOfModificationIterators; - - public VanillaSharedReplicatedHashMap(@NotNull SharedHashMapBuilder builder, - @NotNull Class kClass, - @NotNull Class vClass) throws IOException { - super(builder, kClass, vClass); + private transient ModificationDelegator modificationDelegator; + private transient long startOfModificationIterators; + public ReplicatedChronicleMap(@NotNull ChronicleMapBuilder builder) + throws IOException { + super(builder); this.timeProvider = builder.timeProvider(); - this.localIdentifier = builder.identifier(); - File file = builder.file(); - ObjectSerializer objectSerializer = builder.objectSerializer(); - BytesStore bytesStore = file == null - ? DirectStore.allocateLazy(sizeInBytes(), objectSerializer) - : new MappedStore(file, FileChannel.MapMode.READ_WRITE, sizeInBytes(), objectSerializer); - createMappedStoreAndSegments(bytesStore); - - modificationDelegator = new ModificationDelegator(eventListener, modDelBytes, startOfModificationIterators); - this.eventListener = modificationDelegator; + Replicator firstReplicator = builder.firstReplicator; + this.localIdentifier = firstReplicator != null ? firstReplicator.identifier() : 1; } private int assignedModIterBitSetSizeInBytes() { @@ -127,7 +106,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } @Override - VanillaSharedHashMap.Segment createSegment(NativeBytes bytes, int index) { + Segment createSegment(NativeBytes bytes, int index) { return new Segment(bytes, index); } @@ -135,8 +114,14 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< return Segment.class; } - int modIterBitSetSizeInBytes() { - return (int) align64(bitsPerSegmentInModIterBitSet() * segments.length / 8); + @Override + void initTransients() { + super.initTransients(); + closeables = new CopyOnWriteArraySet(); + } + + long modIterBitSetSizeInBytes() { + return align64(bitsPerSegmentInModIterBitSet() * segments.length / 8); } private long bitsPerSegmentInModIterBitSet() { @@ -149,15 +134,15 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< return 2; } - int getHeaderSize() { - final int headerSize = super.getHeaderSize(); - - return headerSize + LAST_UPDATED_HEADER_SIZE + (modIterBitSetSizeInBytes() * (128 + - RESERVED_MOD_ITER)) + assignedModIterBitSetSizeInBytes(); + @Override + long getHeaderSize() { + return super.getHeaderSize() + LAST_UPDATED_HEADER_SIZE + + (modIterBitSetSizeInBytes() * (128 + RESERVED_MOD_ITER)) + + assignedModIterBitSetSizeInBytes(); } void setLastModificationTime(byte identifier, long timestamp) { - final int offset = identifier * 8; + final long offset = identifier * 8L; // purposely not volatile as this will impact performance, // and the worst that will happen is we'll end up loading more data on a bootstrap @@ -170,38 +155,40 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< public long lastModificationTime(byte remoteIdentifier) { assert remoteIdentifier != this.identifier(); - final int offset = remoteIdentifier * 8; // purposely not volatile as this will impact performance, // and the worst that will happen is we'll end up loading more data on a bootstrap - return identifierUpdatedBytes.readLong(offset); + return identifierUpdatedBytes.readLong(remoteIdentifier * 8L); } - @Override void onHeaderCreated() { - int offset = super.getHeaderSize(); + long offset = super.getHeaderSize(); identifierUpdatedBytes = ms.bytes(offset, LAST_UPDATED_HEADER_SIZE).zeroOut(); offset += LAST_UPDATED_HEADER_SIZE; - modDelBytes = ms.bytes(offset, assignedModIterBitSetSizeInBytes()).zeroOut(); + Bytes modDelBytes = ms.bytes(offset, assignedModIterBitSetSizeInBytes()).zeroOut(); offset += assignedModIterBitSetSizeInBytes(); startOfModificationIterators = offset; + + modificationDelegator = new ModificationDelegator(eventListener, modDelBytes); + } + + @Override + MapEventListener> eventListener() { + return modificationDelegator; } - /** - * {@inheritDoc} - */ @Override public V put(K key, V value) { return put0(key, value, true, localIdentifier, timeProvider.currentTimeMillis()); } /** - * Used in conjunction with map replication, all put events that originate from a remote node will be - * processed using this method. + * Used in conjunction with map replication, all put events that originate from a remote node will be processed + * using this method. * * @param key key with which the specified value is to be associated * @param value value to be associated with the specified key @@ -216,11 +203,8 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< return put0(key, value, true, identifier, timeStamp); } - /** - * {@inheritDoc} - */ @Override - public V putIfAbsent(@net.openhft.lang.model.constraints.NotNull K key, V value) { + public V putIfAbsent(@NotNull K key, V value) { return put0(key, value, false, localIdentifier, timeProvider.currentTimeMillis()); } @@ -263,17 +247,14 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< timeProvider.currentTimeMillis()); } - /** - * {@inheritDoc} - */ @Override public V remove(final Object key) { return removeIfValueIs(key, null, localIdentifier, timeProvider.currentTimeMillis()); } /** - * Used in conjunction with map replication, all remove events that originate from a remote node will be - * processed using this method. + * Used in conjunction with map replication, all remove events that originate from a remote node + * will be processed using this method. * * @param key key with which the specified value is associated * @param value value expected to be associated with the specified key @@ -282,20 +263,15 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< * @return {@code true} if the entry was removed * @see #remove(Object, Object) */ - V remove(K key, V value, byte identifier, long timeStamp) { assert identifier > 0; - return removeIfValueIs(key, null, identifier, timeStamp); + return removeIfValueIs(key, value, identifier, timeStamp); } - void addCloseable(Closeable closeable) { closeables.add(closeable); } - /** - * {@inheritDoc} - */ @Override public void close() { for (Closeable closeable : closeables) { @@ -316,10 +292,6 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } - - /** - * {@inheritDoc} - */ @Override public byte identifier() { return localIdentifier; @@ -330,15 +302,12 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< (short remoteIdentifier, @NotNull final ModificationNotifier modificationNotifier) { - return modificationDelegator.acquireModificationIterator(remoteIdentifier, modificationNotifier); + return modificationDelegator + .acquireModificationIterator(remoteIdentifier, modificationNotifier); } - - /** - * {@inheritDoc} - */ @Override - public boolean remove(@net.openhft.lang.model.constraints.NotNull final Object key, final Object value) { + public boolean remove(@NotNull final Object key, final Object value) { if (value == null) return false; // CHM compatibility; I would throw NPE return removeIfValueIs(key, (V) value, @@ -356,11 +325,8 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< timestamp, identifier); } - /** - * {@inheritDoc} - */ @Override - V replaceIfValueIs(@net.openhft.lang.model.constraints.NotNull final K key, final V existingValue, final V newValue) { + V replaceIfValueIs(@NotNull final K key, final V existingValue, final V newValue) { checkKey(key); checkValue(newValue); Bytes keyBytes = getKeyAsBytes(key); @@ -371,8 +337,217 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< timeProvider.currentTimeMillis()); } + /** + * This method does not set a segment lock, A segment lock should be obtained before calling + * this method, especially when being used in a multi threaded context. + */ + @Override + public void writeExternalEntry(@NotNull Bytes entry, @NotNull Bytes destination, + int chronicleId) { - class Segment extends VanillaSharedHashMap.Segment { + final long initialLimit = entry.limit(); + + final long keyLen = entry.readStopBit(); + final long keyPosition = entry.position(); + entry.skip(keyLen); + final long keyLimit = entry.position(); + final long timeStamp = entry.readLong(); + + final byte identifier = entry.readByte(); + if (identifier != localIdentifier) { + // although unlikely, this may occur if the entry has been updated + return; + } + + final boolean isDeleted = entry.readBoolean(); + long valueLen; + if (!isDeleted) { + valueLen = entry.readStopBit(); + assert valueLen > 0; + } else { + valueLen = 0; + } + + final long valuePosition = entry.position(); + + destination.writeStopBit(keyLen); + destination.writeStopBit(valueLen); + destination.writeStopBit(timeStamp); + + // we store the isDeleted flag in the identifier + // ( when the identifier is negative it is deleted ) + destination.writeByte(isDeleted ? -identifier : identifier); + + // write the key + entry.position(keyPosition); + entry.limit(keyLimit); + destination.write(entry); + + boolean debugEnabled = LOG.isDebugEnabled(); + String message = null; + if (debugEnabled) { + if (isDeleted) { + LOG.debug("WRITING ENTRY TO DEST - into local-id={}, remove(key={})", + localIdentifier, entry.toString().trim()); + } else { + message = String.format( + "WRITING ENTRY TO DEST - into local-id=%d, put(key=%s,", + localIdentifier, entry.toString().trim()); + } + } + + if (isDeleted) + return; + + entry.limit(initialLimit); + entry.position(valuePosition); + // skipping the alignment, as alignment wont work when we send the data over the wire. + alignment.alignPositionAddr(entry); + + // writes the value + entry.limit(entry.position() + valueLen); + destination.write(entry); + + if (debugEnabled) { + LOG.debug(message + "value=" + entry.toString().trim() + ")"); + } + } + + /** + * This method does not set a segment lock, A segment lock should be obtained before calling + * this method, especially when being used in a multi threaded context. + */ + @Override + public void readExternalEntry(@NotNull Bytes source) { + + final long keyLen = source.readStopBit(); + final long valueLen = source.readStopBit(); + final long timeStamp = source.readStopBit(); + final byte id = source.readByte(); + final byte remoteIdentifier; + final boolean isDeleted; + + if (id < 0) { + isDeleted = true; + remoteIdentifier = (byte) -id; + } else if (id != 0) { + isDeleted = false; + remoteIdentifier = id; + } else { + throw new IllegalStateException("identifier can't be 0"); + } + + if (remoteIdentifier == ReplicatedChronicleMap.this.identifier()) { + // this may occur when working with UDP, as we will receive our own data + return; + } + + final long keyPosition = source.position(); + final long keyLimit = keyPosition + keyLen; + + source.limit(keyLimit); + long hash = Hasher.hash(source); + + int segmentNum = hasher.getSegment(hash); + int segmentHash = hasher.segmentHash(hash); + + boolean debugEnabled = LOG.isDebugEnabled(); + + if (isDeleted) { + + if (debugEnabled) { + LOG.debug( + "READING FROM SOURCE - into local-id={}, remote={}, remove(key={})", + localIdentifier, remoteIdentifier, source.toString().trim() + ); + } + + segment(segmentNum).remoteRemove(source, segmentHash, timeStamp, remoteIdentifier); + setLastModificationTime(remoteIdentifier, timeStamp); + return; + } + + String message = null; + if (debugEnabled) { + message = String.format( + "READING FROM SOURCE - into local-id=%d, remote-id=%d, put(key=%s,", + localIdentifier, remoteIdentifier, source.toString().trim()); + } + + final long valuePosition = keyLimit; + final long valueLimit = valuePosition + valueLen; + segment(segmentNum).remotePut(source, segmentHash, remoteIdentifier, timeStamp, + valuePosition, valueLimit); + setLastModificationTime(remoteIdentifier, timeStamp); + + if (debugEnabled) { + source.limit(valueLimit); + source.position(valuePosition); + LOG.debug(message + "value=" + source.toString().trim() + ")"); + } + } + + @NotNull + @Override + public Set> entrySet() { + return new EntrySet(); + } + + @Override + public K key(@NotNull Bytes entry, K usingKey) { + final long start = entry.position(); + try { + // keyLen + entry.readStopBit(); + return keyMarshaller.read(entry, usingKey); + } finally { + entry.position(start); + } + + } + + @Override + public V value(@NotNull Bytes entry, V usingValue) { + final long start = entry.position(); + try { + + // keyLen + entry.skip(entry.readStopBit()); + + //timeStamp + entry.readLong(); + + final byte identifier = entry.readByte(); + if (identifier != localIdentifier) { + return null; + } + + final boolean isDeleted = entry.readBoolean(); + long valueLen; + if (!isDeleted) { + valueLen = entry.readStopBit(); + assert valueLen > 0; + } else { + return null; + } + + return valueMarshaller.read(entry, usingValue); + } finally { + entry.position(start); + } + } + + @Override + public boolean wasRemoved(@NotNull Bytes entry) { + final long start = entry.position(); + try { + return entry.readBoolean(entry.readStopBit() + 10); + } finally { + entry.position(start); + } + } + + class Segment extends VanillaChronicleMap.Segment { private volatile IntIntMultiMap hashLookupLiveAndDeleted; private volatile IntIntMultiMap hashLookupLiveOnly; @@ -384,8 +559,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< @Override void createHashLookups(long start) { hashLookupLiveAndDeleted = createMultiMap(start); - - start += sizeOfMultiMap(); + start += align64(sizeOfMultiMap() + sizeOfMultiMapBitSet()); hashLookupLiveOnly = createMultiMap(start); } @@ -393,9 +567,6 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< return hashLookupLiveOnly; } - - - private long entrySize(long keyLen, long valueLen) { long result = alignment.alignAddr(metaDataBytes + expectedStopBits(keyLen) + keyLen + 10 + @@ -403,7 +574,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< // replication enforces that the entry size will never be larger than an unsigned short if (result > MAX_UNSIGNED_SHORT) throw new IllegalStateException("ENTRY WRITE_BUFFER_SIZE TOO LARGE : Replicated " + - "SharedHashMap's" + + "ChronicleMap's" + " are restricted to an " + "entry size of " + MAX_UNSIGNED_SHORT + ", " + "your entry size=" + result); @@ -412,8 +583,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< /** - * @see net.openhft.collections.VanillaSharedHashMap.Segment#acquire(net.openhft.lang.io.Bytes, - * Object, Object, int, boolean) + * @see VanillaChronicleMap.Segment#acquire */ V acquire(Bytes keyBytes, K key, V usingValue, int hash2, boolean create, long timestamp) { lock(); @@ -429,7 +599,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } else { usingValue = tryObtainUsingValueOnAcquire(keyBytes, key, usingValue, create); if (usingValue != null) { - // see VanillaSharedHashMap.Segment.acquire() for explanation + // see VanillaChronicleMap.Segment.acquire() for explanation // why `usingValue` is `create`. offset = putEntryOnAcquire(keyBytes, hash2, usingValue, create, timestamp); incrementSize(); @@ -651,9 +821,9 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } /** - * Used only with replication, its sometimes possible to receive an old ( or stale update ) from a - * remote map. This method is used to determine if we should ignore such updates. - * + * Used only with replication, its sometimes possible to receive an old ( or stale update ) from a remote map. + * This method is used to determine if we should ignore such updates. + *

*

We can reject put() and removes() when comparing times stamps with remote systems * * @param entry the maps entry @@ -681,23 +851,22 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } /** - * Puts entry. If {@code value} implements {@link net.openhft.lang.model.Byteable} interface and - * {@code usingValue} is {@code true}, the value is backed with the bytes of this entry. + * Puts entry. If {@code value} implements {@link net.openhft.lang.model.Byteable} interface and {@code + * usingValue} is {@code true}, the value is backed with the bytes of this entry. * * @param keyBytes serialized key - * @param hash2 a hash of the {@code keyBytes}. Caller was searching for the key in the - * {@code searchedHashLookup} using this hash. + * @param hash2 a hash of the {@code keyBytes}. Caller was searching for the key in the {@code + * searchedHashLookup} using this hash. * @param value the value to put - * @param usingValue {@code true} if the value should be backed with the bytes of the entry, - * if it implements {@link net.openhft.lang.model.Byteable} interface, - * {@code false} if it should put itself - * @param identifier the identifier of the outer SHM node - * @param timestamp the timestamp when the entry was put (this could be later if it was a - * remote put) this method is called only from usual put or acquire + * @param usingValue {@code true} if the value should be backed with the bytes of the entry, if it + * implements {@link net.openhft.lang.model.Byteable} interface, {@code false} if it + * should put itself + * @param identifier the identifier of the outer CHM node + * @param timestamp the timestamp when the entry was put (this could be later if it was a remote + * put) this method is called only from usual put or acquire * @param searchedHashLookup the hash lookup that used to find the entry based on the key * @return offset of the written entry in the Segment bytes - * @see net.openhft.collections.VanillaSharedHashMap.Segment#putEntry(net.openhft.lang.io.Bytes, - * Object, boolean) + * @see VanillaChronicleMap.Segment#putEntry(Bytes, Object, boolean) */ private long putEntry(Bytes keyBytes, int hash2, V value, boolean usingValue, final int identifier, final long timestamp, @@ -745,8 +914,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } /** - * @see net.openhft.collections.VanillaSharedHashMap.Segment#remove(net.openhft.lang.io.Bytes, Object, - * Object, int) + * @see VanillaChronicleMap.Segment#remove(net.openhft.lang.io.Bytes, Object, Object, int) */ public V remove(Bytes keyBytes, K key, V expectedValue, int hash2, final long timestamp, final byte identifier) { @@ -817,8 +985,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } /** - * @see net.openhft.collections.VanillaSharedHashMap.Segment#remove(net.openhft.lang.io.Bytes, Object, - * Object, int) + * @see VanillaChronicleMap.Segment#remove(net.openhft.lang.io.Bytes, Object, Object, int) */ public V replace(Bytes keyBytes, K key, V expectedValue, V newValue, int hash2, long timestamp) { @@ -875,7 +1042,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< final long entryTimestamp = entry.readLong(); if (entryTimestamp >= timeStamp && - entry.readByte() == VanillaSharedReplicatedHashMap.this.identifier()) + entry.readByte() == ReplicatedChronicleMap.this.identifier()) callback.set(index, pos); } }); @@ -885,227 +1052,121 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } } + @Override + public Entry getEntry(long pos) { + bytes.position(offsetFromPos(pos) + metaDataBytes); + bytes.readStopBit(); + K key = keyMarshaller.read(bytes); + long timestamp = bytes.readLong(); + bytes.skip(2L); // identifier and isDeleted flag + bytes.readStopBit(); + alignment.alignPositionAddr(bytes); + V value = valueMarshaller.read(bytes); + return new TimestampTrackingEntry(key, value, timestamp); + } - /** - * removes all the entries - */ + @Override void clear() { - // we have to make sure that every calls notifies on remove, // so that the replicators can pick it up for (K k : keySet()) { - VanillaSharedReplicatedHashMap.this.remove(k); + ReplicatedChronicleMap.this.remove(k); } - - } - - void visit(IntIntMultiMap.EntryConsumer entryConsumer) { - hashLookupLiveOnly.forEach(entryConsumer); - } - - /** - * returns a null value if the entry has been deleted - * - * @param pos - * @return a null value if the entry has been deleted - */ - @Nullable - public Entry getEntry(long pos) { - long offset = offsetFromPos(pos); - NativeBytes entry = entry(offset); - entry.readStopBit(); - K key = entry.readInstance(kClass, null); //todo: readUsing? - - // skip timestamp and id - entry.skip(10); - - V value = readValue(entry, null); //todo: reusable container - return new WriteThroughEntry(key, value); } } + class TimestampTrackingEntry extends SimpleEntry { + private static final long serialVersionUID = 0L; - /** - * {@inheritDoc} - * - *

This method does not set a segment lock, A segment lock should be obtained before calling this - * method, especially when being used in a multi threaded context. - */ - @Override - public void writeExternalEntry(@NotNull AbstractBytes entry, @NotNull Bytes destination, int chronicleId) { + transient long timestamp; - final long initialLimit = entry.limit(); - - final long keyLen = entry.readStopBit(); - final long keyPosition = entry.position(); - entry.skip(keyLen); - final long keyLimit = entry.position(); - final long timeStamp = entry.readLong(); - - final byte identifier = entry.readByte(); - if (identifier != localIdentifier) { - // although unlikely, this may occur if the entry has been updated - return; + public TimestampTrackingEntry(K key, V value, long timestamp) { + super(key, value); + this.timestamp = timestamp; } - final boolean isDeleted = entry.readBoolean(); - long valueLen; - if (!isDeleted) { - valueLen = entry.readStopBit(); - assert valueLen > 0; - } else { - valueLen = 0; - } - - final long valuePosition = entry.position(); - - destination.writeStopBit(keyLen); - destination.writeStopBit(valueLen); - destination.writeStopBit(timeStamp); - - // we store the isDeleted flag in the identifier - // ( when the identifier is negative it is deleted ) - destination.writeByte(isDeleted ? -identifier : identifier); - - // write the key - entry.position(keyPosition); - entry.limit(keyLimit); - destination.write(entry); - - boolean debugEnabled = LOG.isDebugEnabled(); - String message = null; - if (debugEnabled) { - if (isDeleted) { - LOG.debug("WRITING ENTRY TO DEST - into local-id={}, remove(key={})", - localIdentifier, entry.toString().trim()); - } else { - message = String.format( - "WRITING ENTRY TO DEST - into local-id=%d, put(key=%s,", - localIdentifier, entry.toString().trim()); - } - } - - if (isDeleted) - return; - - entry.limit(initialLimit); - entry.position(valuePosition); - // skipping the alignment, as alignment wont work when we send the data over the wire. - alignment.alignPositionAddr(entry); - - // writes the value - entry.limit(entry.position() + valueLen); - destination.write(entry); - - if (debugEnabled) { - LOG.debug(message + "value=" + entry.toString().trim() + ")"); + @Override + public V setValue(V value) { + long newTimestamp = timestamp = timeProvider.currentTimeMillis(); + put(getKey(), value, localIdentifier, newTimestamp); + return super.setValue(value); } } + class EntryIterator extends VanillaChronicleMap.EntryIterator { + @Override + void removePresent(VanillaChronicleMap.Segment seg, int pos) { + @SuppressWarnings("unchecked") + Segment segment = (Segment) seg; - /** - * {@inheritDoc} - * - *

This method does not set a segment lock, A segment lock should be obtained before calling this - * method, especially when being used in a multi threaded context. - */ - @Override - public void readExternalEntry(@NotNull Bytes source) { + final long offset = segment.offsetFromPos(pos); + final NativeBytes entry = segment.entry(offset); + final long limit = entry.limit(); - final long keyLen = source.readStopBit(); - final long valueLen = source.readStopBit(); - final long timeStamp = source.readStopBit(); - final byte id = source.readByte(); - final byte remoteIdentifier; - final boolean isDeleted; - - if (id < 0) { - isDeleted = true; - remoteIdentifier = (byte) -id; - } else if (id != 0) { - isDeleted = false; - remoteIdentifier = id; - } else { - throw new IllegalStateException("identifier can't be 0"); - } - - if (remoteIdentifier == VanillaSharedReplicatedHashMap.this.identifier()) { - // this may occur when working with UDP, as we will receive our own data - return; - } - - final long keyPosition = source.position(); - final long keyLimit = keyPosition + keyLen; - - source.limit(keyLimit); - long hash = Hasher.hash(source); - - int segmentNum = hasher.getSegment(hash); - int segmentHash = hasher.segmentHash(hash); - - boolean debugEnabled = LOG.isDebugEnabled(); - - if (isDeleted) { - - if (debugEnabled) { - LOG.debug( - "READING FROM SOURCE - into local-id={}, remote={}, remove(key={})", - localIdentifier, remoteIdentifier, source.toString().trim() - ); + final long keyLen = entry.readStopBit(); + long keyPosition = entry.position(); + entry.skip(keyLen); + long timestamp = entry.readLong(); + entry.position(keyPosition); + if (timestamp > ((TimestampTrackingEntry) returnedEntry).timestamp) { + // The entry was updated after being returned from iterator.next() + // Check that it is still the entry with the same key + K key = returnedEntry.getKey(); + DirectBytes returnedKeyBytes = getKeyAsBytes(key); + if (returnedKeyBytes.remaining() != keyLen || !entry.startsWith(returnedKeyBytes)) { + // The case: + // 1. iterator.next() - thread 1 + // 2. map.put() which cause relocation of the key, returned above - thread 2 + // OR map.remove() which remove this key - thread 2 + // 3. map.put() which place a new key on the `pos` in current segment - thread 3 + // 4. iterator.remove() - thread 1 + ReplicatedChronicleMap.this.remove(key); + return; + } } + entry.limit(entry.position() + keyLen); + final int segmentHash = hasher.segmentHash(hash(entry)); + entry.limit(limit); - segment(segmentNum).remoteRemove(source, segmentHash, timeStamp, remoteIdentifier); - setLastModificationTime(remoteIdentifier, timeStamp); - return; - } + segment.hashLookupLiveOnly.remove(segmentHash, pos); + segment.decrementSize(); - String message = null; - if (debugEnabled) { - message = String.format( - "READING FROM SOURCE - into local-id=%d, remote-id=%d, put(key=%s,", - localIdentifier, remoteIdentifier, source.toString().trim()); - } + entry.skip(keyLen); + entry.writeLong(timeProvider.currentTimeMillis()); + entry.writeByte(localIdentifier); + entry.writeBoolean(true); - final long valuePosition = keyLimit; - final long valueLimit = valuePosition + valueLen; - segment(segmentNum).remotePut(source, segmentHash, remoteIdentifier, timeStamp, - valuePosition, valueLimit); - setLastModificationTime(remoteIdentifier, timeStamp); - - if (debugEnabled) { - source.limit(valueLimit); - source.position(valuePosition); - LOG.debug(message + "value=" + source.toString().trim() + ")"); + segment.notifyRemoved(offset, returnedEntry.getKey(), returnedEntry.getValue(), pos); } } + class EntrySet extends VanillaChronicleMap.EntrySet { + @NotNull + @Override + public Iterator> iterator() { + return new EntryIterator(); + } + } /** - * receive an update from the map, via the SharedMapEventListener and delegates the changes to the - * currently active modification iterators + * receive an update from the map, via the MapEventListener and delegates the changes + * to the currently active modification iterators */ - class ModificationDelegator extends SharedMapEventListener> { + class ModificationDelegator extends MapEventListener> { + private static final long serialVersionUID = 0L; // the assigned modification iterators - private final ATSDirectBitSet bitSet; - + private final ATSDirectBitSet modIterSet; + private final MapEventListener> nextListener; private final AtomicReferenceArray modificationIterators = new AtomicReferenceArray(127 + RESERVED_MOD_ITER); - private final SharedMapEventListener> nextListener; - private long startOfModificationIterators; - - public ModificationDelegator(@NotNull final SharedMapEventListener> nextListener, - final Bytes bytes, long startOfModificationIterators) { + public ModificationDelegator(@NotNull final MapEventListener> nextListener, + final Bytes bytes) { this.nextListener = nextListener; - this.startOfModificationIterators = startOfModificationIterators; - bitSet = new ATSDirectBitSet(bytes); - + modIterSet = new ATSDirectBitSet(bytes); } - /** - * {@inheritDoc} - */ public ModificationIterator acquireModificationIterator( final short remoteIdentifier, @NotNull final ModificationNotifier modificationNotifier) { @@ -1129,19 +1190,16 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< bytes, modificationNotifier); modificationIterators.set(remoteIdentifier, newModificationIterator); - bitSet.set(remoteIdentifier); + modIterSet.set(remoteIdentifier); return newModificationIterator; } } - /** - * {@inheritDoc} - */ @Override - public void onPut(SharedHashMap map, Bytes entry, int metaDataBytes, + public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, boolean added, K key, V value, long pos, SharedSegment segment) { - assert VanillaSharedReplicatedHashMap.this == map : + assert ReplicatedChronicleMap.this == map : "ModificationIterator.onPut() is called from outside of the parent map"; try { nextListener.onPut(map, entry, metaDataBytes, added, key, value, pos, segment); @@ -1149,7 +1207,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< LOG.error("", e); } - for (long next = bitSet.nextSetBit(0); next > 0; next = bitSet.nextSetBit(next + 1)) { + for (long next = modIterSet.nextSetBit(0); next > 0; next = modIterSet.nextSetBit(next + 1)) { try { final ModificationIterator modificationIterator = modificationIterators.get((int) next); modificationIterator.onPut(map, entry, metaDataBytes, @@ -1162,13 +1220,10 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } - /** - * {@inheritDoc} - */ @Override - public void onRemove(SharedHashMap map, Bytes entry, int metaDataBytes, + public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, K key, V value, int pos, SharedSegment segment) { - assert VanillaSharedReplicatedHashMap.this == map : + assert ReplicatedChronicleMap.this == map : "ModificationIterator.onRemove() is called from outside of the parent map"; try { nextListener.onRemove(map, entry, metaDataBytes, key, value, pos, segment); @@ -1176,7 +1231,7 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< LOG.error("", e); } - for (long next = bitSet.nextSetBit(0); next > 0; next = bitSet.nextSetBit(next + 1)) { + for (long next = modIterSet.nextSetBit(0); next > 0; next = modIterSet.nextSetBit(next + 1)) { try { final ModificationIterator modificationIterator = modificationIterators.get((int) next); modificationIterator.onRemove(map, entry, metaDataBytes, key, value, pos, segment); @@ -1187,38 +1242,67 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } + @Override + void onRelocation(int pos, SharedSegment segment) { + for (long next = modIterSet.nextSetBit(0); next > 0; next = modIterSet.nextSetBit(next + 1)) { + try { + final ModificationIterator modificationIterator = modificationIterators.get((int) next); + modificationIterator.onRelocation(pos, segment); + } catch (Exception e) { + LOG.error("", e); + } + } + } @Override - public V onGetMissing(SharedHashMap map, Bytes keyBytes, K key, + public V onGetMissing(ChronicleMap map, Bytes keyBytes, K key, V usingValue) { return nextListener.onGetMissing(map, keyBytes, key, usingValue); } @Override - public void onGetFound(SharedHashMap map, Bytes entry, int metaDataBytes, + public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, K key, V value) { nextListener.onGetFound(map, entry, metaDataBytes, key, value); } + /** + * Always throws {@code NotSerializableException} since instances of this class + * are not intended to be serializable. + */ + private void writeObject(ObjectOutputStream out) throws IOException { + throw new NotSerializableException(getClass().getCanonicalName()); + } + + /** + * Always throws {@code NotSerializableException} since instances of this class + * are not intended to be serializable. + */ + private void readObject(ObjectInputStream in) + throws IOException, ClassNotFoundException { + throw new NotSerializableException(getClass().getCanonicalName()); + } + + } /** - * Once a change occurs to a map, map replication requires that these changes are picked up by another - * thread, this class provides an iterator like interface to poll for such changes. - * - *

In most cases the thread that adds data to the node is unlikely to be the same thread that - * replicates the data over to the other nodes, so data will have to be marshaled between the main thread - * storing data to the map, and the thread running the replication. - * - *

One way to perform this marshalling, would be to pipe the data into a queue. However, This class - * takes another approach. It uses a bit set, and marks bits which correspond to the indexes of the - * entries that have changed. It then provides an iterator like interface to poll for such changes. + * Once a change occurs to a map, map replication requires that these changes are picked up by another thread, this + * class provides an iterator like interface to poll for such changes. + *

+ *

In most cases the thread that adds data to the node is unlikely to be the same thread that replicates the data + * over to the other nodes, so data will have to be marshaled between the main thread storing data to the map, and + * the thread running the replication. + *

+ *

One way to perform this marshalling, would be to pipe the data into a queue. However, This class takes another + * approach. It uses a bit set, and marks bits which correspond to the indexes of the entries that have changed. It + * then provides an iterator like interface to poll for such changes. * * @author Rob Austin. */ - class ModificationIterator extends SharedMapEventListener> + class ModificationIterator extends MapEventListener> implements Replica.ModificationIterator { - + private static final long serialVersionUID = 0L; private final ModificationNotifier modificationNotifier; private final ATSDirectBitSet changes; @@ -1255,14 +1339,11 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< return (((long) segmentIndex) << segmentIndexShift) | pos; } - /** - * {@inheritDoc} - */ @Override - public void onPut(SharedHashMap map, Bytes entry, int metaDataBytes, + public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, boolean added, K key, V value, long pos, SharedSegment segment) { - assert VanillaSharedReplicatedHashMap.this == map : + assert ReplicatedChronicleMap.this == map : "ModificationIterator.onPut() is called from outside of the parent map"; @@ -1270,13 +1351,10 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< modificationNotifier.onChange(); } - /** - * {@inheritDoc} - */ @Override - public void onRemove(SharedHashMap map, Bytes entry, int metaDataBytes, + public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, K key, V value, int pos, SharedSegment segment) { - assert VanillaSharedReplicatedHashMap.this == map : + assert ReplicatedChronicleMap.this == map : "ModificationIterator.onRemove() is called from outside of the parent map"; @@ -1297,8 +1375,8 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< /** - * you can continue to poll hasNext() until data becomes available. If are are in the middle of - * processing an entry via {@code nextEntry}, hasNext will return true until the bit is cleared + * you can continue to poll hasNext() until data becomes available. If are are in the middle of processing an + * entry via {@code nextEntry}, hasNext will return true until the bit is cleared * * @return true if there is an entry */ @@ -1311,10 +1389,9 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< /** * @param entryCallback call this to get an entry, this class will take care of the locking - * @param chronicleId * @return true if an entry was processed */ - public boolean nextEntry(@NotNull final AbstractEntryCallback entryCallback, final int chronicleId) { + public boolean nextEntry(@NotNull final EntryCallback entryCallback, final int chronicleId) { long position = this.position; while (true) { long oldPosition = position; @@ -1356,6 +1433,33 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< } } + @Override + public void dirtyEntries(long fromTimeStamp) { + + // iterate over all the segments and mark bit in the modification iterator + // that correspond to entries with an older timestamp + for (final Segment segment : (Segment[]) segments) { + segment.dirtyEntries(fromTimeStamp, entryModifiableCallback); + } + } + + /** + * Always throws {@code NotSerializableException} since instances of this class + * are not intended to be serializable. + */ + private void writeObject(ObjectOutputStream out) throws IOException { + throw new NotSerializableException(getClass().getCanonicalName()); + } + + /** + * Always throws {@code NotSerializableException} since instances of this class + * are not intended to be serializable. + */ + private void readObject(ObjectInputStream in) + throws IOException, ClassNotFoundException { + throw new NotSerializableException(getClass().getCanonicalName()); + } + /** * details about when a modification to an entry was made */ @@ -1372,107 +1476,6 @@ class VanillaSharedReplicatedHashMap extends AbstractVanillaSharedHashMap< changes.set(combine); } } - - @Override - public void dirtyEntries(long fromTimeStamp) { - - // iterate over all the segments and mark bit in the modification iterator - // that correspond to entries with an older timestamp - for (final Segment segment : (Segment[]) segments) { - segment.dirtyEntries(fromTimeStamp, entryModifiableCallback); - } - - } - } - - /** - * {@inheritDoc} - */ - @Override - public K key(@NotNull AbstractBytes entry, K usingKey) { - - final long start = entry.position(); - try { - - // keyLen - entry.readStopBit(); - - final long keyPosition = entry.position(); - - if (generatedValueType) - if (usingKey == null) - usingKey = DataValueClasses.newDirectReference(kClass); - else - assert usingKey instanceof Byteable; - if (usingKey instanceof Byteable) { - ((Byteable) usingKey).bytes(entry, keyPosition); - return usingKey; - } - - return entry.readInstance(kClass, usingKey); - } finally { - entry.position(start); - } - - } - - /** - * {@inheritDoc} - */ - @Override - public V value(@NotNull AbstractBytes entry, V usingValue) { - final long start = entry.position(); - try { - - // keyLen - entry.skip(entry.readStopBit()); - - //timeStamp - entry.readLong(); - - final byte identifier = entry.readByte(); - if (identifier != localIdentifier) { - return null; - } - - final boolean isDeleted = entry.readBoolean(); - long valueLen; - if (!isDeleted) { - valueLen = entry.readStopBit(); - assert valueLen > 0; - } else { - return null; - } - - final long valueOffset = entry.position(); - - if (generatedValueType) - if (usingValue == null) - usingValue = DataValueClasses.newDirectReference(vClass); - else - assert usingValue instanceof Byteable; - if (usingValue instanceof Byteable) { - ((Byteable) usingValue).bytes(entry, valueOffset); - return usingValue; - } - - return entry.readInstance(vClass, usingValue); - } finally { - entry.position(start); - } - } - - /** - * {@inheritDoc} - */ - @Override - public boolean wasRemoved(@NotNull AbstractBytes entry) { - final long start = entry.position(); - try { - return entry.readBoolean(entry.readStopBit() + 10); - } finally { - entry.position(start); - } } } diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatingCluster.java b/src/main/java/net/openhft/chronicle/map/ReplicatingCluster.java new file mode 100644 index 000000000..e555ba63d --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/ReplicatingCluster.java @@ -0,0 +1,416 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import net.openhft.lang.collection.DirectBitSet; +import net.openhft.lang.collection.SingleThreadedDirectBitSet; +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.Bytes; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.atomic.AtomicReferenceArray; + +import static java.lang.Math.min; +import static java.nio.ByteBuffer.wrap; +import static net.openhft.chronicle.map.Replica.EntryExternalizable; +import static net.openhft.chronicle.map.Replica.ModificationIterator; +import static net.openhft.chronicle.map.Replica.ModificationNotifier.NOP; + +/** + * @author Rob Austin. + */ +public final class ReplicatingCluster implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(ReplicatingCluster.class.getName()); + + private static final byte BOOTSTRAP_MESSAGE = 'B'; + final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { + /** + * writes the entry to the chronicle channel provided + * + * @param entry the byte location of the entry to be stored + * @param destination a buffer the entry will be written to, the segment may reject + * this operation and add zeroBytes, if the identifier in the entry + * did not match the maps local + * @param chronicleChannel used in cluster into identify the canonical map or queue + */ + @Override + public void writeExternalEntry(@NotNull Bytes entry, @NotNull Bytes destination, + int chronicleChannel) { + destination.writeStopBit(chronicleChannel); + channelEntryExternalizables[chronicleChannel] + .writeExternalEntry(entry, destination, chronicleChannel); + } + + @Override + public void readExternalEntry(@NotNull Bytes source) { + final int chronicleId = (int) source.readStopBit(); + if (chronicleId < chronicleChannels.length) + channelEntryExternalizables[chronicleId].readExternalEntry(source); + else + LOG.info("skipped entry with chronicleId=" + chronicleId + ", "); + } + }; + private final byte localIdentifier; + final Replica asReplica = new Replica() { + + @Override + public byte identifier() { + return localIdentifier; + } + + @Override + public ModificationIterator acquireModificationIterator( + final short remoteIdentifier, final ModificationNotifier notifier) { + final ModificationIterator result = modificationIterator.get(remoteIdentifier); + if (result != null) + return result; + + final ModificationIterator result0 = new ModificationIterator() { + + @Override + public boolean hasNext() { + for (int i = (int) chronicleChannelBitSet.nextSetBit(0); i >= 0; i = (int) + chronicleChannelBitSet.nextSetBit(i + 1)) { + final ModificationIterator modificationIterator = chronicleChannels[i] + .acquireModificationIterator(remoteIdentifier, notifier); + if (modificationIterator.hasNext()) + return true; + } + return false; + } + + @Override + public boolean nextEntry(@NotNull EntryCallback callback, + final int na) { + for (int i = (int) chronicleChannelBitSet.nextSetBit(0); i >= 0; + i = (int) chronicleChannelBitSet.nextSetBit(i + 1)) { + final ModificationIterator modificationIterator = chronicleChannels[i] + .acquireModificationIterator(remoteIdentifier, notifier); + if (modificationIterator.nextEntry(callback, i)) + return true; + } + return false; + } + + @Override + public void dirtyEntries(long fromTimeStamp) { + for (int i = (int) chronicleChannelBitSet.nextSetBit(0); i >= 0; + i = (int) chronicleChannelBitSet.nextSetBit(i + 1)) { + chronicleChannels[i].acquireModificationIterator(remoteIdentifier, notifier) + .dirtyEntries(fromTimeStamp); + notifier.onChange(); + } + } + }; + + modificationIterator.set((int) remoteIdentifier, result0); + return result0; + + } + + /** + * gets the earliest modification time for all of the chronicles + */ + @Override + public long lastModificationTime(byte remoteIdentifier) { + long t = System.currentTimeMillis(); + for (int i = (int) chronicleChannelBitSet.nextSetBit(0); i > 0; + i = (int) chronicleChannelBitSet.nextSetBit(i + 1)) { + t = min(t, chronicleChannels[i].lastModificationTime(remoteIdentifier)); + } + return t; + } + + @Override + public void close() throws IOException { + ReplicatingCluster.this.close(); + } + }; + private final int maxEntrySize; + private final DirectBitSet chronicleChannelBitSet; + private final Replica[] chronicleChannels; + private final EntryExternalizable[] channelEntryExternalizables; + private final AtomicReferenceArray systemModificationIterator = + new AtomicReferenceArray(128); + private final DirectBitSet systemModificationIteratorBitSet = + newBitSet(systemModificationIterator.length()); + private final AtomicReferenceArray modificationIterator = + new AtomicReferenceArray(128); + private final Set replicators = + new CopyOnWriteArraySet(); + + ReplicatingCluster(ReplicatingClusterBuilder builder) { + localIdentifier = builder.identifier; + maxEntrySize = builder.maxEntrySize; + chronicleChannels = new Replica[builder.maxNumberOfChronicles]; + channelEntryExternalizables = new EntryExternalizable[builder.maxNumberOfChronicles]; + chronicleChannelBitSet = newBitSet(chronicleChannels.length); + MessageHandler systemMessageHandler = new MessageHandler() { + @Override + public void onMessage(Bytes bytes) { + final byte type = bytes.readByte(); + if (type == BOOTSTRAP_MESSAGE) { + onBootstrapMessage(bytes); + } else { + LOG.info("message of type=" + type + " was ignored."); + } + } + }; + SystemQueue systemMessageQueue = new SystemQueue( + systemModificationIteratorBitSet, systemModificationIterator, systemMessageHandler); + add((short) 0, systemMessageQueue.asReplica, systemMessageQueue.asEntryExternalizable); + } + + /** + * creates a bit set based on a number of bits + * + * @param numberOfBits the number of bits the bit set should include + * @return a new DirectBitSet backed by a byteBuffer + */ + private static DirectBitSet newBitSet(int numberOfBits) { + final ByteBufferBytes bytes = new ByteBufferBytes(wrap(new byte[(numberOfBits + 7) / 8])); + return new SingleThreadedDirectBitSet(bytes); + } + + /** + * Returns a replicator, dedicated to the specified channel. Channel is basically + * just a number, that should correspond on different servers for instances + * of the same replicated map. Only one replicator per channel could be obtained from + * a single {@code ReplicatingCluster}. The returned replicator could be applied to a map + * at most once. + * + * @return a replicator, dedicated to the specified channel + */ + public Replicator channelReplicator(short channel) { + return new ChronicleChannel(channel); + } + + /** + * called whenever we receive a bootstrap message + */ + private void onBootstrapMessage(Bytes bytes) { + final short remoteIdentifier = bytes.readByte(); + final int chronicleChannel = bytes.readUnsignedShort(); + final long lastModificationTime = bytes.readLong(); + + // this could be null if once node has a chronicle channel before the other + if (chronicleChannels[chronicleChannel] != null) { + chronicleChannels[chronicleChannel].acquireModificationIterator(remoteIdentifier, NOP) + .dirtyEntries(lastModificationTime); + } + } + + private ByteBufferBytes toBootstrapMessage(short chronicleChannel, final long lastModificationTime) { + final ByteBufferBytes writeBuffer = new ByteBufferBytes(ByteBuffer.allocate(1 + 1 + 2 + 8)); + writeBuffer.writeByte(BOOTSTRAP_MESSAGE); + writeBuffer.writeByte(localIdentifier); + writeBuffer.writeUnsignedShort(chronicleChannel); + writeBuffer.writeLong(lastModificationTime); + writeBuffer.flip(); + return writeBuffer; + } + + private void add(short chronicleChannel, + Replica replica, EntryExternalizable entryExternalizable) { + if (chronicleChannels[chronicleChannel] != null) { + throw new IllegalStateException("chronicleId=" + chronicleChannel + + " is already in use."); + } + chronicleChannels[chronicleChannel] = replica; + channelEntryExternalizables[chronicleChannel] = entryExternalizable; + chronicleChannelBitSet.set(chronicleChannel); + + if (chronicleChannel == 0) + return; + + for (int i = (int) systemModificationIteratorBitSet.nextSetBit(0); i > 0; + i = (int) systemModificationIteratorBitSet.nextSetBit(i + 1)) { + byte remoteIdentifier = (byte) i; + final long lastModificationTime = replica.lastModificationTime(remoteIdentifier); + final ByteBufferBytes message = + toBootstrapMessage(chronicleChannel, lastModificationTime); + systemModificationIterator.get(remoteIdentifier).addPayload(message); + } + + } + + @Override + public void close() throws IOException { + for (AbstractChannelReplicator replicator : replicators) { + replicator.close(); + } + for (int i = (int) chronicleChannelBitSet.nextSetBit(0); i > 0; + i = (int) chronicleChannelBitSet.nextSetBit(i + 1)) { + chronicleChannels[i].close(); + } + } + + void add(AbstractChannelReplicator replicator) { + replicators.add(replicator); + } + + private interface MessageHandler { + void onMessage(Bytes bytes); + } + + private interface PayloadProvider extends ModificationIterator { + void addPayload(final Bytes bytes); + } + + /** + * used to send system messages such as bootstrap from one remote node to another, + * it also can be used in a broadcast context + */ + static class SystemQueue { + + + final Replica asReplica = new Replica() { + + @Override + public byte identifier() { + return 0; + } + + @Override + public ModificationIterator acquireModificationIterator( + final short remoteIdentifier, final ModificationNotifier modificationNotifier) { + + final ModificationIterator result = systemModificationIterator.get(remoteIdentifier); + + if (result != null) + return result; + + final PayloadProvider iterator = new PayloadProvider() { + + final Queue payloads = new LinkedTransferQueue(); + + @Override + public boolean hasNext() { + return payloads.peek() != null; + } + + @Override + public boolean nextEntry(@NotNull EntryCallback callback, int na) { + final Bytes bytes = payloads.poll(); + if (bytes == null) + return false; + callback.onEntry(bytes, 0); + return true; + } + + @Override + public void dirtyEntries(long fromTimeStamp) { + // do nothing + } + + @Override + public void addPayload(Bytes bytes) { + if (bytes.remaining() == 0) + return; + payloads.add(bytes); + // notifies that a change has been made, this will nudge the OP_WRITE + // selector to push this update out over the nio socket + modificationNotifier.onChange(); + } + }; + + systemModificationIterator.set(remoteIdentifier, iterator); + systemModificationIteratorBitSet.set(remoteIdentifier); + + return iterator; + + } + + @Override + public long lastModificationTime(byte remoteIdentifier) { + return 0; + } + + @Override + public void close() throws IOException { + // do nothing + } + }; + final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { + @Override + public void writeExternalEntry(@NotNull Bytes entry, @NotNull Bytes destination, + int na) { + destination.write(entry); + } + + @Override + public void readExternalEntry(@NotNull Bytes source) { + messageHandler.onMessage(source); + } + }; + private final DirectBitSet systemModificationIteratorBitSet; + private final AtomicReferenceArray systemModificationIterator; + private final MessageHandler messageHandler; + + SystemQueue(DirectBitSet systemModificationIteratorBitSet, + AtomicReferenceArray systemModificationIterator, + MessageHandler messageHandler) { + + this.systemModificationIteratorBitSet = systemModificationIteratorBitSet; + this.systemModificationIterator = systemModificationIterator; + this.messageHandler = messageHandler; + } + } + + private class ChronicleChannel extends Replicator implements Closeable { + private final short chronicleChannel; + + private ChronicleChannel(short chronicleChannel) { + this.chronicleChannel = chronicleChannel; + } + + @Override + public byte identifier() { + return localIdentifier; + } + + @Override + protected Closeable applyTo(ChronicleMapBuilder builder, + Replica map, EntryExternalizable entryExternalizable) { + if (builder.entrySize() > maxEntrySize) { + throw new IllegalArgumentException("During ReplicatingClusterBuilder setup, " + + "maxEntrySize=" + maxEntrySize + " was specified, but map with " + + "entrySize=" + builder.entrySize() + " is attempted to apply" + + "to the replicator"); + } + add(chronicleChannel, map, entryExternalizable); + return this; + } + + @Override + public void close() throws IOException { + chronicleChannelBitSet.clear(chronicleChannel); + chronicleChannels[chronicleChannel] = null; + channelEntryExternalizables[chronicleChannel] = null; + } + } +} \ No newline at end of file diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatingClusterBuilder.java b/src/main/java/net/openhft/chronicle/map/ReplicatingClusterBuilder.java new file mode 100644 index 000000000..c5a85fc82 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/ReplicatingClusterBuilder.java @@ -0,0 +1,77 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import java.io.IOException; + +/** + * @author Rob Austin. + */ +public final class ReplicatingClusterBuilder { + + final byte identifier; + final int maxEntrySize; + int maxNumberOfChronicles = 128; + private UdpReplicationConfig udpReplicationConfig = null; + private TcpReplicationConfig tcpReplicationConfig = null; + + ReplicatingClusterBuilder(byte identifier, final int maxEntrySize) { + this.identifier = identifier; + this.maxEntrySize = maxEntrySize; + if (identifier <= 0) { + throw new IllegalArgumentException("Identifier must be positive, identifier=" + + identifier); + } + } + + public ReplicatingClusterBuilder udpReplication(UdpReplicationConfig replicationConfig) { + this.udpReplicationConfig = replicationConfig; + return this; + } + + public ReplicatingClusterBuilder tcpReplication(TcpReplicationConfig replicationConfig) { + this.tcpReplicationConfig = replicationConfig; + return this; + } + + public ReplicatingClusterBuilder maxNumberOfChronicles(int maxNumberOfChronicles) { + this.maxNumberOfChronicles = maxNumberOfChronicles; + return this; + } + + public ReplicatingCluster create() throws IOException { + final ReplicatingCluster replicatingCluster = new ReplicatingCluster(this); + if (tcpReplicationConfig != null) { + final TcpReplicator tcpReplicator = new TcpReplicator( + replicatingCluster.asReplica, + replicatingCluster.asEntryExternalizable, + tcpReplicationConfig, + maxEntrySize); + replicatingCluster.add(tcpReplicator); + } + + if (udpReplicationConfig != null) { + final UdpReplicator udpReplicator = + new UdpReplicator(replicatingCluster.asReplica, + replicatingCluster.asEntryExternalizable, + udpReplicationConfig, + maxEntrySize); + replicatingCluster.add(udpReplicator); + } + return replicatingCluster; + } +} diff --git a/src/main/java/net/openhft/chronicle/map/Replicator.java b/src/main/java/net/openhft/chronicle/map/Replicator.java new file mode 100644 index 000000000..0911c8ba0 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Replicator.java @@ -0,0 +1,62 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import java.io.Closeable; +import java.io.IOException; + +/** + * @see Replicators + * @see ChronicleMapBuilder#addReplicator(Replicator) + */ +public abstract class Replicator { + + /** + * A constructor for use in subclasses. + */ + protected Replicator() { + } + + /** + * Returns this node identifier within the replication topology. + * + * @return the unique identifier among nodes + * @see Replica#identifier() + */ + public abstract byte identifier(); + + /** + * Applies the replicator to the map instance and returns a Closeable token to manage resources, + * associated with the replication. + *

+ *

This method isn't intended to be called from the client code. + * + * @param builder the builder from which the map was constructed. The replicator may obtain some + * map configurations, not accessible via the map instance. + * @param map a replicated map instance. Provides basic tools for replication implementation. + * @param entryExternalizable the callback for ser/deser implementation in the replicator + * @return a {@code Closeable} token to control replication resources. + * It should be closed on closing the replicated map. + * @throws java.io.IOException if an io error occurred during the replicator setup + * @throws IllegalStateException if this replicator doesn't support application to more than + * one map (or the specified number of maps), and this replicator has already been + * applied to a map (or the specified number of maps) + */ + protected abstract Closeable applyTo(ChronicleMapBuilder builder, + Replica map, Replica.EntryExternalizable entryExternalizable) + throws IOException; +} diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java new file mode 100644 index 000000000..f8b44bf52 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -0,0 +1,63 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import java.io.Closeable; +import java.io.IOException; + +public final class Replicators { + + private Replicators() { + } + + public static Replicator tcp(final byte identifier, + final TcpReplicationConfig replicationConfig) { + return new Replicator() { + @Override + public byte identifier() { + return identifier; + } + + @Override + protected Closeable applyTo(ChronicleMapBuilder builder, + Replica map, Replica.EntryExternalizable entryExternalizable) + throws IOException { + return new TcpReplicator(map, entryExternalizable, replicationConfig, + builder.entrySize()); + } + }; + } + + public static Replicator udp(final byte identifier, + final UdpReplicationConfig replicationConfig) { + return new Replicator() { + @Override + public byte identifier() { + return identifier; + } + + @Override + protected Closeable applyTo(ChronicleMapBuilder builder, + Replica map, Replica.EntryExternalizable entryExternalizable) + throws IOException { + return new UdpReplicator(map, entryExternalizable, replicationConfig, + builder.entrySize()); + } + }; + } +} diff --git a/src/main/java/net/openhft/chronicle/map/SharedSegment.java b/src/main/java/net/openhft/chronicle/map/SharedSegment.java new file mode 100644 index 000000000..03c05d430 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/SharedSegment.java @@ -0,0 +1,42 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.NativeBytes; + +import java.util.Map; + +/** + * Allow access to the Segments that user used to make up the shared map, these methods should be used with + * exceptional caution and were initially introduce to facilitate remote map replication. + */ +interface SharedSegment { + + void lock(); + + Map.Entry getEntry(long pos); + + void unlock(); + + NativeBytes entry(long pos); + + int getIndex(); + + long offsetFromPos(long pos); + + +} diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java b/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java new file mode 100644 index 000000000..cf07c522d --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicationConfig.java @@ -0,0 +1,118 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import com.google.auto.value.AutoValue; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableSet; +import static java.util.concurrent.TimeUnit.SECONDS; + +@AutoValue +public abstract class TcpReplicationConfig { + + private static final int DEFAULT_PACKET_SIZE = 1024 * 8; + private static final long DEFAULT_HEART_BEAT_INTERVAL = 20; + private static final TimeUnit DEFAULT_HEART_BEAT_INTERVAL_UNIT = SECONDS; + + /** + * Package-private constructor forbids subclassing from outside of the package + */ + TcpReplicationConfig() { + // nothing to do + } + + public static TcpReplicationConfig of(int serverPort, InetSocketAddress... endpoints) { + return of(serverPort, asList(endpoints)); + } + + public static TcpReplicationConfig of(int serverPort, Collection endpoints) { + // at least in tests, we configure "receive-only" replication without endpoints. + // TODO decide what to do with this case +// if (endpoints.isEmpty()) +// throw new IllegalArgumentException("There should be some endpoints"); + for (final InetSocketAddress endpoint : endpoints) { + if (endpoint.getPort() == serverPort && "localhost".equals(endpoint.getHostName())) + throw new IllegalArgumentException("endpoint=" + endpoint + + " can not point to the same port as the server"); + } + return create(serverPort, + unmodifiableSet(new HashSet(endpoints)), + DEFAULT_PACKET_SIZE, ThrottlingConfig.noThrottling(), + DEFAULT_HEART_BEAT_INTERVAL, DEFAULT_HEART_BEAT_INTERVAL_UNIT); + } + + static TcpReplicationConfig create(int serverPort, Set endpoints, + int packetSize, ThrottlingConfig throttlingConfig, + long heartBeatInterval, TimeUnit heartBeatIntervalUnit) { + return new AutoValue_TcpReplicationConfig(serverPort, endpoints, packetSize, + throttlingConfig, heartBeatInterval, heartBeatIntervalUnit); + } + + public abstract int serverPort(); + + public abstract Set endpoints(); + + public abstract int packetSize(); + + public TcpReplicationConfig packetSize(int packetSize) { + if (packetSize <= 0) + throw new IllegalArgumentException(); + return create(serverPort(), endpoints(), packetSize, throttlingConfig(), + heartBeatInterval(), heartBeatIntervalUnit()); + } + + public abstract ThrottlingConfig throttlingConfig(); + + public TcpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { + ThrottlingConfig.checkMillisecondBucketInterval(throttlingConfig, "TCP"); + return create(serverPort(), endpoints(), packetSize(), throttlingConfig, + heartBeatInterval(), heartBeatIntervalUnit()); + } + + abstract long heartBeatInterval(); + + abstract TimeUnit heartBeatIntervalUnit(); + + public long heartBeatInterval(TimeUnit unit) { + return unit.convert(heartBeatInterval(), heartBeatIntervalUnit()); + } + + /** + * @param heartBeatInterval heart beat interval + * @param unit the time unit of the interval + * @return this builder back + * @throws IllegalArgumentException if the given heart beat interval is unrecognisably small + * for the current TCP replicator implementation or negative. Current minimum interval + * is 1 millisecond. + */ + public TcpReplicationConfig heartBeatInterval(long heartBeatInterval, TimeUnit unit) { + if (unit.toMillis(heartBeatInterval) < 1) { + throw new IllegalArgumentException( + "Minimum heart beat interval is 1 millisecond, " + + heartBeatInterval + " " + unit + " given"); + } + return create(serverPort(), endpoints(), packetSize(), throttlingConfig(), + heartBeatInterval, unit); + } +} diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java new file mode 100644 index 000000000..1089cc697 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -0,0 +1,1028 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.net.*; +import java.nio.ByteBuffer; +import java.nio.channels.*; +import java.util.BitSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.nio.channels.SelectionKey.*; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +/** + * Used with a {@see net.openhft.map.ReplicatedSharedHashMap} to send data between the maps using a + * socket connection

{@see net.openhft.map.OutSocketReplicator} + * + * @author Rob Austin. + */ +class TcpReplicator extends AbstractChannelReplicator implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName()); + private static final int BUFFER_SIZE = 0x100000; // 1MB + final AtomicBoolean forceBootstrap = new AtomicBoolean(); + private final Map connectorBySocket = new ConcurrentHashMap(); + private final SelectionKey[] selectionKeysStore = new SelectionKey[Byte.MAX_VALUE + 1]; + // used to instruct the selector thread to set OP_WRITE on a key correlated by the bit index in the + // bitset + private final KeyInterestUpdater opWriteUpdater = new KeyInterestUpdater(OP_WRITE, selectionKeysStore); + private final BitSet activeKeys = new BitSet(selectionKeysStore.length); + private final long heartBeatInterval; + private final InetSocketAddress serverInetSocketAddress; + private final int packetSize; + private final Iterable endpoints; + + private final Replica replica; + private final byte localIdentifier; + private final int maxEntrySizeBytes; + private final Replica.EntryExternalizable externalizable; + private long selectorTimeout; + + /** + * @param maxEntrySizeBytes used to check that the last entry will fit into the buffer, it can not be + * smaller than the size of and entry, if it is set smaller the buffer will + * over flow, it can be larger then the entry, but setting it too large + * reduces the workable space in the buffer. + * @throws IOException + */ + TcpReplicator(@NotNull final Replica replica, + @NotNull final Replica.EntryExternalizable externalizable, + @NotNull final TcpReplicationConfig replicationConfig, + final int maxEntrySizeBytes) throws IOException { + + super("TcpSocketReplicator-" + replica.identifier(), replicationConfig.throttlingConfig(), + maxEntrySizeBytes); + + serverInetSocketAddress = new InetSocketAddress(replicationConfig.serverPort()); + + heartBeatInterval = replicationConfig.heartBeatInterval(MILLISECONDS); + + long throttleBucketInterval = + replicationConfig.throttlingConfig().bucketInterval(MILLISECONDS); + selectorTimeout = Math.min(heartBeatInterval, throttleBucketInterval); + + packetSize = replicationConfig.packetSize(); + endpoints = replicationConfig.endpoints(); + + this.replica = replica; + this.localIdentifier = replica.identifier(); + this.maxEntrySizeBytes = maxEntrySizeBytes; + this.externalizable = externalizable; + + start(); + } + + @Override + void process() throws IOException { + try { + final Details serverDetails = new Details(serverInetSocketAddress, localIdentifier); + connectorBySocket.put(serverInetSocketAddress, new ServerConnector(serverDetails)); + + for (InetSocketAddress client : endpoints) { + final Details clientDetails = new Details(client, localIdentifier); + connectorBySocket.put(client, new ClientConnector(clientDetails)); + } + + for (AbstractConnector connector : connectorBySocket.values()) { + connector.connect(); + } + + while (selector.isOpen()) { + + registerPendingRegistrations(); + + final int nSelectedKeys = selector.select(selectorTimeout); + + // its less resource intensive to set this less frequently and use an approximation + final long approxTime = System.currentTimeMillis(); + + checkThrottleInterval(); + + // check that we have sent and received heartbeats + heartBeatMonitor(approxTime); + + // set the OP_WRITE when data is ready to send + opWriteUpdater.applyUpdates(); + + if (nSelectedKeys == 0) + continue; // go back and check pendingRegistrations + + final Set selectionKeys = selector.selectedKeys(); + for (final SelectionKey key : selectionKeys) { + try { + + if (!key.isValid()) + continue; + + if (key.isAcceptable()) + onAccept(key); + + if (key.isConnectable()) + onConnect(key); + + if (key.isReadable()) + onRead(key, approxTime); + + if (key.isWritable()) + onWrite(key, approxTime); + + } catch (CancelledKeyException e) { + quietClose(key, e); + } catch (ClosedSelectorException e) { + quietClose(key, e); + } catch (IOException e) { + quietClose(key, e); + } catch (InterruptedException e) { + quietClose(key, e); + } catch (Exception e) { + LOG.info("", e); + closeEarlyAndQuietly(key.channel()); + } + } + selectionKeys.clear(); + } + } catch (CancelledKeyException e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } catch (ClosedSelectorException e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } catch (ClosedChannelException e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } catch (ConnectException e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } catch (Exception e) { + LOG.error("", e); + } finally + + { + if (selector != null) + try { + selector.close(); + } catch (IOException e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } + close(); + } + } + + /** + * checks that we receive heartbeats and send out heart beats. + * + * @param approxTime the approximate time in milliseconds + */ + void heartBeatMonitor(long approxTime) { + for (int i = activeKeys.nextSetBit(0); i >= 0; i = activeKeys.nextSetBit(i + 1)) { + try { + final SelectionKey key = selectionKeysStore[i]; + if (!key.isValid() || !key.channel().isOpen()) { + activeKeys.clear(i); + continue; + } + try { + sendHeartbeatIfRequired(approxTime, key); + } catch (Exception e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } + + try { + heartbeatCheckHasReceived(key, approxTime); + } catch (Exception e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } + + } catch (Exception e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + } + } + } + + /** + * check to see if its time to send a heartbeat, and send one if required + * + * @param approxTime the current time ( approximately ) + * @param key nio selection key + */ + private void sendHeartbeatIfRequired(final long approxTime, + @NotNull final SelectionKey key) { + + final Attached attachment = (Attached) key.attachment(); + + if (attachment.isHandShakingComplete() && attachment.entryWriter.lastSentTime + + heartBeatInterval < approxTime) { + + attachment.entryWriter.lastSentTime = approxTime; + attachment.entryWriter.writeHeartbeatToBuffer(); + + enableOpWrite(key); + + if (LOG.isDebugEnabled()) + LOG.debug("sending heartbeat"); + + } + } + + private void enableOpWrite(SelectionKey key) { + int ops = key.interestOps(); + if ((ops & (OP_CONNECT | OP_ACCEPT)) == 0) + key.interestOps(ops | OP_WRITE); + } + + /** + * check to see if we have lost connection with the remote node and if we have attempts a reconnect. + * + * @param key the key relating to the heartbeat that we are checking + * @param approxTimeOutTime the approximate time in milliseconds + * @throws ConnectException + */ + private void heartbeatCheckHasReceived(@NotNull final SelectionKey key, + final long approxTimeOutTime) throws ConnectException { + + final Attached attached = (Attached) key.attachment(); + + // we wont attempt to reconnect the server socket + if (attached.isServer || !attached.isHandShakingComplete()) + return; + + final SocketChannel channel = (SocketChannel) key.channel(); + + if (approxTimeOutTime > attached.entryReader.lastHeartBeatReceived + attached.remoteHeartbeatInterval) { + if (LOG.isDebugEnabled()) + LOG.debug("lost connection, attempting to reconnect. " + + "missed heartbeat from identifier=" + attached.remoteIdentifier); + try { + channel.socket().close(); + channel.close(); + activeKeys.clear(attached.remoteIdentifier); + closeables.remove(channel); + } catch (IOException e) { + LOG.debug("", e); + } + + attached.connector.connectLater(); + } + } + + /** + * closes and only logs the exception at debug + * + * @param key the SelectionKey + * @param e the Exception that caused the issue + */ + private void quietClose(@NotNull final SelectionKey key, @NotNull final Exception e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + closeEarlyAndQuietly(key.channel()); + } + + @Override + public void forceBootstrap() { + forceBootstrap.lazySet(true); + } + + /** + * called when the selector receives a OP_CONNECT message + */ + private void onConnect(@NotNull final SelectionKey key) + throws IOException, InterruptedException { + + final SocketChannel channel = (SocketChannel) key.channel(); + final Attached attached = (Attached) key.attachment(); + + try { + if (!channel.finishConnect()) { + return; + } + } catch (SocketException e) { + quietClose(key, e); + attached.connector.connect(); + throw e; + } + + attached.connector.setSuccessfullyConnected(); + + if (LOG.isDebugEnabled()) + LOG.debug("successfully connected to {}, local-id={}", + channel.socket().getInetAddress(), localIdentifier); + + channel.configureBlocking(false); + channel.socket().setTcpNoDelay(true); + channel.socket().setSoTimeout(0); + channel.socket().setSoLinger(false, 0); + + attached.entryReader = new TcpSocketChannelEntryReader(); + attached.entryWriter = new TcpSocketChannelEntryWriter(); + + key.interestOps(OP_WRITE | OP_READ); + + throttle(channel); + + // register it with the selector and store the ModificationIterator for this key + attached.entryWriter.identifierToBuffer(localIdentifier); + } + + /** + * called when the selector receives a OP_ACCEPT message + */ + private void onAccept(@NotNull final SelectionKey key) throws IOException { + + final ServerSocketChannel server = (ServerSocketChannel) key.channel(); + final SocketChannel channel = server.accept(); + channel.configureBlocking(false); + channel.socket().setReuseAddress(true); + channel.socket().setTcpNoDelay(true); + channel.socket().setSoTimeout(0); + channel.socket().setSoLinger(false, 0); + + final Attached attached = new Attached(); + channel.register(selector, OP_WRITE | OP_READ, attached); + + throttle(channel); + + attached.entryReader = new TcpSocketChannelEntryReader(); + attached.entryWriter = new TcpSocketChannelEntryWriter(); + + attached.isServer = true; + attached.entryWriter.identifierToBuffer(localIdentifier); + } + + /** + * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear all the + * old bootstrap information + * + * @param key the nio SelectionKey + */ + private void clearHandshaking(SelectionKey key) { + final Attached attached = (Attached) key.attachment(); + activeKeys.clear(attached.remoteIdentifier); + selectionKeysStore[attached.remoteIdentifier] = null; + attached.clearHandShaking(); + } + + /** + * used to exchange identifiers and timestamps and heartbeat intervals between the server and client + * + * @param key the SelectionKey relating to the this cha + * @throws java.io.IOException + * @throws InterruptedException + */ + private void doHandShaking(@NotNull final SelectionKey key) + throws IOException, InterruptedException { + final Attached attached = (Attached) key.attachment(); + final TcpSocketChannelEntryWriter writer = attached.entryWriter; + final TcpSocketChannelEntryReader reader = attached.entryReader; + + if (attached.remoteIdentifier == Byte.MIN_VALUE) { + + final byte remoteIdentifier = reader.identifierFromBuffer(); + + if (remoteIdentifier == Byte.MIN_VALUE) + return; + + attached.remoteIdentifier = remoteIdentifier; + + // we use the as iterating the activeKeys via the bitset wont create and Objects + // but if we use the selector.keys() this will. + selectionKeysStore[remoteIdentifier] = key; + activeKeys.set(remoteIdentifier); + + if (LOG.isDebugEnabled()) { + LOG.debug("server-connection id={}, remoteIdentifier={}", + localIdentifier, remoteIdentifier); + } + + if (remoteIdentifier == localIdentifier) { + throw new IllegalStateException("Where are connecting to a remote " + + "map with the same " + + "identifier as this map, identifier=" + localIdentifier + ", " + + "please change either this maps identifier or the remote one"); + } + + attached.remoteModificationIterator = replica.acquireModificationIterator(remoteIdentifier, + attached); + + writer.writeRemoteBootstrapTimestamp(replica.lastModificationTime(remoteIdentifier)); + + // tell the remote node, what are heartbeat interval is + writer.writeRemoteHeartbeatInterval(heartBeatInterval); + } + + if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) { + attached.remoteBootstrapTimestamp = reader.remoteBootstrapTimestamp(); + if (attached.remoteBootstrapTimestamp == Long.MIN_VALUE) + return; + } + + if (!attached.hasRemoteHeartbeatInterval) { + + long value = reader.remoteHeartbeatIntervalFromBuffer(); + + if (value == Long.MIN_VALUE) + return; + + if (value < 0) { + LOG.error("value=" + value); + } + + // we add a 10% safety margin to the timeout time due to latency fluctuations on the network, + // in other words we wont consider a connection to have + // timed out, unless the heartbeat interval has exceeded 25% of the expected time. + attached.remoteHeartbeatInterval = (long) (value * 1.25); + + // we have to make our selector poll interval at least as short as the minimum selector timeout + selectorTimeout = Math.min(selectorTimeout, value); + + if (selectorTimeout < 0) + LOG.info(""); + + attached.hasRemoteHeartbeatInterval = true; + + // now we're finished we can get on with reading the entries + attached.setHandShakingComplete(); + attached.remoteModificationIterator.dirtyEntries(attached.remoteBootstrapTimestamp); + reader.entriesFromBuffer(); + } + } + + /** + * called when the selector receives a OP_WRITE message + */ + private void onWrite(@NotNull final SelectionKey key, + final long approxTime) throws InterruptedException, IOException { + final SocketChannel socketChannel = (SocketChannel) key.channel(); + final Attached attached = (Attached) key.attachment(); + + if (attached.remoteModificationIterator != null) + attached.entryWriter.entriesToBuffer(attached.remoteModificationIterator, key); + + try { + int bytesJustWritten = attached.entryWriter.writeBufferToSocket(socketChannel, + approxTime); + + contemplateThrottleWrites(bytesJustWritten); + + } catch (IOException e) { + quietClose(key, e); + if (!attached.isServer) + attached.connector.connectLater(); + throw e; + } + } + + /** + * called when the selector receives a OP_READ message + */ + private void onRead(final SelectionKey key, + final long approxTime) throws IOException, InterruptedException { + + final SocketChannel socketChannel = (SocketChannel) key.channel(); + final Attached attached = (Attached) key.attachment(); + + try { + if (attached.entryReader.readSocketToBuffer(socketChannel) <= 0) + return; + + } catch (IOException e) { + if (!attached.isServer) + attached.connector.connectLater(); + throw e; + } + + if (LOG.isDebugEnabled()) + LOG.debug("heartbeat or data received."); + + attached.entryReader.lastHeartBeatReceived = approxTime; + + if (attached.isHandShakingComplete()) { + attached.entryReader.entriesFromBuffer(); + } else { + doHandShaking(key); + } + } + + /** + * sets interestOps to "selector keys",The change to interestOps much be on the same thread as the + * selector. This class, allows via {@link AbstractChannelReplicator + * .KeyInterestUpdater#set(int)} to holds a pending change in interestOps ( via a bitset ), this change + * is processed later on the same thread as the selector + */ + private static class KeyInterestUpdater { + + private final AtomicBoolean wasChanged = new AtomicBoolean(); + private final BitSet changeOfOpWriteRequired; + private final SelectionKey[] selectionKeys; + private final int op; + + KeyInterestUpdater(int op, final SelectionKey[] selectionKeys) { + this.op = op; + this.selectionKeys = selectionKeys; + changeOfOpWriteRequired = new BitSet(selectionKeys.length); + } + + public void applyUpdates() { + if (wasChanged.getAndSet(false)) { + for (int i = changeOfOpWriteRequired.nextSetBit(0); i >= 0; + i = changeOfOpWriteRequired.nextSetBit(i + 1)) { + changeOfOpWriteRequired.clear(i); + final SelectionKey key = selectionKeys[i]; + try { + key.interestOps(key.interestOps() | op); + } catch (Exception e) { + LOG.debug("", e); + } + } + } + } + + /** + * @param keyIndex the index of the key that has changed, the list of keys is provided by the + * constructor {@link KeyInterestUpdater(int, SelectionKey[])} + */ + public void set(int keyIndex) { + changeOfOpWriteRequired.set(keyIndex); + wasChanged.lazySet(true); + } + } + + private class ServerConnector extends AbstractConnector { + + private final Details details; + + private ServerConnector(@NotNull Details details) { + super("TCP-ServerConnector-" + localIdentifier); + this.details = details; + } + + @Override + public String toString() { + return "ServerConnector{" + + "" + details + + '}'; + } + + SelectableChannel doConnect() throws + IOException, InterruptedException { + + final ServerSocketChannel serverChannel = ServerSocketChannel.open(); + + serverChannel.socket().setReceiveBufferSize(BUFFER_SIZE); + serverChannel.configureBlocking(false); + final ServerSocket serverSocket = serverChannel.socket(); + serverSocket.setReuseAddress(true); + + serverSocket.bind(details.address()); + + // these can be run on this thread + addPendingRegistration(new Runnable() { + @Override + public void run() { + final Attached attached = new Attached(); + attached.connector = ServerConnector.this; + try { + serverChannel.register(TcpReplicator.this.selector, OP_ACCEPT, attached); + } catch (ClosedChannelException e) { + LOG.error("", e); + } + + } + }); + + selector.wakeup(); + + return serverChannel; + } + } + + private class ClientConnector extends AbstractConnector { + + private final Details details; + + private ClientConnector(@NotNull Details details) { + super("TCP-ClientConnector-" + details.localIdentifier()); + this.details = details; + } + + + @Override + public String toString() { + return "ClientConnector{" + + "" + details + + '}'; + } + + /** + * blocks until connected + */ + @Override + SelectableChannel doConnect() throws IOException, InterruptedException { + boolean success = false; + final SocketChannel socketChannel = SocketChannel.open(); + try { + socketChannel.configureBlocking(false); + socketChannel.socket().setReuseAddress(true); + socketChannel.socket().setSoLinger(false, 0); + socketChannel.socket().setSoTimeout(0); + socketChannel.socket().setTcpNoDelay(true); + + try { + socketChannel.connect(details.address()); + } catch (UnresolvedAddressException e) { + this.connectLater(); + } + + // Under experiment, the concoction was found to be more successful if we + // paused before registering the OP_CONNECT + Thread.sleep(10); + + // the registration has be be run on the same thread as the selector + addPendingRegistration(new Runnable() { + @Override + public void run() { + + final Attached attached = new Attached(); + attached.connector = ClientConnector.this; + + try { + socketChannel.register(selector, OP_CONNECT, attached); + } catch (ClosedChannelException e) { + if (socketChannel.isOpen()) + LOG.error("", e); + } + + } + }); + + selector.wakeup(); + success = true; + return socketChannel; + + } finally { + if (!success) { + try { + try { + socketChannel.socket().close(); + } catch (Exception e) { + LOG.error("", e); + } + socketChannel.close(); + } catch (IOException e) { + LOG.error("", e); + } + } + } + } + } + + /** + * Attached to the NIO selection key via methods such as {@link SelectionKey#attach(Object)} + */ + class Attached implements Replica.ModificationNotifier { + + public TcpSocketChannelEntryReader entryReader; + public TcpSocketChannelEntryWriter entryWriter; + + public Replica.ModificationIterator remoteModificationIterator; + public AbstractConnector connector; + public long remoteBootstrapTimestamp = Long.MIN_VALUE; + public byte remoteIdentifier = Byte.MIN_VALUE; + public boolean hasRemoteHeartbeatInterval; + // true if its socket is a ServerSocket + public boolean isServer; // the frequency the remote node will send a heartbeat + public long remoteHeartbeatInterval = heartBeatInterval; + private boolean handShakingComplete; + + boolean isHandShakingComplete() { + return handShakingComplete; + } + + void setHandShakingComplete() { + handShakingComplete = true; + } + + void clearHandShaking() { + handShakingComplete = false; + + remoteIdentifier = Byte.MIN_VALUE; + remoteBootstrapTimestamp = Long.MIN_VALUE; + remoteHeartbeatInterval = heartBeatInterval; + hasRemoteHeartbeatInterval = false; + remoteModificationIterator = null; + + + } + + /** + * called whenever there is a change to the modification iterator + */ + @Override + public void onChange() { + + if (remoteIdentifier != Byte.MIN_VALUE) + TcpReplicator.this.opWriteUpdater.set(remoteIdentifier); + + selector.wakeup(); + } + + + } + + /** + * @author Rob Austin. + */ + private class TcpSocketChannelEntryWriter { + + private final ByteBuffer out; + private final ByteBufferBytes in; + private final EntryCallback entryCallback; + private long lastSentTime; + + private TcpSocketChannelEntryWriter() { + out = ByteBuffer.allocateDirect(packetSize + maxEntrySizeBytes); + in = new ByteBufferBytes(out); + entryCallback = new EntryCallback(externalizable, in); + } + + + /** + * writes the timestamp into the buffer + * + * @param localIdentifier the current nodes identifier + */ + void identifierToBuffer(final int localIdentifier) { + in.writeByte(localIdentifier); + } + + /** + * sends the identity and timestamp of this node to a remote node + * + * @param timeStampOfLastMessage the last timestamp we received a message from that node + */ + void writeRemoteBootstrapTimestamp(final long timeStampOfLastMessage) { + in.writeLong(timeStampOfLastMessage); + } + + /** + * writes all the entries that have changed, to the buffer which will later be written to TCP/IP + * + * @param modificationIterator a record of which entries have modification + * @param selectionKey + */ + void entriesToBuffer(@NotNull final Replica.ModificationIterator modificationIterator, + @NotNull final SelectionKey selectionKey) + throws InterruptedException, IOException { + + final SocketChannel socketChannel = (SocketChannel) selectionKey.channel(); + final Attached attached = (Attached) selectionKey.attachment(); + + // this can occur when new CHM's are added to a cluster + final boolean handShakingComplete = attached.isHandShakingComplete(); + + for (; ; ) { + + final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0); + + if (!wasDataRead) { + + // if we have no more data to write to the socket then we will + // un-register OP_WRITE on the selector, until more data becomes available + if (in.position() == 0 && handShakingComplete) + disableWrite(socketChannel, attached); + + return; + } + + // we've filled up the buffer lets give another channel a chance to send some data + if (in.remaining() <= maxEntrySizeBytes) + return; + + // if we have space in the buffer to write more data and we just wrote data into the + // buffer then let try and write some more + } + + } + + /** + * writes the contents of the buffer to the socket + * + * @param socketChannel the socket to publish the buffer to + * @param approxTime an approximation of the current time in millis + * @throws IOException + */ + private int writeBufferToSocket(@NotNull final SocketChannel socketChannel, + final long approxTime) throws IOException { + + if (in.position() == 0) + return 0; + + // if we still have some unwritten writer from last time + lastSentTime = approxTime; + out.limit((int) in.position()); + + final int len = socketChannel.write(out); + + if (LOG.isDebugEnabled()) + LOG.debug("bytes-written=" + len); + + if (out.remaining() == 0) { + out.clear(); + in.clear(); + } else { + out.compact(); + in.position(out.position()); + in.limit(in.capacity()); + out.clear(); + } + + return len; + } + + /** + * used to send an single zero byte if we have not send any data for up to the localHeartbeatInterval + */ + private void writeHeartbeatToBuffer() { + in.writeUnsignedShort(0); + } + + private void writeRemoteHeartbeatInterval(long localHeartbeatInterval) { + in.writeLong(localHeartbeatInterval); + } + + + /** + * removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will get + * added back in as soon as we have data to write + * + * @param socketChannel the socketChannel we wish to stop writing to + * @param attached data associated with the socketChannels key + */ + public synchronized void disableWrite(@NotNull final SocketChannel socketChannel, + @NotNull final Attached attached) { + try { + SelectionKey key = socketChannel.keyFor(selector); + if (key != null) { + if (attached.isHandShakingComplete() && selector.isOpen()) { + if (LOG.isDebugEnabled()) + LOG.debug("Disabling OP_WRITE to remoteIdentifier=" + + attached.remoteIdentifier + + ", localIdentifier=" + localIdentifier); + key.interestOps(key.interestOps() & ~OP_WRITE); + } + } + + } catch (Exception e) { + LOG.error("", e); + } + } + + + } + + /** + * Reads map entries from a socket, this could be a client or server socket + */ + private class TcpSocketChannelEntryReader { + private final ByteBuffer in; + private final ByteBufferBytes out; + public long lastHeartBeatReceived = System.currentTimeMillis(); + // we use Integer.MIN_VALUE as N/A + private int sizeOfNextEntry = Integer.MIN_VALUE; + + private TcpSocketChannelEntryReader() { + in = ByteBuffer.allocateDirect(packetSize + maxEntrySizeBytes); + out = new ByteBufferBytes(in); + out.limit(0); + in.clear(); + } + + /** + * reads from the socket and writes them to the buffer + * + * @param socketChannel the socketChannel to read from + * @return the number of bytes read + * @throws IOException + */ + private int readSocketToBuffer(@NotNull final SocketChannel socketChannel) + throws IOException { + + compactBuffer(); + final int len = socketChannel.read(in); + out.limit(in.position()); + return len; + } + + /** + * reads entries from the buffer till empty + * + * @throws InterruptedException + */ + private void entriesFromBuffer() throws InterruptedException, IOException { + + for (; ; ) { + + out.limit(in.position()); + + // its set to MIN_VALUE when it should be read again + if (sizeOfNextEntry == Integer.MIN_VALUE) { + if (out.remaining() < SIZE_OF_SHORT) { + return; + } + + int value = out.readUnsignedShort(); + + // this is the heartbeat + if (value == 0) + continue; + + sizeOfNextEntry = value; + } + + + if (out.remaining() < sizeOfNextEntry) { + return; + } + + final long nextEntryPos = out.position() + sizeOfNextEntry; + final long limit = out.limit(); + out.limit(nextEntryPos); + externalizable.readExternalEntry(out); + + out.limit(limit); + // skip onto the next entry + out.position(nextEntryPos); + + // to allow the sizeOfNextEntry to be read the next time around + sizeOfNextEntry = Integer.MIN_VALUE; + } + + } + + + /** + * compacts the buffer and updates the {@code in} and {@code out} accordingly + */ + private void compactBuffer() { + + // the maxEntrySizeBytes used here may not be the maximum size of the entry in its serialized form + // however, its only use as an indication that the buffer is becoming full and should be compacted + // the buffer can be compacted at any time + if (in.position() == 0 || in.remaining() > maxEntrySizeBytes) + return; + + in.limit(in.position()); + in.position((int) out.position()); + + in.compact(); + out.position(0); + } + + /** + * @return the identifier or -1 if unsuccessful + */ + byte identifierFromBuffer() { + return (out.remaining() >= 1) ? out.readByte() : Byte.MIN_VALUE; + } + + /** + * @return the timestamp or -1 if unsuccessful + */ + long remoteBootstrapTimestamp() { + return (out.remaining() >= 8) ? out.readLong() : Long.MIN_VALUE; + } + + public long remoteHeartbeatIntervalFromBuffer() { + return (out.remaining() >= 8) ? out.readLong() : Long.MIN_VALUE; + } + } +} + diff --git a/src/main/java/net/openhft/chronicle/map/ThrottlingConfig.java b/src/main/java/net/openhft/chronicle/map/ThrottlingConfig.java new file mode 100644 index 000000000..46b7e0a77 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/ThrottlingConfig.java @@ -0,0 +1,124 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import com.google.auto.value.AutoValue; + +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +@AutoValue +public abstract class ThrottlingConfig { + + private static final long DEFAULT_BUCKET_INTERVAL = 100L; + private static final TimeUnit DEFAULT_BUCKET_INTERVAL_UNIT = MILLISECONDS; + + private static final ThrottlingConfig NO_THROTTLING = create(0L, MILLISECONDS, + DEFAULT_BUCKET_INTERVAL, DEFAULT_BUCKET_INTERVAL_UNIT); + + /** + * Package-private constructor forbids subclassing from outside of the package + */ + ThrottlingConfig() { + // nothing to do + } + + /** + * Returns a config which prescribe that a replicator shouldn't throttle. + * + * @return a config which prescribe that a replicator shouldn't throttle + */ + public static ThrottlingConfig noThrottling() { + return NO_THROTTLING; + } + + /** + * Returns a config which throttle the specified max bits per the given time unit + * with default bucketing interval. + * + * @param maxBits the preferred maximum bits + * @param perUnit the time unit per which maximum bits specified + * @return a config which throttle the specified max bits per the given time unit + * @throws IllegalArgumentException if the specified {@code maxBits} is non-positive value + */ + public static ThrottlingConfig throttle(long maxBits, TimeUnit perUnit) { + if (maxBits <= 0L) + throw new IllegalArgumentException(); + return create(maxBits, perUnit, DEFAULT_BUCKET_INTERVAL, DEFAULT_BUCKET_INTERVAL_UNIT); + } + + static void checkMillisecondBucketInterval(ThrottlingConfig config, String replicator) { + if (config.bucketInterval(MILLISECONDS) < 1) { + throw new IllegalArgumentException( + "Minimum throttle bucketing interval of 1 millisecond is supported by " + + replicator + " replicator"); + } + } + + static ThrottlingConfig create( + long throttle, TimeUnit throttlePerUnit, + long throttleBucketInterval, TimeUnit throttleBucketIntervalUnit) { + return new AutoValue_ThrottlingConfig(throttle, throttlePerUnit, + throttleBucketInterval, throttleBucketIntervalUnit); + } + + abstract long throttle(); + + abstract TimeUnit throttlePerUnit(); + + /** + * Returns maximum bits per the given time unit, i. e. the throttling. {@code 0} (zero) + * designates there is no throttling. + * + * @param perUnit maximum bits is returned per this time unit + * @return maximum bits per the given time unit, or {@code 0} if there is no throttling + */ + public final long throttling(TimeUnit perUnit) { + return throttlePerUnit().convert(throttle(), perUnit); + } + + abstract long bucketInterval(); + + abstract TimeUnit bucketIntervalUnit(); + + /** + * Returns the throttle bucketing interval in the given time units. + *

+ *

Default throttle bucketing interval is 100 millis. + * + * @param unit the time unit of the interval + * @return the bucketing interval for throttling in the given time units + */ + public long bucketInterval(TimeUnit unit) { + return unit.convert(bucketInterval(), bucketIntervalUnit()); + } + + /** + * Returns a copy of this config with the specified bucketing interval. + * + * @param throttleBucketInterval the bucketing interval for throttling + * @param unit the time unit of the interval + * @return a copy of this config with the specified bucketing interval + * @throws IllegalArgumentException if the given bucketing interval is non-positive + */ + public ThrottlingConfig bucketInterval(long throttleBucketInterval, TimeUnit unit) { + if (throttleBucketInterval <= 0L) + throw new IllegalArgumentException(); + return create(throttle(), throttlePerUnit(), throttleBucketInterval, unit); + } +} diff --git a/src/main/java/net/openhft/chronicle/map/TimeProvider.java b/src/main/java/net/openhft/chronicle/map/TimeProvider.java new file mode 100644 index 000000000..0802b5fbd --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/TimeProvider.java @@ -0,0 +1,64 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import java.io.ObjectStreamException; +import java.io.Serializable; + +/** + * TimeProvider was created initially to support testing of ReplicatedChronicleMap, + * with the aim of possibly later providing and optimization to System.currentTimeMillis() + * on every call to put(), get() and remove() + *

+ * Subclasses should be immutable, because {@link ChronicleMapBuilder} doesn't make defensive copies. + * + * @author Rob Austin. + */ +public abstract class TimeProvider implements Serializable { + public static final TimeProvider SYSTEM = new System(); + private static final long serialVersionUID = 0L; + + public abstract long currentTimeMillis(); + + private static class System extends TimeProvider { + private static final long serialVersionUID = 0L; + + public long currentTimeMillis() { + return java.lang.System.currentTimeMillis(); + } + + @Override + public boolean equals(Object o) { + return o != null && o.getClass() == getClass(); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } + + private Object readResolve() throws ObjectStreamException { + return SYSTEM; + } + } + +} diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java new file mode 100644 index 000000000..13725a13d --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicationConfig.java @@ -0,0 +1,69 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import com.google.auto.value.AutoValue; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.net.InetAddress; +import java.net.NetworkInterface; + +@AutoValue +public abstract class UdpReplicationConfig { + + /** + * Package-private constructor forbids subclassing from outside of the package + */ + UdpReplicationConfig() { + // nothing to do + } + + public static UdpReplicationConfig simple(@NotNull InetAddress address, int port) { + if (address.isMulticastAddress()) + throw new IllegalArgumentException(); + return create(address, port, null, ThrottlingConfig.noThrottling()); + } + + public static UdpReplicationConfig multiCast(@NotNull InetAddress address, int port, + @NotNull NetworkInterface networkInterface) { + if (!address.isMulticastAddress() || networkInterface == null) + throw new IllegalArgumentException(); + return create(address, port, networkInterface, ThrottlingConfig.noThrottling()); + } + + static UdpReplicationConfig create(InetAddress address, int port, + NetworkInterface networkInterface, + ThrottlingConfig throttlingConfig) { + return new AutoValue_UdpReplicationConfig(address, port, networkInterface, + throttlingConfig); + } + + public abstract InetAddress address(); + + public abstract int port(); + + @Nullable + public abstract NetworkInterface networkInterface(); + + public abstract ThrottlingConfig throttlingConfig(); + + public UdpReplicationConfig throttlingConfig(ThrottlingConfig throttlingConfig) { + ThrottlingConfig.checkMillisecondBucketInterval(throttlingConfig, "UDP"); + return create(address(), port(), networkInterface(), throttlingConfig); + } +} diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java new file mode 100644 index 000000000..193463b0b --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java @@ -0,0 +1,400 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.NetworkInterface; +import java.net.StandardSocketOptions; +import java.nio.ByteBuffer; +import java.nio.channels.*; +import java.util.Set; + +import static java.net.StandardProtocolFamily.INET; +import static java.net.StandardProtocolFamily.INET6; +import static java.net.StandardSocketOptions.IP_MULTICAST_IF; +import static java.net.StandardSocketOptions.SO_REUSEADDR; +import static java.nio.channels.SelectionKey.OP_READ; +import static java.nio.channels.SelectionKey.OP_WRITE; + + +/** + * The UdpReplicator attempts to read the data ( but it does not enforce or grantee delivery ), + * typically, you should use the UdpReplicator if you have a large number of nodes, + * and you wish to receive the data before it becomes available on TCP/IP. + * In order to not miss data, UdpReplicator should be used in conjunction with the TCP Replicator. + */ +class UdpReplicator extends AbstractChannelReplicator implements Replica.ModificationNotifier, Closeable { + + private static final Logger LOG = + LoggerFactory.getLogger(UdpReplicator.class.getName()); + + private final byte localIdentifier; + private final UdpSocketChannelEntryWriter writer; + private final UdpSocketChannelEntryReader reader; + private final Replica replica; + private final InetAddress address; + private final int port; + private final NetworkInterface networkInterface; + private final ServerConnector serverConnector; + private Replica.ModificationIterator modificationIterator; + private SelectableChannel writeChannel; + private volatile boolean shouldEnableOpWrite; + + UdpReplicator(@NotNull final Replica replica, + @NotNull final Replica.EntryExternalizable entryExternalizable, + @NotNull final UdpReplicationConfig replicationConfig, + final int serializedEntrySize) + throws IOException { + super("UdpReplicator-" + replica.identifier(), replicationConfig.throttlingConfig(), + serializedEntrySize); + + this.localIdentifier = replica.identifier(); + this.replica = replica; + this.writer = new UdpSocketChannelEntryWriter(serializedEntrySize, entryExternalizable); + this.reader = new UdpSocketChannelEntryReader(serializedEntrySize, entryExternalizable); + + address = replicationConfig.address(); + port = replicationConfig.port(); + networkInterface = replicationConfig.networkInterface(); + + serverConnector = new ServerConnector(); + + start(); + } + + @Override + public void close() { + writeChannel = null; + super.close(); + } + + + @Override + public void forceBootstrap() { + modificationIterator = replica.acquireModificationIterator( + ChronicleMapBuilder.UDP_REPLICATION_MODIFICATION_ITERATOR_ID, this); + } + + /** + * binds to the server socket and process data This method will block until interrupted + */ + @Override + void process() throws IOException { + + connectClient().register(selector, OP_READ); + serverConnector.connectLater(); + + while (selector.isOpen()) { + + registerPendingRegistrations(); + + // this may block for a long time, upon return the + // selected set contains keys of the ready channels + final int n = selector.select(100); + + if (shouldEnableOpWrite) + enableWrites(); + + checkThrottleInterval(); + + if (n == 0) { + continue; // nothing to do + } + + final Set selectionKeys = selector.selectedKeys(); + for (final SelectionKey key : selectionKeys) { + + try { + + if (key.isReadable()) { + final DatagramChannel socketChannel = (DatagramChannel) key.channel(); + reader.readAll(socketChannel); + } + + if (key.isWritable()) { + final DatagramChannel socketChannel = (DatagramChannel) key.channel(); + try { + int bytesJustWritten = writer.writeAll(socketChannel); + contemplateThrottleWrites(bytesJustWritten); + } catch (NotYetConnectedException e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + serverConnector.connectLater(); + } catch (IOException e) { + if (LOG.isDebugEnabled()) + LOG.debug("", e); + serverConnector.connectLater(); + } + } + + } catch (Exception e) { + LOG.error("", e); + closeEarlyAndQuietly(key.channel()); + } + } + + selectionKeys.clear(); + } + + + } + + + private DatagramChannel connectClient() throws IOException { + final DatagramChannel client = address.isMulticastAddress() ? + DatagramChannel.open(address.getAddress().length == 4 ? INET : INET6) : + DatagramChannel.open(); + + final InetSocketAddress hostAddress = new InetSocketAddress(port); + client.configureBlocking(false); + synchronized (closeables) { + + if (address.isMulticastAddress()) { + client.setOption(IP_MULTICAST_IF, networkInterface); + client.setOption(SO_REUSEADDR, true); + client.bind(hostAddress); + client.join(address, networkInterface); + if (LOG.isDebugEnabled()) + LOG.debug("Connecting via multicast, group=" + address); + } else { + client.bind(hostAddress); + } + + if (LOG.isDebugEnabled()) + LOG.debug("Listening on port " + port); + closeables.add(client); + } + return client; + } + + /** + * called whenever there is a change to the modification iterator + */ + @Override + public void onChange() { + // the write have to be enabled on the same thread as the selector + shouldEnableOpWrite = true; + selector.wakeup(); + } + + private void enableWrites() { + + if (writeChannel == null) + return; + + try { + final SelectionKey selectionKey = writeChannel.keyFor(this.selector); + if (selectionKey != null) + selectionKey.interestOps(selectionKey.interestOps() | OP_WRITE); + } catch (Exception e) { + LOG.error("", e); + } + } + + private void disableWrites() { + + if (writeChannel == null) + return; + + try { + final SelectionKey selectionKey = writeChannel.keyFor(this.selector); + selectionKey.interestOps(selectionKey.interestOps() & ~OP_WRITE); + } catch (Exception e) { + LOG.error("", e); + } + } + + private static class UdpSocketChannelEntryReader { + + private final Replica.EntryExternalizable externalizable; + private final ByteBuffer in; + private final ByteBufferBytes out; + + /** + * @param serializedEntrySize the maximum size of an entry include the meta data + * @param externalizable supports reading and writing serialize entries + */ + UdpSocketChannelEntryReader(final int serializedEntrySize, + @NotNull final Replica.EntryExternalizable externalizable) { + // we make the buffer twice as large just to give ourselves headroom + in = ByteBuffer.allocateDirect(serializedEntrySize * 2); + this.externalizable = externalizable; + out = new ByteBufferBytes(in); + out.limit(0); + in.clear(); + } + + /** + * reads entries from the socket till it is empty + * + * @param socketChannel the socketChannel that we will read from + * @throws IOException + * @throws InterruptedException + */ + void readAll(@NotNull final DatagramChannel socketChannel) throws IOException, InterruptedException { + + out.clear(); + in.clear(); + + socketChannel.receive(in); + + final int bytesRead = in.position(); + + if (bytesRead < SIZE_OF_SHORT + SIZE_OF_SHORT) + return; + + out.limit(in.position()); + + final short invertedSize = out.readShort(); + final int size = out.readUnsignedShort(); + + // check the the first 4 bytes are the inverted len followed by the len + // we do this to check that this is a valid start of entry, otherwise we throw it away + if (((short) ~size) != invertedSize) + return; + + if (out.remaining() != size) + return; + + externalizable.readExternalEntry(out); + } + + } + + private class UdpSocketChannelEntryWriter { + + private final ByteBuffer out; + private final ByteBufferBytes in; + private final EntryCallback entryCallback; + + UdpSocketChannelEntryWriter(final int serializedEntrySize, + @NotNull final Replica.EntryExternalizable externalizable) { + + // we make the buffer twice as large just to give ourselves headroom + out = ByteBuffer.allocateDirect(serializedEntrySize * 2); + in = new ByteBufferBytes(out); + entryCallback = new EntryCallback(externalizable, in); + + } + + /** + * writes all the entries that have changed, to the tcp socket + * + * @param socketChannel + * @param modificationIterator + * @throws InterruptedException + * @throws java.io.IOException + */ + + /** + * update that are throttled are rejected. + * + * @param socketChannel the socketChannel that we will write to + * @throws InterruptedException + * @throws IOException + */ + int writeAll(@NotNull final DatagramChannel socketChannel) + throws InterruptedException, IOException { + + out.clear(); + in.clear(); + in.skip(SIZE_OF_SHORT); + + final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0); + + if (!wasDataRead) { + disableWrites(); + return 0; + } + + // we'll write the size inverted at the start + in.writeShort(0, ~(in.readUnsignedShort(SIZE_OF_SHORT))); + out.limit((int) in.position()); + + return socketChannel.write(out); + + } + } + + private class ServerConnector extends AbstractChannelReplicator.AbstractConnector { + private final InetSocketAddress socketAddress; + + private ServerConnector() { + super("UDP-Connector"); + this.socketAddress = new InetSocketAddress(address, port); + } + + SelectableChannel doConnect() throws + IOException, InterruptedException { + + final DatagramChannel server = DatagramChannel.open(); + server.configureBlocking(false); + + // Kick off connection establishment + try { + // Create a non-blocking socket channel + server.socket().setBroadcast(true); + server.connect(socketAddress); + } catch (IOException e) { + if (LOG.isDebugEnabled()) + LOG.debug("details=" + new Details(socketAddress, localIdentifier), e); + connectLater(); + return null; + } + + server.setOption(SO_REUSEADDR, true) + .setOption(StandardSocketOptions.IP_MULTICAST_LOOP, false) + .setOption(StandardSocketOptions.SO_BROADCAST, true) + .setOption(SO_REUSEADDR, true); + + // the registration has be be run on the same thread as the selector + addPendingRegistration(new Runnable() { + @Override + public void run() { + + try { + server.register(selector, OP_WRITE); + writeChannel = server; + throttle(server); + } catch (ClosedChannelException e) { + LOG.error("", e); + } + + } + }); + + return server; + } + + + } + + +} + + + + + diff --git a/src/main/java/net/openhft/collections/VanillaSharedHashMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java similarity index 83% rename from src/main/java/net/openhft/collections/VanillaSharedHashMap.java rename to src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 4f360abf2..c4a24be82 100644 --- a/src/main/java/net/openhft/collections/VanillaSharedHashMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -1,11 +1,11 @@ /* - * Copyright 2013 Peter Lawrey + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,137 +14,140 @@ * limitations under the License. */ -package net.openhft.collections; +package net.openhft.chronicle.map; -import net.openhft.chronicle.map.ChronicleMap; import net.openhft.lang.Maths; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.io.serialization.BytesMarshaller; +import net.openhft.lang.io.serialization.ObjectFactory; import net.openhft.lang.io.serialization.ObjectSerializer; import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; import net.openhft.lang.model.Byteable; -import net.openhft.lang.model.DataValueClasses; -import net.openhft.lang.model.constraints.NotNull; -import net.openhft.lang.model.constraints.Nullable; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; import java.lang.reflect.Array; import java.nio.channels.FileChannel; import java.util.*; +import java.util.concurrent.TimeUnit; import static java.lang.Thread.currentThread; -import static net.openhft.collections.AbstractVanillaSharedHashMap.Hasher.hash; +import static net.openhft.chronicle.map.VanillaChronicleMap.Hasher.hash; -class VanillaSharedHashMap extends AbstractVanillaSharedHashMap { - - public VanillaSharedHashMap(SharedHashMapBuilder builder, File file, - Class kClass, Class vClass) throws IOException { - super(builder, kClass, vClass); - ObjectSerializer objectSerializer = builder.objectSerializer(); - BytesStore bytesStore = file == null - ? DirectStore.allocateLazy(sizeInBytes(), objectSerializer) - : new MappedStore(file, FileChannel.MapMode.READ_WRITE, sizeInBytes(), objectSerializer); - createMappedStoreAndSegments(bytesStore); - } -} - -abstract class AbstractVanillaSharedHashMap extends AbstractMap - implements ChronicleMap { - - private static final Logger LOG = LoggerFactory.getLogger(AbstractVanillaSharedHashMap.class); +class VanillaChronicleMap extends AbstractMap + implements ChronicleMap, Serializable { + private static final long serialVersionUID = 0L; + private static final Logger LOG = LoggerFactory.getLogger(VanillaChronicleMap.class); /** * Because DirectBitSet implementations couldn't find more than 64 continuous clear or set bits. */ private static final int MAX_ENTRY_OVERSIZE_FACTOR = 64; -/* private final ObjectSerializer objectSerializer; - private SharedHashMapBuilder builder;*/ - - private static int figureBufferAllocationFactor(SharedHashMapBuilder builder) { - // if expected map size is about 1000, seems rather wasteful to allocate - // key and value serialization buffers each x64 of expected entry size.. - return (int) Math.min(Math.max(2L, builder.entries() >> 10), - MAX_ENTRY_OVERSIZE_FACTOR); - } - - private final int bufferAllocationFactor; - - private final ThreadLocal localBufferForKeys = - new ThreadLocal(); - private final ThreadLocal localBufferForValues = - new ThreadLocal(); - + private static final int SEGMENT_HEADER = 64; final Class kClass; - final Class vClass; - private final long lockTimeOutNS; + final BytesMarshaller keyMarshaller; + final BytesMarshaller valueMarshaller; final int metaDataBytes; - Segment[] segments; // non-final for close() - // non-final for close() and because it is initialized out of constructor - BytesStore ms; final Hasher hasher; - // private final int replicas; final int entrySize; final Alignment alignment; + final int actualSegments; final int entriesPerSegment; - - private final SharedMapErrorListener errorListener; - - /** - * Non-final because could be changed in VanillaSharedReplicatedHashMap. - */ - volatile SharedMapEventListener> eventListener; - - private final boolean generatedKeyType; - final boolean generatedValueType; - + final MapEventListener> eventListener; // if set the ReturnsNull fields will cause some functions to return NULL // rather than as returning the Object can be expensive for something you probably don't use. final boolean putReturnsNull; final boolean removeReturnsNull; - + private final int bufferAllocationFactor; + private final ObjectSerializer objectSerializer; + private final Class vClass; + private final ObjectFactory valueFactory; + private final long lockTimeOutNS; + private final MapErrorListener errorListener; + transient Segment[] segments; // non-final for close() + // non-final for close() and because it is initialized out of constructor + transient BytesStore ms; + transient long headerSize; transient Set> entrySet; + private transient ThreadLocal localBufferForKeys; + private transient ThreadLocal localBufferForValues; - - public AbstractVanillaSharedHashMap(SharedHashMapBuilder builder, - Class kClass, Class vClass) throws IOException { - // this.builder = builder.clone(); + public VanillaChronicleMap(ChronicleMapBuilder builder) + throws IOException { bufferAllocationFactor = figureBufferAllocationFactor(builder); - this.kClass = kClass; - this.vClass = vClass; + this.kClass = builder.keyClass; + this.keyMarshaller = builder.keyMarshaller(); + this.vClass = builder.valueClass; + this.valueMarshaller = builder.valueMarshaller(); + this.valueFactory = builder.valueFactory(); - lockTimeOutNS = builder.lockTimeOutMS() * 1000000; + lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); // this.replicas = builder.replicas(); this.entrySize = builder.alignedEntrySize(); this.alignment = builder.entryAndValueAlignment(); this.errorListener = builder.errorListener(); - this.generatedKeyType = builder.generatedKeyType(); - this.generatedValueType = builder.generatedValueType(); this.putReturnsNull = builder.putReturnsNull(); this.removeReturnsNull = builder.removeReturnsNull(); - // this.objectSerializer = builder.objectSerializer(); + this.objectSerializer = builder.objectSerializer(); - int segments = builder.actualSegments(); - int entriesPerSegment = builder.actualEntriesPerSegment(); - this.entriesPerSegment = entriesPerSegment; + this.actualSegments = builder.actualSegments(); + this.entriesPerSegment = builder.actualEntriesPerSegment(); this.metaDataBytes = builder.metaDataBytes(); this.eventListener = builder.eventListener(); int hashMask = useSmallMultiMaps() ? 0xFFFF : ~0; - this.hasher = new Hasher(segments, hashMask); + this.hasher = new Hasher(actualSegments, hashMask); + initTransients(); + } + + private static int figureBufferAllocationFactor(ChronicleMapBuilder builder) { + // if expected map size is about 1000, seems rather wasteful to allocate + // key and value serialization buffers each x64 of expected entry size.. + return (int) Math.min(Math.max(2L, builder.entries() >> 10), + MAX_ENTRY_OVERSIZE_FACTOR); + } + + /** + * @param size positive number + * @return number of bytes taken by {@link net.openhft.lang.io.AbstractBytes#writeStopBit(long)} applied + * to {@code size} + */ + static int expectedStopBits(long size) { + if (size <= 127) + return 1; + // numberOfLeadingZeros is cheap intrinsic on modern CPUs + // integral division is not... but there is no choice + return ((70 - Long.numberOfLeadingZeros(size)) / 7); + } + + /** + * Cache line alignment, assuming 64-byte cache lines. + */ + static long align64(long l) { + return (l + 63) & ~63; + } + + void initTransients() { @SuppressWarnings("unchecked") - Segment[] ss = (Segment[]) Array.newInstance(segmentType(), segments); + Segment[] ss = (Segment[]) Array.newInstance(segmentType(), actualSegments); this.segments = ss; + localBufferForKeys = new ThreadLocal(); + localBufferForValues = new ThreadLocal(); } Class segmentType() { @@ -165,6 +168,16 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap return offset; } + long createMappedStoreAndSegments(File file) throws IOException { + return createMappedStoreAndSegments(new MappedStore(file, FileChannel.MapMode.READ_WRITE, + sizeInBytes(), objectSerializer)); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + initTransients(); + } + /** * called when the header is created */ @@ -172,8 +185,8 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } - int getHeaderSize() { - return SharedHashMapBuilder.HEADER_SIZE; + long getHeaderSize() { + return headerSize; } Segment createSegment(NativeBytes bytes, int index) { @@ -185,36 +198,22 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap return ms.file(); } - /** - * @param size positive number - * @return number of bytes taken by {@link net.openhft.lang.io.AbstractBytes#writeStopBit(long)} applied - * to {@code size} - */ - static int expectedStopBits(long size) { - if (size <= 127) - return 1; - // numberOfLeadingZeros is cheap intrinsic on modern CPUs - // integral division is not... but there is no choice - return ((70 - Long.numberOfLeadingZeros(size)) / 7); - } - - long sizeInBytes() { return getHeaderSize() + segments.length * segmentSize(); } - long sizeOfMultiMap() { - int np2 = Maths.nextPower2(entriesPerSegment, 8); - return align64(np2 * (entriesPerSegment > (1 << 16) ? 8L : 4L)); + return useSmallMultiMaps() ? + VanillaShortShortMultiMap.sizeInBytes(entriesPerSegment) : + VanillaIntIntMultiMap.sizeInBytes(entriesPerSegment); } - long sizeOfMultiMapBitSet(long numberOfBits) { - long numberOfBytes = (numberOfBits + 7) / 8; - return align64(numberOfBytes); + long sizeOfMultiMapBitSet() { + return useSmallMultiMaps() ? + VanillaShortShortMultiMap.sizeOfBitSetInBytes(entriesPerSegment) : + VanillaIntIntMultiMap.sizeOfBitSetInBytes(entriesPerSegment); } - boolean useSmallMultiMaps() { return entriesPerSegment <= (1 << 16); } @@ -230,9 +229,8 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } long segmentSize() { - long sizeOfMultiMap = sizeOfMultiMap(); - long ss = SharedHashMapBuilder.SEGMENT_HEADER - + (sizeOfMultiMap + sizeOfMultiMapBitSet(sizeOfMultiMap)) * multiMapsPerSegment() + long ss = SEGMENT_HEADER + + align64(sizeOfMultiMap() + sizeOfMultiMapBitSet()) * multiMapsPerSegment() + numberOfBitSets() * sizeOfBitSets() // the free list and 0+ dirty lists. + sizeOfEntriesInSegment(); if ((ss & 63) != 0) @@ -264,16 +262,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap return align64((long) entriesPerSegment * entrySize); } - /** - * Cache line alignment, assuming 64-byte cache lines. - */ - static long align64(long l) { - return (l + 63) & ~63; - } - - /** - * {@inheritDoc} - */ @Override public void close() { if (ms == null) @@ -283,6 +271,10 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap ms = null; } + MapEventListener> eventListener() { + return eventListener; + } + private DirectBytes acquireBufferForKey() { DirectBytes buffer = localBufferForKeys.get(); if (buffer == null) { @@ -322,17 +314,11 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } } - /** - * {@inheritDoc} - */ @Override public V put(K key, V value) { return put0(key, value, true); } - /** - * {@inheritDoc} - */ @Override public V putIfAbsent(K key, V value) { return put0(key, value, false); @@ -350,10 +336,7 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap DirectBytes getKeyAsBytes(K key) { DirectBytes buffer = acquireBufferForKey(); - if (generatedKeyType) - ((BytesMarshallable) key).writeMarshallable(buffer); - else - buffer.writeInstance(kClass, key); + keyMarshaller.write(buffer, key); buffer.flip(); return buffer; } @@ -361,17 +344,11 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap DirectBytes getValueAsBytes(V value) { DirectBytes buffer = acquireBufferForValue(); buffer.clear(); - if (generatedValueType) - ((BytesMarshallable) value).writeMarshallable(buffer); - else - buffer.writeInstance(vClass, value); + valueMarshaller.write(buffer, value); buffer.flip(); return buffer; } - /** - * {@inheritDoc} - */ @Override public V get(Object key) { return lookupUsing((K) key, null, false); @@ -396,9 +373,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap return segments[segmentNum].acquire(keyBytes, key, value, segmentHash, create); } - /** - * {@inheritDoc} - */ @Override public boolean containsKey(final Object key) { checkKey(key); @@ -416,9 +390,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap segment.clear(); } - /** - * {@inheritDoc} - */ @NotNull @Override public Set> entrySet() { @@ -427,8 +398,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap /** - * {@inheritDoc} - * * @throws NullPointerException if the specified key is null */ @Override @@ -437,8 +406,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } /** - * {@inheritDoc} - * * @throws NullPointerException if the specified key is null */ @Override @@ -468,8 +435,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } /** - * {@inheritDoc} - * * @throws NullPointerException if any of the arguments are null */ @Override @@ -480,10 +445,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap /** - * {@inheritDoc} - * - * @return the previous value associated with the specified key, or null if there was no mapping - * for the key * @throws NullPointerException if the specified key or value is null */ @Override @@ -517,7 +478,7 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap * @param newValue the new value you wish to store in the map * @return the value that was replaced */ - V replaceIfValueIs(@net.openhft.lang.model.constraints.NotNull final K key, final V existingValue, final V newValue) { + V replaceIfValueIs(@NotNull final K key, final V existingValue, final V newValue) { checkKey(key); checkValue(newValue); Bytes keyBytes = getKeyAsBytes(key); @@ -537,7 +498,17 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } - static final class Hasher { + static final class Hasher implements Serializable { + private static final long serialVersionUID = 0L; + private final int segments; + private final int bits; + private final int mask; + + Hasher(int segments, int mask) { + this.segments = segments; + this.bits = Maths.intLog2(segments); + this.mask = mask; + } static long hash(Bytes bytes) { long h = 0; @@ -555,16 +526,6 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap return h; } - private final int segments; - private final int bits; - private final int mask; - - Hasher(int segments, int mask) { - this.segments = segments; - this.bits = Maths.intLog2(segments); - this.mask = mask; - } - int segmentHash(long hash) { return (int) (hash >>> bits) & mask; } @@ -588,13 +549,13 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap static final int PAD1_OFFSET = SIZE_OFFSET + 4; // 32-bit static final int REPLICA_OFFSET = PAD1_OFFSET + 4; // 64-bit - private final NativeBytes bytes; - private final int index; + final NativeBytes bytes; final MultiStoreBytes tmpBytes = new MultiStoreBytes(); - private IntIntMultiMap hashLookup; - private final SingleThreadedDirectBitSet freeList; - private int nextPosToSearchFrom = 0; final long entriesOffset; + private final int index; + private final SingleThreadedDirectBitSet freeList; + private IntIntMultiMap hashLookup; + private int nextPosToSearchFrom = 0; /** @@ -605,10 +566,9 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap this.bytes = bytes; this.index = index; - long start = bytes.startAddr() + SharedHashMapBuilder.SEGMENT_HEADER; + long start = bytes.startAddr() + SEGMENT_HEADER; createHashLookups(start); - long sizeOfMultiMap = sizeOfMultiMap(); - start += (sizeOfMultiMap + sizeOfMultiMapBitSet(sizeOfMultiMap)) * multiMapsPerSegment(); + start += align64(sizeOfMultiMap() + sizeOfMultiMapBitSet()) * multiMapsPerSegment(); final NativeBytes bsBytes = new NativeBytes( tmpBytes.objectSerializer(), start, start + sizeOfBitSets(), null); freeList = new SingleThreadedDirectBitSet(bsBytes); @@ -627,14 +587,13 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap IntIntMultiMap createMultiMap(long start) { - long sizeOfMultiMap = sizeOfMultiMap(); final NativeBytes multiMapBytes = new NativeBytes(new VanillaBytesMarshallerFactory(), start, - start = start + sizeOfMultiMap, null); + start = start + sizeOfMultiMap(), null); final NativeBytes sizeOfMultiMapBitSetBytes = new NativeBytes(new VanillaBytesMarshallerFactory(), start, - start + sizeOfMultiMapBitSet(sizeOfMultiMap), null); + start + sizeOfMultiMapBitSet(), null); multiMapBytes.load(); return useSmallMultiMaps() ? new VanillaShortShortMultiMap(multiMapBytes, sizeOfMultiMapBitSetBytes) : @@ -809,14 +768,10 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap if (usingValue != null) { return usingValue; } else { - if (generatedValueType) - return DataValueClasses.newDirectReference(vClass); - else { - try { - return vClass.newInstance(); - } catch (Exception e) { - throw new AssertionError(e); - } + try { + return valueFactory.create(); + } catch (Exception e) { + throw new IllegalStateException(e); } } } else { @@ -987,17 +942,8 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap * @param value the object to reuse (if possible), if {@code null} a new object is created */ V readValue(NativeBytes entry, V value, long valueLen) { - if (generatedValueType) - if (value == null) - value = DataValueClasses.newDirectReference(vClass); - else - assert value instanceof Byteable; - if (value instanceof Byteable) { - long valueOffset = entry.positionAddr() - bytes.address(); - ((Byteable) value).bytes(bytes, valueOffset); - return value; - } - return entry.readInstance(vClass, value); + bytes.positionAddr(entry.positionAddr()); + return valueMarshaller.read(bytes, value); } boolean keyEquals(Bytes keyBytes, long keyLen, Bytes entry) { @@ -1031,7 +977,7 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap ? readValue(entry, null, valueLen) : null; if (expectedValue != null && !expectedValue.equals(valueRemoved)) return null; - hashLookup.remove(hashLookup.getSearchHash(), pos); + hashLookup.removePrevPos(); decrementSize(); free(pos, inBlocks(entryEndAddr - entryStartAddr(offset))); notifyRemoved(offset, key, valueRemoved, pos); @@ -1116,33 +1062,33 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap void notifyPut(long offset, boolean added, K key, V value, final long pos) { - if (eventListener != SharedMapEventListeners.NOP) { + if (eventListener() != MapEventListeners.NOP) { tmpBytes.storePositionAndSize(bytes, offset, entrySize); - eventListener.onPut(AbstractVanillaSharedHashMap.this, tmpBytes, metaDataBytes, + eventListener().onPut(VanillaChronicleMap.this, tmpBytes, metaDataBytes, added, key, value, pos, this); } } void notifyGet(long offset, K key, V value) { - if (eventListener != SharedMapEventListeners.NOP) { + if (eventListener() != MapEventListeners.NOP) { tmpBytes.storePositionAndSize(bytes, offset, entrySize); - eventListener.onGetFound(AbstractVanillaSharedHashMap.this, tmpBytes, metaDataBytes, + eventListener().onGetFound(VanillaChronicleMap.this, tmpBytes, metaDataBytes, key, value); } } V notifyMissed(Bytes keyBytes, K key, V usingValue) { - if (eventListener != SharedMapEventListeners.NOP) { - return eventListener.onGetMissing(AbstractVanillaSharedHashMap.this, keyBytes, + if (eventListener() != MapEventListeners.NOP) { + return eventListener().onGetMissing(VanillaChronicleMap.this, keyBytes, key, usingValue); } return null; } void notifyRemoved(long offset, K key, V value, final int pos) { - if (eventListener != SharedMapEventListeners.NOP) { + if (eventListener() != MapEventListeners.NOP) { tmpBytes.storePositionAndSize(bytes, offset, entrySize); - eventListener.onRemove(AbstractVanillaSharedHashMap.this, tmpBytes, metaDataBytes, + eventListener().onRemove(VanillaChronicleMap.this, tmpBytes, metaDataBytes, key, value, pos, this); } } @@ -1225,7 +1171,7 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap break newValueDoesNotFit; // RELOCATION free(pos, oldSizeInBlocks); - eventListener.onRelocation(pos, this); + eventListener().onRelocation(pos, this); int prevPos = pos; pos = alloc(newSizeInBlocks); // putValue() is called from put() and replace() @@ -1286,20 +1232,13 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } } - void visit(IntIntMultiMap.EntryConsumer entryConsumer) { - hashLookup.forEach(entryConsumer); - } - public Entry getEntry(long pos) { - long offset = offsetFromPos(pos); - NativeBytes entry = entry(offset); - entry.readStopBit(); - K key = entry.readInstance(kClass, null); //todo: readUsing? - - V value = readValue(entry, null); //todo: reusable container - - //notifyGet(offset - metaDataBytes, key, value); //todo: should we call this? - + bytes.position(offsetFromPos(pos) + metaDataBytes); + bytes.readStopBit(); + K key = keyMarshaller.read(bytes); + bytes.readStopBit(); + alignment.alignPositionAddr(bytes); + V value = valueMarshaller.read(bytes); return new WriteThroughEntry(key, value); } @@ -1354,111 +1293,118 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } } - - final class EntryIterator implements Iterator> { - - boolean wasNextCalledSinceLastRemove = false; - private int seg; - private int pos; + class EntryIterator implements Iterator> { + Entry returnedEntry; + private int returnedSeg = -1; + private long returnedPos = -1L; + private int nextSeg; + private long nextPos; public EntryIterator() { - int pos1 = -1; - this.seg = segments.length - 1; - this.pos = pos1; + advance(nextSeg = segments.length - 1, nextPos = -1L); + } + + private boolean advance(int segIndex, long pos) { + while (segIndex >= 0) { + pos = segments[segIndex].getHashLookup().getPositions().nextSetBit(pos + 1L); + if (pos >= 0L) { + nextSeg = segIndex; + nextPos = pos; + return true; + } else { + segIndex--; + pos = -1L; + } + } + nextSeg = -1; + nextPos = -1L; + return false; } @Override - public synchronized boolean hasNext() { - int pos = this.pos; - int segIndex = seg; - - for (; ; ) { - - if (segIndex < 0) - return false; - - if (segments[segIndex].getHashLookup().getPositions().nextSetBit(pos + 1) != -1) - return true; - - segIndex--; - pos = -1; - } - + public boolean hasNext() { + return nextSeg >= 0; } @Override public Entry next() { - int pos = this.pos; - int segIndex = seg; - for (; ; ) { - + int segIndex = nextSeg; + long pos = nextPos; if (segIndex < 0) throw new NoSuchElementException(); - final Segment segment = segments[segIndex]; - try { segment.lock(); - - if ((pos = (int) segment.getHashLookup().getPositions().nextSetBit(pos + 1)) == -1) { - segIndex--; - pos = -1; + if (segment.getHashLookup().getPositions().isClear(pos)) { + // the pos was removed after the previous advance + advance(segIndex, pos); continue; } - - this.wasNextCalledSinceLastRemove = true; - this.seg = segIndex; - this.pos = pos; - return segment.getEntry(pos); - + advance(returnedSeg = segIndex, returnedPos = pos); + return returnedEntry = segment.getEntry(pos); } finally { segment.unlock(); } - } } - @Override public void remove() { - - if (!wasNextCalledSinceLastRemove) + int segIndex = returnedSeg; + if (segIndex < 0) throw new IllegalStateException(); - - - int pos = this.pos; - int segIndex = seg; - final Segment segment = segments[segIndex]; - + final int pos = (int) returnedPos; try { segment.lock(); - - final long offset = segment.offsetFromPos(pos); - final NativeBytes entry = segment.entry(offset); - final long limit = entry.limit(); - - final long keyLen = entry.readStopBit(); - entry.limit(entry.position() + keyLen); - - final int segmentHash = hasher.segmentHash(hash(entry)); - entry.limit(limit); - - final long entryEndAddr = entry.positionAddr() + segment.readValueLen(entry); - segment.decrementSize(); - segment.free(pos, segment.inBlocks(entryEndAddr - segment.entryStartAddr(offset))); - - segment.getHashLookup().remove(segmentHash, pos); - wasNextCalledSinceLastRemove = false; + if (segment.getHashLookup().getPositions().isClear(pos)) { + // The case: + // 1. iterator.next() - thread 1 + // 2. map.put() which cause relocation of the key, returned in above - thread 2 + // OR map.remove() which remove this key - thread 2 + // 3. iterator.remove() - thread 1 + VanillaChronicleMap.this.remove(returnedEntry.getKey()); + } else { + removePresent(segment, pos); + } + returnedSeg = -1; + returnedEntry = null; } finally { segment.unlock(); } + } + void removePresent(Segment segment, int pos) { + // TODO handle the case: + // iterator.next() -- thread 1 + // map.put() which cause relocation of the key, returned above -- thread 2 + // map.put() which place a new key on the `pos` in current segment -- thread 3 + // iterator.remove() -- thread 1 + // The simple solution is to compare bytes in the map with the serialized bytes + // of returnedEntry.getKey(), but it seems rather wasteful to workaround so rare + // case. + final long offset = segment.offsetFromPos(pos); + final NativeBytes entry = segment.entry(offset); + + final long limit = entry.limit(); + final long keyLen = entry.readStopBit(); + entry.limit(entry.position() + keyLen); + final int segmentHash = hasher.segmentHash(hash(entry)); + entry.limit(limit); + + entry.skip(keyLen); + long valueLen = segment.readValueLen(entry); + final long entryEndAddr = entry.positionAddr() + valueLen; + segment.getHashLookup().remove(segmentHash, pos); + segment.decrementSize(); + segment.free(pos, segment.inBlocks(entryEndAddr - segment.entryStartAddr(offset))); + segment.notifyRemoved(offset, returnedEntry.getKey(), returnedEntry.getValue(), pos); } } - final class EntrySet extends AbstractSet> { + class EntrySet extends AbstractSet> { + @NotNull public Iterator> iterator() { return new EntryIterator(); } @@ -1468,7 +1414,7 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap return false; Map.Entry e = (Map.Entry) o; try { - V v = AbstractVanillaSharedHashMap.this.get(e.getKey()); + V v = VanillaChronicleMap.this.get(e.getKey()); return v != null && v.equals(e.getValue()); } catch (ClassCastException ex) { return false; @@ -1484,7 +1430,7 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap try { Object key = e.getKey(); Object value = e.getValue(); - return AbstractVanillaSharedHashMap.this.remove(key, value); + return VanillaChronicleMap.this.remove(key, value); } catch (ClassCastException ex) { return false; } catch (NullPointerException ex) { @@ -1493,15 +1439,15 @@ abstract class AbstractVanillaSharedHashMap extends AbstractMap } public int size() { - return AbstractVanillaSharedHashMap.this.size(); + return VanillaChronicleMap.this.size(); } public boolean isEmpty() { - return AbstractVanillaSharedHashMap.this.isEmpty(); + return VanillaChronicleMap.this.isEmpty(); } public void clear() { - AbstractVanillaSharedHashMap.this.clear(); + VanillaChronicleMap.this.clear(); } } diff --git a/src/main/java/net/openhft/chronicle/map/VanillaIntIntMultiMap.java b/src/main/java/net/openhft/chronicle/map/VanillaIntIntMultiMap.java new file mode 100644 index 000000000..7322bdb9b --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/VanillaIntIntMultiMap.java @@ -0,0 +1,313 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.Maths; +import net.openhft.lang.collection.ATSDirectBitSet; +import net.openhft.lang.collection.DirectBitSet; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.DirectStore; + +/** + * Supports a simple interface for int -> int[] off heap. + */ +class VanillaIntIntMultiMap implements IntIntMultiMap { + + private static final int ENTRY_SIZE = 8; + private static final int ENTRY_SIZE_SHIFT = 3; + private static final int UNSET_KEY = 0; + private static final int HASH_INSTEAD_OF_UNSET_KEY = -1; + private static final int UNSET_VALUE = Integer.MIN_VALUE; + /** + * hash is in 32 higher order bits, because in Intel's little-endian they are written first + * in memory, and in memory we have keys and values in natural order: 4 bytes of k1, 4 bytes + * of v1, 4 bytes of k2, ... and this is somehow compatible with previous version of this class, + * where keys were written before values explicitly. However, this layout increases latency + * of map operations by 1 clock cycle, because we always need to perform shift to obtain the key + * between memory read and comparison with UNSET_KEY. + */ + private static final long UNSET_ENTRY = Integer.MIN_VALUE & 0xFFFFFFFFL; + final DirectBitSet positions; + private final int capacity; + private final int capacityMask; + private final long capacityMask2; + private final Bytes bytes; + private int searchHash = -1; + private long searchPos = -1L; + + public VanillaIntIntMultiMap(int minCapacity) { + if (minCapacity < 0) + throw new IllegalArgumentException(); + capacity = Maths.nextPower2(minCapacity, 16); + capacityMask = capacity - 1; + capacityMask2 = indexToPos(capacity - 1); + bytes = DirectStore.allocateLazy(indexToPos(capacity)).bytes(); + positions = newPositions(capacity); + clear(); + } + + public VanillaIntIntMultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { + capacity = (int) (multiMapBytes.capacity() / ENTRY_SIZE); + assert capacity == Maths.nextPower2(capacity, 16); + capacityMask = capacity - 1; + capacityMask2 = indexToPos(capacity - 1); + this.bytes = multiMapBytes; + positions = new ATSDirectBitSet(multiMapBitSetBytes); + } + + /** + * @param minCapacity as in {@link #VanillaIntIntMultiMap(int)} constructor + * @return size of {@link Bytes} to provide to {@link #VanillaIntIntMultiMap(Bytes, Bytes)} + * constructor as the first argument + */ + public static long sizeInBytes(int minCapacity) { + return indexToPos(Maths.nextPower2(minCapacity, 16)); + } + + /** + * @param minCapacity as in {@link #VanillaIntIntMultiMap(int)} constructor + * @return size of {@link Bytes} to provide to {@link #VanillaIntIntMultiMap(Bytes, Bytes)} + * constructor as the second argument + */ + public static long sizeOfBitSetInBytes(int minCapacity) { + return Maths.nextPower2(minCapacity, 64L) / 8L; + } + + public static ATSDirectBitSet newPositions(int capacity) { + // bit set size should be at least 1 native long (in bits) + capacity = Math.max(capacity, 64); + // no round-up, because capacity is already a power of 2 + int bitSetSizeInBytes = capacity / 8; + return new ATSDirectBitSet(DirectStore.allocateLazy(bitSetSizeInBytes).bytes()); + } + + /** + * Separate method because it is too easy to forget to cast to long before shifting. + */ + private static long indexToPos(int index) { + return ((long) index) << ENTRY_SIZE_SHIFT; + } + + private static int maskUnsetKey(int key) { + return key != UNSET_KEY ? key : HASH_INSTEAD_OF_UNSET_KEY; + } + + @Override + public void put(int key, int value) { + key = maskUnsetKey(key); + long pos = indexToPos(key & capacityMask); + for (int i = 0; i <= capacityMask; i++) { + long entry = bytes.readLong(pos); + int hash2 = (int) (entry >> 32); + if (hash2 == UNSET_KEY) { + bytes.writeLong(pos, (((long) key) << 32) | (value & 0xFFFFFFFFL)); + positions.set(value); + return; + } + if (hash2 == key) { + int value2 = (int) entry; + if (value2 == value) + return; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + } + throw new IllegalStateException(getClass().getSimpleName() + " is full"); + } + + @Override + public boolean remove(int key, int value) { + key = maskUnsetKey(key); + long pos = indexToPos(key & capacityMask); + long posToRemove = -1L; + for (int i = 0; i <= capacityMask; i++) { + long entry = bytes.readLong(pos); + int hash2 = (int) (entry >> 32); + if (hash2 == key) { + int value2 = (int) entry; + if (value2 == value) { + posToRemove = pos; + break; + } + } else if (hash2 == UNSET_KEY) { + break; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + } + if (posToRemove < 0) + return false; + positions.clear(value); + removePos(posToRemove); + return true; + } + + ///////////////////// + // Stateful methods + + @Override + public boolean replace(int key, int oldValue, int newValue) { + key = maskUnsetKey(key); + long pos = indexToPos(key & capacityMask); + for (int i = 0; i <= capacityMask; i++) { + long entry = bytes.readLong(pos); + int hash2 = (int) (entry >> 32); + if (hash2 == key) { + int value2 = (int) entry; + if (value2 == oldValue) { + positions.clear(oldValue); + positions.set(newValue); + bytes.writeLong(pos, (((long) key) << 32) | (newValue & 0xFFFFFFFFL)); + return true; + } + } else if (hash2 == UNSET_KEY) { + break; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + } + return false; + } + + private void removePos(long posToRemove) { + long posToShift = posToRemove; + for (int i = 0; i <= capacityMask; i++) { + posToShift = (posToShift + ENTRY_SIZE) & capacityMask2; + long entryToShift = bytes.readLong(posToShift); + int hash = (int) (entryToShift >> 32); + if (hash == UNSET_KEY) + break; + long insertPos = indexToPos(hash & capacityMask); + // the following condition essentially means circular permutations + // of three (r = posToRemove, s = posToShift, i = insertPos) + // positions are accepted: + // [...i..r...s.] or + // [...r..s...i.] or + // [...s..i...r.] + boolean cond1 = insertPos <= posToRemove; + boolean cond2 = posToRemove <= posToShift; + if ((cond1 && cond2) || + // chain wrapped around capacity + (posToShift < insertPos && (cond1 || cond2))) { + bytes.writeLong(posToRemove, entryToShift); + posToRemove = posToShift; + } + } + bytes.writeLong(posToRemove, UNSET_ENTRY); + } + + @Override + public int startSearch(int key) { + key = maskUnsetKey(key); + searchPos = indexToPos(key & capacityMask); + return searchHash = key; + } + + @Override + public int nextPos() { + long pos = searchPos; + for (int i = 0; i < capacity; i++) { + long entry = bytes.readLong(pos); + int hash2 = (int) (entry >> 32); + if (hash2 == UNSET_KEY) { + searchPos = pos; + return UNSET_VALUE; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + if (hash2 == searchHash) { + searchPos = pos; + return (int) entry; + } + } + // if return UNSET_VALUE, we have 2 cases in putAfterFailedSearch() + throw new IllegalStateException(getClass().getSimpleName() + " is full"); + } + + @Override + public void removePrevPos() { + long prevPos = (searchPos - ENTRY_SIZE) & capacityMask2; + long entry = bytes.readLong(prevPos); + int value = (int) entry; + positions.clear(value); + removePos(prevPos); + } + + @Override + public void replacePrevPos(int newValue) { + long prevPos = ((searchPos - ENTRY_SIZE) & capacityMask2); + long oldEntry = bytes.readLong(prevPos); + int oldValue = (int) oldEntry; + positions.clear(oldValue); + positions.set(newValue); + // Don't need to overwrite searchHash, but we don't know our bytes + // byte order, and can't determine offset of the value within entry. + long newEntry = (((long) searchHash) << 32) | (newValue & 0xFFFFFFFFL); + bytes.writeLong(prevPos, newEntry); + } + + + @Override + public void putAfterFailedSearch(int value) { + positions.set(value); + long entry = (((long) searchHash) << 32) | (value & 0xFFFFFFFFL); + bytes.writeLong(searchPos, entry); + } + + public int getSearchHash() { + return searchHash; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{ "); + for (int i = 0, pos = 0; i < capacity; i++, pos += ENTRY_SIZE) { + long entry = bytes.readLong(pos); + int key = (int) (entry >> 32); + int value = (int) entry; + if (key != UNSET_KEY) + sb.append(key).append('=').append(value).append(", "); + } + if (sb.length() > 2) { + sb.setLength(sb.length() - 2); + return sb.append(" }").toString(); + } + return "{ }"; + } + + @Override + public void forEach(EntryConsumer action) { + for (int i = 0, pos = 0; i < capacity; i++, pos += ENTRY_SIZE) { + long entry = bytes.readLong(pos); + int key = (int) (entry >> 32); + int value = (int) entry; + if (key != UNSET_KEY) { + action.accept(key, value); + } + } + } + + @Override + public DirectBitSet getPositions() { + return positions; + } + + @Override + public void clear() { + positions.clear(); + for (int pos = 0; pos < bytes.capacity(); pos += ENTRY_SIZE) { + bytes.writeLong(pos, UNSET_ENTRY); + } + } +} diff --git a/src/main/java/net/openhft/chronicle/map/VanillaShortShortMultiMap.java b/src/main/java/net/openhft/chronicle/map/VanillaShortShortMultiMap.java new file mode 100644 index 000000000..36de37266 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/VanillaShortShortMultiMap.java @@ -0,0 +1,303 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import net.openhft.lang.Maths; +import net.openhft.lang.collection.ATSDirectBitSet; +import net.openhft.lang.collection.DirectBitSet; +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.DirectStore; + +/** + * Supports a simple interface for int -> int[] off heap. + */ +class VanillaShortShortMultiMap implements IntIntMultiMap { + + private static final int ENTRY_SIZE = 4; + private static final int ENTRY_SIZE_SHIFT = 2; + private static final int UNSET_KEY = 0; + private static final int HASH_INSTEAD_OF_UNSET_KEY = 0xFFFF; + private static final int UNSET_VALUE = Integer.MIN_VALUE; + private static final int UNSET_ENTRY = 0xFFFF; + private final int capacity; + private final int capacityMask; + private final int capacityMask2; + private final Bytes bytes; + private ATSDirectBitSet positions; + private int searchHash = -1; + private int searchPos = -1; + + public VanillaShortShortMultiMap(int minCapacity) { + if (minCapacity < 0 || minCapacity > (1 << 16)) + throw new IllegalArgumentException(); + capacity = Maths.nextPower2(minCapacity, 16); + capacityMask = capacity - 1; + capacityMask2 = (capacity - 1) * ENTRY_SIZE; + bytes = DirectStore.allocateLazy(capacity * ENTRY_SIZE).bytes(); + positions = VanillaIntIntMultiMap.newPositions(capacity); + clear(); + } + + public VanillaShortShortMultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { + capacity = (int) (multiMapBytes.capacity() / ENTRY_SIZE); + assert capacity == Maths.nextPower2(capacity, 16); + capacityMask = capacity - 1; + capacityMask2 = (capacity - 1) * ENTRY_SIZE; + this.bytes = multiMapBytes; + positions = new ATSDirectBitSet(multiMapBitSetBytes); + } + + /** + * @param minCapacity as in {@link #VanillaShortShortMultiMap(int)} constructor + * @return size of {@link Bytes} to provide to {@link #VanillaShortShortMultiMap(Bytes, Bytes)} + * constructor as the first argument + */ + public static long sizeInBytes(int minCapacity) { + return Maths.nextPower2(minCapacity, 16L) * ENTRY_SIZE; + } + + /** + * @param minCapacity as in {@link #VanillaShortShortMultiMap(int)} constructor + * @return size of {@link Bytes} to provide to {@link #VanillaShortShortMultiMap(Bytes, Bytes)} + * constructor as the second argument + */ + public static long sizeOfBitSetInBytes(int minCapacity) { + return VanillaIntIntMultiMap.sizeOfBitSetInBytes(minCapacity); + } + + private static void checkKey(int key) { + if ((key & ~0xFFFF) != 0) + throw new IllegalArgumentException("Key out of range, was " + key); + } + + private static void checkValue(int value) { + if ((value & ~0xFFFF) != 0) + throw new IllegalArgumentException("Value out of range, was " + value); + } + + private static int checkAndMaskUnsetKey(int key) { + if (key == UNSET_KEY) { + return HASH_INSTEAD_OF_UNSET_KEY; + } else { + checkKey(key); + return key; + } + } + + @Override + public void put(int key, int value) { + key = checkAndMaskUnsetKey(key); + checkValue(value); + int pos = (key & capacityMask) << ENTRY_SIZE_SHIFT; + for (int i = 0; i <= capacityMask; i++) { + int entry = bytes.readInt(pos); + int hash2 = entry >>> 16; + if (hash2 == UNSET_KEY) { + bytes.writeInt(pos, ((key << 16) | value)); + positions.set(value); + return; + } + if (hash2 == key) { + int value2 = entry & 0xFFFF; + if (value2 == value) + return; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + } + throw new IllegalStateException(getClass().getSimpleName() + " is full"); + } + + @Override + public boolean remove(int key, int value) { + key = checkAndMaskUnsetKey(key); + checkValue(value); + int pos = (key & capacityMask) << ENTRY_SIZE_SHIFT; + int posToRemove = -1; + for (int i = 0; i <= capacityMask; i++) { + int entry = bytes.readInt(pos); + int hash2 = entry >>> 16; + if (hash2 == key) { + int value2 = entry & 0xFFFF; + if (value2 == value) { + posToRemove = pos; + break; + } + } else if (hash2 == UNSET_KEY) { + break; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + } + if (posToRemove < 0) + return false; + positions.clear(value); + removePos(posToRemove); + return true; + } + + ///////////////////// + // Stateful methods + + @Override + public boolean replace(int key, int oldValue, int newValue) { + key = checkAndMaskUnsetKey(key); + checkValue(oldValue); + checkValue(newValue); + int pos = (key & capacityMask) << ENTRY_SIZE_SHIFT; + for (int i = 0; i <= capacityMask; i++) { + int entry = bytes.readInt(pos); + int hash2 = entry >>> 16; + if (hash2 == key) { + int value2 = entry & 0xFFFF; + if (value2 == oldValue) { + positions.clear(oldValue); + positions.set(newValue); + bytes.writeInt(pos, ((key << 16) | newValue)); + return true; + } + } else if (hash2 == UNSET_KEY) { + break; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + } + return false; + } + + private void removePos(int posToRemove) { + int posToShift = posToRemove; + for (int i = 0; i <= capacityMask; i++) { + posToShift = (posToShift + ENTRY_SIZE) & capacityMask2; + int entryToShift = bytes.readInt(posToShift); + int hash = entryToShift >>> 16; + if (hash == UNSET_KEY) + break; + int insertPos = (hash & capacityMask) << ENTRY_SIZE_SHIFT; + // see comment in VanillaIntIntMultiMap + boolean cond1 = insertPos <= posToRemove; + boolean cond2 = posToRemove <= posToShift; + if ((cond1 && cond2) || + // chain wrapped around capacity + (posToShift < insertPos && (cond1 || cond2))) { + bytes.writeInt(posToRemove, entryToShift); + posToRemove = posToShift; + } + } + bytes.writeInt(posToRemove, UNSET_ENTRY); + } + + @Override + public int startSearch(int key) { + key = checkAndMaskUnsetKey(key); + searchPos = (key & capacityMask) << ENTRY_SIZE_SHIFT; + return searchHash = key; + } + + @Override + public int nextPos() { + int pos = searchPos; + for (int i = 0; i < capacity; i++) { + int entry = bytes.readInt(pos); + int hash2 = entry >>> 16; + if (hash2 == UNSET_KEY) { + searchPos = pos; + return UNSET_VALUE; + } + pos = (pos + ENTRY_SIZE) & capacityMask2; + if (hash2 == searchHash) { + searchPos = pos; + return entry & 0xFFFF; + } + } + // if return UNSET_VALUE, we have 2 cases in putAfterFailedSearch() + throw new IllegalStateException(getClass().getSimpleName() + " is full"); + } + + @Override + public void removePrevPos() { + int prevPos = (searchPos - ENTRY_SIZE) & capacityMask2; + int entry = bytes.readInt(prevPos); + int value = entry & 0xFFFF; + positions.clear(value); + removePos(prevPos); + } + + @Override + public void replacePrevPos(int newValue) { + checkValue(newValue); + int prevPos = ((searchPos - ENTRY_SIZE) & capacityMask2); + int oldEntry = bytes.readInt(prevPos); + int oldValue = oldEntry & 0xFFFF; + positions.clear(oldValue); + positions.set(newValue); + // Don't need to overwrite searchHash, but we don't know our bytes + // byte order, and can't determine offset of the value within entry. + bytes.writeInt(prevPos, searchHash << 16 | newValue); + } + + @Override + public void putAfterFailedSearch(int value) { + checkValue(value); + positions.set(value); + bytes.writeInt(searchPos, searchHash << 16 | value); + } + + public int getSearchHash() { + return searchHash; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{ "); + for (int i = 0, pos = 0; i < capacity; i++, pos += ENTRY_SIZE) { + int entry = bytes.readInt(pos); + int key = entry >>> 16; + int value = entry & 0xFFFF; + if (key != UNSET_KEY) + sb.append(key).append('=').append(value).append(", "); + } + if (sb.length() > 2) { + sb.setLength(sb.length() - 2); + return sb.append(" }").toString(); + } + return "{ }"; + } + + @Override + public void forEach(EntryConsumer action) { + for (int i = 0, pos = 0; i < capacity; i++, pos += ENTRY_SIZE) { + int entry = bytes.readInt(pos); + int key = entry >>> 16; + int value = entry & 0xFFFF; + if (key != UNSET_KEY) + action.accept(key, value); + } + } + + @Override + public DirectBitSet getPositions() { + return positions; + } + + @Override + public void clear() { + positions.clear(); + for (int pos = 0; pos < bytes.capacity(); pos += ENTRY_SIZE) { + bytes.writeInt(pos, UNSET_ENTRY); + } + } +} diff --git a/src/main/java/net/openhft/collections/ClusterReplicator.java b/src/main/java/net/openhft/collections/ClusterReplicator.java deleted file mode 100644 index aa90707e3..000000000 --- a/src/main/java/net/openhft/collections/ClusterReplicator.java +++ /dev/null @@ -1,412 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.collections; - -import net.openhft.lang.collection.DirectBitSet; -import net.openhft.lang.collection.SingleThreadedDirectBitSet; -import net.openhft.lang.io.AbstractBytes; -import net.openhft.lang.io.ByteBufferBytes; -import net.openhft.lang.io.Bytes; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReferenceArray; - -import static java.lang.Math.min; -import static java.nio.ByteBuffer.wrap; -import static net.openhft.collections.Replica.ModificationNotifier.NOP; - -/** - * @author Rob Austin. - */ -class ClusterReplicator implements ReplicaExternalizable, Closeable { - - public static final int WRITE_BUFFER_SIZE = 1024; - private final byte localIdentifer; - - interface MessageHandler { - void onMessage(Bytes bytes); - } - - private static final Logger LOG = LoggerFactory.getLogger(ClusterReplicator.class.getName()); - - private final byte BOOTSTAP_MESSAGE = 'B'; - - private final DirectBitSet chronicalChannelBitSet; - private final ReplicaExternalizable chronicalChannels[]; - - private final AtomicReferenceArray systemModificationIterator = new - AtomicReferenceArray(128); - - private final DirectBitSet systemModificationIteratorBitSet = newBitSet(systemModificationIterator.length()); - - private final AtomicReferenceArray modificationIterator = new - AtomicReferenceArray(128); - - private final Set replicators = new CopyOnWriteArraySet(); - - /** - * handles a message received via the SystemQueue - */ - private final MessageHandler systemMessageHandler = new MessageHandler() { - - @Override - public void onMessage(Bytes bytes) { - final byte type = bytes.readByte(); - - if (type == BOOTSTAP_MESSAGE) { - onBootstrapMessage(bytes); - } else { - LOG.info("message of type=" + type + " was ignored."); - } - - } - - }; - - /** - * used to send an receive system messages - */ - private final SystemQueue systemMessageQueue; - - public ClusterReplicator(final byte identifier, int maxNumberOfChronicleChannels) { - this.localIdentifer = identifier; - - chronicalChannels = new ReplicaExternalizable[maxNumberOfChronicleChannels]; - chronicalChannelBitSet = newBitSet(chronicalChannels.length); - - systemMessageQueue = new SystemQueue(systemModificationIteratorBitSet, systemModificationIterator, systemMessageHandler); - add((short) 0, systemMessageQueue.replicaExternalizable); - } - - - /** - * called whenever we receive a bootstrap message - * - * @param bytes - */ - private void onBootstrapMessage(Bytes bytes) { - final short remoteIdentifier = bytes.readByte(); - final int chronicleChannel = bytes.readUnsignedShort(); - final long lastModificationTime = bytes.readLong(); - - // this could be null if once node has a chronicle channel before the other - if (chronicalChannels[chronicleChannel] != null) - chronicalChannels[chronicleChannel].acquireModificationIterator(remoteIdentifier, NOP).dirtyEntries(lastModificationTime); - - } - - private ByteBufferBytes toBootstrapMessage(short chronicleChannel, final long lastModificationTime) { - final ByteBufferBytes writeBuffer = new ByteBufferBytes(ByteBuffer.allocate(1 + 1 + 2 + 8)); - writeBuffer.writeByte(BOOTSTAP_MESSAGE); - writeBuffer.writeByte(localIdentifer); - writeBuffer.writeUnsignedShort(chronicleChannel); - writeBuffer.writeLong(lastModificationTime); - writeBuffer.flip(); - return writeBuffer; - } - - /** - * creates a bit set based on a number of bits - * - * @param numberOfBits the number of bits the bit set should include - * @return a new DirectBitSet backed by a byteBuffer - */ - private static DirectBitSet newBitSet(int numberOfBits) { - final ByteBufferBytes bytes = new ByteBufferBytes(wrap(new byte[(numberOfBits + 7) / 8])); - return new SingleThreadedDirectBitSet(bytes); - } - - public void add(short chronicleChannel, ReplicaExternalizable replica) { - - if (chronicalChannels[chronicleChannel] != null) - throw new IllegalArgumentException("chronicleId=" + chronicleChannel + " is already in use."); - - chronicalChannels[chronicleChannel] = replica; - chronicalChannelBitSet.set(chronicleChannel); - - - // zero is used for bootstrapping - if (chronicleChannel == 0) - return; - - - // boot strap messages are only sent if the cluster is already established - for (int i = (int) systemModificationIteratorBitSet.nextSetBit(0); i > 0; i = (int) systemModificationIteratorBitSet.nextSetBit(i + 1)) { - byte remoteIdentifier = (byte) i; - final long lastModificationTime = replica.lastModificationTime(remoteIdentifier); - final ByteBufferBytes message = toBootstrapMessage(chronicleChannel, lastModificationTime); - systemModificationIterator.get(remoteIdentifier).addPayload(message); - } - - } - - - /** - * writes the entry to the chronicle channel provided - * - * @param entry the byte location of the entry to be stored - * @param destination a buffer the entry will be written to, the segment may reject this operation - * and add zeroBytes, if the identifier in the entry did not match the maps local - * @param chronicleChannel used in cluster into identify the canonical map or queue - */ - @Override - public void writeExternalEntry(@NotNull AbstractBytes entry, @NotNull Bytes destination, int chronicleChannel) { - destination.writeStopBit(chronicleChannel); - chronicalChannels[chronicleChannel].writeExternalEntry(entry, destination, chronicleChannel); - } - - @Override - public void readExternalEntry(@NotNull Bytes source) { - final int chronicleId = (int) source.readStopBit(); - if (chronicleId < chronicalChannels.length) - chronicalChannels[chronicleId].readExternalEntry(source); - else - LOG.info("skipped entry with chronicleId=" + chronicleId + ", "); - } - - @Override - public byte identifier() { - return this.localIdentifer; - } - - @Override - public ModificationIterator acquireModificationIterator(final short remoteIdentifier, - final ModificationNotifier notifier) { - final ModificationIterator result = modificationIterator.get(remoteIdentifier); - if (result != null) - return result; - - final ModificationIterator result0 = new ModificationIterator() { - - @Override - public boolean hasNext() { - for (int i = (int) chronicalChannelBitSet.nextSetBit(0); i >= 0; i = (int) - chronicalChannelBitSet.nextSetBit(i + 1)) { - final ModificationIterator modificationIterator = chronicalChannels[i] - .acquireModificationIterator(remoteIdentifier, notifier); - if (modificationIterator.hasNext()) - return true; - } - return false; - } - - @Override - public boolean nextEntry(@NotNull AbstractEntryCallback callback, final int na) { - for (int i = (int) chronicalChannelBitSet.nextSetBit(0); i >= 0; i = (int) - chronicalChannelBitSet.nextSetBit(i + 1)) { - final ModificationIterator modificationIterator = chronicalChannels[i] - .acquireModificationIterator(remoteIdentifier, notifier); - if (modificationIterator.nextEntry(callback, i)) - return true; - } - return false; - } - - @Override - public void dirtyEntries(long fromTimeStamp) { - for (int i = (int) chronicalChannelBitSet.nextSetBit(0); i >= 0; i = (int) - chronicalChannelBitSet.nextSetBit(i + 1)) { - chronicalChannels[i].acquireModificationIterator(remoteIdentifier, - notifier).dirtyEntries(fromTimeStamp); - } - notifier.onChange(); - } - }; - - modificationIterator.set((int) remoteIdentifier, result0); - return result0; - - } - - - /** - * gets the earliest modification time for all of the chronicles - * - * @param remoteIdentifier the remote identifier - * @return - */ - @Override - public long lastModificationTime(byte remoteIdentifier) { - - // if isEmpty, then we will ask for all the data from the remote node, as there could be a race - // around the chronicalChannelBitSet and the bootstrap message - if (chronicalChannelBitSet.nextSetBit(0) == -1) { - return 0; - } - - long t = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(1); - for (int i = (int) chronicalChannelBitSet.nextSetBit(0); i > 0; i = (int) chronicalChannelBitSet.nextSetBit(i + 1)) { - t = min(t, chronicalChannels[i].lastModificationTime(remoteIdentifier)); - } - return t; - } - - @Override - public void close() throws IOException { - - for (AbstractChannelReplicator replicator : replicators) { - replicator.close(); - } - - for (int i = (int) chronicalChannelBitSet.nextSetBit(0); i > 0; i = (int) chronicalChannelBitSet.nextSetBit(i + 1)) { - chronicalChannels[i].close(); - } - } - - - public void add(AbstractChannelReplicator replicator) { - replicators.add(replicator); - } - - - interface PayloadProvider extends ModificationIterator { - void addPayload(final Bytes bytes); - } - - - /** - * used to send system messages such as bootstrap from one remote node to another, it also can be used in - * a broadcast context - */ - static class SystemQueue { - - - private final DirectBitSet systemModificationIteratorBitSet; - private final AtomicReferenceArray systemModificationIterator; - private final MessageHandler messageHandler; - - - SystemQueue(DirectBitSet systemModificationIteratorBitSet, - AtomicReferenceArray systemModificationIterator, - MessageHandler messageHandler) { - - this.systemModificationIteratorBitSet = systemModificationIteratorBitSet; - this.systemModificationIterator = systemModificationIterator; - this.messageHandler = messageHandler; - } - - - final ReplicaExternalizable replicaExternalizable = new ReplicaExternalizable() { - - @Override - public byte identifier() { - return 0; - } - - @Override - public ModificationIterator acquireModificationIterator(final short remoteIdentifier, - final ModificationNotifier modificationNotifier) { - - final ModificationIterator result = systemModificationIterator.get(remoteIdentifier); - - if (result != null) - return result; - - final PayloadProvider iterator = new PayloadProvider() { - - @Override - public boolean hasNext() { - return payloads.peek() != null; - } - - final ByteBufferBytes writeBuffer = new ByteBufferBytes(ByteBuffer.allocate(WRITE_BUFFER_SIZE)); - - @Override - public boolean nextEntry(@NotNull AbstractEntryCallback callback, int na) { - - final Bytes bytes = payloads.poll(); - if (bytes == null) - return false; - - if (bytes instanceof AbstractBytes) - callback.onEntry((AbstractBytes) bytes, 0); - else { - writeBuffer.clear(); - writeBuffer.write(bytes); - callback.onEntry(writeBuffer, 0); - } - - return true; - } - - @Override - public void dirtyEntries(long fromTimeStamp) { - // do nothing - } - - final Queue payloads = new LinkedTransferQueue(); - - @Override - public void addPayload(Bytes bytes) { - - if (bytes.remaining() == 0) - return; - - if (bytes.remaining() > writeBuffer.capacity()) - throw new IllegalArgumentException("BUFFER_OVERFLOW: bytes.remaining()=" + bytes - .remaining() + " " + - " the maximum allow number of bytes is " + writeBuffer.capacity()); - - payloads.add(bytes); - - // notifies that a change has been made, this will nudge the OP_WRITE selector to - // push this update out over the nio socket - modificationNotifier.onChange(); - } - }; - - systemModificationIterator.set(remoteIdentifier, iterator); - systemModificationIteratorBitSet.set(remoteIdentifier); - - return iterator; - - } - - @Override - public long lastModificationTime(byte remoteIdentifier) { - return 0; - } - - @Override - public void close() throws IOException { - // do nothing - } - - @Override - public void writeExternalEntry(@NotNull AbstractBytes entry, @NotNull Bytes destination, int na) { - destination.write(entry); - } - - @Override - public void readExternalEntry(@NotNull Bytes source) { - messageHandler.onMessage(source); - } - }; - } -} \ No newline at end of file diff --git a/src/main/java/net/openhft/collections/ClusterReplicatorBuilder.java b/src/main/java/net/openhft/collections/ClusterReplicatorBuilder.java deleted file mode 100644 index 278f25a8d..000000000 --- a/src/main/java/net/openhft/collections/ClusterReplicatorBuilder.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.collections; - -import net.openhft.chronicle.map.ChronicleMapBuilder; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetAddress; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import static net.openhft.collections.SharedHashMapBuilder.UDP_REPLICATION_MODIFICATION_ITERATOR_ID; - -/** - * @author Rob Austin. - */ -public class ClusterReplicatorBuilder { - - Set closeables = new HashSet(); - - private final byte identifier; - private UdpReplicatorBuilder udpReplicatorBuilder = null; - private TcpReplicatorBuilder tcpReplicatorBuilder = null; - - private int maxEntrySize; - private int maxNumberOfChronicles = 128; - private ClusterReplicator clusterReplicator; - - - ClusterReplicatorBuilder(byte identifier, final int maxEntrySize1) { - this.identifier = identifier; - this.maxEntrySize = maxEntrySize1; - if (!(identifier > 0 && identifier < 128)) - throw new IllegalArgumentException("Identifier must be positive and <128, " + - "identifier=" + identifier); - } - - private final Map replicas - = new ConcurrentHashMap(); - - public ClusterReplicatorBuilder udpReplicator(UdpReplicatorBuilder udpReplicatorBuilder) throws IOException { - this.udpReplicatorBuilder = udpReplicatorBuilder; - return this; - } - - public ClusterReplicatorBuilder tcpReplicatorBuilder(TcpReplicatorBuilder tcpReplicatorBuilder) { - this.tcpReplicatorBuilder = tcpReplicatorBuilder; - return this; - } - - /** - * @param chronicleChannel when clustering with a number of maps, each map will be called a chronicle - * channel - */ - public SharedHashMap create(short chronicleChannel, ChronicleMapBuilder builder) throws - IOException { - - final ChronicleMapBuilder builder0 = builder.clone(); - builder0.identifier(identifier); - - final VanillaSharedReplicatedHashMap result = - new VanillaSharedReplicatedHashMap(builder0, builder0.kClass(), - builder0.vClass()); - - if (clusterReplicator == null) - replicas.put(chronicleChannel, (ReplicaExternalizable) result); - else { - clusterReplicator.add(chronicleChannel, result); - } - return result; - } - - - public ClusterReplicator create() throws IOException { - - final ClusterReplicator clusterReplicator = new ClusterReplicator(identifier, maxNumberOfChronicles); - - for (final Map.Entry entry : replicas.entrySet()) { - clusterReplicator.add(entry.getKey(), entry.getValue()); - } - - - if (tcpReplicatorBuilder != null) { - final TcpReplicator tcpReplicator = new TcpReplicator(clusterReplicator, clusterReplicator, tcpReplicatorBuilder, - maxEntrySize); - closeables.add(tcpReplicator); - clusterReplicator.add(tcpReplicator); - } - - if (udpReplicatorBuilder != null) { - final InetAddress address = udpReplicatorBuilder.address(); - - if (address == null) - throw new IllegalArgumentException("address can not be null"); - - if (address.isMulticastAddress() && udpReplicatorBuilder.networkInterface() == null) { - throw new IllegalArgumentException("MISSING: NetworkInterface, " + - "When using a multicast addresses, please provided a networkInterface"); - } - - final UdpReplicator udpReplicator = - new UdpReplicator(clusterReplicator, - udpReplicatorBuilder.clone(), - maxEntrySize, - identifier, - UDP_REPLICATION_MODIFICATION_ITERATOR_ID); - - closeables.add(udpReplicator); - clusterReplicator.add(udpReplicator); - } - - this.clusterReplicator = clusterReplicator; - return clusterReplicator; - - } - -} diff --git a/src/main/java/net/openhft/collections/SharedHashMap.java b/src/main/java/net/openhft/collections/SharedHashMap.java deleted file mode 100644 index 9137f1eab..000000000 --- a/src/main/java/net/openhft/collections/SharedHashMap.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.collections; - -import java.io.Closeable; -import java.io.File; -import java.util.concurrent.ConcurrentMap; - -public interface SharedHashMap extends ConcurrentMap, Closeable { - /** - * @return size as a long. - * @see java.util.Map size() - */ - long longSize(); - - /** - * Get a value for a key if available. If the value is Byteable, it will be assigned to - * reference the value, instead of copying the data for zero copy access to the collection. - * - * @param key to lookup. - * @param value to reuse if possible. If null, a new object will be created. - * @return value found or null if not. - */ - V getUsing(K key, V value); - - /** - * Acquire a value for a key, creating if absent. If the value is Byteable, it will be assigned - * to reference the value, instead of copying the data. - * - * @param key to lookup. - * @param value to reuse if possible. If null, a new object will be created. - * @return value created or found. - */ - V acquireUsing(K key, V value); - - /** - * @return The file or directory for this SharedHashMap - */ - File file(); - -} diff --git a/src/main/java/net/openhft/collections/SharedHashMapBuilder.java b/src/main/java/net/openhft/collections/SharedHashMapBuilder.java deleted file mode 100644 index 01d26cb0a..000000000 --- a/src/main/java/net/openhft/collections/SharedHashMapBuilder.java +++ /dev/null @@ -1,760 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.collections; - -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.lang.Maths; -import net.openhft.lang.io.serialization.BytesMarshallableSerializer; -import net.openhft.lang.io.serialization.BytesMarshallerFactory; -import net.openhft.lang.io.serialization.JDKObjectSerializer; -import net.openhft.lang.io.serialization.ObjectSerializer; -import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; -import net.openhft.lang.model.constraints.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.net.InetAddress; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Arrays; - -public class SharedHashMapBuilder implements Cloneable { - - private static final Logger LOG = LoggerFactory.getLogger(net.openhft.collections.SharedHashMapBuilder.class.getName()); - - static final int HEADER_SIZE = 128; - static final int SEGMENT_HEADER = 64; - private static final byte[] MAGIC = "SharedHM".getBytes(); - - public static final short UDP_REPLICATION_MODIFICATION_ITERATOR_ID = 128; - public static final short JDBC_REPLICATION_MODIFICATION_ITERATOR_ID = 129; - public static final short FILE_REPLICATION_MODIFICATION_ITERATOR_ID = 130; - - // used when configuring the number of segments. - private int minSegments = -1; - private int actualSegments = -1; - - // used when reading the number of entries per - private int actualEntriesPerSegment = -1; - - private int entrySize = 256; - private Alignment alignment = Alignment.OF_4_BYTES; - private long entries = 1 << 20; - private int replicas = 0; - boolean transactional = false; - private long lockTimeOutMS = 20000; - private int metaDataBytes = 0; - private SharedMapEventListener eventListener = SharedMapEventListeners.nop(); - private SharedMapErrorListener errorListener = SharedMapErrorListeners.logging(); - private boolean putReturnsNull = false; - private boolean removeReturnsNull = false; - private boolean generatedKeyType = false; - private boolean generatedValueType = false; - private boolean largeSegments = false; - - // replication - private boolean canReplicate; - byte identifier = Byte.MIN_VALUE; - TcpReplicatorBuilder tcpReplicatorBuilder; - // ExternalReplicatorBuilder externalReplicatorBuilder; - - private TimeProvider timeProvider = TimeProvider.SYSTEM; - UdpReplicatorBuilder udpReplicatorBuilder; - private BytesMarshallerFactory bytesMarshallerFactory; - private ObjectSerializer objectSerializer; - //ExternalReplicator externalReplicator; - File file; - Class kClass; - Class vClass; - - public SharedHashMapBuilder() { - } - - public SharedHashMapBuilder(Class kClass, Class vClass) { - this.kClass = kClass; - this.vClass = vClass; - } - - public static SharedHashMapBuilder of(Class kClass, Class vClass) { - return new SharedHashMapBuilder(kClass, vClass); - } - - @Override - public SharedHashMapBuilder clone() { - - try { - @SuppressWarnings("unchecked") - final SharedHashMapBuilder result = (SharedHashMapBuilder) super.clone(); - if (tcpReplicatorBuilder() != null) - result.tcpReplicatorBuilder(tcpReplicatorBuilder().clone()); - if (udpReplicatorBuilder() != null) - result.udpReplicatorBuilder(udpReplicatorBuilder().clone()); - return result; - - } catch (CloneNotSupportedException e) { - throw new AssertionError(e); - } - } - - - /** - * Set minimum number of segments. See concurrencyLevel in {@link java.util.concurrent.ConcurrentHashMap}. - * - * @return this builder object back - */ - public SharedHashMapBuilder minSegments(int minSegments) { - this.minSegments = minSegments; - return this; - } - - public int minSegments() { - return minSegments < 1 ? tryMinSegments(4, 65536) : minSegments; - } - - private int tryMinSegments(int min, int max) { - for (int i = min; i < max; i <<= 1) { - if (i * i * i >= alignedEntrySize() * 2) - return i; - } - return max; - } - - /** - *

Note that the actual entrySize will be aligned to 4 (default entry alignment). I. e. if you set - * entry size to 30, the actual entry size will be 32 (30 aligned to 4 bytes). If you don't want entry - * size to be aligned, set {@code entryAndValueAlignment(Alignment.NO_ALIGNMENT)}. - * - * @param entrySize the size in bytes - * @return this {@code SharedHashMapBuilder} back - * @see #entryAndValueAlignment(Alignment) - * @see #entryAndValueAlignment() - */ - public SharedHashMapBuilder entrySize(int entrySize) { - this.entrySize = entrySize; - return this; - } - - public int entrySize() { - return entrySize; - } - - int alignedEntrySize() { - return entryAndValueAlignment().alignSize(entrySize()); - } - - /** - * Specifies alignment of address in memory of entries and independently of address in memory of values - * within entries.

Useful when values of the map are updated intensively, particularly fields with - * volatile access, because it doesn't work well if the value crosses cache lines. Also, on some (nowadays - * rare) architectures any misaligned memory access is more expensive than aligned.

Note that - * specified {@link #entrySize()} will be aligned according to this alignment. I. e. if you set {@code - * entrySize(20)} and {@link net.openhft.collections.Alignment#OF_8_BYTES}, actual entry size will be 24 - * (20 aligned to 8 bytes). - * - * @return this {@code SharedHashMapBuilder} back - * @see #entryAndValueAlignment() - */ - public SharedHashMapBuilder entryAndValueAlignment(Alignment alignment) { - this.alignment = alignment; - return this; - } - - /** - * Returns alignment of addresses in memory of entries and independently of values within entries.

- *

Default is {@link net.openhft.collections.Alignment#OF_4_BYTES}. - * - * @see #entryAndValueAlignment(Alignment) - */ - public Alignment entryAndValueAlignment() { - return alignment; - } - - public SharedHashMapBuilder entries(long entries) { - this.entries = entries; - return this; - } - - public long entries() { - return entries; - } - - public SharedHashMapBuilder replicas(int replicas) { - this.replicas = replicas; - return this; - } - - public int replicas() { - return replicas; - } - - public SharedHashMapBuilder actualEntriesPerSegment(int actualEntriesPerSegment) { - this.actualEntriesPerSegment = actualEntriesPerSegment; - return this; - } - - public int actualEntriesPerSegment() { - if (actualEntriesPerSegment > 0) - return actualEntriesPerSegment; - int as = actualSegments(); - // round up to the next multiple of 64. - return (int) (Math.max(1, entries * 2L / as) + 63) & ~63; - } - - public SharedHashMapBuilder actualSegments(int actualSegments) { - this.actualSegments = actualSegments; - return this; - } - - public int actualSegments() { - if (actualSegments > 0) - return actualSegments; - if (!largeSegments && entries > (long) minSegments() << 15) { - long segments = Maths.nextPower2(entries >> 15, 128); - if (segments < 1 << 20) - return (int) segments; - } - // try to keep it 16-bit sizes segments - return (int) Maths.nextPower2(Math.max((entries >> 30) + 1, minSegments()), 1); - } - - /** - * Not supported yet. - * - * @param transactional if the built map should be transactional - * @return this {@code SharedHashMapBuilder} back - */ - public SharedHashMapBuilder transactional(boolean transactional) { - this.transactional = transactional; - return this; - } - - public boolean transactional() { - return transactional; - } - - - public SharedHashMapBuilder kClass(Class kClass) { - this.kClass = kClass; - return this; - } - - public SharedHashMapBuilder vClass(Class vClass) { - this.vClass = vClass; - return this; - } - - public SharedHashMapBuilder file(File file) { - this.file = file; - return this; - } - - public File file() { - return this.file; - } - - public Class kClass() { - return this.kClass; - } - - public Class vClass() { - return this.vClass; - } - - - public ChronicleMap create() throws IOException { - if (kClass == null) - throw new IllegalArgumentException("missing mandatory parameter kClass"); - - if (vClass == null) - throw new IllegalArgumentException("missing mandatory parameter vClass"); - - SharedHashMapBuilder builder = toBuilder(); - - if (!canReplicate()) - return new VanillaSharedHashMap(builder, file, kClass, vClass); - - if (identifier <= 0) - throw new IllegalArgumentException("Identifier must be positive, " + identifier + " given"); - - final VanillaSharedReplicatedHashMap result = - new VanillaSharedReplicatedHashMap(builder, kClass, vClass); - - /* if (externalReplicatorBuilder != null) - externalReplicator = applyExternalReplicator(result, externalReplicatorBuilder, kClass, vClass); -*/ - if (tcpReplicatorBuilder != null) - applyTcpReplication(result, tcpReplicatorBuilder); - - if (udpReplicatorBuilder != null) { - if (tcpReplicatorBuilder == null) - LOG.warn("MISSING TCP REPLICATION : The UdpReplicator only attempts to read data (" + - "it does not enforce or guarantee delivery), you should use the UdpReplicator if " + - "you have a large number of nodes, and you wish to receive the data before it " + - "becomes available on TCP/IP. Since data delivery is not guaranteed, it is " + - "recommended that you only use the UDP" + - " " + - "Replicator " + - "in conjunction with a TCP Replicator"); - applyUdpReplication(result, udpReplicatorBuilder); - } - return result; - } - - - /** - * Its recommended that you use net.openhft.collections.SharedHashMapBuilder#create() instead as this - * method will be shortly removed - */ - @Deprecated - public ChronicleMap create(File file, Class kClass, Class vClass) throws IOException { - return file(file).kClass(kClass).vClass(vClass).create(); - } - - public SharedHashMapBuilder toBuilder() throws IOException { - SharedHashMapBuilder builder = clone(); - - if (file != null) { - for (int i = 0; i < 10; i++) { - if (file.exists() && file.length() > 0) { - readFile(file, builder); - break; - } - if (file.createNewFile() || file.length() == 0) { - newFile(file); - break; - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new IOException(e); - } - } - if (builder == null || !file.exists()) - throw new FileNotFoundException("Unable to create " + file); - } - return builder; - } - - static void readFile(File file, SharedHashMapBuilder builder) throws IOException { - ByteBuffer bb = ByteBuffer.allocateDirect(HEADER_SIZE).order(ByteOrder.nativeOrder()); - FileInputStream fis = new FileInputStream(file); - fis.getChannel().read(bb); - fis.close(); - bb.flip(); - if (bb.remaining() < 22) throw new IOException("File too small, corrupted? " + file); - byte[] bytes = new byte[8]; - bb.get(bytes); - if (!Arrays.equals(bytes, MAGIC)) - throw new IOException("Unknown magic number, was " + new String(bytes, "ISO-8859-1")); - builder.actualSegments(bb.getInt()); - builder.actualEntriesPerSegment(bb.getInt()); - builder.entrySize(bb.getInt()); - builder.entryAndValueAlignment(Alignment.fromOrdinal(bb.get())); - builder.replicas(bb.getInt()); - builder.transactional(bb.get() == 'Y'); - builder.metaDataBytes(bb.get() & 0xFF); - if (builder.actualSegments() <= 0 || builder.actualEntriesPerSegment() <= 0 || builder.entrySize() <= 0) - throw new IOException("Corrupt header for " + file); - } - - void newFile(File file) throws IOException { - ByteBuffer bb = ByteBuffer.allocateDirect(HEADER_SIZE).order(ByteOrder.nativeOrder()); - bb.put(MAGIC); - bb.putInt(actualSegments()); - bb.putInt(actualEntriesPerSegment()); - bb.putInt(entrySize()); - bb.put((byte) entryAndValueAlignment().ordinal()); - bb.putInt(replicas()); - bb.put((byte) (transactional ? 'Y' : 'N')); - bb.put((byte) metaDataBytes); - bb.flip(); - FileOutputStream fos = new FileOutputStream(file); - fos.getChannel().write(bb); - fos.close(); - } - - public SharedHashMapBuilder lockTimeOutMS(long lockTimeOutMS) { - this.lockTimeOutMS = lockTimeOutMS; - return this; - } - - public long lockTimeOutMS() { - return lockTimeOutMS; - } - - public SharedHashMapBuilder errorListener(SharedMapErrorListener errorListener) { - this.errorListener = errorListener; - return this; - } - - public SharedMapErrorListener errorListener() { - return errorListener; - } - - /** - * Map.put() returns the previous value, functionality which is rarely used but fairly cheap for HashMap. - * In the case, for an off heap collection, it has to create a new object (or return a recycled one) - * Either way it's expensive for something you probably don't use. - * - * @param putReturnsNull false if you want SharedHashMap.put() to not return the object that was replaced - * but instead return null - * @return an instance of the map builder - */ - public SharedHashMapBuilder putReturnsNull(boolean putReturnsNull) { - this.putReturnsNull = putReturnsNull; - return this; - } - - /** - * Map.put() returns the previous value, functionality which is rarely used but fairly cheap for HashMap. - * In the case, for an off heap collection, it has to create a new object (or return a recycled one) - * Either way it's expensive for something you probably don't use. - * - * @return true if SharedHashMap.put() is not going to return the object that was replaced but instead - * return null - */ - public boolean putReturnsNull() { - return putReturnsNull; - } - - /** - * Map.remove() returns the previous value, functionality which is rarely used but fairly cheap for - * HashMap. In the case, for an off heap collection, it has to create a new object (or return a recycled - * one) Either way it's expensive for something you probably don't use. - * - * @param removeReturnsNull false if you want SharedHashMap.remove() to not return the object that was - * removed but instead return null - * @return an instance of the map builder - */ - public SharedHashMapBuilder removeReturnsNull(boolean removeReturnsNull) { - this.removeReturnsNull = removeReturnsNull; - return this; - } - - - /** - * Map.remove() returns the previous value, functionality which is rarely used but fairly cheap for - * HashMap. In the case, for an off heap collection, it has to create a new object (or return a recycled - * one) Either way it's expensive for something you probably don't use. - * - * @return true if SharedHashMap.remove() is not going to return the object that was removed but instead - * return null - */ - public boolean removeReturnsNull() { - return removeReturnsNull; - } - - public boolean generatedKeyType() { - return generatedKeyType; - } - - public SharedHashMapBuilder generatedKeyType(boolean generatedKeyType) { - this.generatedKeyType = generatedKeyType; - return this; - } - - public boolean generatedValueType() { - return generatedValueType; - } - - public SharedHashMapBuilder generatedValueType(boolean generatedValueType) { - this.generatedValueType = generatedValueType; - return this; - } - - public boolean largeSegments() { - return entries > 1L << (20 + 15) || largeSegments; - } - - public SharedHashMapBuilder largeSegments(boolean largeSegments) { - this.largeSegments = largeSegments; - return this; - } - - - public SharedHashMapBuilder metaDataBytes(int metaDataBytes) { - if ((metaDataBytes & 0xFF) != metaDataBytes) - throw new IllegalArgumentException("MetaDataBytes must be [0..255] was " + metaDataBytes); - this.metaDataBytes = metaDataBytes; - return this; - } - - public int metaDataBytes() { - return metaDataBytes; - } - - public SharedHashMapBuilder eventListener(SharedMapEventListener eventListener) { - this.eventListener = eventListener; - return this; - } - - public SharedMapEventListener eventListener() { - return eventListener; - } - - @Override - public String toString() { - return "SharedHashMapBuilder{" + - "actualSegments=" + actualSegments() + - ", minSegments=" + minSegments() + - ", actualEntriesPerSegment=" + actualEntriesPerSegment() + - ", entrySize=" + entrySize() + - ", entryAndValueAlignment=" + entryAndValueAlignment() + - ", entries=" + entries() + - ", replicas=" + replicas() + - ", transactional=" + transactional() + - ", lockTimeOutMS=" + lockTimeOutMS() + - ", metaDataBytes=" + metaDataBytes() + - ", eventListener=" + eventListener() + - ", errorListener=" + errorListener() + - ", putReturnsNull=" + putReturnsNull() + - ", removeReturnsNull=" + removeReturnsNull() + - ", generatedKeyType=" + generatedKeyType() + - ", generatedValueType=" + generatedValueType() + - ", largeSegments=" + largeSegments() + - ", canReplicate=" + canReplicate() + - ", identifier=" + identifierToString() + - ", tcpReplicatorBuilder=" + tcpReplicatorBuilder() + - ", udpReplicatorBuilder=" + udpReplicatorBuilder() + - ", timeProvider=" + timeProvider() + - ", bytesMarshallerfactory=" + bytesMarshallerFactory() + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - SharedHashMapBuilder that = (SharedHashMapBuilder) o; - - if (actualEntriesPerSegment != that.actualEntriesPerSegment) return false; - if (actualSegments != that.actualSegments) return false; - if (canReplicate != that.canReplicate) return false; - if (entries != that.entries) return false; - if (entrySize != that.entrySize) return false; - if (generatedKeyType != that.generatedKeyType) return false; - if (generatedValueType != that.generatedValueType) return false; - if (identifier != that.identifier) return false; - if (largeSegments != that.largeSegments) return false; - if (lockTimeOutMS != that.lockTimeOutMS) return false; - if (metaDataBytes != that.metaDataBytes) return false; - if (minSegments != that.minSegments) return false; - if (putReturnsNull != that.putReturnsNull) return false; - if (removeReturnsNull != that.removeReturnsNull) return false; - if (replicas != that.replicas) return false; - if (transactional != that.transactional) return false; - - if (alignment != that.alignment) return false; - if (errorListener != null ? !errorListener.equals(that.errorListener) : that.errorListener != null) - return false; - if (eventListener != null ? !eventListener.equals(that.eventListener) : that.eventListener != null) - return false; - if (tcpReplicatorBuilder != null ? !tcpReplicatorBuilder.equals(that.tcpReplicatorBuilder) : that.tcpReplicatorBuilder != null) - return false; - if (timeProvider != null ? !timeProvider.equals(that.timeProvider) : that.timeProvider != null) - return false; - if (udpReplicatorBuilder != null ? !udpReplicatorBuilder.equals(that.udpReplicatorBuilder) : that.udpReplicatorBuilder != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = minSegments; - result = 31 * result + actualSegments; - result = 31 * result + actualEntriesPerSegment; - result = 31 * result + entrySize; - result = 31 * result + (alignment != null ? alignment.hashCode() : 0); - result = 31 * result + (int) (entries ^ (entries >>> 32)); - result = 31 * result + replicas; - result = 31 * result + (transactional ? 1 : 0); - result = 31 * result + (int) (lockTimeOutMS ^ (lockTimeOutMS >>> 32)); - result = 31 * result + metaDataBytes; - result = 31 * result + (eventListener != null ? eventListener.hashCode() : 0); - result = 31 * result + (errorListener != null ? errorListener.hashCode() : 0); - result = 31 * result + (putReturnsNull ? 1 : 0); - result = 31 * result + (removeReturnsNull ? 1 : 0); - result = 31 * result + (generatedKeyType ? 1 : 0); - result = 31 * result + (generatedValueType ? 1 : 0); - result = 31 * result + (largeSegments ? 1 : 0); - result = 31 * result + (canReplicate ? 1 : 0); - result = 31 * result + (int) identifier; - result = 31 * result + (tcpReplicatorBuilder != null ? tcpReplicatorBuilder.hashCode() : 0); - result = 31 * result + (timeProvider != null ? timeProvider.hashCode() : 0); - result = 31 * result + (udpReplicatorBuilder != null ? udpReplicatorBuilder.hashCode() : 0); - return result; - } - - public boolean canReplicate() { - return canReplicate || tcpReplicatorBuilder != null || udpReplicatorBuilder != null; - } - - public SharedHashMapBuilder canReplicate(boolean canReplicate) { - this.canReplicate = canReplicate; - return this; - } - - - /* ExternalReplicator.AbstractExternalReplicator applyExternalReplicator - (VanillaSharedReplicatedHashMap map, - ExternalReplicatorBuilder builder, - Class kClass, - Class vClass) { - try { - - final ExternalReplicator.AbstractExternalReplicator externalReplicator; - - final short id; - - if (builder instanceof ExternalJDBCReplicatorBuilder) { - externalReplicator = new ExternalJDBCReplicator(kClass, vClass, - (ExternalJDBCReplicatorBuilder) builder, map); - id = JDBC_REPLICATION_MODIFICATION_ITERATOR_ID; - } else if (builder instanceof ExternalFileReplicatorBuilder) { - externalReplicator = new ExternalFileReplicator(kClass, vClass, - (ExternalFileReplicatorBuilder) builder, map); - id = FILE_REPLICATION_MODIFICATION_ITERATOR_ID; - } else - throw new IllegalStateException("builder of type " + builder.getClass() + " is not supported."); - - - final Replica.ModificationIterator modIterator - = map.acquireModificationIterator(id, externalReplicator); - - externalReplicator.setModificationIterator(modIterator); - - map.addCloseable(externalReplicator); - return externalReplicator; - } catch (Exception e) { - LOG.error("", e); - } - - return null; - } -*/ - - void applyUdpReplication(VanillaSharedReplicatedHashMap result, - UdpReplicatorBuilder udpReplicatorBuilder) - throws IOException { - - final InetAddress address = udpReplicatorBuilder.address(); - - if (address == null) { - throw new IllegalArgumentException("address can not be null"); - } - - if (address.isMulticastAddress() && udpReplicatorBuilder.networkInterface() == null) { - throw new IllegalArgumentException("MISSING: NetworkInterface, " + - "When using a multicast addresses, please provided a " + - "networkInterface"); - } - - final UdpReplicator udpReplicator = - new UdpReplicator(result, udpReplicatorBuilder.clone(), entrySize(), result.identifier(), UDP_REPLICATION_MODIFICATION_ITERATOR_ID); - - - result.addCloseable(udpReplicator); - } - - - void applyTcpReplication(@NotNull VanillaSharedReplicatedHashMap result, - @NotNull TcpReplicatorBuilder tcpReplicatorBuilder) - throws IOException { - - result.addCloseable(new TcpReplicator(result, result, tcpReplicatorBuilder, entrySize())); - } - - public SharedHashMapBuilder timeProvider(TimeProvider timeProvider) { - this.timeProvider = timeProvider; - return this; - } - - public TimeProvider timeProvider() { - return timeProvider; - } - - public byte identifier() { - - if (identifier == Byte.MIN_VALUE) - throw new IllegalStateException("identifier is not set."); - - return identifier; - } - - private String identifierToString() { - return identifier == Byte.MIN_VALUE ? "identifier is not set" : (identifier + ""); - } - - public SharedHashMapBuilder identifier(byte identifier) { - this.identifier = identifier; - return this; - } - - public SharedHashMapBuilder tcpReplicatorBuilder(TcpReplicatorBuilder tcpReplicatorBuilder) { - this.tcpReplicatorBuilder = tcpReplicatorBuilder; - return this; - } - - public TcpReplicatorBuilder tcpReplicatorBuilder() { - return tcpReplicatorBuilder; - } - - - public UdpReplicatorBuilder udpReplicatorBuilder() { - return udpReplicatorBuilder; - } - - - public SharedHashMapBuilder udpReplicatorBuilder(UdpReplicatorBuilder udpReplicatorBuilder) { - this.udpReplicatorBuilder = udpReplicatorBuilder; - return this; - } - - public BytesMarshallerFactory bytesMarshallerFactory() { - return bytesMarshallerFactory == null ? bytesMarshallerFactory = new VanillaBytesMarshallerFactory() : bytesMarshallerFactory; - } - - public SharedHashMapBuilder bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) { - this.bytesMarshallerFactory = bytesMarshallerFactory; - return this; - } - - public ObjectSerializer objectSerializer() { - return objectSerializer == null ? objectSerializer = BytesMarshallableSerializer.create(bytesMarshallerFactory(), JDKObjectSerializer.INSTANCE) : objectSerializer; - } - - public SharedHashMapBuilder objectSerializer(ObjectSerializer objectSerializer) { - this.objectSerializer = objectSerializer; - return this; - } - -/* public SharedHashMapBuilder externalReplicatorBuilder(@NotNull ExternalReplicatorBuilder externalReplicatorBuilder) { - this.externalReplicatorBuilder = externalReplicatorBuilder; - return this; - } - - public T externalReplicator() { - return (T) externalReplicator; - }*/ - -} - diff --git a/src/test/java/eg/BigData.java b/src/test/java/eg/BigData.java new file mode 100644 index 000000000..fd2cc894f --- /dev/null +++ b/src/test/java/eg/BigData.java @@ -0,0 +1,226 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eg; + +import net.openhft.affinity.AffinitySupport; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import net.openhft.lang.io.Bytes; + +import java.io.*; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +/* + tune the kernel to maximise the amount of cached write data. + + vm.dirty_background_ratio = 30 + vm.dirty_expire_centisecs = 30000 + vm.dirty_ratio = 60 + vm.dirty_writeback_centisecs = 3000 + */ +public class BigData { + final static long MAXSIZE = 1000 * 1000 * 1000L; + //run 1st test with no map, and Highwatermark set to 0 + //then switch to Highwatermark set to MAXSIZE for subsequent test repeats + static AtomicInteger Highwatermark = new AtomicInteger((int) MAXSIZE); + static final ChronicleMapBuilder builder = + ChronicleMapBuilder.of(Long.class, BigDataStuff.class); + + // static AtomicInteger Highwatermark = new AtomicInteger(0); + static { + builder.largeSegments(true); + builder.actualSegments(8 * 1024); + builder.entries(MAXSIZE); + String dir = System.getProperty("dir", "/ocz/tmp"); + if (!new File("/ocz/tmp").exists()) dir = "."; + String chmPath = dir + "/testmap-" + Long.toString(System.nanoTime(), 36); + new File(chmPath).deleteOnExit(); + System.out.println("ChronicleMap entries() = " + builder.entries()); + try { + theMap = builder.create(new File(chmPath)); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + static Map theMap; + + public static void main(String[] args) throws IOException, InterruptedException { + long start = System.currentTimeMillis(); + initialbuild(); + System.out.println("Start highwatermark " + Highwatermark.get()); + for (int i = 0; i < 10; i++) { + Thread t1 = new Thread("test 1") { + public void run() { + _test(); + } + }; + Thread t2 = new Thread("test 2") { + public void run() { + _test(); + } + }; + Thread t3 = new Thread("test 3") { + public void run() { + _test(); + } + }; + t1.start(); + t2.start(); + t3.start(); + _test(); + t1.join(); + t2.join(); + t3.join(); + } + System.out.println("End highwatermark " + Highwatermark.get()); + long time = System.currentTimeMillis() - start; + System.out.printf("End to end took %.1f%n", time / 1e3); + } + + public static void initialbuild() throws IOException, InterruptedException { + System.out.println("building an empty map"); + long start = System.currentTimeMillis(); + Thread t1 = new Thread("test 1") { + public void run() { + populate(1); + } + }; + t1.start(); + Thread t2 = new Thread("test 2") { + public void run() { + populate(2); + } + }; + t2.start(); + Thread t3 = new Thread("test 3") { + public void run() { + populate(3); + } + }; + t3.start(); + populate(0); + t1.join(); + t2.join(); + t3.join(); + long now = System.currentTimeMillis(); + System.out.println(builder); + System.out.println("Time taken to insert all entries " + ((now - start) / 1000.0) + " seconds"); + } + + public static void populate(int n) { + AffinitySupport.setThreadId(); + + long start = System.currentTimeMillis(); + BigDataStuff value = new BigDataStuff(0); + for (long i = n; i < MAXSIZE; i += 4) { + if (n == 0 && i % (10 * 1000 * 1000) == 0) { + System.out.println("Now inserted to " + i + " seconds since start = " + ((System.currentTimeMillis() - start) / 1000L)); + } + value.x = i; + value.y.setLength(0); + value.y.append(i); + theMap.put(i, value); + } + } + + public static void _test() { + // improves logging of these threads. + AffinitySupport.setThreadId(); + try { + test(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public static void test() throws IOException { + //do a sequence 1m of each of insert/read/update + //inserts + long LOOPCOUNT = 100 * 1000L; + Random rand = new Random(); + long start = System.currentTimeMillis(); + BigDataStuff value = new BigDataStuff(0); + for (long i = 0; i < LOOPCOUNT; i++) { + long current = rand.nextInt(Highwatermark.get()); + value.x = current; + value.y.setLength(0); + value.y.append(current); + theMap.put(i, value); + } + long now = System.currentTimeMillis(); + System.out.println("Time taken to insert 100k entries " + ((now - start) / 1000.0) + " seconds"); + + int count = 0; + start = System.currentTimeMillis(); + for (long i = 0; i < LOOPCOUNT; i++) { + long keyval = rand.nextInt(Highwatermark.get()); + count++; + BigDataStuff stuff = theMap.get(keyval); + if (stuff == null) { + System.out.println("hit an empty at key " + keyval); + } + } + now = System.currentTimeMillis(); + System.out.println("Time taken to read " + count + " entries of 100k attempts " + ((now - start) / 1000.0) + " seconds"); + + start = System.currentTimeMillis(); + count = 0; + for (long i = 0; i < LOOPCOUNT; i++) { + long keyval = rand.nextInt(Highwatermark.get()); + BigDataStuff stuff = theMap.get(keyval); + if (stuff == null) { + System.out.println("hit an empty at key " + keyval); + } else { + count++; + stuff.x++; + stuff.y.append('1'); + theMap.put(keyval, stuff); + } + } + now = System.currentTimeMillis(); + System.out.println("Time taken to read+update " + count + " entries of 100k attempts " + ((now - start) / 1000.0) + " seconds"); + } +} + +class BigDataStuff implements Externalizable { + long x; + StringBuilder y = new StringBuilder(); + + public BigDataStuff(long x) { + this.x = x; + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + Bytes b = (Bytes) out; + b.writeStopBit(x); + b.writeUTFΔ(y); + } + + @Override + public void readExternal(ObjectInput in) throws IOException { + Bytes b = (Bytes) in; + this.x = b.readStopBit(); + if (this.y == null) + this.y = new StringBuilder(); + b.readUTFΔ(this.y); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java b/src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java new file mode 100644 index 000000000..3b465bd6e --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/AcquireGetUsingMain.java @@ -0,0 +1,94 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.model.constraints.MaxSize; + +import java.io.File; + +import static org.junit.Assert.fail; + +public class AcquireGetUsingMain { + public static void main(String[] args) throws Exception { + File file = new File(System.getProperty("java.io.tmpdir") + "/test1"); + ChronicleMap theSharedMap = ChronicleMapBuilder.of(String.class, Data.class) + .create(file); + Data data = DataValueClasses.newDirectReference(Data.class); + String processType = "testkey"; + if (theSharedMap.getUsing(processType, data) == null) { + System.out.println("Key " + processType + " does not exist, " + data); + } else { + System.out.println("Key " + processType + " exists " + data); + } + + // you can't have off heap objects, but you can an on heap object which proxy references off heap data. + // this reference is not usable until it references something concrete. + Data data2 = DataValueClasses.newDirectReference(Data.class); + String processType2 = "testkey2"; + if (theSharedMap.getUsing(processType2, data2) == null) { + // should be unset given we don't set it. + System.out.println("Key " + processType2 + " does not exist, " + data2); + try { + data2.setMaxNumberOfProcessesAllowed(4); + fail("Expected a NPE because this data2 value hasn't set to anything because there was no key in the map."); + } catch (NullPointerException npe) { + // expected as there was not value to set the data2 to. + } + } else { + System.out.println("Key " + processType2 + " exists " + data2); + } + + String processType3 = "testkey3"; + if (theSharedMap.getUsing(processType3, data) == null) { + // should be unset as this key is not present. + System.out.println("Key " + processType3 + " does not exist, " + data); + } else { + System.out.println("Key " + processType3 + " exists " + data); + } + + // populate with an entry, creating as required. + Data data3 = theSharedMap.acquireUsing(processType, data); + assert data3 == data; + + System.out.println("getting " + ((Byteable) data).bytes()); + data.setMaxNumberOfProcessesAllowed(3); + data.setTimeAt(0, 100); + data.setTimeAt(1, 111); + data.setTimeAt(2, 222); + + // an on heap object exists when created as it doesn't use indirection to where the data is actually held. + Data data4 = DataValueClasses.newInstance(Data.class); + + // put is not needed as we created something already. + // theSharedMap.put(processType, data); + System.out.println("1 " + data.getMaxNumberOfProcessesAllowed()); + System.out.println("2 " + data.getTimeAt(1)); + } + + public static interface Data { + void setTimeAt(@MaxSize(8) int index, long time); + + long getTimeAt(int index); + + int getMaxNumberOfProcessesAllowed(); + + void setMaxNumberOfProcessesAllowed(int num); + } + +} diff --git a/src/test/java/net/openhft/chronicle/map/BitUnitTest.java b/src/test/java/net/openhft/chronicle/map/BitUnitTest.java new file mode 100644 index 000000000..2a86710a3 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/BitUnitTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import junit.framework.TestCase; + +public class BitUnitTest extends TestCase { + + public void testToBits() throws Exception { + + } + + public void testToMegaBits() throws Exception { + + } + + public void testToGigaBits() throws Exception { + + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/Builder.java b/src/test/java/net/openhft/chronicle/map/Builder.java new file mode 100644 index 000000000..3f64a8e88 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/Builder.java @@ -0,0 +1,112 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.concurrent.ArrayBlockingQueue; + +/** + * @author Rob Austin. + */ +public class Builder { + + // added to ensure uniqueness + static int count; + static String WIN_OS = "WINDOWS"; + + public static File getPersistenceFile() throws IOException { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/chm-test" + System.nanoTime() + (count++)); + + //Not Guaranteed to work on Windows, since OS file-lock takes precedence + if (System.getProperty("os.name").indexOf(WIN_OS) > 0) { + /*Windows will lock a file that are currently in use. You cannot delete it, however, + using setwritable() and then releasing RandomRW lock adds the file to JVM exit cleanup. + This will only work if the user is an admin on windows. + */ + file.setWritable(true);//just in case relative path was used. + RandomAccessFile raf = new RandomAccessFile(file, "rw"); + raf.close();//allows closing the file access on windows. forcing to close access. Only works for admin-access. + } + + //file.delete(); //isnt guaranteed on windows. + file.deleteOnExit();//isnt guaranteed on windows. + + return file; + } + + + static ChronicleMap newShm( + int size, final ArrayBlockingQueue input, + final ArrayBlockingQueue output, + final byte localIdentifier, byte externalIdentifier, + Class kClass, Class vClass) throws IOException { + Replicator queue = QueueReplicator.of(localIdentifier, externalIdentifier, input, output); + return ChronicleMapBuilder.of(kClass, vClass) + .entries(size) + .addReplicator(queue) + .create(getPersistenceFile()); + } + + static MapProvider> newShmIntInt( + int size, final ArrayBlockingQueue input, + final ArrayBlockingQueue output, + final byte localIdentifier, byte externalIdentifier) throws IOException { + + Replicator queue = QueueReplicator.of(localIdentifier, externalIdentifier, input, output); + final ReplicatedChronicleMap result = + (ReplicatedChronicleMap) ChronicleMapBuilder + .of(Integer.class, Integer.class) + .entries(size) + .addReplicator(queue) + .create(getPersistenceFile()); + QueueReplicator q = null; + for (Closeable closeable : result.closeables) { + if (closeable instanceof QueueReplicator) { + q = (QueueReplicator) closeable; + break; + } + } + if (q == null) + throw new AssertionError(); + final QueueReplicator finalQ = q; + return new MapProvider>() { + + @Override + public ReplicatedChronicleMap getMap() { + return result; + } + + @Override + public boolean isQueueEmpty() { + return finalQ.isEmpty(); + } + + }; + } + + interface MapProvider { + T getMap(); + + boolean isQueueEmpty(); + } + +} diff --git a/src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java b/src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java new file mode 100644 index 000000000..36131acd7 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java @@ -0,0 +1,69 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.NativeBytes; +import net.openhft.lang.io.serialization.BytesMarshaller; +import net.openhft.lang.io.serialization.ObjectFactory; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.values.IntValue; +import org.jetbrains.annotations.Nullable; + +enum ByteableIntValueMarshaller implements BytesMarshaller { + INSTANCE; + + static final Class DIRECT = DataValueClasses.directClassFor(IntValue.class); + + @Override + public void write(Bytes bytes, IntValue intValue) { + Byteable biv = (Byteable) intValue; + bytes.write(biv.bytes(), biv.offset(), biv.maxSize()); + } + + @Nullable + @Override + public IntValue read(Bytes bytes) { + return read(bytes, null); + } + + @Nullable + @Override + public IntValue read(Bytes bytes, @Nullable IntValue intValue) { + try { + if (intValue == null) + intValue = (IntValue) NativeBytes.UNSAFE.allocateInstance(DIRECT); + Byteable biv = (Byteable) intValue; + biv.bytes(bytes, bytes.position()); + bytes.skip(biv.maxSize()); + return intValue; + } catch (Exception e) { + throw new IllegalStateException(e); + } + + } +} + +enum DirectIntValueFactory implements ObjectFactory { + INSTANCE; + + @Override + public IntValue create() throws Exception { + return (IntValue) NativeBytes.UNSAFE.allocateInstance(ByteableIntValueMarshaller.DIRECT); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java b/src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java new file mode 100644 index 000000000..54aee15c3 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java @@ -0,0 +1,70 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.NativeBytes; +import net.openhft.lang.io.serialization.BytesMarshaller; +import net.openhft.lang.io.serialization.ObjectFactory; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.values.LongValue; +import org.jetbrains.annotations.Nullable; + +enum ByteableLongValueMarshaller implements BytesMarshaller { + INSTANCE; + + static final Class DIRECT = DataValueClasses.directClassFor(LongValue.class); + + @Override + public void write(Bytes bytes, LongValue longValue) { + Byteable biv = (Byteable) longValue; + bytes.write(biv.bytes(), biv.offset(), biv.maxSize()); + } + + @Nullable + @Override + public LongValue read(Bytes bytes) { + return read(bytes, null); + } + + @Nullable + @Override + public LongValue read(Bytes bytes, @Nullable LongValue longValue) { + try { + if (longValue == null) + longValue = (LongValue) NativeBytes.UNSAFE.allocateInstance(DIRECT); + Byteable biv = (Byteable) longValue; + biv.bytes(bytes, bytes.position()); + bytes.skip(biv.maxSize()); + return longValue; + } catch (Exception e) { + throw new IllegalStateException(e); + } + + } +} + +enum DirectLongValueFactory implements ObjectFactory { + INSTANCE; + + @Override + public LongValue create() throws Exception { + return (LongValue) NativeBytes.UNSAFE.allocateInstance(ByteableLongValueMarshaller.DIRECT); + } +} + diff --git a/src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java b/src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java new file mode 100644 index 000000000..31e6a0b05 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/CHMLatencyTestMain.java @@ -0,0 +1,165 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.affinity.AffinityLock; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.values.LongValue; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +/** + * Created by peter on 28/02/14. + * 

+ * For 1M entries
+ * run 1 1000000 : 50/90/99/99.9/99.99/worst: 1.0/3.2/14/18/116/172
+ * run 1 500000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/11/16/45/163
+ * run 1 250000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/10/16/20/155
+ * run 1 100000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/9.2/15/20/147
+ * run 1 50000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/9.2/15/20/139
+ * 
+ * For 1M entries to ext4 on laptop with SSD
+ * run 1 2,000,000 : 50/90/99/99.9/99.99/worst : 0.2 / 0.7 / 8.7 / 15 / 60 / 129 micro-seconds.
+ * run 1 1,000,000 : 50/90/99/99.9/99.99/worst : 0.2 / 0.5 / 6.6 / 10 / 15 / 43 micro-seconds.
+ * run 1   500,000 : 50/90/99/99.9/99.99/worst : 0.2 / 0.5 / 5.6 / 10 / 15 / 53 micro-seconds.
+ * 
+ * For 1M entries on server to ext4 with PCI-SSD
+ * run 1 1000000 : 50/90/99/99.9/99.99/worst: 0.9/3.2/14/18/106/188
+ * run 1 500000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/11/16/46/172
+ * run 1 250000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/10/16/20/163
+ * run 1 100000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/9.9/15/20/163
+ * run 1 50000 : 50/90/99/99.9/99.99/worst: 0.2/3.0/9.9/15/20/147
+ * 
+ * For 10M entries
+ * run 1 1000000 : 50/90/99/99.9/99.99/worst: 1.4/6.2/16/34/135/180
+ * run 1 500000 : 50/90/99/99.9/99.99/worst: 0.3/3.1/11/17/54/159
+ * run 1 250000 : 50/90/99/99.9/99.99/worst: 0.3/3.0/10/15/21/147
+ * run 1 100000 : 50/90/99/99.9/99.99/worst: 0.4/3.2/9.7/15/20/151
+ * run 1 50000 : 50/90/99/99.9/99.99/worst: 0.4/3.2/9.4/15/20/147
+ * 
+ * For 100M entries
+ * run 1 1000000 : 50/90/99/99.9/99.99/worst: 570425/2818572/3355443/3422552/3489660/3556769
+ * run 1 500000 : 50/90/99/99.9/99.99/worst: 1.1/11/27/43/94/184
+ * run 1 250000 : 50/90/99/99.9/99.99/worst: 0.7/3.3/12/19/40/167
+ * run 1 100000 : 50/90/99/99.9/99.99/worst: 0.7/3.3/10/16/21/151
+ * run 1 50000 : 50/90/99/99.9/99.99/worst: 0.7/3.3/10/16/22/155
+ *
+ */ +public class CHMLatencyTestMain { + static final int KEYS = 1000 * 1000; + static final int RUN_TIME = 30; + static final long START_TIME = System.currentTimeMillis(); + + // TODO test passes but is under development. + public static void main(String... ignored) throws IOException { + AffinityLock lock = AffinityLock.acquireCore(); + File file = File.createTempFile("testCHMLatency", "deleteme"); +// File file = new File("testCHMLatency.deleteme"); + file.delete(); + ChronicleMap countersMap = ChronicleMapBuilder.of(LongValue.class, LongValue.class) + + .entries(KEYS) + .entrySize(24) + .create(file); + + // add keys + LongValue key = DataValueClasses.newInstance(LongValue.class); + LongValue value = DataValueClasses.newInstance(LongValue.class); + for (long i = 0; i < KEYS; i++) { + key.setValue(i); + value.setValue(0); + countersMap.put(key, value); + } + System.out.println("Keys created"); +// Monitor monitor = new Monitor(); + LongValue value2 = DataValueClasses.newDirectReference(LongValue.class); + for (int t = 0; t < 5; t++) { + for (int rate : new int[]{2 * 1000 * 1000, 1000 * 1000, 500 * 1000/*, 250 * 1000, 100 * 1000, 50 * 1000*/}) { + Histogram times = new Histogram(); + int u = 0; + long start = System.nanoTime(); + long delay = 1000 * 1000 * 1000L / rate; + long next = start + delay; + for (long j = 0; j < RUN_TIME * rate; j += KEYS) { + int stride = Math.max(1, KEYS / (RUN_TIME * rate)); + // the timed part + for (int i = 0; i < KEYS && u < RUN_TIME * rate; i += stride) { + // busy wait for next time. + while (System.nanoTime() < next - 12) ; +// monitor.sample = System.nanoTime(); + long start0 = next; + + // start the update. + key.setValue(i); + LongValue using = countersMap.getUsing(key, value2); + if (using == null) + assertNotNull(using); + value2.addAtomicValue(1); + + // calculate the time using the time it should have started, not when it was able. + long elapse = System.nanoTime() - start0; + times.sample(elapse); + next += delay; + } +// monitor.sample = Long.MAX_VALUE; + } + System.out.printf("run %d %,9d : ", t, rate); + times.printPercentiles(" micro-seconds."); + } + System.out.println(); + } +// monitor.running = false; + countersMap.close(); + file.delete(); + } + + static class Monitor implements Runnable { + final Thread thread; + volatile boolean running = true; + volatile long sample; + + Monitor() { + this.thread = Thread.currentThread(); + sample = Long.MAX_VALUE; + new Thread(this).start(); + } + + @Override + public void run() { + while (running) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); + break; + } + long delay = System.nanoTime() - sample; + if (delay > 1000 * 1000) { + System.out.println("\n" + (System.currentTimeMillis() - START_TIME) + " : Delay of " + delay / 100000 / 10.0 + " ms."); + int count = 0; + for (StackTraceElement ste : thread.getStackTrace()) { + System.out.println("\tat " + ste); + if (count++ > 6) break; + } + } + } + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java b/src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java new file mode 100644 index 000000000..83ad0e11b --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java @@ -0,0 +1,98 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; + +public class CHMMetaDataTest { + private static final String TMP = System.getProperty("java.io.tmpdir"); + + @Test + public void testAccessTimes() throws IOException { + File file = new File(TMP, "testAccessTimes"); + MapEventListener> listener = + new StringStringMapEventListener(new AtomicLong(1)); + ChronicleMap map = ChronicleMapBuilder.of(String.class, String.class) + .metaDataBytes(8) + .eventListener(listener) + .create(file); + + try { + map.put("a", "aye"); + map.put("b", "bee"); + map.put("c", "see"); + map.put("d", "dee"); +// assertEquals(5, timeStamps.longValue()); + assertEquals("aye", map.get("a")); + assertEquals("bee", map.get("b")); + assertEquals("see", map.get("c")); + assertEquals("dee", map.get("d")); +// assertEquals(9, timeStamps.longValue()); + assertEquals("aye", map.remove("a")); + assertEquals("bee", map.remove("b")); + assertEquals("see", map.remove("c")); + assertEquals("dee", map.remove("d")); +// assertEquals(9, timeStamps.longValue()); + } finally { + map.close(); + file.delete(); + } + + } + + private static class StringStringMapEventListener + extends MapEventListener> { + private static final long serialVersionUID = 0L; + + private final AtomicLong timeStamps; + + public StringStringMapEventListener(AtomicLong timeStamps) { + this.timeStamps = timeStamps; + } + + @Override + public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, + String key, String value) { + assertEquals(8, metaDataBytes); + entry.writeLong(0, timeStamps.incrementAndGet()); + } + + @Override + public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, + boolean added, String key, String value, long pos, + SharedSegment segment) { + assertEquals(8, metaDataBytes); + if (added) + assertEquals(0, entry.readLong()); + entry.writeLong(0, timeStamps.incrementAndGet()); + } + + @Override + public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, + String key, String value, int pos, SharedSegment segment) { + assertEquals(8, metaDataBytes); + System.out.println("Removed " + key + "/" + value + " with ts of " + entry.readLong(0)); + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java new file mode 100644 index 000000000..cfbfac040 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java @@ -0,0 +1,1280 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import com.google.common.collect.HashBiMap; +import com.google.common.primitives.Ints; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.model.DataValueGenerator; +import net.openhft.lang.values.IntValue; +import net.openhft.lang.values.LongValue; +import net.openhft.lang.values.LongValue$$Native; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static net.openhft.chronicle.map.Alignment.*; +import static org.junit.Assert.*; + +@SuppressWarnings({"unchecked", "ResultOfMethodCallIgnored"}) +public class ChronicleMapTest { + + static long count = 0; + private StringBuilder sb = new StringBuilder(); + + static void assertKeySet(Set keySet, int[] expectedKeys) { + Set expectedSet = new HashSet(); + for (int expectedKey : expectedKeys) { + expectedSet.add(expectedKey); + } + + assertEquals(expectedSet, keySet); + } + + static void assertValues(Collection values, CharSequence[] expectedValues) { + List expectedList = new ArrayList(); + for (CharSequence expectedValue : expectedValues) { + expectedList.add(expectedValue.toString()); + } + Collections.sort(expectedList); + + List actualList = new ArrayList(); + for (CharSequence actualValue : values) { + actualList.add(actualValue.toString()); + } + Collections.sort(actualList); + + assertEquals(expectedList, actualList); + } + + static void assertEntrySet(Set> entrySet, int[] expectedKeys, CharSequence[] expectedValues) { + Set> expectedSet = new HashSet>(); + for (int i = 0; i < expectedKeys.length; i++) { + expectedSet.add(new AbstractMap.SimpleEntry(expectedKeys[i], expectedValues[i])); + } + + assertEquals(expectedSet, entrySet); + } + + static void assertMap(Map map, int[] expectedKeys, CharSequence[] expectedValues) { + assertEquals(expectedKeys.length, map.size()); + for (int i = 0; i < expectedKeys.length; i++) { + assertEquals("On position " + i, expectedValues[i], map.get(expectedKeys[i])); + } + } + + public static LongValue nativeLongValue() { + return new LongValue$$Native(); + } + + public static IntValue nativeIntValue() { + return DataValueClasses.newDirectReference(IntValue.class); +// return new LongValue$$Native(); + } + + static File getPersistenceFile() { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/chm-test" + System.nanoTime() + count++); + file.deleteOnExit(); + return file; + } + + private static ChronicleMap getSharedMap( + long entries, int segments, int entrySize) throws IOException { + return getSharedMap(entries, segments, entrySize, OF_4_BYTES); + } + + private static ChronicleMap getSharedMap( + long entries, int segments, int entrySize, Alignment alignment) + throws IOException { + return ChronicleMapBuilder.of(CharSequence.class, LongValue.class) + .entries(entries) + .minSegments(segments) + .entrySize(entrySize) + .entryAndValueAlignment(alignment) + .valueMarshallerAndFactory(ByteableLongValueMarshaller.INSTANCE, + DirectLongValueFactory.INSTANCE) + .create(getPersistenceFile()); + } + + private static void printStatus() { + if (!new File("/proc/self/status").exists()) return; + try { + BufferedReader br = new BufferedReader(new FileReader("/proc/self/status")); + for (String line; (line = br.readLine()) != null; ) + if (line.startsWith("Vm")) + System.out.print(line.replaceAll(" +", " ") + ", "); + System.out.println(); + br.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testRemoveWithKey() throws Exception { + + final ChronicleMap map = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .minSegments(2) + .create(getPersistenceFile()); + + assertFalse(map.containsKey("key3")); + map.put("key1", "one"); + map.put("key2", "two"); + assertEquals(2, map.size()); + + assertTrue(map.containsKey("key1")); + assertTrue(map.containsKey("key2")); + assertFalse(map.containsKey("key3")); + + assertEquals("one", map.get("key1")); + assertEquals("two", map.get("key2")); + + final CharSequence result = map.remove("key1"); + + assertEquals(1, map.size()); + + assertEquals("one", result); + assertFalse(map.containsKey("key1")); + + assertEquals(null, map.get("key1")); + assertEquals("two", map.get("key2")); + assertFalse(map.containsKey("key3")); + + // lets add one more item for luck ! + map.put("key3", "three"); + assertEquals("three", map.get("key3")); + assertTrue(map.containsKey("key3")); + assertEquals(2, map.size()); + + // and just for kicks we'll overwrite what we have + map.put("key3", "overwritten"); + assertEquals("overwritten", map.get("key3")); + assertTrue(map.containsKey("key3")); + assertEquals(2, map.size()); + + map.close(); + } + + @Test + public void testSize() throws Exception { + + final ChronicleMap map = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .minSegments(1024) + .removeReturnsNull(true) + .create(getPersistenceFile()); + + + for (int i = 1; i < 1024; i++) { + map.put("key" + i, "value"); + assertEquals(i, map.size()); + } + + for (int i = 1023; i >= 1; ) { + map.remove("key" + i); + i--; + assertEquals(i, map.size()); + } + map.close(); + } + + @Test + @Ignore //todo: fails on my machine + public void testRemoveInteger() throws IOException { + + int count = 3000; + final ChronicleMap map = ChronicleMapBuilder.of(Object.class, Object.class) + .entrySize(count) + .minSegments(2) + .create(getPersistenceFile()); + + + for (int i = 1; i < count; i++) { + map.put(i, i); + assertEquals(i, map.size()); + } + + for (int i = count - 1; i >= 1; ) { + Integer j = (Integer) map.put(i, i); + assertEquals(i, j.intValue()); + Integer j2 = (Integer) map.remove(i); + assertEquals(i, j2.intValue()); + i--; + assertEquals(i, map.size()); + } + map.close(); + } + + @Test + public void testRemoveWithKeyAndRemoveReturnsNull() throws Exception { + + final ChronicleMap map = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .minSegments(2) + .removeReturnsNull(true) + .create(getPersistenceFile()); + + assertFalse(map.containsKey("key3")); + map.put("key1", "one"); + map.put("key2", "two"); + assertEquals(2, map.size()); + + assertTrue(map.containsKey("key1")); + assertTrue(map.containsKey("key2")); + assertFalse(map.containsKey("key3")); + + assertEquals("one", map.get("key1")); + assertEquals("two", map.get("key2")); + + final CharSequence result = map.remove("key1"); + assertEquals(null, result); + + assertEquals(1, map.size()); + + assertFalse(map.containsKey("key1")); + + assertEquals(null, map.get("key1")); + assertEquals("two", map.get("key2")); + assertFalse(map.containsKey("key3")); + + // lets add one more item for luck ! + map.put("key3", "three"); + assertEquals("three", map.get("key3")); + assertTrue(map.containsKey("key3")); + assertEquals(2, map.size()); + + // and just for kicks we'll overwrite what we have + map.put("key3", "overwritten"); + assertEquals("overwritten", map.get("key3")); + assertTrue(map.containsKey("key3")); + assertEquals(2, map.size()); + + map.close(); + } + + @Test + public void testReplaceWithKey() throws Exception { + + final ChronicleMap map = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .minSegments(2) + .create(getPersistenceFile()); + + + map.put("key1", "one"); + map.put("key2", "two"); + assertEquals(2, map.size()); + + assertEquals("one", map.get("key1")); + assertEquals("two", map.get("key2")); + + assertTrue(map.containsKey("key1")); + assertTrue(map.containsKey("key2")); + + final CharSequence result = map.replace("key1", "newValue"); + + assertEquals("one", result); + assertTrue(map.containsKey("key1")); + assertTrue(map.containsKey("key2")); + assertEquals(2, map.size()); + + assertEquals("newValue", map.get("key1")); + assertEquals("two", map.get("key2")); + + assertTrue(map.containsKey("key1")); + assertTrue(map.containsKey("key2")); + assertFalse(map.containsKey("key3")); + + assertEquals(2, map.size()); + + // let and one more item for luck ! + map.put("key3", "three"); + assertEquals(3, map.size()); + + assertTrue(map.containsKey("key1")); + assertTrue(map.containsKey("key2")); + assertTrue(map.containsKey("key3")); + assertEquals("three", map.get("key3")); + + // and just for kicks we'll overwrite what we have + map.put("key3", "overwritten"); + assertEquals("overwritten", map.get("key3")); + + assertTrue(map.containsKey("key1")); + assertTrue(map.containsKey("key2")); + assertTrue(map.containsKey("key3")); + + final CharSequence result2 = map.replace("key2", "newValue"); + + assertEquals("two", result2); + assertEquals("newValue", map.get("key2")); + + final CharSequence result3 = map.replace("rubbish", "newValue"); + assertEquals(null, result3); + + assertFalse(map.containsKey("rubbish")); + assertEquals(3, map.size()); + + map.close(); + } + + @Test + public void testReplaceWithKeyAnd2Params() throws Exception { + + final ChronicleMap map = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .minSegments(2) + .create(getPersistenceFile()); + + map.put("key1", "one"); + map.put("key2", "two"); + + assertEquals("one", map.get("key1")); + assertEquals("two", map.get("key2")); + + final boolean result = map.replace("key1", "one", "newValue"); + + assertEquals(true, result); + + assertEquals("newValue", map.get("key1")); + assertEquals("two", map.get("key2")); + + // let and one more item for luck ! + map.put("key3", "three"); + assertEquals("three", map.get("key3")); + + // and just for kicks we'll overwrite what we have + map.put("key3", "overwritten"); + assertEquals("overwritten", map.get("key3")); + + final boolean result2 = map.replace("key2", "two", "newValue2"); + + assertEquals(true, result2); + assertEquals("newValue2", map.get("key2")); + + final boolean result3 = map.replace("newKey", "", "newValue"); + assertEquals(false, result3); + + final boolean result4 = map.replace("key2", "newValue2", "newValue2"); + assertEquals(true, result4); + + map.close(); + } + + @Test + public void testRemoveWithKeyAndValue() throws Exception { + + final ChronicleMap map = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .minSegments(2) + .create(getPersistenceFile()); + + + map.put("key1", "one"); + map.put("key2", "two"); + + assertEquals("one", map.get("key1")); + assertEquals("two", map.get("key2")); + + + // a false remove + final boolean wasRemoved1 = map.remove("key1", "three"); + + assertFalse(wasRemoved1); + + + assertEquals(null, map.get("key1"), "one"); + assertEquals("two", map.get("key2"), "two"); + + map.put("key1", "one"); + + + final boolean wasRemoved2 = map.remove("key1", "three"); + assertFalse(wasRemoved2); + + // lets add one more item for luck ! + map.put("key3", "three"); + assertEquals("three", map.get("key3")); + + // and just for kicks we'll overwrite what we have + map.put("key3", "overwritten"); + assertEquals("overwritten", map.get("key3")); + + map.close(); + } + + @Test + public void testAcquireWithNullContainer() throws Exception { + ChronicleMap map = getSharedMap(10 * 1000, 128, 24); + map.acquireUsing("key", new LongValue$$Native()); + assertEquals(0, map.acquireUsing("key", null).getValue()); + + map.close(); + } + + // i7-3970X CPU @ 3.50GHz, hex core: -verbose:gc -Xmx64m + // to tmpfs file system + // 10M users, updated 12 times. Throughput 19.3 M ops/sec, no GC! + // 50M users, updated 12 times. Throughput 19.8 M ops/sec, no GC! + // 100M users, updated 12 times. Throughput 19.0M ops/sec, no GC! + // 200M users, updated 12 times. Throughput 18.4 M ops/sec, no GC! + // 400M users, updated 12 times. Throughput 18.4 M ops/sec, no GC! + + // to ext4 file system. + // 10M users, updated 12 times. Throughput 17.7 M ops/sec, no GC! + // 50M users, updated 12 times. Throughput 16.5 M ops/sec, no GC! + // 100M users, updated 12 times. Throughput 15.9 M ops/sec, no GC! + // 200M users, updated 12 times. Throughput 15.4 M ops/sec, no GC! + // 400M users, updated 12 times. Throughput 7.8 M ops/sec, no GC! + // 600M users, updated 12 times. Throughput 5.8 M ops/sec, no GC! + + // dual E5-2650v2 @ 2.6 GHz, 128 GB: -verbose:gc -Xmx32m + // to tmpfs + // TODO small GC on startup should be tidied up, [GC 9216K->1886K(31744K), 0.0036750 secs] + // 10M users, updated 16 times. Throughput 33.0M ops/sec, VmPeak: 5373848 kB, VmRSS: 544252 kB + // 50M users, updated 16 times. Throughput 31.2 M ops/sec, VmPeak: 9091804 kB, VmRSS: 3324732 kB + // 250M users, updated 16 times. Throughput 30.0 M ops/sec, VmPeak: 24807836 kB, VmRSS: 14329112 kB + // 1000M users, updated 16 times, Throughput 24.1 M ops/sec, VmPeak: 85312732 kB, VmRSS: 57165952 kB + // 2500M users, updated 16 times, Throughput 23.5 M ops/sec, VmPeak: 189545308 kB, VmRSS: 126055868 kB + + // to ext4 + // 10M users, updated 16 times. Throughput 28.4 M ops/sec, VmPeak: 5438652 kB, VmRSS: 544624 kB + // 50M users, updated 16 times. Throughput 28.2 M ops/sec, VmPeak: 9091804 kB, VmRSS: 9091804 kB + // 250M users, updated 16 times. Throughput 26.1 M ops/sec, VmPeak: 24807836 kB, VmRSS: 24807836 kB + // 1000M users, updated 16 times, Throughput 1.3 M ops/sec, TODO FIX this + + @Test + public void testGetWithNullContainer() throws Exception { + ChronicleMap map = getSharedMap(10 * 1000, 128, 24); + map.acquireUsing("key", new LongValue$$Native()); + assertEquals(0, map.getUsing("key", null).getValue()); + + map.close(); + } + + @Test + public void testGetWithoutAcquireFirst() throws Exception { + ChronicleMap map = getSharedMap(10 * 1000, 128, 24); + assertNull(map.getUsing("key", new LongValue$$Native())); + + map.close(); + } + + // i7-3970X CPU @ 3.50GHz, hex core: -Xmx30g -verbose:gc + // 10M users, updated 12 times. Throughput 16.2 M ops/sec, longest [Full GC 853669K->852546K(3239936K), 0.8255960 secs] + // 50M users, updated 12 times. Throughput 13.3 M ops/sec, longest [Full GC 5516214K->5511353K(13084544K), 3.5752970 secs] + // 100M users, updated 12 times. Throughput 11.8 M ops/sec, longest [Full GC 11240703K->11233711K(19170432K), 5.8783010 secs] + // 200M users, updated 12 times. Throughput 4.2 M ops/sec, longest [Full GC 25974721K->22897189K(27962048K), 21.7962600 secs] + + // dual E5-2650v2 @ 2.6 GHz, 128 GB: -verbose:gc -Xmx100g + // 10M users, updated 16 times. Throughput 155.3 M ops/sec, VmPeak: 113291428 kB, VmRSS: 9272176 kB, [Full GC 1624336K->1616457K(7299072K), 2.5381610 secs] + // 50M users, updated 16 times. Throughput 120.4 M ops/sec, VmPeak: 113291428 kB, VmRSS: 28436248 kB [Full GC 6545332K->6529639K(18179584K), 6.9053810 secs] + // 250M users, updated 16 times. Throughput 114.1 M ops/sec, VmPeak: 113291428 kB, VmRSS: 76441464 kB [Full GC 41349527K->41304543K(75585024K), 17.3217490 secs] + // 1000M users, OutOfMemoryError. + + @Test + public void testAcquireAndGet() throws IOException, ClassNotFoundException, + IllegalAccessException, InstantiationException { + int entries = 100 * 1000; + testAcquireAndGet(getSharedMap(entries, 128, 24, NO_ALIGNMENT), entries); + testAcquireAndGet(getSharedMap(entries, 128, 24, OF_4_BYTES), entries); + testAcquireAndGet(getSharedMap(entries, 128, 24, OF_8_BYTES), entries); + } + + public void testAcquireAndGet(ChronicleMap map, int entries) + throws IOException, ClassNotFoundException, IllegalAccessException, + InstantiationException { + LongValue value = new LongValue$$Native(); + LongValue value2 = new LongValue$$Native(); + LongValue value3 = new LongValue$$Native(); + + for (int j = 1; j <= 3; j++) { + for (int i = 0; i < entries; i++) { + CharSequence userCS = getUserCharSequence(i); + if (i * 9876 == 829682760) { + System.out.println(1); + } + + if (j > 1) { + if (i == 1) { + System.out.println(1); + } + assertNotNull(userCS.toString(), map.getUsing(userCS, value)); + } else { + map.acquireUsing(userCS, value); + } + if (i >= 1) + assertTrue(userCS.toString(), map.containsKey(getUserCharSequence(1))); + assertEquals(j - 1, value.getValue()); + + value.addAtomicValue(1); + + assertEquals(value2, map.acquireUsing(userCS, value2)); + assertEquals(j, value2.getValue()); + + assertEquals(value3, map.getUsing(userCS, value3)); + assertEquals(j, value3.getValue()); + } + } + + map.close(); + } + + @Test + public void testAcquireFromMultipleThreads() throws Exception { + int entries = 1000 * 1000; + testAcquireFromMultipleThreads(getSharedMap(entries, 128, 24, NO_ALIGNMENT)); + testAcquireFromMultipleThreads(getSharedMap(entries, 128, 24, OF_4_BYTES)); + testAcquireFromMultipleThreads(getSharedMap(entries, 128, 24, OF_8_BYTES)); + } + + public void testAcquireFromMultipleThreads(ChronicleMap map) + throws Exception { + + CharSequence key = getUserCharSequence(0); + map.acquireUsing(key, new LongValue$$Native()); + + int iterations = 1000; + int noOfThreads = 10; + CyclicBarrier barrier = new CyclicBarrier(noOfThreads); + + Thread[] threads = new Thread[noOfThreads]; + for (int t = 0; t < noOfThreads; t++) { + threads[t] = new Thread(new IncrementRunnable(map, key, iterations, barrier)); + threads[t].start(); + } + for (int t = 0; t < noOfThreads; t++) { + threads[t].join(); + } + + assertEquals(noOfThreads * iterations, map.acquireUsing(key, new LongValue$$Native()).getValue()); + + map.close(); + } + + @Test + public void testIntValue() { + DataValueGenerator dvg = new DataValueGenerator(); + dvg.setDumpCode(true); + dvg.acquireNativeClass(IntValue.class); + dvg.acquireNativeClass(LongValue.class); + } + + @Test + @Ignore + public void testAcquirePerf() + throws IOException, ClassNotFoundException, IllegalAccessException, + InstantiationException, InterruptedException, ExecutionException { +// int runs = Integer.getInteger("runs", 10); + for (int runs : new int[]{10, 50, 250, 500, 1000, 2500}) { + final long entries = runs * 1000 * 1000L; + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(CharSequence.class, IntValue.class) + .entries(entries) + .minSegments(1024) + .entrySize(20) + .valueMarshallerAndFactory(ByteableIntValueMarshaller.INSTANCE, + DirectIntValueFactory.INSTANCE); + final ChronicleMap map = builder.create(getPersistenceFile()); + + int procs = Runtime.getRuntime().availableProcessors(); + int threads = procs * 2; // runs > 100 ? procs / 2 : procs; + int count = runs > 500 ? runs > 1200 ? 1 : 3 : 5; + final int independence = 8; // Math.min(procs, runs > 500 ? 8 : 4); + System.out.println("\nKey size: " + runs + " Million entries. " + builder); + for (int j = 0; j < count; j++) { + long start = System.currentTimeMillis(); + ExecutorService es = Executors.newFixedThreadPool(procs); + List futures = new ArrayList(); + for (int i = 0; i < threads; i++) { + final int t = i; + futures.add(es.submit(new Runnable() { + @Override + public void run() { + IntValue value = nativeIntValue(); + StringBuilder sb = new StringBuilder(); + long next = 50 * 1000 * 1000; + // use a factor to give up to 10 digit numbers. + int factor = Math.max(1, (int) ((10 * 1000 * 1000 * 1000L - 1) / entries)); + for (long j = t % independence; j < entries + independence - 1; j += independence) { + sb.setLength(0); + sb.append("us:"); + sb.append(j * factor); + map.acquireUsing(sb, value); + long n = value.addAtomicValue(1); + assert n > 0 && n < 1000 : "Counter corrupted " + n; + if (t == 0 && j >= next) { + long size = map.longSize(); + if (size < 0) throw new AssertionError("size: " + size); + System.out.println(j + ", size: " + size); + next += 50 * 1000 * 1000; + } + } + } + })); + } + for (Future future : futures) { + future.get(); + } + es.shutdown(); + es.awaitTermination(runs / 10 + 1, TimeUnit.MINUTES); + long time = System.currentTimeMillis() - start; + System.out.printf("Throughput %.1f M ops/sec%n", threads * entries / independence / 1000.0 / time); + } + printStatus(); + File file = map.file(); + map.close(); + file.delete(); + } + } + + @Test + @Ignore + public void testCHMAcquirePerf() throws IOException, ClassNotFoundException, IllegalAccessException, InstantiationException, InterruptedException { + for (int runs : new int[]{10, 50, 250, 1000, 2500}) { + System.out.println("Testing " + runs + " million entries"); + final long entries = runs * 1000 * 1000L; + final ConcurrentMap map = new ConcurrentHashMap((int) (entries * 5 / 4), 1.0f, 1024); + + int procs = Runtime.getRuntime().availableProcessors(); + int threads = procs * 2; + int count = runs > 500 ? runs > 1200 ? 1 : 2 : 3; + final int independence = Math.min(procs, runs > 500 ? 8 : 4); + for (int j = 0; j < count; j++) { + long start = System.currentTimeMillis(); + ExecutorService es = Executors.newFixedThreadPool(procs); + for (int i = 0; i < threads; i++) { + final int t = i; + es.submit(new Runnable() { + @Override + public void run() { + StringBuilder sb = new StringBuilder(); + int next = 50 * 1000 * 1000; + // use a factor to give up to 10 digit numbers. + int factor = Math.max(1, (int) ((10 * 1000 * 1000 * 1000L - 1) / entries)); + for (long i = t % independence; i < entries; i += independence) { + sb.setLength(0); + sb.append("u:"); + sb.append(i * factor); + String key = sb.toString(); + AtomicInteger count = map.get(key); + if (count == null) { + map.put(key, new AtomicInteger()); + count = map.get(key); + } + count.getAndIncrement(); + if (t == 0 && i == next) { + System.out.println(i); + next += 50 * 1000 * 1000; + } + } + } + }); + } + es.shutdown(); + es.awaitTermination(10, TimeUnit.MINUTES); + printStatus(); + long time = System.currentTimeMillis() - start; + System.out.printf("Throughput %.1f M ops/sec%n", threads * entries / 1000.0 / time); + } + } + } + + private CharSequence getUserCharSequence(int i) { + sb.setLength(0); + sb.append("u:"); + sb.append(i * 9876); // test 10 digit user numbers. + return sb; + } + + @Test + public void testPutAndRemove() throws IOException, ClassNotFoundException, IllegalAccessException, InstantiationException { + String TMP = System.getProperty("java.io.tmpdir"); + int entries = 100 * 1000; + ChronicleMap map = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(entries) + .minSegments(16) + .entrySize(32) + .putReturnsNull(true) + .removeReturnsNull(true) + .create(getPersistenceFile()); + StringBuilder key = new StringBuilder(); + StringBuilder value = new StringBuilder(); + StringBuilder value2 = new StringBuilder(); + for (int j = 1; j <= 3; j++) { + for (int i = 0; i < entries; i++) { + key.setLength(0); + key.append("user:").append(i); + value.setLength(0); + value.append("value:").append(i); +// System.out.println(key); + assertNull(map.getUsing(key, value)); + assertNull(map.put(key, value)); + assertNotNull(map.getUsing(key, value2)); + assertEquals(value.toString(), value2.toString()); + assertNull(map.remove(key)); + assertNull(map.getUsing(key, value)); + } + } + + map.close(); + } + + @Test + public void mapRemoveReflectedInViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + map.remove(2); + assertMap(map, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertEntrySet(entrySet, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertEntrySet(map.entrySet(), new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertKeySet(keySet, new int[]{1, 3}); + assertKeySet(map.keySet(), new int[]{1, 3}); + assertValues(values, new CharSequence[]{"1", "3"}); + assertValues(map.values(), new CharSequence[]{"1", "3"}); + + map.close(); + } + + @Test + public void mapPutReflectedInViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + map.put(4, "4"); + assertMap(map, new int[]{4, 2, 3, 1}, new CharSequence[]{"4", "2", "3", "1"}); + assertEntrySet(entrySet, new int[]{4, 2, 3, 1}, new CharSequence[]{"4", "2", "3", "1"}); + assertEntrySet(map.entrySet(), new int[]{4, 2, 3, 1}, new CharSequence[]{"4", "2", "3", "1"}); + assertKeySet(keySet, new int[]{4, 2, 3, 1}); + assertKeySet(map.keySet(), new int[]{4, 2, 3, 1}); + assertValues(values, new CharSequence[]{"2", "1", "4", "3"}); + assertValues(map.values(), new CharSequence[]{"2", "1", "4", "3"}); + + map.close(); + } + + @Test + public void entrySetRemoveReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + entrySet.remove(new AbstractMap.SimpleEntry(2, "2")); + assertMap(map, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertEntrySet(entrySet, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertKeySet(keySet, new int[]{1, 3}); + assertValues(values, new CharSequence[]{"1", "3"}); + + map.close(); + } + + @Test + public void keySetRemoveReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + keySet.remove(2); + assertMap(map, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertEntrySet(entrySet, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertKeySet(keySet, new int[]{1, 3}); + assertValues(values, new CharSequence[]{"1", "3"}); + + map.close(); + } + + @Test + public void valuesRemoveReflectedInMap() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + values.remove("2"); + assertMap(map, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertEntrySet(entrySet, new int[]{1, 3}, new CharSequence[]{"1", "3"}); + assertKeySet(keySet, new int[]{1, 3}); + assertValues(values, new CharSequence[]{"1", "3"}); + + map.close(); + } + + @Test + public void entrySetIteratorRemoveReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Map refMap = new HashMap(map); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + Iterator> entryIterator = entrySet.iterator(); + entryIterator.next(); + refMap.remove(entryIterator.next().getKey()); + entryIterator.remove(); + int[] expectedKeys = Ints.toArray(refMap.keySet()); + CharSequence[] expectedValues = refMap.values().toArray(new CharSequence[0]); + assertMap(map, expectedKeys, expectedValues); + assertEntrySet(entrySet, expectedKeys, expectedValues); + assertKeySet(keySet, expectedKeys); + assertValues(values, expectedValues); + + map.close(); + } + + @Test + public void keySetIteratorRemoveReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Map refMap = new HashMap(map); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + Iterator keyIterator = keySet.iterator(); + keyIterator.next(); + refMap.remove(keyIterator.next()); + keyIterator.remove(); + int[] expectedKeys = Ints.toArray(refMap.keySet()); + CharSequence[] expectedValues = refMap.values().toArray(new CharSequence[0]); + assertMap(map, expectedKeys, expectedValues); + assertEntrySet(entrySet, expectedKeys, expectedValues); + assertKeySet(keySet, expectedKeys); + assertValues(values, expectedValues); + + map.close(); + } + + @Test + public void valuesIteratorRemoveReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + HashBiMap refMap = HashBiMap.create(map); + + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + Iterator valueIterator = values.iterator(); + valueIterator.next(); + refMap.inverse().remove(valueIterator.next()); + valueIterator.remove(); + int[] expectedKeys = Ints.toArray(refMap.keySet()); + CharSequence[] expectedValues = new CharSequence[expectedKeys.length]; + for (int i = 0; i < expectedKeys.length; i++) { + expectedValues[i] = refMap.get(expectedKeys[i]); + } + assertMap(map, expectedKeys, expectedValues); + assertEntrySet(entrySet, expectedKeys, expectedValues); + assertKeySet(keySet, expectedKeys); + assertValues(values, expectedValues); + + map.close(); + } + + @Test + public void entrySetRemoveAllReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + entrySet.removeAll( + Arrays.asList( + new AbstractMap.SimpleEntry(1, "1"), + new AbstractMap.SimpleEntry(2, "2") + ) + ); + assertMap(map, new int[]{3}, new CharSequence[]{"3"}); + assertEntrySet(entrySet, new int[]{3}, new CharSequence[]{"3"}); + assertKeySet(keySet, new int[]{3}); + assertValues(values, new CharSequence[]{"3"}); + + map.close(); + } + + @Test + public void keySetRemoveAllReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + keySet.removeAll(Arrays.asList(1, 2)); + assertMap(map, new int[]{3}, new CharSequence[]{"3"}); + assertEntrySet(entrySet, new int[]{3}, new CharSequence[]{"3"}); + assertKeySet(keySet, new int[]{3}); + assertValues(values, new CharSequence[]{"3"}); + + map.close(); + } + + @Test + public void valuesRemoveAllReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + values.removeAll(Arrays.asList("1", "2")); + assertMap(map, new int[]{3}, new CharSequence[]{"3"}); + assertEntrySet(entrySet, new int[]{3}, new CharSequence[]{"3"}); + assertKeySet(keySet, new int[]{3}); + assertValues(values, new CharSequence[]{"3"}); + + map.close(); + } + + @Test + public void entrySetRetainAllReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + entrySet.retainAll( + Arrays.asList( + new AbstractMap.SimpleEntry(1, "1"), + new AbstractMap.SimpleEntry(2, "2") + ) + ); + assertMap(map, new int[]{2, 1}, new CharSequence[]{"2", "1"}); + assertEntrySet(entrySet, new int[]{2, 1}, new CharSequence[]{"2", "1"}); + assertKeySet(keySet, new int[]{2, 1}); + assertValues(values, new CharSequence[]{"2", "1"}); + + map.close(); + } + + @Test + public void keySetRetainAllReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + keySet.retainAll(Arrays.asList(1, 2)); + assertMap(map, new int[]{2, 1}, new CharSequence[]{"2", "1"}); + assertEntrySet(entrySet, new int[]{2, 1}, new CharSequence[]{"2", "1"}); + assertKeySet(keySet, new int[]{2, 1}); + assertValues(values, new CharSequence[]{"2", "1"}); + + map.close(); + } + + @Test + public void valuesRetainAllReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + values.retainAll(Arrays.asList("1", "2")); + assertMap(map, new int[]{2, 1}, new CharSequence[]{"2", "1"}); + assertEntrySet(entrySet, new int[]{2, 1}, new CharSequence[]{"2", "1"}); + assertKeySet(keySet, new int[]{2, 1}); + assertValues(values, new CharSequence[]{"2", "1"}); + + map.close(); + } + + @Test + public void entrySetClearReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + entrySet.clear(); + org.junit.Assert.assertTrue(map.isEmpty()); + org.junit.Assert.assertTrue(entrySet.isEmpty()); + org.junit.Assert.assertTrue(keySet.isEmpty()); + org.junit.Assert.assertTrue(values.isEmpty()); + + map.close(); + } + + @Test + public void keySetClearReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + keySet.clear(); + org.junit.Assert.assertTrue(map.isEmpty()); + org.junit.Assert.assertTrue(entrySet.isEmpty()); + org.junit.Assert.assertTrue(keySet.isEmpty()); + org.junit.Assert.assertTrue(values.isEmpty()); + + map.close(); + } + + @Test + public void valuesClearReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(3); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + values.clear(); + org.junit.Assert.assertTrue(map.isEmpty()); + org.junit.Assert.assertTrue(entrySet.isEmpty()); + org.junit.Assert.assertTrue(keySet.isEmpty()); + org.junit.Assert.assertTrue(values.isEmpty()); + + map.close(); + } + + @Test + public void clearMapViaEntryIteratorRemoves() throws IOException { + int noOfElements = 16 * 1024; + ChronicleMap map = getViewTestMap(noOfElements); + + int sum = 0; + for (Iterator it = map.entrySet().iterator(); it.hasNext(); ) { + it.next(); + it.remove(); + ++sum; + } + map.close(); + + assertEquals(noOfElements, sum); + } + + @Test + public void clearMapViaKeyIteratorRemoves() throws IOException { + int noOfElements = 16 * 1024; + ChronicleMap map = getViewTestMap(noOfElements); + + Set keys = new HashSet(); + for (int i = 1; i <= noOfElements; i++) { + keys.add(i); + } + + int sum = 0; + for (Iterator it = map.keySet().iterator(); it.hasNext(); ) { + Object key = it.next(); + keys.remove(key); + it.remove(); + ++sum; + } + map.close(); + + assertEquals(noOfElements, sum); + } + + @Test(expected = IllegalStateException.class) + public void testRemoveWhenNextIsNotCalled() throws IOException { + + ChronicleMap map = getViewTestMap(2); + + Iterator iterator = map.keySet().iterator(); + iterator.remove(); + } + + @Test + public void clearMapViaValueIteratorRemoves() throws IOException { + int noOfElements = 16 * 1024; + ChronicleMap map = getViewTestMap(noOfElements); + + int sum = 0; + for (Iterator it = map.values().iterator(); it.hasNext(); ) { + it.next(); + it.remove(); + ++sum; + } + map.close(); + + assertEquals(noOfElements, sum); + } + + @Test + public void entrySetValueReflectedInMapAndOtherViews() throws IOException { + ChronicleMap map = getViewTestMap(0); + + map.put(1, "A"); + Set> entrySet = map.entrySet(); + Set keySet = map.keySet(); + Collection values = map.values(); + + assertMap(map, new int[]{1}, new CharSequence[]{"A"}); + assertEntrySet(entrySet, new int[]{1}, new CharSequence[]{"A"}); + assertKeySet(keySet, new int[]{1}); + assertValues(values, new String[]{"A"}); + + entrySet.iterator().next().setValue("B"); + assertMap(map, new int[]{1}, new CharSequence[]{"B"}); + assertEntrySet(entrySet, new int[]{1}, new CharSequence[]{"B"}); + assertEntrySet(map.entrySet(), new int[]{1}, new CharSequence[]{"B"}); + assertKeySet(keySet, new int[]{1}); + assertKeySet(map.keySet(), new int[]{1}); + assertValues(values, new String[]{"B"}); + assertValues(map.values(), new String[]{"B"}); + + map.close(); + } + + private ChronicleMap getViewTestMap(int noOfElements) throws IOException { + int entries = 100 * 1000; + ChronicleMap map = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(entries) + .minSegments(16) + .entrySize(32) + .putReturnsNull(true) + .removeReturnsNull(true) + .create(getPersistenceFile()); + + int[] expectedKeys = new int[noOfElements]; + String[] expectedValues = new String[noOfElements]; + for (int i = 1; i <= noOfElements; i++) { + String value = "" + i; + map.put(i, value); + expectedKeys[i - 1] = i; + expectedValues[i - 1] = value; + } + + return map; + } + + @Test + public void testOversizeEntriesPutRemoveReplace() throws IOException { + ChronicleMapBuilder builder = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(2) + .minSegments(1) + .entrySize(10); + builder.entryAndValueAlignment(NO_ALIGNMENT); + testOversizeEntriesPutRemoveReplace( + (VanillaChronicleMap) + builder.create(getPersistenceFile()) + ); + builder.entryAndValueAlignment(Alignment.OF_4_BYTES); + testOversizeEntriesPutRemoveReplace( + (VanillaChronicleMap) + builder.create(getPersistenceFile()) + ); + builder.entryAndValueAlignment(OF_8_BYTES); + testOversizeEntriesPutRemoveReplace( + (VanillaChronicleMap) + builder.create(getPersistenceFile()) + ); + } + + public void testOversizeEntriesPutRemoveReplace( + VanillaChronicleMap map) { + String key = "k"; + String oversizedKey = "oversized key"; + String value = "v"; + String oversizedValue = "oversized value"; + + map.put(key, oversizedValue); + assertEquals(oversizedValue, map.get(key)); + map.replace(key, value); + map.checkConsistency(); + assertEquals(value, map.get(key)); + map.replace(key, oversizedValue); + map.checkConsistency(); + assertEquals(oversizedValue, map.get(key)); + + map.put(oversizedKey, value); + map.checkConsistency(); + assertEquals(value, map.get(oversizedKey)); + map.replace(oversizedKey, oversizedValue); + map.checkConsistency(); + assertEquals(oversizedValue, map.get(oversizedKey)); + map.remove(oversizedKey); + map.checkConsistency(); + assertEquals(null, map.get(oversizedKey)); + map.put(oversizedKey, oversizedValue); + map.checkConsistency(); + assertEquals(oversizedValue, map.get(oversizedKey)); + map.replace(oversizedKey, oversizedValue, value); + map.checkConsistency(); + assertEquals(value, map.get(oversizedKey)); + + assertEquals(value, map.remove(oversizedKey)); + map.checkConsistency(); + assertEquals(oversizedValue, map.remove(key)); + map.checkConsistency(); + + map.close(); + } + + @Test + public void equalsTest() throws IOException { + final ChronicleMap map1 = ChronicleMapBuilder + .of(Integer.class, String.class).create(getPersistenceFile()); + + map1.put(1, "one"); + map1.put(2, "two"); + + final ChronicleMap map2 = ChronicleMapBuilder + .of(Integer.class, String.class).create(getPersistenceFile()); + + map2.put(1, "one"); + map2.put(2, "two"); + + assertEquals(map1, map2); + } + + private static final class IncrementRunnable implements Runnable { + + private final ChronicleMap map; + + private final CharSequence key; + + private final int iterations; + + private final CyclicBarrier barrier; + + private IncrementRunnable(ChronicleMap map, CharSequence key, int iterations, CyclicBarrier barrier) { + this.map = map; + this.key = key; + this.iterations = iterations; + this.barrier = barrier; + } + + @Override + public void run() { + try { + LongValue value = new LongValue$$Native(); + barrier.await(); + for (int i = 0; i < iterations; i++) { + map.acquireUsing(key, value); + value.addAtomicValue(1); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ClusterReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ClusterReplicationTest.java new file mode 100644 index 000000000..34b9a63c9 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ClusterReplicationTest.java @@ -0,0 +1,144 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import org.junit.*; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static org.junit.Assert.assertTrue; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes + * + * @author Rob Austin. + */ +public class ClusterReplicationTest { + + private ChronicleMap map1a; + private ChronicleMap map2a; + + private ChronicleMap map1b; + private ChronicleMap map2b; + + private ReplicatingCluster clusterB; + private ReplicatingCluster clusterA; + + + @Before + + public void setup() throws IOException { + { + TcpReplicationConfig tcpConfig = TcpReplicationConfig + .of(8086, new InetSocketAddress("localhost", 8087)) + .heartBeatInterval(1, SECONDS); + + clusterA = new ReplicatingClusterBuilder((byte) 1, 1024).tcpReplication(tcpConfig) + .create(); + + map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterA.channelReplicator((short) 1)) + .create(getPersistenceFile()); + } + + { + TcpReplicationConfig tcpConfig = + TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); + + clusterB = new ReplicatingClusterBuilder((byte) 2, 1024).tcpReplication(tcpConfig) + .create(); + + map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterB.channelReplicator((short) 1)) + .create(getPersistenceFile()); + } + } + + @After + public void tearDown() throws InterruptedException { + + for (final Closeable closeable : new Closeable[]{clusterA, clusterB}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + @Test + @Ignore + public void test() throws IOException, InterruptedException { + + // todo remove this sleep + Thread.sleep(100); + + map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterB.channelReplicator((short) 2)) + .create(getPersistenceFile()); + + + map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterA.channelReplicator((short) 2)) + .create(getPersistenceFile()); + + map2a.put(1, "EXAMPLE-2"); + map1a.put(1, "EXAMPLE-1"); + + // allow time for the recompilation to resolve + waitTillEqual(2500); + + Assert.assertEquals("map1a=map1b", map1a, map1b); + Assert.assertEquals("map2a=map2b", map2a, map2b); + + assertTrue("map1a.empty", !map1a.isEmpty()); + assertTrue("map2a.empty", !map2a.isEmpty()); + + } + + + /** + * waits until map1 and map2 show the same value + * + * @param timeOutMs timeout in milliseconds + * @throws InterruptedException + */ + + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + for (int t = 0; t < timeOutMs; t++) { + if (map1a.equals(map1b) && + map2a.equals(map2b)) + break; + Thread.sleep(1); + } + + } + +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java b/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java index c72a535a5..cbc5b4879 100644 --- a/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java +++ b/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java @@ -1,14 +1,12 @@ -package net.openhft.chronicle.map;/* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,6 +14,8 @@ package net.openhft.chronicle.map;/* * limitations under the License. */ +package net.openhft.chronicle.map; + /* * Written by Doug Lea with assistance from members of JCP JSR-166 * Expert Group and released to the public domain, as explained at @@ -25,7 +25,7 @@ package net.openhft.chronicle.map;/* /*import junit.framework.Test; import junit.framework.TestSuite;*/ -import net.openhft.collections.jrs166.JSR166TestCase; +import net.openhft.chronicle.map.jrs166.JSR166TestCase; import org.junit.Test; import java.io.File; @@ -39,8 +39,9 @@ import static org.junit.Assert.*; public class ConcurrentMap8Test extends JSR166TestCase { + static final int SIZE = 10000; private static final double EPSILON = 1E-5; - + static ConcurrentMap longMap; private static ConcurrentMap newLongMap() { //First create (or access if already created) the shared map @@ -52,9 +53,9 @@ public class ConcurrentMap8Test extends JSR166TestCase { assertEquals(8, builder.minSegments()); //// end of test - String shmPath = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "SHMTest" + System.nanoTime(); + String chmPath = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "CHMTest" + System.nanoTime(); try { - return builder.file(new File(shmPath)).create(); + return builder.create(new File(chmPath)); } catch (IOException e) { throw new IllegalStateException(e); } @@ -75,10 +76,10 @@ public class ConcurrentMap8Test extends JSR166TestCase { assertEquals(8, builder.minSegments()); //// end of test - String shmPath = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + - "SHMTest" + System.nanoTime(); + String chmPath = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + + "CHMTest" + System.nanoTime(); try { - return builder.file(new File(shmPath)).create(); + return builder.create(new File(chmPath)); } catch (IOException e) { throw new IllegalStateException(e); } @@ -104,7 +105,6 @@ public class ConcurrentMap8Test extends JSR166TestCase { return map; } - /** * Returns a new map from Integers 1-5 to Strings "A"-"E". */ @@ -123,6 +123,36 @@ public class ConcurrentMap8Test extends JSR166TestCase { return map; } + static Set populatedSet(int n) { + Set a = ConcurrentHashMap.newKeySet(); + assertTrue(a.isEmpty()); + for (int i = 0; i < n; i++) + a.add(i); + assertFalse(a.isEmpty()); + assertEquals(n, a.size()); + return a; + } + + static Set populatedSet(Integer[] elements) { + Set a = ConcurrentHashMap.newKeySet(); + assertTrue(a.isEmpty()); + for (int i = 0; i < elements.length; i++) + a.add(elements[i]); + assertFalse(a.isEmpty()); + assertEquals(elements.length, a.size()); + return a; + } + + static ConcurrentMap longMap() { + if (longMap == null) { + longMap = newLongMap(); + for (int i = 0; i < SIZE; ++i) + longMap.put(Long.valueOf(i), Long.valueOf(2 * i)); + } + + return longMap; + } + /** * getOrDefault returns value if present, else default */ @@ -250,26 +280,6 @@ public class ConcurrentMap8Test extends JSR166TestCase { assertFalse(map.containsKey(one)); } - static Set populatedSet(int n) { - Set a = ConcurrentHashMap.newKeySet(); - assertTrue(a.isEmpty()); - for (int i = 0; i < n; i++) - a.add(i); - assertFalse(a.isEmpty()); - assertEquals(n, a.size()); - return a; - } - - static Set populatedSet(Integer[] elements) { - Set a = ConcurrentHashMap.newKeySet(); - assertTrue(a.isEmpty()); - for (int i = 0; i < elements.length; i++) - a.add(elements[i]); - assertFalse(a.isEmpty()); - assertEquals(elements.length, a.size()); - return a; - } - /* * replaceAll replaces all matching values. */ @@ -298,7 +308,6 @@ public class ConcurrentMap8Test extends JSR166TestCase { assertTrue(a.isEmpty()); } - /** * keySet.addAll adds each element from the given collection */ @@ -605,20 +614,6 @@ public class ConcurrentMap8Test extends JSR166TestCase { assertEquals(y, x); } - static final int SIZE = 10000; - static ConcurrentMap longMap; - - static ConcurrentMap longMap() { - if (longMap == null) { - longMap = newLongMap(); - for (int i = 0; i < SIZE; ++i) - longMap.put(Long.valueOf(i), Long.valueOf(2 * i)); - } - - return longMap; - } - - // explicit function class to avoid type inference problems static class AddKeys implements BiFunction, Map.Entry, Map.Entry> { public Map.Entry apply(Map.Entry x, Map.Entry y) { diff --git a/src/test/java/net/openhft/chronicle/map/GuavaTest.java b/src/test/java/net/openhft/chronicle/map/GuavaTest.java new file mode 100644 index 000000000..116170253 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/GuavaTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import com.google.common.collect.testing.MapTestSuiteBuilder; +import com.google.common.collect.testing.SampleElements; +import com.google.common.collect.testing.TestMapGenerator; +import com.google.common.collect.testing.features.CollectionFeature; +import com.google.common.collect.testing.features.CollectionSize; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.testing.features.MapFeature.*; +import static net.openhft.chronicle.map.ChronicleMapTest.getPersistenceFile; + + +public class GuavaTest extends TestCase { + + public static Test suite() { + TestSuite hhmTests = MapTestSuiteBuilder.using(new HHMTestGenerator()) + .named("HugeHashMap Guava tests") + .withFeatures(GENERAL_PURPOSE) + .withFeatures(CollectionSize.ANY) + .withFeatures(CollectionFeature.REMOVE_OPERATIONS) + .withFeatures(RESTRICTS_KEYS, RESTRICTS_VALUES) + .createTestSuite(); + TestSuite tests = new TestSuite(); + tests.addTest(hhmTests); + return tests; + } + + static abstract class TestGenerator + implements TestMapGenerator { + + abstract Map newMap(); + + public CharSequence[] createKeyArray(int length) { + return new CharSequence[length]; + } + + @Override + public CharSequence[] createValueArray(int length) { + return new CharSequence[length]; + } + + @Override + public SampleElements> samples() { + return SampleElements.mapEntries( + new SampleElements( + "key1", "key2", "key3", "key4", "key5" + ), + new SampleElements( + "val1", "val2", "val3", "val4", "val5" + ) + ); + } + + @Override + public Map create(Object... objects) { + Map map = newMap(); + for (Object obj : objects) { + Map.Entry e = (Map.Entry) obj; + map.put((CharSequence) e.getKey(), + (CharSequence) e.getValue()); + } + return map; + } + + @Override + public Map.Entry[] createArray(int length) { + return new Map.Entry[length]; + } + + @Override + public Iterable> order( + List> insertionOrder) { + return insertionOrder; + } + } + + static class CHMTestGenerator extends TestGenerator { + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(100) + .minSegments(2); + + @Override + Map newMap() { + try { + return builder.create(getPersistenceFile()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + static class HHMTestGenerator extends CHMTestGenerator { + @Override + Map newMap() { + try { + return builder.create(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/Histogram.java b/src/test/java/net/openhft/chronicle/map/Histogram.java new file mode 100644 index 000000000..23b052db5 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/Histogram.java @@ -0,0 +1,84 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +/** + * Created by peter on 28/02/14. + */ +public class Histogram { + static final String[] FRACTION_STR = "worst,99.99,99.9,99,90,50".split(","); + static final int[] FRACTIONS = {Integer.MAX_VALUE, 10000, 1000, 100, 10, 2}; + static final int BITS_OF_ACCURACY = 5; + final int[] counters = new int[40 << BITS_OF_ACCURACY]; + static final int MANTISSA = 52; + static final int BITS_TO_TRUNCATE = MANTISSA - BITS_OF_ACCURACY; + long count = 0; + + public void sample(long value) { + long rawValue = Double.doubleToRawLongBits(value) >> BITS_TO_TRUNCATE; + int bucket = (int) rawValue - (1023 << BITS_OF_ACCURACY); + counters[bucket]++; + count++; + } + + public void printResults() { + for (int i = 0; i < counters.length; i++) { + if (counters[i] == 0) continue; + double d = indexToDouble(i); + if (d < 10000 && d * counters[i] < 20000) continue; + System.out.println((long) d + ":" + counters[i]); + } + } + + private double indexToDouble(int i) { + return Double.longBitsToDouble((i + (1023L << BITS_OF_ACCURACY)) << BITS_TO_TRUNCATE); + } + + public void printPercentiles() { + printPercentiles(""); + } + + public void printPercentiles(String tail) { + long[] times = new long[FRACTIONS.length]; + int f = 0; + long total = 0; + for (int pos = counters.length - 1; pos >= 0 && f < FRACTIONS.length; pos--) { + total += counters[pos]; + if (total > count / FRACTIONS[f]) { + times[f++] = (long) indexToDouble(pos); + } + } + // print in reverse order. + StringBuilder sb = new StringBuilder(); + for (int i = FRACTIONS.length - 1; i >= 0; i--) { + sb.append(FRACTION_STR[i]).append("/"); + } + sb.setLength(sb.length() - 1); + sb.append(" : "); + for (int i = FRACTIONS.length - 1; i >= 0; i--) { + long time = times[i]; + if (time < 10000) + sb.append(time / 100 / 10.0); + else + sb.append(time / 1000); + sb.append(" / "); + } + sb.setLength(sb.length() - 3); + sb.append(tail); + System.out.println(sb); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/HistogramTest.java b/src/test/java/net/openhft/chronicle/map/HistogramTest.java new file mode 100644 index 000000000..9d8809ae0 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/HistogramTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.Test; + +/** + * Created by peter on 28/02/14. + */ +public class HistogramTest { + + @Test + public void testHistogram() { + Histogram hist = new Histogram(); + hist.sample(1); + hist.sample(10); + hist.sample(100); + hist.printResults(); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/IntIntMultiMapTest.java b/src/test/java/net/openhft/chronicle/map/IntIntMultiMapTest.java new file mode 100644 index 000000000..e5381c27f --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/IntIntMultiMapTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * User: peter Date: 09/12/13 + */ +@RunWith(value = Parameterized.class) +public class IntIntMultiMapTest { + + IntIntMultiMap map; + Multimap referenceMap = HashMultimap.create(); + private Class c; + + public IntIntMultiMapTest(Class c) + throws Exception { + this.c = c; + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][]{ + {VanillaIntIntMultiMap.class}, + {VanillaShortShortMultiMap.class} + }); + } + + private void initMap(int capacity) { + try { + map = c.getConstructor(int.class).newInstance(capacity); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void multiMapEquals() { + class Action implements IntIntMultiMap.EntryConsumer { + int mapSize = 0; + + @Override + public void accept(int key, int value) { + mapSize++; + assertTrue(referenceMap.containsEntry(key, value)); + } + } + Action action = new Action(); + map.forEach(action); + assertEquals(referenceMap.size(), action.mapSize); + } + + private void valuesEqualsByKey(int k) { + List values = new ArrayList(); + map.startSearch(k); + int v; + while ((v = map.nextPos()) >= 0) + values.add(v); + Set valueSet = new HashSet(values); + assertEquals(values.size(), valueSet.size()); + assertEquals(new HashSet(referenceMap.get(k)), valueSet); + } + + private void put(int k, int v) { + map.put(k, v); + referenceMap.put(k, v); + } + + private void remove(int k, int v, boolean present) { + assertEquals(present, map.remove(k, v)); + referenceMap.remove(k, v); + } + + @Test + public void testPutRemoveSearch() { + initMap(16); + multiMapEquals(); + put(1, 11); + valuesEqualsByKey(1); + + multiMapEquals(); + put(3, 33); + multiMapEquals(); + put(1, 12); + put(1, 13); + put(1, 14); + put(3, 32); + put(1, 15); + multiMapEquals(); + + remove(1, 11, true); + multiMapEquals(); + remove(1, 11, false); + + valuesEqualsByKey(3); + valuesEqualsByKey(1); + + remove(1, 12, true); + multiMapEquals(); + + remove(1, 15, true); + multiMapEquals(); + + remove(1, 13, true); + multiMapEquals(); + + remove(1, 14, true); + multiMapEquals(); + } + + @Test + public void testRemoveSpecific() { + // Testing a specific case when the remove method on the map + // does (did) not work as expected. The size goes correctly to + // 0 but the value is still present in the map. + initMap(10); + + map.put(15, 1); + map.remove(15, 1); + map.startSearch(15); + assertTrue(map.nextPos() < 0); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/IntValueMapTest.java b/src/test/java/net/openhft/chronicle/map/IntValueMapTest.java new file mode 100644 index 000000000..1f2fb566b --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/IntValueMapTest.java @@ -0,0 +1,58 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.values.IntValue; +import net.openhft.lang.values.IntValue$$Native; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static org.junit.Assert.assertEquals; + + +/** + * @author Rob Austin. + */ +public class IntValueMapTest { + + @Test + public void test() throws IOException { + + final ChronicleMap map = ChronicleMapBuilder + .of(IntValue.class, CharSequence.class) + .entries(20000) + .keyMarshaller(ByteableIntValueMarshaller.INSTANCE) + .create(getPersistenceFile()); + + IntValue$$Native value = new IntValue$$Native(); + value.bytes(new ByteBufferBytes(ByteBuffer.allocateDirect(4)), 0); + + value.setValue(1); + final String expected = "test"; + map.put(value, expected); + + final CharSequence actual = map.get(value); + assertEquals(expected, actual); + + // this will fail + map.toString(); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/KeySizesTest.java b/src/test/java/net/openhft/chronicle/map/KeySizesTest.java new file mode 100644 index 000000000..6b651851e --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/KeySizesTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.Test; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * Created by peter on 28/02/14. + */ +public class KeySizesTest { + @Test + public void testDifferentKeySizes() throws IOException { + File tempFile = File.createTempFile("delete", "me"); + Map map = ChronicleMapBuilder.of(String.class, String.class) + + .create(tempFile); + + String k = ""; + for (int i = 0; i < 100; i++) { + map.put(k, k); + String k2 = map.get(k); + assertEquals(k, k2); + k += "a"; + } + k = ""; + for (int i = 0; i < 100; i++) { + String k2 = map.get(k); + assertEquals(k, k2); + k += "a"; + } + + ((Closeable) map).close(); + tempFile.delete(); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/MultiMapTimeBaseReplicationTest.java b/src/test/java/net/openhft/chronicle/map/MultiMapTimeBaseReplicationTest.java new file mode 100644 index 000000000..f62efe0d5 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/MultiMapTimeBaseReplicationTest.java @@ -0,0 +1,278 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.concurrent.ArrayBlockingQueue; + +import static net.openhft.chronicle.map.Replica.ModificationNotifier.NOP; +import static org.junit.Assert.assertEquals; + +/** + * @author Rob Austin. + */ +public class MultiMapTimeBaseReplicationTest { + + private Replica.ModificationIterator segmentModificationIterator2; + private Replica.ModificationIterator segmentModificationIterator1; + private Builder.MapProvider> mapP1; + private Builder.MapProvider> mapP2; + private ReplicatedChronicleMap map2; + private ReplicatedChronicleMap map1; + + + private ArrayBlockingQueue map1ToMap2; + private ArrayBlockingQueue map2ToMap1; + + @Before + public void setup() throws IOException { + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + + map1ToMap2 = new ArrayBlockingQueue(10000); + map2ToMap1 = new ArrayBlockingQueue(10000); + + mapP1 = Builder.newShmIntInt(20000, map2ToMap1, map1ToMap2, (byte) 1, (byte) 2); + map1 = mapP1.getMap(); + segmentModificationIterator1 = map1.acquireModificationIterator((byte) 2, NOP); + + mapP2 = Builder.newShmIntInt(20000, map1ToMap2, map2ToMap1, (byte) 2, (byte) 1); + map2 = mapP2.getMap(); + + segmentModificationIterator2 = map2.acquireModificationIterator((byte) 1, NOP); + + Mockito.when(timeProvider.currentTimeMillis()).thenReturn((long) 1); + } + + + @Test + public void testPut2Remove2Remove2Put1() throws IOException, InterruptedException { + + map2.put(1, 1, (byte) 2, 1399459457425L); + map2.remove(1, null, (byte) 2, 1399459457425L); + map2.remove(1, null, (byte) 2, 1399459457426L); + // waitTillFinished0(); + map1.put(1, 895, (byte) 1, 1399459457426L); + + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + + assertEquals(map1, map2); + + } + + + @Test + public void testPut1Remove2Remove2() throws IOException, InterruptedException { + + map1.put(1, 895, (byte) 1, 1399459457425L); + waitTillFinished0(); + map2.remove(1, null, (byte) 2, 1399459457425L); + map2.remove(1, null, (byte) 2, 1399459457426L); + + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + assertEquals(map1, map2); + + } + + + @Test + public void testRemove1put2() throws IOException, InterruptedException { + + map2.remove(1, null, (byte) 2, 1399459457425L); + waitTillFinished0(); + map1.put(1, 895, (byte) 1, 1399459457425L); + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + + + assertEquals(map1, map2); + } + + + @Test + public void testRemove1put2Flip() throws IOException, InterruptedException { + + map1.remove(1, null, (byte) 1, 1399459457425L); + waitTillFinished0(); + map2.put(1, 895, (byte) 2, 1399459457425L); + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + + + assertEquals(map1, map2); + } + + + @Test + public void testPut1Put1CrazyTimes() throws IOException, InterruptedException { + + map1.put(1, 894, (byte) 1, 1399459457425L); + waitTillFinished0(); + map2.put(1, 895, (byte) 2, 0L); + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + + + assertEquals(map1, map2); + } + + + @Test + public void testPut1Put1SameTimes() throws IOException, InterruptedException { + + map1.put(1, 10, (byte) 1, 0L); + waitTillFinished0(); + map2.put(1, 20, (byte) 2, 0L); + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + + + assertEquals(map1, map2); + } + + @Test + public void testPut1Put1SameTimesFlip() throws IOException, InterruptedException { + + map2.put(1, 894, (byte) 2, 0L); + waitTillFinished0(); + map1.put(1, 895, (byte) 1, 0L); + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + + + assertEquals(map1, map2); + } + + @Test + public void testPut1Put1CrazyTimesMapFlip() throws IOException, InterruptedException { + + map2.put(1, 894, (byte) 2, 1399459457425L); + waitTillFinished0(); + map1.put(1, 895, (byte) 1, 0L); + + // we will check 10 times that there all the work queues are empty + waitTillEqual(5000); + + + assertEquals(map1, map2); + } + + + @Test + public void testRemovePut() throws IOException, InterruptedException { + + + map2.remove(1, null, (byte) 2, 1399459457425L); + waitTillFinished0(); + map1.put(1, 1, (byte) 1, 1399459457425L); + + waitTillEqual(5000); + + assertEquals(map1, map2); + + } + + + @Test + public void testRemovePutFlip() throws IOException, InterruptedException { + + + map1.remove(1, null, (byte) 1, 1399459457425L); + waitTillFinished0(); + map2.put(1, 1, (byte) 2, 1399459457425L); + waitTillEqual(5000); + // we will check 10 times that there all the work queues are empty + + assertEquals(map1, map2); + + + } + + + @Test + public void testPutRemove() throws IOException, InterruptedException { + + + map2.put(1, 1, (byte) 2, 1399459457425L); + waitTillFinished0(); + map1.remove(1, null, (byte) 1, 1399459457425L); + waitTillEqual(5000); + // we will check 10 times that there all the work queues are empty + + assertEquals(map1, map2); + + } + + + @Test + public void testPutRemoveFilp() throws IOException, InterruptedException { + + + map1.put(1, 1, (byte) 1, 1399459457425L); + waitTillFinished0(); + map2.remove(1, null, (byte) 2, 1399459457425L); + waitTillEqual(5000); + // we will check 10 times that there all the work queues are empty + + assertEquals(map1, map2); + + } + + + private void waitTillFinished0() throws InterruptedException { + int i = 0; + for (; i < 2; i++) { + if (!(map2ToMap1.isEmpty() && map2ToMap1.isEmpty() && + !segmentModificationIterator1.hasNext() && + !segmentModificationIterator2.hasNext() && + mapP1.isQueueEmpty() && mapP2.isQueueEmpty())) { + i = 0; + } + Thread.sleep(1); + } + } + + /** + * waits until map1 and map2 show the same value + * + * @param timeOutMs timeout in milliseconds + * @throws InterruptedException + */ + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + waitTillFinished0(); + if (map1.equals(map2)) + break; + Thread.sleep(1); + } + } + + +} diff --git a/src/test/java/net/openhft/chronicle/map/OSResizesMain.java b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java new file mode 100644 index 000000000..3792462b5 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java @@ -0,0 +1,70 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.Jvm; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.util.Arrays; + +/** + * This example shows that the OS resizes the usage of a CHM as needed. It is not as critical to worry about this. + *

+ * System memory: 7.7 GB, Extents of map: 2199.0 GB, disk used: 13MB, addressRange: 7d380b7bd000-7f380c000000 + *

+ */ +public class OSResizesMain { + public static void main(String[] args) throws IOException, InterruptedException { + File file = File.createTempFile("over-sized", "deleteme"); + ChronicleMap map = ChronicleMapBuilder.of(String.class, String.class) + + .entrySize(1024 * 1024) + .entries(1024 * 1024) + .create(file); + for (int i = 0; i < 10000; i++) { + char[] chars = new char[i]; + Arrays.fill(chars, '+'); + map.put("key-" + i, new String(chars)); + } + System.out.printf("System memory: %.1f GB, Extents of map: %.1f GB, disk used: %sB, addressRange: %s%n", + Double.parseDouble(run("head", "-1", "/proc/meminfo").split("\\s+")[1]) / 1e6, + file.length() / 1e9, + run("du", "-h", file.getAbsolutePath()).split("\\s")[0], + run("grep", "over-sized", "/proc/" + Jvm.getProcessId() + "/maps").split("\\s")[0]); + map.close(); + file.delete(); + } + + static String run(String... cmd) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.redirectErrorStream(true); + Process p = pb.start(); + InputStreamReader reader = new InputStreamReader(p.getInputStream()); + StringWriter sw = new StringWriter(); + char[] chars = new char[512]; + for (int len; (len = reader.read(chars)) > 0; ) + sw.write(chars, 0, len); + int exitValue = p.waitFor(); + if (exitValue != 0) + sw.write("\nexit=" + exitValue); + p.destroy(); + return sw.toString(); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/PageLatencyMain.java b/src/test/java/net/openhft/chronicle/map/PageLatencyMain.java new file mode 100644 index 000000000..fe3f73efe --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/PageLatencyMain.java @@ -0,0 +1,43 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +/** + * Created by peter on 28/02/14. + */ +public class PageLatencyMain { + public static final int PAGES = Integer.getInteger("pages", 1024 * 1024); + public static final int PAGES_SIZE = 512; // longs + public static volatile long b; + + public static void main(String... ignored) { + long[] bytes = new long[PAGES * PAGES_SIZE]; + long maxTime = 0; + for (int j = 0; j < 10; j++) { + for (int i = 0; i < PAGES; i++) { + long start0 = System.nanoTime(); + b = bytes[i * PAGES_SIZE]; + if (b != 0) throw new AssertionError(); + long time = System.nanoTime() - start0; + if (time > maxTime) maxTime = time; + if (time > 1e5) + System.out.println("Page access time was " + time / 100000 / 10.0 + " ms"); + } + } + System.out.println("Longest page access time was " + maxTime / 10000 / 100.0 + " ms"); + } +} diff --git a/src/test/java/net/openhft/collections/PostClusterMapCreationTest.java b/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java similarity index 54% rename from src/test/java/net/openhft/collections/PostClusterMapCreationTest.java rename to src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java index de808410c..dc4b6f962 100644 --- a/src/test/java/net/openhft/collections/PostClusterMapCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PostClusterMapCreationTest.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,10 +14,9 @@ * limitations under the License. */ -package net.openhft.collections; +package net.openhft.chronicle.map; -import net.openhft.chronicle.map.ChronicleMapBuilder; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -34,23 +31,22 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertTrue; /** - * Test VanillaSharedReplicatedHashMap where the Replicated is over a TCP Socket, but with 4 nodes + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes * * @author Rob Austin. */ public class PostClusterMapCreationTest { - private SharedHashMap map1a; - private SharedHashMap map2a; + private ChronicleMap map1a; + private ChronicleMap map2a; - private SharedHashMap map1b; - private SharedHashMap map2b; + private ChronicleMap map1b; + private ChronicleMap map2b; - private ClusterReplicator clusterB; - private ClusterReplicator clusterA; + private ReplicatingCluster clusterB; + private ReplicatingCluster clusterA; - private ClusterReplicatorBuilder clusterReplicatorBuilder; - private ClusterReplicatorBuilder clusterReplicatorBuilder1; + private ReplicatingClusterBuilder replicatingClusterBuilder1; public static File getPersistenceFile() { @@ -66,45 +62,43 @@ public class PostClusterMapCreationTest { public void setup() throws IOException { { - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8086, new InetSocketAddress("localhost", 8087)).heartBeatInterval(1, SECONDS); + final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig + .of(8086, new InetSocketAddress("localhost", 8087)) + .heartBeatInterval(1, SECONDS); - clusterReplicatorBuilder = new ClusterReplicatorBuilder((byte) 1, 1024); - clusterReplicatorBuilder.tcpReplicatorBuilder(tcpReplicatorBuilder); - clusterA = clusterReplicatorBuilder.create(); + clusterA = new ReplicatingClusterBuilder((byte) 1, 1024) + .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created - map1a = clusterReplicatorBuilder.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); + map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterA.channelReplicator((short) 1)) + .create(getPersistenceFile()); - map2a = clusterReplicatorBuilder.create((short) 2, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); + map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterA.channelReplicator((short) 2)) + .create(getPersistenceFile()); } { - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8087).heartBeatInterval(1, SECONDS); + final TcpReplicationConfig tcpReplicationConfig = + TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); - clusterReplicatorBuilder1 = new ClusterReplicatorBuilder((byte) - 2, 1024); - clusterReplicatorBuilder1.tcpReplicatorBuilder(tcpReplicatorBuilder); - clusterB = clusterReplicatorBuilder1.create(); + clusterB = new ReplicatingClusterBuilder((byte) 2, 1024) + .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created - map1b = clusterReplicatorBuilder1.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); + map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterB.channelReplicator((short) 1)) + .create(getPersistenceFile()); - map2b = clusterReplicatorBuilder1.create((short) 2, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); + map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterB.channelReplicator((short) 2)) + .create(getPersistenceFile()); } diff --git a/src/test/java/net/openhft/collections/PreClusterMapCreationTest.java b/src/test/java/net/openhft/chronicle/map/PreClusterMapCreationTest.java similarity index 52% rename from src/test/java/net/openhft/collections/PreClusterMapCreationTest.java rename to src/test/java/net/openhft/chronicle/map/PreClusterMapCreationTest.java index 235d6143b..9fabd9745 100644 --- a/src/test/java/net/openhft/collections/PreClusterMapCreationTest.java +++ b/src/test/java/net/openhft/chronicle/map/PreClusterMapCreationTest.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,10 +14,9 @@ * limitations under the License. */ -package net.openhft.collections; +package net.openhft.chronicle.map; -import net.openhft.chronicle.map.ChronicleMapBuilder; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -34,23 +31,20 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertTrue; /** - * Test VanillaSharedReplicatedHashMap where the Replicated is over a TCP Socket, but with 4 nodes + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes * * @author Rob Austin. */ public class PreClusterMapCreationTest { - private SharedHashMap map1a; - private SharedHashMap map2a; + private ChronicleMap map1a; + private ChronicleMap map2a; - private SharedHashMap map1b; - private SharedHashMap map2b; + private ChronicleMap map1b; + private ChronicleMap map2b; - private ClusterReplicator clusterB; - private ClusterReplicator clusterA; - - private ClusterReplicatorBuilder clusterReplicatorBuilder; - private ClusterReplicatorBuilder clusterReplicatorBuilder1; + private ReplicatingCluster clusterB; + private ReplicatingCluster clusterA; public static File getPersistenceFile() { @@ -66,53 +60,46 @@ public class PreClusterMapCreationTest { public void setup() throws IOException { { - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8086, new InetSocketAddress("localhost", 8087)).heartBeatInterval(1, SECONDS); + final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig + .of(8086, new InetSocketAddress("localhost", 8087)) + .heartBeatInterval(1, SECONDS); - clusterReplicatorBuilder = new ClusterReplicatorBuilder((byte) 1, 1024); - clusterReplicatorBuilder.tcpReplicatorBuilder(tcpReplicatorBuilder); + clusterA = new ReplicatingClusterBuilder((byte) 1, 1024) + .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created - map1a = clusterReplicatorBuilder.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - map2a = clusterReplicatorBuilder.create((short) 2, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - - clusterA = clusterReplicatorBuilder.create(); + map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterA.channelReplicator((short) 1)) + .create(getPersistenceFile()); + map2a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterA.channelReplicator((short) 2)) + .create(getPersistenceFile()); } { - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8087).heartBeatInterval(1, SECONDS); + final TcpReplicationConfig tcpReplicationConfig = + TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); - clusterReplicatorBuilder1 = new ClusterReplicatorBuilder((byte) - 2, 1024); - clusterReplicatorBuilder1.tcpReplicatorBuilder(tcpReplicatorBuilder); + clusterB = new ReplicatingClusterBuilder((byte) 2, 1024) + .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created - map1b = clusterReplicatorBuilder1.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); + map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterB.channelReplicator((short) 1)) + .create(getPersistenceFile()); - map2b = clusterReplicatorBuilder1.create((short) 2, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - - clusterB = clusterReplicatorBuilder1.create(); + map2b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterB.channelReplicator((short) 2)) + .create(getPersistenceFile()); } diff --git a/src/test/java/net/openhft/chronicle/map/ProcessInstanceLimiterMain.java b/src/test/java/net/openhft/chronicle/map/ProcessInstanceLimiterMain.java new file mode 100644 index 000000000..4affbcff1 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ProcessInstanceLimiterMain.java @@ -0,0 +1,158 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +// todo make this example more efficient e.g. not using Serializable. +public class ProcessInstanceLimiterMain implements Runnable { + private static final long TIME_UPDATE_INTERVAL = 100L; + private final String sharedMapName; + private final Map theSharedMap; + private final Callback callback; + private final Map localUpdates = new ConcurrentHashMap(); + + public ProcessInstanceLimiterMain(String sharedMapName, Callback callback) throws IOException { + this.sharedMapName = sharedMapName; + this.callback = callback; + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(String.class, Data.class); + builder.entries(10000); + builder.minSegments(2); + File file = new File(System.getProperty("java.io.tmpdir") + "/" + sharedMapName); + this.theSharedMap = builder.create(file); + Thread t = new Thread(this, "ProcessInstanceLimiterMain updater"); + t.setDaemon(true); + t.start(); + } + + public static void main(String[] args) throws Exception { + Callback callback = new Callback() { + public void tooManyProcessesOfType(String processType) { + System.out.println("Too many processes of type " + processType + " have been started, so exiting this process"); + System.exit(0); + } + + public void noDefinitionForProcessesOfType(String processType) { + System.out.println("No definition for processes of type " + processType + " has been set, so exiting this process"); + System.exit(0); + } + }; + ProcessInstanceLimiterMain limiter = new ProcessInstanceLimiterMain("test", callback); + limiter.setMaxNumberOfProcessesOfType("x", 2); + limiter.startingProcessOfType("x"); + Thread.sleep(60L * 1000L); + } + + public static void pause(long pause) { + long start = System.currentTimeMillis(); + long elapsedTime; + while ((elapsedTime = System.currentTimeMillis() - start) < pause) { + try { + Thread.sleep(pause - elapsedTime); + } catch (InterruptedException e) { + } + } + } + + public void run() { + //every TIME_UPDATE_INTERVAL milliseconds, update the time + while (true) { + try { + pause(TIME_UPDATE_INTERVAL); + String processType; + for (Entry entry : this.localUpdates.entrySet()) { + processType = entry.getKey(); + int index = entry.getValue().intValue; + Data data = this.theSharedMap.get(processType); + if (data != null) { + long timenow = System.currentTimeMillis(); + data.time[index] = timenow; + this.theSharedMap.put(processType, data); + while ((data = this.theSharedMap.get(processType)).time[index] != timenow) { + data.time[index] = timenow; + this.theSharedMap.put(processType, data); + } + } + } + } catch (Exception e) { + // TODO + e.printStackTrace(); + } + } + } + + public void startingProcessOfType(String processType) { + Data data = this.theSharedMap.get(processType); + if (data == null) { + this.callback.noDefinitionForProcessesOfType(processType); + this.callback.tooManyProcessesOfType(processType); + return; + } + long[] times1 = data.time; + pause(2L * TIME_UPDATE_INTERVAL); + long[] times2 = this.theSharedMap.get(processType).time; + for (int i = 0; i < times1.length; i++) { + if (times2[i] == times1[i]) { + //we have an index which has not been updated in 200ms + //so we have a spare slot - use this slot + this.localUpdates.put(processType, new IntWrapper(i)); + return; + } + } + this.callback.tooManyProcessesOfType(processType); + } + + public void setMaxNumberOfProcessesOfType(String processType, int maxNumberOfProcessesAllowed) { + this.theSharedMap.put(processType, new Data(processType, maxNumberOfProcessesAllowed)); + } + + public interface Callback { + public void tooManyProcessesOfType(String processType); + + public void noDefinitionForProcessesOfType(String processType); + } + + public static class Data implements Serializable { + /** + * generated serialVersionUID + */ + private static final long serialVersionUID = 9163018396438735118L; + String processType; + int maxNumberOfProcessesAllowed; + long[] time; + + public Data(String processType, int maxNumberOfProcessesAllowed) { + this.processType = processType; + this.maxNumberOfProcessesAllowed = maxNumberOfProcessesAllowed; + this.time = new long[maxNumberOfProcessesAllowed]; + } + } + + public static class IntWrapper { + int intValue; + + public IntWrapper(int intValue) { + this.intValue = intValue; + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/QueueReplicator.java b/src/test/java/net/openhft/chronicle/map/QueueReplicator.java new file mode 100644 index 000000000..fa0fe9468 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/QueueReplicator.java @@ -0,0 +1,258 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.io.Bytes; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.lang.System.arraycopy; +import static net.openhft.chronicle.map.Replica.EntryExternalizable; +import static net.openhft.chronicle.map.Replica.ModificationIterator; +import static net.openhft.chronicle.map.Replica.ModificationNotifier.NOP; + +/** + * This class replicates data from one ReplicatedShareHashMap to another using a queue it was originally + * written to test the logic in the {@code ReplicatedChronicleMap} + * + * @author Rob Austin. + */ + +public class QueueReplicator implements Closeable { + + public static final short MAX_NUMBER_OF_ENTRIES_PER_CHUNK = 10; + private static final Logger LOG = LoggerFactory.getLogger(ReplicatedChronicleMap.class); + private final AtomicBoolean isWritingEntry = new AtomicBoolean(true); + private final AtomicBoolean isReadingEntry = new AtomicBoolean(true); + private final ByteBufferBytes entryBuffer; + private ByteBufferBytes buffer; + + public QueueReplicator(@NotNull final ModificationIterator modificationIterator, + @NotNull final BlockingQueue input, + @NotNull final BlockingQueue output, + final int entrySize, + @NotNull final EntryExternalizable externalizable) { + + //todo HCOLL-71 fix the 128 padding + final int entrySize0 = entrySize + 128; + entryBuffer = new ByteBufferBytes(ByteBuffer.allocateDirect(entrySize0)); + + // in bound + Executors.newSingleThreadExecutor(new ThreadFactory() { + + @Override + public Thread newThread(Runnable r) { + final Thread thread = new Thread(r, "reader-map" + externalizable); + thread.setDaemon(true); + return thread; + } + + }).execute(new Runnable() { + + @Override + public void run() { + + // this is used in nextEntry() below, its what could be described as callback method + + try { + + for (; ; ) { + + byte[] item = null; + try { + + + for (; ; ) { + isReadingEntry.set(true); + item = input.poll(); + if (item == null) { + isReadingEntry.set(false); + Thread.sleep(1); + } else { + break; + } + + } + + final ByteBufferBytes bufferBytes = new ByteBufferBytes(ByteBuffer.wrap(item)); + + while (bufferBytes.remaining() > 0) { + + final long entrySize = bufferBytes.readStopBit(); + + final long position = bufferBytes.position(); + final long limit = bufferBytes.limit(); + + bufferBytes.limit(position + entrySize); + externalizable.readExternalEntry(bufferBytes); + + bufferBytes.position(position); + bufferBytes.limit(limit); + + // skip onto the next entry + bufferBytes.skip(entrySize); + + } + isReadingEntry.set(false); + + } catch (InterruptedException e1) { + LOG.warn("", e1); + } + + } + } catch (Exception e) { + LOG.warn("", e); + } + + } + + }); + + // out bound + Executors.newSingleThreadExecutor().execute(new Runnable() { + + + @Override + public void run() { + + buffer = new ByteBufferBytes(ByteBuffer.allocate(entrySize0 * MAX_NUMBER_OF_ENTRIES_PER_CHUNK)); + + // this is used in nextEntry() below, its what could be described as callback method + final Replica.EntryCallback entryCallback = + new Replica.EntryCallback() { + @Override + public boolean onEntry(Bytes entry, final int chronicleId) { + + entryBuffer.clear(); + externalizable.writeExternalEntry(entry, entryBuffer, chronicleId); + + if (entryBuffer.position() == 0) + return false; + + // write the entry len + buffer.writeStopBit(entryBuffer.position()); + + // write the entry + entryBuffer.flip(); + buffer.write(entryBuffer); + + return true; + } + + @Override + public void onBeforeEntry() { + isWritingEntry.set(true); + } + + }; + + try { + for (; ; ) { + + final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0); + + if (wasDataRead) { + isWritingEntry.set(false); + } else if (buffer.position() == 0) { + isWritingEntry.set(false); + continue; + } + + if (buffer.remaining() <= entrySize0 && ((wasDataRead || buffer.position() == 0))) + continue; + + // we are going to create an byte[] so that the buffer can be copied into this. + final byte[] source = buffer.buffer().array(); + final int length = (int) buffer.position(); + final byte[] dest = new byte[length]; + + arraycopy(source, 0, dest, 0, length); + + try { + output.put(dest); + } catch (InterruptedException e1) { + LOG.warn("", e1); + break; + } + + // clear the buffer for reuse, we can store a maximum of + // MAX_NUMBER_OF_ENTRIES_PER_CHUNK in this buffer + buffer.clear(); + + } + } catch (Exception e) { + LOG.warn("", e); + } + } + + + }); + + + } + + public static Replicator of(final byte localIdentifier, + final byte externalIdentifier, + @NotNull final BlockingQueue input, + @NotNull final BlockingQueue output) { + return new Replicator() { + boolean used = false; + + @Override + public byte identifier() { + return localIdentifier; + } + + @Override + protected Closeable applyTo(ChronicleMapBuilder builder, + Replica map, + EntryExternalizable entryExternalizable) + throws IOException { + if (used) + throw new IllegalStateException(); + used = true; + return new QueueReplicator( + map.acquireModificationIterator(externalIdentifier, NOP), + input, output, builder.entrySize(), entryExternalizable); + } + }; + } + + /** + * @return true indicates that all the data has been processed ( it lock free so can not be relied upon ) + */ + public boolean isEmpty() { + final boolean b = isWritingEntry.get() || isReadingEntry.get(); + return !b && buffer.position() == 0; + } + + @Override + public void close() throws IOException { + // do nothing + } +} + diff --git a/src/test/java/net/openhft/chronicle/map/RecursiveRefereneChMapTest.java b/src/test/java/net/openhft/chronicle/map/RecursiveRefereneChMapTest.java new file mode 100644 index 000000000..abe3ac3a7 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/RecursiveRefereneChMapTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.serialization.JDKObjectSerializer; +import org.junit.Test; + +import java.io.*; +import java.util.Map; + +import static org.junit.Assert.assertSame; + +public class RecursiveRefereneChMapTest { + public static final String TMP = System.getProperty("java.io.tmpdir"); + + @Test + public void testRecursive() throws IOException { + File file = new File(TMP + "/test." + System.nanoTime() + ".tmp"); + file.deleteOnExit(); + Map map = ChronicleMapBuilder.of(String.class, StupidCycle.class) + .entries(64) + .objectSerializer(JDKObjectSerializer.INSTANCE) + .create(file); + map.put("Test", new StupidCycle()); + map.put("Test2", new StupidCycle2()); + StupidCycle cycle = (StupidCycle) map.get("Test"); + assertSame(cycle, cycle.cycle[0]); + StupidCycle cycle2 = (StupidCycle) map.get("Test2"); + assertSame(cycle2, cycle2.cycle[0]); + } + + public static class StupidCycle implements Serializable { + int dummy; + Object cycle[] = {this}; + } + + public static class StupidCycle2 extends StupidCycle implements Externalizable { + public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(dummy); + out.writeObject(cycle); + } + + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + dummy = in.readInt(); + cycle = (Object[]) in.readObject(); + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java new file mode 100644 index 000000000..404bb4ef1 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java @@ -0,0 +1,858 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.*; + +/* + * Originally written by Doug Lea with assistance from members of JCP JSR-166 + * Expert Group and released to the public domain, as explained at + * http://creativecommons.org/publicdomain/zero/1.0/ + * Other contributors include Andrew Wright, Jeffrey Hayes, + * Pat Fisher, Mike Judd. Then modified by the Open HFT team. + */ +public class ReplicatedChronicleMapTest extends JSR166TestCase { + + private static File getPersistenceFile() { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/chm-test" + System.nanoTime()); + file.deleteOnExit(); + return file; + } + + ChronicleMap newShmIntString(int size) throws IOException { + return ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(size) + .forceReplicatedImpl() + .create(getPersistenceFile()); + + } + + ChronicleMap newShmListBoolean(int size) throws IOException { + return ChronicleMapBuilder.of(ArrayList.class, CharSequence.class) + .entries(size) + .forceReplicatedImpl() + .create(getPersistenceFile()); + + } + + + ChronicleMap newShmListBoolean() throws IOException { + return ChronicleMapBuilder.of(ArrayList.class, CharSequence.class) + .forceReplicatedImpl() + .create(getPersistenceFile()); + + } + + ChronicleMap newShmStringString(int size) throws IOException { + return ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(size) + .forceReplicatedImpl() + .create(getPersistenceFile()); + + } + + + ChronicleMap newShmIntString() throws IOException { + return ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .forceReplicatedImpl() + .create(getPersistenceFile()); + + } + + ChronicleMap newShmBiBoolean() throws IOException { + return ChronicleMapBuilder.of(BI.class, Boolean.class) + .forceReplicatedImpl().create(getPersistenceFile()); + + } + + /** + * Returns a new map from Integers 1-5 to Strings "A"-"E". + */ + private ChronicleMap map5() throws IOException { + ChronicleMap map = newShmIntString(5); + assertTrue(map.isEmpty()); + map.put(JSR166TestCase.one, "A"); + map.put(JSR166TestCase.two, "B"); + map.put(JSR166TestCase.three, "C"); + map.put(JSR166TestCase.four, "D"); + map.put(JSR166TestCase.five, "E"); + assertFalse(map.isEmpty()); + assertEquals(5, map.size()); + return map; + } + + /** + * clear removes all pairs + */ + @Test + public void testClear() throws IOException { + ChronicleMap map = map5(); + map.clear(); + assertEquals(0, map.size()); + } + + /** + * Maps with same contents are equal + */ + @Test + public void testEquals() throws IOException { + ChronicleMap map1 = map5(); + ChronicleMap map2 = map5(); + assertEquals(map1, map2); + assertEquals(map2, map1); + map1.clear(); + assertFalse(map1.equals(map2)); + assertFalse(map2.equals(map1)); + } + + /** + * contains returns true for contained value + */ + @Test + public void testContains() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsValue("A")); + assertFalse(map.containsValue("Z")); + } + + /** + * containsKey returns true for contained key + */ + @Test + public void testContainsKey() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsKey(JSR166TestCase.one)); + assertFalse(map.containsKey(JSR166TestCase.zero)); + } + + /** + * containsValue returns true for held values + */ + @Test + public void testContainsValue() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsValue("A")); + assertFalse(map.containsValue("Z")); + } + + + /** + * Inserted elements that are subclasses of the same Comparable + * class are found. + */ + /* @Test public void testComparableFamily() throws IOException { + ChronicleMap m = + newShmBiBoolean(); + for (int i = 0; i < 1000; i++) { + assertTrue(m.put(new CI(i), true) == null); + } + for (int i = 0; i < 1000; i++) { + assertTrue(m.containsKey(new CI(i))); + assertTrue(m.containsKey(new DI(i))); + } + }*/ + + /** + * TODO : fix test + * Elements of classes with erased generic type parameters based + * on Comparable can be inserted and found. + */ + /* @Test public void testGenericComparable() throws IOException { + ChronicleMap m = + newShmBiBoolean(); + for (int i = 0; i < 1000; i++) { + BI bi = new BI(i); + BS bs = new BS(String.valueOf(i)); + LexicographicList bis = new LexicographicList(bi); + LexicographicList bss = new LexicographicList(bs); + assertTrue(m.putIfAbsent(bis, true) == null); + assertTrue(m.containsKey(bis)); + if (m.putIfAbsent(bss, true) == null) + assertTrue(m.containsKey(bss)); + assertTrue(m.containsKey(bis)); + } + for (int i = 0; i < 1000; i++) { + assertTrue(m.containsKey(new ArrayList(Collections.singleton(new BI(i))))); + } + } +*/ + + /** + * Elements of non-comparable classes equal to those of classes + * with erased generic type parameters based on Comparable can be + * inserted and found. + */ + /* @Test public void testGenericComparable2() throws IOException { + ChronicleMap m = + newShmListBoolean(); + for (int i = 0; i < 1000; i++) { + m.put(new ArrayList(Collections.singleton(new BI(i))), true); + } + + for (int i = 0; i < 1000; i++) { + LexicographicList bis = new LexicographicList(new BI(i)); + assertTrue(m.containsKey(bis)); + } + } +*/ + + /** + * get returns the correct element at the given key, or null if not present + */ + @Test + public void testGet() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", (String) map.get(JSR166TestCase.one)); + ChronicleMap empty = newShmIntString(); + assertNull(map.get(JSR166TestCase.notPresent)); + } + + /** + * isEmpty is true of empty map and false for non-empty + */ + @Test + public void testIsEmpty() throws IOException { + ChronicleMap empty = newShmIntString(); + ChronicleMap map = map5(); + assertTrue(empty.isEmpty()); + assertFalse(map.isEmpty()); + } + + /** + * keySet returns a Set containing all the keys + */ + @Test + public void testKeySet() throws IOException { + ChronicleMap map = map5(); + Set s = map.keySet(); + assertEquals(5, s.size()); + assertTrue(s.contains(JSR166TestCase.one)); + assertTrue(s.contains(JSR166TestCase.two)); + assertTrue(s.contains(JSR166TestCase.three)); + assertTrue(s.contains(JSR166TestCase.four)); + assertTrue(s.contains(JSR166TestCase.five)); + } + + /** + * keySet.toArray returns contains all keys + */ + @Test + public void testKeySetToArray() throws IOException { + ChronicleMap map = map5(); + Set s = map.keySet(); + Object[] ar = s.toArray(); + assertTrue(s.containsAll(Arrays.asList(ar))); + assertEquals(5, ar.length); + ar[0] = JSR166TestCase.m10; + assertFalse(s.containsAll(Arrays.asList(ar))); + } + + /** + * Values.toArray contains all values + */ + @Test + public void testValuesToArray() throws IOException { + ChronicleMap map = map5(); + Collection v = map.values(); + Object[] ar = v.toArray(); + ArrayList s = new ArrayList(Arrays.asList(ar)); + assertEquals(5, ar.length); + assertTrue(s.contains("A")); + assertTrue(s.contains("B")); + assertTrue(s.contains("C")); + assertTrue(s.contains("D")); + assertTrue(s.contains("E")); + } + + /** + * TODO : enumeration returns an enumeration containing the correct + * elements + */ + /* @Test public void testEnumeration() throws IOException { + ChronicleMap map = map5(); + Enumeration e = map.elements(); + int count = 0; + while (e.hasMoreElements()) { + count++; + e.nextElement(); + } + assertEquals(5, count); + }*/ + + /** + * entrySet.toArray contains all entries + */ + @Test + public void testEntrySetToArray() throws IOException { + ChronicleMap map = map5(); + Set s = map.entrySet(); + Object[] ar = s.toArray(); + assertEquals(5, ar.length); + for (int i = 0; i < 5; ++i) { + assertTrue(map.containsKey(((Map.Entry) (ar[i])).getKey())); + assertTrue(map.containsValue(((Map.Entry) (ar[i])).getValue())); + } + } + + /** + * values collection contains all values + */ + @Test + public void testValues() throws IOException { + ChronicleMap map = map5(); + Collection s = map.values(); + assertEquals(5, s.size()); + assertTrue(s.contains("A")); + assertTrue(s.contains("B")); + assertTrue(s.contains("C")); + assertTrue(s.contains("D")); + assertTrue(s.contains("E")); + } + + /** + * TODO : keys returns an enumeration containing all the keys from the map + */ + /* @Test public void testKeys() throws IOException { + ChronicleMap map = map5(); + Enumeration e = map.keys(); + int count = 0; + while (e.hasMoreElements()) { + count++; + e.nextElement(); + } + assertEquals(5, count); + }*/ + + /** + * entrySet contains all pairs + */ + @Test + public void testEntrySet() throws IOException { + ChronicleMap map = map5(); + Set s = map.entrySet(); + assertEquals(5, s.size()); + Iterator it = s.iterator(); + while (it.hasNext()) { + Map.Entry e = (Map.Entry) it.next(); + assertTrue( + (e.getKey().equals(JSR166TestCase.one) && e.getValue().equals("A")) || + (e.getKey().equals(JSR166TestCase.two) && e.getValue().equals("B")) || + (e.getKey().equals(JSR166TestCase.three) && e.getValue().equals("C")) || + (e.getKey().equals(JSR166TestCase.four) && e.getValue().equals("D")) || + (e.getKey().equals(JSR166TestCase.five) && e.getValue().equals("E")) + ); + } + } + + /** + * putAll adds all key-value pairs from the given map + */ + @Test + public void testPutAll() throws IOException { + ChronicleMap empty = newShmIntString(); + ChronicleMap map = map5(); + empty.putAll(map); + assertEquals(5, empty.size()); + assertTrue(empty.containsKey(JSR166TestCase.one)); + assertTrue(empty.containsKey(JSR166TestCase.two)); + assertTrue(empty.containsKey(JSR166TestCase.three)); + assertTrue(empty.containsKey(JSR166TestCase.four)); + assertTrue(empty.containsKey(JSR166TestCase.five)); + } + + /** + * putIfAbsent works when the given key is not present + */ + @Test + public void testPutIfAbsent() throws IOException { + ChronicleMap map = map5(); + map.putIfAbsent(JSR166TestCase.six, "Z"); + assertTrue(map.containsKey(JSR166TestCase.six)); + } + + /** + * putIfAbsent does not add the §pair if the key is already present + */ + @Test + public void testPutIfAbsent2() throws IOException { + ChronicleMap map = map5(); + final Object z = map.putIfAbsent(JSR166TestCase.one, "Z"); + assertEquals("A", z); + } + + /** + * replace fails when the given key is not present + */ + @Test + public void testReplace() throws IOException { + ChronicleMap map = map5(); + assertNull(map.replace(JSR166TestCase.six, "Z")); + assertFalse(map.containsKey(JSR166TestCase.six)); + } + + /** + * replace succeeds if the key is already present + */ + @Test + public void testReplace2() throws IOException { + ChronicleMap map = map5(); + assertNotNull(map.replace(JSR166TestCase.one, "Z")); + assertEquals("Z", map.get(JSR166TestCase.one)); + } + + /** + * replace value fails when the given key not mapped to expected value + */ + @Test + public void testReplaceValue() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", map.get(JSR166TestCase.one)); + assertFalse(map.replace(JSR166TestCase.one, "Z", "Z")); + assertEquals("A", map.get(JSR166TestCase.one)); + } + + /** + * replace value succeeds when the given key mapped to expected value + */ + @Test + public void testReplaceValue2() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", map.get(JSR166TestCase.one)); + assertTrue(map.replace(JSR166TestCase.one, "A", "Z")); + assertEquals("Z", map.get(JSR166TestCase.one)); + } + + /** + * remove removes the correct key-value pair from the map + */ + @Test + public void testRemove() throws IOException { + ChronicleMap map = map5(); + map.remove(JSR166TestCase.five); + assertEquals(4, map.size()); + assertFalse(map.containsKey(JSR166TestCase.five)); + } + + /** + * remove(key,value) removes only if pair present + */ + @Test + public void testRemove2() throws IOException { + ChronicleMap map = map5(); + map.remove(JSR166TestCase.five, "E"); + assertEquals(4, map.size()); + assertFalse(map.containsKey(JSR166TestCase.five)); + map.remove(JSR166TestCase.four, "A"); + assertEquals(4, map.size()); + assertTrue(map.containsKey(JSR166TestCase.four)); + } + + /** + * size returns the correct values + */ + @Test + public void testSize() throws IOException { + ChronicleMap map = map5(); + ChronicleMap empty = newShmIntString(); + assertEquals(0, empty.size()); + assertEquals(5, map.size()); + } + + /** + * toString contains toString of elements + */ + @Test + public void testToString() throws IOException { + ChronicleMap map = map5(); + String s = map.toString(); + for (int i = 1; i <= 5; ++i) { + assertTrue(s.contains(String.valueOf(i))); + } + } + + /** + * get(null) throws NPE + */ + @Test + public void testGet_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.get(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * containsKey(null) throws NPE + */ + @Test + public void testContainsKey_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.containsKey(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * put(null,x) throws NPE + */ + @Test + public void testPut1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.put(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * put(x, null) throws NPE + */ + @Test + public void testPut2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.put(JSR166TestCase.notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * putIfAbsent(null, x) throws NPE + */ + @Test + public void testPutIfAbsent1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.putIfAbsent(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + // Exception tests + + /** + * TODO : Cannot create with negative capacity + */ + /* @Test public void testConstructor1() { + try { + newShmIntString(-1, 0, 1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * TODO : Cannot create with negative concurrency level + */ + /*@Test public void testConstructor2() { + try { + newShmIntString(1, 0, -1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * TODO :Cannot create with only negative capacity + */ + /* @Test public void testConstructor3() { + try { + newShmIntString(-1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * replace(null, x) throws NPE + */ + @Test + public void testReplace_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(null, x, y) throws NPE + */ + @Test + public void testReplaceValue_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(null, "A", "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * containsValue(null) throws NPE + */ +/* @Test public void testContainsValue_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.containsValue(null); + shouldThrow(); + } catch (NullPointerException success) { + } + }*/ + + /** + * todo : contains(null) throws NPE + */ +/* + @Test public void testContains_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.contains(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } +*/ + + /** + * putIfAbsent(x, null) throws NPE + */ + @Test + public void testPutIfAbsent2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.putIfAbsent(JSR166TestCase.notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, null) throws NPE + */ + @Test + public void testReplace2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(JSR166TestCase.notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, null, y) throws NPE + */ + @Test + public void testReplaceValue2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(JSR166TestCase.notPresent, null, "A"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, y, null) throws NPE + */ + @Test + public void testReplaceValue3_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(JSR166TestCase.notPresent, "A", null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(null) throws NPE + */ + @Test + public void testRemove1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + c.remove(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(null, x) throws NPE + */ + @Test + public void testRemove2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + c.remove(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(x, null) returns false + */ + @Test + public void testRemove3() throws IOException { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + assertFalse(c.remove("sadsdf", null)); + } + + // classes for testing Comparable fallbacks + static class BI implements Comparable { + private final int value; + + BI(int value) { + this.value = value; + } + + public int compareTo(BI other) { + return Integer.compare(value, other.value); + } + + public boolean equals(Object x) { + return (x instanceof BI) && ((BI) x).value == value; + } + + public int hashCode() { + return 42; + } + } + + static class CI extends BI { + CI(int value) { + super(value); + } + } + + static class DI extends BI { + DI(int value) { + super(value); + } + } + + static class BS implements Comparable { + private final String value; + + BS(String value) { + this.value = value; + } + + public int compareTo(BS other) { + return value.compareTo(other.value); + } + + public boolean equals(Object x) { + return (x instanceof BS) && value.equals(((BS) x).value); + } + + public int hashCode() { + return 42; + } + } + + static class LexicographicList> extends ArrayList + implements Comparable> { + private static final long serialVersionUID = 0; + static long total; + static long n; + + LexicographicList(Collection c) { + super(c); + } + + LexicographicList(E e) { + super(Collections.singleton(e)); + } + + public int compareTo(LexicographicList other) { + long start = System.currentTimeMillis(); + int common = Math.min(size(), other.size()); + int r = 0; + for (int i = 0; i < common; i++) { + if ((r = get(i).compareTo(other.get(i))) != 0) + break; + } + if (r == 0) + r = Integer.compare(size(), other.size()); + total += System.currentTimeMillis() - start; + n++; + return r; + } + } + + /** + * A deserialized map equals original + */ + /* @Test public void testSerialization() throws Exception { + Map x = map5(); + Map y = serialClone(x); + + assertNotSame(x, y); + assertEquals(x.size(), y.size()); + assertEquals(x, y); + assertEquals(y, x); + }*/ + + /** + * TODO : SetValue of an EntrySet entry sets value in the map. + */ + /*@Test public void testSetValueWriteThrough() { + // Adapted from a bug report by Eric Zoerner + ChronicleMap map = newShmIntString(2, 5.0f, 1); + assertTrue(map.isEmpty()); + for (int i = 0; i < 20; i++) + map.put(new Integer(i), new Integer(i)); + assertFalse(map.isEmpty()); + Map.Entry entry1 = (Map.Entry) map.entrySet().iterator().next(); + // Unless it happens to be first (in which case remainder of + // test is skipped), remove a possibly-colliding key from map + // which, under some implementations, may cause entry1 to be + // cloned in map + if (!entry1.getKey().equals(new Integer(16))) { + map.remove(new Integer(16)); + entry1.setValue("XYZ"); + assertTrue(map.containsValue("XYZ")); // fails if write-through broken + } + }*/ + +} + diff --git a/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest2.java b/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest2.java new file mode 100644 index 000000000..90e6aa327 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest2.java @@ -0,0 +1,892 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ArrayBlockingQueue; + +import static org.junit.Assert.*; + +/* + * Originally written by Doug Lea with assistance from members of JCP JSR-166 + * Expert Group and released to the public domain, as explained at + * http://creativecommons.org/publicdomain/zero/1.0/ + * Other contributors include Andrew Wright, Jeffrey Hayes, + * Pat Fisher, Mike Judd. Then modified by the Open HFT team. + */ + +@RunWith(value = Parameterized.class) +public class ReplicatedChronicleMapTest2 extends JSR166TestCase { + + boolean canReplicate; + + public ReplicatedChronicleMapTest2(Boolean canReplicate) { + this.canReplicate = canReplicate; + } + + @Parameterized.Parameters + public static Collection data() { + + return Arrays.asList(new Object[][]{ + { + Boolean.TRUE + } + + }); + } + + private static File getPersistenceFile() { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/chm-test" + System.nanoTime()); + file.deleteOnExit(); + return file; + } + + ChronicleMap newShmIntString(int size) throws IOException { + + final ArrayBlockingQueue map1ToMap2 = new ArrayBlockingQueue(100); + final ArrayBlockingQueue map2ToMap1 = new ArrayBlockingQueue(100); + + final ChronicleMap map1 = + Builder.newShm(size, map1ToMap2, map2ToMap1, (byte) 1, (byte) 2, + Integer.class, CharSequence.class); + final ChronicleMap map2 = + Builder.newShm(size, map2ToMap1, map1ToMap2, (byte) 2, (byte) 1, + Integer.class, CharSequence.class); + + return new ReplicationCheckingMap(map1, map2); + + + } + + + ChronicleMap newShmStringString(int size) throws IOException { + + final ArrayBlockingQueue map1ToMap2 = new ArrayBlockingQueue(100); + final ArrayBlockingQueue map2ToMap1 = new ArrayBlockingQueue(100); + + final ChronicleMap map1 = + Builder.newShm(size, map1ToMap2, map2ToMap1, (byte) 1, (byte) 2, + CharSequence.class, CharSequence.class); + final ChronicleMap map2 = + Builder.newShm(size, map2ToMap1, map1ToMap2, (byte) 2, (byte) 1, + CharSequence.class, CharSequence.class); + + return new ReplicationCheckingMap(map1, map2); + + } + + + ChronicleMap newShmIntString() throws IOException { + return ChronicleMapBuilder.of(Integer.class, CharSequence.class) + + .forceReplicatedImpl() + .create(getPersistenceFile()); + } + + + /** + * Returns a new map from Integers 1-5 to Strings "A"-"E". + */ + private ChronicleMap map5() throws IOException { + ChronicleMap map = newShmIntString(5); + assertTrue(map.isEmpty()); + map.put(JSR166TestCase.one, "A"); + map.put(JSR166TestCase.two, "B"); + map.put(JSR166TestCase.three, "C"); + map.put(JSR166TestCase.four, "D"); + map.put(JSR166TestCase.five, "E"); + assertFalse(map.isEmpty()); + assertEquals(5, map.size()); + return map; + } + + /** + * clear removes all pairs + */ + @Test + public void testClear() throws IOException { + ChronicleMap map = map5(); + map.clear(); + assertEquals(0, map.size()); + } + + /** + * clear removes all pairs + */ + @Test + public void testKeySetRemove() throws IOException { + ChronicleMap map = map5(); + + System.out.print("map.keySet()=" + map.keySet()); + for (Object k : map.keySet()) { + map.remove(k); + } + + + assertEquals(0, map.size()); + } + + + /** + * Maps with same contents are equal + */ + @Test + public void testEquals() throws IOException { + ChronicleMap map1 = map5(); + ChronicleMap map2 = map5(); + assertEquals(map1, map2); + assertEquals(map2, map1); + map1.clear(); + assertFalse(map1.equals(map2)); + assertFalse(map2.equals(map1)); + } + + /** + * contains returns true for contained value + */ + @Test + public void testContains() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsValue("A")); + assertFalse(map.containsValue("Z")); + } + + /** + * containsKey returns true for contained key + */ + @Test + public void testContainsKey() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsKey(JSR166TestCase.one)); + assertFalse(map.containsKey(JSR166TestCase.zero)); + } + + /** + * containsValue returns true for held values + */ + @Test + public void testContainsValue() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsValue("A")); + assertFalse(map.containsValue("Z")); + } + + + /** + * Inserted elements that are subclasses of the same Comparable + * class are found. + */ + /* @Test public void testComparableFamily() throws IOException { + ChronicleMap m = + newShmBiBoolean(); + for (int i = 0; i < 1000; i++) { + assertTrue(m.put(new CI(i), true) == null); + } + for (int i = 0; i < 1000; i++) { + assertTrue(m.containsKey(new CI(i))); + assertTrue(m.containsKey(new DI(i))); + } + }*/ + + /** + * TODO : fix test + * Elements of classes with erased generic type parameters based + * on Comparable can be inserted and found. + */ + /* @Test public void testGenericComparable() throws IOException { + ChronicleMap m = + newShmBiBoolean(); + for (int i = 0; i < 1000; i++) { + BI bi = new BI(i); + BS bs = new BS(String.valueOf(i)); + LexicographicList bis = new LexicographicList(bi); + LexicographicList bss = new LexicographicList(bs); + assertTrue(m.putIfAbsent(bis, true) == null); + assertTrue(m.containsKey(bis)); + if (m.putIfAbsent(bss, true) == null) + assertTrue(m.containsKey(bss)); + assertTrue(m.containsKey(bis)); + } + for (int i = 0; i < 1000; i++) { + assertTrue(m.containsKey(new ArrayList(Collections.singleton(new BI(i))))); + } + } +*/ + + /** + * Elements of non-comparable classes equal to those of classes + * with erased generic type parameters based on Comparable can be + * inserted and found. + */ + /* @Test public void testGenericComparable2() throws IOException { + ChronicleMap m = + newShmListBoolean(); + for (int i = 0; i < 1000; i++) { + m.put(new ArrayList(Collections.singleton(new BI(i))), true); + } + + for (int i = 0; i < 1000; i++) { + LexicographicList bis = new LexicographicList(new BI(i)); + assertTrue(m.containsKey(bis)); + } + } +*/ + + /** + * get returns the correct element at the given key, or null if not present + */ + @Test + public void testGet() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", (String) map.get(JSR166TestCase.one)); + ChronicleMap empty = newShmIntString(); + assertNull(map.get(JSR166TestCase.notPresent)); + } + + /** + * isEmpty is true of empty map and false for non-empty + */ + @Test + public void testIsEmpty() throws IOException { + ChronicleMap empty = newShmIntString(); + ChronicleMap map = map5(); + assertTrue(empty.isEmpty()); + assertFalse(map.isEmpty()); + } + + /** + * keySet returns a Set containing all the keys + */ + @Test + public void testKeySet() throws IOException { + ChronicleMap map = map5(); + Set s = map.keySet(); + assertEquals(5, s.size()); + assertTrue(s.contains(JSR166TestCase.one)); + assertTrue(s.contains(JSR166TestCase.two)); + assertTrue(s.contains(JSR166TestCase.three)); + assertTrue(s.contains(JSR166TestCase.four)); + assertTrue(s.contains(JSR166TestCase.five)); + } + + /** + * keySet.toArray returns contains all keys + */ + @Test + public void testKeySetToArray() throws IOException { + ChronicleMap map = map5(); + Set s = map.keySet(); + Object[] ar = s.toArray(); + assertTrue(s.containsAll(Arrays.asList(ar))); + assertEquals(5, ar.length); + ar[0] = JSR166TestCase.m10; + assertFalse(s.containsAll(Arrays.asList(ar))); + } + + /** + * Values.toArray contains all values + */ + @Test + public void testValuesToArray() throws IOException { + ChronicleMap map = map5(); + Collection v = map.values(); + Object[] ar = v.toArray(); + ArrayList s = new ArrayList(Arrays.asList(ar)); + assertEquals(5, ar.length); + assertTrue(s.contains("A")); + assertTrue(s.contains("B")); + assertTrue(s.contains("C")); + assertTrue(s.contains("D")); + assertTrue(s.contains("E")); + } + + /** + * TODO : enumeration returns an enumeration containing the correct + * elements + */ + /* @Test public void testEnumeration() throws IOException { + ChronicleMap map = map5(); + Enumeration e = map.elements(); + int count = 0; + while (e.hasMoreElements()) { + count++; + e.nextElement(); + } + assertEquals(5, count); + }*/ + + /** + * entrySet.toArray contains all entries + */ + @Test + public void testEntrySetToArray() throws IOException { + ChronicleMap map = map5(); + Set s = map.entrySet(); + Object[] ar = s.toArray(); + assertEquals(5, ar.length); + for (int i = 0; i < 5; ++i) { + assertTrue(map.containsKey(((Map.Entry) (ar[i])).getKey())); + assertTrue(map.containsValue(((Map.Entry) (ar[i])).getValue())); + } + } + + /** + * values collection contains all values + */ + @Test + public void testValues() throws IOException { + ChronicleMap map = map5(); + Collection s = map.values(); + assertEquals(5, s.size()); + assertTrue(s.contains("A")); + assertTrue(s.contains("B")); + assertTrue(s.contains("C")); + assertTrue(s.contains("D")); + assertTrue(s.contains("E")); + } + + /** + * TODO : keys returns an enumeration containing all the keys from the map + */ + /* @Test public void testKeys() throws IOException { + ChronicleMap map = map5(); + Enumeration e = map.keys(); + int count = 0; + while (e.hasMoreElements()) { + count++; + e.nextElement(); + } + assertEquals(5, count); + }*/ + + /** + * entrySet contains all pairs + */ + @Test + public void testEntrySet() throws IOException { + ChronicleMap map = map5(); + Set s = map.entrySet(); + assertEquals(5, s.size()); + Iterator it = s.iterator(); + while (it.hasNext()) { + Map.Entry e = (Map.Entry) it.next(); + assertTrue( + (e.getKey().equals(JSR166TestCase.one) && e.getValue().equals("A")) || + (e.getKey().equals(JSR166TestCase.two) && e.getValue().equals("B")) || + (e.getKey().equals(JSR166TestCase.three) && e.getValue().equals("C")) || + (e.getKey().equals(JSR166TestCase.four) && e.getValue().equals("D")) || + (e.getKey().equals(JSR166TestCase.five) && e.getValue().equals("E")) + ); + } + } + + /** + * putAll adds all key-value pairs from the given map + */ + @Test + public void testPutAll() throws IOException { + ChronicleMap empty = newShmIntString(); + ChronicleMap map = map5(); + empty.putAll(map); + assertEquals(5, empty.size()); + assertTrue(empty.containsKey(JSR166TestCase.one)); + assertTrue(empty.containsKey(JSR166TestCase.two)); + assertTrue(empty.containsKey(JSR166TestCase.three)); + assertTrue(empty.containsKey(JSR166TestCase.four)); + assertTrue(empty.containsKey(JSR166TestCase.five)); + } + + /** + * putIfAbsent works when the given key is not present + */ + @Test + public void testPutIfAbsent() throws IOException { + ChronicleMap map = map5(); + map.putIfAbsent(JSR166TestCase.six, "Z"); + assertTrue(map.containsKey(JSR166TestCase.six)); + } + + /** + * putIfAbsent does not add the §pair if the key is already present + */ + @Test + public void testPutIfAbsent2() throws IOException { + ChronicleMap map = map5(); + final Object z = map.putIfAbsent(JSR166TestCase.one, "Z"); + assertEquals("A", z); + } + + /** + * replace fails when the given key is not present + */ + @Test + public void testReplace() throws IOException { + ChronicleMap map = map5(); + assertNull(map.replace(JSR166TestCase.six, "Z")); + assertFalse(map.containsKey(JSR166TestCase.six)); + } + + /** + * replace succeeds if the key is already present + */ + @Test + public void testReplace2() throws IOException { + ChronicleMap map = map5(); + assertNotNull(map.replace(JSR166TestCase.one, "Z")); + assertEquals("Z", map.get(JSR166TestCase.one)); + } + + /** + * replace value fails when the given key not mapped to expected value + */ + @Test + public void testReplaceValue() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", map.get(JSR166TestCase.one)); + assertFalse(map.replace(JSR166TestCase.one, "Z", "Z")); + assertEquals("A", map.get(JSR166TestCase.one)); + } + + /** + * replace value succeeds when the given key mapped to expected value + */ + @Test + public void testReplaceValue2() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", map.get(JSR166TestCase.one)); + assertTrue(map.replace(JSR166TestCase.one, "A", "Z")); + assertEquals("Z", map.get(JSR166TestCase.one)); + } + + /** + * remove removes the correct key-value pair from the map + */ + @Test + public void testRemove() throws IOException { + ChronicleMap map = map5(); + map.remove(JSR166TestCase.five); + assertEquals(4, map.size()); + assertFalse(map.containsKey(JSR166TestCase.five)); + } + + /** + * remove(key,value) removes only if pair present + */ + @Test + public void testRemove2() throws IOException { + ChronicleMap map = map5(); + map.remove(JSR166TestCase.five, "E"); + assertEquals(4, map.size()); + assertFalse(map.containsKey(JSR166TestCase.five)); + map.remove(JSR166TestCase.four, "A"); + assertEquals(4, map.size()); + assertTrue(map.containsKey(JSR166TestCase.four)); + } + + /** + * size returns the correct values + */ + @Test + public void testSize() throws IOException { + ChronicleMap map = map5(); + ChronicleMap empty = newShmIntString(); + assertEquals(0, empty.size()); + assertEquals(5, map.size()); + } + + /** + * toString contains toString of elements + */ + @Test + public void testToString() throws IOException { + ChronicleMap map = map5(); + String s = map.toString(); + for (int i = 1; i <= 5; ++i) { + assertTrue(s.contains(String.valueOf(i))); + } + } + + /** + * get(null) throws NPE + */ + @Test + public void testGet_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.get(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * containsKey(null) throws NPE + */ + @Test + public void testContainsKey_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.containsKey(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * put(null,x) throws NPE + */ + @Test + public void testPut1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.put(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * put(x, null) throws NPE + */ + @Test + public void testPut2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.put(JSR166TestCase.notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * putIfAbsent(null, x) throws NPE + */ + @Test + public void testPutIfAbsent1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.putIfAbsent(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + // Exception tests + + /** + * TODO : Cannot create with negative capacity + */ + /* @Test public void testConstructor1() { + try { + newShmIntString(-1, 0, 1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * TODO : Cannot create with negative concurrency level + */ + /*@Test public void testConstructor2() { + try { + newShmIntString(1, 0, -1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * TODO :Cannot create with only negative capacity + */ + /* @Test public void testConstructor3() { + try { + newShmIntString(-1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * replace(null, x) throws NPE + */ + @Test + public void testReplace_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(null, x, y) throws NPE + */ + @Test + public void testReplaceValue_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(null, "A", "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * containsValue(null) throws NPE + */ +/* @Test public void testContainsValue_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.containsValue(null); + shouldThrow(); + } catch (NullPointerException success) { + } + }*/ + + /** + * todo : contains(null) throws NPE + */ +/* + @Test public void testContains_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.contains(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } +*/ + + /** + * putIfAbsent(x, null) throws NPE + */ + @Test + public void testPutIfAbsent2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.putIfAbsent(JSR166TestCase.notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, null) throws NPE + */ + @Test + public void testReplace2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(JSR166TestCase.notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, null, y) throws NPE + */ + @Test + public void testReplaceValue2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(JSR166TestCase.notPresent, null, "A"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, y, null) throws NPE + */ + @Test + public void testReplaceValue3_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(JSR166TestCase.notPresent, "A", null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(null) throws NPE + */ + @Test + public void testRemove1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + c.remove(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(null, x) throws NPE + */ + @Test + public void testRemove2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + c.remove(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(x, null) returns false + */ + @Test + public void testRemove3() throws IOException { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + assertFalse(c.remove("sadsdf", null)); + } + + // classes for testing Comparable fallbacks + static class BI implements Comparable { + private final int value; + + BI(int value) { + this.value = value; + } + + public int compareTo(BI other) { + return Integer.compare(value, other.value); + } + + public boolean equals(Object x) { + return (x instanceof BI) && ((BI) x).value == value; + } + + public int hashCode() { + return 42; + } + } + + static class CI extends BI { + CI(int value) { + super(value); + } + } + + static class DI extends BI { + DI(int value) { + super(value); + } + } + + static class BS implements Comparable { + private final String value; + + BS(String value) { + this.value = value; + } + + public int compareTo(BS other) { + return value.compareTo(other.value); + } + + public boolean equals(Object x) { + return (x instanceof BS) && value.equals(((BS) x).value); + } + + public int hashCode() { + return 42; + } + } + + static class LexicographicList> extends ArrayList + implements Comparable> { + private static final long serialVersionUID = 0; + static long total; + static long n; + + LexicographicList(Collection c) { + super(c); + } + + LexicographicList(E e) { + super(Collections.singleton(e)); + } + + public int compareTo(LexicographicList other) { + long start = System.currentTimeMillis(); + int common = Math.min(size(), other.size()); + int r = 0; + for (int i = 0; i < common; i++) { + if ((r = get(i).compareTo(other.get(i))) != 0) + break; + } + if (r == 0) + r = Integer.compare(size(), other.size()); + total += System.currentTimeMillis() - start; + n++; + return r; + } + } + + /** + * A deserialized map equals original + */ + /* @Test public void testSerialization() throws Exception { + Map x = map5(); + Map y = serialClone(x); + + assertNotSame(x, y); + assertEquals(x.size(), y.size()); + assertEquals(x, y); + assertEquals(y, x); + }*/ + + /** + * TODO : SetValue of an EntrySet entry sets value in the map. + */ + /*@Test public void testSetValueWriteThrough() { + // Adapted from a bug report by Eric Zoerner + ChronicleMap map = newShmIntString(2, 5.0f, 1); + assertTrue(map.isEmpty()); + for (int i = 0; i < 20; i++) + map.put(new Integer(i), new Integer(i)); + assertFalse(map.isEmpty()); + Map.Entry entry1 = (Map.Entry) map.entrySet().iterator().next(); + // Unless it happens to be first (in which case remainder of + // test is skipped), remove a possibly-colliding key from map + // which, under some implementations, may cause entry1 to be + // cloned in map + if (!entry1.getKey().equals(new Integer(16))) { + map.remove(new Integer(16)); + entry1.setValue("XYZ"); + assertTrue(map.containsValue("XYZ")); // fails if write-through broken + } + }*/ + +} + diff --git a/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java new file mode 100644 index 000000000..cf06064cc --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java @@ -0,0 +1,282 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Rob Austin. + */ +public class ReplicationCheckingMap implements ChronicleMap { + + ChronicleMap map1; + ChronicleMap map2; + + public ReplicationCheckingMap(ChronicleMap map1, ChronicleMap map2) { + this.map1 = map1; + this.map2 = map2; + } + + @Override + public V putIfAbsent(K key, V value) { + return map1.putIfAbsent(key, value); + } + + @Override + public boolean remove(Object key, Object value) { + return map1.remove(key, value); + } + + @Override + public boolean replace(K key, V oldValue, V newValue) { + return map1.replace(key, oldValue, newValue); + } + + @Override + public V replace(final K key, final V value) { + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return map.replace(key, value); + } + } + ); + } + + @Override + public int size() { + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return map.size(); + } + } + ); + + } + + public R check(Call instance) { + R r1 = null; + R r2 = null; + for (int i = 0; i < 50; i++) { + + r1 = (R) instance.method(map1); + r2 = (R) instance.method(map2); + + if (r1 != null && r1.equals(r2)) + return r1; + + if (i > 30) { + try { + Thread.sleep(i); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } else + Thread.yield(); + + } + + + Assert.assertEquals(map1, map2); + System.out.print(map1); + System.out.print(map2); + + if (r1 != null) + Assert.assertEquals(r1.toString(), r2.toString()); + + + return (R) r1; + + } + + @Override + public boolean isEmpty() { + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return map.isEmpty(); + } + } + ); + + } + + @Override + public boolean containsKey(final Object key) { + + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return map.containsKey(key); + } + } + ); + + } + + @Override + public boolean containsValue(final Object value) { + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return map.containsValue(value); + } + } + ); + + } + + @Override + public V get(final Object key) { + + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return map.get(key); + } + } + ); + + } + + @Override + public V put(K key, V value) { + return map1.put(key, value); + } + + @Override + public V remove(Object key) { + return map1.remove(key); + } + + @Override + public void putAll(Map m) { + map1.putAll(m); + } + + @Override + public void clear() { + map1.clear(); + } + + @NotNull + @Override + public Set keySet() { + + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return map.keySet(); + } + } + ); + + } + + @NotNull + @Override + public Collection values() { + + + return check(new Call() { + @Override + public Collection method(ConcurrentMap map) { + return map.values(); + } + } + ); + + } + + @NotNull + @Override + public Set> entrySet() { + + + return check(new Call() { + @Override + public Object method(ConcurrentMap map) { + return (map.entrySet()); + } + } + ); + + } + + @Override + public long longSize() { + return map1.longSize(); + } + + @Override + public V getUsing(K key, V value) { + return map1.getUsing(key, value); + } + + @Override + public V acquireUsing(K key, V value) { + return map1.acquireUsing(key, value); + } + + @Override + public File file() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws IOException { + map1.close(); + map2.close(); + } + + @Override + public boolean equals(Object o) { + return map1.equals(o); + } + + @Override + public int hashCode() { + return map1.hashCode(); + } + + public String toString() { + return map1.toString(); + } + + interface Call { + Object method(ConcurrentMap map); + } + + +} diff --git a/src/test/java/net/openhft/chronicle/map/SampleValues.java b/src/test/java/net/openhft/chronicle/map/SampleValues.java new file mode 100644 index 000000000..bc2ac49d7 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/SampleValues.java @@ -0,0 +1,83 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map;/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshallable; +import org.jetbrains.annotations.NotNull; + +enum BuySell { + Buy, Sell +} + +/** + * Sample entry of 10 fields with two String, Enum, int, double, long + */ +public class SampleValues implements BytesMarshallable { + String aa = "aaaaaaaaaa"; + String bb = "bbbbbbbbbb"; + BuySell cc = BuySell.Buy; + BuySell dd = BuySell.Sell; + int ee = 123456; + int ff = 654321; + double gg = 1.23456789; + double hh = 9.87654321; + long ii = 987654321; + long jj = 123456789; + + @Override + public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { + aa = in.readEnum(String.class); + bb = in.readEnum(String.class); + cc = in.readEnum(BuySell.class); + dd = in.readEnum(BuySell.class); + ee = in.readInt(); + ff = in.readInt(); + gg = in.readDouble(); + hh = in.readDouble(); + ii = in.readLong(); + jj = in.readLong(); + } + + @Override + public void writeMarshallable(@NotNull Bytes out) { + out.writeUTFΔ(aa); + out.writeUTFΔ(bb); + out.writeEnum(cc); + out.writeEnum(dd); + out.writeInt(ee); + out.writeInt(ff); + out.writeDouble(gg); + out.writeDouble(hh); + out.writeLong(ii); + out.writeLong(jj); + } +} diff --git a/src/test/java/net/openhft/collections/SingleMapClusterTest.java b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java similarity index 56% rename from src/test/java/net/openhft/collections/SingleMapClusterTest.java rename to src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java index 5e9f319ab..3677f8555 100644 --- a/src/test/java/net/openhft/collections/SingleMapClusterTest.java +++ b/src/test/java/net/openhft/chronicle/map/SingleMapClusterTest.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,10 +14,9 @@ * limitations under the License. */ -package net.openhft.collections; +package net.openhft.chronicle.map; -import net.openhft.chronicle.map.ChronicleMapBuilder; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -34,21 +31,21 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertTrue; /** - * Test VanillaSharedReplicatedHashMap where the Replicated is over a TCP Socket, but with 4 nodes + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes * * @author Rob Austin. */ public class SingleMapClusterTest { - private SharedHashMap map1a; + private ChronicleMap map1a; - private SharedHashMap map1b; + private ChronicleMap map1b; - private ClusterReplicator clusterB; - private ClusterReplicator clusterA; + private ReplicatingCluster clusterB; + private ReplicatingCluster clusterA; - private ClusterReplicatorBuilder clusterReplicatorBuilder; - private ClusterReplicatorBuilder clusterReplicatorBuilder1; + private ReplicatingClusterBuilder replicatingClusterBuilder; + private ReplicatingClusterBuilder replicatingClusterBuilder1; public static File getPersistenceFile() { @@ -60,53 +57,36 @@ public class SingleMapClusterTest { @Before - public void setup() throws IOException { { - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8086, new InetSocketAddress("localhost", 8087)).heartBeatInterval(1, SECONDS); + final TcpReplicationConfig tcpReplicationConfig = + TcpReplicationConfig.of(8086, new InetSocketAddress("localhost", 8087)) + .heartBeatInterval(1, SECONDS); - clusterReplicatorBuilder = new ClusterReplicatorBuilder((byte) 1, 1024); - clusterReplicatorBuilder.tcpReplicatorBuilder(tcpReplicatorBuilder); + clusterA = new ReplicatingClusterBuilder((byte) 1, 1024) + .tcpReplication(tcpReplicationConfig).create(); // this is how you add maps after the custer is created - map1a = clusterReplicatorBuilder.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - clusterA = clusterReplicatorBuilder.create(); - - - - - - - + map1a = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(1000) + .addReplicator(clusterA.channelReplicator((short) 1)) + .create(getPersistenceFile()); } { - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8087).heartBeatInterval(1, SECONDS); - - clusterReplicatorBuilder1 = new ClusterReplicatorBuilder((byte) - 2, 1024); - clusterReplicatorBuilder1.tcpReplicatorBuilder(tcpReplicatorBuilder); - + final TcpReplicationConfig replicationConfig = + TcpReplicationConfig.of(8087).heartBeatInterval(1, SECONDS); + clusterB = new ReplicatingClusterBuilder((byte) 2, 1024) + .tcpReplication(replicationConfig).create(); // this is how you add maps after the custer is created - map1b = clusterReplicatorBuilder1.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - clusterB = clusterReplicatorBuilder1.create(); - + map1b = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .addReplicator(clusterB.channelReplicator((short) 1)) + .entries(1000) + .create(getPersistenceFile()); } - - } @After @@ -128,7 +108,6 @@ public class SingleMapClusterTest { // todo remove this sleep - map1a.put(1, "EXAMPLE-1"); // allow time for the recompilation to resolve @@ -152,7 +131,7 @@ public class SingleMapClusterTest { private void waitTillEqual(final int timeOutMs) throws InterruptedException { for (int t = 0; t < timeOutMs; t++) { - if (map1a.equals(map1b) ) + if (map1a.equals(map1b)) break; Thread.sleep(1); diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java new file mode 100644 index 000000000..19f07350d --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java @@ -0,0 +1,187 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + + +import net.openhft.lang.values.IntValue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static net.openhft.chronicle.map.Replicators.tcp; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket, but with 4 nodes + * + * @author Rob Austin. + */ +public class TCPSocketReplication4WayMapTest { + + private ChronicleMap map1; + private ChronicleMap map2; + private ChronicleMap map3; + private ChronicleMap map4; + + public static ChronicleMapBuilder newTcpSocketShmBuilder( + Class kClass, Class vClass, + final byte identifier, + final int serverPort, + final InetSocketAddress... endpoints) throws IOException { + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints) + .heartBeatInterval(1L, SECONDS); + return ChronicleMapBuilder.of(kClass, vClass) + .entries(20000L) + .addReplicator(tcp(identifier, tcpConfig)); + } + + public static > T newTcpSocketShmIntString( + final byte identifier, + final int serverPort, + final InetSocketAddress... endpoints) throws IOException { + return (T) newTcpSocketShmBuilder(Integer.class, CharSequence.class, + identifier, serverPort, endpoints).create(getPersistenceFile()); + } + + static ChronicleMap newTcpSocketShmIntValueString( + final byte identifier, + final int serverPort, + final InetSocketAddress... endpoints) throws IOException { + return newTcpSocketShmBuilder(IntValue.class, CharSequence.class, + identifier, serverPort, endpoints) + .entries(20000L) + .keyMarshaller(ByteableIntValueMarshaller.INSTANCE) + .create(getPersistenceFile()); + } + + + @Before + public void setup() throws IOException { + + map1 = newTcpSocketShmIntString((byte) 1, 8086, new InetSocketAddress("localhost", 8087), + new InetSocketAddress("localhost", 8088), new InetSocketAddress("localhost", 8089)); + map2 = newTcpSocketShmIntString((byte) 2, 8087, new InetSocketAddress("localhost", 8088), + new InetSocketAddress("localhost", 8089)); + map3 = newTcpSocketShmIntString((byte) 3, 8088, new InetSocketAddress("localhost", 8089)); + map4 = newTcpSocketShmIntString((byte) 4, 8089); + } + + @After + public void tearDown() throws InterruptedException { + + for (final Closeable closeable : new Closeable[]{map1, map2, map3, map4}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + @Test + public void test() throws IOException, InterruptedException { + Thread.sleep(1000); + map1.put(1, "EXAMPLE-1"); + map2.put(2, "EXAMPLE-1"); + map3.put(3, "EXAMPLE-1"); + map4.remove(3); + + // allow time for the recompilation to resolve + waitTillEqual(2500); + + assertEquals("map2", map1, map2); + assertEquals("map3", map1, map3); + assertEquals("map4", map1, map4); + assertTrue("map2.empty", !map2.isEmpty()); + + } + + + @Test + public void testBufferOverflow() throws IOException, InterruptedException { + Thread.sleep(1000); + for (int i = 0; i < 50; i++) { + map1.put(i, "EXAMPLE-1"); + } + + // allow time for the recompilation to resolve + waitTillEqual(15000); + + assertEquals("map2", map1, map2); + assertEquals("map3", map1, map3); + assertEquals("map4", map1, map4); + assertTrue("map2.empty", !map2.isEmpty()); + + } + + + @Test + public void testBufferOverflowPutIfAbsent() throws IOException, InterruptedException { + Thread.sleep(1000); + + for (int i = 0; i < 1024; i++) { + map1.putIfAbsent(i, "EXAMPLE-1"); + } + + for (int i = 0; i < 1024; i++) { + map1.putIfAbsent(i, ""); + } + + // allow time for the recompilation to resolve + waitTillEqual(10000); + + assertEquals("map2", map1, map2); + assertEquals("map3", map1, map3); + assertEquals("map4", map1, map4); + assertTrue("map2.empty", !map2.isEmpty()); + + } + + + /** + * waits until map1 and map2 show the same value + * + * @param timeOutMs timeout in milliseconds + * @throws InterruptedException + */ + + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + if (map1.equals(map2) && + map1.equals(map3) && + map2.equals(map3) && + map4.equals(map3) && + map1.equals(map4)) + break; + Thread.sleep(1); + } + + } + +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java new file mode 100644 index 000000000..8dc3428e6 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationBootStrapTests.java @@ -0,0 +1,161 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.After; +import org.junit.Test; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; + +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmBuilder; +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmIntString; +import static org.junit.Assert.assertEquals; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class TCPSocketReplicationBootStrapTests { + + private ReplicatedChronicleMap map1; + private ChronicleMap map2; + + @Test + public void testBootstrap() throws IOException, InterruptedException { + + map1 = newTcpSocketShmIntString((byte) 1, 8079); + + ChronicleMapBuilder map2aBuilder = + newTcpSocketShmBuilder(Integer.class, CharSequence.class, + (byte) 2, 8076, new InetSocketAddress("localhost", 8079)); + final ChronicleMap map2a = + map2aBuilder.create(getPersistenceFile()); + map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 + + long lastModificationTime; + + // lets make sure that the message has got to map 1 + do { + lastModificationTime = map1.lastModificationTime((byte) 2); + Thread.yield(); + } while (lastModificationTime == 0); + + final File map2File = map2a.file(); + map2a.close(); + + { + // restart map 2 but don't doConnect it to map one + final ChronicleMap map2b = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .create(map2File); + // add data into it + map2b.put(11, "ADDED WHEN DISCONNECTED TO MAP1"); + map2b.close(); + } + + // now restart map2a and doConnect it to map1, map1 should bootstrap the missing entry + map2 = map2aBuilder.create(map2File); + + // add data into it + waitTillEqual(5000); + assertEquals("ADDED WHEN DISCONNECTED TO MAP1", map1.get(11)); + + } + + + // todo we have to fix this + @Test + public void testBootstrapAndHeartbeat() throws IOException, InterruptedException { + map1 = newTcpSocketShmIntString((byte) 1, 8079, new InetSocketAddress("localhost", 8076)); + ChronicleMapBuilder map2aBuilder = + newTcpSocketShmBuilder(Integer.class, CharSequence.class, (byte) 2, 8076); + final ReplicatedChronicleMap map2a = + (ReplicatedChronicleMap) + map2aBuilder.create(getPersistenceFile()); + + map2a.put(10, "EXAMPLE-10"); // this will be the last time that map1 go an update from map2 + + long lastModificationTime; + + // lets make sure that the message has got to map 1 + do { + lastModificationTime = map1.lastModificationTime((byte) 2); + Thread.yield(); + } while (lastModificationTime == 0); + + final File map2File = map2a.file(); + map2a.close(); + + { + // restart map 2 but don't doConnect it to map one + final ChronicleMap map2b = + ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .create(map2File); + // add data into it + map2b.put(11, "ADDED WHEN DISCONNECTED TO MAP1"); + map2b.close(); + } + + // now restart map2a and doConnect it to map1, map1 should bootstrap the missing entry + map2 = map2aBuilder.create(map2File); + + // add data into it + waitTillEqual(20000); + assertEquals("ADDED WHEN DISCONNECTED TO MAP1", map1.get(11)); + + + } + + @After + public void tearDown() { + + for (final Closeable closeable : new Closeable[]{map1, map2}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + /** + * waits until map1 and map2 show the same value + * + * @param timeOutMs timeout in milliseconds + * @throws InterruptedException + */ + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + if (map1.equals(map2)) + break; + Thread.sleep(1); + } + + } + +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java new file mode 100644 index 000000000..2671e4d40 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationIntValueTest.java @@ -0,0 +1,195 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.values.IntValue; +import net.openhft.lang.values.IntValue$$Native; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.Random; + +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmBuilder; +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmIntValueString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class TCPSocketReplicationIntValueTest { + + + private ChronicleMapBuilder map1Builder; + private ChronicleMap map1; + private ChronicleMap map2; + private IntValue$$Native value; + + @Before + public void setup() throws IOException { + value = new IntValue$$Native(); + value.bytes(new ByteBufferBytes(ByteBuffer.allocateDirect(4)), 0); + map1Builder = newTcpSocketShmBuilder(IntValue.class, CharSequence.class, + (byte) 1, 8076, new InetSocketAddress("localhost", 8077)); + map1 = map1Builder.keyMarshaller(ByteableIntValueMarshaller.INSTANCE) + .create(getPersistenceFile()); + map2 = newTcpSocketShmIntValueString((byte) 2, 8077); + } + + + @After + public void tearDown() throws InterruptedException { + + for (final Closeable closeable : new Closeable[]{map1, map2}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + @Test + @Ignore + public void test3() throws IOException, InterruptedException { + + map1.put(set(5), "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertTrue(!map1.isEmpty()); + } + + private IntValue$$Native set(int x) { + + value.setValue(x); + return value; + } + + + @Test + @Ignore + public void test() throws IOException, InterruptedException { + + map1.put(set(1), "EXAMPLE-1"); + map1.put(set(2), "EXAMPLE-2"); + map1.put(set(3), "EXAMPLE-1"); + + map2.put(set(5), "EXAMPLE-2"); + map2.put(set(6), "EXAMPLE-2"); + + map1.remove(2); + map2.remove(3); + map1.remove(3); + map2.put(set(5), "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertTrue(!map1.isEmpty()); + + } + + + @Test + public void testBufferOverflow() throws IOException, InterruptedException { + + for (int i = 0; i < 1000; i++) { + map1.put(set(i), "EXAMPLE-1"); + } + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertTrue(!map2.isEmpty()); + + } + + /** + * * waits until map1 and map2 show the same value + * + * @param timeOutMs timeout in milliseconds + * @throws InterruptedException + */ + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + if (map1.equals(map2)) + break; + Thread.sleep(1); + } + + } + + + @Test + public void testSoakTestWithRandomData() throws IOException, InterruptedException { + + + final long start = System.currentTimeMillis(); + System.out.print("SoakTesting "); + for (int j = 1; j < 100 * 1000; j++) { + if (j % 5000 == 0) + System.out.print("."); + Random rnd = new Random(j); + for (int i = 1; i < 10; i++) { + + final int select = rnd.nextInt(2); + final ChronicleMap map = select > 0 ? map1 : map2; + + + switch (rnd.nextInt(2)) { + case 0: + map.put(set(rnd.nextInt(1000)) /* + select * 100 */, "test"); + break; + case 1: + map.remove(set(rnd.nextInt(1000)) /*+ select * 100 */); + break; + } + } + + } + + waitTillEqual(5000); + + final long time = System.currentTimeMillis() - start; + + //assertTrue("timeTaken="+time, time < 2200); + System.out.println("\ntime taken millis=" + time); + + } + +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java new file mode 100644 index 000000000..52fc00cac --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest.java @@ -0,0 +1,203 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Random; + +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmBuilder; +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmIntString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class TCPSocketReplicationTest { + + + private ChronicleMap map1; + private ChronicleMap map2; + + @Before + public void setup() throws IOException { + ChronicleMapBuilder map1Builder = + newTcpSocketShmBuilder(Integer.class, CharSequence.class, + (byte) 1, 8076, new InetSocketAddress("localhost", 8077)); + map1 = map1Builder.create(getPersistenceFile()); + map2 = newTcpSocketShmIntString((byte) 2, 8077); + } + + @After + public void tearDown() throws InterruptedException { + + for (final Closeable closeable : new Closeable[]{map1, map2}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + @Test + public void test3() throws IOException, InterruptedException { + + map1.put(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertTrue(!map1.isEmpty()); + } + + + @Test + public void test() throws IOException, InterruptedException { + + map1.put(1, "EXAMPLE-1"); + map1.put(2, "EXAMPLE-2"); + map1.put(3, "EXAMPLE-1"); + + map2.put(5, "EXAMPLE-2"); + map2.put(6, "EXAMPLE-2"); + + map1.remove(2); + map2.remove(3); + map1.remove(3); + map2.put(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertTrue(!map1.isEmpty()); + + } + + + @Test + public void testBufferOverflow() throws IOException, InterruptedException { + + for (int i = 0; i < 1000; i++) { + map1.put(i, "EXAMPLE-1"); + } + + // allow time for the recompilation to resolve + waitTillEqual(9000); + + assertEquals(map1, map2); + assertTrue(!map2.isEmpty()); + + } + + /** + * * waits until map1 and map2 show the same value + * + * @param timeOutMs timeout in milliseconds + * @throws InterruptedException + */ + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + if (map1.equals(map2)) + break; + Thread.sleep(1); + } + + } + + + @Test + @Ignore + public void testSoakTestWithRandomData() throws IOException, InterruptedException { + final long start = System.currentTimeMillis(); + System.out.print("SoakTesting "); + for (int j = 1; j < 90 * 1000; j++) { + if (j % 9000 == 0) + System.out.print("."); + Random rnd = new Random(j); + for (int i = 1; i < 10; i++) { + + final int select = rnd.nextInt(2); + final ChronicleMap map = select > 0 ? map1 : map2; + + + switch (rnd.nextInt(2)) { + case 0: + map.put(rnd.nextInt(1000) /* + select * 100 */, "test"); + break; + case 1: + map.remove(rnd.nextInt(1000) /*+ select * 100 */); + break; + } + } + + } + System.out.println(""); + waitTillEqual(5000); + + } + + + @Ignore + @Test + public void testObjectAllocationWithYourKit() throws IOException, InterruptedException { + + + System.out.print("SoakTesting "); + for (; ; ) { + + Random rnd = new Random(0); + for (int i = 1; i < 10; i++) { + + final int select = rnd.nextInt(2); + final ChronicleMap map = select > 0 ? map1 : map2; + + switch (rnd.nextInt(2)) { + case 0: + map.put(rnd.nextInt(1000) /* + select * 100 */, "test"); + break; + case 1: + map.remove(rnd.nextInt(1000) /*+ select * 100 */); + break; + } + } + + } + + + } + + +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3way.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3way.java new file mode 100644 index 000000000..519b225bf --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3way.java @@ -0,0 +1,165 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmIntString; +import static org.junit.Assert.*; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class TCPSocketReplicationTest3way { + + + private ChronicleMap map1; + private ChronicleMap map2; + private ChronicleMap map3; + + @Before + public void setup() throws IOException { + map1 = newTcpSocketShmIntString((byte) 1, 8076, new InetSocketAddress("localhost", 8077), + new InetSocketAddress("localhost", 8079)); + map2 = newTcpSocketShmIntString((byte) 2, 8077, new InetSocketAddress("localhost", 8079)); + map3 = newTcpSocketShmIntString((byte) 3, 8079); + } + + @After + public void tearDown() throws InterruptedException { + Thread.sleep(5); + for (final Closeable closeable : new Closeable[]{map1, map2, map3}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + @Test + public void test3() throws IOException, InterruptedException { + + assertEquals(null, map3.put(5, "EXAMPLE-2")); + + // allow time for the recompilation to resolve + waitTillEqual(15000); + + assertEquals(map1, map2); + assertEquals(map3, map2); + assertTrue(!map1.isEmpty()); + + } + + @Test + public void test() throws IOException, InterruptedException { + + assertEquals(null, map1.put(1, "EXAMPLE-1")); + assertEquals(null, map1.put(2, "EXAMPLE-2")); + assertNotEquals(null, map1.put(2, "EXAMPLE-1")); + + assertEquals(null, map2.put(5, "EXAMPLE-2")); + assertEquals(null, map2.put(6, "EXAMPLE-2")); + + map1.remove(2); + map2.remove(3); + map1.remove(3); + map2.put(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertEquals(map3, map3); + assertTrue(!map1.isEmpty()); + + } + + + @Test + public void testClear() throws IOException, InterruptedException { + + assertEquals(null, map1.put(1, "EXAMPLE-1")); + assertEquals(null, map1.put(2, "EXAMPLE-2")); + assertNotEquals(null, map1.put(2, "EXAMPLE-1")); + + assertEquals(null, map2.put(5, "EXAMPLE-2")); + assertEquals(null, map2.put(6, "EXAMPLE-2")); + + map1.clear(); + + map2.put(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertEquals(map3, map3); + assertTrue(!map1.isEmpty()); + + } + + + @Test + public void testPutIfAbsent() throws IOException, InterruptedException { + + assertEquals(null, map1.putIfAbsent(1, "EXAMPLE-1")); + assertNotEquals(null, map1.putIfAbsent(1, "EXAMPLE-2")); + assertEquals(null, map1.putIfAbsent(2, "EXAMPLE-2")); + assertEquals(null, map1.putIfAbsent(3, "EXAMPLE-1")); + + assertEquals(null, map2.putIfAbsent(5, "EXAMPLE-2")); + assertEquals(null, map2.putIfAbsent(6, "EXAMPLE-2")); + + map1.remove(2); + map2.remove(3); + map1.remove(3); + map2.putIfAbsent(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertEquals(map3, map3); + assertTrue(!map1.isEmpty()); + + } + + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + if (map1.equals(map2) && + map1.equals(map3)) + break; + Thread.sleep(1); + } + + } +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java new file mode 100644 index 000000000..922fd26f2 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3wayPutReturnsNull.java @@ -0,0 +1,155 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class TCPSocketReplicationTest3wayPutReturnsNull { + + + private ChronicleMap map1; + private ChronicleMap map2; + private ChronicleMap map3; + + static > T newTcpSocketShmIntString( + final byte identifier, + final int serverPort, + final InetSocketAddress... endpoints) throws IOException { + TcpReplicationConfig tcpConfig = TcpReplicationConfig.of(serverPort, endpoints); + return (T) ChronicleMapBuilder.of(Integer.class, CharSequence.class) + + .entries(1000) + .putReturnsNull(true) + .addReplicator(Replicators.tcp(identifier, tcpConfig)) + .create(Builder.getPersistenceFile()); + } + + + @Before + public void setup() throws IOException { + map1 = newTcpSocketShmIntString((byte) 1, 8076, new InetSocketAddress("localhost", 8077), + new InetSocketAddress("localhost", 8078)); + map2 = newTcpSocketShmIntString((byte) 2, 8077, new InetSocketAddress("localhost", 8078)); + map3 = newTcpSocketShmIntString((byte) 3, 8078); + } + + @After + public void tearDown() throws InterruptedException { + + for (final Closeable closeable : new Closeable[]{map1, map2, map3}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + @Test + public void test3() throws IOException, InterruptedException { + + map3.put(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertEquals(map3, map2); + assertTrue(!map1.isEmpty()); + + } + + @Test + public void test() throws IOException, InterruptedException { + + assertEquals(null, map1.put(1, "EXAMPLE-1")); + assertEquals(null, map1.put(2, "EXAMPLE-2")); + assertEquals(null, map1.put(2, "EXAMPLE-1")); + + assertEquals(null, map2.put(5, "EXAMPLE-2")); + assertEquals(null, map2.put(6, "EXAMPLE-2")); + + map1.remove(2); + map2.remove(3); + map1.remove(3); + map2.put(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertEquals(map3, map3); + assertTrue(!map1.isEmpty()); + + } + + + @Test + public void testPutIfAbsent() throws IOException, InterruptedException { + + assertEquals(null, map1.putIfAbsent(1, "EXAMPLE-1")); + assertEquals(null, map1.putIfAbsent(1, "EXAMPLE-2")); + assertEquals(null, map1.putIfAbsent(2, "EXAMPLE-2")); + assertEquals(null, map1.putIfAbsent(3, "EXAMPLE-1")); + + assertEquals(null, map2.putIfAbsent(5, "EXAMPLE-2")); + assertEquals(null, map2.putIfAbsent(6, "EXAMPLE-2")); + + map1.remove(2); + map2.remove(3); + map1.remove(3); + map2.putIfAbsent(5, "EXAMPLE-2"); + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertEquals(map3, map3); + assertTrue(!map1.isEmpty()); + + } + + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + if (map1.equals(map2) && + map1.equals(map3)) + break; + Thread.sleep(1); + } + + } +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java new file mode 100644 index 000000000..9b6dc69eb --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTestPostConnection.java @@ -0,0 +1,140 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.After; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import static net.openhft.chronicle.map.TCPSocketReplication4WayMapTest.newTcpSocketShmBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class TCPSocketReplicationTestPostConnection { + + private ChronicleMap map1; + private ChronicleMap map2; + + @Test + public void testPostConnection() throws IOException, InterruptedException { + + + map1 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 1, 8076); + map1.put(5, "EXAMPLE-2"); + Thread.sleep(1); + map2 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 2, 8077, new InetSocketAddress("localhost", 8076)); + + // allow time for the recompilation to resolve + waitTillEqual(500); + + assertEquals(map1, map2); + assertTrue(!map1.isEmpty()); + + } + + @Test + public void testPostConnectionNoSleep() throws IOException, InterruptedException { + + + map1 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 1, 8076); + map1.put(5, "EXAMPLE-2"); + map2 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 2, 8077, new InetSocketAddress("localhost", 8076)); + + // allow time for the recompilation to resolve + waitTillEqual(1000); + + assertEquals(map1, map2); + assertTrue(!map1.isEmpty()); + + } + + @Test + public void testBootStrapIntoNewMapWithNewFile() throws IOException, InterruptedException { + + ChronicleMapBuilder map2aBuilder = + newTcpSocketShmBuilder(Integer.class, CharSequence.class, + (byte) 2, 8077, new InetSocketAddress("localhost", 8076)); + final ChronicleMap map2a = + map2aBuilder.create(Builder.getPersistenceFile()); + map1 = TCPSocketReplication4WayMapTest.newTcpSocketShmIntString((byte) 1, 8076); + + Thread.sleep(1); + map1.put(5, "EXAMPLE-2"); + + map2a.close(); + + + Thread.sleep(1); + map1.put(6, "EXAMPLE-1"); + + // recreate map2 with new unique file + map2 = map2aBuilder.create(Builder.getPersistenceFile()); + + + // allow time for the recompilation to resolve + waitTillEqual(5000); + + assertEquals(map1, map2); + assertTrue(!map1.isEmpty()); + assertTrue(map2.get(6).equals("EXAMPLE-1")); + + } + + + @After + public void tearDown() { + + for (final Closeable closeable : new Closeable[]{map1, map2}) { + try { + if (closeable != null) + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + /** + * * waits until map1 and map2 show the same value + * + * @param timeOutMs timeout in milliseconds + * @throws InterruptedException + */ + private void waitTillEqual(final int timeOutMs) throws InterruptedException { + int t = 0; + for (; t < timeOutMs; t++) { + if (map1.equals(map2)) + break; + Thread.sleep(1); + } + + } + +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java new file mode 100644 index 000000000..6635b84cf --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java @@ -0,0 +1,359 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; + +/** + * @author Rob Austin. + */ +public class TimeBasedReplicationTests extends JSR166TestCase { + + + public static final byte IDENTIFIER = 1; + + private static File getPersistenceFile() { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/chm-test" + System.nanoTime()); + file.deleteOnExit(); + return file; + } + + + @Test + public void testIgnoreALatePut() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + + ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(10) + .forceReplicatedImpl() + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(map.size(), 1); + assertEquals(map.get("key-1"), "value-1"); + + // now test assume that we receive a late update to the map, the following update should be ignored + late(timeProvider); + + + map.put("key-1", "value-2"); + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + assertEquals(map.size(), 1); + assertEquals(map.get("key-1"), "value-1"); + + } + + @Test + public void testIgnoreALatePutIfAbsent() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(10) + .forceReplicatedImpl() + .entries(10) + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(map.size(), 1); + assertEquals(map.get("key-1"), "value-1"); + + // now test assume that we receive a late update to the map, the following update should be ignored + late(timeProvider); + + + final Object o = map.putIfAbsent("key-1", "value-2"); + assertEquals(o, null); + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + assertEquals(1, map.size()); + assertEquals(map.get("key-1"), "value-1"); + + } + + @Test + public void testIgnoreALateReplace() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(10) + .forceReplicatedImpl() + .entries(10) + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + + // now test assume that we receive a late update to the map, + // the following update should be ignored + late(timeProvider); + + + final Object o = map.replace("key-1", "value-2"); + assertEquals(o, null); + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + assertEquals(map.size(), 1); + assertEquals("value-1", map.get("key-1")); + + } + + @Test + public void testIgnoreALateReplaceWithValue() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(10) + .forceReplicatedImpl() + .entries(10) + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + // now test assume that we receive a late update to the map, the following update should be ignored + late(timeProvider); + + + assertEquals(null, map.replace("key-1", "value-2")); + + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + } + + @Test + public void testIgnoreALateRemoveWithValue() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + + .forceReplicatedImpl() + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + // now test assume that we receive a late update to the map, the following update should be ignored + late(timeProvider); + + + assertEquals(false, map.remove("key-1", "value-1")); + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + } + + @Test + public void testIgnoreALateRemove() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + + .forceReplicatedImpl() + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + // now test assume that we receive a late update to the map, the following update should be ignored + late(timeProvider); + + map.remove("key-1"); + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + } + + + @Test + public void testIgnoreWithRemoteRemove() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ReplicatedChronicleMap map = (ReplicatedChronicleMap) ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + + .forceReplicatedImpl() + .entries(10) + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + // now test assume that we receive a late update to the map, the following update should be ignored + final long late = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(5); + assertEquals(null, map.remove("key-1", "value-2", IDENTIFIER, late)); + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + assertTrue(map.containsValue("value-1")); + assertFalse(map.containsValue("value-2")); + } + + + @Test + public void testIgnoreWithRemotePut() throws IOException { + + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ReplicatedChronicleMap map = (ReplicatedChronicleMap) ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + + .forceReplicatedImpl() + .entries(10) + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + assertEquals(1, map.size()); + assertEquals("value-1", map.get("key-1")); + + // now test assume that we receive a late update to the map, the following update should be ignored + // now test assume that we receive a late update to the map, the following update should be ignored + final long late = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(5); + assertEquals(null, map.put("key-1", "value-2", IDENTIFIER, late)); + + + // we'll now flip the time back to the current in order to do the read the result + current(timeProvider); + + assertEquals("value-1", map.get("key-1")); + assertEquals(1, map.size(), 0); + } + + + @Test + public void testRemoveFollowedByLatePut() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ReplicatedChronicleMap map = (ReplicatedChronicleMap) ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(10) + .forceReplicatedImpl() + .entries(10) + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + map.remove("key-1", "value-1"); + assertEquals(0, map.size()); + assertEquals(null, map.get("key-1")); + assertEquals(false, map.containsKey("key-1")); + + // test assume that we receive a late update to the map, the following update should be ignored + final long late = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(50); + assertEquals(null, map.put("key-1", "value-2", IDENTIFIER, late)); + + assertEquals(null, map.get("key-1")); + assertEquals(false, map.containsKey("key-1")); + assertEquals(0, map.size(), 0); + } + + + @Test + public void testPutRemovePut() throws IOException { + + final TimeProvider timeProvider = Mockito.mock(TimeProvider.class); + ChronicleMap map = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .entries(10) + .forceReplicatedImpl() + .entries(10) + .timeProvider(timeProvider) + .create(getPersistenceFile()); + + current(timeProvider); + + // we do a put at the current time + map.put("key-1", "value-1"); + map.remove("key-1"); + assertEquals(0, map.size()); + assertEquals(null, map.put("key-1", "new-value-2")); + assertEquals(true, map.containsKey("key-1")); + assertEquals("new-value-2", map.get("key-1")); + assertEquals(1, map.size(), 0); + } + + private void current(TimeProvider timeProvider) { + Mockito.when(timeProvider.currentTimeMillis()).thenReturn(System.currentTimeMillis()); + } + + private void late(TimeProvider timeProvider) { + Mockito.when(timeProvider.currentTimeMillis()).thenReturn(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(5)); + } +} diff --git a/src/test/java/net/openhft/collections/TwoMapOnDifferentServers.java b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java similarity index 59% rename from src/test/java/net/openhft/collections/TwoMapOnDifferentServers.java rename to src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java index 21684cdaf..91671206d 100644 --- a/src/test/java/net/openhft/collections/TwoMapOnDifferentServers.java +++ b/src/test/java/net/openhft/chronicle/map/TwoMapOnDifferentServers.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,10 +14,8 @@ * limitations under the License. */ -package net.openhft.collections; +package net.openhft.chronicle.map; -import net.openhft.chronicle.map.ChronicleMap; -import net.openhft.chronicle.map.ChronicleMapBuilder; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -30,6 +26,8 @@ import java.io.IOException; import java.net.InetSocketAddress; import static java.util.concurrent.TimeUnit.SECONDS; +import static net.openhft.chronicle.map.Builder.getPersistenceFile; +import static net.openhft.chronicle.map.Replicators.tcp; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -45,30 +43,22 @@ public class TwoMapOnDifferentServers { @Before public void setup() throws IOException { - final TcpReplicatorBuilder tcpReplicatorBuilder1 = - new TcpReplicatorBuilder(8076, new InetSocketAddress[]{new InetSocketAddress("localhost", 8077)}) - .heartBeatInterval(1, SECONDS); + final TcpReplicationConfig tcpReplicationConfig = TcpReplicationConfig + .of(8076, new InetSocketAddress("localhost", 8077)) + .heartBeatInterval(1, SECONDS); - File persistenceFile = ClusterReplicationTest.getPersistenceFile(); - map1 = (ChronicleMap) ChronicleMapBuilder.of(Integer.class, - CharSequence.class) - .entries(1000) - .identifier((byte) 1) - .tcpReplicatorBuilder(tcpReplicatorBuilder1) - .entries(20000).file(persistenceFile).create(); + File persistenceFile = getPersistenceFile(); + map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000) + .addReplicator(tcp((byte) 1, tcpReplicationConfig)) + .create(persistenceFile); - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8077, new InetSocketAddress[]{}) - .heartBeatInterval(1, SECONDS); - - - map2 = (ChronicleMap) ChronicleMapBuilder.of(Integer.class, - CharSequence.class) - .entries(1000) - .identifier((byte) 2) - .tcpReplicatorBuilder(tcpReplicatorBuilder) - .entries(20000).file(persistenceFile).kClass(Integer.class).vClass(CharSequence.class).create(); + map2 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(20000) + .addReplicator(tcp((byte) 2, + TcpReplicationConfig.of(8077).heartBeatInterval(1, SECONDS))) + .create(persistenceFile); } @After diff --git a/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java new file mode 100644 index 000000000..71690a2b5 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/UDPSocketReplicationTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.net.Inet4Address; + +import static net.openhft.chronicle.map.Replicators.udp; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class UDPSocketReplicationTest { + + private ChronicleMap map2; + + static ChronicleMap newUdpSocketShmIntString( + final int identifier, + final int udpPort) throws IOException { + + UdpReplicationConfig udpConfig = UdpReplicationConfig + .simple(Inet4Address.getByName("255.255.255.255"), udpPort); + return ChronicleMapBuilder.of(Integer.class, CharSequence.class) + + .entries(1000) + .addReplicator(udp((byte) identifier, udpConfig)) + .create(Builder.getPersistenceFile()); + } + + @Before + public void setup() throws IOException { + + map2 = newUdpSocketShmIntString(1, 1234); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @After + public void tearDown() throws InterruptedException { + + for (final Closeable closeable : new Closeable[]{map2}) { + try { + closeable.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + @Test + @Ignore + public void testBufferOverflow() throws IOException, InterruptedException { + + for (int i = 0; i < 1024; i++) { + Thread.sleep(5000); + map2.put(i * 2, "E"); + System.out.println("" + map2); + } + + } + + +} + + + diff --git a/src/test/java/net/openhft/chronicle/map/blackboard/CHMTest5.java b/src/test/java/net/openhft/chronicle/map/blackboard/CHMTest5.java new file mode 100644 index 000000000..1a322f450 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/blackboard/CHMTest5.java @@ -0,0 +1,256 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.blackboard; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.model.constraints.MaxSize; + +import java.io.File; + +import static org.junit.Assert.assertEquals; + +/** + * @author jack shirazi + */ +public class CHMTest5 { + public static final String TEST_KEY = "whatever"; + public static int NUMBER_OF_PROCESSES_ALLOWED = 2; + + public static void main(String[] args) throws Exception { + //First create (or access if already created) the shared map + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(String.class, CHMTest5Data.class) + .entries(1000); + + //// don't include this, just to check it is as expected. + assertEquals(8, builder.minSegments()); + //// end of test + + String chmPath = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "CHMTest5"; + ChronicleMap theSharedMap = builder + .create(new File(chmPath)); + + //Now get the shared data object, auto-creating if it's not there + CHMTest5Data data = DataValueClasses.newDirectReference(CHMTest5Data.class); + theSharedMap.acquireUsing(TEST_KEY, data); + //if this was newly created, we need to set the max allowed + //Note, the object is pointing at shared memory and writes + //directly to it, so no need to put() it back into the map + if (data.getMaxNumberOfProcessesAllowed() != NUMBER_OF_PROCESSES_ALLOWED) { + //it's either a new object, set to 0, or + //another process set it to an invalid value + if (data.compareAndSwapMaxNumberOfProcessesAllowed(0, NUMBER_OF_PROCESSES_ALLOWED)) { + //What we expected, everything's good + } else { + //something else set a value, if it's not 2 we've got a conflict + if (data.getMaxNumberOfProcessesAllowed() != NUMBER_OF_PROCESSES_ALLOWED) { + System.out.println("Incorrect configuration found, expected " + + NUMBER_OF_PROCESSES_ALLOWED + " slots, instead found " + + data.getMaxNumberOfProcessesAllowed() + " - exiting"); + System.exit(0); + } + } + } + + //Now, we look for an empty slot in the "time" array. + //Because this initial implementation of the data object + //is directly accessing the shared memory, we just operate + //on it as a shared object + + // There would be no need to lock the MaxNumberOfProcessesAllowed field + // as we used CAS to update and it's read-only after that + // but we need to lock access to the Time array + long[] times1 = new long[NUMBER_OF_PROCESSES_ALLOWED]; + boolean locked = false; + for (int i = 0; i < 1000000; i++) { + //try up to 1 second + if (data.tryLockNanosEntry(1000L)) { + locked = true; + break; + } + } + if (!locked) { + System.out.println("Unable to acquire a lock on the time array - exiting"); + System.exit(0); + + } + try { + //we've got the lock, now copy the array + for (int i = 0; i < times1.length; i++) { + times1[i] = data.getTimeAt(i); + } + } finally { + //and release the lock + try { + data.unlockEntry(); + } catch (IllegalMonitorStateException e) { + //odd, but we'll be unlocked either way + System.out.println("Unexpected state: " + e); + e.printStackTrace(); + } + } + pause(300L); + long[] times2 = new long[NUMBER_OF_PROCESSES_ALLOWED]; + locked = false; + for (int i = 0; i < 1000000; i++) { + //try up to 1 second + if (data.tryLockNanosEntry(1000L)) { + locked = true; + break; + } + } + if (!locked) { + System.out.println("Unable to acquire a lock on the time array - exiting"); + System.exit(0); + + } + try { + //we've got the lock, now copy the array + for (int i = 0; i < times2.length; i++) { + times2[i] = data.getTimeAt(i); + } + } finally { + //and release the lock + try { + data.unlockEntry(); + } catch (IllegalMonitorStateException e) { + //odd, but we'll be unlocked either way + System.out.println("Unexpected state: " + e); + e.printStackTrace(); + } + } + //look for a slot that hasn't changed in that 300ms pause + int slotindex = 0; + long lastUpdateTime = -1; + for (; slotindex < times1.length; slotindex++) { + if (times2[slotindex] == times1[slotindex]) { + //we have an index which has not been updated by anything else + //in the 300ms pause, so we have a spare slot - we use this slot + long timenow = System.currentTimeMillis(); + locked = false; + for (int i = 0; i < 1000000; i++) { + //try up to 1 second + if (data.tryLockNanosEntry(1000L)) { + locked = true; + break; + } + } + if (!locked) { + System.out.println("Unable to acquire a lock on the time array - exiting"); + System.exit(0); + + } + try { + data.setTimeAt(slotindex, timenow); + } finally { + //and release the lock + try { + data.unlockEntry(); + } catch (IllegalMonitorStateException e) { + //odd, but we'll be unlocked either way + System.out.println("Unexpected state: " + e); + e.printStackTrace(); + } + } + + //Now we have successfully acquired a slot + lastUpdateTime = timenow; + System.out.println("We have started on slot " + slotindex); + break; + } + } + + if (lastUpdateTime == -1) { + System.out.println("We failed to find a free slot, so terminating now"); + System.exit(0); + } + + //Now let's run for 60 seconds, updating the slot every 100ms + for (int count = 0; count < 600; count++) { + pause(100L); + long timenow = System.currentTimeMillis(); + locked = false; + for (int i = 0; i < 1000000; i++) { + //try up to 1 second + if (data.tryLockNanosEntry(1000L)) { + locked = true; + break; + } + } + if (!locked) { + System.out.println("Unable to acquire a lock on the time array - exiting"); + System.exit(0); + + } + try { + if (lastUpdateTime == data.getTimeAt(slotindex)) { + //That's what we expect so just update the slot + data.setTimeAt(slotindex, timenow); + lastUpdateTime = timenow; + } else { + //Some other process has hijacked our slot - so + //the only thing we can do is terminate + System.out.println("Another process has hijacked our slot " + slotindex + ", so terminating now"); + System.exit(0); + } + } finally { + //and release the lock + try { + data.unlockEntry(); + } catch (IllegalMonitorStateException e) { + //odd, but we'll be unlocked either way + System.out.println("Unexpected state: " + e); + e.printStackTrace(); + } + } + } + System.out.println("Exiting slot " + slotindex + " after completing the full test."); + } + + public static void pause(long pause) { + long start = System.currentTimeMillis(); + long elapsedTime; + while ((elapsedTime = System.currentTimeMillis() - start) < pause) { + try { + Thread.sleep(pause - elapsedTime); + } catch (InterruptedException e) { + } + } + } + + public static interface CHMTest5Data { + public int getMaxNumberOfProcessesAllowed(); + + public void setMaxNumberOfProcessesAllowed(int max); + + public void setTimeAt(@MaxSize(4) int index, long time); + + public long getTimeAt(int index); + + boolean compareAndSwapMaxNumberOfProcessesAllowed(int expected, int value); + + void busyLockEntry() throws InterruptedException, IllegalStateException; + + boolean tryLockNanosEntry(long nanos); + + boolean tryLockEntry(); + + void unlockEntry() throws IllegalMonitorStateException; + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/blackboard/CHMTestIterator1.java b/src/test/java/net/openhft/chronicle/map/blackboard/CHMTestIterator1.java new file mode 100644 index 000000000..6d7d4e96a --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/blackboard/CHMTestIterator1.java @@ -0,0 +1,64 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.blackboard; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; + +import java.io.File; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author Anshul Shelley + */ + +public class CHMTestIterator1 { + public static void main(String[] args) throws Exception { + AtomicLong alValue = new AtomicLong(); + AtomicLong alKey = new AtomicLong(); + int runs = 3000000; + ChronicleMapBuilder builder = ChronicleMapBuilder.of(String.class, Long.class) + .entries(runs); + String chmPath = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "CHMTestIterator1"; + ChronicleMap chm = builder.create(new File(chmPath)); + /*chm.put("k1", alValue.incrementAndGet()); + chm.put("k2", alValue.incrementAndGet()); + chm.put("k3", alValue.incrementAndGet()); + chm.put("k4", alValue.incrementAndGet()); + chm.put("k5", alValue.incrementAndGet());*/ + //chm.keySet(); + + + for (int i = 0; i < runs; i++) { + chm.put("k" + alKey.incrementAndGet(), alValue.incrementAndGet()); + } + + long start = System.nanoTime(); + for (Map.Entry entry : chm.entrySet()) { + entry.getKey(); + entry.getValue(); + + } + long time = System.nanoTime() - start; + System.out.println("Average iteration time was " + time / runs / 1e3 + "us, for " + runs / 1e6 + "m entries"); + + + } + + +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/BondVOInterface.java b/src/test/java/net/openhft/chronicle/map/fromdocs/BondVOInterface.java new file mode 100644 index 000000000..d38903b09 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/BondVOInterface.java @@ -0,0 +1,84 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.lang.model.constraints.MaxSize; + +public interface BondVOInterface { + /* add support for entry based locking */ + void busyLockEntry() throws InterruptedException; + + void unlockEntry(); + + long getIssueDate(); + + void setIssueDate(long issueDate); /* time in millis */ + + long getMaturityDate(); + + void setMaturityDate(long maturityDate); /* time in millis */ + + long addAtomicMaturityDate(long toAdd); + + boolean compareAndSwapCoupon(double expected, double value); + + double getCoupon(); + + void setCoupon(double coupon); + + double addAtomicCoupon(double toAdd); + + String getSymbol(); + + void setSymbol(@MaxSize(20) String symbol); + + // OpenHFT Off-Heap array[ ] processing notice ‘At’ suffix + void setMarketPxIntraDayHistoryAt(@MaxSize(7) int tradingDayHour, MarketPx mPx); + + /* 7 Hours in the Trading Day: + * index_0 = 9.30am, + * index_1 = 10.30am, + …, + * index_6 = 4.30pm + */ + + MarketPx getMarketPxIntraDayHistoryAt(int tradingDayHour); + + /* nested interface - empowering an Off-Heap hierarchical “TIER of prices” + as array[ ] value */ + interface MarketPx { + double getCallPx(); + + void setCallPx(double px); + + double getParPx(); + + void setParPx(double px); + + double getMaturityPx(); + + void setMaturityPx(double px); + + double getBidPx(); + + void setBidPx(double px); + + double getAskPx(); + + void setAskPx(double px); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/GettingStartedTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/GettingStartedTest.java new file mode 100644 index 000000000..cde364bb9 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/GettingStartedTest.java @@ -0,0 +1,66 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.map.ChronicleMapBuilder; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * Created by peter on 28/02/14. + */ +public class GettingStartedTest { + @Test + @Ignore + public void testTheCodeInGuide() throws IOException { + String tmpdir = System.getProperty("java.io.tmpdir"); + Map map = ChronicleMapBuilder.of(String.class, String.class) + + .create(new File(tmpdir + "/shared.map")); + map.put("some.json", "missing"); + map.get("some.json"); + + map.put("Hello", "World"); + String hi = map.get("Hello"); + map.put("Long String", "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"); + String json = "{\"menu\": {\n" + + " \"id\": \"file\",\n" + + " \"value\": \"File\",\n" + + " \"popup\": {\n" + + " \"menuitem\": [\n" + + " {\"value\": \"New\", \"onclick\": \"CreateNew()\"},\n" + + " {\"value\": \"Open\", \"onclick\": \"OpenDoc()\"},\n" + + " {\"value\": \"Close\", \"onclick\": \"CloseDoc()\"}\n" + + " ]\n" + + " }\n" + + "}}"; + map.put("some.json", json); + + String s2 = map.get("some.json"); + assertEquals(json, s2); + + ((Closeable) map).close(); + + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java new file mode 100644 index 000000000..2d9de48d6 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java @@ -0,0 +1,151 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.affinity.AffinitySupport; +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import net.openhft.lang.model.DataValueClasses; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +import static org.junit.Assert.assertEquals; + +/** + * These code fragments will appear in an article on OpenHFT. These tests to ensure that the examples compile and behave as expected. + */ +public class OpenJDKAndHashMapExamplesTest { + private static final SimpleDateFormat YYYYMMDD = new SimpleDateFormat("yyyyMMdd"); + + + private static final String TMP = System.getProperty("java.io.tmpdir"); + + private static long parseYYYYMMDD(String s) { + try { + return YYYYMMDD.parse(s).getTime(); + } catch (ParseException e) { + throw new AssertionError(e); + } + } + + @Test + @Ignore + public void bondExample() throws IOException, InterruptedException { + + + ChronicleMap chm = ChronicleMapBuilder + .of(String.class, BondVOInterface.class) + .entrySize(512) + .create(new File(TMP + "/chm-myBondPortfolioCHM")); + + + BondVOInterface bondVO = DataValueClasses.newDirectReference(BondVOInterface.class); + chm.acquireUsing("369604103", bondVO); + bondVO.setIssueDate(parseYYYYMMDD("20130915")); + bondVO.setMaturityDate(parseYYYYMMDD("20140915")); + bondVO.setCoupon(5.0 / 100); // 5.0% + + BondVOInterface.MarketPx mpx930 = bondVO.getMarketPxIntraDayHistoryAt(0); + mpx930.setAskPx(109.2); + mpx930.setBidPx(106.9); + + BondVOInterface.MarketPx mpx1030 = bondVO.getMarketPxIntraDayHistoryAt(1); + mpx1030.setAskPx(109.7); + mpx1030.setBidPx(107.6); + + + ChronicleMap chmB = ChronicleMapBuilder + .of(String.class, BondVOInterface.class) + .entrySize(320) + .create(new File(TMP + "/chm-myBondPortfolioCHM")); + + // ZERO Copy but creates a new off heap reference each time + + + BondVOInterface bondVOB = chmB.get("369604103"); + assertEquals(5.0 / 100, bondVOB.getCoupon(), 0.0); + + BondVOInterface.MarketPx mpx930B = bondVOB.getMarketPxIntraDayHistoryAt(0); + assertEquals(109.2, mpx930B.getAskPx(), 0.0); + assertEquals(106.9, mpx930B.getBidPx(), 0.0); + + BondVOInterface.MarketPx mpx1030B = bondVOB.getMarketPxIntraDayHistoryAt(1); + assertEquals(109.7, mpx1030B.getAskPx(), 0.0); + assertEquals(107.6, mpx1030B.getBidPx(), 0.0); + + + //ZERO-COPY + // our reusable, mutable off heap reference, generated from the interface. + BondVOInterface bondZC = DataValueClasses.newDirectReference(BondVOInterface.class); + + // lookup the key and give me a reference to the data. + if (chm.getUsing("369604103", bondZC) != null) { + // found a key and bondZC has been set + // get directly without touching the rest of the record. + long _matDate = bondZC.getMaturityDate(); + // write just this field, again we need to assume we are the only writer. + bondZC.setMaturityDate(parseYYYYMMDD("20440315")); + + //demo of how to do OpenHFT off-heap array[ ] processing + int tradingHour = 2; //current trading hour intra-day + BondVOInterface.MarketPx mktPx = bondZC.getMarketPxIntraDayHistoryAt(tradingHour); + if (mktPx.getCallPx() < 103.50) { + mktPx.setParPx(100.50); + mktPx.setAskPx(102.00); + mktPx.setBidPx(99.00); + // setMarketPxIntraDayHistoryAt is not needed as we are using zero copy, + // the original has been changed. + } + } + + // bondZC will be full of default values and zero length string the first time. + + // from this point, all operations are completely record/entry local, + // no other resource is involved. + // now perform thread safe operations on my reference + bondZC.addAtomicMaturityDate(16 * 24 * 3600 * 1000L); //20440331 + + + bondZC.addAtomicCoupon(-1 * bondZC.getCoupon()); //MT-safe! now a Zero Coupon Bond. + + // say I need to do something more complicated + // set the Threads getId() to match the process id of the thread. + AffinitySupport.setThreadId(); + + bondZC.busyLockEntry(); + try { + String str = bondZC.getSymbol(); + if (str.equals("IBM_HY_2044")) + bondZC.setSymbol("OPENHFT_IG_2044"); + } finally { + bondZC.unlockEntry(); + } + + // cleanup. + chm.close(); + chmB.close(); + new File("/dev/chm/myBondPortfolioCHM").delete(); + + } + + +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddLeft.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddLeft.java new file mode 100644 index 000000000..e7beb0025 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddLeft.java @@ -0,0 +1,68 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs.pingpong_latency; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.fromdocs.BondVOInterface; + +import java.io.IOException; +import java.util.Arrays; + +import static net.openhft.lang.model.DataValueClasses.newDirectReference; + +public class PingPongAddLeft { + public static void main(String... ignored) throws IOException { + ChronicleMap chm = PingPongCASLeft.acquireCHM(); + + playPingPong(chm, +1, true, "PingPongAddLEFT"); + } + + static void playPingPong(ChronicleMap chm, double add, boolean setFirst, final String desc) { + BondVOInterface bond1 = newDirectReference(BondVOInterface.class); + BondVOInterface bond2 = newDirectReference(BondVOInterface.class); + BondVOInterface bond3 = newDirectReference(BondVOInterface.class); + BondVOInterface bond4 = newDirectReference(BondVOInterface.class); + + chm.acquireUsing("369604101", bond1); + chm.acquireUsing("369604102", bond2); + chm.acquireUsing("369604103", bond3); + chm.acquireUsing("369604104", bond4); + System.out.printf("\n\n" + desc + ": Timing off-heap operations on /dev/chm/RDR_DIM_Mock\n"); + if (setFirst) { + bond1.setCoupon(add); + bond2.setCoupon(add); + bond3.setCoupon(add); + bond4.setCoupon(add); + } + int timeToCallNanoTime = 30; + int runs = 1000000; + long[] timings = new long[runs]; + for (int j = 0; j < 100; j++) { + for (int i = 0; i < runs; i++) { + long _start = System.nanoTime(); + bond1.addAtomicCoupon(add); + bond2.addAtomicCoupon(add); + bond3.addAtomicCoupon(add); + bond4.addAtomicCoupon(add); + timings[i] = (System.nanoTime() - _start - timeToCallNanoTime) / 4; + } + Arrays.sort(timings); + System.out.printf("#%d: atomic add coupon 50/90/99%%tile was %,d / %,d / %,d%n", + j, timings[runs / 2], timings[runs * 9 / 10], timings[runs * 99 / 100]); + } + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddRight.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddRight.java new file mode 100644 index 000000000..bfd65d81d --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongAddRight.java @@ -0,0 +1,33 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs.pingpong_latency; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.fromdocs.BondVOInterface; + +import java.io.IOException; + +import static net.openhft.chronicle.map.fromdocs.pingpong_latency.PingPongAddLeft.playPingPong; + +public class PingPongAddRight { + public static void main(String... ignored) throws IOException { + ChronicleMap chm = PingPongCASLeft.acquireCHM(); + + playPingPong(chm, -1, false, "PingPongAddRIGHT"); + } + +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java new file mode 100644 index 000000000..28aa4332e --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java @@ -0,0 +1,99 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs.pingpong_latency; + +import net.openhft.affinity.AffinitySupport; +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import net.openhft.chronicle.map.fromdocs.BondVOInterface; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import static net.openhft.lang.model.DataValueClasses.newDirectReference; + +/* on an i7-4500 laptop. + +PingPongLEFT: Timing 1 x off-heap operations on /dev/chm/RDR_DIM_Mock +#0: compareAndSwapCoupon() 50/90/99%tile was 41 / 52 / 132 +#1: compareAndSwapCoupon() 50/90/99%tile was 40 / 56 / 119 +#2: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 54 +#3: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 56 +#4: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 54 +#5: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 54 +#6: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 55 +#7: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 55 +#8: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 54 +#9: compareAndSwapCoupon() 50/90/99%tile was 39 / 48 / 54 + + */ +public class PingPongCASLeft { + public static void main(String... ignored) throws IOException { + ChronicleMap chm = PingPongCASLeft.acquireCHM(); + + playPingPong(chm, 4, 5, true, "PingPongCASLEFT"); + } + + static void playPingPong(ChronicleMap chm, double _coupon, double _coupon2, boolean setFirst, final String desc) { + BondVOInterface bond1 = newDirectReference(BondVOInterface.class); + BondVOInterface bond2 = newDirectReference(BondVOInterface.class); + BondVOInterface bond3 = newDirectReference(BondVOInterface.class); + BondVOInterface bond4 = newDirectReference(BondVOInterface.class); + + chm.acquireUsing("369604101", bond1); + chm.acquireUsing("369604102", bond2); + chm.acquireUsing("369604103", bond3); + chm.acquireUsing("369604104", bond4); + System.out.printf("\n\n" + desc + ": Timing 1 x off-heap operations on /dev/chm/RDR_DIM_Mock\n"); + if (setFirst) { + bond1.setCoupon(_coupon); + bond2.setCoupon(_coupon); + bond3.setCoupon(_coupon); + bond4.setCoupon(_coupon); + } + int timeToCallNanoTime = 30; + int runs = 1000000; + long[] timings = new long[runs]; + for (int j = 0; j < 10; j++) { + for (int i = 0; i < runs; i++) { + long _start = System.nanoTime(); // + while (!bond1.compareAndSwapCoupon(_coupon, _coupon2)) ; + while (!bond2.compareAndSwapCoupon(_coupon, _coupon2)) ; + while (!bond3.compareAndSwapCoupon(_coupon, _coupon2)) ; + while (!bond4.compareAndSwapCoupon(_coupon, _coupon2)) ; + + timings[i] = (System.nanoTime() - _start - timeToCallNanoTime) / 4; + } + Arrays.sort(timings); + System.out.printf("#%d: compareAndSwapCoupon() 50/90/99%%tile was %,d / %,d / %,d%n", + j, timings[runs / 2], timings[runs * 9 / 10], timings[runs * 99 / 100]); + } + } + + static ChronicleMap acquireCHM() throws IOException { + // ensure thread ids are globally unique. + AffinitySupport.setThreadId(); + + String TMP = System.getProperty("java.io.tmpdir"); + return ChronicleMapBuilder.of(String.class, BondVOInterface.class) + + .entries(16) + .entrySize(64) + .create(new File(TMP + "/BondPortfolioCHM")); + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASRight.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASRight.java new file mode 100644 index 000000000..ab699a942 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASRight.java @@ -0,0 +1,32 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs.pingpong_latency; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.fromdocs.BondVOInterface; + +import java.io.IOException; + +import static net.openhft.chronicle.map.fromdocs.pingpong_latency.PingPongCASLeft.playPingPong; + +public class PingPongCASRight { + public static void main(String... ignored) throws IOException { + ChronicleMap chm = PingPongCASLeft.acquireCHM(); + + playPingPong(chm, 5, 4, false, "PingPongCASLEFT"); + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockLeft.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockLeft.java new file mode 100644 index 000000000..a10437ba0 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockLeft.java @@ -0,0 +1,85 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs.pingpong_latency; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.fromdocs.BondVOInterface; + +import java.io.IOException; +import java.util.Arrays; + +import static net.openhft.lang.model.DataValueClasses.newDirectReference; + +public class PingPongLockLeft { + public static void main(String... ignored) throws IOException, InterruptedException { + ChronicleMap chm = PingPongCASLeft.acquireCHM(); + + playPingPong(chm, 4, 5, true, "PingPongLockLEFT"); + } + + static void playPingPong(ChronicleMap chm, double _coupon, double _coupon2, boolean setFirst, final String desc) throws InterruptedException { + BondVOInterface bond1 = newDirectReference(BondVOInterface.class); + BondVOInterface bond2 = newDirectReference(BondVOInterface.class); + BondVOInterface bond3 = newDirectReference(BondVOInterface.class); + BondVOInterface bond4 = newDirectReference(BondVOInterface.class); + + chm.acquireUsing("369604101", bond1); + chm.acquireUsing("369604102", bond2); + chm.acquireUsing("369604103", bond3); + chm.acquireUsing("369604104", bond4); + System.out.printf("\n\n" + desc + ": Timing 1 x off-heap operations on " + chm.file() + "\n"); + if (setFirst) { + bond1.setCoupon(_coupon); + bond2.setCoupon(_coupon); + bond3.setCoupon(_coupon); + bond4.setCoupon(_coupon); + } + int timeToCallNanoTime = 30; + int runs = 1000000; + long[] timings = new long[runs]; + for (int j = 0; j < 10; j++) { + for (int i = 0; i < runs; i++) { + long _start = System.nanoTime(); + toggleCoupon(bond1, _coupon, _coupon2); + toggleCoupon(bond2, _coupon, _coupon2); + toggleCoupon(bond3, _coupon, _coupon2); + toggleCoupon(bond4, _coupon, _coupon2); + + timings[i] = (System.nanoTime() - _start - timeToCallNanoTime) / 4; + } + Arrays.sort(timings); + System.out.printf("#%d: lock,compare,set,unlock 50/90/99%%tile was %,d / %,d / %,d%n", + j, timings[runs / 2], timings[runs * 9 / 10], timings[runs * 99 / 100]); + } + } + + private static void toggleCoupon(BondVOInterface bond, double _coupon, double _coupon2) throws InterruptedException { + for (int i = 0; ; i++) { + bond.busyLockEntry(); + try { + if (bond.getCoupon() == _coupon) { + bond.setCoupon(_coupon2); + return; + } + if (i > 1000) + Thread.yield(); + } finally { + bond.unlockEntry(); + } + } + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockRight.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockRight.java new file mode 100644 index 000000000..8faccb20e --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongLockRight.java @@ -0,0 +1,30 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.fromdocs.pingpong_latency; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.fromdocs.BondVOInterface; + +import java.io.IOException; + +public class PingPongLockRight { + public static void main(String... ignored) throws IOException, InterruptedException { + ChronicleMap chm = PingPongCASLeft.acquireCHM(); + + PingPongLockLeft.playPingPong(chm, 5, 4, false, "PingPongLockRIGHT"); + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/ipc/StateMachineData.java b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineData.java new file mode 100644 index 000000000..ab91c933a --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineData.java @@ -0,0 +1,173 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.ipc; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.model.Byteable; + +/** + * + */ +public class StateMachineData implements Byteable { + private Bytes bytes; + private long offset; + + /** + * c-tor + */ + public StateMachineData() { + this.bytes = null; + this.offset = -1; + } + + // ************************************************************************* + // + // ************************************************************************* + + /** + * @param states + * @return + */ + public boolean stateIn(StateMachineState... states) { + StateMachineState currentState = getState(); + for (StateMachineState state : states) { + if (state == currentState) { + return true; + } + } + + return false; + } + + /** + * @param from + * @param to + */ + public boolean setState(StateMachineState from, StateMachineState to) { + if (this.bytes == null) throw new NullPointerException("Byteable is not set to off heap"); + + return this.bytes.compareAndSwapInt(this.offset, from.value(), to.value()); + } + + /** + * @return + */ + public StateMachineState getState() { + if (this.bytes == null) throw new NullPointerException("Byteable is not set to off heap"); + + int value = this.bytes.readVolatileInt(this.offset); + return StateMachineState.fromValue(value); + } + + /** + * @param state + */ + public void setState(StateMachineState state) { + if (this.bytes == null) throw new NullPointerException("Byteable is not set to off heap"); + + this.bytes.writeInt(this.offset, state.value()); + } + + /** + * Wait for a state and make a transition. + * It spins initially (1000 iterations), then uses a Thread.yield() . + * + * @param from + * @param to + */ + public void waitForState(StateMachineState from, StateMachineState to) { + if (this.bytes == null) throw new NullPointerException("Byteable is not set to off heap"); + + // spin + for (int i = 0; !setState(from, to); i++) { + if (i > 1000) { + Thread.yield(); // back off a little. + } + } + } + + // ************************************************************************* + // + // ************************************************************************* + + /** + * @return + */ + public int getStateData() { + if (this.bytes != null) { + return this.bytes.readVolatileInt(this.offset + 4); + } + + return -1; + } + + /** + * @param data + */ + public void setStateData(int data) { + if (this.bytes != null) { + this.bytes.writeInt(this.offset + 4, data); + } + } + + /** + * @return + */ + public int incStateData() { + if (this.bytes != null) { + return this.bytes.addInt(this.offset + 4, 1); + } + + return -1; + } + + /** + * @return + */ + public boolean done() { + if (this.bytes != null) { + return getStateData() > 100; + } + + return true; + } + + // ************************************************************************* + // + // ************************************************************************* + + @Override + public void bytes(Bytes bytes, long offset) { + this.bytes = bytes; + this.offset = offset; + } + + @Override + public Bytes bytes() { + return this.bytes; + } + + @Override + public long offset() { + return this.offset; + } + + @Override + public int maxSize() { + return 16; + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ipc/StateMachineProcessor.java b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineProcessor.java new file mode 100644 index 000000000..8c4d83c2b --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineProcessor.java @@ -0,0 +1,81 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.ipc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class StateMachineProcessor implements Runnable { + private final StateMachineData smd; + private final StateMachineState from; + private final StateMachineState transition; + private final StateMachineState to; + private final Logger logger; + + /** + * @param smd + * @param from + * @param to + */ + public StateMachineProcessor(final StateMachineData smd, StateMachineState from, StateMachineState transition, StateMachineState to) { + this.smd = smd; + this.from = from; + this.transition = transition; + this.to = to; + + this.logger = LoggerFactory.getLogger(from + " => " + transition + " => " + to); + } + + /** + * @param smd + * @param from + * @param transition + * @param to + */ + public static void runProcessor(final StateMachineData smd, StateMachineState from, StateMachineState transition, StateMachineState to) { + new StateMachineProcessor(smd, from, transition, to).run(); + } + + @Override + public void run() { + while (!smd.done()) { + if (smd.stateIn(transition)) { + doProcess(); + } + + logger.info("Wait for {}", from); + smd.waitForState(from, transition); + + doProcess(); + } + } + + private void doProcess() { + smd.incStateData(); + + logger.info("Status {}, Next {}, Data {}", + from, + to, + smd.getStateData() + ); + + smd.setState(transition, to); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ipc/StateMachineState.java b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineState.java new file mode 100644 index 000000000..859a317ae --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineState.java @@ -0,0 +1,60 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.ipc; + +/** + * + */ +public enum StateMachineState { + UNKNOWN(-1), + STATE_0(0), + STATE_0_WORKING(1), + STATE_1(10), + STATE_1_WORKING(11), + STATE_2(20), + STATE_2_WORKING(21), + STATE_3(30), + STATE_3_WORKING(31); + + private int state; + + /** + * c-tor + * + * @param state + */ + StateMachineState(int state) { + this.state = state; + } + + public static StateMachineState fromValue(int value) { + for (StateMachineState sms : StateMachineState.values()) { + if (sms.value() == value) { + return sms; + } + } + + return StateMachineState.UNKNOWN; + } + + /** + * @return + */ + public int value() { + return this.state; + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java new file mode 100644 index 000000000..00196986a --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ipc/StateMachineTutorial.java @@ -0,0 +1,97 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.ipc; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +/** + * + */ +public class StateMachineTutorial { + private static final Logger LOGGER = LoggerFactory.getLogger(StateMachineTutorial.class); + + // ************************************************************************* + // + // ************************************************************************* + + public static void main(String[] args) throws Exception { + ChronicleMap map = null; + + try { + File dataFile = new File(System.getProperty("java.io.tmpdir"), "hft-state-machine"); + + map = ChronicleMapBuilder.of(Integer.class, StateMachineData.class) + + .entries(8) + .create(dataFile); + + if (args.length > 0) { + if ("0".equalsIgnoreCase(args[0])) { + StateMachineData smd = + map.acquireUsing(0, new StateMachineData()); + + StateMachineState st = smd.getState(); + if (st == StateMachineState.STATE_0) { + long start = System.nanoTime(); + + //fire the first state change + smd.setStateData(0); + smd.setState(StateMachineState.STATE_0, StateMachineState.STATE_1); + + while (!smd.done()) { + // busy wait + } + + long end = System.nanoTime(); + + LOGGER.info("Took {} us for 100 transiction", (end - start) / 1E3); + } + } else if ("1".equalsIgnoreCase(args[0])) { + StateMachineProcessor.runProcessor( + map.acquireUsing(0, new StateMachineData()), + StateMachineState.STATE_1, + StateMachineState.STATE_1_WORKING, + StateMachineState.STATE_2); + } else if ("2".equalsIgnoreCase(args[0])) { + StateMachineProcessor.runProcessor( + map.acquireUsing(0, new StateMachineData()), + StateMachineState.STATE_2, + StateMachineState.STATE_2_WORKING, + StateMachineState.STATE_3); + } else if ("3".equalsIgnoreCase(args[0])) { + StateMachineProcessor.runProcessor( + map.acquireUsing(0, new StateMachineData()), + StateMachineState.STATE_3, + StateMachineState.STATE_3_WORKING, + StateMachineState.STATE_1); + } else if ("clean".equalsIgnoreCase(args[0])) { + LOGGER.info("deleting {}", dataFile.getAbsolutePath()); + dataFile.delete(); + } + } + } finally { + if (map != null) { + map.close(); + } + } + } +} diff --git a/src/test/java/net/openhft/collections/jrs166/JSR166TestCase.java b/src/test/java/net/openhft/chronicle/map/jrs166/JSR166TestCase.java similarity index 99% rename from src/test/java/net/openhft/collections/jrs166/JSR166TestCase.java rename to src/test/java/net/openhft/chronicle/map/jrs166/JSR166TestCase.java index c808e5f8b..5856e652b 100644 --- a/src/test/java/net/openhft/collections/jrs166/JSR166TestCase.java +++ b/src/test/java/net/openhft/chronicle/map/jrs166/JSR166TestCase.java @@ -1,14 +1,12 @@ /* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

+ * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +14,7 @@ * limitations under the License. */ -package net.openhft.collections.jrs166; +package net.openhft.chronicle.map.jrs166; import junit.framework.AssertionFailedError; import org.junit.After; @@ -800,7 +798,7 @@ public class JSR166TestCase { * A security policy where new permissions can be dynamically added * or all cleared. */ - public static class AdjustablePolicy extends Policy { + public static class AdjustablePolicy extends java.security.Policy { Permissions perms = new Permissions(); AdjustablePolicy(Permission... permissions) { diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java new file mode 100644 index 000000000..46242e233 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java @@ -0,0 +1,863 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.jrs166.map; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.*; + +/* + * Written by Doug Lea with assistance from members of JCP JSR-166 + * Expert Group and released to the public domain, as explained at + * http://creativecommons.org/publicdomain/zero/1.0/ + * Other contributors include Andrew Wright, Jeffrey Hayes, + * Pat Fisher, Mike Judd. + */ + + +public class ChronicleMapTest extends JSR166TestCase { + + + private static File getPersistenceFile() { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/chm-test" + System.nanoTime()); + file.deleteOnExit(); + return file; + } + + static ChronicleMap newShmIntString(int size) throws IOException { + + return ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(size) + .create(getPersistenceFile()); + + } + + static ChronicleMap newShmListBoolean(int size) throws IOException { + + return ChronicleMapBuilder.of(ArrayList.class, CharSequence.class) + + .entries(size).create(getPersistenceFile()); + + } + + + static ChronicleMap newShmListBoolean() throws IOException { + + return ChronicleMapBuilder.of(ArrayList.class, CharSequence.class) + + .create(getPersistenceFile()); + + } + + static ChronicleMap newShmStringString(int size) throws IOException { + + return ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + + .entries(size).create(getPersistenceFile()); + + } + + + static ChronicleMap newShmIntString() throws IOException { + + return ChronicleMapBuilder.of(Integer.class, CharSequence.class) + + .create(getPersistenceFile()); + + } + + static ChronicleMap newShmBiBoolean() throws IOException { + + return ChronicleMapBuilder.of(BI.class, Boolean.class).create(getPersistenceFile()); + + } + + /** + * Returns a new map from Integers 1-5 to Strings "A"-"E". + */ + private static ChronicleMap map5() throws IOException { + ChronicleMap map = newShmIntString(5); + assertTrue(map.isEmpty()); + map.put(one, "A"); + map.put(two, "B"); + map.put(three, "C"); + map.put(four, "D"); + map.put(five, "E"); + assertFalse(map.isEmpty()); + assertEquals(5, map.size()); + return map; + } + + /** + * clear removes all pairs + */ + @Test + public void testClear() throws IOException { + ChronicleMap map = map5(); + map.clear(); + assertEquals(0, map.size()); + } + + /** + * Maps with same contents are equal + */ + @Test + public void testEquals() throws IOException { + ChronicleMap map1 = map5(); + ChronicleMap map2 = map5(); + assertEquals(map1, map2); + assertEquals(map2, map1); + map1.clear(); + assertFalse(map1.equals(map2)); + assertFalse(map2.equals(map1)); + } + + /** + * contains returns true for contained value + */ + @Test + public void testContains() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsValue("A")); + assertFalse(map.containsValue("Z")); + } + + /** + * containsKey returns true for contained key + */ + @Test + public void testContainsKey() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsKey(one)); + assertFalse(map.containsKey(zero)); + } + + /** + * containsValue returns true for held values + */ + @Test + public void testContainsValue() throws IOException { + ChronicleMap map = map5(); + assertTrue(map.containsValue("A")); + assertFalse(map.containsValue("Z")); + } + + + /** + * Inserted elements that are subclasses of the same Comparable + * class are found. + */ + /* @Test public void testComparableFamily() throws IOException { + ChronicleMap m = + newShmBiBoolean(); + for (int i = 0; i < 1000; i++) { + assertTrue(m.put(new CI(i), true) == null); + } + for (int i = 0; i < 1000; i++) { + assertTrue(m.containsKey(new CI(i))); + assertTrue(m.containsKey(new DI(i))); + } + }*/ + + /** + * TODO : fix test + * Elements of classes with erased generic type parameters based + * on Comparable can be inserted and found. + */ + /* @Test public void testGenericComparable() throws IOException { + ChronicleMap m = + newShmBiBoolean(); + for (int i = 0; i < 1000; i++) { + BI bi = new BI(i); + BS bs = new BS(String.valueOf(i)); + LexicographicList bis = new LexicographicList(bi); + LexicographicList bss = new LexicographicList(bs); + assertTrue(m.putIfAbsent(bis, true) == null); + assertTrue(m.containsKey(bis)); + if (m.putIfAbsent(bss, true) == null) + assertTrue(m.containsKey(bss)); + assertTrue(m.containsKey(bis)); + } + for (int i = 0; i < 1000; i++) { + assertTrue(m.containsKey(new ArrayList(Collections.singleton(new BI(i))))); + } + } +*/ + + /** + * Elements of non-comparable classes equal to those of classes + * with erased generic type parameters based on Comparable can be + * inserted and found. + */ + /* @Test public void testGenericComparable2() throws IOException { + ChronicleMap m = + newShmListBoolean(); + for (int i = 0; i < 1000; i++) { + m.put(new ArrayList(Collections.singleton(new BI(i))), true); + } + + for (int i = 0; i < 1000; i++) { + LexicographicList bis = new LexicographicList(new BI(i)); + assertTrue(m.containsKey(bis)); + } + } +*/ + + /** + * get returns the correct element at the given key, + * or null if not present + */ + @Test + public void testGet() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", (String) map.get(one)); + ChronicleMap empty = newShmIntString(); + assertNull(map.get(notPresent)); + } + + /** + * isEmpty is true of empty map and false for non-empty + */ + @Test + public void testIsEmpty() throws IOException { + ChronicleMap empty = newShmIntString(); + ChronicleMap map = map5(); + assertTrue(empty.isEmpty()); + assertFalse(map.isEmpty()); + } + + /** + * keySet returns a Set containing all the keys + */ + @Test + public void testKeySet() throws IOException { + ChronicleMap map = map5(); + Set s = map.keySet(); + assertEquals(5, s.size()); + assertTrue(s.contains(one)); + assertTrue(s.contains(two)); + assertTrue(s.contains(three)); + assertTrue(s.contains(four)); + assertTrue(s.contains(five)); + } + + /** + * keySet.toArray returns contains all keys + */ + @Test + public void testKeySetToArray() throws IOException { + ChronicleMap map = map5(); + Set s = map.keySet(); + Object[] ar = s.toArray(); + assertTrue(s.containsAll(Arrays.asList(ar))); + assertEquals(5, ar.length); + ar[0] = m10; + assertFalse(s.containsAll(Arrays.asList(ar))); + } + + /** + * Values.toArray contains all values + */ + @Test + public void testValuesToArray() throws IOException { + ChronicleMap map = map5(); + Collection v = map.values(); + Object[] ar = v.toArray(); + ArrayList s = new ArrayList(Arrays.asList(ar)); + assertEquals(5, ar.length); + assertTrue(s.contains("A")); + assertTrue(s.contains("B")); + assertTrue(s.contains("C")); + assertTrue(s.contains("D")); + assertTrue(s.contains("E")); + } + + /** + * TODO : enumeration returns an enumeration containing the correct + * elements + */ + /* @Test public void testEnumeration() throws IOException { + ChronicleMap map = map5(); + Enumeration e = map.elements(); + int count = 0; + while (e.hasMoreElements()) { + count++; + e.nextElement(); + } + assertEquals(5, count); + }*/ + + /** + * entrySet.toArray contains all entries + */ + @Test + public void testEntrySetToArray() throws IOException { + ChronicleMap map = map5(); + Set s = map.entrySet(); + Object[] ar = s.toArray(); + assertEquals(5, ar.length); + for (int i = 0; i < 5; ++i) { + assertTrue(map.containsKey(((Map.Entry) (ar[i])).getKey())); + assertTrue(map.containsValue(((Map.Entry) (ar[i])).getValue())); + } + } + + /** + * values collection contains all values + */ + @Test + public void testValues() throws IOException { + ChronicleMap map = map5(); + Collection s = map.values(); + assertEquals(5, s.size()); + assertTrue(s.contains("A")); + assertTrue(s.contains("B")); + assertTrue(s.contains("C")); + assertTrue(s.contains("D")); + assertTrue(s.contains("E")); + } + + /** + * TODO : keys returns an enumeration containing all the keys from the map + */ + /* @Test public void testKeys() throws IOException { + ChronicleMap map = map5(); + Enumeration e = map.keys(); + int count = 0; + while (e.hasMoreElements()) { + count++; + e.nextElement(); + } + assertEquals(5, count); + }*/ + + /** + * entrySet contains all pairs + */ + @Test + public void testEntrySet() throws IOException { + ChronicleMap map = map5(); + Set s = map.entrySet(); + assertEquals(5, s.size()); + Iterator it = s.iterator(); + while (it.hasNext()) { + Map.Entry e = (Map.Entry) it.next(); + assertTrue( + (e.getKey().equals(one) && e.getValue().equals("A")) || + (e.getKey().equals(two) && e.getValue().equals("B")) || + (e.getKey().equals(three) && e.getValue().equals("C")) || + (e.getKey().equals(four) && e.getValue().equals("D")) || + (e.getKey().equals(five) && e.getValue().equals("E")) + ); + } + } + + /** + * putAll adds all key-value pairs from the given map + */ + @Test + public void testPutAll() throws IOException { + ChronicleMap empty = newShmIntString(); + ChronicleMap map = map5(); + empty.putAll(map); + assertEquals(5, empty.size()); + assertTrue(empty.containsKey(one)); + assertTrue(empty.containsKey(two)); + assertTrue(empty.containsKey(three)); + assertTrue(empty.containsKey(four)); + assertTrue(empty.containsKey(five)); + } + + /** + * putIfAbsent works when the given key is not present + */ + @Test + public void testPutIfAbsent() throws IOException { + ChronicleMap map = map5(); + map.putIfAbsent(six, "Z"); + assertTrue(map.containsKey(six)); + } + + /** + * putIfAbsent does not add the pair if the key is already present + */ + @Test + public void testPutIfAbsent2() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", map.putIfAbsent(one, "Z")); + } + + /** + * replace fails when the given key is not present + */ + @Test + public void testReplace() throws IOException { + ChronicleMap map = map5(); + assertNull(map.replace(six, "Z")); + assertFalse(map.containsKey(six)); + } + + /** + * replace succeeds if the key is already present + */ + @Test + public void testReplace2() throws IOException { + ChronicleMap map = map5(); + assertNotNull(map.replace(one, "Z")); + assertEquals("Z", map.get(one)); + } + + /** + * replace value fails when the given key not mapped to expected value + */ + @Test + public void testReplaceValue() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", map.get(one)); + assertFalse(map.replace(one, "Z", "Z")); + assertEquals("A", map.get(one)); + } + + /** + * replace value succeeds when the given key mapped to expected value + */ + @Test + public void testReplaceValue2() throws IOException { + ChronicleMap map = map5(); + assertEquals("A", map.get(one)); + assertTrue(map.replace(one, "A", "Z")); + assertEquals("Z", map.get(one)); + } + + /** + * remove removes the correct key-value pair from the map + */ + @Test + public void testRemove() throws IOException { + ChronicleMap map = map5(); + map.remove(five); + assertEquals(4, map.size()); + assertFalse(map.containsKey(five)); + } + + /** + * remove(key,value) removes only if pair present + */ + @Test + public void testRemove2() throws IOException { + ChronicleMap map = map5(); + map.remove(five, "E"); + assertEquals(4, map.size()); + assertFalse(map.containsKey(five)); + map.remove(four, "A"); + assertEquals(4, map.size()); + assertTrue(map.containsKey(four)); + } + + /** + * size returns the correct values + */ + @Test + public void testSize() throws IOException { + ChronicleMap map = map5(); + ChronicleMap empty = newShmIntString(); + assertEquals(0, empty.size()); + assertEquals(5, map.size()); + } + + /** + * toString contains toString of elements + */ + @Test + public void testToString() throws IOException { + ChronicleMap map = map5(); + String s = map.toString(); + for (int i = 1; i <= 5; ++i) { + assertTrue(s.contains(String.valueOf(i))); + } + } + + /** + * get(null) throws NPE + */ + @Test + public void testGet_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.get(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * containsKey(null) throws NPE + */ + @Test + public void testContainsKey_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.containsKey(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * put(null,x) throws NPE + */ + @Test + public void testPut1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.put(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * put(x, null) throws NPE + */ + @Test + public void testPut2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.put(notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * putIfAbsent(null, x) throws NPE + */ + @Test + public void testPutIfAbsent1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.putIfAbsent(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + // Exception tests + + /** + * TODO : Cannot create with negative capacity + */ + /* @Test public void testConstructor1() { + try { + newShmIntString(-1, 0, 1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * TODO : Cannot create with negative concurrency level + */ + /*@Test public void testConstructor2() { + try { + newShmIntString(1, 0, -1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * TODO :Cannot create with only negative capacity + */ + /* @Test public void testConstructor3() { + try { + newShmIntString(-1); + shouldThrow(); + } catch (IllegalArgumentException success) { + } + }*/ + + /** + * replace(null, x) throws NPE + */ + @Test + public void testReplace_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(null, x, y) throws NPE + */ + @Test + public void testReplaceValue_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(null, "A", "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * containsValue(null) throws NPE + */ +/* @Test public void testContainsValue_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.containsValue(null); + shouldThrow(); + } catch (NullPointerException success) { + } + }*/ + + /** + * todo : contains(null) throws NPE + */ +/* + @Test public void testContains_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.contains(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } +*/ + + /** + * putIfAbsent(x, null) throws NPE + */ + @Test + public void testPutIfAbsent2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.putIfAbsent(notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, null) throws NPE + */ + @Test + public void testReplace2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(notPresent, null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, null, y) throws NPE + */ + @Test + public void testReplaceValue2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(notPresent, null, "A"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * replace(x, y, null) throws NPE + */ + @Test + public void testReplaceValue3_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmIntString(5); + c.replace(notPresent, "A", null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(null) throws NPE + */ + @Test + public void testRemove1_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + c.remove(null); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(null, x) throws NPE + */ + @Test + public void testRemove2_NullPointerException() throws IOException { + try { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + c.remove(null, "whatever"); + shouldThrow(); + } catch (NullPointerException success) { + } + } + + /** + * remove(x, null) returns false + */ + @Test + public void testRemove3() throws IOException { + ChronicleMap c = newShmStringString(5); + c.put("sadsdf", "asdads"); + assertFalse(c.remove("sadsdf", null)); + } + + // classes for testing Comparable fallbacks + static class BI implements Comparable { + private final int value; + + BI(int value) { + this.value = value; + } + + public int compareTo(BI other) { + return Integer.compare(value, other.value); + } + + public boolean equals(Object x) { + return (x instanceof BI) && ((BI) x).value == value; + } + + public int hashCode() { + return 42; + } + } + + static class CI extends BI { + CI(int value) { + super(value); + } + } + + static class DI extends BI { + DI(int value) { + super(value); + } + } + + static class BS implements Comparable { + private final String value; + + BS(String value) { + this.value = value; + } + + public int compareTo(BS other) { + return value.compareTo(other.value); + } + + public boolean equals(Object x) { + return (x instanceof BS) && value.equals(((BS) x).value); + } + + public int hashCode() { + return 42; + } + } + + static class LexicographicList> extends ArrayList + implements Comparable> { + private static final long serialVersionUID = 0; + static long total; + static long n; + + LexicographicList(Collection c) { + super(c); + } + + LexicographicList(E e) { + super(Collections.singleton(e)); + } + + public int compareTo(LexicographicList other) { + long start = System.currentTimeMillis(); + int common = Math.min(size(), other.size()); + int r = 0; + for (int i = 0; i < common; i++) { + if ((r = get(i).compareTo(other.get(i))) != 0) + break; + } + if (r == 0) + r = Integer.compare(size(), other.size()); + total += System.currentTimeMillis() - start; + n++; + return r; + } + } + + /** + * A deserialized map equals original + */ + /* @Test public void testSerialization() throws Exception { + Map x = map5(); + Map y = serialClone(x); + + assertNotSame(x, y); + assertEquals(x.size(), y.size()); + assertEquals(x, y); + assertEquals(y, x); + }*/ + + /** + * TODO : SetValue of an EntrySet entry sets value in the map. + */ + /*@Test public void testSetValueWriteThrough() { + // Adapted from a bug report by Eric Zoerner + ChronicleMap map = newShmIntString(2, 5.0f, 1); + assertTrue(map.isEmpty()); + for (int i = 0; i < 20; i++) + map.put(new Integer(i), new Integer(i)); + assertFalse(map.isEmpty()); + Map.Entry entry1 = (Map.Entry) map.entrySet().iterator().next(); + // Unless it happens to be first (in which case remainder of + // test is skipped), remove a possibly-colliding key from map + // which, under some implementations, may cause entry1 to be + // cloned in map + if (!entry1.getKey().equals(new Integer(16))) { + map.remove(new Integer(16)); + entry1.setValue("XYZ"); + assertTrue(map.containsValue("XYZ")); // fails if write-through broken + } + }*/ + +} + diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/LoopHelpers.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/LoopHelpers.java new file mode 100644 index 000000000..f3dd5efcc --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/LoopHelpers.java @@ -0,0 +1,234 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.jrs166.map; + +/* + * Written by Doug Lea with assistance from members of JCP JSR-166 + * Expert Group and released to the public domain, as explained at + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Misc utilities in JSR166 performance tests + */ +class LoopHelpers { + + static final SimpleRandom staticRNG = new SimpleRandom(); + + // Some mindless computation to do between synchronizations... + + /** + * generates 32 bit pseudo-random numbers. + * Adapted from http://www.snippets.org + */ + public static int compute1(int x) { + int lo = 16807 * (x & 0xFFFF); + int hi = 16807 * (x >>> 16); + lo += (hi & 0x7FFF) << 16; + if ((lo & 0x80000000) != 0) { + lo &= 0x7fffffff; + ++lo; + } + lo += hi >>> 15; + if (lo == 0 || (lo & 0x80000000) != 0) { + lo &= 0x7fffffff; + ++lo; + } + return lo; + } + + /** + * Computes a linear congruential random number a random number + * of times. + */ + public static int compute2(int x) { + int loops = (x >>> 4) & 7; + while (loops-- > 0) { + x = (x * 2147483647) % 16807; + } + return x; + } + + /** + * Yet another random number generator + */ + public static int compute3(int x) { + int t = (x % 127773) * 16807 - (x / 127773) * 2836; + return (t > 0) ? t : t + 0x7fffffff; + } + + /** + * Yet another random number generator + */ + public static int compute4(int x) { + return x * 134775813 + 1; + } + + + /** + * Yet another random number generator + */ + public static int compute5(int x) { + return 36969 * (x & 65535) + (x >> 16); + } + + /** + * Marsaglia xorshift (1, 3, 10) + */ + public static int compute6(int seed) { + seed ^= seed << 1; + seed ^= seed >>> 3; + seed ^= (seed << 10); + return seed; + } + + /** + * Marsaglia xorshift (6, 21, 7) + */ + public static int compute7(int y) { + y ^= y << 6; + y ^= y >>> 21; + y ^= (y << 7); + return y; + } + + // FNV: (x ^ 0x811c9dc5) * 0x01000193; 15485863; + + /** + * Marsaglia xorshift for longs + */ + public static long compute8(long x) { + x ^= x << 13; + x ^= x >>> 7; + x ^= (x << 17); + return x; + } + + public static String rightJustify(long n) { + // There's probably a better way to do this... + String field = " "; + String num = Long.toString(n); + if (num.length() >= field.length()) + return num; + StringBuffer b = new StringBuffer(field); + b.replace(b.length() - num.length(), b.length(), num); + return b.toString(); + } + + public static final class XorShift32Random { + static final AtomicInteger seq = new AtomicInteger(8862213); + int x = -1831433054; + + public XorShift32Random(int seed) { + x = seed; + } + + public XorShift32Random() { + this((int) System.nanoTime() + seq.getAndAdd(129)); + } + + public int next() { + x ^= x << 6; + x ^= x >>> 21; + x ^= (x << 7); + return x; + } + } + + /** + * Multiplication-free RNG from Marsaglia "Xorshift RNGs" paper + */ + public static final class MarsagliaRandom { + static final AtomicInteger seq = new AtomicInteger(3122688); + int x; + int y = 842502087; + int z = -715159705; + int w = 273326509; + + public MarsagliaRandom(int seed) { + x = seed; + } + + public MarsagliaRandom() { + this((int) System.nanoTime() + seq.getAndAdd(129)); + } + + public int next() { + int t = x ^ (x << 11); + x = y; + y = z; + z = w; + return w = (w ^ (w >>> 19) ^ (t ^ (t >>> 8))); + } + } + + /** + * Unsynchronized version of java.util.Random algorithm. + */ + public static final class SimpleRandom { + static final AtomicLong seq = new AtomicLong(-715159705); + private static final long multiplier = 0x5DEECE66DL; + private static final long addend = 0xBL; + private static final long mask = (1L << 48) - 1; + private long seed; + + SimpleRandom(long s) { + seed = s; + } + + SimpleRandom() { + seed = System.nanoTime() + seq.getAndAdd(129); + } + + public void setSeed(long s) { + seed = s; + } + + public int next() { + long nextseed = (seed * multiplier + addend) & mask; + seed = nextseed; + return ((int) (nextseed >>> 17)) & 0x7FFFFFFF; + } + } + + public static class BarrierTimer implements Runnable { + volatile boolean started; + volatile long startTime; + volatile long endTime; + + public void run() { + long t = System.nanoTime(); + if (!started) { + started = true; + startTime = t; + } else + endTime = t; + } + + public void clear() { + started = false; + } + + public long getTime() { + return endTime - startTime; + } + } + +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java new file mode 100644 index 000000000..805a3013a --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java @@ -0,0 +1,770 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.jrs166.map; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; + +import java.io.*; +import java.util.*; + +/* + * Written by Doug Lea with assistance from members of JCP JSR-166 + * Expert Group and released to the public domain, as explained at + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +/** + * @test + * @synopsis Times and checks basic map operations + *

+ * When run with "s" second arg, this requires file "testwords", which + * is best used with real words. We can't check in this file, but you + * can create one from a real dictionary (1 line per word) and then run + * linux "shuf" to randomize entries. + */ + +public class MapCheck { + static final String MISSING = "MISSING"; + static final LoopHelpers.SimpleRandom srng = new LoopHelpers.SimpleRandom(); + static final Random rng = new Random(3152688); + static TestTimer timer = new TestTimer(); + static Class eclass; + static volatile int checkSum; + static int counter = 0; + + static void reallyAssert(boolean b) { + if (!b) throw new Error("Failed Assertion"); + } + + public static void main(String[] args) throws Exception { + int numTests = 20; + int size = 36864; // about midway of HashMap resize interval + + if (args.length == 0) + System.out.println("Usage: MapCheck mapclass [int|float|string|object] [trials] [size] [serialtest]"); + + + if (args.length > 1) { + String et = args[1].toLowerCase(); + if (et.startsWith("i")) + eclass = java.lang.Integer.class; + else if (et.startsWith("f")) + eclass = java.lang.Float.class; + else if (et.startsWith("s")) + eclass = java.lang.String.class; + else if (et.startsWith("d")) + eclass = java.lang.Double.class; + } + if (eclass == null) + eclass = Integer.class; + + if (args.length > 2) + numTests = Integer.parseInt(args[2]); + + if (args.length > 3) + size = Integer.parseInt(args[3]); + + boolean doSerializeTest = args.length > 4; + + while ((size & 3) != 0) ++size; + + + System.out.print(" elements: " + eclass.getName()); + System.out.print(" trials: " + numTests); + System.out.print(" size: " + size); + System.out.println(); + + Object[] key = new Object[size]; + Object[] absent = new Object[size]; + initializeKeys(key, absent, size); + + precheck(size, key, absent); + + for (int rep = 0; rep < numTests; ++rep) { + mainTest(key, absent); + if ((rep & 3) == 3 && rep < numTests - 1) { + shuffle(key); + // Thread.sleep(10); + } + } + + TestTimer.printStats(); + + checkNullKey(); + + if (doSerializeTest) + serTest(size); + } + + private static File getPersistenceFile() { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/chm-test" + System.nanoTime()); + file.deleteOnExit(); + return file; + } + + + static Map newMap() { + try { + +// return new ConcurrentHashMap(); + return ChronicleMapBuilder.of(Object.class, Object.class) + .create(getPersistenceFile()); + + } catch (Exception e) { + throw new RuntimeException("Can't instantiate CHM : " + e); + } + } + + static void closeMap(Map map) { + if (map instanceof ChronicleMap) { + ChronicleMap chm = (ChronicleMap) map; + try { + chm.close(); + } catch (IOException e) { + throw new RuntimeException("Can't close CHM : " + e); + } + } + } + + static void precheck(int n, Object[] key, Object[] abs) { + int ck = 0; + Map s = newMap(); + for (int i = 0; i < n; i++) { + Object k = key[i]; + if (k == null) throw new Error("Null key at" + i); + ck += System.identityHashCode(k); + if (i == 409) + Thread.yield(); + Object v = s.put(k, k); + if (v != null) + throw new Error("Duplicate " + k + " / " + v); + } + for (int i = 0; i < n; i++) { + Object k = abs[i]; + if (k == null) throw new Error("Null key at" + i); + ck += System.identityHashCode(k); + Object v = s.put(k, k); + if (v != null) + throw new Error("Duplicate " + k + " / " + v); + } + checkSum += ck; + closeMap(s); + } + + static void checkNullKey() { + Map m = newMap(); + Object x = (byte) 1; + Object v; + try { + m.put(null, x); + v = m.get(null); + } catch (NullPointerException npe) { + System.out.println("Map does not allow null keys"); + return; + } catch (IllegalArgumentException npe) { + System.out.println("Map does not allow null keys"); + return; + } + if (!v.equals(x)) throw new Error(); + if (m.remove(null) != v) throw new Error(); + if (m.get(null) != null) throw new Error(); + closeMap(m); + } + + + static void getTest(String nm, int n, Map s, Object[] key, int expect) { + int sum = 0; + timer.start(nm, n); + for (int i = 0; i < n; i++) { + Object v = s.get(key[i]); + if (v != null && v.getClass() == eclass) + ++sum; + } + timer.finish(); + reallyAssert(sum == expect); + checkSum += sum; + } + + + // unused + static void getTestBoxed(String nm, int n, Map s, Object[] key, int expect) { + int sum = 0; + Map intMap = (Map) s; + timer.start(nm, n); + for (int i = 0; i < n; i++) { + if (intMap.get(i) != i) ++sum; + } + timer.finish(); + reallyAssert(sum == expect); + } + + static void remTest(String nm, int n, Map s, Object[] key, int expect) { + int sum = 0; + timer.start(nm, n); + for (int i = 0; i < n; i++) { + if (s.remove(key[i]) != null) ++sum; + } + timer.finish(); + reallyAssert(sum == expect); + checkSum += sum; + } + + static void clrTest(int n, Map s) { + String nm = "Remove Present "; + timer.start(nm, n); + s.clear(); + timer.finish(); + reallyAssert(s.isEmpty()); + } + + static void putTest(String nm, int n, Map s, Object[] key, int expect) { + int sum = 0; + timer.start(nm, n); + for (int i = 0; i < n; i++) { + Object k = key[i]; + Object v = s.put(k, k); + if (v == null) ++sum; + } + timer.finish(); + reallyAssert(sum == expect); + checkSum += sum; + } + + static void keyTest(String nm, int n, Map s, Object[] key, int expect) { + int sum = 0; + timer.start(nm, n); + for (int i = 0; i < n; i++) { + if (s.containsKey(key[i])) ++sum; + } + timer.finish(); + reallyAssert(sum == expect); + checkSum += sum; + } + + // version without timing for uncategorized tests + static void untimedKeyTest(String nm, int n, Map s, Object[] key, int expect) { + int sum = 0; + for (int i = 0; i < n; i++) { + if (s.containsKey(key[i])) ++sum; + } + reallyAssert(sum == expect); + checkSum += sum; + } + + static void remHalfTest(String nm, int n, Map s, Object[] key, int expect) { + int sum = 0; + timer.start(nm, n / 2); + for (int i = n - 2; i >= 0; i -= 2) { + if (s.remove(key[i]) != null) ++sum; + } + timer.finish(); + reallyAssert(sum == expect); + checkSum += sum; + } + + static void valTest(Map s, Object[] key) { + int size = s.size(); + int sum = 0; + timer.start("Traverse key or value ", size); + if (s.containsValue(MISSING)) ++sum; + timer.finish(); + reallyAssert(sum == 0); + checkSum += sum; + } + + + static Object kitTest(Map s, int size) { + Object last = null; + int sum = 0; + timer.start("Traverse key or value ", size); + for (Iterator it = s.keySet().iterator(); it.hasNext(); ) { + Object x = it.next(); + if (x != last && x != null && x.getClass() == eclass) + ++sum; + last = x; + } + timer.finish(); + reallyAssert(sum == size); + checkSum += sum; + return last; + } + + static Object vitTest(Map s, int size) { + Object last = null; + int sum = 0; + timer.start("Traverse key or value ", size); + for (Iterator it = s.values().iterator(); it.hasNext(); ) { + Object x = it.next(); + if (x != last && x != null && x.getClass() == eclass) + ++sum; + last = x; + } + timer.finish(); + reallyAssert(sum == size); + checkSum += sum; + return last; + } + + static void eitTest(Map s, int size) { + int sum = 0; + timer.start("Traverse entry ", size); + for (Iterator it = s.entrySet().iterator(); it.hasNext(); ) { + Map.Entry e = (Map.Entry) it.next(); + Object k = e.getKey(); + Object v = e.getValue(); + if (k != null && k.getClass() == eclass && + v != null && v.getClass() == eclass) + ++sum; + } + timer.finish(); + reallyAssert(sum == size); + checkSum += sum; + } + + static void itRemTest(Map s, int size) { + int sz = s.size(); + reallyAssert(sz == size); + timer.start("Remove Present ", size); + int sum = 0; + for (Iterator it = s.keySet().iterator(); it.hasNext(); ) { + it.next(); + it.remove(); + ++sum; + } + timer.finish(); + reallyAssert(sum == sz); + checkSum += sum; + } + + static void itHalfRemTest(Map s, int size) { + int sz = s.size(); + reallyAssert(sz == size); + timer.start("Remove Present ", size); + int sum = 0; + for (Iterator it = s.keySet().iterator(); it.hasNext(); ) { + it.next(); + it.remove(); + if (it.hasNext()) + it.next(); + ++sum; + } + timer.finish(); + reallyAssert(sum == sz / 2); + checkSum += sum; + } + + static void putAllTest(String nm, int n, Map src, Map dst) { + timer.start(nm, n); + dst.putAll(src); + timer.finish(); + reallyAssert(src.size() == dst.size()); + } + + static void serTest(int size) throws Exception { + Map s = newMap(); + if (!(s instanceof Serializable)) + return; + System.out.print("Serialize : "); + + for (int i = 0; i < size; i++) { + s.put(new Integer(i), Boolean.TRUE); + } + + long startTime = System.currentTimeMillis(); + + FileOutputStream fs = new FileOutputStream("MapCheck.dat"); + ObjectOutputStream out = new ObjectOutputStream(new BufferedOutputStream(fs)); + out.writeObject(s); + out.close(); + + FileInputStream is = new FileInputStream("MapCheck.dat"); + ObjectInputStream in = new ObjectInputStream(new BufferedInputStream(is)); + Map m = (Map) in.readObject(); + + long endTime = System.currentTimeMillis(); + long time = endTime - startTime; + + System.out.print(time + "ms"); + + if (s instanceof IdentityHashMap) return; + reallyAssert(s.equals(m)); + closeMap(s); + } + + static void mainTest(Object[] key, Object[] absent) { + Map s = newMap(); + int size = key.length; + + putTest("Add Absent ", size, s, key, size); + reallyAssert(s.size() == size); + getTest("Access Present ", size, s, key, size); + getTest("Search Absent ", size, s, absent, 0); + kitTest(s, size); + vitTest(s, size); + eitTest(s, size); + putTest("Modify Present ", size, s, key, 0); + reallyAssert(s.size() == size); + untimedKeyTest("Access Present ", size, s, key, size); + keyTest("Search Absent ", size, s, absent, 0); +// No supported: valTest(s, key); + remTest("Search Absent ", size, s, absent, 0); + reallyAssert(s.size() == size); + remHalfTest("Remove Present ", size, s, key, size / 2); + reallyAssert(s.size() == size / 2); + getTest("Access Present ", size, s, key, size / 2); + putTest("Add Absent ", size, s, key, size / 2); + reallyAssert(s.size() == size); + getTest("Access Present ", size, s, key, size); + getTest("Search Absent ", size, s, absent, 0); + itRemTest(s, size); + s.clear(); + + putTest("Add Absent ", size, s, key, size); + reallyAssert(s.size() == size); + getTest("Access Present ", size, s, key, size); + untimedKeyTest("Access Present ", size, s, key, size); + kitTest(s, size); + vitTest(s, size); + eitTest(s, size); + twoMapTest1(s, key, absent); + twoMapTest2(s, key, absent); + closeMap(s); + } + + static void twoMapTest1(Map s, Object[] key, Object[] absent) { + int size = s.size(); + Map s2 = newMap(); + putAllTest("Add Absent ", size, s, s2); + getTest("Access Present ", size, s2, key, size); + itHalfRemTest(s2, size); + reallyAssert(s2.size() == size / 2); + itHalfRemTest(s2, size / 2); + reallyAssert(s2.size() == size / 4); + putTest("Add Absent ", size, s2, absent, size); + putTest("Add Absent ", size, s2, key, size * 3 / 4); + reallyAssert(s2.size() == size * 2); + clrTest(size, s2); + closeMap(s2); + } + + static void twoMapTest2(Map s, Object[] key, Object[] absent) { + int size = key.length; + + Map s2 = newMap(); + putAllTest("Add Absent ", size, s, s2); + putAllTest("Modify Present ", size, s, s2); + + Object lastkey = kitTest(s2, size); + Object hold = s2.get(lastkey); + int sum = 0; + + timer.start("Traverse entry ", size * 12); // 12 until finish + + int sh1 = s.hashCode() - s2.hashCode(); + reallyAssert(sh1 == 0); + boolean eq1 = s2.equals(s); + boolean eq2 = s.equals(s2); + reallyAssert(eq1 && eq2); + + Set es2 = s2.entrySet(); + for (Iterator it = s.entrySet().iterator(); it.hasNext(); ) { + Object entry = it.next(); + if (es2.contains(entry)) ++sum; + } + reallyAssert(sum == size); + + s2.put(lastkey, MISSING); + + int sh2 = s.hashCode() - s2.hashCode(); + reallyAssert(sh2 != 0); + + eq1 = s2.equals(s); + eq2 = s.equals(s2); + reallyAssert(!eq1 && !eq2); + + sum = 0; + for (Iterator it = s.entrySet().iterator(); it.hasNext(); ) { + Map.Entry e = (Map.Entry) it.next(); + e.setValue(absent[sum++]); + } + reallyAssert(sum == size); + for (Iterator it = s2.entrySet().iterator(); it.hasNext(); ) { + Map.Entry e = (Map.Entry) it.next(); + e.setValue(s.get(e.getKey())); + } + + timer.finish(); + + int rmiss = 0; + timer.start("Remove Present ", size * 2); + Iterator s2i = s2.entrySet().iterator(); + Set es = s.entrySet(); + while (s2i.hasNext()) { + if (!es.remove(s2i.next())) + ++rmiss; + } + timer.finish(); + reallyAssert(rmiss == 0); + + clrTest(size, s2); + reallyAssert(s2.isEmpty() && s.isEmpty()); + closeMap(s2); + } + + static void itTest4(Map s, int size, int pos) { + IdentityHashMap seen = new IdentityHashMap(size); + reallyAssert(s.size() == size); + int sum = 0; + timer.start("Iter XEntry ", size); + Iterator it = s.entrySet().iterator(); + Object k = null; + Object v = null; + for (int i = 0; i < size - pos; ++i) { + Map.Entry x = (Map.Entry) (it.next()); + k = x.getKey(); + v = x.getValue(); + seen.put(k, k); + if (!x.equals(MISSING)) + ++sum; + } + reallyAssert(s.containsKey(k)); + it.remove(); + reallyAssert(!s.containsKey(k)); + while (it.hasNext()) { + Map.Entry x = (Map.Entry) (it.next()); + Object k2 = x.getKey(); + seen.put(k2, k2); + if (!x.equals(MISSING)) + ++sum; + } + + reallyAssert(s.size() == size - 1); + s.put(k, v); + reallyAssert(seen.size() == size); + timer.finish(); + reallyAssert(sum == size); + reallyAssert(s.size() == size); + } + + + static void initializeKeys(Object[] key, Object[] absent, int size) { + // Object cannot be used as it cannot be serialized. + if (eclass == Object.class || eclass == Integer.class) { + initInts(key, absent, size); + } else if (eclass == Float.class) { + initFloats(key, absent, size); + } else if (eclass == Double.class) { + initDoubles(key, absent, size); + } else if (eclass == String.class) { + initWords(size, key, absent); + } else + throw new Error("unknown type"); + } + + static void initInts(Object[] key, Object[] absent, int size) { + for (int i = 0; i < size; ++i) + key[i] = Integer.valueOf(i); + Map m = newMap(); + int k = 0; + while (k < size) { + int r = srng.next(); + if (r < 0 || r >= size) { + Integer ir = Integer.valueOf(r); + if (m.put(ir, ir) == null) + absent[k++] = ir; + } + } + closeMap(m); + } + + static void initFloats(Object[] key, Object[] absent, int size) { + Map m = newMap(); + for (int i = 0; i < size; ++i) { + float r = Float.valueOf(i); + key[i] = r; + m.put(r, r); + } + int k = 0; + while (k < size) { + float r = rng.nextFloat(); + Float ir = Float.valueOf(r); + if (m.put(ir, ir) == null) + absent[k++] = ir; + } + closeMap(m); + } + + static void initDoubles(Object[] key, Object[] absent, int size) { + Map m = newMap(); + for (int i = 0; i < size; ++i) { + double r = Double.valueOf(i); + key[i] = r; + m.put(r, r); + } + int k = 0; + while (k < size) { + double r = rng.nextDouble(); + Double ir = Double.valueOf(r); + if (m.put(ir, ir) == null) + absent[k++] = ir; + } + closeMap(m); + } + + // Use as many real words as possible, then use fake random words + + static void initWords(int size, Object[] key, Object[] abs) { + String fileName = "testwords.txt"; + int ki = 0; + int ai = 0; + try { + FileInputStream fr = new FileInputStream(fileName); + BufferedInputStream in = new BufferedInputStream(fr); + while (ki < size || ai < size) { + StringBuffer sb = new StringBuffer(); + for (; ; ) { + int c = in.read(); + if (c < 0) { + if (ki < size) + randomWords(key, ki, size); + if (ai < size) + randomWords(abs, ai, size); + in.close(); + return; + } + if (c == '\n') { + String s = sb.toString(); + if (ki < size) + key[ki++] = s; + else + abs[ai++] = s; + break; + } + sb.append((char) c); + } + } + in.close(); + } catch (IOException ex) { + System.out.println("Can't read words file:" + ex); + throw new Error(ex); + } + } + + + static void randomWords(Object[] ws, int origin, int size) { + for (int i = origin; i < size; ++i) { + int k = 0; + int len = 2 + (srng.next() & 0xf); + char[] c = new char[len * 4 + 1]; + for (int j = 1; j < len; ++j) { + int r = srng.next(); + c[k++] = (char) (' ' + (r & 0x7f)); + r >>>= 8; + c[k++] = (char) (' ' + (r & 0x7f)); + r >>>= 8; + c[k++] = (char) (' ' + (r & 0x7f)); + r >>>= 8; + c[k++] = (char) (' ' + (r & 0x7f)); + } + c[k++] = (char) ((i & 31) | 1); // never == to any testword + ws[i] = new String(c); + } + } + + static void shuffle(Object[] keys) { + int size = keys.length; + for (int i = size; i > 1; i--) { + int r = rng.nextInt(i); + Object t = keys[i - 1]; + keys[i - 1] = keys[r]; + keys[r] = t; + } + } + + static void shuffle(ArrayList keys) { + int size = keys.size(); + for (int i = size; i > 1; i--) { + int r = rng.nextInt(i); + Object t = keys.get(i - 1); + keys.set(i - 1, keys.get(r)); + keys.set(r, t); + } + } + + static final class TestTimer { + static final java.util.TreeMap accum = new java.util.TreeMap(); + private String name; + private long numOps; + private long startTime; + + static void printStats() { + for (Iterator it = accum.entrySet().iterator(); it.hasNext(); ) { + Map.Entry e = (Map.Entry) it.next(); + Stats stats = (Stats) e.getValue(); + System.out.print(e.getKey() + ": "); + long s; + long n = stats.number; + if (n == 0) { + n = stats.firstn; + s = stats.first; + } else + s = stats.sum; + + double t = ((double) s) / n; + long nano = Math.round(t); + System.out.printf("%6d", +nano); + System.out.println(); + } + } + + void start(String name, long numOps) { + this.name = name; + this.numOps = numOps; + startTime = System.nanoTime(); + } + + void finish() { + long elapsed = System.nanoTime() - startTime; + Object st = accum.get(name); + if (st == null) + accum.put(name, new Stats(elapsed, numOps)); + else + ((Stats) st).addTime(elapsed, numOps); + } + + } + + static final class Stats { + long sum; + long number; + long first; + long firstn; + + Stats(long t, long n) { + first = t; + firstn = n; + } + + void addTime(long t, long n) { + sum += t; + number += n; + } + } + +} diff --git a/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java b/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java new file mode 100644 index 000000000..7053d4492 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java @@ -0,0 +1,596 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map.utility; + +import net.openhft.affinity.AffinitySupport; +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleMapBuilder; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.model.constraints.MaxSize; + +import java.io.File; +import java.io.IOException; +import java.util.Date; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * ProcessInstanceLimiter limits the number of JVM processes of a particular + * type that can be started on a particular machine. It does this be using a + * shared map (using ChronicleMap) to maintain shared data across any processes + * which use a ProcessInstanceLimiter, and checking on startup and regularly + * after whether it is allowed to run. + *

+ * Typically, you need to specify two things to create an instance of + * ProcessInstanceLimiter: a path to a file that will hold the shared map; and a + * callback object (an instance implementing ProcessInstanceLimiter.Callback) to + * handle the various possible callback messages that the ProcessInstanceLimiter + * can generate. + *

+ * Once you have a ProcessInstanceLimiter instance, you specify a type of + * process (any string) which will be limited to up to N processes running at + * the same time by calling the setMaxNumberOfProcessesOfType() method. Finally + * you tell the instance you are starting your process of type X by calling + * startingProcessOfType(X). This last is deliberately not done automatically as + * you may wish for one type of process to define limitations on other types of + * processes. + *

+ * The are some convenience methods which allow you to quickly specify a limit + * without consideration of the above. For example, if during your application + * startup you call ProcessInstanceLimiter.limitTo(2), then you need not call + * anything else and you have limited your application to running at most 2 JVM + * instances of your application. Under the covers, this call is identical to + * the sequence: + *

+ * ProcessInstanceLimiter limiter = new ProcessInstanceLimiter(); + * limiter.setMaxNumberOfProcessesOfType(processType,numProcesses); + * limiter.startingProcessOfType(processType); + *

+ * This: + * 1. Creates a shared file called ProcessInstanceLimiter_DEFAULT_SHARED_MAP_ in + * the temp directory to hold an instance of ChronicleMap + * 2. Creates an instance of ProcessInstanceLimiter.DefaultCallback to handle + * all callbacks in a reasonable way - all callbacks will emit a message on stdout + * (using System.out) and those that indicate a conflict will exit the process + * (using System.exit) + * 3. Call setMaxNumberOfProcessesOfType("_DEFAULT_",2) to specify that at most only + * 2 processes designated as type _DEFAULT_ will be allowed to run + * 4. Calls startingProcessOfType("_DEFAULT_") to indicate that this process is an + * instance of a _DEFAULT_ type process, and so should be limited appropriately + */ +public class ProcessInstanceLimiter implements Runnable { + private static final long DEFAULT_TIME_UPDATE_INTERVAL_MS = 100L; + private long timeUpdateInterval = DEFAULT_TIME_UPDATE_INTERVAL_MS; + private static final String DEFAULT_SHARED_MAP_NAME = "ProcessInstanceLimiter_DEFAULT_SHARED_MAP_"; + private static final String DEFAULT_SHARED_MAP_DIRECTORY = System.getProperty("java.io.tmpdir"); + private static final String DEFAULT_PROCESS_NAME = "_DEFAULT_"; + + static { + AffinitySupport.setThreadId(); + } + + private final String sharedMapPath; + private final ChronicleMap theSharedMap; + private final Callback callback; + private final Map localUpdates = new ConcurrentHashMap(); + private final Map processTypeToStartTimeType = new ConcurrentHashMap(); + private long startTime; + private long[] lastStartTimes; + private Map timedata = new ConcurrentHashMap(); + private Map starttimedata = new ConcurrentHashMap(); + + /** + * Create a ProcessInstanceLimiter instance with a default callback, an + * instance of "DefaultCallback", and using the default shareed file named + * ProcessInstanceLimiter_DEFAULT_SHARED_MAP_ in the temp directory. + * + * @throws IOException - if the default shared file cannot be created + */ + public ProcessInstanceLimiter() throws IOException { + this(new DefaultCallback()); + ((DefaultCallback) this.getCallback()).setLimiter(this); + } + + /** + * Create a ProcessInstanceLimiter instance using the default shared file + * named ProcessInstanceLimiter_DEFAULT_SHARED_MAP_ in the temp directory. + * + * @param callback - An instance of the Callback interface, which will receive + * callbacks + * @throws IOException - if the default shared file cannot be created + */ + public ProcessInstanceLimiter(Callback callback) throws IOException { + this(DEFAULT_SHARED_MAP_DIRECTORY + System.getProperty("file.separator") + DEFAULT_SHARED_MAP_NAME, callback); + } + + /** + * Create a ProcessInstanceLimiter instance using the default shareed file + * named ProcessInstanceLimiter_DEFAULT_SHARED_MAP_ in the tmp directory. + * + * @param sharedMapPath - The path to a file which will be used to store the shared + * map (the file need not pre-exist) + * @param callback - An instance of the Callback interface, which will receive + * callbacks + * @throws IOException - if the default shared file cannot be created + */ + public ProcessInstanceLimiter(String sharedMapPath, Callback callback) throws IOException { + this.sharedMapPath = sharedMapPath; + this.callback = callback; + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(String.class, Data.class); + builder.entries(1000); + builder.entrySize(1024); + this.theSharedMap = builder.create(new File(sharedMapPath)); + Thread t = new Thread(this, "ProcessInstanceLimiter updater"); + t.setDaemon(true); + t.start(); + } + + public static void main(String[] args) throws Exception { + ProcessInstanceLimiter.limitTo(2); + Thread.sleep(60L * 1000L); + } + + /** + * Convenience method. + *

+ * Create a ProcessInstanceLimiter instance which is limited to one OS + * process instance of the DEFAULT type. This will enforce that any JVM on + * the same box which runs the code + * "ProcessInstanceLimiter.limitToOneProcess()" will only have at most one + * JVM instance running at a time. + * + * @return - the ProcessInstanceLimiter instance + * @throws IOException - if the default shared file cannot be created + */ + public static ProcessInstanceLimiter limitToOneProcess() throws IOException { + return limitTo(1); + } + + /** + * Convenience method. + *

+ * Create a ProcessInstanceLimiter instance which is limited to + * "numProcesses" OS process instances of the DEFAULT type. This will + * enforce that any JVM on the same box which runs the code + * "ProcessInstanceLimiter.limitTo(numProcesses)" will only have at most + * numProcesses JVM instances running at a time. All the JVMs must use the + * same "numProcesses" value or the process will immediately exit with a + * configuration error. + * + * @param numProcesses - the number of JVM processes that can run at any one time + * @return - the ProcessInstanceLimiter instance + * @throws IOException - if the default shared file cannot be created + */ + public static ProcessInstanceLimiter limitTo(int numProcesses) throws IOException { + return limitTo(numProcesses, DEFAULT_PROCESS_NAME); + } + + /** + * Convenience method. + *

+ * Create a ProcessInstanceLimiter instance which is limited to + * "numProcesses" OS process instances of the "processType" type. This will + * enforce that any JVM on the same box which runs the code + * "ProcessInstanceLimiter.limitTo(numProcesses, processType)" will only + * have at most numProcesses JVM instances running at a time. All the JVMs + * must use the same "numProcesses" value for a "processType" or the process + * will immediately exit with a configuration error. + * + * @param numProcesses - the number of JVM processes that can run at any one time + * @param processType - any string, specifies the type of process that is limited to + * numProcesses processes + * @return - the ProcessInstanceLimiter instance + * @throws IOException - if the default shared file cannot be created + */ + public static ProcessInstanceLimiter limitTo(int numProcesses, String processType) throws IOException { + ProcessInstanceLimiter limiter = new ProcessInstanceLimiter(); + limiter.setMaxNumberOfProcessesOfType(processType, numProcesses); + limiter.startingProcessOfType(processType); + return limiter; + } + + /** + * Sleeps the thread for the specified number of milliseconds, ignoring + * interruptions. + * + * @param pause - time in milliseconds to sleep + */ + public static void pause(long pause) { + long start = System.currentTimeMillis(); + long elapsedTime; + while ((elapsedTime = System.currentTimeMillis() - start) < pause) { + try { + Thread.sleep(pause - elapsedTime); + } catch (InterruptedException e) { + } + } + } + + /** + * The path to the shared file which stored the shared map. + */ + public String getSharedMapPath() { + return sharedMapPath; + } + + /** + * The instance of the Callback interface held by the instance, which will + * receive callbacks + */ + public Callback getCallback() { + return this.callback; + } + + /** + * Returns the MaxNumberOfProcesses allowed for processes of type + * "processType", as specified in the shared map. If that type hasn't been + * set, then this returns -1 (which is an invalid value, as it must be a + * positive value) + */ + public int getMaxNumberOfProcessesAllowedFor(String processType) { + Data data = this.starttimedata.get(processType); + if (data == null) { + return -1; + } else { + return data.getMaxNumberOfProcessesAllowed(); + } + } + + /** + * The interval between updates to the shared map timestamps - i.e. this is + * the interval between notifications of other processes starting + */ + public long getTimeUpdateInterval() { + return timeUpdateInterval; + } + + /** + * Set the interval between updates to the shared map timestamps - i.e. this + * is the interval between notifications of other processes starting + */ + public void setTimeUpdateInterval(long timeUpdateInterval) { + this.timeUpdateInterval = timeUpdateInterval; + } + + /** + * run() method for the ProcessInstanceLimiter which it starts in a thread + * called "ProcessInstanceLimiter updater" + */ + public void run() { + //every timeUpdateInterval milliseconds, update the time + while (true) { + try { + pause(timeUpdateInterval); + String processType; + Set> entrySet = this.localUpdates.entrySet(); + for (Entry entry : entrySet) { + processType = entry.getKey(); + int index = entry.getValue().intValue(); + Data data = this.timedata.get(processType); + if (data == null) { + entrySet.remove(entry); + } else { + if (!lock(data, 100000)) { + entrySet.remove(entry); + this.callback.lockConflictDetected(processType, index); + } else { + try { + if (!updateTheSharedMap(processType, index, data)) { + entrySet.remove(entry); + } + } finally { + //and release the lock + unlock(data); + } + } + } + } + } catch (Exception e) { + // TODO + e.printStackTrace(); + } + } + } + + /** + * Call this near the start of the process - if the process can acquire a + * slot, it will callback thisProcessOfTypeHasStartedAtSlot(), otherwise one + * of the other callback interface methods will be called. + * + * @param processType + */ + public void startingProcessOfType(String processType) { + Data data = this.timedata.get(processType); + if (data == null) { + this.callback.noDefinitionForProcessesOfType(processType); + this.callback.tooManyProcessesOfType(processType); + return; + } + // We need to lock access to the Time array, try up to 1 second + long[] times1 = new long[data.getMaxNumberOfProcessesAllowed()]; + if (!lock(data, 1000000)) { + this.callback.tooManyProcessesOfType(processType); + return; + } + //try {Thread.sleep(60L*1000L);} catch (InterruptedException e) {} + //we've got the lock, now copy the array + try { + for (int i = 0; i < times1.length; i++) { + times1[i] = data.getTimeAt(i); + } + } finally { + //and release the lock + unlock(data); + } + pause(3L * timeUpdateInterval); + if (!lock(data, 1000000)) { + this.callback.tooManyProcessesOfType(processType); + return; + } + boolean alreadyUnlocked = false; + try { + for (int i = 0; i < times1.length; i++) { + if (data.getTimeAt(i) == times1[i]) { + //we have an index which has not been updated in 3x the + //time interval, so we have a spare slot - use this slot + this.startTime = System.currentTimeMillis(); + this.starttimedata.get(processType).setTimeAt(i, this.startTime); + if (updateTheSharedMap(processType, i, data)) { + this.localUpdates.put(processType, new Integer(i)); + unlock(data); + alreadyUnlocked = true; + this.callback.thisProcessOfTypeHasStartedAtSlot(processType, i); + return; + } + } + } + } finally { + //and release the lock + if (!alreadyUnlocked) { + unlock(data); + } + } + this.callback.tooManyProcessesOfType(processType); + } + + /** + * Set the maximum number of processes of type processType that can run + * concurrently on the same machine + * + * @param processType - any string, specifies the type of process that is limited to + * maxNumberOfProcessesAllowed processes + * @param maxNumberOfProcessesAllowed - any positive number, specifies the maximum number of + * processes of this type that can run concurrently on the same + * machine + */ + public void setMaxNumberOfProcessesOfType(String processType, int maxNumberOfProcessesAllowed) { + if (maxNumberOfProcessesAllowed <= 0) { + throw new IllegalArgumentException("maxNumberOfProcessesAllowed must be a positive number, not " + maxNumberOfProcessesAllowed); + } + Data data = DataValueClasses.newDirectReference(Data.class); + this.timedata.put(processType, data); + this.theSharedMap.acquireUsing(processType, data); + if (data.getMaxNumberOfProcessesAllowed() != maxNumberOfProcessesAllowed) { + //it's either a new object, set to 0, or + //another process set it to an invalid value + if (data.compareAndSwapMaxNumberOfProcessesAllowed(0, maxNumberOfProcessesAllowed)) { + //What we expected, everything's good + } else { + //something else set a value, if it's not 2 we've got a conflict + if (data.getMaxNumberOfProcessesAllowed() != maxNumberOfProcessesAllowed) { + throw new IllegalArgumentException("The existing shared map already specifies that the maximum number of processes allowed is " + data.getMaxNumberOfProcessesAllowed() + " and changing that to " + maxNumberOfProcessesAllowed + " is not supported"); + } + } + } + String name = processType + '#'; + data = DataValueClasses.newDirectReference(Data.class); + this.starttimedata.put(processType, data); + this.processTypeToStartTimeType.put(processType, name); + this.theSharedMap.acquireUsing(name, data); + //this time just set it, we've done the guarding with the other value + if (data.getMaxNumberOfProcessesAllowed() == 0) { + data.setMaxNumberOfProcessesAllowed(maxNumberOfProcessesAllowed); + } + } + + /** + * Assumes that the data object is non-null and already locked + * If true is returned, the update has been applied, otherwise + * this slot is conflicted + */ + private boolean updateTheSharedMap(String processType, int index, Data data) { + long timenow = System.currentTimeMillis(); + data.setTimeAt(index, timenow); + Data startTimesData = this.starttimedata.get(processType); + if (this.startTime != startTimesData.getTimeAt(index)) { + //something else is updating this index, so we assume we're + //conflicted and give up - with a callback + this.callback.anotherProcessHasHijackedThisSlot(processType, index); + return false; + } + if (this.lastStartTimes != null) { + for (int i = 0; i < this.lastStartTimes.length; i++) { + if ((i != index) && (this.lastStartTimes[i] != startTimesData.getTimeAt(i))) { + this.callback.anotherProcessHasStartedOnSlot(processType, i, startTimesData.getTimeAt(i)); + } + } + } else { + this.lastStartTimes = new long[startTimesData.getMaxNumberOfProcessesAllowed()]; + } + for (int i = 0; i < this.lastStartTimes.length; i++) { + this.lastStartTimes[i] = startTimesData.getTimeAt(i); + } + return true; + } + + private boolean lock(Data data, int microsecondsToTry) { + return data.tryLockNanosTimelock(1000L * microsecondsToTry); + } + + private void unlock(Data data) { + try { + data.unlockTimelock(); + } catch (IllegalMonitorStateException e) { + //odd, but we'll be unlocked either way + System.out.println("Unexpected state: " + e); + e.printStackTrace(); + } + } + + /** + * The Callback interface holds all the calls that can be made by the + * process instance limiter. + */ + public static interface Callback { + /** + * Called when there are already the specified number of processes of + * the given type running, and this process is one too many. + * + * @param processType - the name of the type of process being limited + */ + public void tooManyProcessesOfType(String processType); + + /** + * Called when there is a lock conflict in the limiter + * which probably means the process must exit + * + * @param processType - the name of the type of process being limited + * @param index - the slot number held by the other process + */ + public void lockConflictDetected(String processType, int index); + + /** + * Called when another process has started and successfully acquired a + * slot that allows it to continue running. + * + * @param processType - the name of the type of process being limited + * @param slot - the slot number held by the other process + * @param startTime - the start timestamp of the other process + */ + public void anotherProcessHasStartedOnSlot(String processType, int slot, long startTime); + + /** + * Called when this process has started and successfully acquired a slot + * that allows it to continue running. + * + * @param processType - the name of the type of process being limited + * @param slot - the slot number held by this process + */ + public void thisProcessOfTypeHasStartedAtSlot(String processType, int slot); + + /** + * Called if the process was started but there was no data defined in + * the shared map that would limit processes of this type. + * + * @param processType - the name of the type of process being limited + */ + public void noDefinitionForProcessesOfType(String processType); + + /** + * Called when another process somehow managed to steal the slot that + * this process had acquired. + * + * @param processType - the name of the type of process being limited + * @param slot - the slot number held by this process + */ + public void anotherProcessHasHijackedThisSlot(String processType, int slot); + } + + /** + * The Data object holds an array of timestamps and a maximum number of + * processes allowed to be running concurrently + *

+ * The Timelock field is just for locking the time field + */ + public static interface Data { + void setTimeAt(@MaxSize(50) int index, long time); + + long getTimeAt(int index); + + int getMaxNumberOfProcessesAllowed(); + + void setMaxNumberOfProcessesAllowed(int num); + + boolean compareAndSwapMaxNumberOfProcessesAllowed(int expected, int value); + + boolean tryLockNanosTimelock(long nanos); + + void unlockTimelock() throws IllegalMonitorStateException; + + //void resetlockTimelock() throws IllegalMonitorStateException; + int getTimelock(); + + void setTimelock(int num); + } + + /** + * A default implementation of the Callback interface, which prints an + * information line to System.out for each callback, and calls + * System.exit(0) for those methods which don't leave the current process + * owning a slot. + */ + public static class DefaultCallback implements Callback { + ProcessInstanceLimiter limiter; + + public DefaultCallback(ProcessInstanceLimiter limiter) { + this.limiter = limiter; + } + + public DefaultCallback() { + } + + public ProcessInstanceLimiter getLimiter() { + return limiter; + } + + public void setLimiter(ProcessInstanceLimiter limiter) { + this.limiter = limiter; + } + + public void tooManyProcessesOfType(String processType) { + System.out.println("Sufficient processes (" + this.limiter.getMaxNumberOfProcessesAllowedFor(processType) + ") of type " + processType + " have already been started, so exiting this process"); + System.exit(0); + } + + public void noDefinitionForProcessesOfType(String processType) { + System.out.println("No definition for processes of type " + processType + " has been set, so exiting this process"); + System.exit(0); + } + + public void anotherProcessHasHijackedThisSlot(String processType, int slot) { + System.out.println("Another process of type " + processType + " has hijacked the slot (" + slot + "/" + this.limiter.getMaxNumberOfProcessesAllowedFor(processType) + ") allocated to this process, so exiting this process"); + System.exit(0); + } + + public void thisProcessOfTypeHasStartedAtSlot(String processType, int slot) { + System.out.println("This process of type " + processType + " has started at slot " + slot + "/" + this.limiter.getMaxNumberOfProcessesAllowedFor(processType)); + } + + public void anotherProcessHasStartedOnSlot(String processType, int slot, long startTime) { + System.out.println("Another process of type " + processType + " has started at slot " + slot + "/" + this.limiter.getMaxNumberOfProcessesAllowedFor(processType) + " at time " + new Date(startTime)); + } + + public void lockConflictDetected(String processType, int slot) { + System.out.println("The limiter lock has become conflicted for type " + processType + " on slot (" + slot + "/" + this.limiter.getMaxNumberOfProcessesAllowedFor(processType) + ") allocated to this process, so exiting this process"); + System.exit(0); + } + } + +} diff --git a/src/test/java/net/openhft/collections/ClusterReplicationTest.java b/src/test/java/net/openhft/collections/ClusterReplicationTest.java deleted file mode 100644 index a5c3ce5b8..000000000 --- a/src/test/java/net/openhft/collections/ClusterReplicationTest.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - *

- * http://www.higherfrequencytrading.com - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.collections; - - -import net.openhft.chronicle.map.ChronicleMapBuilder; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.junit.Assert.assertTrue; - -/** - * Test VanillaSharedReplicatedHashMap where the Replicated is over a TCP Socket, but with 4 nodes - * - * @author Rob Austin. - */ -public class ClusterReplicationTest { - - private SharedHashMap map1a; - private SharedHashMap map2a; - - private SharedHashMap map1b; - private SharedHashMap map2b; - - private ClusterReplicator clusterB; - private ClusterReplicator clusterA; - - private ClusterReplicatorBuilder clusterReplicatorBuilder; - private ClusterReplicatorBuilder clusterReplicatorBuilder1; - - - public static File getPersistenceFile() { - String TMP = System.getProperty("java.io.tmpdir"); - File file = new File(TMP + "/test" + System.nanoTime()); - file.deleteOnExit(); - return file; - } - - - @Before - - public void setup() throws IOException { - - { - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8086, new InetSocketAddress("localhost", 8087)).heartBeatInterval(1, SECONDS); - - clusterReplicatorBuilder = new ClusterReplicatorBuilder((byte) 1, 1024); - clusterReplicatorBuilder.tcpReplicatorBuilder(tcpReplicatorBuilder); - - - // this is how you add maps after the custer is created - map1a = clusterReplicatorBuilder.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - clusterA = clusterReplicatorBuilder.create(); - - } - - - { - - final TcpReplicatorBuilder tcpReplicatorBuilder = - new TcpReplicatorBuilder(8087).heartBeatInterval(1, SECONDS); - - clusterReplicatorBuilder1 = new ClusterReplicatorBuilder((byte) - 2, 1024); - clusterReplicatorBuilder1.tcpReplicatorBuilder(tcpReplicatorBuilder); - - - // this is how you add maps after the custer is created - map1b = clusterReplicatorBuilder1.create((short) 1, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - clusterB = clusterReplicatorBuilder1.create(); - } - - - } - - @After - public void tearDown() throws InterruptedException { - - for (final Closeable closeable : new Closeable[]{clusterA, clusterB}) { - try { - closeable.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - - @Test - public void test() throws IOException, InterruptedException { - - // todo remove this sleep - - // Thread.sleep(100); - - map2b = clusterReplicatorBuilder1.create((short) 2, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - map2a = clusterReplicatorBuilder.create((short) 2, - ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .entries(1000) - .file(getPersistenceFile())); - - map2a.put(1, "EXAMPLE-2"); - map1a.put(1, "EXAMPLE-1"); - - // allow time for the recompilation to resolve - waitTillEqual(2500); - - Assert.assertEquals("map1a=map1b", map1a, map1b); - Assert.assertEquals("map2a=map2b", map2a, map2b); - - assertTrue("map1a.empty", !map1a.isEmpty()); - assertTrue("map2a.empty", !map2a.isEmpty()); - - } - - - /** - * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - - private void waitTillEqual(final int timeOutMs) throws InterruptedException { - for (int t = 0; t < timeOutMs; t++) { - if (map1a.equals(map1b) && - map2a.equals(map2b)) - break; - Thread.sleep(1); - } - - } - -} - - - diff --git a/src/test/java/net/openhft/lang/values/IntValue$$Native.java b/src/test/java/net/openhft/lang/values/IntValue$$Native.java new file mode 100644 index 000000000..e01dc0782 --- /dev/null +++ b/src/test/java/net/openhft/lang/values/IntValue$$Native.java @@ -0,0 +1,124 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.values; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.Copyable; + +import static net.openhft.lang.Compare.calcLongHashCode; +import static net.openhft.lang.Compare.isEqual; + +public class IntValue$$Native implements IntValue, BytesMarshallable, Byteable, Copyable { + private static final int VALUE = 0; + + + private Bytes _bytes; + private long _offset; + + + + public void setValue(int $) { + _bytes.writeInt(_offset + VALUE, $); + } + + public int getValue() { + return _bytes.readInt(_offset + VALUE); + } + + + + public void setOrderedValue(int $) { + _bytes.writeOrderedInt(_offset + VALUE, $); + } + + public int getVolatileValue() { + return _bytes.readVolatileInt(_offset + VALUE); + } + + public int addValue(int $) { + return _bytes.addInt(_offset + VALUE, $); + } public int addAtomicValue(int $) { + return _bytes.addAtomicInt(_offset + VALUE, $); + } public boolean compareAndSwapValue(int _1, int _2) { + return _bytes.compareAndSwapInt(_offset + VALUE, _1, _2); + } public boolean tryLockNanosValue(long nanos) { + return _bytes.tryLockNanosInt(_offset + VALUE, nanos); + } public boolean tryLockValue() { + return _bytes.tryLockInt(_offset + VALUE); + } public void unlockValue() { + _bytes.unlockInt(_offset + VALUE); + } public void busyLockValue() throws InterruptedException { + _bytes.busyLockInt(_offset + VALUE); + } @Override + public void copyFrom(net.openhft.lang.values.IntValue from) { + setValue(from.getValue()); + } + + @Override + public void writeMarshallable(Bytes out) { + out.writeInt(getValue()); + } + @Override + public void readMarshallable(Bytes in) { + setValue(in.readInt()); + } + @Override + public void bytes(Bytes bytes, long offset) { + this._bytes = bytes; + this._offset = offset; + } + @Override + public Bytes bytes() { + return _bytes; + } + @Override + public long offset() { + return _offset; + } + @Override + public int maxSize() { + return 4; + } + public int hashCode() { + long lhc = longHashCode(); + return (int) ((lhc >>> 32) ^ lhc); + } + + public long longHashCode() { + return calcLongHashCode(getValue()); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof IntValue)) return false; + IntValue that = (IntValue) o; + + if(!isEqual(getValue(), that.getValue())) return false; + return true; + } + + public String toString() { + if (_bytes == null) return "bytes is null"; + StringBuilder sb = new StringBuilder(); + sb.append("IntValue{ "); + sb.append("value= ").append(getValue()); + sb.append(" }"); + return sb.toString(); + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/lang/values/LongValue$$Native.java b/src/test/java/net/openhft/lang/values/LongValue$$Native.java new file mode 100644 index 000000000..3e20fc0a9 --- /dev/null +++ b/src/test/java/net/openhft/lang/values/LongValue$$Native.java @@ -0,0 +1,116 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.values; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.Copyable; + +import static net.openhft.lang.Compare.calcLongHashCode; +import static net.openhft.lang.Compare.isEqual; + +public class LongValue$$Native implements LongValue, BytesMarshallable, Byteable, Copyable { + private static final int VALUE = 0; + + + private Bytes _bytes; + private long _offset; + + + + public void setValue(long $) { + _bytes.writeLong(_offset + VALUE, $); + } + + public long getValue() { + return _bytes.readLong(_offset + VALUE); + } + + + + public void setOrderedValue(long $) { + _bytes.writeOrderedLong(_offset + VALUE, $); + } + + public long getVolatileValue() { + return _bytes.readVolatileLong(_offset + VALUE); + } + + public long addValue(long $) { + return _bytes.addLong(_offset + VALUE, $); + } public long addAtomicValue(long $) { + return _bytes.addAtomicLong(_offset + VALUE, $); + } public boolean compareAndSwapValue(long _1, long _2) { + return _bytes.compareAndSwapLong(_offset + VALUE, _1, _2); + } @Override + public void copyFrom(net.openhft.lang.values.LongValue from) { + setValue(from.getValue()); + } + + @Override + public void writeMarshallable(Bytes out) { + out.writeLong(getValue()); + } + @Override + public void readMarshallable(Bytes in) { + setValue(in.readLong()); + } + @Override + public void bytes(Bytes bytes, long offset) { + this._bytes = bytes; + this._offset = offset; + } + @Override + public Bytes bytes() { + return _bytes; + } + @Override + public long offset() { + return _offset; + } + @Override + public int maxSize() { + return 8; + } + public int hashCode() { + long lhc = longHashCode(); + return (int) ((lhc >>> 32) ^ lhc); + } + + public long longHashCode() { + return calcLongHashCode(getValue()); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof LongValue)) return false; + LongValue that = (LongValue) o; + + if(!isEqual(getValue(), that.getValue())) return false; + return true; + } + + public String toString() { + if (_bytes == null) return "bytes is null"; + StringBuilder sb = new StringBuilder(); + sb.append("LongValue{ "); + sb.append("value= ").append(getValue()); + sb.append(" }"); + return sb.toString(); + } +}