From e14461249d54d5f4096dffca5cb5ca9e727f42cb Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 10:44:48 +0000 Subject: [PATCH 01/21] HCOLL-231 Close the connection when the attached is dropped. --- src/main/java/net/openhft/chronicle/map/TcpReplicator.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index efec51608..004c63cb0 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -592,7 +592,11 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable final long approxTime) throws IOException { final SocketChannel socketChannel = (SocketChannel) key.channel(); final Attached attached = (Attached) key.attachment(); - if (attached == null) throw new NullPointerException("No attached"); + if (attached == null) { + LOG.info("Closing connection " + socketChannel + ", nothing attached"); + socketChannel.close(); + return; + } TcpSocketChannelEntryWriter entryWriter = attached.entryWriter; if (entryWriter == null) throw new NullPointerException("No entryWriter"); if (entryWriter.isWorkIncomplete()) { From f6ebed3c7c790aee7869cb8ff72405f011ef163a Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 11:46:34 +0000 Subject: [PATCH 02/21] Update lang to the latest version --- .../map/AbstractChronicleMapBuilder.java | 14 ++++++++ .../chronicle/map/ChronicleMapBuilder.java | 7 ++-- .../chronicle/map/ChronicleMapBuilderI.java | 3 ++ .../chronicle/map/StatelessChronicleMap.java | 27 +++++++++------- .../chronicle/map/CHMUseCasesTest.java | 32 +++++++++---------- .../chronicle/map/StatelessClientTest.java | 2 +- 6 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java index 2b92941a5..0b4002388 100755 --- a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java @@ -143,6 +143,8 @@ abstract class AbstractChronicleMapBuilder keyClass, Class valueClass) { keyBuilder = new SerializationBuilder(keyClass, SerializationBuilder.Role.KEY); @@ -1003,5 +1005,17 @@ abstract class AbstractChronicleMapBuilder checkSerializedValues() { + this.checkSerializedValues = true; + return this; + } + + @Override + public ChronicleMapBuilderI disableOversizedEntries() { + this.disableOversizedEntries = true; + return this; + } } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 8283f5037..eb6ed4dee 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -541,13 +541,12 @@ public final class ChronicleMapBuilder implements ChronicleMapBuilderI checkSerializedValues() { - // throw new UnsupportedOperationException("Not implemented"); + delegate.checkSerializedValues(); return this; } - public ChronicleMapBuilder disableOversizedEntries(boolean disableOversizedEntries) { - // throw an exception rather than use oversized entries to test the size is as expected. - // throw new UnsupportedOperationException("Not implemented"); + public ChronicleMapBuilder disableOversizedEntries() { + delegate.disableOversizedEntries(); return this; } } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java index 8268a2008..454356f04 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java @@ -315,4 +315,7 @@ public interface ChronicleMapBuilderI extends Serializable { String name(); + ChronicleMapBuilderI checkSerializedValues(); + + ChronicleMapBuilderI disableOversizedEntries(); } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index bf5cf7f3f..e592fb2a9 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -68,8 +68,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon private volatile ByteBuffer outBuffer; private volatile ByteBufferBytes outBytes; - private ByteBuffer inBuffer; - private ByteBufferBytes inBytes; + private volatile ByteBuffer inBuffer; + private volatile ByteBufferBytes inBytes; @NotNull private final ReaderWithSize keyReaderWithSize; @@ -331,13 +331,14 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon * @return a unique transactionId */ private long nextUniqueTransaction(long time) { - long old = transactionID.getAndSet(time); - - if (old == time) - return transactionID.incrementAndGet(); - else - return time; - + long id = time * 1000; + for (; ; ) { + long old = transactionID.get(); + if (old > id) id = old + 1; + if (transactionID.compareAndSet(old, id)) + break; + } + return id; } @SuppressWarnings("NullableProblems") @@ -997,13 +998,15 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon receive(SIZE_OF_SIZE + SIZE_OF_TRANSACTION_ID, timeoutTime); final int messageSize = inBytes.readInt(); + assert messageSize > 0 : "Invalid message size " + messageSize; + assert messageSize < 16 << 20 : "Invalid message size " + messageSize; + final int remainingBytes0 = messageSize - (SIZE_OF_SIZE + SIZE_OF_TRANSACTION_ID); final long transactionId0 = inBytes.readLong(); - assert transactionId0 != 0; - // check the transaction id is reasonable - assert String.valueOf(transactionId0).startsWith("14"); + assert transactionId0 > 1410000000000000L : "TransactionId too small " + transactionId0 + " messageSize " + messageSize; + assert transactionId0 < 2100000000000000L : "TransactionId too large " + transactionId0 + " messageSize " + messageSize; // if the transaction id is for this thread process it if (transactionId0 == transactionId) { diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index ce6187196..a52a72a9b 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -246,7 +246,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = ChronicleMapBuilder .of(Integer.class, Integer.class) .entrySize(8) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .create()) { Integer key1; Integer key2; @@ -300,7 +300,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = ChronicleMapBuilder .of(Long.class, Long.class) .entrySize(16) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .create()) { map.put(1L, 11L); assertEquals((Long) 11L, map.get(1L)); @@ -342,7 +342,7 @@ public class CHMUseCasesTest { public void testDoubleDoubleMap() throws ExecutionException, InterruptedException { try (ChronicleMap map = ChronicleMapBuilder .of(Double.class, Double.class) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(16) .create()) { map.put(1.0, 11.0); @@ -386,7 +386,7 @@ public class CHMUseCasesTest { public void testByteArrayByteArrayMap() throws ExecutionException, InterruptedException { try (ChronicleMap map = ChronicleMapBuilder .of(byte[].class, byte[].class) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(12) .create()) { byte[] key1 = {1, 1, 1, 1}; @@ -436,7 +436,7 @@ public class CHMUseCasesTest { .of(ByteBuffer.class, ByteBuffer.class) .valueMarshaller(ByteBufferMarshaller.INSTANCE) .keyMarshaller(ByteBufferMarshaller.INSTANCE) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(20) .create()) { ByteBuffer key1 = ByteBuffer.wrap(new byte[]{1, 1, 1, 1}); @@ -522,7 +522,7 @@ public class CHMUseCasesTest { ByteBuffer bb1 = ByteBuffer.allocate(8); bb1.put(value1); - bb1.putShort((short)12345); + bb1.putShort((short) 12345); bb1.flip(); assertBBEquals(bb1, valueA); } @@ -543,7 +543,7 @@ public class CHMUseCasesTest { .of(ByteBuffer.class, ByteBuffer.class) .valueMarshaller(ByteBufferMarshaller.INSTANCE) // we should not have to to this ! .keyMarshaller(ByteBufferMarshaller.INSTANCE) // we should not have to to this ! - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(12) .create()) { ByteBuffer key1 = ByteBuffer.wrap(new byte[]{1, 1, 1, 1}); @@ -595,7 +595,7 @@ public class CHMUseCasesTest { public void testIntValueIntValueMap() { try (ChronicleMap map = ChronicleMapBuilder .of(IntValue.class, IntValue.class) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(8) .create()) { IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); @@ -677,7 +677,7 @@ public class CHMUseCasesTest { public void testUnsignedIntValueUnsignedIntValueMap() { try (ChronicleMap map = ChronicleMapBuilder .of(UnsignedIntValue.class, UnsignedIntValue.class) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(8) .create()) { UnsignedIntValue key1 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, true); @@ -759,7 +759,7 @@ public class CHMUseCasesTest { public void testIntValueShortValueMap() { try (ChronicleMap map = ChronicleMapBuilder .of(IntValue.class, ShortValue.class) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(6) .create()) { IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); @@ -842,7 +842,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = ChronicleMapBuilder .of(IntValue.class, UnsignedShortValue.class) .entrySize(6) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .create()) { IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); @@ -923,7 +923,7 @@ public class CHMUseCasesTest { public void testIntValueCharValueMap() { try (ChronicleMap map = ChronicleMapBuilder .of(IntValue.class, CharValue.class) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(6) .create()) { IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); @@ -1006,7 +1006,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = ChronicleMapBuilder .of(IntValue.class, UnsignedByteValue.class) .entrySize(5) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .create()) { IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); @@ -1087,7 +1087,7 @@ public class CHMUseCasesTest { public void testIntValueBooleanValueMap() { try (ChronicleMap map = ChronicleMapBuilder .of(IntValue.class, BooleanValue.class) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .entrySize(5) .create()) { IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); @@ -1170,7 +1170,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = ChronicleMapBuilder .of(FloatValue.class, FloatValue.class) .entrySize(8) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .create()) { FloatValue key1 = ChronicleMapBuilder.newInstance(FloatValue.class, false); FloatValue key2 = ChronicleMapBuilder.newInstance(FloatValue.class, false); @@ -1337,7 +1337,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = ChronicleMapBuilder .of(LongValue.class, LongValue.class) .entrySize(16) - .disableOversizedEntries(true) // disabled for testing purposes only. + .disableOversizedEntries() // disabled for testing purposes only. .create()) { LongValue key1 = ChronicleMapBuilder.newInstance(LongValue.class, false); LongValue key2 = ChronicleMapBuilder.newInstance(LongValue.class, false); diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index 9f87f5d1a..c25fcfa6c 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -38,7 +38,7 @@ import static org.junit.Assert.assertTrue; public class StatelessClientTest { public static final int SIZE = 2500; - static int s_port = 9050; + static int s_port = 9070; enum ToString implements Function { INSTANCE; From ed27869e40b09648e2f5ab1ea4e9199969f87ad4 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 10 Dec 2014 12:06:13 +0000 Subject: [PATCH 03/21] changed to always reconnect if fails while establishing a connection --- src/main/java/net/openhft/chronicle/map/TcpReplicator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index efec51608..90409177e 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -413,8 +413,8 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable // so these nodes will establish the connection when they come back up, // hence under these circumstances, polling a dropped node to attempt to reconnect is no-longer // required as the remote node will establish the connection its self on startup. - if (replicationConfig.autoReconnectedUponDroppedConnection()) - attached.connector.connect(); + + attached.connector.connect(); throw e; } From 9ac22da7f74f63d9486548d8ac65668f6ba80c62 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 10 Dec 2014 12:14:37 +0000 Subject: [PATCH 04/21] change the auto reconnect to true by default --- .../hash/replication/TcpTransportAndNetworkConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java b/src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java index a37a4a45d..3e8b025d9 100644 --- a/src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java +++ b/src/main/java/net/openhft/chronicle/hash/replication/TcpTransportAndNetworkConfig.java @@ -77,7 +77,7 @@ public final class TcpTransportAndNetworkConfig implements Serializable { return new TcpTransportAndNetworkConfig( serverPort, unmodifiableSet(s), DEFAULT_PACKET_SIZE, - false, // autoReconnectedUponDroppedConnection + true, // autoReconnectedUponDroppedConnection ThrottlingConfig.noThrottling(), DEFAULT_HEART_BEAT_INTERVAL, DEFAULT_HEART_BEAT_INTERVAL_UNIT); From eb7338ef7d9e90feeb21fe54bee1a02657809d11 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 12:57:30 +0000 Subject: [PATCH 05/21] Add an annotation to make it clearer when the argument is expected to be altered. --- src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index a52a72a9b..838fa9d99 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -434,10 +434,10 @@ public class CHMUseCasesTest { public void testByteBufferByteBufferMap() throws ExecutionException, InterruptedException { try (ChronicleMap map = ChronicleMapBuilder .of(ByteBuffer.class, ByteBuffer.class) - .valueMarshaller(ByteBufferMarshaller.INSTANCE) .keyMarshaller(ByteBufferMarshaller.INSTANCE) + .valueMarshaller(ByteBufferMarshaller.INSTANCE) .disableOversizedEntries() // disabled for testing purposes only. - .entrySize(20) + .entrySize(12) .create()) { ByteBuffer key1 = ByteBuffer.wrap(new byte[]{1, 1, 1, 1}); ByteBuffer key2 = ByteBuffer.wrap(new byte[]{2, 2, 2, 2}); From 8d347e4f28158c9bf87fe9d068eb29edecb8af19 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 10 Dec 2014 15:33:25 +0000 Subject: [PATCH 06/21] renamed to putWith() to putMapped() --- .../openhft/chronicle/map/ChronicleMap.java | 8 +-- .../chronicle/map/StatelessChronicleMap.java | 8 +-- .../openhft/chronicle/map/TcpReplicator.java | 12 ++--- .../map/{Mutator.java => UnaryOperator.java} | 2 +- .../chronicle/map/VanillaChronicleMap.java | 6 +-- .../chronicle/map/CHMUseCasesTest.java | 54 +++++++++---------- .../chronicle/map/ReplicationCheckingMap.java | 8 +-- .../chronicle/map/StatelessClientTest.java | 4 +- .../jrs166/map/StatelessChronicleMapTest.java | 8 +-- 9 files changed, 55 insertions(+), 55 deletions(-) rename src/main/java/net/openhft/chronicle/map/{Mutator.java => UnaryOperator.java} (88%) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java index 115c598dc..8482ae791 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java @@ -197,17 +197,17 @@ public interface ChronicleMap extends ConcurrentMap, ChronicleHash { * @param return type. * @return the result of the function, or null if there is no entry for the key. */ - R mapForKey(K key, @NotNull Function function); + R getMapped(K key, @NotNull Function function); /** - * Apply a mutator to the value for a key and return a result. A write lock is assumed.

If + * Apply a unaryOperator to the value for a key and return a result. A write lock is assumed.

If * there is no entry for this key null will be returned * * @param key to apply the mapping to - * @param mutator to alter the value and calculate a result + * @param unaryOperator to alter the value and calculate a result * @return the result of the function. */ - V putWith(K key, @NotNull Mutator mutator); + V putMapped(K key, @NotNull UnaryOperator unaryOperator); /** * exports all the entries to a {@link java.io.File} storing them in JSON format, an attempt is diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index e592fb2a9..727195d2a 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -123,7 +123,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon PUT_ALL_WITHOUT_ACC, HASH_CODE, MAP_FOR_KEY, - UPDATE_FOR_KEY + PUT_MAPPED } @@ -482,7 +482,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } @Nullable - public R mapForKey(@Nullable K key, @NotNull Function function) { + public R getMapped(@Nullable K key, @NotNull Function function) { if (key == null) throw keyNotNullNPE(); return fetchObject(MAP_FOR_KEY, key, function); @@ -491,10 +491,10 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon @Nullable @Override - public V putWith(@Nullable K key, @NotNull Mutator mutator) { + public V putMapped(@Nullable K key, @NotNull UnaryOperator unaryOperator) { if (key == null) throw keyNotNullNPE(); - return fetchObject(UPDATE_FOR_KEY, key, mutator); + return fetchObject(PUT_MAPPED, key, unaryOperator); } diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index c4c82d6db..dbb480ba7 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -1417,8 +1417,8 @@ class StatelessServerConnector { case MAP_FOR_KEY: return mapForKey(reader, writer, sizeLocation); - case UPDATE_FOR_KEY: - return updateForKey(reader, writer, sizeLocation); + case PUT_MAPPED: + return putMapped(reader, writer, sizeLocation); default: throw new IllegalStateException("unsupported event=" + event); @@ -1430,7 +1430,7 @@ class StatelessServerConnector { final K key = keyReaderWithSize.read(reader, null); final Function function = (Function) reader.readObject(); try { - Object result = map.mapForKey(key, function); + Object result = map.getMapped(key, function); writer.writeObject(result); } catch (Throwable e) { LOG.info("", e); @@ -1442,11 +1442,11 @@ class StatelessServerConnector { } @Nullable - public Work updateForKey(@NotNull ByteBufferBytes reader, @NotNull Bytes writer, long sizeLocation) { + public Work putMapped(@NotNull ByteBufferBytes reader, @NotNull Bytes writer, long sizeLocation) { final K key = keyReaderWithSize.read(reader, null); - final Mutator mutator = (Mutator) reader.readObject(); + final UnaryOperator unaryOperator = (UnaryOperator) reader.readObject(); try { - Object result = map.putWith(key, mutator); + Object result = map.putMapped(key, unaryOperator); writer.writeObject(result); } catch (Throwable e) { LOG.info("", e); diff --git a/src/main/java/net/openhft/chronicle/map/Mutator.java b/src/main/java/net/openhft/chronicle/map/UnaryOperator.java similarity index 88% rename from src/main/java/net/openhft/chronicle/map/Mutator.java rename to src/main/java/net/openhft/chronicle/map/UnaryOperator.java index 5d527604b..888eb4489 100755 --- a/src/main/java/net/openhft/chronicle/map/Mutator.java +++ b/src/main/java/net/openhft/chronicle/map/UnaryOperator.java @@ -11,7 +11,7 @@ import java.io.Serializable; */ -public interface Mutator extends Serializable { +public interface UnaryOperator extends Serializable { /** * Applies this mutator to the given mutable argument. * diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index c55448db0..2864f578f 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -551,7 +551,7 @@ class VanillaChronicleMap, } @Override - public R mapForKey(K key, @NotNull Function function) { + public R getMapped(K key, @NotNull Function function) { try (ReadContext entry = lookupUsing(key, null, false, false, LockType.READ_LOCK)) { return entry.present() ? function.apply(entry.value()) : null; @@ -559,7 +559,7 @@ class VanillaChronicleMap, } @Override - public V putWith(K key, @NotNull Mutator mutator) { + public V putMapped(K key, @NotNull UnaryOperator unaryOperator) { final V using = (vClass.equals(CharSequence.class)) ? (V) new StringBuilder() : null; @@ -576,7 +576,7 @@ class VanillaChronicleMap, if (entry.value() == null) return null; - V result = mutator.update(entry.value()); + V result = unaryOperator.update(entry.value()); if (entry.value() instanceof StringBuilder) { ((StringBuilder) using).setLength(0); ((StringBuilder) using).append((CharSequence) result); diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index 838fa9d99..616357008 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -38,13 +38,13 @@ public class CHMUseCasesTest { map.put("Hello", "World"); assertEquals("World", map.get("Hello")); - assertEquals("New World", map.mapForKey("Hello", new Function() { + assertEquals("New World", map.getMapped("Hello", new Function() { @Override public String apply(String s) { return "New " + s; } })); - assertEquals(null, map.mapForKey("No key", new Function() { + assertEquals(null, map.getMapped("No key", new Function() { @Override public String apply(String s) { return "New " + s; @@ -73,7 +73,7 @@ public class CHMUseCasesTest { map.put("Hello", "World"); - map.putWith("Hello", new Mutator() { + map.putMapped("Hello", new UnaryOperator() { @Override public String update(String s) { return "New " + s; @@ -109,14 +109,14 @@ public class CHMUseCasesTest { assertNull(map.getUsing(key, value)); - assertEquals("New World", map.mapForKey("Hello", new + assertEquals("New World", map.getMapped("Hello", new Function() { @Override public CharSequence apply(CharSequence s) { return "New " + s; } })); - assertEquals(null, map.mapForKey("No key", new + assertEquals(null, map.getMapped("No key", new Function() { @Override public CharSequence apply(CharSequence s) { @@ -124,8 +124,8 @@ public class CHMUseCasesTest { } })); - assertEquals("New World !!", map.putWith("Hello", new - Mutator() { + assertEquals("New World !!", map.putMapped("Hello", new + UnaryOperator() { @Override public CharSequence update(CharSequence s) { ((StringBuilder) s).append(" !!"); @@ -135,8 +135,8 @@ public class CHMUseCasesTest { assertEquals("New World !!", map.get("Hello").toString()); - assertEquals(null, map.putWith("no-key", new - Mutator() { + assertEquals(null, map.putMapped("no-key", new + UnaryOperator() { @Override public CharSequence update(CharSequence s) { ((StringBuilder) s).append("!!"); @@ -268,13 +268,13 @@ public class CHMUseCasesTest { assertEquals(null, map.get(3)); assertEquals(null, map.get(4)); - assertEquals((Integer) 110, map.mapForKey(1, new Function() { + assertEquals((Integer) 110, map.getMapped(1, new Function() { @Override public Integer apply(Integer s) { return 10 * s; } })); - assertEquals(null, map.mapForKey(-1, new Function() { + assertEquals(null, map.getMapped(-1, new Function() { @Override public Integer apply(Integer s) { return 10 * s; @@ -282,7 +282,7 @@ public class CHMUseCasesTest { })); try { - map.putWith(1, new Mutator() { + map.putMapped(1, new UnaryOperator() { @Override public Integer update(Integer s) { return s + 1; @@ -311,13 +311,13 @@ public class CHMUseCasesTest { assertEquals(null, map.get(3L)); assertEquals(null, map.get(4L)); - assertEquals((Long) 110L, map.mapForKey(1L, new Function() { + assertEquals((Long) 110L, map.getMapped(1L, new Function() { @Override public Long apply(Long s) { return 10 * s; } })); - assertEquals(null, map.mapForKey(-1L, new Function() { + assertEquals(null, map.getMapped(-1L, new Function() { @Override public Long apply(Long s) { return 10 * s; @@ -325,7 +325,7 @@ public class CHMUseCasesTest { })); try { - map.putWith(1L, new Mutator() { + map.putMapped(1L, new UnaryOperator() { @Override public Long update(Long s) { return s + 1; @@ -354,13 +354,13 @@ public class CHMUseCasesTest { assertEquals(null, map.get(3.0)); assertEquals(null, map.get(4.0)); - assertEquals((Double) 110.0, map.mapForKey(1.0, new Function() { + assertEquals((Double) 110.0, map.getMapped(1.0, new Function() { @Override public Double apply(Double s) { return 10 * s; } })); - assertEquals(null, map.mapForKey(-1.0, new Function() { + assertEquals(null, map.getMapped(-1.0, new Function() { @Override public Double apply(Double s) { return 10 * s; @@ -368,7 +368,7 @@ public class CHMUseCasesTest { })); try { - map.putWith(1.0, new Mutator() { + map.putMapped(1.0, new UnaryOperator() { @Override public Double update(Double s) { return s + 1; @@ -401,20 +401,20 @@ public class CHMUseCasesTest { map.put(key1, value1); - assertTrue(Arrays.equals(new byte[]{11, 11}, map.mapForKey(key1, new Function() { + assertTrue(Arrays.equals(new byte[]{11, 11}, map.getMapped(key1, new Function() { @Override public byte[] apply(byte[] s) { return Arrays.copyOf(s, 2); } }))); - assertEquals(null, map.mapForKey(key2, new Function() { + assertEquals(null, map.getMapped(key2, new Function() { @Override public byte[] apply(byte[] s) { return Arrays.copyOf(s, 2); } })); - assertTrue(Arrays.equals(new byte[]{12, 10}, map.putWith(key1, new Mutator() { + assertTrue(Arrays.equals(new byte[]{12, 10}, map.putMapped(key1, new UnaryOperator() { @Override public byte[] update(byte[] s) { s[0]++; @@ -458,10 +458,10 @@ public class CHMUseCasesTest { }; map.put(key1, value1); - assertBBEquals(ByteBuffer.wrap(new byte[]{11, 11}), map.mapForKey(key1, function)); - assertEquals(null, map.mapForKey(key2, function)); + assertBBEquals(ByteBuffer.wrap(new byte[]{11, 11}), map.getMapped(key1, function)); + assertEquals(null, map.getMapped(key2, function)); - assertBBEquals(ByteBuffer.wrap(new byte[]{12, 10}), map.putWith(key1, new Mutator() { + assertBBEquals(ByteBuffer.wrap(new byte[]{12, 10}), map.putMapped(key1, new UnaryOperator() { @Override public ByteBuffer update(ByteBuffer s) { s.put(0, (byte) (s.get(0) + 1)); @@ -568,10 +568,10 @@ public class CHMUseCasesTest { return slice; } }; - assertBBEquals(ByteBuffer.wrap(new byte[]{11, 11}), map.mapForKey(key1, function)); - assertEquals(null, map.mapForKey(key2, function)); + assertBBEquals(ByteBuffer.wrap(new byte[]{11, 11}), map.getMapped(key1, function)); + assertEquals(null, map.getMapped(key2, function)); - assertBBEquals(ByteBuffer.wrap(new byte[]{12, 10}), map.putWith(key1, new Mutator() { + assertBBEquals(ByteBuffer.wrap(new byte[]{12, 10}), map.putMapped(key1, new UnaryOperator() { @Override public ByteBuffer update(ByteBuffer s) { s.put(0, (byte) (s.get(0) + 1)); diff --git a/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java index 757a728c5..41c314780 100644 --- a/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java +++ b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java @@ -218,13 +218,13 @@ public class ReplicationCheckingMap implements ChronicleMap { } @Override - public R mapForKey(K key, @NotNull Function function) { - return map1.mapForKey(key, function); + public R getMapped(K key, @NotNull Function function) { + return map1.getMapped(key, function); } @Override - public V putWith(K key, @NotNull Mutator mutator) { - return map1.putWith(key, mutator); + public V putMapped(K key, @NotNull UnaryOperator unaryOperator) { + return map1.putMapped(key, unaryOperator); } @Override diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index c25fcfa6c..a169e7060 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -111,7 +111,7 @@ public class StatelessClientTest { .class, StringBuilder.class) .putReturnsNull(true) .statelessClient(new InetSocketAddress("localhost", port)).create()) { - String actual = statelessMap.mapForKey(10, ToString.INSTANCE); + String actual = statelessMap.getMapped(10, ToString.INSTANCE); assertEquals("Hello World", actual); } @@ -131,7 +131,7 @@ public class StatelessClientTest { try (ChronicleMap statelessMap = ChronicleMapBuilder.of(Integer .class, StringBuilder.class) .statelessClient(new InetSocketAddress("localhost", port)).create()) { - String actual = statelessMap.mapForKey(11, ToString.INSTANCE); + String actual = statelessMap.getMapped(11, ToString.INSTANCE); assertEquals(null, actual); } diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java index 5d7a9551c..e4ddc060f 100755 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java @@ -170,13 +170,13 @@ public class StatelessChronicleMapTest extends JSR166TestCase { } @Override - public R mapForKey(K key, @NotNull Function function) { - return d.mapForKey(key, function); + public R getMapped(K key, @NotNull Function function) { + return d.getMapped(key, function); } @Override - public V putWith(K key, @NotNull Mutator mutator) { - return d.putWith(key, mutator); + public V putMapped(K key, @NotNull UnaryOperator unaryOperator) { + return d.putMapped(key, unaryOperator); } @Override From 18554f4ccced86bc220f06b775d0c620ea28bd53 Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 10 Dec 2014 17:21:30 +0000 Subject: [PATCH 07/21] introduced newKeyInstance and newValueInstance --- .../openhft/chronicle/map/ChronicleMap.java | 9 ++ .../chronicle/map/ChronicleMapBuilder.java | 20 +--- .../chronicle/map/StatelessChronicleMap.java | 10 ++ .../chronicle/map/VanillaChronicleMap.java | 35 +++++- .../chronicle/map/CHMUseCasesTest.java | 110 +++++++++++------- .../chronicle/map/ReplicationCheckingMap.java | 10 ++ .../jrs166/map/StatelessChronicleMapTest.java | 10 ++ 7 files changed, 140 insertions(+), 64 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java index 8482ae791..db2acbc35 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java @@ -247,5 +247,14 @@ public interface ChronicleMap extends ConcurrentMap, ChronicleHash { */ void putAll(File fromFile) throws IOException; + /** + * @return a new empty instance based on the Value type + */ + V newValueInstance(); + + /** + * @return a new empty instance based on the Key type + */ + K newKeyInstance(); } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index eb6ed4dee..21e2b5884 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -69,26 +69,8 @@ public final class ChronicleMapBuilder implements ChronicleMapBuilderI(builder); } - // todo - work in progress this will be changed me not today, so don't refactor this ! - public static T newInstance(Class interfaceClass, boolean isKey) { - if (interfaceClass.isEnum()) - return isKey ? DataValueClasses.newDirectReference(interfaceClass) : DataValueClasses - .newDirectInstance(interfaceClass); - - else if (!offHeapReference(interfaceClass) && !interfaceClass.isInterface() && - !builtInType(interfaceClass)) { - - // if (()) - // interfaceClass = DataValueClasses.directClassFor(interfaceClass); - return DataValueClasses.newInstance(interfaceClass); - - } - return DataValueClasses.newDirectInstance(interfaceClass); - - } - static boolean builtInType(Class clazz) { return clazz.getClassLoader() == Class.class.getClassLoader(); } @@ -97,7 +79,7 @@ public final class ChronicleMapBuilder implements ChronicleMapBuilderI implements ChronicleMap, Closeable, Clon throw new UnsupportedOperationException(); } + @Override + public V newValueInstance() { + return VanillaChronicleMap.newInstance(vClass, false); + } + + @Override + public K newKeyInstance() { + return VanillaChronicleMap.newInstance(kClass, true); + } + private ExecutorService lazyExecutorService() { if (executorService == null) { synchronized (this) { diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 2864f578f..3c3f538e9 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -37,6 +37,7 @@ import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.JDKObjectSerializer; import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; +import net.openhft.lang.model.DataValueClasses; import net.openhft.lang.threadlocal.Provider; import net.openhft.lang.threadlocal.StatefulCopyable; import net.openhft.lang.threadlocal.ThreadLocalCopies; @@ -58,6 +59,8 @@ import static java.lang.Math.max; import static java.lang.Thread.currentThread; import static java.nio.ByteBuffer.allocateDirect; import static net.openhft.chronicle.map.Asserts.assertNotNull; +import static net.openhft.chronicle.map.ChronicleMapBuilder.builtInType; +import static net.openhft.chronicle.map.ChronicleMapBuilder.offHeapReference; import static net.openhft.lang.MemoryUnit.*; class VanillaChronicleMap, @@ -559,7 +562,7 @@ class VanillaChronicleMap, } @Override - public V putMapped(K key, @NotNull UnaryOperator unaryOperator) { + public V putMapped(K key, @NotNull UnaryOperator unaryOperator) { final V using = (vClass.equals(CharSequence.class)) ? (V) new StringBuilder() : null; @@ -616,6 +619,36 @@ class VanillaChronicleMap, } } + @Override + public V newValueInstance() { + return newInstance(vClass, false); + } + + @Override + public K newKeyInstance() { + return newInstance(kClass, true); + } + + + static T newInstance(Class interfaceClass, boolean isKey) { + + + if (interfaceClass.isEnum()) + return isKey ? DataValueClasses.newDirectReference(interfaceClass) : DataValueClasses + .newDirectInstance(interfaceClass); + + else if (!offHeapReference(interfaceClass) && !interfaceClass.isInterface() && + !builtInType(interfaceClass)) { + + // if (()) + // interfaceClass = DataValueClasses.directClassFor(interfaceClass); + return DataValueClasses.newInstance(interfaceClass); + + } + return DataValueClasses.newDirectInstance(interfaceClass); + + } + private XStreamConverter xStreamConverter; /** diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index 616357008..4c8b3ddf3 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -156,10 +156,10 @@ public class CHMUseCasesTest { .of(StringValue.class, StringValue.class) .checkSerializedValues() // for testing purposes only .create()) { - StringValue key1 = ChronicleMapBuilder.newInstance(StringValue.class, true); - StringValue key2 = ChronicleMapBuilder.newInstance(StringValue.class, true); - StringValue value1 = ChronicleMapBuilder.newInstance(StringValue.class, false); - StringValue value2 = ChronicleMapBuilder.newInstance(StringValue.class, false); + StringValue key1 = map.newKeyInstance(); + StringValue key2 = map.newKeyInstance(); + StringValue value1 = map.newValueInstance(); + StringValue value2 = map.newValueInstance(); key1.setValue(new StringBuilder("1")); value1.setValue("11"); @@ -598,10 +598,15 @@ public class CHMUseCasesTest { .disableOversizedEntries() // disabled for testing purposes only. .entrySize(8) .create()) { - IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue value1 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue value2 = ChronicleMapBuilder.newInstance(IntValue.class, false); + IntValue key1 = map.newKeyInstance(); + IntValue key2 = map.newKeyInstance(); + IntValue value1 = map.newValueInstance(); + IntValue value2 = map.newValueInstance(); + + + + + key1.setValue(1); value1.setValue(11); @@ -680,10 +685,13 @@ public class CHMUseCasesTest { .disableOversizedEntries() // disabled for testing purposes only. .entrySize(8) .create()) { - UnsignedIntValue key1 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, true); - UnsignedIntValue key2 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, true); - UnsignedIntValue value1 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, false); - UnsignedIntValue value2 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, false); + + + + UnsignedIntValue key1 = map.newKeyInstance(); + UnsignedIntValue key2 = map.newKeyInstance(); + UnsignedIntValue value1 = map.newValueInstance(); + UnsignedIntValue value2 = map.newValueInstance(); key1.setValue(1); value1.setValue(11); @@ -762,10 +770,10 @@ public class CHMUseCasesTest { .disableOversizedEntries() // disabled for testing purposes only. .entrySize(6) .create()) { - IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); - ShortValue value1 = ChronicleMapBuilder.newInstance(ShortValue.class, false); - ShortValue value2 = ChronicleMapBuilder.newInstance(ShortValue.class, false); + IntValue key1 = map.newKeyInstance(); + IntValue key2 = map.newKeyInstance(); + ShortValue value1 = map.newValueInstance(); + ShortValue value2 = map.newValueInstance(); key1.setValue(1); value1.setValue((short) 11); @@ -844,10 +852,11 @@ public class CHMUseCasesTest { .entrySize(6) .disableOversizedEntries() // disabled for testing purposes only. .create()) { - IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); - UnsignedShortValue value1 = ChronicleMapBuilder.newInstance(UnsignedShortValue.class, false); - UnsignedShortValue value2 = ChronicleMapBuilder.newInstance(UnsignedShortValue.class, false); + + IntValue key1 = map.newKeyInstance(); + IntValue key2 = map.newKeyInstance(); + UnsignedShortValue value1 = map.newValueInstance(); + UnsignedShortValue value2 = map.newValueInstance(); key1.setValue(1); value1.setValue(11); @@ -926,10 +935,11 @@ public class CHMUseCasesTest { .disableOversizedEntries() // disabled for testing purposes only. .entrySize(6) .create()) { - IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); - CharValue value1 = ChronicleMapBuilder.newInstance(CharValue.class, false); - CharValue value2 = ChronicleMapBuilder.newInstance(CharValue.class, false); + + IntValue key1 = map.newKeyInstance(); + IntValue key2 = map.newKeyInstance(); + CharValue value1 = map.newValueInstance(); + CharValue value2 = map.newValueInstance(); key1.setValue(1); value1.setValue((char) 11); @@ -1008,10 +1018,14 @@ public class CHMUseCasesTest { .entrySize(5) .disableOversizedEntries() // disabled for testing purposes only. .create()) { - IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); - UnsignedByteValue value1 = ChronicleMapBuilder.newInstance(UnsignedByteValue.class, false); - UnsignedByteValue value2 = ChronicleMapBuilder.newInstance(UnsignedByteValue.class, false); + + + IntValue key1 = map.newKeyInstance(); + IntValue key2 = map.newKeyInstance(); + UnsignedByteValue value1 = map.newValueInstance(); + UnsignedByteValue value2 = map.newValueInstance(); + + key1.setValue(1); value1.setValue(11); @@ -1090,10 +1104,14 @@ public class CHMUseCasesTest { .disableOversizedEntries() // disabled for testing purposes only. .entrySize(5) .create()) { - IntValue key1 = ChronicleMapBuilder.newInstance(IntValue.class, false); - IntValue key2 = ChronicleMapBuilder.newInstance(IntValue.class, false); - BooleanValue value1 = ChronicleMapBuilder.newInstance(BooleanValue.class, false); - BooleanValue value2 = ChronicleMapBuilder.newInstance(BooleanValue.class, false); + + + + IntValue key1 = map.newKeyInstance(); + IntValue key2 = map.newKeyInstance(); + BooleanValue value1 = map.newValueInstance(); + BooleanValue value2 = map.newValueInstance(); + key1.setValue(1); value1.setValue(true); @@ -1172,10 +1190,12 @@ public class CHMUseCasesTest { .entrySize(8) .disableOversizedEntries() // disabled for testing purposes only. .create()) { - FloatValue key1 = ChronicleMapBuilder.newInstance(FloatValue.class, false); - FloatValue key2 = ChronicleMapBuilder.newInstance(FloatValue.class, false); - FloatValue value1 = ChronicleMapBuilder.newInstance(FloatValue.class, false); - FloatValue value2 = ChronicleMapBuilder.newInstance(FloatValue.class, false); + + FloatValue key1 = map.newKeyInstance(); + FloatValue key2 = map.newKeyInstance(); + FloatValue value1 = map.newValueInstance(); + FloatValue value2 = map.newValueInstance(); + key1.setValue(1); value1.setValue(11); @@ -1254,10 +1274,10 @@ public class CHMUseCasesTest { .entrySize(16) .create()) { - DoubleValue key1 = ChronicleMapBuilder.newInstance(DoubleValue.class, false); - DoubleValue key2 = ChronicleMapBuilder.newInstance(DoubleValue.class, false); - DoubleValue value1 = ChronicleMapBuilder.newInstance(DoubleValue.class, false); - DoubleValue value2 = ChronicleMapBuilder.newInstance(DoubleValue.class, false); + DoubleValue key1 = map.newKeyInstance(); + DoubleValue key2 = map.newKeyInstance(); + DoubleValue value1 = map.newValueInstance(); + DoubleValue value2 = map.newValueInstance(); key1.setValue(1); @@ -1339,10 +1359,12 @@ public class CHMUseCasesTest { .entrySize(16) .disableOversizedEntries() // disabled for testing purposes only. .create()) { - LongValue key1 = ChronicleMapBuilder.newInstance(LongValue.class, false); - LongValue key2 = ChronicleMapBuilder.newInstance(LongValue.class, false); - LongValue value1 = ChronicleMapBuilder.newInstance(LongValue.class, false); - LongValue value2 = ChronicleMapBuilder.newInstance(LongValue.class, false); + + LongValue key1 = map.newKeyInstance(); + LongValue key2 = map.newKeyInstance(); + LongValue value1 = map.newValueInstance(); + LongValue value2 = map.newValueInstance(); + key1.setValue(1); diff --git a/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java index 41c314780..a973809e1 100644 --- a/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java +++ b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java @@ -237,6 +237,16 @@ public class ReplicationCheckingMap implements ChronicleMap { throw new UnsupportedOperationException(); } + @Override + public V newValueInstance() { + throw new UnsupportedOperationException(); + } + + @Override + public K newKeyInstance() { + throw new UnsupportedOperationException(); + } + @Override public V acquireUsing(@NotNull K key, V usingValue) { return map1.acquireUsing(key, usingValue); diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java index e4ddc060f..7f5ddc084 100755 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java @@ -189,6 +189,16 @@ public class StatelessChronicleMapTest extends JSR166TestCase { d.putAll(fromFile); } + @Override + public V newValueInstance() { + throw new UnsupportedOperationException(); + } + + @Override + public K newKeyInstance() { + throw new UnsupportedOperationException(); + } + @Override public V putIfAbsent(K key, V value) { return d.putIfAbsent(key, value); From c5396b23e8776815d55bddd584cae695bd7aa7f2 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 17:49:00 +0000 Subject: [PATCH 08/21] HCOLL-211 Fixing a few tests to cover the use cases. --- .../map/AbstractChronicleMapBuilder.java | 16 +-- .../chronicle/map/ChronicleMapBuilder.java | 5 - .../chronicle/map/ChronicleMapBuilderI.java | 2 - .../chronicle/map/StatelessChronicleMap.java | 38 +++---- .../openhft/chronicle/map/TcpReplicator.java | 7 +- .../chronicle/map/VanillaChronicleMap.java | 103 +++++++++--------- .../chronicle/map/CHMUseCasesTest.java | 52 +++++---- .../chronicle/map/ConcurrentMap8Test.java | 2 +- .../map/ReplicatedChronicleMapTest.java | 2 +- .../map/TimeBasedReplicationTest.java | 2 +- .../{jrs166 => jsr166}/JSR166TestCase.java | 2 +- .../map/ChronicleMapTest.java | 4 +- .../{jrs166 => jsr166}/map/LoopHelpers.java | 2 +- .../map/{jrs166 => jsr166}/map/MapCheck.java | 2 +- .../map/StatelessChronicleMapTest.java | 4 +- 15 files changed, 124 insertions(+), 119 deletions(-) rename src/test/java/net/openhft/chronicle/map/{jrs166 => jsr166}/JSR166TestCase.java (99%) rename src/test/java/net/openhft/chronicle/map/{jrs166 => jsr166}/map/ChronicleMapTest.java (99%) rename src/test/java/net/openhft/chronicle/map/{jrs166 => jsr166}/map/LoopHelpers.java (99%) rename src/test/java/net/openhft/chronicle/map/{jrs166 => jsr166}/map/MapCheck.java (99%) rename src/test/java/net/openhft/chronicle/map/{jrs166 => jsr166}/map/StatelessChronicleMapTest.java (99%) diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java index 0b4002388..bee59229d 100755 --- a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java @@ -143,7 +143,6 @@ abstract class AbstractChronicleMapBuilder keyClass, Class valueClass) { @@ -935,7 +934,7 @@ abstract class AbstractChronicleMapBuilder> 10), - VanillaChronicleMap.MAX_ENTRY_OVERSIZE_FACTOR); + return (int) Math.max(4L, entries() >> 10); } @Override @@ -1006,16 +1004,14 @@ abstract class AbstractChronicleMapBuilder checkSerializedValues() { - this.checkSerializedValues = true; - return this; - } - @Override public ChronicleMapBuilderI disableOversizedEntries() { this.disableOversizedEntries = true; return this; } + + public int maxEntryOversizeFactor() { + return disableOversizedEntries ? 1 : VanillaChronicleMap.MAX_ENTRY_OVERSIZE_FACTOR; + } } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 21e2b5884..5dcee5383 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -522,11 +522,6 @@ public final class ChronicleMapBuilder implements ChronicleMapBuilderI checkSerializedValues() { - delegate.checkSerializedValues(); - return this; - } - public ChronicleMapBuilder disableOversizedEntries() { delegate.disableOversizedEntries(); return this; diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java index 454356f04..99cc3455b 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java @@ -315,7 +315,5 @@ public interface ChronicleMapBuilderI extends Serializable { String name(); - ChronicleMapBuilderI checkSerializedValues(); - ChronicleMapBuilderI disableOversizedEntries(); } diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index 4b4da89ac..23bd41e4a 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -47,7 +47,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; import static java.nio.ByteBuffer.allocateDirect; -import static net.openhft.chronicle.map.AbstractChannelReplicator.*; +import static net.openhft.chronicle.map.AbstractChannelReplicator.SIZE_OF_SIZE; +import static net.openhft.chronicle.map.AbstractChannelReplicator.SIZE_OF_TRANSACTION_ID; import static net.openhft.chronicle.map.StatelessChronicleMap.EventId.*; /** @@ -235,16 +236,14 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } /** - * attempts a single connect without a timeout and eat any IOException used typically in the - * constructor, its possible the server is not running with this instance is created, {@link - * net.openhft.chronicle.map.StatelessChronicleMap#lazyConnect(long, - * java.net.InetSocketAddress)} will attempt to establish the connection when the client make - * the first map method call. + * attempts a single connect without a timeout and eat any IOException used typically in the constructor, its + * possible the server is not running with this instance is created, {@link net.openhft.chronicle.map.StatelessChronicleMap#lazyConnect(long, + * java.net.InetSocketAddress)} will attempt to establish the connection when the client make the first map method + * call. * * @param remoteAddress the Inet Socket Address * @throws java.io.IOException - * @see net.openhft.chronicle.map.StatelessChronicleMap#lazyConnect(long, - * java.net.InetSocketAddress) + * @see net.openhft.chronicle.map.StatelessChronicleMap#lazyConnect(long, java.net.InetSocketAddress) */ private synchronized void attemptConnect(final InetSocketAddress remoteAddress) { @@ -273,9 +272,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } /** - * initiates a very simple level of handshaking with the remote server, we send a special ID of - * -127 ( when the server receives this it knows its dealing with a stateless client, receive - * back an identifier from the server + * initiates a very simple level of handshaking with the remote server, we send a special ID of -127 ( when the + * server receives this it knows its dealing with a stateless client, receive back an identifier from the server * * @param clientChannel clientChannel * @throws java.io.IOException @@ -341,10 +339,10 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon * @return a unique transactionId */ private long nextUniqueTransaction(long time) { - long id = time * 1000; + long id = time * TcpReplicator.TIMESTAMP_FACTOR; for (; ; ) { long old = transactionID.get(); - if (old > id) id = old + 1; + if (old >= id) id = old + 1; if (transactionID.compareAndSet(old, id)) break; } @@ -393,10 +391,9 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } /** - * calling this method should be avoided at all cost, as the entire {@code object} is - * serialized. This equals can be used to compare map that extends ChronicleMap. So two - * Chronicle Maps that contain the same data are considered equal, even if the instances of the - * chronicle maps were of different types + * calling this method should be avoided at all cost, as the entire {@code object} is serialized. This equals can be + * used to compare map that extends ChronicleMap. So two Chronicle Maps that contain the same data are considered + * equal, even if the instances of the chronicle maps were of different types * * @param object the object that you are comparing against * @return true if the contain the same data @@ -825,8 +822,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon // skips for the transaction id outBytes.skip(SIZE_OF_TRANSACTION_ID); - outBytes.skip(SIZE_OF_TIME_SHIFT); outBytes.writeByte(identifier); + outBytes.writeInt(0); return sizeLocation; } @@ -1015,8 +1012,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon final long transactionId0 = inBytes.readLong(); // check the transaction id is reasonable - assert transactionId0 > 1410000000000000L : "TransactionId too small " + transactionId0 + " messageSize " + messageSize; - assert transactionId0 < 2100000000000000L : "TransactionId too large " + transactionId0 + " messageSize " + messageSize; + assert transactionId0 > 1410000000000L * TcpReplicator.TIMESTAMP_FACTOR : "TransactionId too small " + transactionId0 + " messageSize " + messageSize; + assert transactionId0 < 2100000000000L * TcpReplicator.TIMESTAMP_FACTOR : "TransactionId too large " + transactionId0 + " messageSize " + messageSize; // if the transaction id is for this thread process it if (transactionId0 == transactionId) { @@ -1143,7 +1140,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon outBytes.writeInt(size0); assert transactionId != 0; outBytes.writeLong(transactionId); - outBytes.writeUnsignedShort((int) (transactionId - startTime)); outBytes.position(pos); } diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index dbb480ba7..88fcd6183 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -64,6 +64,7 @@ interface Work { */ final class TcpReplicator extends AbstractChannelReplicator implements Closeable { + public static final long TIMESTAMP_FACTOR = 10000; private static final int STATELESS_CLIENT = -127; private static final byte NOT_SET = (byte) HEARTBEAT.ordinal(); private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName()); @@ -1336,10 +1337,10 @@ class StatelessServerConnector { long transactionId = reader.readLong(); // the time stamp and the transaction are usually the same, or out by the shift - int timestampShift = reader.readUnsignedShort(); - long timestamp = transactionId - timestampShift; - + long timestamp = transactionId / TcpReplicator.TIMESTAMP_FACTOR; byte identifier = reader.readByte(); + int headerSize = reader.readInt(); + reader.skip(headerSize); // these methods don't return a result to the client or don't return a result to the diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 3c3f538e9..18b3d0379 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -36,8 +36,7 @@ import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.JDKObjectSerializer; -import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; -import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.io.serialization.ObjectSerializer; import net.openhft.lang.threadlocal.Provider; import net.openhft.lang.threadlocal.StatefulCopyable; import net.openhft.lang.threadlocal.ThreadLocalCopies; @@ -71,8 +70,7 @@ class VanillaChronicleMap, private static final Logger LOG = LoggerFactory.getLogger(VanillaChronicleMap.class); /** - * Because DirectBitSet implementations couldn't find more than 64 continuous clear or set - * bits. + * Because DirectBitSet implementations couldn't find more than 64 continuous clear or set bits. */ static final int MAX_ENTRY_OVERSIZE_FACTOR = 64; private static final long serialVersionUID = 2L; @@ -108,6 +106,7 @@ class VanillaChronicleMap, private final HashSplitting hashSplitting; final boolean isNativeValueClass; final MultiMapFactory multiMapFactory; + final int maxEntryOversizeFactor; transient Provider> keyReaderProvider; transient Provider keyInteropProvider; transient Provider> valueReaderProvider; @@ -116,7 +115,7 @@ class VanillaChronicleMap, // non-final for close() and because it is initialized out of constructor transient BytesStore ms; transient long headerSize; - private transient Set> entrySet; + transient Set> entrySet; public VanillaChronicleMap(AbstractChronicleMapBuilder builder) throws IOException { @@ -159,6 +158,7 @@ class VanillaChronicleMap, this.eventListener = builder.eventListener(); this.bytesEventListener = builder.bytesEventListener(); this.segmentHeaderSize = builder.segmentHeaderSize(); + this.maxEntryOversizeFactor = builder.maxEntryOversizeFactor(); hashSplitting = HashSplitting.Splitting.forSegments(actualSegments); initTransients(); @@ -564,33 +564,44 @@ class VanillaChronicleMap, @Override public V putMapped(K key, @NotNull UnaryOperator unaryOperator) { + return (vClass.equals(CharSequence.class) || vClass.equals(StringBuilder.class)) + ? putMappedSB(key, unaryOperator) + : putMapped0(key, unaryOperator); + } - final V using = (vClass.equals(CharSequence.class)) ? (V) new StringBuilder() : null; + V putMappedSB(K key, @NotNull UnaryOperator unaryOperator) { - try (WriteContext entry = lookupUsing(key, using, - false, false, LockType.WRITE_LOCK)) { + StringBuilder usingSB = new StringBuilder("\uFFFF"); + try (WriteContext entry = lookupUsing(key, (V) usingSB, + false, true, LockType.WRITE_LOCK)) { - if (CharSequence.class.isAssignableFrom(vClass)) { - CharSequence value = (CharSequence) entry.value(); - if (entry.value() == null || value.length() == 0) - return null; + usingSB = (StringBuilder) entry.value(); + if (usingSB.length() == 1 && usingSB.charAt(0) == '\uFFFF') + return null; + + V result = unaryOperator.update((V) usingSB); + if (usingSB != result) { + usingSB.setLength(0); + usingSB.append((CharSequence) result); } + return result; + } + } + + V putMapped0(K key, @NotNull UnaryOperator unaryOperator) { + try (WriteContext entry = lookupUsing(key, null, + false, true, LockType.WRITE_LOCK)) { if (entry.value() == null) return null; V result = unaryOperator.update(entry.value()); - if (entry.value() instanceof StringBuilder) { - ((StringBuilder) using).setLength(0); - ((StringBuilder) using).append((CharSequence) result); - } else - ((MutableLockedEntry) entry).value(result); + ((MutableLockedEntry) entry).value(result); return result; } - - } + @Override public synchronized void getAll(File toFile) throws IOException { @@ -1040,9 +1051,9 @@ class VanillaChronicleMap, } /** - * removes ( if there exists ) an entry from the map, if the {@param key} and {@param - * expectedValue} match that of a maps.entry. If the {@param expectedValue} equals null then ( - * if there exists ) an entry whose key equals {@param key} this is removed. + * removes ( if there exists ) an entry from the map, if the {@param key} and {@param expectedValue} match that of a + * maps.entry. If the {@param expectedValue} equals null then ( if there exists ) an entry whose key equals {@param + * key} this is removed. * * @param k the key of the entry to remove * @param expectedValue null if not required @@ -1174,8 +1185,8 @@ class VanillaChronicleMap, * replace the value in a map, only if the existing entry equals {@param expectedValue} * * @param key the key into the map - * @param expectedValue the expected existing value in the map ( could be null when we don't - * wish to do this check ) + * @param expectedValue the expected existing value in the map ( could be null when we don't wish to do this check + * ) * @param newValue the new value you wish to store in the map * @return the value that was replaced */ @@ -1501,11 +1512,11 @@ class VanillaChronicleMap, private MultiMap createMultiMap(long start) { final NativeBytes multiMapBytes = - new NativeBytes(new VanillaBytesMarshallerFactory(), start, + new NativeBytes((ObjectSerializer) null, start, start = start + sizeOfMultiMap(), null); final NativeBytes sizeOfMultiMapBitSetBytes = - new NativeBytes(new VanillaBytesMarshallerFactory(), start, + new NativeBytes((ObjectSerializer) null, start, start + sizeOfMultiMapBitSet(), null); // multiMapBytes.load(); return multiMapFactory.create(multiMapBytes, sizeOfMultiMapBitSetBytes); @@ -1930,9 +1941,9 @@ class VanillaChronicleMap, } /** - * Returns value size, writes the entry (key, value, sizes) to the entry, after this method - * call entry positioned after value bytes written (i. e. at the end of entry), sets entry - * position (in segment) and value size position in the given segmentState + * Returns value size, writes the entry (key, value, sizes) to the entry, after this method call entry + * positioned after value bytes written (i. e. at the end of entry), sets entry position (in segment) and value + * size position in the given segmentState */ final , E, EW> long putEntry(SegmentState segmentState, @@ -1975,9 +1986,9 @@ class VanillaChronicleMap, //TODO refactor/optimize private long alloc(int blocks) { - if (blocks > MAX_ENTRY_OVERSIZE_FACTOR) + if (blocks > maxEntryOversizeFactor) throw new IllegalArgumentException("Entry is too large: requires " + blocks + - " entry size chucks, " + MAX_ENTRY_OVERSIZE_FACTOR + " is maximum."); + " entry size chucks, " + maxEntryOversizeFactor + " is maximum."); long ret = freeList.setNextNContinuousClearBits(nextPosToSearchFrom, blocks); if (ret == DirectBitSet.NOT_FOUND || ret + blocks > entriesPerSegment) { if (ret + blocks > entriesPerSegment) @@ -2056,10 +2067,9 @@ class VanillaChronicleMap, } /** - * - if expectedValue is not null, returns Boolean.TRUE (removed) or Boolean.FALSE (entry - * not found), regardless the expectedValue object is Bytes instance (RPC call) or the value - * instance - if expectedValue is null: - if resultUnused is false, null or removed value is - * returned - if resultUnused is true, null is always returned + * - if expectedValue is not null, returns Boolean.TRUE (removed) or Boolean.FALSE (entry not found), regardless + * the expectedValue object is Bytes instance (RPC call) or the value instance - if expectedValue is null: - if + * resultUnused is false, null or removed value is returned - if resultUnused is true, null is always returned */ , RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> @@ -2203,9 +2213,8 @@ class VanillaChronicleMap, } /** - * Replaces the specified value for the key with the given value. {@code newValue} is set - * only if the existing value corresponding to the specified key is equal to {@code - * expectedValue} or {@code expectedValue == null}. + * Replaces the specified value for the key with the given value. {@code newValue} is set only if the existing + * value corresponding to the specified key is equal to {@code expectedValue} or {@code expectedValue == null}. * * @param hash2 a hash code related to the {@code keyBytes} * @return the replaced value or {@code null} if the value was not replaced @@ -2303,17 +2312,15 @@ class VanillaChronicleMap, } /** - * Replaces value in existing entry. May cause entry relocation, because there may be not - * enough space for new value in location already allocated for this entry. + * Replaces value in existing entry. May cause entry relocation, because there may be not enough space for new + * value in location already allocated for this entry. * * @param pos index of the first block occupied by the entry - * @param offset relative offset of the entry in Segment bytes (before, i. e. - * including metaData) + * @param offset relative offset of the entry in Segment bytes (before, i. e. including metaData) * @param entry relative pointer in Segment bytes * @param valueSizePos relative position of value size in entry * @param entryEndAddr absolute address of the entry end - * @return relative offset of the entry in Segment bytes after putting value (that may cause - * entry relocation) + * @return relative offset of the entry in Segment bytes after putting value (that may cause entry relocation) */ final long putValue( long pos, long offset, MultiStoreBytes entry, long valueSizePos, long entryEndAddr, @@ -2332,10 +2339,10 @@ class VanillaChronicleMap, int oldSizeInBlocks = inBlocks(oldEntrySize); int newSizeInBlocks = inBlocks(newEntryEndAddr - entryStartAddr); if (newSizeInBlocks > oldSizeInBlocks) { - if (newSizeInBlocks > MAX_ENTRY_OVERSIZE_FACTOR) { + if (newSizeInBlocks > maxEntryOversizeFactor) { throw new IllegalArgumentException("Value too large: " + "entry takes " + newSizeInBlocks + " blocks, " + - MAX_ENTRY_OVERSIZE_FACTOR + " is maximum."); + maxEntryOversizeFactor + " is maximum."); } if (realloc(pos, oldSizeInBlocks, newSizeInBlocks)) break newValueDoesNotFit; @@ -2664,7 +2671,6 @@ class VanillaChronicleMap, private K key; private V value; - boolean present; MutableLockedEntry(SegmentState segmentState) { this.segmentState = segmentState; @@ -2683,7 +2689,6 @@ class VanillaChronicleMap, } void value(V value) { - this.present = value != null; this.value = value; } @@ -2709,7 +2714,7 @@ class VanillaChronicleMap, * @return if the value is not null */ public boolean present() { - return present; + return value() != null; } } diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index 4c8b3ddf3..328a4ceea 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -2,6 +2,7 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.serialization.impl.*; import net.openhft.lang.values.*; +import org.junit.Ignore; import org.junit.Test; import java.nio.ByteBuffer; @@ -18,8 +19,8 @@ import static org.junit.Assert.*; public class CHMUseCasesTest { /** - * String is not as efficient as CharSequence as a key or value but easier to use The key can - * only be on heap and variable length serialised. + * String is not as efficient as CharSequence as a key or value but easier to use The key can only be on heap and + * variable length serialised. */ @Test public void testStringStringMap() throws ExecutionException, InterruptedException { @@ -32,8 +33,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = chmb.create()) { */ try (ChronicleMap map = ChronicleMapBuilder - .of(String.class, String.class) - .checkSerializedValues() // for testing purposes only + .of(String.class, String.class) // for testing purposes only .create()) { map.put("Hello", "World"); assertEquals("World", map.get("Hello")); @@ -67,8 +67,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = chmb.create()) { */ try (ChronicleMap map = ChronicleMapBuilder - .of(String.class, String.class) - .checkSerializedValues() // for testing purposes only + .of(String.class, String.class) // for testing purposes only .create()) { map.put("Hello", "World"); @@ -79,20 +78,18 @@ public class CHMUseCasesTest { return "New " + s; } }); - - } } /** - * CharSequence is more efficient when object creation is avoided. The key can only be on heap - * and variable length serialised. + * CharSequence is more efficient when object creation is avoided. The key can only be on heap and variable length + * serialised. */ @Test public void testCharSequenceCharSequenceMap() throws ExecutionException, InterruptedException { try (ChronicleMap map = ChronicleMapBuilder - .of(CharSequence.class, CharSequence.class) - .checkSerializedValues() // for testing purposes only + .of(CharSequence.class, CharSequence.class) // for testing purposes only + .defaultValue("") .create()) { map.put("Hello", "World"); StringBuilder key = new StringBuilder(); @@ -135,7 +132,7 @@ public class CHMUseCasesTest { assertEquals("New World !!", map.get("Hello").toString()); - assertEquals(null, map.putMapped("no-key", new + assertEquals("New !!", map.putMapped("no-key", new UnaryOperator() { @Override public CharSequence update(CharSequence s) { @@ -153,8 +150,7 @@ public class CHMUseCasesTest { @Test public void testStringValueStringValueMap() { try (ChronicleMap map = ChronicleMapBuilder - .of(StringValue.class, StringValue.class) - .checkSerializedValues() // for testing purposes only + .of(StringValue.class, StringValue.class) // for testing purposes only .create()) { StringValue key1 = map.newKeyInstance(); StringValue key2 = map.newKeyInstance(); @@ -437,7 +433,8 @@ public class CHMUseCasesTest { .keyMarshaller(ByteBufferMarshaller.INSTANCE) .valueMarshaller(ByteBufferMarshaller.INSTANCE) .disableOversizedEntries() // disabled for testing purposes only. - .entrySize(12) + // TODO does this have to be 14 bytes? + .entrySize(14) .create()) { ByteBuffer key1 = ByteBuffer.wrap(new byte[]{1, 1, 1, 1}); ByteBuffer key2 = ByteBuffer.wrap(new byte[]{2, 2, 2, 2}); @@ -510,7 +507,6 @@ public class CHMUseCasesTest { try (WriteContext wc = map.acquireUsingLocked(key1, valueB)) { - assertBBEquals(value1, valueB); appendMode(valueB); valueB.putShort((short) 12345); @@ -675,6 +671,26 @@ public class CHMUseCasesTest { } } + @Test + @Ignore("Generated code creates a field too large ie. it ignores the @Range") + public void testUnsignedIntValueUnsignedIntValueMapEntrySize() { + // TODO once this is working, merge the next test. + try (ChronicleMap map = ChronicleMapBuilder + .of(UnsignedIntValue.class, UnsignedIntValue.class) + .disableOversizedEntries() // disabled for testing purposes only. + .entrySize(8) + .create()) { + UnsignedIntValue key1 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, true); + UnsignedIntValue value1 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, false); + + key1.setValue(1); + value1.setValue(11); + map.put(key1, value1); + assertEquals(value1, map.get(key1)); + + } + } + /** * For unsigned int -> unsigned int entries, the key can be on heap or off heap. */ @@ -682,8 +698,6 @@ public class CHMUseCasesTest { public void testUnsignedIntValueUnsignedIntValueMap() { try (ChronicleMap map = ChronicleMapBuilder .of(UnsignedIntValue.class, UnsignedIntValue.class) - .disableOversizedEntries() // disabled for testing purposes only. - .entrySize(8) .create()) { diff --git a/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java b/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java index b98e46e2b..fb7a3f2bb 100644 --- a/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java +++ b/src/test/java/net/openhft/chronicle/map/ConcurrentMap8Test.java @@ -27,7 +27,7 @@ package net.openhft.chronicle.map; /*import junit.framework.Test; import junit.framework.TestSuite;*/ -import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import net.openhft.chronicle.map.jsr166.JSR166TestCase; import java.util.concurrent.ConcurrentMap; diff --git a/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java index 6e7174d7c..eaee51f2c 100644 --- a/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/ReplicatedChronicleMapTest.java @@ -18,7 +18,7 @@ package net.openhft.chronicle.map; -import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import net.openhft.chronicle.map.jsr166.JSR166TestCase; import org.junit.Test; import java.io.IOException; diff --git a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java index 1919c861a..1bee40e87 100644 --- a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java @@ -19,7 +19,7 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.hash.replication.TimeProvider; -import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import net.openhft.chronicle.map.jsr166.JSR166TestCase; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/JSR166TestCase.java b/src/test/java/net/openhft/chronicle/map/jsr166/JSR166TestCase.java similarity index 99% rename from src/test/java/net/openhft/chronicle/map/jrs166/JSR166TestCase.java rename to src/test/java/net/openhft/chronicle/map/jsr166/JSR166TestCase.java index 4471e2ab5..529a4bf17 100644 --- a/src/test/java/net/openhft/chronicle/map/jrs166/JSR166TestCase.java +++ b/src/test/java/net/openhft/chronicle/map/jsr166/JSR166TestCase.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package net.openhft.chronicle.map.jrs166; +package net.openhft.chronicle.map.jsr166; import junit.framework.AssertionFailedError; import org.junit.After; diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jsr166/map/ChronicleMapTest.java similarity index 99% rename from src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java rename to src/test/java/net/openhft/chronicle/map/jsr166/map/ChronicleMapTest.java index 5024f490b..3e4ef6fdb 100644 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jsr166/map/ChronicleMapTest.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package net.openhft.chronicle.map.jrs166.map; +package net.openhft.chronicle.map.jsr166.map; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; -import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import net.openhft.chronicle.map.jsr166.JSR166TestCase; import org.junit.Test; import java.io.IOException; diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/LoopHelpers.java b/src/test/java/net/openhft/chronicle/map/jsr166/map/LoopHelpers.java similarity index 99% rename from src/test/java/net/openhft/chronicle/map/jrs166/map/LoopHelpers.java rename to src/test/java/net/openhft/chronicle/map/jsr166/map/LoopHelpers.java index 04e1a3e8a..3ace7f0f1 100644 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/LoopHelpers.java +++ b/src/test/java/net/openhft/chronicle/map/jsr166/map/LoopHelpers.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package net.openhft.chronicle.map.jrs166.map; +package net.openhft.chronicle.map.jsr166.map; /* * Written by Doug Lea with assistance from members of JCP JSR-166 diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java b/src/test/java/net/openhft/chronicle/map/jsr166/map/MapCheck.java similarity index 99% rename from src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java rename to src/test/java/net/openhft/chronicle/map/jsr166/map/MapCheck.java index 6f065768f..9682814f7 100644 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/MapCheck.java +++ b/src/test/java/net/openhft/chronicle/map/jsr166/map/MapCheck.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package net.openhft.chronicle.map.jrs166.map; +package net.openhft.chronicle.map.jsr166.map; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jsr166/map/StatelessChronicleMapTest.java similarity index 99% rename from src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java rename to src/test/java/net/openhft/chronicle/map/jsr166/map/StatelessChronicleMapTest.java index 7f5ddc084..12f69defa 100755 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jsr166/map/StatelessChronicleMapTest.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package net.openhft.chronicle.map.jrs166.map; +package net.openhft.chronicle.map.jsr166.map; import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; import net.openhft.chronicle.map.*; -import net.openhft.chronicle.map.jrs166.JSR166TestCase; +import net.openhft.chronicle.map.jsr166.JSR166TestCase; import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Before; From 698a49d068457217e89772a2a4fc0c2da4ba0443 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 17:52:32 +0000 Subject: [PATCH 09/21] HCOLL-211 Fixing a few tests to cover the use cases. --- .../java/net/openhft/chronicle/map/VanillaChronicleMap.java | 5 +++-- src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 18b3d0379..af5314f64 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -37,6 +37,7 @@ import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.JDKObjectSerializer; import net.openhft.lang.io.serialization.ObjectSerializer; +import net.openhft.lang.model.DataValueClasses; import net.openhft.lang.threadlocal.Provider; import net.openhft.lang.threadlocal.StatefulCopyable; import net.openhft.lang.threadlocal.ThreadLocalCopies; @@ -645,8 +646,8 @@ class VanillaChronicleMap, if (interfaceClass.isEnum()) - return isKey ? DataValueClasses.newDirectReference(interfaceClass) : DataValueClasses - .newDirectInstance(interfaceClass); + return isKey ? DataValueClasses.newDirectReference(interfaceClass) + : DataValueClasses.newDirectInstance(interfaceClass); else if (!offHeapReference(interfaceClass) && !interfaceClass.isInterface() && !builtInType(interfaceClass)) { diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index 328a4ceea..4860e2903 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -680,8 +680,8 @@ public class CHMUseCasesTest { .disableOversizedEntries() // disabled for testing purposes only. .entrySize(8) .create()) { - UnsignedIntValue key1 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, true); - UnsignedIntValue value1 = ChronicleMapBuilder.newInstance(UnsignedIntValue.class, false); + UnsignedIntValue key1 = map.newKeyInstance(); + UnsignedIntValue value1 = map.newValueInstance(); key1.setValue(1); value1.setValue(11); From 64178023aa69d2125aaf3bb838190060049d2f41 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 18:09:56 +0000 Subject: [PATCH 10/21] HCOLL-228 Add support for .json.gz files. --- .../chronicle/map/VanillaChronicleMap.java | 12 ++++++-- .../chronicle/map/LargeEntriesTest.java | 29 +++++++++++++------ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index af5314f64..47a293e89 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -52,6 +52,8 @@ import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import static java.lang.Class.forName; import static java.lang.Long.numberOfTrailingZeros; @@ -612,7 +614,10 @@ class VanillaChronicleMap, // ideally we will use java serialization if we can lazyXStreamConverter().registerConverter(xstream); - try (FileOutputStream out = new FileOutputStream(toFile)) { + OutputStream outputStream = new FileOutputStream(toFile); + if (toFile.getName().toLowerCase().endsWith(".gz")) + outputStream = new GZIPOutputStream(outputStream); + try (OutputStream out = outputStream) { xstream.toXML(entrySet(), out); } } @@ -626,7 +631,10 @@ class VanillaChronicleMap, // ideally we will use java serialization if we can lazyXStreamConverter().registerConverter(xstream); - try (FileInputStream out = new FileInputStream(fromFile)) { + InputStream inputStream = new FileInputStream(fromFile); + if (fromFile.getName().toLowerCase().endsWith(".gz")) + inputStream = new GZIPInputStream(inputStream); + try (InputStream out = inputStream) { xstream.fromXML(out); } } diff --git a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java index 12b73c8c2..e84939acf 100644 --- a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java +++ b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java @@ -32,16 +32,16 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; /** * Created by peter on 06/12/14. */ public class LargeEntriesTest { - static final int ENTRIES = 640; + static final int ENTRIES = 6400; static final int ENTRY_SIZE = 1024; + private static final boolean TESTING = Boolean.getBoolean("testing"); @Test @Ignore @@ -55,7 +55,7 @@ public class LargeEntriesTest { .putReturnsNull(true) .create(); { - int threads = 2; //Runtime.getRuntime().availableProcessors(); + int threads = 4; //Runtime.getRuntime().availableProcessors(); ExecutorService es = Executors.newFixedThreadPool(threads); final int block = ENTRIES / threads; for (int i = 0; i < 3; i++) { @@ -124,11 +124,20 @@ public class LargeEntriesTest { SnappyStringMarshaller.INSTANCE.read(bytes); bytes.release(); } + Object lock = TESTING ? Thread.currentThread() : map; + List keys = new ArrayList<>(); for (int i = start; i < finish; i++) { String key = "key-" + i; - map.put(key, value); - String object = map.get(key); + String object; + synchronized (lock) { + map.put(key, value); + keys.add(key); + for (String s : keys) + assertTrue(s, map.containsKey(s)); + object = map.get(key); + assertTrue(key, map.containsKey(key)); + } assertNotNull(key, object); assertEquals(key, entrySize, object.length()); @@ -138,10 +147,12 @@ public class LargeEntriesTest { for (int i = start; i < finish; i++) { // System.out.println(i); String key = "key-" + i; - String object = map.get(key); + synchronized (lock) { + String object = map.get(key); - assertNotNull(key, object); - assertEquals(key, entrySize, object.length()); + assertNotNull(key, object); + assertEquals(key, entrySize, object.length()); + } } } } From 3a36ee0d7be982458426afc605c0db4dda4e83da Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Wed, 10 Dec 2014 18:45:07 +0000 Subject: [PATCH 11/21] updated to latest release of lang --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 55df114de..ce973db4a 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ net.openhft lang - 6.4.16-SNAPSHOT + 6.4.16 From 692edb3cebec4fd106f1fbf4e99b40b674eb364b Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 18:47:37 +0000 Subject: [PATCH 12/21] Remove an unused field and parameter --- .../openhft/chronicle/map/ReplicatedChronicleMap.java | 2 +- .../openhft/chronicle/map/VanillaChronicleMap.java | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 4aa7a37d6..24f960bd5 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -718,7 +718,7 @@ final class ReplicatedChronicleMap, implements ChronicleMap, Serializable, ReadValue, InstanceOrBytesToInstance, GetValueInterops { - private static final Logger LOG = LoggerFactory.getLogger(VanillaChronicleMap.class); +// private static final Logger LOG = LoggerFactory.getLogger(VanillaChronicleMap.class); /** * Because DirectBitSet implementations couldn't find more than 64 continuous clear or set bits. @@ -607,7 +605,6 @@ class VanillaChronicleMap, @Override public synchronized void getAll(File toFile) throws IOException { - final XStream xstream = new XStream(new JettisonMappedXmlDriver()); xstream.setMode(XStream.NO_REFERENCES); xstream.alias("chronicle-entries", VanillaChronicleMap.EntrySet.class); @@ -1926,7 +1923,7 @@ class VanillaChronicleMap, } // putValue may relocate entry and change offset - putValue(pos, offset, entry, valueSizePos, entryEndAddr, copies, segmentState, + putValue(pos, offset, entry, valueSizePos, entryEndAddr, segmentState, metaValueInterop, valueInterop, value, valueSize, searchedHashLookup); // put callbacks @@ -2303,7 +2300,7 @@ class VanillaChronicleMap, MVBI metaValueInterop = getNewValueInterops.getMetaValueInterop( copies, valueInterop, newValue); long newValueSize = metaValueInterop.size(valueInterop, newValue); - putValue(pos, offset, entry, valueSizePos, entryEndAddr, copies, segmentState, + putValue(pos, offset, entry, valueSizePos, entryEndAddr, segmentState, metaValueInterop, valueInterop, newValue, newValueSize, searchedHashLookup); // put callbacks @@ -2333,7 +2330,7 @@ class VanillaChronicleMap, */ final long putValue( long pos, long offset, MultiStoreBytes entry, long valueSizePos, long entryEndAddr, - ThreadLocalCopies copies, SegmentState segmentState, + SegmentState segmentState, MetaBytesWriter metaElemWriter, EW elemWriter, E newElem, long newElemSize, MultiMap searchedHashLookup) { From 66ab33cfe7ad9f01e7bb2a7f876bcd3e00ded8e2 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 10 Dec 2014 18:49:42 +0000 Subject: [PATCH 13/21] [maven-release-plugin] prepare release chronicle-map-2.0.12b --- pom.xml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index ce973db4a..5b23ff18e 100644 --- a/pom.xml +++ b/pom.xml @@ -17,20 +17,18 @@ ~ limitations under the License. --> - + net.openhft java-parent-pom 1.1.2 - + 4.0.0 chronicle-map - 2.0.12b-SNAPSHOT + 2.0.12b OpenHFT/Chronicle-Map Chronicle-Map @@ -181,7 +179,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - master + chronicle-map-2.0.12b From 024547864d69ff91d0170e0a4fbc08af871f7070 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 10 Dec 2014 18:49:47 +0000 Subject: [PATCH 14/21] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5b23ff18e..7092ad641 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 4.0.0 chronicle-map - 2.0.12b + 2.0.13b-SNAPSHOT OpenHFT/Chronicle-Map Chronicle-Map @@ -179,7 +179,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - chronicle-map-2.0.12b + master From 628e2c610025bae955d30996a83d1f8a8adddbee Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 19:06:16 +0000 Subject: [PATCH 15/21] HCOLL-231 Use only write locking while we solve a concurrency issue. --- .../chronicle/map/VanillaChronicleMap.java | 28 +++++++++++++--- .../chronicle/map/LargeEntriesTest.java | 32 ++++++++----------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 0243639c0..4038e986e 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -34,7 +34,6 @@ import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.chronicle.hash.serialization.internal.*; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; -import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.JDKObjectSerializer; import net.openhft.lang.io.serialization.ObjectSerializer; import net.openhft.lang.model.DataValueClasses; @@ -50,6 +49,7 @@ import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -1477,7 +1477,7 @@ class VanillaChronicleMap, */ static final long LOCK_OFFSET = 0L; // 64-bit static final long SIZE_OFFSET = LOCK_OFFSET + 8L; // 32-bit - + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); final NativeBytes segmentHeader; final NativeBytes bytes; final long entriesOffset; @@ -1566,7 +1566,16 @@ class VanillaChronicleMap, public final ReadLocked readLock( @Nullable SegmentState segmentState) { while (true) { - final boolean success = segmentHeader.tryRWReadLock(LOCK_OFFSET, lockTimeOutNS); +// final boolean success = segmentHeader.tryRWReadLock(LOCK_OFFSET, lockTimeOutNS); + final boolean success = segmentHeader.tryRWWriteLock(LOCK_OFFSET, lockTimeOutNS); +/* + boolean success = false; + try { + success = lock.readLock().tryLock(lockTimeOutNS, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } +*/ if (success) { if (segmentState != null) { return segmentState.readLocked(this); @@ -1592,6 +1601,14 @@ class VanillaChronicleMap, startWriteLock = System.nanoTime(); while (true) { final boolean success = segmentHeader.tryRWWriteLock(LOCK_OFFSET, lockTimeOutNS); +/* + boolean success = false; + try { + success = lock.writeLock().tryLock(lockTimeOutNS, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } +*/ if (success) { if (segmentState != null) { if (isNativeValueClass) { @@ -1617,7 +1634,9 @@ class VanillaChronicleMap, @Override public final void readUnlock() { try { - segmentHeader.unlockRWReadLock(LOCK_OFFSET); +// segmentHeader.unlockRWReadLock(LOCK_OFFSET); + segmentHeader.unlockRWWriteLock(LOCK_OFFSET); +// lock.readLock().unlock(); } catch (IllegalMonitorStateException e) { errorListener.errorOnUnlock(e); } @@ -1627,6 +1646,7 @@ class VanillaChronicleMap, public final void writeUnlock() { try { segmentHeader.unlockRWWriteLock(LOCK_OFFSET); +// lock.writeLock().unlock(); } catch (IllegalMonitorStateException e) { errorListener.errorOnUnlock(e); } diff --git a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java index e84939acf..20421483f 100644 --- a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java +++ b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java @@ -26,6 +26,8 @@ import org.junit.Test; import java.beans.XMLEncoder; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -40,12 +42,13 @@ import static org.junit.Assert.*; public class LargeEntriesTest { static final int ENTRIES = 6400; - static final int ENTRY_SIZE = 1024; + static final int ENTRY_SIZE = 102400; private static final boolean TESTING = Boolean.getBoolean("testing"); @Test @Ignore - public void testLargeStrings() throws ExecutionException, InterruptedException { + public void testLargeStrings() throws ExecutionException, InterruptedException, IOException { + File file = File.createTempFile("largeEntries", ".deleteme"); final ChronicleMap map = ChronicleMapBuilder .of(String.class, String.class) .valueMarshaller(SnappyStringMarshaller.INSTANCE) @@ -53,7 +56,7 @@ public class LargeEntriesTest { .entries(ENTRIES * 2) .entrySize(ENTRY_SIZE / 4) .putReturnsNull(true) - .create(); + .createPersistedTo(file); { int threads = 4; //Runtime.getRuntime().availableProcessors(); ExecutorService es = Executors.newFixedThreadPool(threads); @@ -124,20 +127,14 @@ public class LargeEntriesTest { SnappyStringMarshaller.INSTANCE.read(bytes); bytes.release(); } - Object lock = TESTING ? Thread.currentThread() : map; - List keys = new ArrayList<>(); for (int i = start; i < finish; i++) { String key = "key-" + i; String object; - synchronized (lock) { - map.put(key, value); - keys.add(key); - for (String s : keys) - assertTrue(s, map.containsKey(s)); - object = map.get(key); - assertTrue(key, map.containsKey(key)); - } + + map.put(key, value); + object = map.get(key); + assertTrue(key, map.containsKey(key)); assertNotNull(key, object); assertEquals(key, entrySize, object.length()); @@ -147,12 +144,11 @@ public class LargeEntriesTest { for (int i = start; i < finish; i++) { // System.out.println(i); String key = "key-" + i; - synchronized (lock) { - String object = map.get(key); - assertNotNull(key, object); - assertEquals(key, entrySize, object.length()); - } + String object = map.get(key); + + assertNotNull(key, object); + assertEquals(key, entrySize, object.length()); } } } From e71dfa9734a495ae2202fe990e43ed07816fbc94 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Wed, 10 Dec 2014 19:13:31 +0000 Subject: [PATCH 16/21] HCOLL-231 Use only write locking while we solve a concurrency issue. --- src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 4038e986e..48a8219c9 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -34,6 +34,7 @@ import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.chronicle.hash.serialization.internal.*; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; +import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.JDKObjectSerializer; import net.openhft.lang.io.serialization.ObjectSerializer; import net.openhft.lang.model.DataValueClasses; From e5323b6195f77be8aaa2ba296173051db8f5200d Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 10 Dec 2014 19:19:33 +0000 Subject: [PATCH 17/21] [maven-release-plugin] prepare release chronicle-map-2.0.13b --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7092ad641..5e4e5f3a9 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 4.0.0 chronicle-map - 2.0.13b-SNAPSHOT + 2.0.13b OpenHFT/Chronicle-Map Chronicle-Map @@ -179,7 +179,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - master + chronicle-map-2.0.13b From ab8f1148c65c6d9971e791b8532e9591ca4a5fd1 Mon Sep 17 00:00:00 2001 From: Team City Date: Wed, 10 Dec 2014 19:19:38 +0000 Subject: [PATCH 18/21] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5e4e5f3a9..37a3dae7b 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 4.0.0 chronicle-map - 2.0.13b + 2.0.14b-SNAPSHOT OpenHFT/Chronicle-Map Chronicle-Map @@ -179,7 +179,7 @@ scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git scm:git:git@github.com:OpenHFT/Chronicle-Map.git - chronicle-map-2.0.13b + master From 41eef849206bb5b4da58a9554b08a05b4f1d0fc5 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Thu, 11 Dec 2014 05:19:55 +0000 Subject: [PATCH 19/21] Update lang and chronicle-map snapshots --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9e26588df..d3243d8a1 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ and define the snapshot version in your pom.xml, for example: net.openhft chronicle-map - 1.0.1-SNAPSHOT + ``` diff --git a/pom.xml b/pom.xml index 37a3dae7b..22fc570c0 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ net.openhft lang - 6.4.16 + 6.4.17-SNAPSHOT From 356e6c8181d0d6baf9f76a6b9dfc7a4e1112c275 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Thu, 11 Dec 2014 07:46:05 +0000 Subject: [PATCH 20/21] Improve support for large entries --- .../chronicle/map/LargeEntriesTest.java | 112 +++++++++++++----- 1 file changed, 85 insertions(+), 27 deletions(-) diff --git a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java index 20421483f..955e9c1b9 100644 --- a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java +++ b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java @@ -41,23 +41,23 @@ import static org.junit.Assert.*; */ public class LargeEntriesTest { - static final int ENTRIES = 6400; - static final int ENTRY_SIZE = 102400; - private static final boolean TESTING = Boolean.getBoolean("testing"); - @Test - @Ignore public void testLargeStrings() throws ExecutionException, InterruptedException, IOException { + final int ENTRIES = 250; + final int ENTRY_SIZE = 100 * 1024; + File file = File.createTempFile("largeEntries", ".deleteme"); + file.deleteOnExit(); final ChronicleMap map = ChronicleMapBuilder .of(String.class, String.class) .valueMarshaller(SnappyStringMarshaller.INSTANCE) .actualSegments(1) // to force an error. - .entries(ENTRIES * 2) - .entrySize(ENTRY_SIZE / 4) + .entries(ENTRIES) + .entrySize(ENTRY_SIZE / 6) .putReturnsNull(true) .createPersistedTo(file); { + warmUpCompression(ENTRY_SIZE); int threads = 4; //Runtime.getRuntime().availableProcessors(); ExecutorService es = Executors.newFixedThreadPool(threads); final int block = ENTRIES / threads; @@ -66,11 +66,10 @@ public class LargeEntriesTest { List> futureList = new ArrayList<>(); for (int t = 0; t < threads; t++) { final int finalT = t; - final int finalI = i; futureList.add(es.submit(new Runnable() { @Override public void run() { - exerciseLargeStrings(map, finalI, finalT * block, finalT * block + block, ENTRY_SIZE); + exerciseLargeStrings(map, finalT * block, finalT * block + block, ENTRY_SIZE); } })); } @@ -87,7 +86,70 @@ public class LargeEntriesTest { } } - void exerciseLargeStrings(ChronicleMap map, int run, int start, int finish, int entrySize) { + private void warmUpCompression(int entrySize) { + String value = generateValue(entrySize); + DirectBytes bytes = DirectStore.allocate(entrySize / 6).bytes(); + for (int i = 0; i < 5; i++) { + // warmup to compression. + bytes.clear(); + SnappyStringMarshaller.INSTANCE.write(bytes, value); + bytes.flip(); + SnappyStringMarshaller.INSTANCE.read(bytes); + } + bytes.release(); + } + + @Test + @Ignore + public void testLargeStringsPerf() throws ExecutionException, InterruptedException, IOException { + doLargeEntryPerf(10000, 100 * 1024); + doLargeEntryPerf(100000, 10 * 1024); + doLargeEntryPerf(10000, 100 * 1024); + doLargeEntryPerf(3000, 1024 * 1024); + } + + private void doLargeEntryPerf(int ENTRIES, final int ENTRY_SIZE) throws IOException, InterruptedException, ExecutionException { + System.out.printf("Testing %,d entries of %,d KB%n", ENTRIES, ENTRY_SIZE); + File file = File.createTempFile("largeEntries", ".deleteme"); + file.deleteOnExit(); + final ChronicleMap map = ChronicleMapBuilder + .of(String.class, String.class) + .valueMarshaller(SnappyStringMarshaller.INSTANCE) + .entries(ENTRIES * 2) + .entrySize(ENTRY_SIZE / 6) + .putReturnsNull(true) + .createPersistedTo(file); + { + warmUpCompression(ENTRY_SIZE); + int threads = Runtime.getRuntime().availableProcessors(); + ExecutorService es = Executors.newFixedThreadPool(threads); + final int block = ENTRIES / threads; + for (int i = 0; i < 3; i++) { + long start = System.currentTimeMillis(); + List> futureList = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + final int finalT = t; + futureList.add(es.submit(new Runnable() { + @Override + public void run() { + exerciseLargeStrings(map, finalT * block, finalT * block + block, ENTRY_SIZE); + } + })); + } + for (Future future : futureList) { + future.get(); + } + long time = System.currentTimeMillis() - start; + long operations = 3; + System.out.printf("Put/Get rate was %.1f MB/s%n", operations * ENTRIES * ENTRY_SIZE / 1e6 / (time / 1e3)); + } + es.shutdown(); + if (es.isTerminated()) + map.close(); + } + } + + void exerciseLargeStrings(ChronicleMap map, int start, int finish, int entrySize) { /* final Thread thisThread = Thread.currentThread(); Thread monitor = new Thread(new Runnable() { @@ -110,23 +172,7 @@ public class LargeEntriesTest { }); monitor.start(); */ - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - XMLEncoder xml = new XMLEncoder(baos); - Map map2 = new HashMap<>(); - Random rand = new Random(1); - for (int i = 0; i < entrySize / 80; i++) - map2.put("key-" + i, "value-" + rand.nextInt(1000)); - xml.writeObject(map2); - xml.close(); - String value = baos.toString().substring(0, entrySize); - // warmup to compression. - if (run == 0) { - DirectBytes bytes = DirectStore.allocate(entrySize).bytes(); - SnappyStringMarshaller.INSTANCE.write(bytes, value); - bytes.flip(); - SnappyStringMarshaller.INSTANCE.read(bytes); - bytes.release(); - } + String value = generateValue(entrySize); for (int i = start; i < finish; i++) { String key = "key-" + i; @@ -151,4 +197,16 @@ public class LargeEntriesTest { assertEquals(key, entrySize, object.length()); } } + + private String generateValue(int entrySize) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + XMLEncoder xml = new XMLEncoder(baos); + Map map2 = new HashMap<>(); + Random rand = new Random(1); + for (int i = 0; i < entrySize / 80; i++) + map2.put("key-" + i, "value-" + rand.nextInt(1000)); + xml.writeObject(map2); + xml.close(); + return baos.toString().substring(0, entrySize); + } } From 97753304c65faa44a2525b9a44caac38554a646f Mon Sep 17 00:00:00 2001 From: Rob Austin Date: Thu, 11 Dec 2014 09:14:03 +0000 Subject: [PATCH 21/21] added images --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index d3243d8a1..c95703212 100644 --- a/README.md +++ b/README.md @@ -599,6 +599,9 @@ entries are no where near the max entry size. ## Serialization +![Serialization](http://openhft.net/wp-content/uploads/2014/09/Serialization_01.jpg) + + Chronicle Map stores your data into off heap memory, so when you give it a Key or Value, it will serialise these objects into bytes. @@ -1117,6 +1120,9 @@ click [here](https://github.com/OpenHFT/Chronicle-Map#sharing-data-between-two-o ### How to speed up the Chronicle Map Stateless Client +![Chronicle Map](http://openhft.net/wp-content/uploads/2014/09/State-Transition_1-thread_02.jpg) +![Chronicle Map](http://openhft.net/wp-content/uploads/2014/09/State-Transition_2-thread_03.jpg) + By default when you make a method call to a `ChronicleMap` stateless client, your method call is wrapped into an event which is sent over tcp and processed by the server, your stateless client will block until an acknowledgement has been received from the server that