From 1774a4d3311085f93cae44122368d0b2c4808f50 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Mon, 10 Nov 2014 15:07:08 +0000 Subject: [PATCH 1/4] Tweek segment sizes again. --- .../map/AbstractChronicleMapBuilder.java | 36 +++++++++---------- .../chronicle/map/ChronicleMapTest.java | 3 +- .../chronicle/map/EntryCountMapTest.java | 36 +++++++++---------- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java index a89942f33..e0a689148 100644 --- a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java @@ -372,10 +372,8 @@ public abstract class AbstractChronicleMapBuilder 100 ? procs / 2 : procs; + int threads = procs * 3; ExecutorService es = Executors.newFixedThreadPool(procs); for (int runs : new int[]{10, 50, 250, 500, 1000, 2500}) { // JAVA 8 produces more garbage than previous versions for internal work. @@ -585,6 +585,7 @@ public class ChronicleMapTest { OffHeapUpdatableChronicleMapBuilder builder = OffHeapUpdatableChronicleMapBuilder .of(CharSequence.class, LongValue.class) .entries(entries) + .actualSegments(8 * 1024) .entryAndValueAlignment(OF_8_BYTES) .entrySize(24); diff --git a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java index d3cad018a..742366961 100644 --- a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java @@ -53,7 +53,7 @@ public class EntryCountMapTest { throws IOException { OffHeapUpdatableChronicleMapBuilder mapBuilder = OffHeapUpdatableChronicleMapBuilder.of(CharSequence.class, LongValue.class) .entries(entries) - .minSegments(segments) + .actualSegments(segments) .entrySize(entrySize) .entryAndValueAlignment(alignment); return mapBuilder.createPersistedTo(getPersistenceFile()); @@ -103,33 +103,33 @@ public class EntryCountMapTest { testEntriesMaxSize(s, 2000, 3100, i); testEntriesMaxSize(s, 4000, 5100, i); testEntriesMaxSize(s, 5000, 7100, i); - testEntriesMaxSize(s, 8000, 10200, i); - testEntriesMaxSize(s, 12000, 16000, i); - testEntriesMaxSize(s, 16000, 21300, i); + testEntriesMaxSize(s, 8000, 11000, i); + testEntriesMaxSize(s, 12000, 17000, i); + testEntriesMaxSize(s, 16000, 22000, i); s = 32; testEntriesMaxSize(s, 2000, 3900, i); testEntriesMaxSize(s, 4000, 5900, i); testEntriesMaxSize(s, 5000, 7900, i); testEntriesMaxSize(s, 8000, 12000, i); - testEntriesMaxSize(s, 12000, 15900, i); - testEntriesMaxSize(s, 16000, 21300, i); - testEntriesMaxSize(s, 32000, 42200, i); + testEntriesMaxSize(s, 12000, 17000, i); + testEntriesMaxSize(s, 16000, 23000, i); + testEntriesMaxSize(s, 32000, 46000, i); s = 64; testEntriesMaxSize(s, 5000, 7550, i); testEntriesMaxSize(s, 8000, 11400, i); - testEntriesMaxSize(s, 12000, 16000, i); - testEntriesMaxSize(s, 16000, 22000, i); - testEntriesMaxSize(s, 32000, 43000, i); - testEntriesMaxSize(s, 40000, 54000, i); + testEntriesMaxSize(s, 12000, 18000, i); + testEntriesMaxSize(s, 16000, 24000, i); + testEntriesMaxSize(s, 32000, 47000, i); + testEntriesMaxSize(s, 40000, 60000, i); s = 128; - testEntriesMaxSize(s, 16000, 23000, i); - testEntriesMaxSize(s, 32000, 44200, i); - testEntriesMaxSize(s, 40000, 56000, i); - testEntriesMaxSize(s, 64000, 92000, i); - testEntriesMaxSize(s, 129000, 190000, i); + testEntriesMaxSize(s, 16000, 24000, i); + testEntriesMaxSize(s, 32000, 50000, i); + testEntriesMaxSize(s, 40000, 62000, i); + testEntriesMaxSize(s, 64000, 98000, i); + testEntriesMaxSize(s, 129000, 204000, i); s = 256; - testEntriesMaxSize(s, 64000, 92000, i); - testEntriesMaxSize(s, 129000, 192000, i); + testEntriesMaxSize(s, 64000, 100000, i); + testEntriesMaxSize(s, 129000, 213000, i); } // hyperbolic average gives more weight to small numbers. System.out.printf("Score: %.2f%n", scoreCount / score); From 9be0d58baf72aa31bfa629461eec6df13a40051f Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 10 Nov 2014 17:33:53 +0300 Subject: [PATCH 2/4] Reduced accessability of multimaps --- .../map/AbstractChronicleMapBuilder.java | 23 ++--- ...4MultiMap.java => Int24Int24MultiMap.java} | 22 ++--- ...ntIntMultiMap.java => IntIntMultiMap.java} | 16 ++-- .../chronicle/map/{utils => }/MultiMap.java | 4 +- .../map/{utils => }/MultiMapFactory.java | 32 +++---- .../chronicle/map/ReplicatedChronicleMap.java | 2 +- ...tMultiMap.java => ShortShortMultiMap.java} | 24 +++--- .../chronicle/map/VanillaChronicleMap.java | 4 +- .../chronicle/map/utils/IntIntMultiMap.java | 83 ------------------- .../map/{utils => }/IntIntMultiMapTest.java | 11 ++- 10 files changed, 71 insertions(+), 150 deletions(-) rename src/main/java/net/openhft/chronicle/map/{utils/VanillaI24I24MultiMap.java => Int24Int24MultiMap.java} (91%) rename src/main/java/net/openhft/chronicle/map/{utils/VanillaIntIntMultiMap.java => IntIntMultiMap.java} (94%) rename src/main/java/net/openhft/chronicle/map/{utils => }/MultiMap.java (96%) rename src/main/java/net/openhft/chronicle/map/{utils => }/MultiMapFactory.java (66%) rename src/main/java/net/openhft/chronicle/map/{utils/VanillaShortShortMultiMap.java => ShortShortMultiMap.java} (90%) delete mode 100644 src/main/java/net/openhft/chronicle/map/utils/IntIntMultiMap.java rename src/test/java/net/openhft/chronicle/map/{utils => }/IntIntMultiMapTest.java (94%) diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java index a89942f33..252689498 100644 --- a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java @@ -26,7 +26,7 @@ import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.chronicle.hash.serialization.internal.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.internal.MetaBytesWriter; import net.openhft.chronicle.hash.serialization.internal.MetaProvider; -import net.openhft.chronicle.map.utils.MultiMapFactory; +import net.openhft.chronicle.map.MultiMapFactory; import net.openhft.chronicle.set.ChronicleSetBuilder; import net.openhft.lang.Maths; import net.openhft.lang.io.ByteBufferBytes; @@ -439,6 +439,17 @@ public abstract class AbstractChronicleMapBuilder> 10), VanillaChronicleMap.MAX_ENTRY_OVERSIZE_FACTOR); } - - int segmentHeaderSize() { - int segments = actualSegments(); - // reduce false sharing unless we have a lot of segments. - return segments <= 8192 ? 64 : 16; - } - - public MultiMapFactory multiMapFactory() { - return MultiMapFactory.forCapacity(actualEntriesPerSegment()); - } } diff --git a/src/main/java/net/openhft/chronicle/map/utils/VanillaI24I24MultiMap.java b/src/main/java/net/openhft/chronicle/map/Int24Int24MultiMap.java similarity index 91% rename from src/main/java/net/openhft/chronicle/map/utils/VanillaI24I24MultiMap.java rename to src/main/java/net/openhft/chronicle/map/Int24Int24MultiMap.java index 01933c5db..c2f0c3bed 100644 --- a/src/main/java/net/openhft/chronicle/map/utils/VanillaI24I24MultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/Int24Int24MultiMap.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package net.openhft.chronicle.map.utils; +package net.openhft.chronicle.map; import net.openhft.lang.collection.ATSDirectBitSet; import net.openhft.lang.collection.DirectBitSet; @@ -28,7 +28,7 @@ import static net.openhft.lang.Maths.isPowerOf2; /** * Supports a simple interface for int -> int[] off heap. */ -class VanillaI24I24MultiMap implements MultiMap { +class Int24Int24MultiMap implements MultiMap { static final int MAX_CAPACITY = (1 << 24); @@ -48,8 +48,8 @@ class VanillaI24I24MultiMap implements MultiMap { private int searchHash = -1; private int searchPos = -1; - public VanillaI24I24MultiMap(long minCapacity) { - capacity = (int) VanillaIntIntMultiMap.multiMapCapacity(minCapacity); + public Int24Int24MultiMap(long minCapacity) { + capacity = (int) IntIntMultiMap.multiMapCapacity(minCapacity); capacityMask = capacity - 1; capacityMod2 = indexToPos(capacity - 1); bytes = DirectStore.allocateLazy(indexToPos(capacity)).bytes(); @@ -57,7 +57,7 @@ class VanillaI24I24MultiMap implements MultiMap { clear(); } - public VanillaI24I24MultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { + public Int24Int24MultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { capacity = (int) Math.min(MAX_CAPACITY, (multiMapBytes.capacity() / ENTRY_SIZE)); assert capacity <= MAX_CAPACITY; capacityMask = capacity - 1; @@ -67,21 +67,21 @@ class VanillaI24I24MultiMap implements MultiMap { } /** - * @param minCapacity as in {@link #VanillaI24I24MultiMap(long)} constructor - * @return size of {@link Bytes} to provide to {@link #VanillaI24I24MultiMap(Bytes, Bytes)} + * @param minCapacity as in {@link #Int24Int24MultiMap(long)} constructor + * @return size of {@link Bytes} to provide to {@link #Int24Int24MultiMap(Bytes, Bytes)} * constructor as the first argument */ public static long sizeInBytes(long minCapacity) { - return indexToPos((int) VanillaIntIntMultiMap.multiMapCapacity(minCapacity)); + return indexToPos((int) IntIntMultiMap.multiMapCapacity(minCapacity)); } /** - * @param minCapacity as in {@link #VanillaI24I24MultiMap(long)} constructor - * @return size of {@link Bytes} to provide to {@link #VanillaI24I24MultiMap(Bytes, Bytes)} + * @param minCapacity as in {@link #Int24Int24MultiMap(long)} constructor + * @return size of {@link Bytes} to provide to {@link #Int24Int24MultiMap(Bytes, Bytes)} * constructor as the second argument */ public static long sizeOfBitSetInBytes(long minCapacity) { - return (int) (Math.max(VanillaIntIntMultiMap.multiMapCapacity(minCapacity), 64L) * 6L + 63) / 64; + return (int) (Math.max(IntIntMultiMap.multiMapCapacity(minCapacity), 64L) * 6L + 63) / 64; } public static ATSDirectBitSet newPositions(int capacity) { diff --git a/src/main/java/net/openhft/chronicle/map/utils/VanillaIntIntMultiMap.java b/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java similarity index 94% rename from src/main/java/net/openhft/chronicle/map/utils/VanillaIntIntMultiMap.java rename to src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java index d1f2bd646..9f0b0c113 100644 --- a/src/main/java/net/openhft/chronicle/map/utils/VanillaIntIntMultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package net.openhft.chronicle.map.utils; +package net.openhft.chronicle.map; import net.openhft.lang.Maths; import net.openhft.lang.collection.ATSDirectBitSet; @@ -29,7 +29,7 @@ import static net.openhft.lang.Maths.isPowerOf2; /** * Supports a simple interface for int -> int[] off heap. */ -class VanillaIntIntMultiMap implements MultiMap { +class IntIntMultiMap implements MultiMap { static final long MAX_CAPACITY = (1L << 32); @@ -59,7 +59,7 @@ class VanillaIntIntMultiMap implements MultiMap { private long searchHash = -1L; private long searchPos = -1L; - public VanillaIntIntMultiMap(long minCapacity) { + public IntIntMultiMap(long minCapacity) { capacity = multiMapCapacity(minCapacity); capacityMask = capacity - 1L; capacityMask2 = indexToPos(capacity - 1L); @@ -68,7 +68,7 @@ class VanillaIntIntMultiMap implements MultiMap { clear(); } - public VanillaIntIntMultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { + public IntIntMultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { capacity = multiMapBytes.capacity() / ENTRY_SIZE; assert capacity == Maths.nextPower2(capacity, 16L); capacityMask = capacity - 1L; @@ -78,8 +78,8 @@ class VanillaIntIntMultiMap implements MultiMap { } /** - * @param minCapacity as in {@link #VanillaIntIntMultiMap(long)} constructor - * @return size of {@link Bytes} to provide to {@link #VanillaIntIntMultiMap(Bytes, Bytes)} + * @param minCapacity as in {@link #IntIntMultiMap(long)} constructor + * @return size of {@link Bytes} to provide to {@link #IntIntMultiMap(Bytes, Bytes)} * constructor as the first argument */ public static long sizeInBytes(long minCapacity) { @@ -87,8 +87,8 @@ class VanillaIntIntMultiMap implements MultiMap { } /** - * @param minCapacity as in {@link #VanillaIntIntMultiMap(long)} constructor - * @return size of {@link Bytes} to provide to {@link #VanillaIntIntMultiMap(Bytes, Bytes)} + * @param minCapacity as in {@link #IntIntMultiMap(long)} constructor + * @return size of {@link Bytes} to provide to {@link #IntIntMultiMap(Bytes, Bytes)} * constructor as the second argument */ public static long sizeOfBitSetInBytes(long minCapacity) { diff --git a/src/main/java/net/openhft/chronicle/map/utils/MultiMap.java b/src/main/java/net/openhft/chronicle/map/MultiMap.java similarity index 96% rename from src/main/java/net/openhft/chronicle/map/utils/MultiMap.java rename to src/main/java/net/openhft/chronicle/map/MultiMap.java index c7104ae87..60d013d1d 100644 --- a/src/main/java/net/openhft/chronicle/map/utils/MultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/MultiMap.java @@ -14,14 +14,14 @@ * limitations under the License. */ -package net.openhft.chronicle.map.utils; +package net.openhft.chronicle.map; import net.openhft.lang.collection.DirectBitSet; /** * This is only used to store keys and positions, but it could store int/int key/values for another purpose. */ -public interface MultiMap { +interface MultiMap { /** * @return gets all the active positions as a bitset diff --git a/src/main/java/net/openhft/chronicle/map/utils/MultiMapFactory.java b/src/main/java/net/openhft/chronicle/map/MultiMapFactory.java similarity index 66% rename from src/main/java/net/openhft/chronicle/map/utils/MultiMapFactory.java rename to src/main/java/net/openhft/chronicle/map/MultiMapFactory.java index a2c445c17..3c5ab1e7e 100644 --- a/src/main/java/net/openhft/chronicle/map/utils/MultiMapFactory.java +++ b/src/main/java/net/openhft/chronicle/map/MultiMapFactory.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package net.openhft.chronicle.map.utils; +package net.openhft.chronicle.map; import net.openhft.lang.io.NativeBytes; @@ -25,65 +25,65 @@ import net.openhft.lang.io.NativeBytes; * * In future we might support I32 -> I16 or similar. */ -public enum MultiMapFactory { +enum MultiMapFactory { I16 { @Override public long sizeInBytes(long minCapacity) { - return VanillaShortShortMultiMap.sizeInBytes(minCapacity); + return ShortShortMultiMap.sizeInBytes(minCapacity); } @Override public long sizeOfBitSetInBytes(long minCapacity) { - return VanillaShortShortMultiMap.sizeOfBitSetInBytes(minCapacity); + return ShortShortMultiMap.sizeOfBitSetInBytes(minCapacity); } @Override public MultiMap create(NativeBytes bytes, NativeBytes bitSetBytes) { - return new VanillaShortShortMultiMap(bytes, bitSetBytes); + return new ShortShortMultiMap(bytes, bitSetBytes); } }, I24 { @Override public long sizeInBytes(long minCapacity) { - return VanillaI24I24MultiMap.sizeInBytes(minCapacity); + return Int24Int24MultiMap.sizeInBytes(minCapacity); } @Override public long sizeOfBitSetInBytes(long minCapacity) { - return VanillaI24I24MultiMap.sizeOfBitSetInBytes(minCapacity); + return Int24Int24MultiMap.sizeOfBitSetInBytes(minCapacity); } @Override public MultiMap create(NativeBytes bytes, NativeBytes bitSetBytes) { - return new VanillaI24I24MultiMap(bytes, bitSetBytes); + return new Int24Int24MultiMap(bytes, bitSetBytes); } }, I32 { @Override public long sizeInBytes(long minCapacity) { - return VanillaIntIntMultiMap.sizeInBytes(minCapacity); + return IntIntMultiMap.sizeInBytes(minCapacity); } @Override public long sizeOfBitSetInBytes(long minCapacity) { - return VanillaIntIntMultiMap.sizeOfBitSetInBytes(minCapacity); + return IntIntMultiMap.sizeOfBitSetInBytes(minCapacity); } @Override public MultiMap create(NativeBytes bytes, NativeBytes bitSetBytes) { - return new VanillaIntIntMultiMap(bytes, bitSetBytes); + return new IntIntMultiMap(bytes, bitSetBytes); } }; - public static final long I16_MAX_CAPACITY = VanillaShortShortMultiMap.MAX_CAPACITY; - public static final long MAX_CAPACITY = VanillaIntIntMultiMap.MAX_CAPACITY; + public static final long I16_MAX_CAPACITY = ShortShortMultiMap.MAX_CAPACITY; + public static final long MAX_CAPACITY = IntIntMultiMap.MAX_CAPACITY; public static MultiMapFactory forCapacity(long capacity) { - if (capacity <= VanillaShortShortMultiMap.MAX_CAPACITY) + if (capacity <= ShortShortMultiMap.MAX_CAPACITY) return I16; -// if (capacity <= VanillaI24I24MultiMap.MAX_CAPACITY) +// if (capacity <= Int24Int24MultiMap.MAX_CAPACITY) // return I24; - if (capacity <= VanillaIntIntMultiMap.MAX_CAPACITY) + if (capacity <= IntIntMultiMap.MAX_CAPACITY) return I32; throw new IllegalArgumentException("Capacity " + capacity + " not supported"); } diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index b41e420c0..01358a276 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -23,7 +23,7 @@ import net.openhft.chronicle.hash.replication.TimeProvider; import net.openhft.chronicle.hash.serialization.BytesReader; import net.openhft.chronicle.hash.serialization.internal.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.internal.MetaBytesWriter; -import net.openhft.chronicle.map.utils.MultiMap; +import net.openhft.chronicle.map.MultiMap; import net.openhft.lang.Maths; import net.openhft.lang.collection.ATSDirectBitSet; import net.openhft.lang.io.Bytes; diff --git a/src/main/java/net/openhft/chronicle/map/utils/VanillaShortShortMultiMap.java b/src/main/java/net/openhft/chronicle/map/ShortShortMultiMap.java similarity index 90% rename from src/main/java/net/openhft/chronicle/map/utils/VanillaShortShortMultiMap.java rename to src/main/java/net/openhft/chronicle/map/ShortShortMultiMap.java index 00c9ea67a..7bd73eecf 100644 --- a/src/main/java/net/openhft/chronicle/map/utils/VanillaShortShortMultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/ShortShortMultiMap.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package net.openhft.chronicle.map.utils; +package net.openhft.chronicle.map; import net.openhft.lang.collection.ATSDirectBitSet; @@ -24,13 +24,13 @@ import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.DirectStore; -import static net.openhft.chronicle.map.utils.VanillaIntIntMultiMap.multiMapCapacity; +import static net.openhft.chronicle.map.IntIntMultiMap.multiMapCapacity; import static net.openhft.lang.Maths.isPowerOf2; /** * Supports a simple interface for int -> int[] off heap. */ -class VanillaShortShortMultiMap implements MultiMap { +class ShortShortMultiMap implements MultiMap { static final long MAX_CAPACITY = (1L << 16); @@ -47,7 +47,7 @@ class VanillaShortShortMultiMap implements MultiMap { private long searchHash = -1L; private long searchPos = -1L; - public VanillaShortShortMultiMap(long minCapacity) { + public ShortShortMultiMap(long minCapacity) { assert minCapacity <= MAX_CAPACITY; long capacity = multiMapCapacity(minCapacity); assert isPowerOf2(capacity); @@ -55,11 +55,11 @@ class VanillaShortShortMultiMap implements MultiMap { capacityMask = capacity - 1L; capacityMask2 = (capacity - 1L) * ENTRY_SIZE; bytes = DirectStore.allocateLazy(capacity * ENTRY_SIZE).bytes(); - positions = VanillaIntIntMultiMap.newPositions(capacity); + positions = IntIntMultiMap.newPositions(capacity); clear(); } - public VanillaShortShortMultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { + public ShortShortMultiMap(Bytes multiMapBytes, Bytes multiMapBitSetBytes) { long capacity = multiMapBytes.capacity() / ENTRY_SIZE; assert isPowerOf2(capacity); assert (capacity / 2L) <= MAX_CAPACITY; @@ -71,8 +71,8 @@ class VanillaShortShortMultiMap implements MultiMap { } /** - * @param minCapacity as in {@link #VanillaShortShortMultiMap(long)} constructor - * @return size of {@link Bytes} to provide to {@link #VanillaShortShortMultiMap(Bytes, Bytes)} + * @param minCapacity as in {@link #ShortShortMultiMap(long)} constructor + * @return size of {@link Bytes} to provide to {@link #ShortShortMultiMap(Bytes, Bytes)} * constructor as the first argument */ public static long sizeInBytes(long minCapacity) { @@ -80,14 +80,14 @@ class VanillaShortShortMultiMap implements MultiMap { } /** - * @param minCapacity as in {@link #VanillaShortShortMultiMap(long)} constructor - * @return size of {@link Bytes} to provide to {@link #VanillaShortShortMultiMap(Bytes, Bytes)} + * @param minCapacity as in {@link #ShortShortMultiMap(long)} constructor + * @return size of {@link Bytes} to provide to {@link #ShortShortMultiMap(Bytes, Bytes)} * constructor as the second argument */ public static long sizeOfBitSetInBytes(long minCapacity) { - return VanillaIntIntMultiMap.sizeOfBitSetInBytes(minCapacity); + return IntIntMultiMap.sizeOfBitSetInBytes(minCapacity); } private void checkValueForPut(long value) { @@ -183,7 +183,7 @@ class VanillaShortShortMultiMap implements MultiMap { if (entryToShift == UNSET_ENTRY) break; long insertPos = pos(key(entryToShift)); - // see comment in VanillaIntIntMultiMap + // see comment in IntIntMultiMap boolean cond1 = insertPos <= posToRemove; boolean cond2 = posToRemove <= posToShift; if ((cond1 && cond2) || diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index ce6d28416..749e0dfc2 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -25,8 +25,8 @@ import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.chronicle.hash.serialization.internal.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.internal.MetaBytesWriter; import net.openhft.chronicle.hash.serialization.internal.MetaProvider; -import net.openhft.chronicle.map.utils.MultiMap; -import net.openhft.chronicle.map.utils.MultiMapFactory; +import net.openhft.chronicle.map.MultiMap; +import net.openhft.chronicle.map.MultiMapFactory; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.*; diff --git a/src/main/java/net/openhft/chronicle/map/utils/IntIntMultiMap.java b/src/main/java/net/openhft/chronicle/map/utils/IntIntMultiMap.java deleted file mode 100644 index 7f444e02c..000000000 --- a/src/main/java/net/openhft/chronicle/map/utils/IntIntMultiMap.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.map.utils; - -import net.openhft.lang.collection.DirectBitSet; - -/** - * This is only used to store keys and positions, but it could store int/int key/values for another purpose. - */ -interface IntIntMultiMap { - - /** - * @return gets all the active positions as a bitset - */ - DirectBitSet getPositions(); - - /** - * Add an entry. Allow duplicate hashes, but not key/position pairs. - * - * @param key to add - * @param value to add - */ - void put(int key, int value); - - /** - * Remove a key/value pair. - * - * @param key to remove - * @param value to remove - * @return whether a match was found. - */ - boolean remove(int key, int value); - - boolean replace(int key, int oldValue, int newValue); - - /** - * Used for start a search for a given key - * - * @return normalized key value, better to be used in subsequent calls - */ - int startSearch(int key); - - int getSearchHash(); - - /** - * Used for getting the next position for a given key - * - * @return the next position for the last search or negative value - */ - int nextPos(); - - void removePrevPos(); - - void replacePrevPos(int newValue); - - void putAfterFailedSearch(int value); - - void clear(); - - void forEach(EntryConsumer action); - - static interface EntryConsumer { - void accept(int key, int value); - } - - -} diff --git a/src/test/java/net/openhft/chronicle/map/utils/IntIntMultiMapTest.java b/src/test/java/net/openhft/chronicle/map/IntIntMultiMapTest.java similarity index 94% rename from src/test/java/net/openhft/chronicle/map/utils/IntIntMultiMapTest.java rename to src/test/java/net/openhft/chronicle/map/IntIntMultiMapTest.java index 9d1ad11a3..66f918dc9 100644 --- a/src/test/java/net/openhft/chronicle/map/utils/IntIntMultiMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/IntIntMultiMapTest.java @@ -16,11 +16,14 @@ * limitations under the License. */ -package net.openhft.chronicle.map.utils; +package net.openhft.chronicle.map; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import net.openhft.chronicle.map.Int24Int24MultiMap; +import net.openhft.chronicle.map.IntIntMultiMap; +import net.openhft.chronicle.map.MultiMap; import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.DirectStore; import org.junit.Assert; @@ -52,9 +55,9 @@ public class IntIntMultiMapTest { @Parameterized.Parameters public static Collection data() { return Arrays.asList(new Object[][]{ - {VanillaShortShortMultiMap.class}, - {VanillaI24I24MultiMap.class}, - {VanillaIntIntMultiMap.class}, + {ShortShortMultiMap.class}, + {Int24Int24MultiMap.class}, + {IntIntMultiMap.class}, }); } From daac52ba354c2a37055b192226f675ca1e37a871 Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 10 Nov 2014 18:14:30 +0300 Subject: [PATCH 3/4] Changed acquireUsingLocked and getUsingLocked behaviour -- now throwing IllegalState if the given value is not reused --- .../chronicle/map/ReplicatedChronicleMap.java | 14 +++--- .../chronicle/map/VanillaChronicleMap.java | 43 +++++++++---------- .../chronicle/map/ChronicleMapTest.java | 3 +- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 01358a276..4d899f503 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -273,7 +273,7 @@ final class ReplicatedChronicleMap, } @Override - T lookupUsing(K key, V value, LockType lockTypeType) { + T lookupUsing(K key, V usingValue, LockType lockType, boolean mustReuseValue) { checkKey(key); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); @@ -284,24 +284,26 @@ final class ReplicatedChronicleMap, int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); - boolean create = lockTypeType == LockType.WRITE_LOCK; + boolean create = lockType == LockType.WRITE_LOCK; Segment segment = segment(segmentNum); - MutableLockedEntry lock = (lockTypeType == LockType.WRITE_LOCK) + MutableLockedEntry lock = (lockType == LockType.WRITE_LOCK) ? segment.writeLock() : segment.readLock(); V v = segment.acquireWithoutLock(copies, metaKeyWriter, keyWriter, key, - value, + usingValue, segmentHash, create, create ? timeProvider.currentTimeMillis() : 0L); + checkReallyUsingValue(usingValue, mustReuseValue, v); + lock.copies = copies; lock.metaKeyInterop = metaKeyWriter; lock.keyInterop = keyWriter; lock.segmentHash = segmentHash; - if (!isNativeValueClass && lockTypeType == LockType.WRITE_LOCK && v == null) - v = value; + if (!isNativeValueClass && lockType == LockType.WRITE_LOCK && v == null) + v = usingValue; lock.value(v); lock.key(key); diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 749e0dfc2..051520842 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -25,8 +25,6 @@ import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.chronicle.hash.serialization.internal.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.internal.MetaBytesWriter; import net.openhft.chronicle.hash.serialization.internal.MetaProvider; -import net.openhft.chronicle.map.MultiMap; -import net.openhft.chronicle.map.MultiMapFactory; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.*; @@ -357,7 +355,7 @@ class VanillaChronicleMap, public V get(Object key) { try (Context entry = lookupUsing((K) key, null, - LockType.READ_LOCK)) { + LockType.READ_LOCK, false)) { return entry.value(); } } @@ -366,7 +364,7 @@ class VanillaChronicleMap, public V getUsing(K key, V usingValue) { try (Context entry = lookupUsing(key, usingValue, - LockType.READ_LOCK)) { + LockType.READ_LOCK, false)) { return entry.value(); } } @@ -374,20 +372,21 @@ class VanillaChronicleMap, @NotNull @Override public WriteContext acquireUsingLocked(@NotNull K key, @NotNull V usingValue) { - return this.lookupUsing(key, usingValue, LockType.WRITE_LOCK); + return this.lookupUsing(key, usingValue, LockType.WRITE_LOCK, true); } @NotNull @Override public ReadContext getUsingLocked(@NotNull K key, @NotNull V usingValue) { - return this.lookupUsing(key, usingValue, LockType.READ_LOCK); + return this.lookupUsing(key, usingValue, LockType.READ_LOCK, true); } @Override public V acquireUsing(@NotNull K key, @NotNull V usingValue) { - try (WriteContext kvContext = lookupUsing(key, usingValue, LockType.WRITE_LOCK)) { + try (WriteContext kvContext = + lookupUsing(key, usingValue, LockType.WRITE_LOCK, false)) { if (!isNativeValueClass) kvContext.dontPutOnClose(); @@ -397,7 +396,7 @@ class VanillaChronicleMap, } - T lookupUsing(K key, V value, LockType lockTypeType) { + T lookupUsing(K key, V usingValue, LockType lockType, boolean mustReuseValue) { checkKey(key); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); @@ -409,20 +408,22 @@ class VanillaChronicleMap, long segmentHash = segmentHash(hash); Segment segment = segments[segmentNum]; - MutableLockedEntry lock = (lockTypeType == LockType.WRITE_LOCK) + MutableLockedEntry lock = (lockType == LockType.WRITE_LOCK) ? segment.writeLock() : segment.readLock(); - V v = segment.acquireWithoutLock(copies, metaKeyInterop, keyInterop, key, value, - segmentHash, lockTypeType == LockType.WRITE_LOCK); + V v = segment.acquireWithoutLock(copies, metaKeyInterop, keyInterop, key, usingValue, + segmentHash, lockType == LockType.WRITE_LOCK); + + checkReallyUsingValue(usingValue, mustReuseValue, v); lock.copies = copies; lock.metaKeyInterop = metaKeyInterop; lock.keyInterop = keyInterop; lock.segmentHash = segmentHash; - if (!isNativeValueClass && lockTypeType == LockType.WRITE_LOCK && v == null) - v = value; + if (!isNativeValueClass && lockType == LockType.WRITE_LOCK && v == null) + v = usingValue; lock.value(v); lock.key(key); @@ -431,6 +432,12 @@ class VanillaChronicleMap, } + void checkReallyUsingValue(V usingValue, boolean mustReuseValue, V returnedValue) { + if (mustReuseValue && returnedValue != null && returnedValue != usingValue) + throw new IllegalArgumentException("acquireUsingLocked/getUsingLocked MUST reuse given " + + "values. Given value" + usingValue + " cannot be reused to read " + returnedValue); + } + @Override public boolean containsKey(final Object k) { checkKey(k); @@ -849,17 +856,7 @@ class VanillaChronicleMap, } V acquireWithoutLock(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V usingValue, - long hash2, boolean create) { - V v = acquire0(copies, metaKeyInterop, keyInterop, key, usingValue, hash2, false); - if (create && v == null) - v = acquire0(copies, metaKeyInterop, keyInterop, key, usingValue, hash2, true); - return v; - } - - V acquire0(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V usingValue, long hash2, boolean create) { - - long keySize = metaKeyInterop.size(keyInterop, key); MultiStoreBytes entry = acquireTmpBytes(); long offset = searchKey(keyInterop, metaKeyInterop, key, keySize, hash2, entry, diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java index 098f16f25..f4310763b 100644 --- a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java @@ -1362,7 +1362,6 @@ public class ChronicleMapTest { // check that the entry was removed try (ReadContext context = map.getUsingLocked("one", value)) { - assert value == context.value(); assertEquals(false, context.present()); assertEquals(null, context.value()); } @@ -1389,7 +1388,7 @@ public class ChronicleMapTest { } - @Test(expected = IllegalAccessException.class) + @Test(expected = IllegalStateException.class) public void testAcquireUsingLockedWithString() throws IOException { ChronicleMapBuilder builder = ChronicleMapBuilder From 1cbdd91640b66e665b4cc7091620caead0d6d4d3 Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 10 Nov 2014 18:38:14 +0300 Subject: [PATCH 4/4] Fix tests --- pom.xml | 2 +- src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f1c471919..0b70d144a 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ net.openhft lang - 6.4.12 + 6.4.13-SNAPSHOT diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java index 9aac4a616..6c7337300 100644 --- a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java @@ -1389,12 +1389,13 @@ public class ChronicleMapTest { } - @Test(expected = IllegalStateException.class) + @Test(expected = IllegalArgumentException.class) public void testAcquireUsingLockedWithString() throws IOException { ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, String.class) .entries(1000) + .defaultValue("") .entrySize(16);