diff --git a/src/main/java/net/openhft/chronicle/common/serialization/Hasher.java b/src/main/java/net/openhft/chronicle/common/serialization/Hasher.java index f0a15da56..12b69be1d 100644 --- a/src/main/java/net/openhft/chronicle/common/serialization/Hasher.java +++ b/src/main/java/net/openhft/chronicle/common/serialization/Hasher.java @@ -23,11 +23,12 @@ import sun.misc.Unsafe; import static java.nio.ByteOrder.LITTLE_ENDIAN; import static java.nio.ByteOrder.nativeOrder; -public final class Hasher { +public enum Hasher { + ; private static final long LONG_LEVEL_PRIME_MULTIPLE = 1011001110001111L; - private static final int SHORT_LEVEL_PRIME_MULTIPLE = 101111; - private static final long BYTE_LEVEL_PRIME_MULTIPLE = 2111; + private static final short SHORT_LEVEL_PRIME_MULTIPLE = 10191; + private static final byte BYTE_LEVEL_PRIME_MULTIPLE = 109; private static final int INT_HASH_LOW_SHORT_MULTIPLE = nativeOrder() == LITTLE_ENDIAN ? SHORT_LEVEL_PRIME_MULTIPLE : 1; @@ -72,10 +73,8 @@ public final class Hasher { } public static long hash(long value) { - value *= 11018881818881011L; - value ^= (value >>> 41) ^ (value >>> 21); + value ^= (value >>> 41) - (value >>> 21); + value ^= (value >>> 15) + (value >>> 7); return value; } - - private Hasher() {} } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 4b1cf7477..46a780a7e 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -390,10 +390,10 @@ public class ChronicleMapBuilder implements Cloneable, if (actualEntriesPerSegment > 0) return actualEntriesPerSegment; int as = actualSegments(); - long actualEntries = - as == 1 ? entries + 1 : // The extra 1 might not be needed. - as <= 4 ? entries * 10L / 9 : // 10%, assumes you are trying to be tight. - entries * 5L / 4; // 25% + long actualEntries = entries + + (as == 1 ? 1 : // The extra 1 might not be needed. + as <= 8 ? entries / 12 : // 6%, (3% was min for tests) + entries / 6); // 16% (8% was min for tests) // round up to the next multiple of 64. return (int) (Math.max(1, actualEntries / as) + 63) & ~63; diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 33329352f..0f5f3ad69 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -262,7 +262,7 @@ class ReplicatedChronicleMap, int segmentNum = getSegment(hash); int segmentHash = segmentHash(hash); return segment(segmentNum).acquire(copies, metaKeyWriter, keyWriter, key, value, - segmentHash, create, timeProvider.currentTimeMillis()); + segmentHash, create, create ? timeProvider.currentTimeMillis() : 0L); } @Override diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index ba9e584e7..e6bb3a8b1 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -17,9 +17,9 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.common.ChronicleHashErrorListener; +import net.openhft.chronicle.common.serialization.*; import net.openhft.chronicle.common.threadlocal.Provider; import net.openhft.chronicle.common.threadlocal.ThreadLocalCopies; -import net.openhft.chronicle.common.serialization.*; import net.openhft.lang.Maths; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; @@ -910,10 +910,10 @@ class VanillaChronicleMap, ret = (int) freeList.setNextNContinuousClearBits(0, blocks); if (ret == DirectBitSet.NOT_FOUND) { if (blocks == 1) { - throw new IllegalArgumentException( + throw new IllegalStateException( "Segment is full, no free entries found"); } else { - throw new IllegalArgumentException( + throw new IllegalStateException( "Segment is full or has no ranges of " + blocks + " continuous free blocks" ); @@ -970,7 +970,7 @@ class VanillaChronicleMap, /** * Removes a key (or key-value pair) from the Segment. - * + *

*

The entry will only be removed if {@code expectedValue} equals to {@code null} * or the value previously corresponding to the specified key. * @@ -1446,4 +1446,11 @@ class VanillaChronicleMap, return super.setValue(value); } } + + int[] segmentSizes() { + int[] sizes = new int[segments.length]; + for (int i = 0; i < segments.length; i++) + sizes[i] = segments[i].getSize(); + return sizes; + } } diff --git a/src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java b/src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java index 36131acd7..31331bfe3 100644 --- a/src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java +++ b/src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java @@ -63,7 +63,11 @@ enum DirectIntValueFactory implements ObjectFactory { INSTANCE; @Override - public IntValue create() throws Exception { - return (IntValue) NativeBytes.UNSAFE.allocateInstance(ByteableIntValueMarshaller.DIRECT); + public IntValue create() { + try { + return (IntValue) NativeBytes.UNSAFE.allocateInstance(ByteableIntValueMarshaller.DIRECT); + } catch (InstantiationException e) { + throw new AssertionError(e); + } } } diff --git a/src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java b/src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java index 54aee15c3..e35eacb91 100644 --- a/src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java +++ b/src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java @@ -63,8 +63,12 @@ enum DirectLongValueFactory implements ObjectFactory { INSTANCE; @Override - public LongValue create() throws Exception { - return (LongValue) NativeBytes.UNSAFE.allocateInstance(ByteableLongValueMarshaller.DIRECT); + public LongValue create() { + try { + return (LongValue) NativeBytes.UNSAFE.allocateInstance(ByteableLongValueMarshaller.DIRECT); + } catch (InstantiationException e) { + throw new AssertionError(e); + } } } diff --git a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java new file mode 100644 index 000000000..ec35eaa29 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java @@ -0,0 +1,168 @@ +package net.openhft.chronicle.map; + +import net.openhft.lang.values.LongValue; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; + +import static net.openhft.chronicle.map.Alignment.OF_4_BYTES; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class EntryCountMapTest { + @Test + public void testVerySmall() throws Exception { + int s = 1, i = 1; + // regression test. + testEntriesMaxSize(s, 1, 64, i); + testEntriesMaxSize(s, 2, 64, i); + testEntriesMaxSize(s, 4, 64, i); + testEntriesMaxSize(s, 8, 64, i); + testEntriesMaxSize(s, 16, 64, i); + testEntriesMaxSize(s, 32, 64, i); + testEntriesMaxSize(s, 64, 128, i); + } + + @Test + public void testSmall() throws Exception { + for (int i = 0; i < 10; i++) { + // regression test. + for (int s : new int[]{1, 2, 4, 8}) { + if (s <= 4) + testEntriesMaxSize(s, 128, 256, i); + testEntriesMaxSize(s, 256, 510, i); + testEntriesMaxSize(s, 512, 1010, i); + testEntriesMaxSize(s, 1000, 1730, i); + testEntriesMaxSize(s, 2000, 2800, i); + testEntriesMaxSize(s, 4000, 5100, i); + testEntriesMaxSize(s, 5000, 6800, i); + testEntriesMaxSize(s, 8000, 9800, i); + testEntriesMaxSize(s, 12000, 15000, i); + testEntriesMaxSize(s, 16000, 20000, i); + } + } + // hyperbolic average gives more weight to small numbers. + System.out.printf("Score: %.2f%n", scoreCount / score); + } + + @Test + public void testMedium() throws Exception { + for (int i = 0; i < 3; i++) { + // regression test. + int s = 16; + testEntriesMaxSize(s, 512, 1000, i); + testEntriesMaxSize(s, 1000, 2100, i); + testEntriesMaxSize(s, 2000, 3100, i); + testEntriesMaxSize(s, 4000, 5100, i); + testEntriesMaxSize(s, 5000, 7100, i); + testEntriesMaxSize(s, 8000, 10100, i); + testEntriesMaxSize(s, 12000, 15100, i); + testEntriesMaxSize(s, 16000, 20200, i); + s = 32; + testEntriesMaxSize(s, 2000, 3900, i); + testEntriesMaxSize(s, 4000, 5900, i); + testEntriesMaxSize(s, 5000, 7900, i); + testEntriesMaxSize(s, 8000, 12000, i); + testEntriesMaxSize(s, 12000, 15900, i); + testEntriesMaxSize(s, 16000, 20100, i); + testEntriesMaxSize(s, 32000, 41000, i); + s = 64; + testEntriesMaxSize(s, 5000, 7550, i); + testEntriesMaxSize(s, 8000, 11400, i); + testEntriesMaxSize(s, 12000, 15400, i); + testEntriesMaxSize(s, 16000, 20000, i); + testEntriesMaxSize(s, 32000, 39400, i); + testEntriesMaxSize(s, 40000, 52200, i); + s = 128; + testEntriesMaxSize(s, 16000, 23000, i); + testEntriesMaxSize(s, 32000, 39000, i); + testEntriesMaxSize(s, 40000, 55200, i); + testEntriesMaxSize(s, 64000, 79800, i); + testEntriesMaxSize(s, 129000, 160000, i); + s = 256; + testEntriesMaxSize(s, 64000, 79800, i); + testEntriesMaxSize(s, 129000, 160000, i); + } + // hyperbolic average gives more weight to small numbers. + System.out.printf("Score: %.2f%n", scoreCount / score); + } + + double score = 0; + int scoreCount = 0; + + private void testEntriesMaxSize(int segments, int minSize, int maxSize, int seed) throws IOException { + int counter = minSize + new Random(seed + segments * 1000).nextInt(9999); + ChronicleMap map = getSharedMap(minSize, segments, 32); + LongValue longValue = DirectLongValueFactory.INSTANCE.create(); + try { + for (int j = 0; j < maxSize * 12 / 10 + 300; j++) { + String key = "key:" + counter++; + // give a biased hashcode distribution. + if ((Integer.bitCount(key.hashCode()) & 7) != 0) { + j--; + continue; + } + map.acquireUsing(key, longValue); + longValue.setValue(1); + } + int[] a = ((VanillaChronicleMap) map).segmentSizes(); + System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " " + + Arrays.toString(a) + + " sum: " + sum(a)); + fail("Expected the map to be full."); + } catch (IllegalStateException e) { + // calculate the hyperbolic average. + score += (double) minSize / map.size(); + scoreCount++; + boolean condition = minSize <= map.size() && map.size() <= maxSize; + if (!condition) { + int[] a = ((VanillaChronicleMap) map).segmentSizes(); + System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " " + + Arrays.toString(a) + + " sum: " + sum(a)); + assertTrue("min: " + minSize + ", size: " + map.size(), condition); + } + } finally { + map.close(); + } + } + + private long sum(int[] ints) { + long sum = 0; + for (int i : ints) { + sum += i; + } + return sum; + } + + static File getPersistenceFile() { + String TMP = System.getProperty("java.io.tmpdir"); + File file = new File(TMP + "/small-chm-test" + System.nanoTime()); + file.deleteOnExit(); + return file; + } + + private static ChronicleMap getSharedMap( + long entries, int segments, int entrySize) throws IOException { + return getSharedMap(entries, segments, entrySize, OF_4_BYTES); + } + + private static ChronicleMap getSharedMap( + long entries, int segments, int entrySize, Alignment alignment) + throws IOException { + ChronicleMapBuilder mapBuilder = ChronicleMapBuilder.of(CharSequence.class, LongValue.class) + .entries(entries) + .minSegments(segments) + .entrySize(entrySize) + .entryAndValueAlignment(alignment) + .valueMarshallerAndFactory(ByteableLongValueMarshaller.INSTANCE, + DirectLongValueFactory.INSTANCE) + .file(getPersistenceFile()); + return mapBuilder.create(); + } + + +} diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3VoidValueTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3VoidValueTest.java index e116e1681..50015ce16 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3VoidValueTest.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3VoidValueTest.java @@ -25,7 +25,8 @@ import java.io.IOException; import java.net.InetSocketAddress; import static net.openhft.chronicle.map.Builder.newMapVoid; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket @@ -65,9 +66,8 @@ public class TCPSocketReplicationTest3VoidValueTest { assertEquals(null, map3.put(5, null)); // allow time for the recompilation to resolve - waitTillEqual(15000); + assertTrue(waitTillEqual(15000)); - assertEquals(map1, map2); assertEquals(map3, map2); assertTrue(!map1.isEmpty()); @@ -88,9 +88,8 @@ public class TCPSocketReplicationTest3VoidValueTest { map2.put(5,null); // allow time for the recompilation to resolve - waitTillEqual(5000); + assertTrue(waitTillEqual(5000)); - assertEquals(map1, map2); assertEquals(map3, map3); assertTrue(!map1.isEmpty()); @@ -111,22 +110,22 @@ public class TCPSocketReplicationTest3VoidValueTest { map2.put(5, null); // allow time for the recompilation to resolve - waitTillEqual(5000); + assertTrue(waitTillEqual(5000)); - assertEquals(map1, map2); assertEquals(map3, map3); assertTrue(!map1.isEmpty()); } - private void waitTillEqual(final int timeOutMs) throws InterruptedException { + private boolean waitTillEqual(final int timeOutMs) throws InterruptedException { int t = 0; for (; t < timeOutMs; t++) { if (map1.equals(map2) && map1.equals(map3)) - break; + return true; Thread.sleep(1); } + return false; } }