Alawn/Chronicle-Map
1
0
Fork 0
mirror of https://github.com/OpenHFT/Chronicle-Map.git synced 2021-09-13 01:26:12 +00:00
Code Releases Activity

Tweek the sizing rules as well as optimise the hashing algo so it creates a more even distribution (for at least one set of test cases)

Browse Source
This commit is contained in:
Peter Lawrey
2014-10-23 21:20:46 +01:00
parent 113c51566a
commit 9a8a2512f0
8 changed files with 210 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/net/openhft/chronicle/common/serialization/Hasher.java
+6 -7
View File
@@ -23,11 +23,12 @@ import sun.misc.Unsafe;
import static java.nio.ByteOrder.LITTLE_ENDIAN;
import static java.nio.ByteOrder.nativeOrder;
public final class Hasher {
public enum Hasher {
;
private static final long LONG_LEVEL_PRIME_MULTIPLE = 1011001110001111L;
private static final int SHORT_LEVEL_PRIME_MULTIPLE = 101111;
private static final long BYTE_LEVEL_PRIME_MULTIPLE = 2111;
private static final short SHORT_LEVEL_PRIME_MULTIPLE = 10191;
private static final byte BYTE_LEVEL_PRIME_MULTIPLE = 109;
private static final int INT_HASH_LOW_SHORT_MULTIPLE =
nativeOrder() == LITTLE_ENDIAN ? SHORT_LEVEL_PRIME_MULTIPLE : 1;
@@ -72,10 +73,8 @@ public final class Hasher {
}
public static long hash(long value) {
value *= 11018881818881011L;
value ^= (value >>> 41) ^ (value >>> 21);
value ^= (value >>> 41) - (value >>> 21);
value ^= (value >>> 15) + (value >>> 7);
return value;
}
private Hasher() {}
}
src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java
+4 -4
View File
@@ -390,10 +390,10 @@ public class ChronicleMapBuilder<K, V> implements Cloneable,
if (actualEntriesPerSegment > 0)
return actualEntriesPerSegment;
int as = actualSegments();
long actualEntries =
as == 1 ? entries + 1 : // The extra 1 might not be needed.
as <= 4 ? entries * 10L / 9 : // 10%, assumes you are trying to be tight.
entries * 5L / 4; // 25%
long actualEntries = entries +
(as == 1 ? 1 : // The extra 1 might not be needed.
as <= 8 ? entries / 12 : // 6%, (3% was min for tests)
entries / 6); // 16% (8% was min for tests)
// round up to the next multiple of 64.
return (int) (Math.max(1, actualEntries / as) + 63) & ~63;
src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java
+1 -1
View File
@@ -262,7 +262,7 @@ class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
int segmentNum = getSegment(hash);
int segmentHash = segmentHash(hash);
return segment(segmentNum).acquire(copies, metaKeyWriter, keyWriter, key, value,
segmentHash, create, timeProvider.currentTimeMillis());
segmentHash, create, create ? timeProvider.currentTimeMillis() : 0L);
}
@Override
src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java
+11 -4
View File
@@ -17,9 +17,9 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.common.ChronicleHashErrorListener;
import net.openhft.chronicle.common.serialization.*;
import net.openhft.chronicle.common.threadlocal.Provider;
import net.openhft.chronicle.common.threadlocal.ThreadLocalCopies;
import net.openhft.chronicle.common.serialization.*;
import net.openhft.lang.Maths;
import net.openhft.lang.collection.DirectBitSet;
import net.openhft.lang.collection.SingleThreadedDirectBitSet;
@@ -910,10 +910,10 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
ret = (int) freeList.setNextNContinuousClearBits(0, blocks);
if (ret == DirectBitSet.NOT_FOUND) {
if (blocks == 1) {
throw new IllegalArgumentException(
throw new IllegalStateException(
"Segment is full, no free entries found");
} else {
throw new IllegalArgumentException(
throw new IllegalStateException(
"Segment is full or has no ranges of " + blocks
+ " continuous free blocks"
);
@@ -970,7 +970,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
/**
* Removes a key (or key-value pair) from the Segment.
*
* <p/>
* <p>The entry will only be removed if {@code expectedValue} equals to {@code null}
* or the value previously corresponding to the specified key.
*
@@ -1446,4 +1446,11 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
return super.setValue(value);
}
}
int[] segmentSizes() {
int[] sizes = new int[segments.length];
for (int i = 0; i < segments.length; i++)
sizes[i] = segments[i].getSize();
return sizes;
}
}
src/test/java/net/openhft/chronicle/map/ByteableIntValueMarshaller.java
+6 -2
View File
@@ -63,7 +63,11 @@ enum DirectIntValueFactory implements ObjectFactory<IntValue> {
INSTANCE;
@Override
public IntValue create() throws Exception {
return (IntValue) NativeBytes.UNSAFE.allocateInstance(ByteableIntValueMarshaller.DIRECT);
public IntValue create() {
try {
return (IntValue) NativeBytes.UNSAFE.allocateInstance(ByteableIntValueMarshaller.DIRECT);
} catch (InstantiationException e) {
throw new AssertionError(e);
}
}
}
src/test/java/net/openhft/chronicle/map/ByteableLongValueMarshaller.java
+6 -2
View File
@@ -63,8 +63,12 @@ enum DirectLongValueFactory implements ObjectFactory<LongValue> {
INSTANCE;
@Override
public LongValue create() throws Exception {
return (LongValue) NativeBytes.UNSAFE.allocateInstance(ByteableLongValueMarshaller.DIRECT);
public LongValue create() {
try {
return (LongValue) NativeBytes.UNSAFE.allocateInstance(ByteableLongValueMarshaller.DIRECT);
} catch (InstantiationException e) {
throw new AssertionError(e);
}
}
}
src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java
+168
View File
@@ -0,0 +1,168 @@
package net.openhft.chronicle.map;
import net.openhft.lang.values.LongValue;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import static net.openhft.chronicle.map.Alignment.OF_4_BYTES;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class EntryCountMapTest {
@Test
public void testVerySmall() throws Exception {
int s = 1, i = 1;
// regression test.
testEntriesMaxSize(s, 1, 64, i);
testEntriesMaxSize(s, 2, 64, i);
testEntriesMaxSize(s, 4, 64, i);
testEntriesMaxSize(s, 8, 64, i);
testEntriesMaxSize(s, 16, 64, i);
testEntriesMaxSize(s, 32, 64, i);
testEntriesMaxSize(s, 64, 128, i);
}
@Test
public void testSmall() throws Exception {
for (int i = 0; i < 10; i++) {
// regression test.
for (int s : new int[]{1, 2, 4, 8}) {
if (s <= 4)
testEntriesMaxSize(s, 128, 256, i);
testEntriesMaxSize(s, 256, 510, i);
testEntriesMaxSize(s, 512, 1010, i);
testEntriesMaxSize(s, 1000, 1730, i);
testEntriesMaxSize(s, 2000, 2800, i);
testEntriesMaxSize(s, 4000, 5100, i);
testEntriesMaxSize(s, 5000, 6800, i);
testEntriesMaxSize(s, 8000, 9800, i);
testEntriesMaxSize(s, 12000, 15000, i);
testEntriesMaxSize(s, 16000, 20000, i);
}
}
// hyperbolic average gives more weight to small numbers.
System.out.printf("Score: %.2f%n", scoreCount / score);
}
@Test
public void testMedium() throws Exception {
for (int i = 0; i < 3; i++) {
// regression test.
int s = 16;
testEntriesMaxSize(s, 512, 1000, i);
testEntriesMaxSize(s, 1000, 2100, i);
testEntriesMaxSize(s, 2000, 3100, i);
testEntriesMaxSize(s, 4000, 5100, i);
testEntriesMaxSize(s, 5000, 7100, i);
testEntriesMaxSize(s, 8000, 10100, i);
testEntriesMaxSize(s, 12000, 15100, i);
testEntriesMaxSize(s, 16000, 20200, i);
s = 32;
testEntriesMaxSize(s, 2000, 3900, i);
testEntriesMaxSize(s, 4000, 5900, i);
testEntriesMaxSize(s, 5000, 7900, i);
testEntriesMaxSize(s, 8000, 12000, i);
testEntriesMaxSize(s, 12000, 15900, i);
testEntriesMaxSize(s, 16000, 20100, i);
testEntriesMaxSize(s, 32000, 41000, i);
s = 64;
testEntriesMaxSize(s, 5000, 7550, i);
testEntriesMaxSize(s, 8000, 11400, i);
testEntriesMaxSize(s, 12000, 15400, i);
testEntriesMaxSize(s, 16000, 20000, i);
testEntriesMaxSize(s, 32000, 39400, i);
testEntriesMaxSize(s, 40000, 52200, i);
s = 128;
testEntriesMaxSize(s, 16000, 23000, i);
testEntriesMaxSize(s, 32000, 39000, i);
testEntriesMaxSize(s, 40000, 55200, i);
testEntriesMaxSize(s, 64000, 79800, i);
testEntriesMaxSize(s, 129000, 160000, i);
s = 256;
testEntriesMaxSize(s, 64000, 79800, i);
testEntriesMaxSize(s, 129000, 160000, i);
}
// hyperbolic average gives more weight to small numbers.
System.out.printf("Score: %.2f%n", scoreCount / score);
}
double score = 0;
int scoreCount = 0;
private void testEntriesMaxSize(int segments, int minSize, int maxSize, int seed) throws IOException {
int counter = minSize + new Random(seed + segments * 1000).nextInt(9999);
ChronicleMap<CharSequence, LongValue> map = getSharedMap(minSize, segments, 32);
LongValue longValue = DirectLongValueFactory.INSTANCE.create();
try {
for (int j = 0; j < maxSize * 12 / 10 + 300; j++) {
String key = "key:" + counter++;
// give a biased hashcode distribution.
if ((Integer.bitCount(key.hashCode()) & 7) != 0) {
j--;
continue;
}
map.acquireUsing(key, longValue);
longValue.setValue(1);
}
int[] a = ((VanillaChronicleMap) map).segmentSizes();
System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " "
+ Arrays.toString(a)
+ " sum: " + sum(a));
fail("Expected the map to be full.");
} catch (IllegalStateException e) {
// calculate the hyperbolic average.
score += (double) minSize / map.size();
scoreCount++;
boolean condition = minSize <= map.size() && map.size() <= maxSize;
if (!condition) {
int[] a = ((VanillaChronicleMap) map).segmentSizes();
System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " "
+ Arrays.toString(a)
+ " sum: " + sum(a));
assertTrue("min: " + minSize + ", size: " + map.size(), condition);
}
} finally {
map.close();
}
}
private long sum(int[] ints) {
long sum = 0;
for (int i : ints) {
sum += i;
}
return sum;
}
static File getPersistenceFile() {
String TMP = System.getProperty("java.io.tmpdir");
File file = new File(TMP + "/small-chm-test" + System.nanoTime());
file.deleteOnExit();
return file;
}
private static ChronicleMap<CharSequence, LongValue> getSharedMap(
long entries, int segments, int entrySize) throws IOException {
return getSharedMap(entries, segments, entrySize, OF_4_BYTES);
}
private static ChronicleMap<CharSequence, LongValue> getSharedMap(
long entries, int segments, int entrySize, Alignment alignment)
throws IOException {
ChronicleMapBuilder<CharSequence, LongValue> mapBuilder = ChronicleMapBuilder.of(CharSequence.class, LongValue.class)
.entries(entries)
.minSegments(segments)
.entrySize(entrySize)
.entryAndValueAlignment(alignment)
.valueMarshallerAndFactory(ByteableLongValueMarshaller.INSTANCE,
DirectLongValueFactory.INSTANCE)
.file(getPersistenceFile());
return mapBuilder.create();
}
}
src/test/java/net/openhft/chronicle/map/TCPSocketReplicationTest3VoidValueTest.java
+8 -9
View File
@@ -25,7 +25,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import static net.openhft.chronicle.map.Builder.newMapVoid;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Test ReplicatedChronicleMap where the Replicated is over a TCP Socket
@@ -65,9 +66,8 @@ public class TCPSocketReplicationTest3VoidValueTest {
assertEquals(null, map3.put(5, null));
// allow time for the recompilation to resolve
waitTillEqual(15000);
assertTrue(waitTillEqual(15000));
assertEquals(map1, map2);
assertEquals(map3, map2);
assertTrue(!map1.isEmpty());
@@ -88,9 +88,8 @@ public class TCPSocketReplicationTest3VoidValueTest {
map2.put(5,null);
// allow time for the recompilation to resolve
waitTillEqual(5000);
assertTrue(waitTillEqual(5000));
assertEquals(map1, map2);
assertEquals(map3, map3);
assertTrue(!map1.isEmpty());
@@ -111,22 +110,22 @@ public class TCPSocketReplicationTest3VoidValueTest {
map2.put(5, null);
// allow time for the recompilation to resolve
waitTillEqual(5000);
assertTrue(waitTillEqual(5000));
assertEquals(map1, map2);
assertEquals(map3, map3);
assertTrue(!map1.isEmpty());
}
private void waitTillEqual(final int timeOutMs) throws InterruptedException {
private boolean waitTillEqual(final int timeOutMs) throws InterruptedException {
int t = 0;
for (; t < timeOutMs; t++) {
if (map1.equals(map2) &&
map1.equals(map3))
break;
return true;
Thread.sleep(1);
}
return false;
}
}