diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 7f2042198..04b9b53e7 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -525,15 +525,17 @@ public final class ChronicleMapBuilder implements Cloneable, if (entries * 4 <= s3) entries *= 1.8; else if (entries <= s3) - entries *= 1.5; + entries *= 1.45; else if (entries <= s3 * 4) - entries *= 1.35; + entries *= 1.33; else if (entries <= s3 * 8) entries *= 1.22; + else if (entries <= s3 * 16) + entries *= 1.15; else - entries *= 1.16; + entries *= 1.1; return Math.min(segments * this.entries, - entries + 6 * segments); + entries + 4 * segments + 8); } @Override diff --git a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java index caf91387a..89290d4f8 100644 --- a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java @@ -24,8 +24,15 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; import static net.openhft.chronicle.map.Alignment.OF_4_BYTES; import static net.openhft.lang.model.DataValueClasses.newDirectReference; @@ -35,157 +42,171 @@ import static org.junit.Assert.fail; public class EntryCountMapTest { double score = 0; int scoreCount = 0; + static final String TMP = System.getProperty("java.io.tmpdir"); - static File getPersistenceFile() { - String TMP = System.getProperty("java.io.tmpdir"); - File file = new File(TMP + "/small-chm-test" + System.nanoTime()); + static File getPersistenceFile() throws IOException { + File file = File.createTempFile("ecm-chm-test", ".deleteme"); file.deleteOnExit(); return file; } private static ChronicleMap getSharedMap( long entries, int segments, int entrySize) throws IOException { - return getSharedMap(entries, segments, entrySize, OF_4_BYTES); - } - - private static ChronicleMap getSharedMap( - long entries, int segments, int entrySize, Alignment alignment) - throws IOException { ChronicleMapBuilder mapBuilder = ChronicleMapBuilder.of(CharSequence .class, LongValue.class) .entries(entries) .actualSegments(segments) - .entrySize(entrySize) - .entryAndValueAlignment(alignment); + .entrySize(entrySize); return mapBuilder.createPersistedTo(getPersistenceFile()); } - @Ignore("HCOLL-251 Investigate test failure of net.openhft.chronicle.map.EntryCountMapTest") + static final int ecmTests = Integer.getInteger("ecm.tests", 5); + @Test public void testVerySmall() throws Exception { - for (int t = 0; t < 5; t++) { + System.out.print("testVerySmall seeds"); + for (int t = 0; t < ecmTests; t++) { + System.out.print("."); int s = 1; // regression test. - testEntriesMaxSize(s, 1, 1); - testEntriesMaxSize(s, 2, 2); - testEntriesMaxSize(s, 4, 4); - testEntriesMaxSize(s, 8, 8); - testEntriesMaxSize(s, 16, 16); - testEntriesMaxSize(s, 32, 32); - testEntriesMaxSize(s, 64, 64); - testEntriesMaxSize(s, 128, 128); - testEntriesMaxSize(s, 256, 256); - testEntriesMaxSize(s, 512, 512); + testEntriesMaxSize(s, 1, 1, t); + testEntriesMaxSize(s, 2, 2, t); + testEntriesMaxSize(s, 4, 4, t); + testEntriesMaxSize(s, 8, 8, t); + testEntriesMaxSize(s, 16, 16, t); + testEntriesMaxSize(s, 32, 32, t); + testEntriesMaxSize(s, 64, 64, t); + testEntriesMaxSize(s, 128, 128, t); + testEntriesMaxSize(s, 256, 256, t); + testEntriesMaxSize(s, 512, 512, t); s = 2; // regression test. - testEntriesMaxSize(s, 8, 16); - testEntriesMaxSize(s, 16, 32); - testEntriesMaxSize(s, 32, 56); - testEntriesMaxSize(s, 64, 95); - testEntriesMaxSize(s, 128, 168); - testEntriesMaxSize(s, 256, 322); - testEntriesMaxSize(s, 512, 640); + testEntriesMaxSize(s, 8, 16, t); + testEntriesMaxSize(s, 16, 32, t); + testEntriesMaxSize(s, 32, 60, t); + testEntriesMaxSize(s, 64, 95, t); + testEntriesMaxSize(s, 128, 168, t); + testEntriesMaxSize(s, 256, 322, t); + testEntriesMaxSize(s, 512, 640, t); s = 4; // regression test. - testEntriesMaxSize(s, 32, 70); - testEntriesMaxSize(s, 64, 120); - testEntriesMaxSize(s, 128, 200); - testEntriesMaxSize(s, 256, 380); - testEntriesMaxSize(s, 512, 650); + testEntriesMaxSize(s, 32, 72, t); + testEntriesMaxSize(s, 64, 120, t); + testEntriesMaxSize(s, 128, 200, t); + testEntriesMaxSize(s, 256, 380, t); + testEntriesMaxSize(s, 512, 650, t); for (int s2 : new int[]{1, 2, 4}) { - testEntriesMaxSize(s2, 1000, 1300); - testEntriesMaxSize(s2, 2000, 2500); - testEntriesMaxSize(s2, 4000, 5000); - testEntriesMaxSize(s2, 5000, 6200); - testEntriesMaxSize(s2, 8000, 9900); - testEntriesMaxSize(s2, 12000, 15000); - testEntriesMaxSize(s2, 16000, 20000); + testEntriesMaxSize(s2, 1000, 1300, t); + testEntriesMaxSize(s2, 2000, 2500, t); + testEntriesMaxSize(s2, 4000, 5000, t); + testEntriesMaxSize(s2, 5000, 6200, t); + testEntriesMaxSize(s2, 8000, 9900, t); + testEntriesMaxSize(s2, 12000, 15000, t); + testEntriesMaxSize(s2, 16000, 20000, t); } } // hyperbolic average gives more weight to small numbers. - System.out.printf("Score: %.2f%n", scoreCount / score); + System.out.printf(" Score: %.2f%n", scoreCount / score); } - @Ignore("HCOLL-251 Investigate test failure of net.openhft.chronicle.map.EntryCountMapTest") @Test public void testSmall() throws Exception { - for (int i = 0; i < 5; i++) { + System.out.print("testSmall seeds"); + int procs = Runtime.getRuntime().availableProcessors(); + ExecutorService es = Executors.newFixedThreadPool(procs); + for (int t = 0; t < ecmTests; t++) { + List> futures = new ArrayList<>(); { int s = 8; - testEntriesMaxSize(s, 250, 400); - testEntriesMaxSize(s, 500, 770); - testEntriesMaxSize(s, 1000, 1400); - testEntriesMaxSize(s, 2000, 2800); + futures.add(testEntriesMaxSize(es, s, 250, 400, t)); + futures.add(testEntriesMaxSize(es, s, 500, 770, t)); + futures.add(testEntriesMaxSize(es, s, 1000, 1400, t)); + futures.add(testEntriesMaxSize(es, s, 2000, 2800, t)); } // regression test. for (int s : new int[]{8, 16}) { - testEntriesMaxSize(s, 3000, 4200); - testEntriesMaxSize(s, 5000, 6900); - testEntriesMaxSize(s, 10000, 13000); + futures.add(testEntriesMaxSize(es, s, 3000, 4200, t)); + futures.add(testEntriesMaxSize(es, s, 5000, 6900, t)); + futures.add(testEntriesMaxSize(es, s, 10000, 13000, t)); } - for (int s : new int[]{8, 16, 32}) { - testEntriesMaxSize(s, 8000, 11600); - testEntriesMaxSize(s, 12000, 16500); - testEntriesMaxSize(s, 16000, 22000); - testEntriesMaxSize(s, 24000, 31000); - testEntriesMaxSize(s, 32000, 41000); + for(int s : new int[] { 32, 64, 128}) + futures.add(testEntriesMaxSize(es, s, 250 * s, 250 * s * 3 / 2, t)); + + int s = 32; + for (int e : new int[]{ 16000, 25000, 50000}) + futures.add(testEntriesMaxSize(es, s, e, e * 4 / 3, t)); + s = 64; + for (int e : new int[]{ 25000, 50000, 100000}) + futures.add(testEntriesMaxSize(es, s, e, e * 4 / 3, t)); + s = 128; + for (int e : new int[]{ 50000, 100000, 200000}) + futures.add(testEntriesMaxSize(es, s, e, e * 4 / 3, t)); + for (Future future : futures) { + System.out.print("."); + future.get(); } } + es.shutdown(); // hyperbolic average gives more weight to small numbers. - System.out.printf("Score: %.2f%n", scoreCount / score); + System.out.printf(" Score: %.2f%n", scoreCount / score); } - @Ignore("Long running test") + @Ignore("Long running, large tests test") @Test public void testMedium() throws Exception { - for (int i = 0; i < 10; i++) { + System.out.print("testMedium seeds"); + int procs = Runtime.getRuntime().availableProcessors(); + ExecutorService es = Executors.newFixedThreadPool(procs); + for (int t = 0; t < ecmTests; t++) { // regression test. - for (int s : new int[]{16, 32, 64}) { - int s3 = s * s * s; - testEntriesMaxSize(s, s3, s3 * 14 / 10); - testEntriesMaxSize(s, s3 * 3 / 2, s3 * 2); - testEntriesMaxSize(s, s3 * 2, s3 * 27 / 10); + List> futures = new ArrayList<>(); + for (int s : new int[]{512, 256, 128, 64}) { + int s3 = s * s * 128; + futures.add(testEntriesMaxSize(es, s, s3 * 2, s3 * 24 / 10, t)); + if (s < 512) { + futures.add(testEntriesMaxSize(es, s, s3 * 3 / 2, s3 * 9 / 5, t)); + futures.add(testEntriesMaxSize(es, s, s3, s3 * 12 / 10, t)); + } } - for (int s : new int[]{128, 256, 512}) { - int s2 = s * s; - testEntriesMaxSize(s, s2, s2 * 15 / 10); - } - for (int s : new int[]{1024, 2048, 4096}) { - int s2 = s * 1024; - testEntriesMaxSize(s, s2, s2 * 14 / 10); + for (Future future : futures) { + System.out.print("."); + future.get(); } } + es.shutdown(); // hyperbolic average gives more weight to small numbers. System.out.printf("Score: %.2f%n", scoreCount / score); } - @Test - @Ignore("ignore because its a scalability test") - public void testLarge() throws Exception { - int s = 8 * 1024; - for (int i = 0; i < 10; i++) { - System.out.println("i:" + i); - int entries = s * s; - testEntriesMaxSize(s, entries * 4 / 5, entries * 4 / 3); -// testEntriesMaxSize(s, entries + 100, entries * 3 / 2, i); - entries *= 2; - testEntriesMaxSize(s, entries * 4 / 5, entries * 4 / 3); -// testEntriesMaxSize(s, entries + 100, entries * 3 / 2, i); - } - // hyperbolic average gives more weight to small numbers. - System.out.printf("Score: %.2f%n", scoreCount / score); - } - - private void testEntriesMaxSize(int segments, int minSize, int maxSize) throws IOException { + private Future testEntriesMaxSize(ExecutorService es, final int segments, final int minSize, final int maxSize, final int seed) throws IOException { assert minSize <= maxSize; - Random random = new Random(); + Random random = new Random(seed); + int counter = minSize + random.nextInt(9999 + maxSize); + final int stride = 1 + random.nextInt(100); + final ChronicleMap map = getSharedMap(minSize, segments, 32); + return es.submit(new Callable() { + @Override + public Void call() throws Exception { + testEntriesMaxSize0(segments, minSize, maxSize, seed, stride, map); + return null; + } + }); + } + + void testEntriesMaxSize(int segments, int minSize, int maxSize, int seed) throws IOException { + assert minSize <= maxSize; + Random random = new Random(seed); int counter = minSize + random.nextInt(9999 + maxSize); int stride = 1 + random.nextInt(100); ChronicleMap map = getSharedMap(minSize, segments, 32); + testEntriesMaxSize0(segments, minSize, maxSize, counter, stride, map); + } + + void testEntriesMaxSize0(int segments, int minSize, int maxSize, int counter, int stride, ChronicleMap map) { LongValue longValue = newDirectReference(LongValue.class); try { for (int j = 0; j < maxSize * 14 / 10 + 300; j++) { @@ -199,21 +220,15 @@ public class EntryCountMapTest { map.acquireUsing(key, longValue); longValue.setValue(1); } - long[] a = ((VanillaChronicleMap) map).segmentSizes(); - System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " " - + Arrays.toString(a) - + " sum: " + sum(a)); + dumpMapStats(segments, minSize, map); fail("Expected the map to be full."); } catch (IllegalStateException e) { // calculate the hyperbolic average. score += (double) minSize / map.size(); scoreCount++; boolean condition = minSize <= map.size() && map.size() <= maxSize; - long[] a = ((VanillaChronicleMap) map).segmentSizes(); - System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " " - + Arrays.toString(a) - + " sum: " + sum(a)); if (!condition) { + dumpMapStats(segments, minSize, map); assertTrue("stride: " + stride + ", seg: " + segments + ", min: " + minSize + ", size: " + map.size(), condition); } } finally { @@ -221,6 +236,13 @@ public class EntryCountMapTest { } } + private void dumpMapStats(int segments, int minSize, ChronicleMap map) { + long[] a = ((VanillaChronicleMap) map).segmentSizes(); + System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " " + + Arrays.toString(a) + + " sum: " + sum(a)); + } + private long sum(long[] longs) { long sum = 0; for (long i : longs) {