From 8d639908814a0e6e48d9918a144704c83c633412 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Tue, 16 Dec 2014 17:40:01 +0000 Subject: [PATCH 1/3] JLANG-42 Remove unchecked warnings in generated code. --- .../hash/serialization/internal/DataValueBytesMarshallers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DataValueBytesMarshallers.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DataValueBytesMarshallers.java index 1128424d7..c12280146 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DataValueBytesMarshallers.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DataValueBytesMarshallers.java @@ -228,7 +228,7 @@ public final class DataValueBytesMarshallers { String simpleName = tClass.getSimpleName(); String bytesReaderName = bytesReaderName(tClass, false); String simpleReaderName = bytesReaderName(simpleName(tClass)); - sb.append("\npublic class ").append(simpleReaderName) + sb.append("\n@SuppressWarnings(\"unchecked\") public class ").append(simpleReaderName) .append(" implements DeserializationFactoryConfigurableBytesReader<") .append(simpleName).append(", ").append(simpleReaderName).append("> {\n"); declareSerialVersionUID(sb); From db4c9550037aca8f92547cea993ecabb1d152ebe Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 17 Dec 2014 02:16:41 +0700 Subject: [PATCH 2/3] HCOLL-249 fixed Bytes as CharSequence value test case --- src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index ab1222dbc..413ff372e 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -543,7 +543,6 @@ public class CHMUseCasesTest { } } - @Ignore("HCOLL-249 issue when using acquireUsingLocked() with ByteBufferBytes wrong string.length()") @Test public void testAcquireUsingWithByteBufferBytesValue() throws IOException { @@ -564,6 +563,7 @@ public class CHMUseCasesTest { try (WriteContext rc = map.acquireUsingLocked(key, value)) { assertTrue(key instanceof IntValue); assertTrue(value instanceof CharSequence); + value.limit(value.position()); } assertTrue( map.get(key).length() == 0); From 6aaa5b9d6b8f12d9ca5d5b965db652ef4763c067 Mon Sep 17 00:00:00 2001 From: Peter Lawrey Date: Tue, 16 Dec 2014 20:48:18 +0000 Subject: [PATCH 3/3] HCOLL-251 Re-tune the estimates for number of segment and how much entry headroom to use. --- .../chronicle/map/ChronicleMapBuilder.java | 123 ++++++++----- .../chronicle/map/ChronicleMapTest.java | 24 ++- .../chronicle/map/EntryCountMapTest.java | 173 ++++++++++-------- 3 files changed, 189 insertions(+), 131 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index d5c8268d4..c06e2545e 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -66,23 +66,23 @@ import static net.openhft.lang.model.DataValueGenerator.firstPrimitiveFieldType; * ChronicleMapBuilder builder = ChronicleMapBuilder * .of(Key.class, Value.class) * .entries(100500); - * + *

* ChronicleMap map1 = builder.create(); * ChronicleMap map2 = builder.create();} * i. e. created {@code ChronicleMap} instances don't depend on the builder. - * + *

*

{@code ChronicleMapBuilder} is mutable, see a note in {@link ChronicleHashBuilder} interface * documentation. - * + *

*

Later in this documentation, "ChronicleMap" means "ChronicleMaps, created by {@code * ChronicleMapBuilder}", unless specified different, because theoretically someone might provide * {@code ChronicleMap} implementations with completely different properties. - * + *

*

{@code ChronicleMap} ("ChronicleMaps, created by {@code ChronicleMapBuilder}") currently * doesn't support resizing. That is why you should always configure {@linkplain * #entries(long) number of entries} you are going to insert into the created map at most. * See {@link #entries(long)} method documentation for more information on this. - * + *

*

{@code ChronicleMap} allocates memory by equally sized chunks. This size is called {@linkplain * #entrySize(int) entry size}, you are strongly recommended to configure it to achieve least memory * consumption and best speed. See {@link #entrySize(int)} method documentation for more information @@ -220,7 +220,7 @@ public final class ChronicleMapBuilder implements Cloneable, /** * {@inheritDoc} - * + *

*

Example: if keys in your map(s) are English words in {@link String} form, keys size 10 * (a bit more than average English word length) would be a good choice:

{@code
      * ChronicleMap wordFrequencies = ChronicleMapBuilder
@@ -250,7 +250,7 @@ public final class ChronicleMapBuilder implements Cloneable,
 
     /**
      * {@inheritDoc}
-     *
+     * 

*

For example, if your keys are Git commit hashes:

{@code
      * Map gitCommitMessagesByHash =
      *     ChronicleMapBuilder.of(byte[].class, String.class)
@@ -275,10 +275,10 @@ public final class ChronicleMapBuilder implements Cloneable,
      * Configures the optimal number of bytes, taken by serialized form of values, put into maps,
      * created by this builder. If value size is always the same, call {@link
      * #constantValueSizeBySample(Object)} method instead of this one.
-     *
+     * 

*

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is * known statically, it is automatically accounted and shouldn't be specified by user. - * + *

*

If value size varies moderately, specify the size higher than average, but lower than the * maximum possible, to minimize average memory overuse. If value size varies in a wide range, * it's better to use {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it @@ -311,10 +311,10 @@ public final class ChronicleMapBuilder implements Cloneable, * Configures the constant number of bytes, taken by serialized form of values, put into maps, * created by this builder. This is done by providing the {@code sampleValue}, all values should * take the same number of bytes in serialized form, as this sample object. - * + *

*

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is * known statically, it is automatically accounted and this method shouldn't be called. - * + *

*

If value size varies, method {@link #valueSize(int)} or {@link #entrySize(int)} should be * called instead of this one. * @@ -342,16 +342,16 @@ public final class ChronicleMapBuilder implements Cloneable, /** * {@inheritDoc} - * + *

*

In fully default case you can expect entry size to be about 256 bytes. But it is strongly * recommended always to configure {@linkplain #keySize(int) key size} and {@linkplain * #valueSize(int) value size}, if they couldn't be derived statically. - * + *

*

If entry size is not configured explicitly by calling this method, it is computed based on * {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size}, * plus {@linkplain #valueSize(int) value size}, plus a few bytes required by implementations, * with respect to {@linkplain #entryAndValueAlignment(Alignment) alignment}. - * + *

*

Note that the actual entrySize will be aligned by the configured {@linkplain * #entryAndValueAlignment(Alignment) entry and value alignment}. I. e. if you set entry size to * 30, and entry alignment is set to {@link Alignment#OF_4_BYTES}, the actual entry size will be @@ -424,18 +424,18 @@ public final class ChronicleMapBuilder implements Cloneable, /** * Configures alignment strategy of address in memory of entries and independently of address in * memory of values within entries in ChronicleMaps, created by this builder. - * + *

*

Useful when values of the map are updated intensively, particularly fields with volatile * access, because it doesn't work well if the value crosses cache lines. Also, on some * (nowadays rare) architectures any misaligned memory access is more expensive than aligned. - * + *

*

Note that {@linkplain #entrySize(int) entry size} will be aligned according to this * alignment. I. e. if you set {@code entrySize(20)} and {@link Alignment#OF_8_BYTES}, actual * entry size will be 24 (20 aligned to 8 bytes). - * + *

*

If values couldn't reference off-heap memory (i. e. it is not {@link Byteable} or "data * value generated"), alignment configuration makes no sense and forbidden. - * + *

*

Default is {@link Alignment#NO_ALIGNMENT} if values couldn't reference off-heap memory, * otherwise chosen heuristically (configure explicitly for being sure and to compare * performance in your case). @@ -519,9 +519,21 @@ public final class ChronicleMapBuilder implements Cloneable, private long totalEntriesIfPoorDistribution(int segments) { if (segments == 1) return entries; - double poorDistEntriesScale = Math.log(segments) * entries; - return Math.min(entries * segments, - (long) (entries + poorDistEntriesScale * 0.14 + 32)); // 12% was min for tests + long entries = this.entries; + // check if the number of entries is small compared with the number of segments. + long s3 = (long) Math.min(8, segments) * Math.min(32, segments) * Math.min(128, segments); + if (entries * 4 <= s3) + entries *= 1.8; + else if (entries <= s3) + entries *= 1.5; + else if (entries <= s3 * 4) + entries *= 1.35; + else if (entries <= s3 * 8) + entries *= 1.22; + else + entries *= 1.16; + return Math.min(segments * this.entries, + entries + 6 * segments); } @Override @@ -536,23 +548,34 @@ public final class ChronicleMapBuilder implements Cloneable, } private int estimateSegments() { - return (int) Math.min(Maths.nextPower2(entries / 32, 1), estimateSegementBasedOnSize()); + return (int) Math.min(Maths.nextPower2(entries / 32, 1), estimateSegmentsBasedOnSize()); } - private int estimateSegementBasedOnSize() { + private int estimateSegmentsBasedOnSize() { // based on entries with multimap of 100 bytes. - long size = (long) (Math.log10(entrySize + 32) / 2 * entries); - if (size < 10 * 1000) - return 512; - if (size < 100 * 1000) - return 1024; - if (size < 1000 * 1000) - return 2 * 1024; - if (size < 10 * 1000 * 1000) - return 4 * 1024; - if (size < 100 * 1000 * 1000) - return 8 * 1024; - return 16 * 1024; + int segmentsForEntries = estimateSegmentsForEntries(); + return entrySize >= 100000 + ? segmentsForEntries * 8 + : entrySize >= 10000 + ? segmentsForEntries * 4 + : entrySize >= 1000 + ? segmentsForEntries * 2 + : segmentsForEntries; + } + + private int estimateSegmentsForEntries() { + long size = entries; + if (size > 200 << 20) + return 256; + if (size >= 1 << 20) + return 128; + if (size >= 128 << 10) + return 64; + if (size >= 16 << 10) + return 32; + if (size >= 4 << 10) + return 16; + return 1; } @Override @@ -618,12 +641,12 @@ public final class ChronicleMapBuilder implements Cloneable, * Configures if the maps created by this builder should return {@code null} instead of previous * mapped values on {@link ChronicleMap#put(Object, Object) ChornicleMap.put(key, value)} * calls. - * + *

*

{@link Map#put(Object, Object) Map.put()} returns the previous value, functionality which * is rarely used but fairly cheap for {@link HashMap}. In the case, for an off heap collection, * it has to create a new object and deserialize the data from off-heap memory. It's expensive * for something you probably don't use. - * + *

*

By default, of cause, {@code ChronicleMap} conforms the general {@code Map} contract and * returns the previous mapped value on {@code put()} calls. * @@ -645,12 +668,12 @@ public final class ChronicleMapBuilder implements Cloneable, /** * Configures if the maps created by this builder should return {@code null} instead of the last * mapped value on {@link ChronicleMap#remove(Object) ChronicleMap.remove(key)} calls. - * + *

*

{@link Map#remove(Object) Map.remove()} returns the previous value, functionality which is * rarely used but fairly cheap for {@link HashMap}. In the case, for an off heap collection, it * has to create a new object and deserialize the data from off-heap memory. It's expensive for * something you probably don't use. - * + *

*

By default, of cause, {@code ChronicleMap} conforms the general {@code Map} contract and * returns the mapped value on {@code remove()} calls. * @@ -754,7 +777,7 @@ public final class ChronicleMapBuilder implements Cloneable, /** * {@inheritDoc} - * + *

*

Example:

{@code Map map =
      *     ChronicleMapBuilder.of(Key.class, Value.class)
      *     .entries(1_000_000)
@@ -762,7 +785,7 @@ public final class ChronicleMapBuilder implements Cloneable,
      *     // this class hasn't implemented yet, just for example
      *     .objectSerializer(new KryoObjectSerializer())
      *     .create();}
- * + *

*

This serializer is used to serialize both keys and values, if they both require this: * loosely typed, nullable, and custom {@linkplain #keyMarshaller(BytesMarshaller) key} and * {@linkplain #valueMarshaller(BytesMarshaller) value} marshallers are not configured. @@ -832,11 +855,11 @@ public final class ChronicleMapBuilder implements Cloneable, * Configures the marshallers, used to serialize/deserialize values to/from off-heap memory in * maps, created by this builder. See the * section about serialization in ChronicleMap manual for more information. - * + *

*

Configuring marshalling this way results to a little bit more compact in-memory layout of * the map, comparing to a single interface configuration: {@link #valueMarshaller( *BytesMarshaller)}. - * + *

*

Passing {@link BytesInterop} instead of plain {@link BytesWriter} is, of cause, possible, * but currently pointless for values. * @@ -857,7 +880,7 @@ public final class ChronicleMapBuilder implements Cloneable, /** * Configures the marshaller used to serialize actual value sizes to off-heap memory in maps, * created by this builder. - * + *

*

Default value size marshaller is so-called {@linkplain SizeMarshallers#stopBit() stop bit * encoding marshalling}, unless {@link #constantValueSizeBySample(Object)} or the builder * statically knows the value size is constant -- {@link SizeMarshallers#constant(long)} or @@ -879,7 +902,7 @@ public final class ChronicleMapBuilder implements Cloneable, * {@link Byteable}, {@link BytesMarshallable} or {@link Externalizable} subclass, or value type * is eligible for data value generation, or configured custom value reader is an instance of * {@link DeserializationFactoryConfigurableBytesReader}, in maps, created by this builder. - * + *

*

Default value deserialization factory is {@link NewInstanceObjectFactory}, which creates a * new value instance using {@link Class#newInstance()} default constructor. You could provide * an {@link AllocateInstanceObjectFactory}, which uses {@code Unsafe.allocateInstance(Class)} @@ -921,7 +944,7 @@ public final class ChronicleMapBuilder implements Cloneable, /** * Specifies the value to be put for each key queried in {@link ChronicleMap#acquireUsing * acquireUsing()} method, if the key is absent in the map, created by this builder. - * + *

*

This configuration overrides any previous {@link #defaultValueProvider( *DefaultValueProvider)} and {@link #prepareDefaultValueBytes(PrepareValueBytes)} * configurations to this builder. @@ -944,7 +967,7 @@ public final class ChronicleMapBuilder implements Cloneable, /** * Specifies the function to obtain a value for the key during {@link ChronicleMap#acquireUsing * acquireUsing()} calls, if the key is absent in the map, created by this builder. - * + *

*

This configuration overrides any previous {@link #defaultValue(Object)} and {@link * #prepareDefaultValueBytes(PrepareValueBytes)} configurations to this builder. * @@ -966,14 +989,14 @@ public final class ChronicleMapBuilder implements Cloneable, * pointing to or deserialized from, if the key is absent, on {@link ChronicleMap#acquireUsing * acquireUsing()} call on maps, created by this builder. See {@link PrepareValueBytes} for more * information. - * + *

*

This method of value initialization on {@code acquireUsing()} calls is allowed only if * value size is constant. Otherwise you should use either {@link #defaultValue(Object)} or * {@link #defaultValueProvider(DefaultValueProvider)} methods. - * + *

*

This configuration overrides any previous {@link #defaultValue(Object)} and {@link * #defaultValueProvider(DefaultValueProvider)} configurations to this builder. - * + *

*

The default preparation callback zeroes out the value bytes. * * @param prepareValueBytes what to do with the value bytes before assigning them into the diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java index ae1d7bb9f..b11bb10b6 100755 --- a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java @@ -643,24 +643,24 @@ public class ChronicleMapTest { InstantiationException, InterruptedException, ExecutionException { // int runs = Integer.getInteger("runs", 10); int procs = Runtime.getRuntime().availableProcessors(); - int threads = procs * 3; // runs > 100 ? procs / 2 : procs; + if (procs > 8) procs--; + int threads = procs * 3; ExecutorService es = Executors.newFixedThreadPool(procs); - for (int runs : new int[]{10, 50, 250, 500, 1000, 2500}) { - // JAVA 8 produces more garbage than previous versions for internal work. -// System.gc(); + for (int runs : new int[]{1, 2, 5, 10, 25, 50, 100, 500, 1000, 2500}) { + final long entries = runs * 1000 * 1000L; ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, LongValue.class) .entries(entries) .entryAndValueAlignment(OF_8_BYTES) - .actualSegments(8 * 1024) + .actualSegments(256) .entrySize(24); File tmpFile = File.createTempFile("testAcquirePerf", ".deleteme"); tmpFile.deleteOnExit(); final ChronicleMap map = builder.createPersistedTo(tmpFile); - int count = runs > 500 ? runs > 1200 ? 3 : 5 : 5; + int count = runs >= 5 ? 2 : 3; final int independence = Math.min(procs, runs > 500 ? 8 : 4); System.out.println("\nKey size: " + runs + " Million entries. " + builder); for (int j = 0; j < count; j++) { @@ -674,6 +674,7 @@ public class ChronicleMapTest { LongValue value = nativeLongValue(); StringBuilder sb = new StringBuilder(); long next = 50 * 1000 * 1000; + Random rand = new Random(); // use a factor to give up to 10 digit numbers. int factor = Math.max(1, (int) ((10 * 1000 * 1000 * 1000L - 1) / entries)); for (long j = t % independence; j < entries + independence - 1; j += independence) { @@ -681,8 +682,15 @@ public class ChronicleMapTest { sb.append("us:"); sb.append(j * factor); long n; - try (WriteContext wc = map.acquireUsingLocked(sb, value)) { - n = value.addValue(1); + // 75% read + if (rand.nextBoolean() || rand.nextBoolean()) { + try (ReadContext rc = map.getUsingLocked(sb, value)) { + n = rc.present() ? value.getValue() + 1 : 1; + } + } else { + try (WriteContext wc = map.acquireUsingLocked(sb, value)) { + n = value.addValue(1); + } } assert n > 0 && n < 1000 : "Counter corrupted " + n; if (t == 0 && j >= next) { diff --git a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java index 98cdaf909..1e129cadc 100644 --- a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java @@ -62,106 +62,133 @@ public class EntryCountMapTest { @Test public void testVerySmall() throws Exception { - int s = 1, i = 1; - // regression test. - testEntriesMaxSize(s, 1, 1, i); - testEntriesMaxSize(s, 2, 4, i); - testEntriesMaxSize(s, 4, 6, i); - testEntriesMaxSize(s, 8, 13, i); - testEntriesMaxSize(s, 16, 55, i); // todo fix the max. - testEntriesMaxSize(s, 32, 82, i); // todo fix the max. - testEntriesMaxSize(s, 64, 128, i); - testEntriesMaxSize(s, 128, 256, i); - } - - @Test - public void testSmall() throws Exception { - for (int i = 0; i < 5; i++) { + for (int t = 0; t < 5; t++) { + int s = 1; // regression test. - for (int s : new int[]{1, 2, 4, 8}) { - testEntriesMaxSize(s, 256, 510, i); - testEntriesMaxSize(s, 512, 1010, i); - testEntriesMaxSize(s, 1000, 1730, i); - testEntriesMaxSize(s, 2000, 2800, i); - testEntriesMaxSize(s, 4000, 5300, i); - testEntriesMaxSize(s, 5000, 6800, i); - testEntriesMaxSize(s, 8000, 10600, i); - testEntriesMaxSize(s, 12000, 16000, i); - testEntriesMaxSize(s, 16000, 21300, i); + testEntriesMaxSize(s, 1, 1); + testEntriesMaxSize(s, 2, 2); + testEntriesMaxSize(s, 4, 4); + testEntriesMaxSize(s, 8, 8); + testEntriesMaxSize(s, 16, 16); + testEntriesMaxSize(s, 32, 32); + testEntriesMaxSize(s, 64, 64); + testEntriesMaxSize(s, 128, 128); + testEntriesMaxSize(s, 256, 256); + testEntriesMaxSize(s, 512, 512); + + s = 2; + // regression test. + testEntriesMaxSize(s, 8, 16); + testEntriesMaxSize(s, 16, 32); + testEntriesMaxSize(s, 32, 56); + testEntriesMaxSize(s, 64, 95); + testEntriesMaxSize(s, 128, 168); + testEntriesMaxSize(s, 256, 322); + testEntriesMaxSize(s, 512, 640); + + s = 4; + // regression test. + testEntriesMaxSize(s, 32, 70); + testEntriesMaxSize(s, 64, 120); + testEntriesMaxSize(s, 128, 200); + testEntriesMaxSize(s, 256, 380); + testEntriesMaxSize(s, 512, 650); + + for (int s2 : new int[]{1, 2, 4}) { + testEntriesMaxSize(s2, 1000, 1300); + testEntriesMaxSize(s2, 2000, 2500); + testEntriesMaxSize(s2, 4000, 5000); + testEntriesMaxSize(s2, 5000, 6200); + testEntriesMaxSize(s2, 8000, 9900); + testEntriesMaxSize(s2, 12000, 15000); + testEntriesMaxSize(s2, 16000, 20000); } } // hyperbolic average gives more weight to small numbers. System.out.printf("Score: %.2f%n", scoreCount / score); } - @Ignore("HCOLL-251 Investigate test failure of net.openhft.chronicle.map" + - ".EntryCountMapTest#testMedium") + @Test + public void testSmall() throws Exception { + for (int i = 0; i < 5; i++) { + { + int s = 8; + testEntriesMaxSize(s, 250, 400); + testEntriesMaxSize(s, 500, 770); + testEntriesMaxSize(s, 1000, 1400); + testEntriesMaxSize(s, 2000, 2800); + } + // regression test. + for (int s : new int[]{8, 16}) { + testEntriesMaxSize(s, 3000, 4200); + testEntriesMaxSize(s, 5000, 6900); + testEntriesMaxSize(s, 10000, 13000); + } + + for (int s : new int[]{8, 16, 32}) { + testEntriesMaxSize(s, 8000, 11600); + testEntriesMaxSize(s, 12000, 16500); + testEntriesMaxSize(s, 16000, 22000); + testEntriesMaxSize(s, 24000, 31000); + testEntriesMaxSize(s, 32000, 41000); + } + } + // hyperbolic average gives more weight to small numbers. + System.out.printf("Score: %.2f%n", scoreCount / score); + } + + @Ignore("Long running test") @Test public void testMedium() throws Exception { - for (int i = 0; i < 2; i++) { + for (int i = 0; i < 10; i++) { // regression test. - int s = 16; - testEntriesMaxSize(s, 512, 1000, i); - testEntriesMaxSize(s, 1000, 2100, i); - testEntriesMaxSize(s, 2000, 3100, i); - testEntriesMaxSize(s, 4000, 5100, i); - testEntriesMaxSize(s, 5000, 7100, i); - testEntriesMaxSize(s, 8000, 11000, i); - testEntriesMaxSize(s, 12000, 17000, i); - testEntriesMaxSize(s, 16000, 22000, i); - s = 32; - testEntriesMaxSize(s, 2000, 3900, i); - testEntriesMaxSize(s, 4000, 5900, i); - testEntriesMaxSize(s, 5000, 7900, i); - testEntriesMaxSize(s, 8000, 12000, i); - testEntriesMaxSize(s, 12000, 17000, i); - testEntriesMaxSize(s, 16000, 23000, i); - testEntriesMaxSize(s, 32000, 46000, i); - s = 64; - testEntriesMaxSize(s, 5000, 7550, i); - testEntriesMaxSize(s, 8000, 11400, i); - testEntriesMaxSize(s, 12000, 18000, i); - testEntriesMaxSize(s, 16000, 24000, i); - testEntriesMaxSize(s, 32000, 47000, i); - testEntriesMaxSize(s, 40000, 60000, i); - s = 128; - testEntriesMaxSize(s, 16000, 24000, i); - testEntriesMaxSize(s, 32000, 50000, i); - testEntriesMaxSize(s, 40000, 62000, i); - testEntriesMaxSize(s, 64000, 98000, i); - testEntriesMaxSize(s, 129000, 204000, i); - s = 256; - testEntriesMaxSize(s, 64000, 100000, i); - testEntriesMaxSize(s, 129000, 213000, i); + for (int s : new int[]{16, 32, 64}) { + int s3 = s * s * s; + testEntriesMaxSize(s, s3, s3 * 14 / 10); + testEntriesMaxSize(s, s3 * 3 / 2, s3 * 2); + testEntriesMaxSize(s, s3 * 2, s3 * 27 / 10); + } + for (int s : new int[]{128, 256, 512}) { + int s2 = s * s; + testEntriesMaxSize(s, s2, s2 * 15 / 10); + } + for (int s : new int[]{1024, 2048, 4096}) { + int s2 = s * 1024; + testEntriesMaxSize(s, s2, s2 * 14 / 10); + } } // hyperbolic average gives more weight to small numbers. System.out.printf("Score: %.2f%n", scoreCount / score); } @Test - @Ignore ("ignore because its a scalability test") + @Ignore("ignore because its a scalability test") public void testLarge() throws Exception { int s = 8 * 1024; for (int i = 0; i < 10; i++) { System.out.println("i:" + i); int entries = s * s; - testEntriesMaxSize(s, entries * 4 / 5, entries * 4 / 3, i); + testEntriesMaxSize(s, entries * 4 / 5, entries * 4 / 3); // testEntriesMaxSize(s, entries + 100, entries * 3 / 2, i); entries *= 2; - testEntriesMaxSize(s, entries * 4 / 5, entries * 4 / 3, i); + testEntriesMaxSize(s, entries * 4 / 5, entries * 4 / 3); // testEntriesMaxSize(s, entries + 100, entries * 3 / 2, i); } // hyperbolic average gives more weight to small numbers. System.out.printf("Score: %.2f%n", scoreCount / score); } - private void testEntriesMaxSize(int segments, int minSize, int maxSize, int seed) throws IOException { - int counter = minSize + new Random(seed + segments * 1000).nextInt(9999); + private void testEntriesMaxSize(int segments, int minSize, int maxSize) throws IOException { + assert minSize <= maxSize; + Random random = new Random(); + int counter = minSize + random.nextInt(9999 + maxSize); + int stride = 1 + random.nextInt(100); ChronicleMap map = getSharedMap(minSize, segments, 32); LongValue longValue = newDirectReference(LongValue.class); try { - for (int j = 0; j < maxSize * 12 / 10 + 300; j++) { - String key = "key:" + counter++; + for (int j = 0; j < maxSize * 14 / 10 + 300; j++) { + String key = "key:" + counter; + counter += stride; // give a biased hashcode distribution. if ((Integer.bitCount(key.hashCode()) & 7) != 0) { j--; @@ -180,12 +207,12 @@ public class EntryCountMapTest { score += (double) minSize / map.size(); scoreCount++; boolean condition = minSize <= map.size() && map.size() <= maxSize; + long[] a = ((VanillaChronicleMap) map).segmentSizes(); + System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " " + + Arrays.toString(a) + + " sum: " + sum(a)); if (!condition) { - long[] a = ((VanillaChronicleMap) map).segmentSizes(); - System.out.println("segs: " + segments + " min: " + minSize + " was " + map.size() + " " - + Arrays.toString(a) - + " sum: " + sum(a)); - assertTrue("seg: " + segments + ", min: " + minSize + ", size: " + map.size(), condition); + assertTrue("stride: " + stride + ", seg: " + segments + ", min: " + minSize + ", size: " + map.size(), condition); } } finally { map.close();