From 81f6142af7dd9b441f3a20272e84f2a8fbb09496 Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 25 Dec 2014 21:39:26 +0700 Subject: [PATCH] Clearly separated notion of entries and chunks, averageKeySize()/ValueSize() instead of keySize()/valueSize(), minor fixes and refactorings --- README.md | 13 - .../chronicle/hash/ChronicleHashBuilder.java | 183 +++--- .../hash/serialization/SizeMarshaller.java | 4 + .../hash/serialization/SizeMarshallers.java | 20 + .../internal/ByteableMarshaller.java | 10 + .../internal/DoubleMarshaller.java | 10 + .../internal/IntegerMarshaller.java | 10 + .../internal/LongMarshaller.java | 10 + .../internal/VoidMarshaller.java | 10 + .../net/openhft/chronicle/map/Alignment.java | 17 + .../chronicle/map/ChronicleMapBuilder.java | 607 ++++++++++++------ .../chronicle/map/MultiMapFactory.java | 1 + .../chronicle/map/ReplicatedChronicleMap.java | 4 +- .../map/ReplicationHubFindByName.java | 3 +- .../chronicle/map/SerializationBuilder.java | 19 +- .../chronicle/map/StatelessMapConfig.java | 3 +- .../chronicle/map/VanillaChronicleMap.java | 181 +++--- .../chronicle/set/ChronicleSetBuilder.java | 50 +- .../chronicle/set/DummyValueMarshaller.java | 10 + .../java/eg/TestNesterInterfaceMarshal.java | 8 +- .../chronicle/map/CHMUseCasesTest.java | 82 +-- .../map/ChronicleMapImportExportTest.java | 74 ++- .../chronicle/map/ChronicleMapTest.java | 66 +- .../chronicle/map/EntryCountMapTest.java | 5 +- .../net/openhft/chronicle/map/GuavaTest.java | 1 + .../openhft/chronicle/map/Issue42Test.java | 4 +- .../chronicle/map/LargeEntriesTest.java | 15 +- .../openhft/chronicle/map/MapByNameTest.java | 2 +- .../net/openhft/chronicle/map/MarkTest.java | 6 +- ...va => NoUpperBoundChunksPerEntryTest.java} | 6 +- .../openhft/chronicle/map/OSResizesMain.java | 2 +- .../chronicle/map/StatelessClientTest.java | 8 +- .../map/TCPSocketReplication4WayMapTest.java | 2 +- ...ReplicationWithAcquireUsingLockedTest.java | 8 +- ...sSpanningSeveralChunksReplicationHub.java} | 19 +- .../map/example/LotsOfEntriesMain.java | 16 +- .../chronicle/map/example/StringsMapMain.java | 5 +- .../OpenJDKAndHashMapExamplesTest.java | 6 +- .../fromdocs/StringEncodingExamplesTest.java | 2 +- .../pingpong_latency/PingPongCASLeft.java | 2 +- .../map/jsr166/map/ChronicleMapTest.java | 1 + .../jsr166/map/StatelessChronicleMapTest.java | 1 + .../map/utility/ProcessInstanceLimiter.java | 2 +- .../net/openhft/chronicle/set/Issue3Test.java | 2 +- 44 files changed, 923 insertions(+), 587 deletions(-) rename src/test/java/net/openhft/chronicle/map/{NoEntryOversizeFactorTest.java => NoUpperBoundChunksPerEntryTest.java} (85%) rename src/test/java/net/openhft/chronicle/map/{TestOversizedReplicationHub.java => TestEntriesSpanningSeveralChunksReplicationHub.java} (87%) diff --git a/README.md b/README.md index d53322fd3..c70e942cb 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ Click here to get the [Latest Version Number](http://search.maven.org/#search%7C * [Entries](https://github.com/OpenHFT/Chronicle-Map#entries) * [Size of Space Reserved on Disk](https://github.com/OpenHFT/Chronicle-Map#size-of-space-reserved-on-disk) * [Chronicle Map Interface](https://github.com/OpenHFT/Chronicle-Map#chronicle-map-interface) -* [Oversized Entries Support] (https://github.com/OpenHFT/Chronicle-Map/blob/master/README.md#oversized-entries-support) * [Serialization](https://github.com/OpenHFT/Chronicle-Map#serialization) * [Simple Types](https://github.com/OpenHFT/Chronicle-Map#simple-types) * [Complex Types](https://github.com/OpenHFT/Chronicle-Map#complex-types) @@ -599,18 +598,6 @@ If after some business logic, in our example after reading the 'issueDate' and 'context' is already aware of the entries location in memory. So it will be quicker to call `context.removeEntry()` rather than `map.remove(key)`. -## Oversized Entries Support - -It is possible for the size of your entry to be larger than the space which chronicle map -internally allocates to each entry. We refer to this type of entry as an oversized entry. -Oversized entries are there to cater for the case -where only a small percentage of your entries are larger than the others, in this case your large entry will -span across a number of entries. The alternative would be to increase the internal entry size to be similar -to the size of the largest entry, but this approach is wasteful of memory, especially when most -entries are no where near the max entry size. So put simply Chronicle map is able to accept entries -where each can have a different size but it does this in a way that is efficient. - - ## Serialization ![Serialization](http://openhft.net/wp-content/uploads/2014/09/Serialization_01.jpg) diff --git a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java index 8dce56d17..4ffa24c73 100755 --- a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java +++ b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java @@ -55,6 +55,11 @@ import java.util.concurrent.TimeUnit; * the corresponding configuration changed. To make an independent configuration, {@linkplain * #clone} the builder. * + *

There are some "low-level" configurations in this builder, + * that require deep understanding of the Chronicle implementation design to be properly used. + * Know what you do. These configurations are picked up strictly as-is, without extra round-ups, + * adjustments, etc. + * * @param the type of keys in hash containers, created by this builder * @param the container type, created by this builder, i. e. {@link ChronicleMap} or {@link * ChronicleSet} @@ -83,27 +88,29 @@ public interface ChronicleHashBuilderIf key size varies moderately, specify the size higher than average, but lower than the - * maximum possible, to minimize average memory overuse. If key size varies in a wide range, - * it's better to use {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it - * directly. + *

{@code ChronicleHashBuilder} implementation heuristically chooses + * {@linkplain #actualChunkSize(int) the actual chunk size} based on this configuration, that, + * however, might result to quite high internal fragmentation, i. e. losses because only + * integral number of chunks could be allocated for the entry. If you want to avoid this, you + * should manually configure the actual chunk size in addition to this average key size + * configuration, which is anyway needed. * *

If key is a boxed primitive type or {@link Byteable} subclass, i. e. if key size is known * statically, it is automatically accounted and shouldn't be specified by user. * - * @param keySize number of bytes, taken by serialized form of keys + * @param averageKeySize the average number of bytes, taken by serialized form of keys * @return this builder back * @throws IllegalStateException if key size is known statically and shouldn't be configured * by user * @throws IllegalArgumentException if the given {@code keySize} is non-positive * @see #constantKeySizeBySample(Object) - * @see #entrySize(int) + * @see #actualChunkSize(int) */ - B keySize(int keySize); + B averageKeySize(double averageKeySize); /** * Configures the constant number of bytes, taken by serialized form of keys, put into hash @@ -113,12 +120,12 @@ public interface ChronicleHashBuilderIf keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size is * known statically, it is automatically accounted and this method shouldn't be called. * - *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be - * called instead of this one. + *

If key size varies, method {@link #averageKeySize(double)} should be called instead of + * this one. * * @param sampleKey the sample key * @return this builder back - * @see #keySize(int) + * @see #averageKeySize(double) */ B constantKeySizeBySample(K sampleKey); @@ -130,104 +137,124 @@ public interface ChronicleHashBuilderentry size is a minimum allocation portion in the - * hash containers, created by this builder. E. g. if entry size is 100, the created container + * ChronicleSet} should allocate. So chunk size is the minimum allocation portion in the + * hash containers, created by this builder. E. g. if chunk size is 100, the created container * could only allocate 100, 200, 300... bytes for an entry. If say 150 bytes of entry space are - * required by the entry, 200 bytes will be allocated, 150 used and 50 wasted. To minimize - * memory overuse and improve speed, you should pay decent attention to this configuration. + * required by the entry, 200 bytes will be allocated, 150 used and 50 wasted. This is called + * internal fragmentation. * - *

There are three major patterns of this configuration usage:

  1. Key (and value, in - * {@code ChronicleMap} case) sizes are constant. Configure them via {@link - * #constantKeySizeBySample(Object)} and {@link ChronicleMapBuilder#constantValueSizeBySample(Object)} - * methods, and you will experience no memory waste at all.
  2. Key (and/or value size, in - * {@code ChronicleMap} case) varies moderately. Specify them using corresponding methods, or - * specify entry size directly by calling this method, by sizes somewhere between average and - * maximum possible. The idea is to have most (90% or more) entries to fit a single "entry size" - * with moderate memory waste (10-20% on average), rest 10% or less of entries should take 2 - * "entry sizes", thus with ~50% memory overuse.
  3. Key (and/or value size, in {@code - * ChronicleMap} case) varies in a wide range. Then it's best to use entry size configuration in - * chunk mode. Specify entry size so that most entries should take from 5 to several - * dozens of "chunks". With this approach, average memory waste should be very low. + *

    To minimize memory overuse and improve speed, you should pay decent attention to this + * configuration. Alternatively, you can just trust the heuristics and doesn't configure + * the chunk size. * - *

    However, remember that

    • Operations with entries that span several "entry sizes" - * are a bit slower, than entries which take a single "entry size". That is why "chunk" approach - * is not recommended, when key (and/or value size) varies moderately.
    • The maximum - * number of chunks could be taken by an entry is {@link #maxEntryOversizeFactor(int)}, of max - * 64 chunks. {@link IllegalArgumentException} is thrown on attempt to insert too large entry, - * compared to the configured or computed entry size.
    • The number of "entries" - * {@linkplain #entries(long) in the whole map} and {@linkplain #actualEntriesPerSegment(long) - * per segment} is actually the number "chunks".
    + *

    Specify chunk size so that most entries would take from 5 to several dozens of chunks. + * However, remember that operations with entries that span several chunks are a bit slower, + * than with entries which take a single chunk. Particularly avoid entries to take more than + * 64 chunks. * *

    Example: if values in your {@code ChronicleMap} are adjacency lists of some social graph, * where nodes are represented as {@code long} ids, and adjacency lists are serialized in * efficient manner, for example as {@code long[]} arrays. Typical number of connections is - * 100-300, maximum is 3000. In this case entry size of - * 50 * (8 bytes for each id) = 400 bytes would be a good choice:

    {@code
    +     * 100-300, maximum is 3000. In this case chunk size of
    +     * 30 * (8 bytes for each id) = 240 bytes would be a good choice: 
    {@code
          * Map socialGraph = ChronicleMapOnHeapUpdatableBuilder
          *     .of(Long.class, long[].class)
    -     *     // given that graph should have of 1 billion nodes, and 150 average adjacency list size
    -     *     // => values takes 3 chuncks on average
    -     *     .entries(1_000_000_000L * (150 / 50))
    -     *     .entrySize(50 * 8)
    +     *     .entries(1_000_000_000L)
    +     *     .averageValueSize(150 * 8) // 150 is average adjacency list size
    +     *     .actualChunkSize(30 * 8) // average 5-6 chunks per entry
          *     .create();}
    - * It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max chunks by - * single entry" limit, and ensures moderate average memory overuse (not more than 20%).
  4. - *
* - * @param entrySize the "chunk size" in bytes + *

This is a low-level configuration. The configured number + * of bytes is strictly used as-is, without anything like round-up to the multiple of 8 or + * 16, or any other adjustment. + * + * @param actualChunkSize the "chunk size" in bytes * @return this builder back * @see #entries(long) - * @see #maxEntryOversizeFactor(int) + * @see #maxChunksPerEntry(int) */ - B entrySize(int entrySize); + B actualChunkSize(int actualChunkSize); /** - * Configures how much the actual entry size is allowed to be larger than configured or derived - * {@linkplain #entrySize(int) entry size}. + * Configures how many chunks a single entry, inserted into {@code ChronicleHash}es, created + * by this builder, could take. If you try to insert larger entry, {@link IllegalStateException} + * is fired. This is useful as self-check, that you configured chunk size right and you + * keys (and values, in {@link ChronicleMap} case) take expected number of bytes. For example, + * if {@link #constantKeySizeBySample(Object)} is configured or key size is statically known + * to be constant (boxed primitives, data value generated implementations, {@link Byteable}s, + * etc.), and the same for value objects in {@code ChronicleMap} case, max chunks per entry + * is configured to 1, to ensure keys and values are actually constantly-sized. * - * @param maxEntryOversizeFactor number of times the actual entry size could oversize the - * configured "entry size" (chunk size, actually) + * @param maxChunksPerEntry how many chunks a single entry could span at most * @return this builder back - * @throws IllegalArgumentException if the given {@code maxEntryOversizeFactor} is lesser than 1 + * @throws IllegalArgumentException if the given {@code maxChunksPerEntry} is lesser than 1 * or greater than 64 + * @see #actualChunkSize(int) */ - B maxEntryOversizeFactor(int maxEntryOversizeFactor); + B maxChunksPerEntry(int maxChunksPerEntry); /** - * Configures the maximum number of {@linkplain #entrySize(int) "entry size chunks"}, which - * could be taken by the maximum number of entries, inserted into the hash containers, created - * by this builder. If you try to insert more data, {@link IllegalStateException} might be - * thrown, because currently {@link ChronicleMap} and {@link ChronicleSet} doesn't support - * resizing. + * Configures the maximum number of entries, that could be inserted into the hash containers, + * created by this builder. If you try to insert more data, {@link IllegalStateException} + * might be thrown, because currently {@link ChronicleMap} and {@link ChronicleSet} + * don't support resizing. * - *

  1. If key size (and value size, in {@code ChronicleMap} case) is constant, this number - * is equal to the maximum number of entries (because each entry takes exactly one "entry size" - * memory unit).
  2. If key (and/or value, in {@code ChronicleMap} case) size varies - * moderately, you should pass to this method the maximum number of entries + 5-25%, depending - * on your data properties and configured {@linkplain #keySize(int) key}/{@linkplain - * ChronicleMapBuilder#valueSize(int) - * value}/{@linkplain #entrySize(int) entry} sizes.
  3. If your data size varies in a wide - * range, pass the maximum number of entries multiplied by average data size and divided by the - * configured "entry size" (i. e. chunk size). See an example in the documentation to {@link - * #entrySize(int)} method.
- * - *

You shouldn't put additional margin over the number, computed according the rules above. + *

You shouldn't put additional margin over the actual maximum number of entries. * This bad practice was popularized by {@link HashMap#HashMap(int)} and {@link - * HashSet#HashSet(int)} constructors, which accept "capacity", that should be multiplied by - * "load factor" to obtain actual maximum expected number of entries. {@code ChronicleMap} and - * {@code ChronicleSet} don't have a notion of load factor. + * HashSet#HashSet(int)} constructors, which accept capacity, that should be multiplied + * by load factor to obtain the actual maximum expected number of entries. + * {@code ChronicleMap} and {@code ChronicleSet} don't have a notion of load factor. * - *

Default value is 2^20 (~ 1 million). + *

Default maximum entries is 2^20 (~ 1 million). * - * @param entries maximum size of the created maps, in memory allocation units, so-called "entry - * size" + * @param entries maximum size of the maps or sets, created by this builder * @return this builder back - * @see #entrySize(int) */ B entries(long entries); - B actualEntriesPerSegment(long actualEntriesPerSegment); + /** + * Configures the actual maximum number entries, that could be inserted into any single segment + * of the hash containers, created by this builder. Configuring both the actual number of + * entries per segment and {@linkplain #actualSegments(int) actual segments} replaces a single + * {@link #entries(long)} configuration. + * + *

This is a low-level configuration. + * + * @param entriesPerSegment the actual maximum number entries per segment in the + * hash containers, created by this builder + * @return this builder back + * @see #entries(long) + * @see #actualSegments(int) + */ + B entriesPerSegment(long entriesPerSegment); + /** + * Configures the actual number of chunks, that will be reserved for any single segment of the + * hash containers, created by this builder. This configuration is a lower-level version of + * {@link #entriesPerSegment(long)}. Makes sense only if {@link #actualChunkSize(int)}, + * {@link #actualSegments(int)} and {@link #entriesPerSegment(long)} are also configured + * manually. + * + * @param actualChunksPerSegment the actual number of segments, reserved per segment in the + * hash containers, created by this builder + * @return this builder back + */ + B actualChunksPerSegment(long actualChunksPerSegment); + + /** + * Configures the actual number of segments in the hash containers, created by this builder. + * With {@linkplain #entriesPerSegment(long) actual number of segments}, this + * configuration replaces a single {@link #entries(long)} call. + * + *

This is a low-level configuration. The configured number + * is used as-is, without anything like round-up to the closest power of 2. + * + * @param actualSegments the actual number of segments in hash containers, created by + * this builder + * @return this builder back + * @see #minSegments(int) + * @see #entriesPerSegment(long) + */ B actualSegments(int actualSegments); /** diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java index d564e45e3..5c988298d 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java @@ -26,6 +26,10 @@ public interface SizeMarshaller extends Serializable { int sizeEncodingSize(long size); + int minSizeEncodingSize(); + + int maxSizeEncodingSize(); + void writeSize(Bytes bytes, long size); long readSize(Bytes bytes); diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshallers.java b/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshallers.java index 7c006738f..490a34a58 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshallers.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshallers.java @@ -36,6 +36,16 @@ public final class SizeMarshallers { return stopBitLength(size); } + @Override + public int minSizeEncodingSize() { + return 1; + } + + @Override + public int maxSizeEncodingSize() { + return 9; + } + @Override public void writeSize(Bytes bytes, long size) { bytes.writeStopBit(size); @@ -65,6 +75,16 @@ public final class SizeMarshallers { return 0; } + @Override + public int minSizeEncodingSize() { + return 0; + } + + @Override + public int maxSizeEncodingSize() { + return 0; + } + @Override public void writeSize(Bytes bytes, long size) { // do nothing diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java index bb39475c7..0f32d5c03 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java @@ -69,6 +69,16 @@ public abstract class ByteableMarshaller return 0; } + @Override + public int minSizeEncodingSize() { + return 0; + } + + @Override + public int maxSizeEncodingSize() { + return 0; + } + @Override public void writeSize(Bytes bytes, long size) { // do nothing diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DoubleMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DoubleMarshaller.java index 2a13e1577..20fb35773 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DoubleMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DoubleMarshaller.java @@ -39,6 +39,16 @@ public enum DoubleMarshaller implements BytesInterop, BytesReader builder = ChronicleMapBuilder * .of(Key.class, Value.class) * .entries(100500); - *

+ * * ChronicleMap map1 = builder.create(); * ChronicleMap map2 = builder.create();} - * i. e. created {@code ChronicleMap} instances don't depend on the builder.

{@code - * ChronicleMapBuilder} is mutable, see a note in {@link ChronicleHashBuilder} interface - * documentation.

Later in this documentation, "ChronicleMap" means "ChronicleMaps, created - * by {@code ChronicleMapBuilder}", unless specified different, because theoretically someone might - * provide {@code ChronicleMap} implementations with completely different properties.

{@code - * ChronicleMap} ("ChronicleMaps, created by {@code ChronicleMapBuilder}") currently doesn't support - * resizing. That is why you should always configure {@linkplain #entries(long) number of - * entries} you are going to insert into the created map at most. See {@link #entries(long)} - * method documentation for more information on this.

{@code ChronicleMap} allocates memory - * by equally sized chunks. This size is called {@linkplain #entrySize(int) entry size}, you are - * strongly recommended to configure it to achieve least memory consumption and best speed. See - * {@link #entrySize(int)} method documentation for more information on this. + * i. e. created {@code ChronicleMap} instances don't depend on the builder. + * + *

{@code ChronicleMapBuilder} is mutable, see a note in {@link ChronicleHashBuilder} interface + * documentation. + * + *

Later in this documentation, "ChronicleMap" means "ChronicleMaps, created by {@code + * ChronicleMapBuilder}", unless specified different, because theoretically someone might provide + * {@code ChronicleMap} implementations with completely different properties. + * + *

{@code ChronicleMap} ("ChronicleMaps, created by {@code ChronicleMapBuilder}") currently + * doesn't support resizing. That is why you must configure {@linkplain + * #entries(long) number of entries} you are going to insert into the created map at most. + * See {@link #entries(long)} method documentation for more information on this. + * + *

If you key or value type is not constantly sized and known to {@code ChronicleHashBuilder}, + * i. e. it is not a boxed primitive, data value generated interface, {@link Byteable}, etc. (see + * the complete list TODO insert the link to the complete list), + * you must provide the {@code ChronicleHashBuilder} with some information about you keys + * or values: if they are constantly-sized, call {@link #constantKeySizeBySample(Object)}, otherwise + * {@link ChronicleHashBuilder#averageKeySize(double)} method, accordingly for values. * * @param key type of the maps, produced by this builder * @param value type of the maps, produced by this builder @@ -93,6 +98,7 @@ public final class ChronicleMapBuilder implements Cloneable, static final byte UDP_REPLICATION_MODIFICATION_ITERATOR_ID = (byte) 128; private static final int DEFAULT_KEY_OR_VALUE_SIZE = 120; + private static final long DEFAULT_ENTRIES = 1 << 20; private static final int MAX_SEGMENTS = (1 << 30); private static final int MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS = (1 << 20); private static final Logger LOG = @@ -101,6 +107,15 @@ public final class ChronicleMapBuilder implements Cloneable, private static final StringBuilder EMTRY_STRING_BUILDER = new StringBuilder(); + private static final double UNDEFINED_DOUBLE_CONFIG = Double.NaN; + + private static boolean isDefined(double config) { + return !Double.isNaN(config); + } + + private static final boolean strictStateChecks = + Boolean.getBoolean("chronicle.strictStateChecks"); + SerializationBuilder keyBuilder; SerializationBuilder valueBuilder; @@ -110,15 +125,16 @@ public final class ChronicleMapBuilder implements Cloneable, private int minSegments = -1; private int actualSegments = -1; // used when reading the number of entries per - private long actualEntriesPerSegment = -1L; - private int keySize = 0; + private long entriesPerSegment = -1L; + private long actualChunksPerSegment = -1L; + private double averageKeySize = UNDEFINED_DOUBLE_CONFIG; private K sampleKey; - private int valueSize = 0; + private double averageValueSize = UNDEFINED_DOUBLE_CONFIG; private V sampleValue; - private int entrySize = 0; - private int maxEntryOversizeFactor = -1; + private int actualChunkSize = 0; + private int maxChunksPerEntry = -1; private Alignment alignment = null; - private long entries = 1 << 20; // 1 million by default + private long entries = -1; private long lockTimeOut = 20000; private TimeUnit lockTimeOutUnit = TimeUnit.MILLISECONDS; private int metaDataBytes = 0; @@ -214,36 +230,37 @@ public final class ChronicleMapBuilder implements Cloneable, } /** - * {@inheritDoc}

Example: if keys in your map(s) are English words in {@link String} - * form, keys size 10 - * (a bit more than average English word length) would be a good choice:

{@code
+     * {@inheritDoc}
+     *
+     * 

Example: if keys in your map(s) are English words in {@link String} form, average English + * word length is 5.1, configure average key size of 6:

{@code
      * ChronicleMap wordFrequencies = ChronicleMapBuilder
      *     .of(String.class, LongValue.class)
      *     .entries(50000)
-     *     .keySize(10)
-     *     // shouldn't specify valueSize(), because it is statically known
+     *     .averageKeySize(6)
      *     .create();}
- * (Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded (and - * each character takes 2 bytes on-heap), because default off-heap {@link String} encoding is - * UTF-8 in {@code ChronicleMap}.) + * (Note that 6 is chosen as average key size in bytes despite strings in Java are UTF-16 + * encoded (and each character takes 2 bytes on-heap), because default off-heap {@link String} + * encoding is UTF-8 in {@code ChronicleMap}.) * * @throws IllegalStateException {@inheritDoc} * @throws IllegalArgumentException {@inheritDoc} * @see #constantKeySizeBySample(Object) - * @see #valueSize(int) - * @see #entrySize(int) + * @see #averageValueSize(double) + * @see #actualChunkSize(int) + * @param averageKeySize */ @Override - public ChronicleMapBuilder keySize(int keySize) { + public ChronicleMapBuilder averageKeySize(double averageKeySize) { checkSizeIsNotStaticallyKnown(keyBuilder); - if (keySize <= 0) - throw new IllegalArgumentException("Key size must be positive"); - this.keySize = keySize; + checkAverageSize(averageKeySize, "key"); + this.averageKeySize = averageKeySize; return this; } /** - * {@inheritDoc}

+ * {@inheritDoc} + * *

For example, if your keys are Git commit hashes:

{@code
      * Map gitCommitMessagesByHash =
      *     ChronicleMapBuilder.of(byte[].class, String.class)
@@ -251,7 +268,7 @@ public final class ChronicleMapBuilder implements Cloneable,
      *     .immutableKeys()
      *     .create();}
* - * @see #keySize(int) + * @see ChronicleHashBuilder#averageKeySize(double) * @see #constantValueSizeBySample(Object) */ @Override @@ -260,37 +277,49 @@ public final class ChronicleMapBuilder implements Cloneable, return this; } - private int keySize() { - return keyOrValueSize(keySize, keyBuilder); + private double averageKeySize() { + return averageKeyOrValueSize(averageKeySize, keyBuilder); } /** - * Configures the optimal number of bytes, taken by serialized form of values, put into maps, + * Configures the average number of bytes, taken by serialized form of values, put into maps, * created by this builder. If value size is always the same, call {@link - * #constantValueSizeBySample(Object)} method instead of this one.

If values are of boxed - * primitive type or {@link Byteable} subclass, i. e. if value size is known statically, it is - * automatically accounted and shouldn't be specified by user.

If value size varies - * moderately, specify the size higher than average, but lower than the maximum possible, to - * minimize average memory overuse. If value size varies in a wide range, it's better to use - * {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it directly. + * #constantValueSizeBySample(Object)} method instead of this one. * - * @param valueSize number of bytes, taken by serialized form of values + *

{@code ChronicleHashBuilder} implementation heuristically chooses + * {@linkplain #actualChunkSize(int) the actual chunk size} based on this configuration and + * the key size, that, however, might result to quite high internal fragmentation, + * i. e. losses because only integral number of chunks could be allocated for the entry. If you + * want to avoid this, you should manually configure the actual chunk size in addition to this + * average value size configuration, which is anyway needed. + * + *

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is + * known statically, it is automatically accounted and shouldn't be specified by user. + * + * @param averageValueSize number of bytes, taken by serialized form of values * @return this builder back * @throws IllegalStateException if value size is known statically and shouldn't be * configured by user - * @throws IllegalArgumentException if the given {@code valueSize} is non-positive + * @throws IllegalArgumentException if the given {@code averageValueSize} is non-positive * @see #constantValueSizeBySample(Object) - * @see #keySize(int) - * @see #entrySize(int) + * @see ChronicleHashBuilder#averageKeySize(double) + * @see #actualChunkSize(int) */ - public ChronicleMapBuilder valueSize(int valueSize) { + public ChronicleMapBuilder averageValueSize(double averageValueSize) { checkSizeIsNotStaticallyKnown(valueBuilder); - if (valueSize <= 0) - throw new IllegalArgumentException("Value size must be positive"); - this.valueSize = valueSize; + checkAverageSize(averageValueSize, "value"); + this.averageValueSize = averageValueSize; return this; } + private static void checkAverageSize(double averageSize, String role) { + if (averageSize <= 0 || Double.isNaN(averageSize) || + Double.isInfinite(averageSize)) { + throw new IllegalArgumentException("Average " + role + " size must be a positive, " + + "finite number"); + } + } + private static void checkSizeIsNotStaticallyKnown(SerializationBuilder builder) { if (builder.sizeIsStaticallyKnown) throw new IllegalStateException("Size of type " + builder.eClass + @@ -300,15 +329,17 @@ public final class ChronicleMapBuilder implements Cloneable, /** * Configures the constant number of bytes, taken by serialized form of values, put into maps, * created by this builder. This is done by providing the {@code sampleValue}, all values should - * take the same number of bytes in serialized form, as this sample object.

If values are - * of boxed primitive type or {@link Byteable} subclass, i. e. if value size is known - * statically, it is automatically accounted and this method shouldn't be called.

If - * value size varies, method {@link #valueSize(int)} or {@link #entrySize(int)} should be called - * instead of this one. + * take the same number of bytes in serialized form, as this sample object. + * + *

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is + * known statically, it is automatically accounted and this method shouldn't be called. + * + *

If value size varies, method {@link #averageValueSize(double)} should be called instead of + * this one. * * @param sampleValue the sample value * @return this builder back - * @see #valueSize(int) + * @see #averageValueSize(double) * @see #constantKeySizeBySample(Object) */ public ChronicleMapBuilder constantValueSizeBySample(V sampleValue) { @@ -316,12 +347,12 @@ public final class ChronicleMapBuilder implements Cloneable, return this; } - int valueSize() { - return keyOrValueSize(valueSize, valueBuilder); + double averageValueSize() { + return averageKeyOrValueSize(averageValueSize, valueBuilder); } - private int keyOrValueSize(int configuredSize, SerializationBuilder builder) { - if (configuredSize > 0) + private double averageKeyOrValueSize(double configuredSize, SerializationBuilder builder) { + if (isDefined(configuredSize)) return configuredSize; if (builder.constantSizeMarshaller()) return builder.pseudoReadConstantSize(); @@ -329,110 +360,195 @@ public final class ChronicleMapBuilder implements Cloneable, } /** - * {@inheritDoc}

In fully default case you can expect entry size to be about 256 bytes. - * But it is strongly recommended always to configure {@linkplain #keySize(int) key size} and - * {@linkplain #valueSize(int) value size}, if they couldn't be derived statically.

If - * entry size is not configured explicitly by calling this method, it is computed based on - * {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size}, - * plus {@linkplain #valueSize(int) value size}, plus a few bytes required by implementations, - * with respect to {@linkplain #entryAndValueAlignment(Alignment) alignment}.

Note that - * the actual entrySize will be aligned by the configured {@linkplain - * #entryAndValueAlignment(Alignment) entry and value alignment}. I. e. if you set entry size to - * 30, and entry alignment is set to {@link Alignment#OF_4_BYTES}, the actual entry size will be - * 32 (30 aligned to 4 bytes). + * {@inheritDoc} * * @throws IllegalStateException is sizes of both keys and values of maps created by this - * builder are statically known, hence entry size shouldn't be + * builder are constant, hence chunk size shouldn't be * configured by user * @see #entryAndValueAlignment(Alignment) * @see #entries(long) - * @see #maxEntryOversizeFactor(int) + * @see #maxChunksPerEntry(int) */ @Override - public ChronicleMapBuilder entrySize(int entrySize) { - if (keyBuilder.sizeIsStaticallyKnown && valueBuilder.sizeIsStaticallyKnown) { + public ChronicleMapBuilder actualChunkSize(int actualChunkSize) { + if (constantlySizedEntries()) { throw new IllegalStateException("Sizes of key type: " + keyBuilder.eClass + " and " + - "value type: " + valueBuilder.eClass + " are both statically known, " + - "so entry size shouldn't be specified manually"); + "value type: " + valueBuilder.eClass + " are both constant, " + + "so chunk size shouldn't be specified manually"); } - if (entrySize <= 0) - throw new IllegalArgumentException("Entry Size must be positive"); - this.entrySize = entrySize; + if (actualChunkSize <= 0) + throw new IllegalArgumentException("Chunk size must be positive"); + this.actualChunkSize = actualChunkSize; return this; } - int entrySize(boolean replicated) { - if (entrySize > 0) - return entryAndValueAlignment().alignSize(entrySize); - int size = metaDataBytes; - int keySize = keySize(); - size += keyBuilder.sizeMarshaller().sizeEncodingSize(keySize); + double averageEntrySize(boolean replicated) { + double size = metaDataBytes; + double keySize = averageKeySize(); + size += averageSizeEncodingSize(keyBuilder, keySize); size += keySize; if (replicated) size += ReplicatedChronicleMap.ADDITIONAL_ENTRY_BYTES; - int valueSize = valueSize(); - size += valueBuilder.sizeMarshaller().sizeEncodingSize(valueSize); - size = entryAndValueAlignment().alignSize(size); - size += valueSize; - // Some cache line heuristics - for (int i = 1; i <= 4; i++) { - int bound = i * 64; - // Not more than 5% oversize. - // DEFAULT_KEY_OR_VALUE_SIZE and this heuristic are specially adjusted to produce - // entry size 256 -- the default value prior to keySize and valueSize -- when key and - // value sizes are both not static nor configured, in both vanilla and replicated modes. - if (size < bound && (bound - size) <= bound / 20) { - size = bound; - break; - } - } - return entryAndValueAlignment().alignSize(size); - } - - @Override - public ChronicleMapBuilder maxEntryOversizeFactor(int maxEntryOversizeFactor) { - if (maxEntryOversizeFactor < 1) - throw new IllegalArgumentException("maxEntryOversizeFactor should be more than 1, " + - maxEntryOversizeFactor + " given"); - this.maxEntryOversizeFactor = maxEntryOversizeFactor; - return this; - } - - private void checkMaxEntryOversizeFactorIfReplicated(boolean replicated) { - if (replicated && maxEntryOversizeFactor > MAX_ENTRY_OVERSIZE_FACTOR) - throw new IllegalStateException("As of now Replicated Chornicle Map doesn't support " + - "more than " + MAX_ENTRY_OVERSIZE_FACTOR + " entry oversize factor, " + - maxEntryOversizeFactor + " is configured"); - } - - int maxEntryOversizeFactor(boolean replicated) { - if (maxEntryOversizeFactor < 0) { - if (keyBuilder.constantSizeMarshaller() && valueBuilder.constantSizeMarshaller()) { - return 1; - } else { - if (replicated) { - return MAX_ENTRY_OVERSIZE_FACTOR; + double valueSize = averageValueSize(); + size += averageSizeEncodingSize(valueBuilder, valueSize); + Alignment alignment = valueAlignment(); + if (alignment != Alignment.NO_ALIGNMENT) { + if (constantlySizedKeys() && valueBuilder.constantSizeEncodingSizeMarshaller()) { + long constantSizeBeforeAlignment = round(size); + if (constantlySizedValues()) { + // see specialEntrySpaceOffset() + long totalDataSize = constantSizeBeforeAlignment + constantValueSize(); + size += alignment.alignAddr(totalDataSize) - totalDataSize; } else { - return Integer.MAX_VALUE; + if (actualChunkSize > 0) { + size += averageAlignmentAssumingChunkSize(constantSizeBeforeAlignment, + actualChunkSize); + } else { + for (int chunkSize : new int[]{8, 4}) { + double expectedAverageAlignment = averageAlignmentAssumingChunkSize( + constantSizeBeforeAlignment, chunkSize); + if (size + expectedAverageAlignment + valueSize >= + maxDefaultChunksPerAverageEntry(replicated) * chunkSize || + chunkSize == 4) { + size += expectedAverageAlignment; + break; + } + } + } + } + } else { + if (alignment.alignment() > 0) { + // assume worst case, we always lose most possible bytes for alignment + size += alignment.alignment() - 1; } } } - return maxEntryOversizeFactor; + size += valueSize; + return size; + } + + /** + * This is needed, if chunkSize = constant entry size is not aligned, for entry alignment to + * be always the same, we should _misalign_ the first chunk. + */ + int specialEntrySpaceOffset(boolean replicated) { + if (!constantlySizedEntries() || valueAlignment() == Alignment.NO_ALIGNMENT) + return 0; + return (int) (constantValueSize() % valueAlignment().alignment()); + } + + private long constantValueSize() { + return valueBuilder.pseudoReadConstantSize(); + } + + private boolean constantlySizedKeys() { + return keyBuilder.constantSizeMarshaller(); + } + + private static double averageSizeEncodingSize( + SerializationBuilder builder, double averageSize) { + SizeMarshaller sizeMarshaller = builder.sizeMarshaller(); + if (averageSize == round(averageSize)) + return sizeMarshaller.sizeEncodingSize(round(averageSize)); + long lower = (long) averageSize; + long upper = lower + 1; + int lowerEncodingSize = sizeMarshaller.sizeEncodingSize(lower); + int upperEncodingSize = sizeMarshaller.sizeEncodingSize(upper); + if (lowerEncodingSize == upperEncodingSize) + return lowerEncodingSize; + return lower * (upper - averageSize) + upper * (averageSize - lower); + } + + private double averageAlignmentAssumingChunkSize( + long constantSizeBeforeAlignment, int chunkSize) { + if (valueAlignment() == Alignment.NO_ALIGNMENT) + return 0.0; + Alignment valueAlignment = valueAlignment(); + long firstAlignment = valueAlignment.alignAddr(constantSizeBeforeAlignment) - + constantSizeBeforeAlignment; + int alignment = valueAlignment.alignment(); + int gcdOfAlignmentAndChunkSize = greatestCommonDivisor(alignment, chunkSize); + if (gcdOfAlignmentAndChunkSize == alignment) + return firstAlignment; + // assume worst by now because we cannot predict alignment in VanillaCM.entrySize() method + // before allocation + return alignment - gcdOfAlignmentAndChunkSize; + } + + static int greatestCommonDivisor(int a, int b) { + if (b == 0) return a; + return greatestCommonDivisor(b, a % b); + } + + long chunkSize(boolean replicated) { + if (actualChunkSize > 0) + return actualChunkSize; + double averageEntrySize = averageEntrySize(replicated); + if (constantlySizedEntries()) + return round(averageEntrySize); + int maxChunkSize = 1 << 30; + for (long chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) { + if (maxDefaultChunksPerAverageEntry(replicated) * chunkSize > averageEntrySize) + return chunkSize; + } + return maxChunkSize; + } + + boolean constantlySizedEntries() { + return constantlySizedKeys() && constantlySizedValues(); + } + + double averageChunksPerEntry(boolean replicated) { + if (constantlySizedEntries()) + return 1.0; + long chunkSize = chunkSize(replicated); + // assuming we always has worst internal fragmentation. This affects total segment + // entry space which is allocated lazily on Linux (main target platform) + // so we can afford this + return (averageEntrySize(replicated) + chunkSize - 1) / chunkSize; + } + + private static int maxDefaultChunksPerAverageEntry(boolean replicated) { + return replicated ? 4 : 8; + } + + @Override + public ChronicleMapBuilder maxChunksPerEntry(int maxChunksPerEntry) { + if (maxChunksPerEntry < 1) + throw new IllegalArgumentException("maxChunksPerEntry should be >= 1, " + + maxChunksPerEntry + " given"); + this.maxChunksPerEntry = maxChunksPerEntry; + return this; + } + + int maxChunksPerEntry() { + if (maxChunksPerEntry > 0) + return maxChunksPerEntry; + if (constantlySizedEntries()) { + return 1; + } else { + return Integer.MAX_VALUE; + } + } + + private boolean constantlySizedValues() { + return valueBuilder.constantSizeMarshaller(); } /** * Configures alignment strategy of address in memory of entries and independently of address in - * memory of values within entries in ChronicleMaps, created by this builder.

Useful when - * values of the map are updated intensively, particularly fields with volatile access, because - * it doesn't work well if the value crosses cache lines. Also, on some (nowadays rare) - * architectures any misaligned memory access is more expensive than aligned.

Note that - * {@linkplain #entrySize(int) entry size} will be aligned according to this alignment. I. e. if - * you set {@code entrySize(20)} and {@link Alignment#OF_8_BYTES}, actual entry size will be 24 - * (20 aligned to 8 bytes).

If values couldn't reference off-heap memory (i. e. it is not - * {@link Byteable} or "data value generated"), alignment configuration makes no sense and - * forbidden.

Default is {@link Alignment#NO_ALIGNMENT} if values couldn't reference - * off-heap memory, otherwise chosen heuristically (configure explicitly for being sure and to - * compare performance in your case). + * memory of values within entries in ChronicleMaps, created by this builder. + * + *

Useful when values of the map are updated intensively, particularly fields with volatile + * access, because it doesn't work well if the value crosses cache lines. Also, on some + * (nowadays rare) architectures any misaligned memory access is more expensive than aligned. + * + *

If values couldn't reference off-heap memory (i. e. it is not {@link Byteable} or "data + * value generated"), alignment configuration makes no sense and forbidden. + * + *

Default is {@link Alignment#NO_ALIGNMENT} if values couldn't reference off-heap memory, + * otherwise chosen heuristically (configure explicitly for being sure and to compare + * performance in your case). * * @param alignment the new alignment of the maps constructed by this builder * @return this {@code ChronicleMapOnHeapUpdatableBuilder} back @@ -452,7 +568,7 @@ public final class ChronicleMapBuilder implements Cloneable, "if values might point to off-heap memory"); } - Alignment entryAndValueAlignment() { + Alignment valueAlignment() { if (alignment != null) return alignment; Class firstPrimitiveFieldType = firstPrimitiveFieldType(valueBuilder.eClass); @@ -463,11 +579,6 @@ public final class ChronicleMapBuilder implements Cloneable, return Alignment.NO_ALIGNMENT; } - /** - * Package-private for tests - */ - - @Override public ChronicleMapBuilder entries(long entries) { if (entries <= 0L) @@ -477,43 +588,80 @@ public final class ChronicleMapBuilder implements Cloneable, } long entries() { + if (entries < 0) + return DEFAULT_ENTRIES; return entries; } @Override - public ChronicleMapBuilder actualEntriesPerSegment(long actualEntriesPerSegment) { - if (actualEntriesPerSegment <= 0L) - throw new IllegalArgumentException("entries per segment should be positive, " + - actualEntriesPerSegment + " given"); - if (tooManyEntriesPerSegment(actualEntriesPerSegment)) - throw new IllegalArgumentException("max entries per segment is " + - MultiMapFactory.MAX_CAPACITY + ", " + actualEntriesPerSegment + " given"); - this.actualEntriesPerSegment = actualEntriesPerSegment; + public ChronicleMapBuilder entriesPerSegment(long entriesPerSegment) { + if (entriesPerSegment <= 0L) + throw new IllegalArgumentException("Entries per segment should be positive, " + + entriesPerSegment + " given"); + this.entriesPerSegment = entriesPerSegment; return this; } - private boolean tooManyEntriesPerSegment(long entriesPerSegment) { - return entriesPerSegment > MultiMapFactory.MAX_CAPACITY; + long entriesPerSegment(boolean replicated) { + long entriesPerSegment; + if (this.entriesPerSegment > 0L) { + entriesPerSegment = this.entriesPerSegment; + } else { + int actualSegments = actualSegments(replicated); + long totalEntries = totalEntriesIfPoorDistribution(actualSegments); + entriesPerSegment = divideUpper(totalEntries, actualSegments); + } + if (actualChunksPerSegment > 0) + return entriesPerSegment; + double averageChunksPerEntry = averageChunksPerEntry(replicated); + if (entriesPerSegment * averageChunksPerEntry > MultiMapFactory.MAX_CAPACITY) + throw new IllegalStateException("Max chunks per segment is " + + MultiMapFactory.MAX_CAPACITY + " configured entries() and " + + "actualSegments() so that there should be " + entriesPerSegment + + " entries per segment, while average chunks per entry is " + + averageChunksPerEntry); + return entriesPerSegment; } - long actualEntriesPerSegment() { - if (actualEntriesPerSegment > 0L) - return actualEntriesPerSegment; - int actualSegments = actualSegments(); - long actualEntries = totalEntriesIfPoorDistribution(actualSegments); - long actualEntriesPerSegment = divideUpper(actualEntries, actualSegments); - if (tooManyEntriesPerSegment(actualEntriesPerSegment)) - throw new IllegalStateException("max entries per segment is " + - MultiMapFactory.MAX_CAPACITY + " configured entries() and " + - "actualSegments() so that there should be " + actualEntriesPerSegment + - " entries per segment"); - return actualEntriesPerSegment; + @Override + public ChronicleMapBuilder actualChunksPerSegment(long actualChunksPerSegment) { + if (actualChunksPerSegment <= 0) + throw new IllegalArgumentException("Actual chunks per segment should be positive, " + + actualChunksPerSegment + " given"); + this.actualChunksPerSegment = actualChunksPerSegment; + return this; + } + + private void checkActualChunksPerSegmentIsConfiguredOnlyIfOtherLowLevelConfigsAreManual() { + if (actualChunksPerSegment > 0) { + if (entriesPerSegment <= 0 || (actualChunkSize <= 0 && !constantlySizedEntries()) || + actualSegments <= 0) + throw new IllegalStateException("Actual chunks per entry could be configured " + + "only if other three low level configs are manual: " + + "entriesPerSegment(), actualSegments() and actualChunkSize(), unless " + + "both keys and value sizes are constant"); + } + } + + private void checkActualChunksPerSegmentGreaterOrEqualToEntries() { + if (actualChunksPerSegment > 0 && entriesPerSegment > 0 && + entriesPerSegment > actualChunksPerSegment) { + throw new IllegalStateException("Entries per segment couldn't be greater than " + + "actual chunks per segment. Entries: " + entriesPerSegment + ", " + + "chunks: " + actualChunksPerSegment + " is configured"); + } + } + + long actualChunksPerSegment(boolean replicated) { + if (actualChunksPerSegment > 0) + return actualChunksPerSegment; + return round(entriesPerSegment(replicated) * averageChunksPerEntry(replicated)); } private long totalEntriesIfPoorDistribution(int segments) { if (segments == 1) - return entries; - long entries = this.entries; + return entries(); + long entries = entries(); // check if the number of entries is small compared with the number of segments. long s3 = (long) Math.min(8, segments) * Math.min(32, segments) * Math.min(128, segments); if (entries * 4 <= s3) @@ -528,7 +676,7 @@ public final class ChronicleMapBuilder implements Cloneable, entries *= 1.15; else entries *= 1.1; - return Math.min(segments * this.entries, + return Math.min(segments * entries(), entries + 4 * segments + 8); } @@ -544,19 +692,19 @@ public final class ChronicleMapBuilder implements Cloneable, } private int estimateSegments() { - return (int) Math.min(Maths.nextPower2(entries / 32, 1), estimateSegmentsBasedOnSize()); + return (int) Math.min(Maths.nextPower2(entries() / 32, 1), estimateSegmentsBasedOnSize()); } private int estimateSegmentsBasedOnSize() { // based on entries with multimap of 100 bytes. - int segmentsForEntries = estimateSegmentsForEntries(entries); - return entrySize >= 1000000 + int segmentsForEntries = estimateSegmentsForEntries(entries()); + return actualChunkSize >= 1000000 ? segmentsForEntries * 16 - : entrySize >= 100000 + : actualChunkSize >= 100000 ? segmentsForEntries * 8 - : entrySize >= 10000 + : actualChunkSize >= 10000 ? segmentsForEntries * 4 - : entrySize >= 1000 + : actualChunkSize >= 1000 ? segmentsForEntries * 2 : segmentsForEntries; } @@ -584,36 +732,51 @@ public final class ChronicleMapBuilder implements Cloneable, return this; } - int actualSegments() { + int actualSegments(boolean replicated) { if (actualSegments > 0) return actualSegments; + if (entriesPerSegment > 0) { + long segments = 1; + for (int i = 0; i < 3; i++) { // try to converge + long totalEntries = totalEntriesIfPoorDistribution((int) segments); + segments = divideUpper(totalEntries, entriesPerSegment); + if (segments > MAX_SEGMENTS) + throw new IllegalStateException(); + } + if (minSegments > 0) + segments = Math.max(minSegments, segments); + return (int) segments; + } long shortMMapSegments = trySegments(MultiMapFactory.I16_MAX_CAPACITY, - MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS); + MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS, replicated); + // TODO why not try I24 multiMap? if (shortMMapSegments > 0L) return (int) shortMMapSegments; - long intMMapSegments = trySegments(MultiMapFactory.MAX_CAPACITY, MAX_SEGMENTS); + long intMMapSegments = trySegments(MultiMapFactory.MAX_CAPACITY, MAX_SEGMENTS, replicated); if (intMMapSegments > 0L) return (int) intMMapSegments; throw new IllegalStateException("Max segments is " + MAX_SEGMENTS + ", configured so much" + - " entries() that builder automatically decided to use " + + " entries (" + entries() + ") or average chunks per entry is too high (" + + averageChunksPerEntry(replicated) + ") that builder automatically decided to use " + (-intMMapSegments) + " segments"); } - private long trySegments(long maxSegmentCapacity, int maxSegments) { - long segments = divideUpper(totalEntriesIfPoorDistribution(minSegments()), - maxSegmentCapacity); + private long trySegments(long maxSegmentCapacity, int maxSegments, boolean replicated) { + long totalChunks = round(totalEntriesIfPoorDistribution(minSegments()) * + averageChunksPerEntry(replicated)); + long segments = divideUpper(totalChunks, maxSegmentCapacity); segments = Maths.nextPower2(Math.max(segments, minSegments()), 1L); return segments <= maxSegments ? segments : -segments; } - int segmentHeaderSize() { - int segments = actualSegments(); + int segmentHeaderSize(boolean replicated) { + int segments = actualSegments(replicated); // reduce false sharing unless we have a lot of segments. return segments <= 16 * 1024 ? 64 : 32; } - MultiMapFactory multiMapFactory() { - return MultiMapFactory.forCapacity(actualEntriesPerSegment()); + MultiMapFactory multiMapFactory(boolean replicated) { + return MultiMapFactory.forCapacity(actualChunksPerSegment(replicated)); } public ChronicleMapBuilder lockTimeOut(long lockTimeOut, TimeUnit unit) { @@ -703,13 +866,14 @@ public final class ChronicleMapBuilder implements Cloneable, "name=" + name + ", actualSegments=" + pretty(actualSegments) + ", minSegments=" + pretty(minSegments) + - ", actualEntriesPerSegment=" + pretty(actualEntriesPerSegment) + - ", keySize=" + pretty(keySize) + + ", entriesPerSegment=" + pretty(entriesPerSegment) + + ", actualChunksPerSegment=" + pretty(actualChunksPerSegment) + + ", averageKeySize=" + pretty(averageKeySize) + ", sampleKeyForConstantSizeComputation=" + pretty(sampleKey) + - ", valueSize=" + pretty(valueSize) + + ", averageValueSize=" + pretty(averageValueSize) + ", sampleValueForConstantSizeComputation=" + pretty(sampleValue) + - ", entrySize=" + pretty(entrySize) + - ", entryAndValueAlignment=" + entryAndValueAlignment() + + ", actualChunkSize=" + pretty(actualChunkSize) + + ", valueAlignment=" + valueAlignment() + ", entries=" + entries() + ", lockTimeOut=" + lockTimeOut + " " + lockTimeOutUnit + ", metaDataBytes=" + metaDataBytes() + @@ -773,7 +937,7 @@ public final class ChronicleMapBuilder implements Cloneable, *

Example:

{@code Map map =
      *     ChronicleMapBuilder.of(Key.class, Value.class)
      *     .entries(1_000_000)
-     *     .keySize(50).valueSize(200)
+     *     .averageKeySize(50).averageValueSize(200)
      *     // this class hasn't implemented yet, just for example
      *     .objectSerializer(new KryoObjectSerializer())
      *     .create();}
@@ -997,7 +1161,7 @@ public final class ChronicleMapBuilder implements Cloneable, } private void checkPrepareValueBytesOnlyIfConstantValueSize() { - if (prepareValueBytes != null && !valueBuilder.constantSizeMarshaller()) + if (prepareValueBytes != null && !constantlySizedValues()) throw new IllegalStateException("Prepare value bytes could be used only if " + "value size is constant"); } @@ -1005,11 +1169,12 @@ public final class ChronicleMapBuilder implements Cloneable, PrepareValueBytesAsWriter prepareValueBytesAsWriter() { PrepareValueBytes prepareValueBytes = this.prepareValueBytes; if ((prepareValueBytes == null && defaultValueProvider() != null) || - !valueBuilder.constantSizeMarshaller()) + !constantlySizedValues()) return null; + long constantValueSize = constantValueSize(); if (prepareValueBytes == null) - prepareValueBytes = new ZeroOutValueBytes<>(valueSize()); - return new PrepareValueBytesAsWriter<>(prepareValueBytes, valueSize()); + prepareValueBytes = new ZeroOutValueBytes<>(constantValueSize); + return new PrepareValueBytesAsWriter<>(prepareValueBytes, constantValueSize); } /** @@ -1207,21 +1372,35 @@ public final class ChronicleMapBuilder implements Cloneable, keyBuilder.objectSerializer(acquireObjectSerializer(JDKObjectSerializer.INSTANCE)); valueBuilder.objectSerializer(acquireObjectSerializer(JDKObjectSerializer.INSTANCE)); - long maxSize = (long) entrySize(replicated) * figureBufferAllocationFactor(replicated); - keyBuilder.maxSize(maxSize); - valueBuilder.maxSize(maxSize); + double largeKeySize = averageKeySize(); + if (!constantlySizedKeys()) + largeKeySize *= figureBufferAllocationFactor(); + keyBuilder.maxSize(round(largeKeySize)); + double largeValueSize = averageValueSize(); + if (!constantlySizedValues()) + largeValueSize *= figureBufferAllocationFactor(); + valueBuilder.maxSize(round(largeValueSize)); if (sampleKey != null) keyBuilder.constantSizeBySample(sampleKey); if (sampleValue != null) valueBuilder.constantSizeBySample(sampleValue); - stateChecks(replicated); + stateChecks(); } - private void stateChecks(boolean replicated) { + private void stateChecks() { + if (strictStateChecks) { + if (entries < 0) + throw new IllegalStateException("Entries must be specified"); + if (!constantlySizedKeys() && !isDefined(averageKeySize)) + throw new IllegalStateException("No info about key size"); + if (!constantlySizedValues() && !isDefined(averageValueSize)) + throw new IllegalStateException("No info about value size"); + } checkAlignmentOnlyIfValuesPossiblyReferenceOffHeap(); checkPrepareValueBytesOnlyIfConstantValueSize(); - checkMaxEntryOversizeFactorIfReplicated(replicated); + checkActualChunksPerSegmentIsConfiguredOnlyIfOtherLowLevelConfigsAreManual(); + checkActualChunksPerSegmentGreaterOrEqualToEntries(); } private ChronicleMap establishReplication( @@ -1257,11 +1436,11 @@ public final class ChronicleMapBuilder implements Cloneable, return map; } - private int figureBufferAllocationFactor(boolean replicated) { + private int figureBufferAllocationFactor() { // if expected map size is about 1000, seems rather wasteful to allocate // key and value serialization buffers each x64 of expected entry size.. return (int) Math.min(Math.max(2L, entries() >> 10), - maxEntryOversizeFactor(replicated)); + Math.min(64, maxChunksPerEntry())); } } diff --git a/src/main/java/net/openhft/chronicle/map/MultiMapFactory.java b/src/main/java/net/openhft/chronicle/map/MultiMapFactory.java index d88de2fb7..49fcaf1ba 100644 --- a/src/main/java/net/openhft/chronicle/map/MultiMapFactory.java +++ b/src/main/java/net/openhft/chronicle/map/MultiMapFactory.java @@ -74,6 +74,7 @@ enum MultiMapFactory { public static MultiMapFactory forCapacity(long capacity) { if (capacity <= ShortShortMultiMap.MAX_CAPACITY) return I16; + // TODO why not I24? // if (capacity <= Int24Int24MultiMap.MAX_CAPACITY) // return I24; if (capacity <= IntIntMultiMap.MAX_CAPACITY) diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 066a0aab5..64ee58d25 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -150,7 +150,7 @@ final class ReplicatedChronicleMap implements FindByName { this.map = (Map) ChronicleMapBuilder .of(CharSequence.class, MapInstanceConfig.class) - .entrySize(4000) + .averageKeySize(10) + .averageValueSize(4000) .entries(128) .eventListener(listener) .instance() diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index eef36ec81..77ba08d36 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -274,7 +274,7 @@ final class SerializationBuilder implements Cloneable, Serializable { } long constantSize = metaInterop.size(interop, sampleObject); if (sizeIsStaticallyKnown) { - int expectedConstantSize = pseudoReadConstantSize(); + long expectedConstantSize = pseudoReadConstantSize(); if (constantSize != expectedConstantSize) { throw new IllegalStateException("Although configuring constant size by sample " + "is not forbidden for types which size we already know statically, they " + @@ -306,11 +306,22 @@ final class SerializationBuilder implements Cloneable, Serializable { } boolean constantSizeMarshaller() { - return sizeMarshaller().sizeEncodingSize(0L) == 0; + try { + return sizeMarshaller().sizeEncodingSize(1L) == 0; + } catch (Exception e) { + // size marshaller thrown likely IllegalArgumentException, that means it doesn't + // expect size 1. marshaller of constant size shouldn't do that, it's body should be + // just `return 0;` + return false; + } } - int pseudoReadConstantSize() { - return (int) sizeMarshaller().readSize(EMPTY_BYTES); + boolean constantSizeEncodingSizeMarshaller() { + return sizeMarshaller().minSizeEncodingSize() == sizeMarshaller().maxSizeEncodingSize(); + } + + long pseudoReadConstantSize() { + return sizeMarshaller().readSize(EMPTY_BYTES); } public SerializationBuilder sizeMarshaller(SizeMarshaller sizeMarshaller) { diff --git a/src/main/java/net/openhft/chronicle/map/StatelessMapConfig.java b/src/main/java/net/openhft/chronicle/map/StatelessMapConfig.java index 256e9eefa..204b317a2 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessMapConfig.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessMapConfig.java @@ -19,6 +19,7 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.hash.StatelessClientConfig; +import net.openhft.lang.MemoryUnit; import java.io.IOException; import java.net.InetSocketAddress; @@ -35,7 +36,7 @@ class StatelessMapConfig implements StatelessClientConfig, @@ -62,11 +63,6 @@ class VanillaChronicleMap, private static final Logger LOG = LoggerFactory.getLogger(VanillaChronicleMap.class); - /** - * Because DirectBitSet implementations couldn't find more than 64 continuous clear or set - * bits. - */ - static final int MAX_ENTRY_OVERSIZE_FACTOR = 64; private static final long serialVersionUID = 2L; final Class kClass; final String dataFileVersion; @@ -84,10 +80,13 @@ class VanillaChronicleMap, final DefaultValueProvider defaultValueProvider; final PrepareValueBytesAsWriter prepareValueBytesAsWriter; final int metaDataBytes; - // private final int replicas; - final long entrySize; + final long chunkSize; final Alignment alignment; + final boolean constantlySizedEntry; + final int worstAlignment; + final int specialEntrySpaceOffset; final int actualSegments; + final long actualChunksPerSegment; final long entriesPerSegment; final MapEventListener eventListener; final BytesMapEventListener bytesEventListener; @@ -101,7 +100,7 @@ class VanillaChronicleMap, private final HashSplitting hashSplitting; final Class nativeValueClass; final MultiMapFactory multiMapFactory; - final int maxEntryOversizeFactor; + final int maxChunksPerEntry; transient Provider> keyReaderProvider; transient Provider keyInteropProvider; transient Provider> valueReaderProvider; @@ -152,22 +151,30 @@ class VanillaChronicleMap, lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); boolean replicated = getClass() == ReplicatedChronicleMap.class; - this.entrySize = builder.entrySize(replicated); - this.alignment = builder.entryAndValueAlignment(); + this.chunkSize = builder.chunkSize(replicated); + this.alignment = builder.valueAlignment(); + this.constantlySizedEntry = builder.constantlySizedEntries(); + int alignment = this.alignment.alignment(); + if (this.alignment != Alignment.NO_ALIGNMENT) { + this.worstAlignment = alignment - greatestCommonDivisor((int) chunkSize, alignment); + } else { + this.worstAlignment = 0; + } + this.specialEntrySpaceOffset = builder.specialEntrySpaceOffset(replicated); this.errorListener = builder.errorListener(); this.putReturnsNull = builder.putReturnsNull(); this.removeReturnsNull = builder.removeReturnsNull(); - this.actualSegments = builder.actualSegments(); - this.entriesPerSegment = builder.actualEntriesPerSegment(); - this.multiMapFactory = builder.multiMapFactory(); + this.actualSegments = builder.actualSegments(replicated); + this.actualChunksPerSegment = builder.actualChunksPerSegment(replicated); + this.entriesPerSegment = builder.entriesPerSegment(replicated); + this.multiMapFactory = builder.multiMapFactory(replicated); this.metaDataBytes = builder.metaDataBytes(); this.eventListener = builder.eventListener(); this.bytesEventListener = builder.bytesEventListener(); - this.segmentHeaderSize = builder.segmentHeaderSize(); - this.maxEntryOversizeFactor = - builder.maxEntryOversizeFactor(this instanceof ReplicatedChronicleMap); + this.segmentHeaderSize = builder.segmentHeaderSize(replicated); + this.maxChunksPerEntry = builder.maxChunksPerEntry(); hashSplitting = HashSplitting.Splitting.forSegments(actualSegments); initTransients(); @@ -319,11 +326,11 @@ class VanillaChronicleMap, } private long sizeOfMultiMapBitSet() { - return MultiMapFactory.sizeOfBitSetInBytes(entriesPerSegment); + return MultiMapFactory.sizeOfBitSetInBytes(actualChunksPerSegment); } - private long sizeOfBitSets() { - return CACHE_LINES.align(BYTES.alignAndConvert(entriesPerSegment, BITS), BYTES); + private long sizeOfSegmentFreeListBitSets() { + return CACHE_LINES.align(BYTES.alignAndConvert(actualChunksPerSegment, BITS), BYTES); } private int numberOfBitSets() { @@ -333,8 +340,9 @@ class VanillaChronicleMap, private long segmentSize() { long ss = (CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES) * multiMapsPerSegment()) - + numberOfBitSets() * sizeOfBitSets() // the free list and 0+ dirty lists. - + sizeOfEntriesInSegment(); + // the free list and 0+ dirty lists. + + numberOfBitSets() * sizeOfSegmentFreeListBitSets() + + sizeOfEntrySpaceInSegment(); if ((ss & 63L) != 0) throw new AssertionError(); return breakL1CacheAssociativityContention(ss); @@ -356,8 +364,9 @@ class VanillaChronicleMap, return 1; } - private long sizeOfEntriesInSegment() { - return CACHE_LINES.align(entriesPerSegment * entrySize, BYTES); + private long sizeOfEntrySpaceInSegment() { + return CACHE_LINES.align(specialEntrySpaceOffset + actualChunksPerSegment * chunkSize, + BYTES); } @Override @@ -1435,12 +1444,13 @@ class VanillaChronicleMap, * multiMapsPerSegment(); final NativeBytes bsBytes = new NativeBytes(ms.objectSerializer(), start, - start + LONGS.align(BYTES.alignAndConvert(entriesPerSegment, BITS), BYTES), + start + LONGS.align(BYTES.alignAndConvert(actualChunksPerSegment, BITS), BYTES), null); freeList = new SingleThreadedDirectBitSet(bsBytes); - start += numberOfBitSets() * sizeOfBitSets(); + start += numberOfBitSets() * sizeOfSegmentFreeListBitSets(); + start += specialEntrySpaceOffset; entriesOffset = start - bytes.startAddr(); - assert bytes.capacity() >= entriesOffset + entriesPerSegment * entrySize; + assert bytes.capacity() >= entriesOffset + actualChunksPerSegment * chunkSize; } final MultiMap hashLookup() { @@ -1591,7 +1601,7 @@ class VanillaChronicleMap, @Override public final long offsetFromPos(long pos) { - return entriesOffset + pos * entrySize; + return entriesOffset + pos * chunkSize; } final MultiStoreBytes reuse(MultiStoreBytes entry, long offset) { @@ -1601,19 +1611,26 @@ class VanillaChronicleMap, } long entrySize(long keySize, long valueSize) { - return alignment.alignAddr(metaDataBytes + + long sizeOfEverythingBeforeValue = metaDataBytes + keySizeMarshaller.sizeEncodingSize(keySize) + keySize + - valueSizeMarshaller.sizeEncodingSize(valueSize)) + valueSize; + valueSizeMarshaller.sizeEncodingSize(valueSize); + if (constantlySizedEntry) { + return alignment.alignAddr(sizeOfEverythingBeforeValue + valueSize); + } else if (worstAlignment > 0) { + return sizeOfEverythingBeforeValue + worstAlignment + valueSize; + } else { + return alignment.alignAddr(sizeOfEverythingBeforeValue) + valueSize; + } } - final int inBlocks(long sizeInBytes) { - if (sizeInBytes <= entrySize) + final int inChunks(long sizeInBytes) { + if (sizeInBytes <= chunkSize) return 1; // int division is MUCH faster than long on Intel CPUs sizeInBytes -= 1L; if (sizeInBytes <= Integer.MAX_VALUE) - return (((int) sizeInBytes) / (int) entrySize) + 1; - return (int) (sizeInBytes / entrySize) + 1; + return (((int) sizeInBytes) / (int) chunkSize) + 1; + return (int) (sizeInBytes / chunkSize) + 1; } , RV> @@ -1890,7 +1907,7 @@ class VanillaChronicleMap, MultiStoreBytes entry, boolean writeDefaultInitialReplicationValues) { long valueSize = metaElemWriter.size(elemWriter, elem); long entrySize = entrySize(keySize, valueSize); - long pos = alloc(inBlocks(entrySize)); + long pos = alloc(inChunks(entrySize)); segmentState.pos = pos; long offset = offsetFromPos(pos); clearMetaData(offset); @@ -1923,56 +1940,58 @@ class VanillaChronicleMap, } //TODO refactor/optimize - final long alloc(int blocks) { - if (blocks > maxEntryOversizeFactor) - throw new IllegalArgumentException("Entry is too large: requires " + blocks + - " entry size chucks, " + maxEntryOversizeFactor + " is maximum."); - long ret = freeList.setNextNContinuousClearBits(nextPosToSearchFrom, blocks); - if (ret == DirectBitSet.NOT_FOUND || ret + blocks > entriesPerSegment) { - if (ret + blocks > entriesPerSegment) - freeList.clear(ret, ret + blocks); - ret = freeList.setNextNContinuousClearBits(0L, blocks); - if (ret == DirectBitSet.NOT_FOUND || ret + blocks > entriesPerSegment) { - if (ret + blocks > entriesPerSegment) - freeList.clear(ret, ret + blocks); - if (blocks == 1) { + final long alloc(int chunks) { + if (chunks > maxChunksPerEntry) + throw new IllegalArgumentException("Entry is too large: requires " + chunks + + " entry size chucks, " + maxChunksPerEntry + " is maximum."); + long ret = freeList.setNextNContinuousClearBits(nextPosToSearchFrom, chunks); + if (ret == DirectBitSet.NOT_FOUND || ret + chunks > actualChunksPerSegment) { + if (ret != DirectBitSet.NOT_FOUND && + ret + chunks > actualChunksPerSegment && ret < actualChunksPerSegment) + freeList.clear(ret, actualChunksPerSegment); + ret = freeList.setNextNContinuousClearBits(0L, chunks); + if (ret == DirectBitSet.NOT_FOUND || ret + chunks > actualChunksPerSegment) { + if (ret != DirectBitSet.NOT_FOUND && + ret + chunks > actualChunksPerSegment && ret < actualChunksPerSegment) + freeList.clear(ret, actualChunksPerSegment); + if (chunks == 1) { throw new IllegalStateException( "Segment is full, no free entries found"); } else { throw new IllegalStateException( - "Segment is full or has no ranges of " + blocks - + " continuous free blocks" + "Segment is full or has no ranges of " + chunks + + " continuous free chunks" ); } } - updateNextPosToSearchFrom(ret, blocks); + updateNextPosToSearchFrom(ret, chunks); } else { // if bit at nextPosToSearchFrom is clear, it was skipped because - // more than 1 block was requested. Don't move nextPosToSearchFrom - // in this case. blocks == 1 clause is just a fast path. - if (blocks == 1 || freeList.isSet(nextPosToSearchFrom)) { - updateNextPosToSearchFrom(ret, blocks); + // more than 1 chunk was requested. Don't move nextPosToSearchFrom + // in this case. chunks == 1 clause is just a fast path. + if (chunks == 1 || freeList.isSet(nextPosToSearchFrom)) { + updateNextPosToSearchFrom(ret, chunks); } } return ret; } - private void updateNextPosToSearchFrom(long allocated, int blocks) { - if ((nextPosToSearchFrom = allocated + blocks) >= entriesPerSegment) + private void updateNextPosToSearchFrom(long allocated, int chunks) { + if ((nextPosToSearchFrom = allocated + chunks) >= actualChunksPerSegment) nextPosToSearchFrom = 0L; } - private boolean realloc(long fromPos, int oldBlocks, int newBlocks) { - if (freeList.allClear(fromPos + oldBlocks, fromPos + newBlocks)) { - freeList.set(fromPos + oldBlocks, fromPos + newBlocks); + private boolean realloc(long fromPos, int oldChunks, int newChunks) { + if (freeList.allClear(fromPos + oldChunks, fromPos + newChunks)) { + freeList.set(fromPos + oldChunks, fromPos + newChunks); return true; } else { return false; } } - private void free(long fromPos, int blocks) { - freeList.clear(fromPos, fromPos + blocks); + private void free(long fromPos, int chunks) { + freeList.clear(fromPos, fromPos + chunks); if (fromPos < nextPosToSearchFrom) nextPosToSearchFrom = fromPos; } @@ -2079,7 +2098,7 @@ class VanillaChronicleMap, if (removeFromMultiMap) { hashLookup.removePrevPos(searchState); long entrySizeInBytes = entry.positionAddr() + valueSize - entry.startAddr(); - free(pos, inBlocks(entrySizeInBytes)); + free(pos, inChunks(entrySizeInBytes)); } else { hashLookup.removePosition(pos); } @@ -2237,7 +2256,7 @@ class VanillaChronicleMap, * Replaces value in existing entry. May cause entry relocation, because there may be not * enough space for new value in location already allocated for this entry. * - * @param pos index of the first block occupied by the entry + * @param pos index of the first chunk occupied by the entry * @param offset relative offset of the entry in Segment bytes (before, i. e. * including metaData) * @param entry relative pointer in Segment bytes @@ -2260,20 +2279,20 @@ class VanillaChronicleMap, newValueDoesNotFit: if (newEntryEndAddr != entryEndAddr) { long oldEntrySize = entryEndAddr - entryStartAddr; - int oldSizeInBlocks = inBlocks(oldEntrySize); - int newSizeInBlocks = inBlocks(newEntryEndAddr - entryStartAddr); - if (newSizeInBlocks > oldSizeInBlocks) { - if (newSizeInBlocks > maxEntryOversizeFactor) { + int oldSizeInChunks = inChunks(oldEntrySize); + int newSizeInChunks = inChunks(newEntryEndAddr - entryStartAddr); + if (newSizeInChunks > oldSizeInChunks) { + if (newSizeInChunks > maxChunksPerEntry) { throw new IllegalArgumentException("Value too large: " + - "entry takes " + newSizeInBlocks + " blocks, " + - maxEntryOversizeFactor + " is maximum."); + "entry takes " + newSizeInChunks + " chunks, " + + maxChunksPerEntry + " is maximum."); } - if (realloc(pos, oldSizeInBlocks, newSizeInBlocks)) + if (realloc(pos, oldSizeInChunks, newSizeInChunks)) break newValueDoesNotFit; // RELOCATION - free(pos, oldSizeInBlocks); + free(pos, oldSizeInChunks); onRelocation(this, pos); - pos = alloc(newSizeInBlocks); + pos = alloc(newSizeInChunks); // putValue() is called from put() and replace() // after successful search by key searchedHashLookup.replacePrevPos(segmentState.searchState, pos); @@ -2286,12 +2305,12 @@ class VanillaChronicleMap, newEntryStartAddr, valueSizeAddr - entryStartAddr); entry = reuse(entry, offset); // END OF RELOCATION - } else if (newSizeInBlocks < oldSizeInBlocks) { - // Freeing extra blocks - freeList.clear(pos + newSizeInBlocks, pos + oldSizeInBlocks); + } else if (newSizeInChunks < oldSizeInChunks) { + // Freeing extra chunks + freeList.clear(pos + newSizeInChunks, pos + oldSizeInChunks); // Do NOT reset nextPosToSearchFrom, because if value - // once was larger it could easily became oversized again, - // But if these blocks will be taken by that time, + // once was larger it could easily became larger again, + // But if these chunks will be taken by that time, // this entry will need to be relocated. } } @@ -2356,10 +2375,10 @@ class VanillaChronicleMap, manageReplicationBytes(entry, false, false); long valueSize = valueSizeMarshaller.readSize(entry); long sizeInBytes = entrySize(keySize, valueSize); - int entrySizeInBlocks = inBlocks(sizeInBytes); - if (!freeList.allSet(pos, pos + entrySizeInBlocks)) + int entrySizeInChunks = inChunks(sizeInBytes); + if (!freeList.allSet(pos, pos + entrySizeInChunks)) throw new AssertionError(); - pos += entrySizeInBlocks; + pos += entrySizeInChunks; } } finally { readUnlock(); @@ -2503,7 +2522,7 @@ class VanillaChronicleMap, final long entryEndAddr = entry.positionAddr() + valueSize; if (removeFromMultiMap) { segment.hashLookup().remove(segmentHash, pos); - segment.free(pos, segment.inBlocks(entryEndAddr - entry.address())); + segment.free(pos, segment.inChunks(entryEndAddr - entry.address())); } else { segment.hashLookup().removePosition(pos); } diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index 6ecb96ac3..1db8d0c3d 100755 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -92,31 +92,38 @@ public final class ChronicleSetBuilder } @Override - public ChronicleSetBuilder actualEntriesPerSegment(long actualEntriesPerSegment) { - chronicleMapBuilder.actualEntriesPerSegment(actualEntriesPerSegment); + public ChronicleSetBuilder entriesPerSegment(long entriesPerSegment) { + chronicleMapBuilder.entriesPerSegment(entriesPerSegment); + return this; + } + + @Override + public ChronicleSetBuilder actualChunksPerSegment(long actualChunksPerSegment) { + chronicleMapBuilder.actualChunksPerSegment(actualChunksPerSegment); return this; } /** * {@inheritDoc} * - *

Example: if keys in your set(s) are English words in {@link String} form, keys size 10 - * (a bit more than average English word length) would be a good choice:

{@code
+     * 

Example: if keys in your set(s) are English words in {@link String} form, average English + * word length is 5.1, configure average key size of 6:

{@code
      * ChronicleSet uniqueWords = ChronicleSetBuilder.of(String.class)
      *     .entries(50000)
-     *     .keySize(10)
+     *     .averageKeySize(6)
      *     .create();}
* - *

(Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded - * (and each character takes 2 bytes on-heap), because default off-heap {@link String} encoding - * is UTF-8 in {@code ChronicleSet}.) + *

(Note that 6 is chosen as average key size in bytes despite strings in Java are UTF-16 + * encoded (and each character takes 2 bytes on-heap), because default off-heap {@link String} + * encoding is UTF-8 in {@code ChronicleSet}.) * * @see #constantKeySizeBySample(Object) - * @see #entrySize(int) + * @see #actualChunkSize(int) + * @param averageKeySize */ @Override - public ChronicleSetBuilder keySize(int keySize) { - chronicleMapBuilder.keySize(keySize); + public ChronicleSetBuilder averageKeySize(double averageKeySize) { + chronicleMapBuilder.averageKeySize(averageKeySize); return this; } @@ -128,7 +135,7 @@ public final class ChronicleSetBuilder * .constantKeySizeBySample(new byte[20]) * .create();}

* - * @see #keySize(int) + * @see ChronicleHashBuilder#averageKeySize(double) */ @Override public ChronicleSetBuilder constantKeySizeBySample(E sampleKey) { @@ -136,26 +143,15 @@ public final class ChronicleSetBuilder return this; } - /** - * {@inheritDoc} - * - *

In fully default case you can expect entry size to be about 120-130 bytes. But it is - * strongly recommended always to configure {@linkplain #keySize(int) key size}, if they - * couldn't be derived statically. - * - *

If entry size is not configured explicitly by calling this method, it is computed based on - * {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size}, - * plus a few bytes required by implementations. - */ @Override - public ChronicleSetBuilder entrySize(int entrySize) { - chronicleMapBuilder.entrySize(entrySize); + public ChronicleSetBuilder actualChunkSize(int actualChunkSize) { + chronicleMapBuilder.actualChunkSize(actualChunkSize); return this; } @Override - public ChronicleSetBuilder maxEntryOversizeFactor(int maxEntryOversizeFactor) { - chronicleMapBuilder.maxEntryOversizeFactor(maxEntryOversizeFactor); + public ChronicleSetBuilder maxChunksPerEntry(int maxChunksPerEntry) { + chronicleMapBuilder.maxChunksPerEntry(maxChunksPerEntry); return this; } diff --git a/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java b/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java index ddb3956f7..877757afa 100644 --- a/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java +++ b/src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java @@ -64,6 +64,16 @@ enum DummyValueMarshaller return 0; } + @Override + public int minSizeEncodingSize() { + return 0; + } + + @Override + public int maxSizeEncodingSize() { + return 0; + } + @Override public void writeSize(Bytes bytes, long size) { // do nothing diff --git a/src/test/java/eg/TestNesterInterfaceMarshal.java b/src/test/java/eg/TestNesterInterfaceMarshal.java index e03d8d69f..eff7d284e 100644 --- a/src/test/java/eg/TestNesterInterfaceMarshal.java +++ b/src/test/java/eg/TestNesterInterfaceMarshal.java @@ -35,7 +35,7 @@ public class TestNesterInterfaceMarshal { ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, TestInstrumentVOInterface.class) .entries(5000L) - .keySize(10) + .averageKeySize(10) .replication((byte) 1, tcpTransportAndNetworkConfig); @@ -56,7 +56,7 @@ public class TestNesterInterfaceMarshal { .of(CharSequence.class, TestInstrumentVOInterface.class) .putReturnsNull(true) .removeReturnsNull(true) - .entries(5000L).keySize(10) + .entries(5000L).averageKeySize(10) .replication((byte) 2, tcpTransportAndNetworkConfig1); @@ -120,7 +120,7 @@ public class TestNesterInterfaceMarshal { ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, TestInstrumentVOInterface.class) .entries(5000L) - .keySize(10) + .averageKeySize(10) .replication((byte) 1, tcpTransportAndNetworkConfig); @@ -145,7 +145,7 @@ public class TestNesterInterfaceMarshal { .putReturnsNull(true) .removeReturnsNull(true) .entries(5000L) - .keySize(10) + .averageKeySize(10) .replication((byte) 2, tcpTransportAndNetworkConfig); diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index 1ff200aeb..8dbc45a82 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -186,9 +186,15 @@ public class CHMUseCasesTest { map1.getAll(file); - try (ChronicleMap actual = ChronicleMapBuilder.of(map1.keyClass(), + VanillaChronicleMap vanillaMap = (VanillaChronicleMap) map1; + ChronicleMapBuilder builder = ChronicleMapBuilder.of(map1.keyClass(), map1.valueClass()) - .create()) { + .entriesPerSegment(vanillaMap.entriesPerSegment) + .actualSegments(vanillaMap.actualSegments) + .actualChunksPerSegment(vanillaMap.actualChunksPerSegment); + if (!vanillaMap.constantlySizedEntry) + builder.actualChunkSize((int) vanillaMap.chunkSize); + try (ChronicleMap actual = builder.create()) { actual.putAll(file); @@ -211,7 +217,6 @@ public class CHMUseCasesTest { private ChronicleMap newInstance(ChronicleMapBuilder builder) throws IOException { - switch (typeOfMap) { case SIMPLE: @@ -378,7 +383,7 @@ public class CHMUseCasesTest { return; // acquireUsingLocked not supported by the STATELESS client ChronicleMapBuilder builder = ChronicleMapBuilder.of(String.class, BondVOInterface.class) - .keySize(10); + .averageKeySize(10); try (ChronicleMap chm = newInstance(builder)) { BondVOInterface bondVO = chm.newValueInstance(); @@ -419,7 +424,8 @@ public class CHMUseCasesTest { @Test - public void testOversizedEntry() throws ExecutionException, InterruptedException, IOException { + public void testEntrySpanningSeveralChunks() + throws ExecutionException, InterruptedException, IOException { int salefactor = 100; int valueSize = 10 * salefactor; @@ -429,8 +435,8 @@ public class CHMUseCasesTest { ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, char[].class) - .keySize(10) - .valueSize(10); + .averageKeySize(10) + .averageValueSize(10); try (ChronicleMap map = newInstance(builder)) { map.put("Key", expected); @@ -445,8 +451,8 @@ public class CHMUseCasesTest { ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, CharSequence.class) - .constantKeySizeBySample("Key") - .constantValueSizeBySample("Value") + .averageKeySize("Key".length()) + .averageValueSize("Value".length()) .entries(1); try (ChronicleMap map = newInstance(builder)) { @@ -974,7 +980,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = newInstance(builder)) { - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); Integer key1; Integer key2; Integer value1; @@ -1043,7 +1049,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = newInstance(builder)) { // assertEquals(16, entrySize(map)); -// assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); +// assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); map.put(1L, 11L); assertEquals((Long) 11L, map.get(1L)); @@ -1097,7 +1103,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = newInstance(builder)) { - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); map.put(1.0, 11.0); assertEquals((Double) 11.0, map.get(1.0)); @@ -1143,8 +1149,8 @@ public class CHMUseCasesTest { return; ChronicleMapBuilder builder = ChronicleMapBuilder - .of(byte[].class, byte[].class).keySize(4).valueSize(4) - .maxEntryOversizeFactor(1); + .of(byte[].class, byte[].class).averageKeySize(4).averageValueSize(4) + .entries(1000); try (ChronicleMap map = newInstance(builder)) { byte[] key1 = {1, 1, 1, 1}; @@ -1196,9 +1202,9 @@ public class CHMUseCasesTest { ChronicleMapBuilder builder = ChronicleMapBuilder .of(ByteBuffer.class, ByteBuffer.class) - .keySize(8) - .valueSize(8) - .maxEntryOversizeFactor(1); + .averageKeySize(8) + .averageValueSize(8) + .entries(1000); try (ChronicleMap map = newInstance(builder)) { @@ -1221,18 +1227,17 @@ public class CHMUseCasesTest { @Test - public void testByteBufferByteBufferMap() throws ExecutionException, InterruptedException, IOException { + public void testByteBufferByteBufferMap() + throws ExecutionException, InterruptedException, IOException { if (typeOfMap == TypeOfMap.STATELESS) return; // getUsingLocked supported by the STATELESS client ChronicleMapBuilder builder = ChronicleMapBuilder .of(ByteBuffer.class, ByteBuffer.class) - .keyMarshaller(ByteBufferMarshaller.INSTANCE) - .valueMarshaller(ByteBufferMarshaller.INSTANCE) - .keySize(8) - .valueSize(8) - .maxEntryOversizeFactor(1); + .averageKeySize(8) + .averageValueSize(8) + .entries(1000); try (ChronicleMap map = newInstance(builder)) { @@ -1259,7 +1264,8 @@ public class CHMUseCasesTest { assertBBEquals(ByteBuffer.wrap(new byte[]{11, 11}), map.getMapped(key1, function)); assertEquals(null, map.getMapped(key2, function)); mapChecks(); - assertBBEquals(ByteBuffer.wrap(new byte[]{12, 10}), map.putMapped(key1, new UnaryOperator() { + assertBBEquals(ByteBuffer.wrap(new byte[]{12, 10}), + map.putMapped(key1, new UnaryOperator() { @Override public ByteBuffer update(ByteBuffer s) { s.put(0, (byte) (s.get(0) + 1)); @@ -1344,10 +1350,8 @@ public class CHMUseCasesTest { ChronicleMapBuilder builder = ChronicleMapBuilder .of(ByteBuffer.class, ByteBuffer.class) - .valueMarshaller(ByteBufferMarshaller.INSTANCE) // we should not have to to this ! - .keyMarshaller(ByteBufferMarshaller.INSTANCE) // we should not have to to this ! - .keySize(5).valueSize(5) - .maxEntryOversizeFactor(1); + .averageKeySize(5).averageValueSize(5) + .entries(1000); try (ChronicleMap map = newInstance(builder)) { @@ -1408,7 +1412,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = newInstance(builder)) { // this may change due to alignment // assertEquals(8, entrySize(map)); - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); IntValue key1 = map.newKeyInstance(); IntValue key2 = map.newKeyInstance(); IntValue value1 = map.newValueInstance(); @@ -1496,7 +1500,7 @@ public class CHMUseCasesTest { // this may change due to alignment //assertEquals(8, entrySize(map)); - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); UnsignedIntValue key1 = map.newKeyInstance(); UnsignedIntValue value1 = map.newValueInstance(); @@ -1615,7 +1619,7 @@ public class CHMUseCasesTest { // assertEquals(6, entrySize(map)); - // assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + // assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); IntValue key1 = map.newKeyInstance(); IntValue key2 = map.newKeyInstance(); ShortValue value1 = map.newValueInstance(); @@ -1707,7 +1711,7 @@ public class CHMUseCasesTest { // this may change due to alignment // assertEquals(8, entrySize(map)); - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); IntValue key1 = map.newKeyInstance(); IntValue key2 = map.newKeyInstance(); UnsignedShortValue value1 = map.newValueInstance(); @@ -1796,7 +1800,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = newInstance(builder)) { - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); IntValue key1 = map.newKeyInstance(); IntValue key2 = map.newKeyInstance(); CharValue value1 = map.newValueInstance(); @@ -1887,7 +1891,7 @@ public class CHMUseCasesTest { // TODO should be 5, but shorter fields based on range doesn't seem to be implemented // on data value generation level yet //assertEquals(8, entrySize(map)); this may change due to alignmented - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); IntValue key1 = map.newKeyInstance(); IntValue key2 = map.newKeyInstance(); @@ -1978,7 +1982,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = newInstance(builder)) { - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); IntValue key1 = map.newKeyInstance(); IntValue key2 = map.newKeyInstance(); @@ -2069,7 +2073,7 @@ public class CHMUseCasesTest { try (ChronicleMap map = newInstance(builder)) { - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); FloatValue key1 = map.newKeyInstance(); FloatValue key2 = map.newKeyInstance(); @@ -2163,7 +2167,7 @@ public class CHMUseCasesTest { // this may change due to alignment //assertEquals(16, entrySize(map)); - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); DoubleValue key1 = map.newKeyInstance(); DoubleValue key2 = map.newKeyInstance(); @@ -2258,7 +2262,7 @@ public class CHMUseCasesTest { // this may change due to alignment // assertEquals(16, entrySize(map)); - assertEquals(1, ((VanillaChronicleMap) map).maxEntryOversizeFactor); + assertEquals(1, ((VanillaChronicleMap) map).maxChunksPerEntry); LongValue key1 = map.newKeyInstance(); LongValue key2 = map.newKeyInstance(); @@ -2536,7 +2540,7 @@ public class CHMUseCasesTest { return; // acquireUsingLocked not supported by the STATELESS client ChronicleMapBuilder builder = ChronicleMapBuilder - .of(String.class, IBean.class).keySize(5); + .of(String.class, IBean.class).averageKeySize(5).entries(1000); try (ChronicleMap map = newInstance(builder)) { IBean iBean = map.newValueInstance(); diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapImportExportTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapImportExportTest.java index 0f49892e9..49d6f3ec8 100644 --- a/src/test/java/net/openhft/chronicle/map/ChronicleMapImportExportTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapImportExportTest.java @@ -32,16 +32,16 @@ public class ChronicleMapImportExportTest { File file = new File(TMP + "/chronicle-map-" + System.nanoTime() + ".json"); file.deleteOnExit(); - try (ChronicleMap expected = ChronicleMapBuilder.of(String.class, String - .class) - .create()) { + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(String.class, String.class) + .averageKeySize(10).averageValueSize(10) + .entries(1000); + try (ChronicleMap expected = builder.create()) { expected.put("hello", "world"); expected.put("aKey", "aValue"); expected.getAll(file); - try (ChronicleMap actual = ChronicleMapBuilder.of(String.class, String - .class) - .create()) { + try (ChronicleMap actual = builder.create()) { actual.putAll(file); Assert.assertEquals(expected, actual); @@ -58,17 +58,19 @@ public class ChronicleMapImportExportTest { file.deleteOnExit(); System.out.println(file.getAbsoluteFile()); - try (ChronicleMap expected = ChronicleMapBuilder.of(String.class, Map - .class) + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(String.class, Map.class) + .averageKeySize("hello".length()) + .averageValueSize(100) + .entries(1000); + try (ChronicleMap expected = builder .create()) { HashMap data = new HashMap<>(); data.put("myKey", "myValue"); expected.put("hello", data); expected.getAll(file); - try (ChronicleMap actual = ChronicleMapBuilder.of(String.class, Map - .class) - .create()) { + try (ChronicleMap actual = builder.create()) { actual.putAll(file); Assert.assertEquals(expected, actual); @@ -85,8 +87,10 @@ public class ChronicleMapImportExportTest { file.deleteOnExit(); System.out.println(file.getAbsoluteFile()); - try (ChronicleMap expected = ChronicleMapBuilder.of - (String.class, Map.class).create()) { + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(String.class, Map.class) + .averageKeySize("hello".length()).averageValueSize(100).entries(1000); + try (ChronicleMap expected = builder.create()) { HashMap data = new HashMap<>(); HashMap data2 = new HashMap<>(); data2.put("nested", "map"); @@ -95,8 +99,7 @@ public class ChronicleMapImportExportTest { expected.getAll(file); - try (ChronicleMap actual = ChronicleMapBuilder.of(String.class, Map - .class) + try (ChronicleMap actual = builder .create()) { actual.putAll(file); @@ -139,15 +142,17 @@ public class ChronicleMapImportExportTest { file.deleteOnExit(); System.out.println(file.getAbsolutePath()); - try (ChronicleMap expected = ChronicleMapBuilder.of(CharSequence.class, CharSequence - .class) + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) + .averageKeySize("hello".length()).averageValueSize("world".length()) + .entries(1000); + try (ChronicleMap expected = builder .create()) { expected.put("hello", "world"); expected.getAll(file); - try (ChronicleMap actual = ChronicleMapBuilder.of(CharSequence.class, CharSequence - .class) + try (ChronicleMap actual = builder .create()) { actual.putAll(file); @@ -179,8 +184,11 @@ public class ChronicleMapImportExportTest { xstream.toXML(map, new FileOutputStream(file)); - try (ChronicleMap expected = ChronicleMapBuilder.of(Integer.class, - String.class).create()) { + try (ChronicleMap expected = ChronicleMapBuilder + .of(Integer.class, String.class) + .averageValueSize(10) + .entries(1000) + .create()) { expected.put(1, "one"); expected.put(2, "two"); @@ -233,9 +241,11 @@ public class ChronicleMapImportExportTest { //file.deleteOnExit(); System.out.println(file.getAbsolutePath()); - try (ChronicleMap expected = ChronicleMapBuilder.of(CharSequence.class, LongValue - .class) - .create()) { + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(CharSequence.class, LongValue.class) + .averageKeySize("one".length()) + .entries(1000); + try (ChronicleMap expected = builder.create()) { LongValue value = expected.newValueInstance(); // this will add the entry @@ -248,9 +258,7 @@ public class ChronicleMapImportExportTest { expected.getAll(file); - try (ChronicleMap actual = ChronicleMapBuilder.of(CharSequence.class, LongValue - .class) - .create()) { + try (ChronicleMap actual = builder.create()) { actual.putAll(file); @@ -269,8 +277,11 @@ public class ChronicleMapImportExportTest { file.deleteOnExit(); System.out.println(file.getAbsolutePath()); + ChronicleMapBuilder builder = + ChronicleMapBuilder.of(CharSequence.class, BondVOInterface.class) + .averageKeySize("one".length()).entries(1000); try (ChronicleMap expected = - ChronicleMapBuilder.of(CharSequence.class, BondVOInterface.class).create()) { + builder.create()) { final BondVOInterface value = expected.newValueInstance(); @@ -286,13 +297,12 @@ public class ChronicleMapImportExportTest { expected.getAll(file); - try (ChronicleMap actual = - ChronicleMapBuilder.of(CharSequence.class, - DataValueClasses.directClassFor(BondVOInterface.class)).create()) { + try (ChronicleMap actual = builder.create()) { actual.putAll(file); - Assert.assertEquals(expected.get("one").getCoupon(), actual.get("one").getCoupon(), 0); + Assert.assertEquals(expected.get("one").getCoupon(), + actual.get("one").getCoupon(), 0); } } finally { file.delete(); diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java index b00df52b5..9b7e5c26c 100755 --- a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java @@ -108,7 +108,7 @@ public class ChronicleMapTest { return ChronicleMapBuilder.of(CharSequence.class, LongValue.class) .entries(entries) .minSegments(segments) - .keySize(keySize) + .averageKeySize(keySize) .entryAndValueAlignment(alignment) .create(); } @@ -653,7 +653,7 @@ public class ChronicleMapTest { .entries(entries) .actualSegments(8 * 1024) .entryAndValueAlignment(OF_8_BYTES) - .keySize(13); + .averageKeySize(13); File tmpFile = File.createTempFile("testAcquirePerf", ".deleteme"); tmpFile.deleteOnExit(); @@ -728,7 +728,7 @@ public class ChronicleMapTest { .entries(entries) .entryAndValueAlignment(OF_8_BYTES) .actualSegments(256) - .keySize(13); + .averageKeySize(13); File tmpFile = File.createTempFile("testAcquirePerf", ".deleteme"); tmpFile.deleteOnExit(); @@ -936,8 +936,8 @@ public class ChronicleMapTest { ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) .entries(entries) .minSegments(16) - .keySize("user:".length() + 6) - .valueSize("value:".length() + 6) + .averageKeySize("user:".length() + 6) + .averageValueSize("value:".length() + 6) .putReturnsNull(true) .removeReturnsNull(true).create(); StringBuilder key = new StringBuilder(); @@ -1365,7 +1365,7 @@ public class ChronicleMapTest { ChronicleMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) .entries(noOfElements * 2 + 100) - .valueSize((noOfElements + "").length()) + .averageValueSize((noOfElements + "").length()) .putReturnsNull(true) .removeReturnsNull(true).create(); @@ -1382,52 +1382,52 @@ public class ChronicleMapTest { } @Test - public void testOversizeEntriesPutRemoveReplace() throws IOException { + public void testEntriesSpanningSeveralChunksPutRemoveReplace() throws IOException { ChronicleMapBuilder builder = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) .entries(10) .minSegments(1) - .entrySize(10); - testOversizeEntriesPutRemoveReplace( + .actualChunkSize(10); + testEntriesSpanningSeveralChunksPutRemoveReplace( (VanillaChronicleMap) builder.createPersistedTo(getPersistenceFile()) ); } - public void testOversizeEntriesPutRemoveReplace( + public void testEntriesSpanningSeveralChunksPutRemoveReplace( VanillaChronicleMap map) { String key = "k"; - String oversizedKey = "oversized key"; + String largeKey = "very large key"; String value = "v"; - String oversizedValue = "oversized value"; + String largeValue = "vary large value"; - map.put(key, oversizedValue); - assertEquals(oversizedValue, map.get(key)); + map.put(key, largeValue); + assertEquals(largeValue, map.get(key)); map.replace(key, value); map.checkConsistency(); assertEquals(value, map.get(key)); - map.replace(key, oversizedValue); + map.replace(key, largeValue); map.checkConsistency(); - assertEquals(oversizedValue, map.get(key)); + assertEquals(largeValue, map.get(key)); - map.put(oversizedKey, value); + map.put(largeKey, value); map.checkConsistency(); - assertEquals(value, map.get(oversizedKey)); - map.replace(oversizedKey, oversizedValue); + assertEquals(value, map.get(largeKey)); + map.replace(largeKey, largeValue); map.checkConsistency(); - assertEquals(oversizedValue, map.get(oversizedKey)); - map.remove(oversizedKey); + assertEquals(largeValue, map.get(largeKey)); + map.remove(largeKey); map.checkConsistency(); - assertEquals(null, map.get(oversizedKey)); - map.put(oversizedKey, oversizedValue); + assertEquals(null, map.get(largeKey)); + map.put(largeKey, largeValue); map.checkConsistency(); - assertEquals(oversizedValue, map.get(oversizedKey)); - map.replace(oversizedKey, oversizedValue, value); + assertEquals(largeValue, map.get(largeKey)); + map.replace(largeKey, largeValue, value); map.checkConsistency(); - assertEquals(value, map.get(oversizedKey)); + assertEquals(value, map.get(largeKey)); - assertEquals(value, map.remove(oversizedKey)); + assertEquals(value, map.remove(largeKey)); map.checkConsistency(); - assertEquals(oversizedValue, map.remove(key)); + assertEquals(largeValue, map.remove(key)); map.checkConsistency(); map.close(); @@ -1488,7 +1488,7 @@ public class ChronicleMapTest { final ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, LongValue.class) .entries(1000) - .keySize("x".length()); + .averageKeySize("x".length()); final ChronicleMap map = builder.create(); @@ -1501,7 +1501,7 @@ public class ChronicleMapTest { ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, LongValue.class) .entries(1000) - .keySize("one".length()); + .averageKeySize("one".length()); final ChronicleMap map = builder.create(); @@ -1575,8 +1575,8 @@ public class ChronicleMapTest { try (final ChronicleMap map = ChronicleMapBuilder .of(CharSequence.class, CharSequence.class) .entries(1000) - .keySize("one".length()) - .valueSize("Hello World".length()) + .averageKeySize("one".length()) + .averageValueSize("Hello World".length()) .defaultValue("") .create()) { StringBuilder value = new StringBuilder(); @@ -1594,7 +1594,7 @@ public class ChronicleMapTest { ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, LongValue.class) .entries(1000) - .keySize("one".length()); + .averageKeySize("one".length()); File tmpFile = File.createTempFile("testAcquireUsingLocked", ".deleteme"); tmpFile.deleteOnExit(); final ChronicleMap map = builder.createPersistedTo(tmpFile); diff --git a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java index fc53b204a..0feeffe44 100644 --- a/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/EntryCountMapTest.java @@ -32,11 +32,9 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicLong; import static java.lang.Math.log10; import static java.lang.Math.round; -import static net.openhft.chronicle.map.Alignment.OF_4_BYTES; import static net.openhft.lang.model.DataValueClasses.newDirectReference; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -58,8 +56,7 @@ public class EntryCountMapTest { ChronicleMapBuilder.of(CharSequence.class, LongValue.class) .entries(entries) .actualSegments(segments) - .keySize(keySize) - .maxEntryOversizeFactor(1); + .averageKeySize(keySize); return mapBuilder.createPersistedTo(getPersistenceFile()); } diff --git a/src/test/java/net/openhft/chronicle/map/GuavaTest.java b/src/test/java/net/openhft/chronicle/map/GuavaTest.java index 10e6f6d48..b06dd2f15 100644 --- a/src/test/java/net/openhft/chronicle/map/GuavaTest.java +++ b/src/test/java/net/openhft/chronicle/map/GuavaTest.java @@ -100,6 +100,7 @@ public class GuavaTest extends TestCase { ChronicleMapBuilder builder = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) .entries(100) + .averageKeySize(10).averageValueSize(10) .minSegments(2); @Override diff --git a/src/test/java/net/openhft/chronicle/map/Issue42Test.java b/src/test/java/net/openhft/chronicle/map/Issue42Test.java index 16cd331b8..494b7b1b6 100755 --- a/src/test/java/net/openhft/chronicle/map/Issue42Test.java +++ b/src/test/java/net/openhft/chronicle/map/Issue42Test.java @@ -34,8 +34,8 @@ public class Issue42Test { try (final ChronicleMap map = ChronicleMapBuilder .of(CharSequence.class, CharSequence.class) - .entrySize(18) - .entries(1500000) + .averageKeySize(5.9).averageValueSize(5.9) + .entries(1000000) .minSegments(128).create()) { for (int i = 0; i < 1000000; ++i) { String s = String.valueOf(i); diff --git a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java index a732624a5..d4bbe7699 100644 --- a/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java +++ b/src/test/java/net/openhft/chronicle/map/LargeEntriesTest.java @@ -53,7 +53,8 @@ public class LargeEntriesTest { .valueMarshaller(SnappyStringMarshaller.INSTANCE) .actualSegments(1) // to force an error. .entries(ENTRIES) - .entrySize(ENTRY_SIZE / 6) + .averageKeySize(10) + .averageValueSize(ENTRY_SIZE) .putReturnsNull(true) .createPersistedTo(file); { @@ -69,7 +70,8 @@ public class LargeEntriesTest { futureList.add(es.submit(new Runnable() { @Override public void run() { - exerciseLargeStrings(map, finalT * block, finalT * block + block, ENTRY_SIZE); + exerciseLargeStrings(map, finalT * block, finalT * block + block, + ENTRY_SIZE); } })); } @@ -78,7 +80,8 @@ public class LargeEntriesTest { } long time = System.currentTimeMillis() - start; long operations = 3; - System.out.printf("Put/Get rate was %.1f MB/s%n", operations * ENTRIES * ENTRY_SIZE / 1e6 / (time / 1e3)); + System.out.printf("Put/Get rate was %.1f MB/s%n", + operations * ENTRIES * ENTRY_SIZE / 1e6 / (time / 1e3)); } es.shutdown(); if (es.isTerminated()) @@ -117,7 +120,8 @@ public class LargeEntriesTest { .of(String.class, String.class) .valueMarshaller(SnappyStringMarshaller.INSTANCE) .entries(ENTRIES) - .entrySize(ENTRY_SIZE / 7 + 200) + .averageKeySize(10) + .averageValueSize(ENTRY_SIZE) .putReturnsNull(true) .createPersistedTo(file); { @@ -133,7 +137,8 @@ public class LargeEntriesTest { futureList.add(es.submit(new Runnable() { @Override public void run() { - exerciseLargeStrings(map, finalT * block, finalT * block + block, ENTRY_SIZE); + exerciseLargeStrings(map, finalT * block, finalT * block + block, + ENTRY_SIZE); } })); } diff --git a/src/test/java/net/openhft/chronicle/map/MapByNameTest.java b/src/test/java/net/openhft/chronicle/map/MapByNameTest.java index ec48d929d..8a1dc3db8 100644 --- a/src/test/java/net/openhft/chronicle/map/MapByNameTest.java +++ b/src/test/java/net/openhft/chronicle/map/MapByNameTest.java @@ -49,7 +49,7 @@ public class MapByNameTest { final MapInstanceConfig config = (MapInstanceConfig) ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .keySize("hello".length()).valueSize("world".length()) + .averageKeySize("hello".length()).averageValueSize("world".length()) .entries(2) .minSegments(2) .replication((byte) 1, TcpTransportAndNetworkConfig.of(8244)) diff --git a/src/test/java/net/openhft/chronicle/map/MarkTest.java b/src/test/java/net/openhft/chronicle/map/MarkTest.java index 9ae684c44..dd72edc40 100644 --- a/src/test/java/net/openhft/chronicle/map/MarkTest.java +++ b/src/test/java/net/openhft/chronicle/map/MarkTest.java @@ -69,12 +69,10 @@ public class MarkTest { private static void test(Function, ChronicleMap> createMap) { long ms = System.currentTimeMillis(); - double entriesWithMargin = ENTRIES * 1.1; - long actualEntriesPerSegment = (1L << 17) / 3L; try (ChronicleMap map = createMap.apply(ChronicleMapBuilder .of(Integer.class, Integer.class) - .actualSegments((int) (entriesWithMargin / actualEntriesPerSegment)) - .actualEntriesPerSegment(actualEntriesPerSegment) + .entries(ENTRIES) + .entriesPerSegment((1 << 17) / 3) .putReturnsNull(true) .removeReturnsNull(true))) { diff --git a/src/test/java/net/openhft/chronicle/map/NoEntryOversizeFactorTest.java b/src/test/java/net/openhft/chronicle/map/NoUpperBoundChunksPerEntryTest.java similarity index 85% rename from src/test/java/net/openhft/chronicle/map/NoEntryOversizeFactorTest.java rename to src/test/java/net/openhft/chronicle/map/NoUpperBoundChunksPerEntryTest.java index 7c7671f94..e0cd4fc64 100644 --- a/src/test/java/net/openhft/chronicle/map/NoEntryOversizeFactorTest.java +++ b/src/test/java/net/openhft/chronicle/map/NoUpperBoundChunksPerEntryTest.java @@ -19,13 +19,13 @@ package net.openhft.chronicle.map; import org.junit.Assert; import org.junit.Test; -public class NoEntryOversizeFactorTest { +public class NoUpperBoundChunksPerEntryTest { @Test - public void noOversizeFactorTest() { + public void noUpperBoundChunksPerEntryTest() { ChronicleMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .valueSize(1).entries(10000L).actualSegments(1).create(); + .averageValueSize(1).entries(10000L).actualSegments(1).create(); String ultraLargeValue = ""; for (int i = 0; i < 100; i++) { ultraLargeValue += "Hello"; diff --git a/src/test/java/net/openhft/chronicle/map/OSResizesMain.java b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java index 758041c7f..6bb636c6f 100755 --- a/src/test/java/net/openhft/chronicle/map/OSResizesMain.java +++ b/src/test/java/net/openhft/chronicle/map/OSResizesMain.java @@ -36,7 +36,7 @@ public class OSResizesMain { byte[] chars = new byte[valueSize]; ByteArray ba = new ByteArray(chars); ChronicleMap map = ChronicleMapBuilder.of(String.class, ByteArray.class) - .keySize("key-".length() + 4) + .averageKeySize("key-".length() + 4) .constantValueSizeBySample(ba) .createPersistedTo(file); Arrays.fill(chars, (byte) '+'); diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index 8400e39da..73dcdcda7 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -656,15 +656,15 @@ public class StatelessClientTest { try (ChronicleMap serverMap = ChronicleMapBuilder .of(byte[].class, byte[].class) - .keySize(4) - .valueSize(4) + .averageKeySize(4) + .averageValueSize(4) .replication((byte) 2, TcpTransportAndNetworkConfig.of(8056)) .create()) { try (ChronicleMap statelessMap = ChronicleMapBuilder .of(byte[].class, byte[].class) - .keySize(4) - .valueSize(4) + .averageKeySize(4) + .averageValueSize(4) .statelessClient(new InetSocketAddress("localhost", 8056)) .create()) { diff --git a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java index 7ace69cb2..50113a515 100644 --- a/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/TCPSocketReplication4WayMapTest.java @@ -48,7 +48,7 @@ public class TCPSocketReplication4WayMapTest { final int serverPort, final InetSocketAddress... endpoints) throws IOException { return (T) newTcpSocketShmBuilder(Integer.class, CharSequence.class, - identifier, serverPort, endpoints).create(); + identifier, serverPort, endpoints).averageValueSize(10).create(); } @Before diff --git a/src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireUsingLockedTest.java b/src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireUsingLockedTest.java index 5a8bbeea7..cd085170d 100644 --- a/src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireUsingLockedTest.java +++ b/src/test/java/net/openhft/chronicle/map/TcpReplicationWithAcquireUsingLockedTest.java @@ -80,7 +80,7 @@ public class TcpReplicationWithAcquireUsingLockedTest { (CharSequence.class, TestInstrumentVOInterface .class) - .entries(5000L).keySize("hello".length()) + .entries(5000L).averageKeySize("hello".length()) .replication((byte) 1, config1).create()) { map1a = map1; @@ -95,7 +95,7 @@ public class TcpReplicationWithAcquireUsingLockedTest { .putReturnsNull(true) .removeReturnsNull(true) .replication((byte) 2, config2) - .entries(5000L).keySize("hello".length()).create()) { + .entries(5000L).averageKeySize("hello".length()).create()) { map2a = map2; @@ -133,7 +133,7 @@ public class TcpReplicationWithAcquireUsingLockedTest { (CharSequence.class, TestInstrumentVOInterface .class) - .entries(5000L).keySize("hello".length()) + .entries(5000L).averageKeySize("hello".length()) .replication((byte) 1, config1).create()) { map1a = map1; @@ -148,7 +148,7 @@ public class TcpReplicationWithAcquireUsingLockedTest { .putReturnsNull(true) .removeReturnsNull(true) .replication((byte) 2, config2) - .entries(5000L).keySize("hello".length()).create()) { + .entries(5000L).averageKeySize("hello".length()).create()) { map2a = map2; diff --git a/src/test/java/net/openhft/chronicle/map/TestOversizedReplicationHub.java b/src/test/java/net/openhft/chronicle/map/TestEntriesSpanningSeveralChunksReplicationHub.java similarity index 87% rename from src/test/java/net/openhft/chronicle/map/TestOversizedReplicationHub.java rename to src/test/java/net/openhft/chronicle/map/TestEntriesSpanningSeveralChunksReplicationHub.java index 790404c8e..7bc23c9d8 100644 --- a/src/test/java/net/openhft/chronicle/map/TestOversizedReplicationHub.java +++ b/src/test/java/net/openhft/chronicle/map/TestEntriesSpanningSeveralChunksReplicationHub.java @@ -15,16 +15,17 @@ import static org.junit.Assert.assertEquals; /** * @author Rob Austin. */ -public class TestOversizedReplicationHub { +public class TestEntriesSpanningSeveralChunksReplicationHub { @Test - public void testReplicationHubHandlesOverSizeEntries() throws IOException, InterruptedException { + public void testReplicationHubHandlesSpanningSeveralChunksEntries() + throws IOException, InterruptedException { ChronicleMap favoriteColourServer1, favoriteColourServer2; ChronicleMap favoriteComputerServer1, favoriteComputerServer2; - int OVERSIZE_FACTOR = 65; - char[] largeEntry = new char[100 * OVERSIZE_FACTOR]; + int chunksPerEntry = 65; + char[] largeEntry = new char[100 * chunksPerEntry]; Arrays.fill(largeEntry, 'X'); String largeString = new String(largeEntry); @@ -33,8 +34,9 @@ public class TestOversizedReplicationHub { { ChronicleMapBuilder builder = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .keySize(10) - .valueSize(100); + .averageKeySize(10) + .averageValueSize(100) + .entries(1000); byte identifier = (byte) 1; @@ -68,8 +70,9 @@ public class TestOversizedReplicationHub { { ChronicleMapBuilder builder = ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .keySize(10) - .valueSize(100); + .averageKeySize(10) + .averageValueSize(100) + .entries(1000); byte identifier = (byte) 2; diff --git a/src/test/java/net/openhft/chronicle/map/example/LotsOfEntriesMain.java b/src/test/java/net/openhft/chronicle/map/example/LotsOfEntriesMain.java index dc2f4a151..d72da2711 100644 --- a/src/test/java/net/openhft/chronicle/map/example/LotsOfEntriesMain.java +++ b/src/test/java/net/openhft/chronicle/map/example/LotsOfEntriesMain.java @@ -28,21 +28,11 @@ public class LotsOfEntriesMain { throws IOException, ExecutionException, InterruptedException { final long entries = 100_000_000; File file = new File("/tmp/lotsOfEntries.dat"); - double averageEntrySize = - 1 + // key size marshalling size - // + 2 is average oversize because we append 4-letter "-key" in a loop - (Math.log(1.024) - Math.log(0.024)) * 24 + 2 + // average key size - 24 + // value size - 2 // average loss due to value alignment. defaults to OF_4_BYTES because - // there are floats in values - ; final ChronicleMap map = ChronicleMapBuilder .of(CharSequence.class, MyFloats.class) - .entries((long) (entries * - (averageEntrySize / 8.0 + - 0.5))) // account average internal fragmentation -- - // because we need to round up to integral number of chunks - .entrySize(8) // "chunk mode" + .entries(entries) + // + 2 is average oversize because we append 4-letter "-key" in a loop + .averageKeySize((Math.log(1.024) - Math.log(0.024)) * 24 + 2) .createPersistedTo(file); int threads = Runtime.getRuntime().availableProcessors(); ExecutorService es = Executors.newFixedThreadPool(threads); diff --git a/src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java b/src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java index 4d55a5bfc..34fa7da8b 100644 --- a/src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java +++ b/src/test/java/net/openhft/chronicle/map/example/StringsMapMain.java @@ -41,9 +41,10 @@ public class StringsMapMain { int keyAndValueSize = (int) Math.round(Math.log10(entries)); final ChronicleMap serverMap = ChronicleMapBuilder .of(CharSequence.class, CharSequence.class) + .entries(entries) .putReturnsNull(true) - .keySize(keyAndValueSize) - .valueSize(keyAndValueSize) + .averageKeySize(keyAndValueSize) + .averageValueSize(keyAndValueSize) .replication((byte) 2, TcpTransportAndNetworkConfig.of(port)) .createPersistedTo(file); System.out.println("Server started"); diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java index ed0dbcf24..19aa38517 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java @@ -56,7 +56,8 @@ public class OpenJDKAndHashMapExamplesTest { ChronicleMap chm = ChronicleMapBuilder .of(String.class, BondVOInterface.class) - .keySize(10) + .averageKeySize(10) + .entries(1000) .createPersistedTo(file); BondVOInterface bondVO = chm.newValueInstance(); @@ -76,7 +77,8 @@ public class OpenJDKAndHashMapExamplesTest { ChronicleMap chmB = ChronicleMapBuilder .of(String.class, BondVOInterface.class) - .keySize(10) + .averageKeySize(10) + .entries(1000) .createPersistedTo(file); // ZERO Copy but creates a new off heap reference each time diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/StringEncodingExamplesTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/StringEncodingExamplesTest.java index d302d14f6..5686819e5 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/StringEncodingExamplesTest.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/StringEncodingExamplesTest.java @@ -193,7 +193,7 @@ public class StringEncodingExamplesTest { try (ChronicleSet chineseWordSet = ChronicleSetBuilder.of(CharSequence.class) .keyMarshaller(marshaller) .actualSegments(1) - .actualEntriesPerSegment(1000) + .entriesPerSegment(1000) .create()) { chineseWordSet.add("新闻"); chineseWordSet.add("地图"); diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java index 1b02f5a6a..88abab8dd 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/pingpong_latency/PingPongCASLeft.java @@ -92,6 +92,6 @@ public class PingPongCASLeft { AffinitySupport.setThreadId(); return ChronicleMapBuilder.of(String.class, BondVOInterface.class) .entries(16) - .keySize("369604101".length()).create(); + .averageKeySize("369604101".length()).create(); } } \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/jsr166/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jsr166/map/ChronicleMapTest.java index 3e4ef6fdb..cb64bcbf6 100644 --- a/src/test/java/net/openhft/chronicle/map/jsr166/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jsr166/map/ChronicleMapTest.java @@ -41,6 +41,7 @@ public class ChronicleMapTest extends JSR166TestCase { static ChronicleMap newShmIntString(int size) throws IOException { return ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .averageValueSize(1) .entries(size).create(); } diff --git a/src/test/java/net/openhft/chronicle/map/jsr166/map/StatelessChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jsr166/map/StatelessChronicleMapTest.java index 95c6589b0..097c38e75 100755 --- a/src/test/java/net/openhft/chronicle/map/jsr166/map/StatelessChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jsr166/map/StatelessChronicleMapTest.java @@ -234,6 +234,7 @@ public class StatelessChronicleMapTest extends JSR166TestCase { final ChronicleMap serverMap = ChronicleMapBuilder.of(Integer.class, String.class) + .averageValueSize(1) .entries(SIZE) .replication((byte) 1, TcpTransportAndNetworkConfig.of(port)) .create(); diff --git a/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java b/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java index 3a861300d..ed4974317 100644 --- a/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java +++ b/src/test/java/net/openhft/chronicle/map/utility/ProcessInstanceLimiter.java @@ -137,7 +137,7 @@ public class ProcessInstanceLimiter implements Runnable { ChronicleMapBuilder builder = ChronicleMapBuilder.of(String.class, Data.class); builder.entries(1000); - builder.keySize((DEFAULT_PROCESS_NAME + "#").length()); + builder.averageKeySize((DEFAULT_PROCESS_NAME + "#").length()); this.theSharedMap = builder.createPersistedTo(new File(sharedMapPath)); Thread t = new Thread(this, "ProcessInstanceLimiter updater"); t.setDaemon(true); diff --git a/src/test/java/net/openhft/chronicle/set/Issue3Test.java b/src/test/java/net/openhft/chronicle/set/Issue3Test.java index b9df5a0e7..dd6f518b2 100644 --- a/src/test/java/net/openhft/chronicle/set/Issue3Test.java +++ b/src/test/java/net/openhft/chronicle/set/Issue3Test.java @@ -29,7 +29,7 @@ public class Issue3Test { public void test() throws IOException { try (ChronicleSet set = ChronicleSetBuilder.of(Long.class) .actualSegments(1) - .actualEntriesPerSegment(1000) + .entriesPerSegment(1000) .create()) { Random r = new Random(); for (int i = 0; i < 10; i++) {