From 1a0ecc621ce751d52ea000444519d3c214987895 Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 13 Oct 2014 07:12:48 +0400 Subject: [PATCH 1/2] HCOLL-136 HCOLL-137 Improved Alignment and ChronicleMapBuilder javadocs --- .../net/openhft/chronicle/map/Alignment.java | 15 +++++- .../chronicle/map/ChronicleMapBuilder.java | 46 ++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/Alignment.java b/src/main/java/net/openhft/chronicle/map/Alignment.java index 91f9800d9..764cd33d2 100644 --- a/src/main/java/net/openhft/chronicle/map/Alignment.java +++ b/src/main/java/net/openhft/chronicle/map/Alignment.java @@ -18,8 +18,15 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.Bytes; - +/** + * Memory addresses alignment strategies. + * + * @see ChronicleMapBuilder#entryAndValueAlignment(Alignment) + */ public enum Alignment { + /** + * Doesn't align memory addresses. + */ NO_ALIGNMENT { @Override void alignPositionAddr(Bytes bytes) { @@ -37,6 +44,9 @@ public enum Alignment { } }, + /** + * Aligns memory addresses on 4-byte boundary. + */ OF_4_BYTES { @Override void alignPositionAddr(Bytes bytes) { @@ -54,6 +64,9 @@ public enum Alignment { } }, + /** + * Aligns memory addresses on 8-byte boundary. + */ OF_8_BYTES { @Override void alignPositionAddr(Bytes bytes) { diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 64d52cfc0..474abdaf8 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -39,6 +39,49 @@ import java.util.concurrent.TimeUnit; import static net.openhft.chronicle.map.Objects.builderEquals; +/** + * {@code ChronicleMapBuilder} manages the whole set of {@link ChronicleMap} configurations, could + * be used as a classic builder and/or factory. This means that in addition to the standard builder + * usage pattern: 
{@code
+ * ChronicleMap map = ChronicleMapBuilder
+ *     .of(Key.class, Value.class)
+ *     .entries(100500)
+ *     // ... other configurations
+ *     .create();}
+ * {@code ChronicleMapBuilder} could be prepared and used to create many similar maps: 
{@code
+ * ChronicleMapBuilder builder = ChronicleMapBuilder
+ *     .of(Key.class, Value.class)
+ *     .entries(100500);
+ *
+ * ChronicleMap map1 = builder.create();
+ * ChronicleMap map2 = builder.create();}
+ * i. e. created {@code ChronicleMap} instances don't depend on the builder. + * + *

Use static {@link #of(Class, Class)} method to obtain a {@code ChronicleMapBuilder} instance. + * + *

{@code ChronicleMapBuilder} is mutable. Configuration methods mutate the builder and + * return the builder itself back to support chaining pattern, rather than the builder copies + * with the corresponding configuration changed. To make an independent configuration, + * {@linkplain #clone} the builder. + * + *

Later in this documentation, "ChronicleMap" means + * "ChronicleMaps, created by {@code ChronicleMapBuilder}", unless specified different, because + * theoretically someone might provide {@code ChronicleMap} implementations with completely + * different properties. + * + *

{@code ChronicleMap} ("ChronicleMaps, created by {@code ChronicleMapBuilder}") currently + * doesn't support resizing. That is why you should always configure {@linkplain + * #entries(long) number of entries} you are going to insert into the created map at most. + * See {@link #entries(long)} method documentation for more information on this. + * + *

{@code ChronicleMap} allocates memory by equally sized chunks. This size is called + * {@linkplain #entrySize(int) entry size}, you are strongly recommended to configure it to achieve + * least memory consumption and best speed. See {@link #entrySize(int)} method documentation for + * more information on this. + * + * @param key type of the maps, produced by this builder + * @param value type of the maps, produced by this builder + */ public class ChronicleMapBuilder implements Cloneable { private static final Bytes EMPTY_BYTES = new ByteBufferBytes(ByteBuffer.allocate(0)); @@ -109,8 +152,7 @@ public class ChronicleMapBuilder implements Cloneable { } /** - * Set minimum number of segments. See concurrencyLevel - * in {@link java.util.concurrent.ConcurrentHashMap}. + * Set minimum number of segments. See concurrencyLevel in {@link ConcurrentHashMap}. * * @param minSegments the minimum number of segments in maps, constructed by this builder * @return this builder object back From 6c02947ffb7bd5c9191db1bcc072e1e184d8e12b Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 13 Oct 2014 18:12:58 +0400 Subject: [PATCH 2/2] HCOLL-164 Exctracted ChronicleHashBuilder interface and pulled some classes up from n.o.chronicle.map package to n.o.chronicle. Fixed ChronicleSetBuilder.equals() implementation. --- .../chronicle/ChronicleHashBuilder.java | 222 ++++++++++++++++++ ...r.java => ChronicleHashErrorListener.java} | 4 +- ....java => ChronicleHashErrorListeners.java} | 12 +- .../chronicle/{map => }/TimeProvider.java | 20 +- .../chronicle/map/ChronicleMapBuilder.java | 171 +++----------- .../chronicle/map/ReplicatedChronicleMap.java | 1 + .../chronicle/map/VanillaChronicleMap.java | 3 +- .../chronicle/set/ChronicleSetBuilder.java | 123 +++++----- .../map/TimeBasedReplicationTests.java | 1 + 9 files changed, 351 insertions(+), 206 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java rename src/main/java/net/openhft/chronicle/{map/MapErrorListener.java => ChronicleHashErrorListener.java} (89%) rename src/main/java/net/openhft/chronicle/{map/MapErrorListeners.java => ChronicleHashErrorListeners.java} (84%) rename src/main/java/net/openhft/chronicle/{map => }/TimeProvider.java (74%) diff --git a/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java new file mode 100644 index 000000000..53345e6b9 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/ChronicleHashBuilder.java @@ -0,0 +1,222 @@ +package net.openhft.chronicle; + +import net.openhft.chronicle.map.*; +import net.openhft.chronicle.set.ChronicleSetBuilder; +import org.jetbrains.annotations.NotNull; + +import net.openhft.lang.io.serialization.*; +import net.openhft.lang.model.Byteable; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * This interface defines the meaning of configurations, common to {@link ChronicleMapBuilder} + * and {@link ChronicleSetBuilder}, i. e. Chronicle hash container configurations. + * + * @param the type of keys in hash containers, created by this builder + * @param the container type, created by this builder, i. e. {@link ChronicleMap} + * or {@link ChronicleSet} + * @param the concrete builder type, i. e. {@link ChronicleMapBuilder} + * or {@link ChronicleSetBuilder} + */ +public interface ChronicleHashBuilder> + extends Cloneable { + + B clone(); + + /** + * Set minimum number of segments. See concurrencyLevel in {@link ConcurrentHashMap}. + * + * @param minSegments the minimum number of segments in containers, constructed by this builder + * @return this builder object back + */ + B minSegments(int minSegments); + + /** + * Configures the optimal number of bytes, taken by serialized form of keys, put into hash + * containers, created by this builder. If key size is always the same, call + * {@link #constantKeySizeBySample(Object)} method instead of this one. + * + *

If key size varies moderately, specify the size higher than average, but lower than + * the maximum possible, to minimize average memory overuse. If key size varies in a wide range, + * it's better to use {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it + * directly. + * + * @param keySize number of bytes, taken by serialized form of keys + * @return this builder back + * @see #constantKeySizeBySample(Object) + * @see #entrySize(int) + */ + B keySize(int keySize); + + /** + * Configures the constant number of bytes, taken by serialized form of keys, put into hash + * containers, created by this builder. This is done by providing the {@code sampleKey}, + * all keys should take the same number of bytes in serialized form, as this sample object. + * + *

If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size + * is known statically, it is automatically accounted and this method shouldn't be called. + * + *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be + * called instead of this one. + * + * @param sampleKey the sample key + * @return this builder back + * @see #keySize(int) + */ + B constantKeySizeBySample(K sampleKey); + + /** + * Configures the size in bytes of allocation unit of hash container instances, created by + * this builder. + * + *

{@link ChronicleMap} and {@link ChronicleSet} store their data off-heap, so it is required + * to serialize key (and values, in {@code ChronicleMap} case) (unless they are direct + * {@link Byteable} instances). Serialized key bytes (+ serialized value bytes, + * in {@code ChronicleMap} case) + some metadata bytes comprise "entry space", which + * {@code ChronicleMap} or {@code ChronicleSet} should allocate. So entry size + * is a minimum allocation portion in the hash containers, created by this builder. + * E. g. if entry size is 100, the created container could only allocate 100, 200, 300... bytes + * for an entry. If say 150 bytes of entry space are required by the entry, 200 bytes + * will be allocated, 150 used and 50 wasted. To minimize memory overuse and improve speed, you + * should pay decent attention to this configuration. + * + *

There are three major patterns of this configuration usage: + *

    + *
  1. Key (and value, in {@code ChronicleMap} case) sizes are constant. Configure them + * via {@link #constantKeySizeBySample(Object)} and + * {@link ChronicleMapBuilder#constantValueSizeBySample(Object)} methods, and you will + * experience no memory waste at all.
    2. + *
  2. Key (and/or value size, in {@code ChronicleMap} case) varies moderately. Specify them + * using corresponding methods, or specify entry size directly by calling this method, + * by sizes somewhere between average and maximum possible. The idea is to have most + * (90% or more) entries to fit a single "entry size" with moderate memory waste + * (10-20% on average), rest 10% or less of entries should take 2 "entry sizes", thus with + * ~50% memory overuse.
    3. + *
  3. Key (and/or value size, in {@code ChronicleMap} case) varies in a wide range. + * Then it's best to use entry size configuration in chunk mode. Specify entry size + * so that most entries should take from 5 to several dozens of "chunks". With this + * approach, average memory waste should be very low. + * + *

    However, remember that + *

      + *
    • Operations with entries that span several "entry sizes" are a bit slower, than + * entries which take a single "entry size". That is why "chunk" approach + * is not recommended, when key (and/or value size) varies moderately.
      • + *
    • The maximum number of chunks could be taken by an entry is 64. + * {@link IllegalArgumentException} is thrown on attempt to insert too large entry, + * compared to the configured or computed entry size.
      • + *
    • The number of "entries" {@linkplain #entries(long) in the whole map} and + * {@linkplain #actualEntriesPerSegment(int) per segment} is actually the number + * "chunks".
      • + *
    + * + *

    Example: if values in your {@code ChronicleMap} are adjacency lists of some social + * graph, where nodes are represented as {@code long} ids, and adjacency lists + * are serialized in efficient manner, for example as {@code long[]} arrays. Typical number + * of connections is 100-300, maximum is 3000. In this case entry size of + * 50 * (8 bytes for each id) = 400 bytes would be a good choice: 

    {@code
+     * Map socialGraph = ChronicleMapBuilder
+     *     .of(Long.class, long[].class)
+     *     // given that graph should have of 1 billion nodes, and 150 average adjacency list size
+     *     // => values takes 3 chuncks on average
+     *     .entries(1_000_000_000L * (150 / 50))
+     *     .entrySize(50 * 8)
+     *     .create();}
    + * It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max + * chunks by single entry" limit, and ensures moderate average memory overuse (not more + * than 20%). + *
    4. + *
+ * + * @param entrySize the "chunk size" in bytes + * @return this builder back + * @see #entries(long) + */ + B entrySize(int entrySize); + + /** + * Configures the maximum number of {@linkplain #entrySize(int) "entry size chunks"}, which + * could be taken by the maximum number of entries, inserted into the hash containers, created + * by this builder. If you try to insert more data, {@link IllegalStateException} might be + * thrown, because currently {@link ChronicleMap} and {@link ChronicleSet} doesn't support + * resizing. + * + *
    + *
  1. If key size (and value size, in {@code ChronicleMap} case) is constant, this number + * is equal to the maximum number of entries (because each entry takes exactly one + * "entry size" memory unit).
    2. + *
  2. If key (and/or value, in {@code ChronicleMap} case) size varies moderately, you + * should pass to this method the maximum number of entries + 5-25%, depending on your data + * properties and configured {@linkplain #keySize(int) key}/{@linkplain + * ChronicleMapBuilder#valueSize(int) value}/{@linkplain #entrySize(int) entry} sizes.
    3. + *
  3. If your data size varies in a wide range, pass the maximum number of entries + * multiplied by average data size and divided by the configured "entry size" (i. e. chunk + * size). See an example in the documentation to {@link #entrySize(int)} method.
    4. + *
+ * + *

You shouldn't put additional margin over the number, computed according the rules above. + * This bad practice was popularized by {@link HashMap#HashMap(int)} + * and {@link HashSet#HashSet(int)} constructors, which accept "capacity", that should + * be multiplied by "load factor" to obtain actual maximum expected number of entries. + * {@code ChronicleMap} and {@code ChronicleSet} don't have a notion of load factor. + * + *

Default value is 2^20 (~ 1 million). + * + * @param entries maximum size of the created maps, in memory allocation units, + * so-called "entry size" + * @return this builder back + * @see #entrySize(int) + */ + B entries(long entries); + + B actualEntriesPerSegment(int actualEntriesPerSegment); + + B actualSegments(int actualSegments); + + B lockTimeOut(long lockTimeOut, TimeUnit unit); + + B errorListener(ChronicleHashErrorListener errorListener); + + B largeSegments(boolean largeSegments); + + B metaDataBytes(int metaDataBytes); + + B addReplicator(Replicator replicator); + + B setReplicators(Replicator... replicators); + + B timeProvider(TimeProvider timeProvider); + + B bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory); + + B objectSerializer(ObjectSerializer objectSerializer); + + B keyMarshaller(@NotNull BytesMarshaller keyMarshaller); + + /** + * Specifies that key objects, queried with the hash containers, created by this builder, + * are inherently immutable. Keys in {@link ChronicleMap} or {@link ChronicleSet} + * are not required to be immutable, as in ordinary {@link Map} or {@link Set} implementations, + * because they are serialized off-heap. However, {@code ChronicleMap} and {@code ChronicleSet} + * implementations can benefit from the knowledge that keys are not mutated between queries. + * + *

By default, {@code ChronicleHashBuilder}s detects immutability automatically + * only for very few standard JDK types (for example, for {@link String}), it is not recommended + * to rely on {@code ChronicleHashBuilder} to be smart enough about this. + * + * @return this builder back + */ + B immutableKeys(); + + C create(File file) throws IOException; + + C create() throws IOException; +} diff --git a/src/main/java/net/openhft/chronicle/map/MapErrorListener.java b/src/main/java/net/openhft/chronicle/ChronicleHashErrorListener.java similarity index 89% rename from src/main/java/net/openhft/chronicle/map/MapErrorListener.java rename to src/main/java/net/openhft/chronicle/ChronicleHashErrorListener.java index 2517bd5ba..30f404a4e 100644 --- a/src/main/java/net/openhft/chronicle/map/MapErrorListener.java +++ b/src/main/java/net/openhft/chronicle/ChronicleHashErrorListener.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package net.openhft.chronicle.map; +package net.openhft.chronicle; import java.io.Serializable; -public interface MapErrorListener extends Serializable { +public interface ChronicleHashErrorListener extends Serializable { void onLockTimeout(long threadId) throws IllegalStateException; void errorOnUnlock(IllegalMonitorStateException e); diff --git a/src/main/java/net/openhft/chronicle/map/MapErrorListeners.java b/src/main/java/net/openhft/chronicle/ChronicleHashErrorListeners.java similarity index 84% rename from src/main/java/net/openhft/chronicle/map/MapErrorListeners.java rename to src/main/java/net/openhft/chronicle/ChronicleHashErrorListeners.java index 67749d28d..52ff73c4d 100644 --- a/src/main/java/net/openhft/chronicle/map/MapErrorListeners.java +++ b/src/main/java/net/openhft/chronicle/ChronicleHashErrorListeners.java @@ -14,19 +14,19 @@ * limitations under the License. */ -package net.openhft.chronicle.map; +package net.openhft.chronicle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public final class MapErrorListeners { +public final class ChronicleHashErrorListeners { - public static MapErrorListener logging() { + public static ChronicleHashErrorListener logging() { return LoggingErrorListener.INSTANCE; } - public static MapErrorListener error() { + public static ChronicleHashErrorListener error() { return ThrowingErrorListener.INSTANCE; } @@ -35,7 +35,7 @@ public final class MapErrorListeners { * We can add some configuration-on-the-first-call in the future. */ - private static enum LoggingErrorListener implements MapErrorListener { + private static enum LoggingErrorListener implements ChronicleHashErrorListener { INSTANCE; final Logger LOG = LoggerFactory.getLogger(getClass()); @@ -55,7 +55,7 @@ public final class MapErrorListeners { } } - private static enum ThrowingErrorListener implements MapErrorListener { + private static enum ThrowingErrorListener implements ChronicleHashErrorListener { INSTANCE; @Override diff --git a/src/main/java/net/openhft/chronicle/map/TimeProvider.java b/src/main/java/net/openhft/chronicle/TimeProvider.java similarity index 74% rename from src/main/java/net/openhft/chronicle/map/TimeProvider.java rename to src/main/java/net/openhft/chronicle/TimeProvider.java index 0802b5fbd..5f48dcb51 100644 --- a/src/main/java/net/openhft/chronicle/map/TimeProvider.java +++ b/src/main/java/net/openhft/chronicle/TimeProvider.java @@ -14,21 +14,29 @@ * limitations under the License. */ -package net.openhft.chronicle.map; +package net.openhft.chronicle; + +import net.openhft.chronicle.map.ChronicleMap; +import net.openhft.chronicle.map.ChronicleSet; import java.io.ObjectStreamException; import java.io.Serializable; /** - * TimeProvider was created initially to support testing of ReplicatedChronicleMap, - * with the aim of possibly later providing and optimization to System.currentTimeMillis() - * on every call to put(), get() and remove() - *

- * Subclasses should be immutable, because {@link ChronicleMapBuilder} doesn't make defensive copies. + * TimeProvider was aims to possibly later provide an optimization to + * {@link System#currentTimeMillis()} on every call to replicated {@link ChronicleMap} + * or {@link ChronicleSet}. + * + * Subclasses should be immutable, because {@link ChronicleHashBuilder} doesn't make defensive + * copies. * * @author Rob Austin. + * @see ChronicleHashBuilder#timeProvider(TimeProvider) */ public abstract class TimeProvider implements Serializable { + /** + * Delegates {@link #currentTimeMillis()} to {@link System#currentTimeMillis()}. + */ public static final TimeProvider SYSTEM = new System(); private static final long serialVersionUID = 0L; diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 474abdaf8..bbcaaa7de 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -16,6 +16,10 @@ package net.openhft.chronicle.map; +import net.openhft.chronicle.ChronicleHashBuilder; +import net.openhft.chronicle.ChronicleHashErrorListener; +import net.openhft.chronicle.ChronicleHashErrorListeners; +import net.openhft.chronicle.TimeProvider; import net.openhft.chronicle.map.serialization.MetaBytesInterop; import net.openhft.chronicle.map.serialization.MetaBytesWriter; import net.openhft.chronicle.map.serialization.MetaProvider; @@ -34,7 +38,6 @@ import java.io.*; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import static net.openhft.chronicle.map.Objects.builderEquals; @@ -82,7 +85,8 @@ import static net.openhft.chronicle.map.Objects.builderEquals; * @param key type of the maps, produced by this builder * @param value type of the maps, produced by this builder */ -public class ChronicleMapBuilder implements Cloneable { +public class ChronicleMapBuilder implements Cloneable, + ChronicleHashBuilder, ChronicleMapBuilder> { private static final Bytes EMPTY_BYTES = new ByteBufferBytes(ByteBuffer.allocate(0)); private static final int DEFAULT_KEY_OR_VALUE_SIZE = 120; @@ -109,7 +113,7 @@ public class ChronicleMapBuilder implements Cloneable { private long lockTimeOut = 2000; private TimeUnit lockTimeOutUnit = TimeUnit.MILLISECONDS; private int metaDataBytes = 0; - private MapErrorListener errorListener = MapErrorListeners.logging(); + private ChronicleHashErrorListener errorListener = ChronicleHashErrorListeners.logging(); private boolean putReturnsNull = false; private boolean removeReturnsNull = false; private boolean largeSegments = false; @@ -151,12 +155,7 @@ public class ChronicleMapBuilder implements Cloneable { } } - /** - * Set minimum number of segments. See concurrencyLevel in {@link ConcurrentHashMap}. - * - * @param minSegments the minimum number of segments in maps, constructed by this builder - * @return this builder object back - */ + @Override public ChronicleMapBuilder minSegments(int minSegments) { this.minSegments = minSegments; return this; @@ -175,14 +174,7 @@ public class ChronicleMapBuilder implements Cloneable { } /** - * Configures the optimal number of bytes, taken by serialized form of keys, put into maps, - * created by this builder. If key size is always the same, call - * {@link #constantKeySizeBySample(Object)} method instead of this one. - * - *

If key size varies moderately, specify the size higher than average, but lower than - * the maximum possible, to minimize average memory overuse. If key size varies in a wide range, - * it's better to use {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it - * directly. + * {@inheritDoc} * *

Example: if keys in your map(s) are English words in {@link String} form, keys size 10 * (a bit more than average English word length) would be a good choice: 

{@code
@@ -196,12 +188,11 @@ public class ChronicleMapBuilder implements Cloneable {
      * (and each character takes 2 bytes on-heap), because default off-heap {@link String} encoding
      * is UTF-8 in {@code ChronicleMap}.)
      *
-     * @param keySize number of bytes, taken by serialized form of keys
-     * @return this {@code ChronicleMapBuilder} back
      * @see #constantKeySizeBySample(Object)
      * @see #valueSize(int)
      * @see #entrySize(int)
      */
+    @Override
     public ChronicleMapBuilder keySize(int keySize) {
         if (keySize <= 0)
             throw new IllegalArgumentException("Key size must be positive");
@@ -210,28 +201,18 @@ public class ChronicleMapBuilder implements Cloneable {
     }
 
     /**
-     * Configures the constant number of bytes, taken by serialized form of keys, put into maps,
-     * created by this builder. This is done by providing the {@code sampleKey}, all keys should
-     * take the same number of bytes in serialized form, as this sample object.
+     * {@inheritDoc}
      *
      * 
For example, if your keys are Git commit hashes:
{@code
      * Map gitCommitMessagesByHash =
      *     ChronicleMapBuilder.of(byte[].class, String.class)
      *     .constantKeySizeBySample(new byte[20])
-     *     .create();
-     * }

+     *     .create();}
* - *

If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size - * is known statically, it is automatically accounted and this method shouldn't be called. - * - *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be - * called instead of this one. - * - * @param sampleKey the sample key - * @return this builder back * @see #keySize(int) * @see #constantValueSizeBySample(Object) */ + @Override public ChronicleMapBuilder constantKeySizeBySample(K sampleKey) { this.sampleKey = sampleKey; return this; @@ -301,83 +282,26 @@ public class ChronicleMapBuilder implements Cloneable { } /** - * Configures the size in bytes of allocation unit of {@code ChronicleMap} instances, created by - * this builder. - * - *

{@code ChronicleMap} stores it's data off-heap, so it is required to serialize - * key and values (unless they are direct {@link Byteable} instances). Serialized key bytes + - * serialized value bytes + some metadata bytes comprise "entry space", which - * {@code ChronicleMap} should allocate. So entry size is a minimum allocation portion - * in the maps, created by this builder. E. g. if entry size is 100, the created map could only - * allocate 100, 200, 300... bytes for an entry. If say 150 bytes of entry space are required - * by the entry, 200 bytes will be allocated, 150 used and 50 wasted. To minimize memory overuse - * and improve speed, you should pay decent attention to this configuration. + * {@inheritDoc} * *

In fully default case you can expect entry size to be about 256 bytes. But it is strongly * recommended always to configure {@linkplain #keySize(int) key size} and * {@linkplain #valueSize(int) value size}, if they couldn't be derived statically. * - *

If entry size is not {@linkplain #entrySize(int) configured} explicitly, it is computed - * based on {@linkplain #metaDataBytes() meta data bytes}, plus {@linkplain #keySize(int) key + *

If entry size is not configured explicitly by calling this method, it is computed + * based on {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key * size}, plus {@linkplain #valueSize(int) value size}, plus a few bytes required by * implementations, with respect to {@linkplain #entryAndValueAlignment(Alignment) alignment}. * - *

There are three major patterns of this configuration usage: - *

    - *
  1. Key and value sizes are constant. Configure them via {@link #constantKeySizeBySample( - * Object)} and {@link #constantValueSizeBySample(Object)} methods, and you will experience - * no memory waste at all.
    2. - *
  2. Key and/or value size varies moderately. Specify them using corresponding methods, - * or {@linkplain #entrySize(int) specify entry size} directly, by values somewhere between - * average and maximum possible. The idea is to have most (90% or more) entries to fit - * a single "entry size" with moderate memory waste (10-20% on average), rest 10% or less of - * entries should take 2 "entry sizes", thus with ~50% memory overuse.
    3. - *
  3. Key and/or value size varies in a wide range. Then it's best to use entry size - * configuration in chunk mode. Specify entry size so that most entries should take - * from 5 to several dozens of "chunks". With this approach, average memory waste should be - * very low. - * - *

    However, remember that - *

      - *
    • Operations with entries that span several "entry sizes" are a bit slower, than - * entries which take a single "entry size". That is why "chunk" approach - * is not recommended, when key and/or value size varies moderately.
      • - *
    • The maximum number of chunks could be taken by an entry is 64. - * {@link IllegalArgumentException} is thrown on attempt to insert too large entry, - * compared to the configured or computed entry size.
      • - *
    • The number of "entries" {@linkplain #entries(long) in the whole map} and - * {@linkplain #actualEntriesPerSegment(int) per segment} is actually the number - * "chunks".
      • - *
    - * - *

    Example: if values in your map are adjacency lists of some social graph, where nodes - * are represented as {@code long} ids, and adjacency lists are serialized in efficient - * manner, for example as {@code long[]} arrays. Typical number of connections is 100-300, - * maximum is 3000. In this case entry size of 50 * (8 bytes for each id) = 400 bytes would - * be a good choice: 

    {@code
-     * Map socialGraph = ChronicleMapBuilder
-     *     .of(Long.class, long[].class)
-     *     // given that graph should have of 1 billion nodes, and 150 average adjacency list size
-     *     // => values takes 3 chuncks on average
-     *     .entries(1_000_000_000L * (150 / 50))
-     *     .entrySize(50 * 8)
-     *     .create();}
    - * It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max - * chunks by single entry" limit, and ensures moderate average memory overuse (not more - * than 20%). - *
    4. - *
- * *

Note that the actual entrySize will be aligned to 4 (default {@linkplain * #entryAndValueAlignment(Alignment) entry alignment}). I. e. if you set entry size to 30, * the actual entry size will be 32 (30 aligned to 4 bytes). If you don't want entry size * to be aligned, set {@code entryAndValueAlignment(Alignment.NO_ALIGNMENT)}. * - * @param entrySize the size in bytes - * @return this {@code ChronicleMapBuilder} back * @see #entryAndValueAlignment(Alignment) * @see #entries(long) */ + @Override public ChronicleMapBuilder entrySize(int entrySize) { if (entrySize <= 0) throw new IllegalArgumentException("Entry Size must be positive"); @@ -439,36 +363,7 @@ public class ChronicleMapBuilder implements Cloneable { return alignment; } - /** - * Configures the maximum number of {@linkplain #entrySize(int) "entry size chunks"}, which - * could be taken by the maximum number of entries, inserted into the maps, created by this - * builder. If you try to insert more data, {@link IllegalStateException} might be thrown, - * because currently {@code ChronicleMap} doesn't support resizing. - * - *

    - *
  1. If key and value sizes are constant, this number is equal to the maximum number - * of entries (because each entry takes exactly one "entry size" memory unit).
    2. - *
  2. If key and/or value size varies moderately, you should pass to this method - * the maximum number of entries + 5-25%, depending on your data properties and configured - * {@linkplain #keySize(int) key}/{@linkplain #valueSize(int) value}/{@linkplain - * #entrySize(int) entry} sizes.
    3. - *
  3. If your data size varies in a wide range, pass the maximum number of entries - * multiplied by average data size and divided by the configured "entry size" (i. e. chunk - * size). See an example in the documentation to {@link #entrySize(int)} method.
    4. - *
- * - *

You shouldn't put additional margin over the number, computed according the rules above. - * This bad practice was popularized by {@link HashMap#HashMap(int)} constructor, which accept - * "capacity", that should be multiplied by "load factor" to obtain actual maximum expected - * number of entries. {@code ChronicleMap} doesn't have a notion of load factor. - * - *

Default value is 2^20 (~ 1 million). - * - * @param entries maximum size of the created maps, in memory allocation units, - * so-called "entry size" - * @return this builder back - * @see #entrySize(int) - */ + @Override public ChronicleMapBuilder entries(long entries) { this.entries = entries; return this; @@ -478,6 +373,7 @@ public class ChronicleMapBuilder implements Cloneable { return entries; } + @Override public ChronicleMapBuilder actualEntriesPerSegment(int actualEntriesPerSegment) { this.actualEntriesPerSegment = actualEntriesPerSegment; return this; @@ -491,6 +387,7 @@ public class ChronicleMapBuilder implements Cloneable { return (int) (Math.max(1, entries * 2L / as) + 63) & ~63; } + @Override public ChronicleMapBuilder actualSegments(int actualSegments) { this.actualSegments = actualSegments; return this; @@ -508,6 +405,7 @@ public class ChronicleMapBuilder implements Cloneable { return (int) Maths.nextPower2(Math.max((entries >> 30) + 1, minSegments()), 1); } + @Override public ChronicleMapBuilder lockTimeOut(long lockTimeOut, TimeUnit unit) { this.lockTimeOut = lockTimeOut; lockTimeOutUnit = unit; @@ -518,12 +416,13 @@ public class ChronicleMapBuilder implements Cloneable { return unit.convert(lockTimeOut, lockTimeOutUnit); } - public ChronicleMapBuilder errorListener(MapErrorListener errorListener) { + @Override + public ChronicleMapBuilder errorListener(ChronicleHashErrorListener errorListener) { this.errorListener = errorListener; return this; } - MapErrorListener errorListener() { + ChronicleHashErrorListener errorListener() { return errorListener; } @@ -582,11 +481,13 @@ public class ChronicleMapBuilder implements Cloneable { return entries > 1L << (20 + 15) || largeSegments; } + @Override public ChronicleMapBuilder largeSegments(boolean largeSegments) { this.largeSegments = largeSegments; return this; } + @Override public ChronicleMapBuilder metaDataBytes(int metaDataBytes) { if ((metaDataBytes & 0xFF) != metaDataBytes) throw new IllegalArgumentException("MetaDataBytes must be [0..255] was " + metaDataBytes); @@ -647,6 +548,7 @@ public class ChronicleMapBuilder implements Cloneable { return toString().hashCode(); } + @Override public ChronicleMapBuilder addReplicator(Replicator replicator) { if (firstReplicator == null) { firstReplicator = replicator; @@ -665,6 +567,7 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + @Override public ChronicleMapBuilder setReplicators(Replicator... replicators) { firstReplicator = null; this.replicators.clear(); @@ -674,6 +577,7 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + @Override public ChronicleMapBuilder timeProvider(TimeProvider timeProvider) { this.timeProvider = timeProvider; return this; @@ -689,6 +593,7 @@ public class ChronicleMapBuilder implements Cloneable { bytesMarshallerFactory; } + @Override public ChronicleMapBuilder bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) { this.bytesMarshallerFactory = bytesMarshallerFactory; return this; @@ -701,6 +606,7 @@ public class ChronicleMapBuilder implements Cloneable { objectSerializer; } + @Override public ChronicleMapBuilder objectSerializer(ObjectSerializer objectSerializer) { this.objectSerializer = objectSerializer; return this; @@ -714,22 +620,13 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + @Override public ChronicleMapBuilder keyMarshaller(@NotNull BytesMarshaller keyMarshaller) { keyBuilder.marshaller(keyMarshaller, null); return this; } - /** - * Specifies that key objects, queried with the maps created by this builder, are inherently - * immutable. Keys in {@code ChronicleMap} are not required to be immutable, as in ordinary - * {@link Map} implementations, because they are serialized off-heap. However, - * {@code ChronicleMap} implementations can benefit from the knowledge that keys are not mutated - * between queries. By default, {@code ChronicleMapBuilder} detects immutability automatically - * only for very few standard JDK types (for example, for {@link String}), it is not recommended - * to rely on {@code ChronicleMapBuilder} to be smart enough about this. - * - * @return this builder back - */ + @Override public ChronicleMapBuilder immutableKeys() { keyBuilder.instancesAreMutable(false); return this; @@ -815,6 +712,7 @@ public class ChronicleMapBuilder implements Cloneable { return new ConstantValueProvider(defaultValue, metaValueWriter, valueWriter); } + @Override public ChronicleMap create(File file) throws IOException { for (int i = 0; i < 10; i++) { if (file.exists() && file.length() > 0) { @@ -861,6 +759,7 @@ public class ChronicleMapBuilder implements Cloneable { return establishReplication(map); } + @Override public ChronicleMap create() throws IOException { VanillaChronicleMap map = newMap(); BytesStore bytesStore = new DirectStore(JDKObjectSerializer.INSTANCE, diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 55d982d99..bcf0a5d0c 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -17,6 +17,7 @@ package net.openhft.chronicle.map; +import net.openhft.chronicle.TimeProvider; import net.openhft.lang.Maths; import net.openhft.lang.collection.ATSDirectBitSet; import net.openhft.lang.io.Bytes; diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index ec60352ba..1a2189a09 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -16,6 +16,7 @@ package net.openhft.chronicle.map; +import net.openhft.chronicle.ChronicleHashErrorListener; import net.openhft.chronicle.map.serialization.*; import net.openhft.chronicle.map.threadlocal.Provider; import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; @@ -88,7 +89,7 @@ class VanillaChronicleMap, final boolean removeReturnsNull; private final long lockTimeOutNS; - private final MapErrorListener errorListener; + private final ChronicleHashErrorListener errorListener; transient Segment[] segments; // non-final for close() // non-final for close() and because it is initialized out of constructor transient BytesStore ms; diff --git a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java index 6b37d2fb7..26a27f8f5 100644 --- a/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java +++ b/src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java @@ -16,6 +16,10 @@ package net.openhft.chronicle.set; +import net.openhft.chronicle.TimeProvider; +import net.openhft.chronicle.map.Alignment; +import net.openhft.chronicle.ChronicleHashBuilder; +import net.openhft.chronicle.ChronicleHashErrorListener; import net.openhft.chronicle.map.*; import net.openhft.lang.io.serialization.BytesMarshaller; import net.openhft.lang.io.serialization.BytesMarshallerFactory; @@ -29,10 +33,12 @@ import java.util.AbstractSet; import java.util.Collection; import java.util.Iterator; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -public class ChronicleSetBuilder implements Cloneable { +public class ChronicleSetBuilder + implements ChronicleHashBuilder, ChronicleSetBuilder> { private ChronicleMapBuilder chronicleMapBuilder; @@ -57,98 +63,104 @@ public class ChronicleSetBuilder implements Cloneable { } } + @Override public ChronicleSetBuilder actualSegments(int actualSegments) { chronicleMapBuilder.actualSegments(actualSegments); return this; } - /** - * Set minimum number of segments. See concurrencyLevel in {@link java.util.concurrent.ConcurrentHashMap}. - * - * @param minSegments the minimum number of segments in maps, constructed by this builder - * @return this builder object back - */ + @Override public ChronicleSetBuilder minSegments(int minSegments) { chronicleMapBuilder.minSegments(minSegments); return this; } - public ChronicleSetBuilder actualElementsPerSegment(int actualEntriesPerSegment) { + @Override + public ChronicleSetBuilder actualEntriesPerSegment(int actualEntriesPerSegment) { chronicleMapBuilder.actualEntriesPerSegment(actualEntriesPerSegment); return this; } /** - * Configures the optimal number of bytes for an element. If elementSize is always the same, call {@link - * #constantElementsSizeBySample(Object)} method instead of this one. + * {@inheritDoc} * - *

If the size varies moderately, specify the size higher than average, but lower than the maximum - * possible, to minimize average memory overuse. If the size varies in a wide range, it's better to use - * - * - *

Example: if element in your set(s) are English words in {@link String} form, keys size 10 + *

Example: if keys in your set(s) are English words in {@link String} form, keys size 10 * (a bit more than average English word length) would be a good choice: 

{@code
-     * ChronicleSetBuilder wordFrequencies = ChronicleSetBuilder
-     *     .of(String.class)
+     * ChronicleSet uniqueWords = ChronicleSetBuilder.of(String.class)
      *     .entries(50000)
-     *     .elementSize(10)
+     *     .keySize(10)
      *     .create();}
+ * (Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded + * (and each character takes 2 bytes on-heap), because default off-heap {@link String} encoding + * is UTF-8 in {@code ChronicleSet}.) * - * @param elementSize number of bytes, taken by serialized form of the element - * @return this {@code ChronicleSetBuilder} back - * @see #constantElementsSizeBySample(Object) + * @see #constantKeySizeBySample(Object) + * @see #entrySize(int) */ - public ChronicleSetBuilder elementSize(int elementSize) { - chronicleMapBuilder.keySize(elementSize); + @Override + public ChronicleSetBuilder keySize(int keySize) { + chronicleMapBuilder.keySize(keySize); return this; } - /** - * Configures the constant number of bytes, taken by serialized form of elements, created by this builder. - * This is done by providing the {@code element}, all elements should take the same number of bytes in - * serialized form, as this sample object. + * {@inheritDoc} * - *

For example, if your elements are Git commit hashes:

{@code
-     * Map gitCommitMessagesByHash =
-     *     ChronicleSetBuilder.of(byte[].class)
-     *     .constantElementsSizeBySample(new byte[20])
-     *     .create();
-     * }
+ *

For example, if your keys are Git commit hashes:

{@code
+     * Set gitCommitsOfInterest = ChronicleSetBuilder.of(byte[].class)
+     *     .constantKeySizeBySample(new byte[20])
+     *     .create();}
* - *

If elements are of boxed primitive type or {@link net.openhft.lang.model.Byteable} subclass, i. e. - * if element size is known statically, it is automatically accounted and this method shouldn't be - * called. - * - * @param element the sample element - * @return this builder back + * @see #keySize(int) */ - public ChronicleSetBuilder constantElementsSizeBySample(E element) { - chronicleMapBuilder.constantKeySizeBySample(element); + @Override + public ChronicleSetBuilder constantKeySizeBySample(E sampleKey) { + chronicleMapBuilder.constantKeySizeBySample(sampleKey); return this; } - public ChronicleSetBuilder elements(long entries) { + /** + * {@inheritDoc} + * + *

In fully default case you can expect entry size to be about 120-130 bytes. But it is + * strongly recommended always to configure {@linkplain #keySize(int) key size}, + * if they couldn't be derived statically. + * + *

If entry size is not configured explicitly by calling this method, it is computed + * based on {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key + * size}, plus a few bytes required by implementations. + */ + @Override + public ChronicleSetBuilder entrySize(int entrySize) { + chronicleMapBuilder.entrySize(entrySize); + return this; + } + + @Override + public ChronicleSetBuilder entries(long entries) { chronicleMapBuilder.entries(entries); return this; } + @Override public ChronicleSetBuilder lockTimeOut(long lockTimeOut, TimeUnit unit) { chronicleMapBuilder.lockTimeOut(lockTimeOut, unit); return this; } - - public ChronicleSetBuilder errorListener(MapErrorListener errorListener) { + @Override + public ChronicleSetBuilder errorListener(ChronicleHashErrorListener errorListener) { chronicleMapBuilder.errorListener(errorListener); return this; } + @Override public ChronicleSetBuilder largeSegments(boolean largeSegments) { chronicleMapBuilder.largeSegments(largeSegments); return this; } + @Override public ChronicleSetBuilder metaDataBytes(int metaDataBytes) { chronicleMapBuilder.metaDataBytes(metaDataBytes); return this; @@ -164,7 +176,9 @@ public class ChronicleSetBuilder implements Cloneable { @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override public boolean equals(Object o) { - return chronicleMapBuilder.equals(o); + if (!(o instanceof ChronicleSetBuilder)) + return false; + return chronicleMapBuilder.equals(((ChronicleSetBuilder) o).chronicleMapBuilder); } @Override @@ -182,45 +196,44 @@ public class ChronicleSetBuilder implements Cloneable { return this; } + @Override public ChronicleSetBuilder timeProvider(TimeProvider timeProvider) { chronicleMapBuilder.timeProvider(timeProvider); return this; } - public ChronicleSetBuilder bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) { + @Override + public ChronicleSetBuilder bytesMarshallerFactory( + BytesMarshallerFactory bytesMarshallerFactory) { chronicleMapBuilder.bytesMarshallerFactory(bytesMarshallerFactory); return this; } + @Override public ChronicleSetBuilder objectSerializer(ObjectSerializer objectSerializer) { chronicleMapBuilder.objectSerializer(objectSerializer); return this; } + @Override public ChronicleSetBuilder keyMarshaller(@NotNull BytesMarshaller keyMarshaller) { chronicleMapBuilder.keyMarshaller(keyMarshaller); return this; } - /** - * Specifies that elements, are immutable. {@code ChronicleSet} implementations can benefit from the - * knowledge that elements are not mutated between queries. By default, {@code ChronicleMapBuilder} - * detects immutability automatically only for very few standard JDK types (for example, for {@link - * String}), it is not recommended to rely on {@code ChronicleSetBuilder} to be smart enough about this. - * - * @return this builder back - */ + @Override public ChronicleSetBuilder immutableKeys() { chronicleMapBuilder.immutableKeys(); return this; } - + @Override public ChronicleSet create(File file) throws IOException { final ChronicleMap map = chronicleMapBuilder.create(file); return new SetFromMap(map); } + @Override public ChronicleSet create() throws IOException { final ChronicleMap map = chronicleMapBuilder.create(); return new SetFromMap(map); diff --git a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java index 6635b84cf..d5ffc4545 100644 --- a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java +++ b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTests.java @@ -16,6 +16,7 @@ package net.openhft.chronicle.map; +import net.openhft.chronicle.TimeProvider; import net.openhft.chronicle.map.jrs166.JSR166TestCase; import org.junit.Test; import org.mockito.Mockito;