Alawn/Chronicle-Map
1
0
Fork 0
mirror of https://github.com/OpenHFT/Chronicle-Map.git synced 2021-09-13 01:26:12 +00:00
Code Releases Activity

HCOLL-164 Exctracted ChronicleHashBuilder interface and pulled some classes up from n.o.chronicle.map package to n.o.chronicle. Fixed ChronicleSetBuilder.equals() implementation.

Browse Source
This commit is contained in:
leventov
2014-10-13 18:13:31 +04:00
parent 1a0ecc621c
commit 6c02947ffb
9 changed files with 351 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java
+35 -136
View File
@@ -16,6 +16,10 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.ChronicleHashBuilder;
import net.openhft.chronicle.ChronicleHashErrorListener;
import net.openhft.chronicle.ChronicleHashErrorListeners;
import net.openhft.chronicle.TimeProvider;
import net.openhft.chronicle.map.serialization.MetaBytesInterop;
import net.openhft.chronicle.map.serialization.MetaBytesWriter;
import net.openhft.chronicle.map.serialization.MetaProvider;
@@ -34,7 +38,6 @@ import java.io.*;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import static net.openhft.chronicle.map.Objects.builderEquals;
@@ -82,7 +85,8 @@ import static net.openhft.chronicle.map.Objects.builderEquals;
* @param <K> key type of the maps, produced by this builder
* @param <V> value type of the maps, produced by this builder
*/
public class ChronicleMapBuilder<K, V> implements Cloneable {
public class ChronicleMapBuilder<K, V> implements Cloneable,
ChronicleHashBuilder<K, ChronicleMap<K, V>, ChronicleMapBuilder<K, V>> {
private static final Bytes EMPTY_BYTES = new ByteBufferBytes(ByteBuffer.allocate(0));
private static final int DEFAULT_KEY_OR_VALUE_SIZE = 120;
@@ -109,7 +113,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
private long lockTimeOut = 2000;
private TimeUnit lockTimeOutUnit = TimeUnit.MILLISECONDS;
private int metaDataBytes = 0;
private MapErrorListener errorListener = MapErrorListeners.logging();
private ChronicleHashErrorListener errorListener = ChronicleHashErrorListeners.logging();
private boolean putReturnsNull = false;
private boolean removeReturnsNull = false;
private boolean largeSegments = false;
@@ -151,12 +155,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
}
}
/**
* Set minimum number of segments. See concurrencyLevel in {@link ConcurrentHashMap}.
*
* @param minSegments the minimum number of segments in maps, constructed by this builder
* @return this builder object back
*/
@Override
public ChronicleMapBuilder<K, V> minSegments(int minSegments) {
this.minSegments = minSegments;
return this;
@@ -175,14 +174,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
}
/**
* Configures the optimal number of bytes, taken by serialized form of keys, put into maps,
* created by this builder. If key size is always the same, call
* {@link #constantKeySizeBySample(Object)} method instead of this one.
*
* <p>If key size varies moderately, specify the size higher than average, but lower than
* the maximum possible, to minimize average memory overuse. If key size varies in a wide range,
* it's better to use {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it
* directly.
* {@inheritDoc}
*
* <p>Example: if keys in your map(s) are English words in {@link String} form, keys size 10
* (a bit more than average English word length) would be a good choice: <pre>{@code
@@ -196,12 +188,11 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
* (and each character takes 2 bytes on-heap), because default off-heap {@link String} encoding
* is UTF-8 in {@code ChronicleMap}.)
*
* @param keySize number of bytes, taken by serialized form of keys
* @return this {@code ChronicleMapBuilder} back
* @see #constantKeySizeBySample(Object)
* @see #valueSize(int)
* @see #entrySize(int)
*/
@Override
public ChronicleMapBuilder<K, V> keySize(int keySize) {
if (keySize <= 0)
throw new IllegalArgumentException("Key size must be positive");
@@ -210,28 +201,18 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
}
/**
* Configures the constant number of bytes, taken by serialized form of keys, put into maps,
* created by this builder. This is done by providing the {@code sampleKey}, all keys should
* take the same number of bytes in serialized form, as this sample object.
* {@inheritDoc}
*
* <p>For example, if your keys are Git commit hashes:<pre>{@code
* Map<byte[], String> gitCommitMessagesByHash =
* ChronicleMapBuilder.of(byte[].class, String.class)
* .constantKeySizeBySample(new byte[20])
* .create();
* }</pre>
* .create();}</pre>
*
* <p>If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size
* is known statically, it is automatically accounted and this method shouldn't be called.
*
* <p>If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be
* called instead of this one.
*
* @param sampleKey the sample key
* @return this builder back
* @see #keySize(int)
* @see #constantValueSizeBySample(Object)
*/
@Override
public ChronicleMapBuilder<K, V> constantKeySizeBySample(K sampleKey) {
this.sampleKey = sampleKey;
return this;
@@ -301,83 +282,26 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
}
/**
* Configures the size in bytes of allocation unit of {@code ChronicleMap} instances, created by
* this builder.
*
* <p>{@code ChronicleMap} stores it's data off-heap, so it is required to serialize
* key and values (unless they are direct {@link Byteable} instances). Serialized key bytes +
* serialized value bytes + some metadata bytes comprise "entry space", which
* {@code ChronicleMap} should allocate. So <i>entry size</i> is a minimum allocation portion
* in the maps, created by this builder. E. g. if entry size is 100, the created map could only
* allocate 100, 200, 300... bytes for an entry. If say 150 bytes of entry space are required
* by the entry, 200 bytes will be allocated, 150 used and 50 wasted. To minimize memory overuse
* and improve speed, you should pay decent attention to this configuration.
* {@inheritDoc}
*
* <p>In fully default case you can expect entry size to be about 256 bytes. But it is strongly
* recommended always to configure {@linkplain #keySize(int) key size} and
* {@linkplain #valueSize(int) value size}, if they couldn't be derived statically.
*
* <p>If entry size is not {@linkplain #entrySize(int) configured} explicitly, it is computed
* based on {@linkplain #metaDataBytes() meta data bytes}, plus {@linkplain #keySize(int) key
* <p>If entry size is not configured explicitly by calling this method, it is computed
* based on {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key
* size}, plus {@linkplain #valueSize(int) value size}, plus a few bytes required by
* implementations, with respect to {@linkplain #entryAndValueAlignment(Alignment) alignment}.
*
* <p>There are three major patterns of this configuration usage:
* <ol>
* <li>Key and value sizes are constant. Configure them via {@link #constantKeySizeBySample(
* Object)} and {@link #constantValueSizeBySample(Object)} methods, and you will experience
* no memory waste at all.</li>
* <li>Key and/or value size varies moderately. Specify them using corresponding methods,
* or {@linkplain #entrySize(int) specify entry size} directly, by values somewhere between
* average and maximum possible. The idea is to have most (90% or more) entries to fit
* a single "entry size" with moderate memory waste (10-20% on average), rest 10% or less of
* entries should take 2 "entry sizes", thus with ~50% memory overuse.</li>
* <li>Key and/or value size varies in a wide range. Then it's best to use entry size
* configuration in <i>chunk mode</i>. Specify entry size so that most entries should take
* from 5 to several dozens of "chunks". With this approach, average memory waste should be
* very low.
*
* <p>However, remember that
* <ul>
* <li>Operations with entries that span several "entry sizes" are a bit slower, than
* entries which take a single "entry size". That is why "chunk" approach
* is not recommended, when key and/or value size varies moderately.</li>
* <li>The maximum number of chunks could be taken by an entry is 64.
* {@link IllegalArgumentException} is thrown on attempt to insert too large entry,
* compared to the configured or computed entry size.</li>
* <li>The number of "entries" {@linkplain #entries(long) in the whole map} and
* {@linkplain #actualEntriesPerSegment(int) per segment} is actually the number
* "chunks".</li>
* </ul>
*
* <p>Example: if values in your map are adjacency lists of some social graph, where nodes
* are represented as {@code long} ids, and adjacency lists are serialized in efficient
* manner, for example as {@code long[]} arrays. Typical number of connections is 100-300,
* maximum is 3000. In this case entry size of 50 * (8 bytes for each id) = 400 bytes would
* be a good choice: <pre>{@code
* Map<Long, long[]> socialGraph = ChronicleMapBuilder
* .of(Long.class, long[].class)
* // given that graph should have of 1 billion nodes, and 150 average adjacency list size
* // => values takes 3 chuncks on average
* .entries(1_000_000_000L * (150 / 50))
* .entrySize(50 * 8)
* .create();}</pre>
* It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max
* chunks by single entry" limit, and ensures moderate average memory overuse (not more
* than 20%).
* </li>
* </ol>
*
* <p>Note that the actual entrySize will be aligned to 4 (default {@linkplain
* #entryAndValueAlignment(Alignment) entry alignment}). I. e. if you set entry size to 30,
* the actual entry size will be 32 (30 aligned to 4 bytes). If you don't want entry size
* to be aligned, set {@code entryAndValueAlignment(Alignment.NO_ALIGNMENT)}.
*
* @param entrySize the size in bytes
* @return this {@code ChronicleMapBuilder} back
* @see #entryAndValueAlignment(Alignment)
* @see #entries(long)
*/
@Override
public ChronicleMapBuilder<K, V> entrySize(int entrySize) {
if (entrySize <= 0)
throw new IllegalArgumentException("Entry Size must be positive");
@@ -439,36 +363,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return alignment;
}
/**
* Configures the maximum number of {@linkplain #entrySize(int) "entry size chunks"}, which
* could be taken by the maximum number of entries, inserted into the maps, created by this
* builder. If you try to insert more data, {@link IllegalStateException} might be thrown,
* because currently {@code ChronicleMap} doesn't support resizing.
*
* <ol>
* <li>If key and value sizes are constant, this number is equal to the maximum number
* of entries (because each entry takes exactly one "entry size" memory unit).</li>
* <li>If key and/or value size varies moderately, you should pass to this method
* the maximum number of entries + 5-25%, depending on your data properties and configured
* {@linkplain #keySize(int) key}/{@linkplain #valueSize(int) value}/{@linkplain
* #entrySize(int) entry} sizes.</li>
* <li>If your data size varies in a wide range, pass the maximum number of entries
* multiplied by average data size and divided by the configured "entry size" (i. e. chunk
* size). See an example in the documentation to {@link #entrySize(int)} method.</li>
* </ol>
*
* <p>You shouldn't put additional margin over the number, computed according the rules above.
* This bad practice was popularized by {@link HashMap#HashMap(int)} constructor, which accept
* "capacity", that should be multiplied by "load factor" to obtain actual maximum expected
* number of entries. {@code ChronicleMap} doesn't have a notion of load factor.
*
* <p>Default value is 2^20 (~ 1 million).
*
* @param entries maximum size of the created maps, in memory allocation units,
* so-called "entry size"
* @return this builder back
* @see #entrySize(int)
*/
@Override
public ChronicleMapBuilder<K, V> entries(long entries) {
this.entries = entries;
return this;
@@ -478,6 +373,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return entries;
}
@Override
public ChronicleMapBuilder<K, V> actualEntriesPerSegment(int actualEntriesPerSegment) {
this.actualEntriesPerSegment = actualEntriesPerSegment;
return this;
@@ -491,6 +387,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return (int) (Math.max(1, entries * 2L / as) + 63) & ~63;
}
@Override
public ChronicleMapBuilder<K, V> actualSegments(int actualSegments) {
this.actualSegments = actualSegments;
return this;
@@ -508,6 +405,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return (int) Maths.nextPower2(Math.max((entries >> 30) + 1, minSegments()), 1);
}
@Override
public ChronicleMapBuilder<K, V> lockTimeOut(long lockTimeOut, TimeUnit unit) {
this.lockTimeOut = lockTimeOut;
lockTimeOutUnit = unit;
@@ -518,12 +416,13 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return unit.convert(lockTimeOut, lockTimeOutUnit);
}
public ChronicleMapBuilder<K, V> errorListener(MapErrorListener errorListener) {
@Override
public ChronicleMapBuilder<K, V> errorListener(ChronicleHashErrorListener errorListener) {
this.errorListener = errorListener;
return this;
}
MapErrorListener errorListener() {
ChronicleHashErrorListener errorListener() {
return errorListener;
}
@@ -582,11 +481,13 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return entries > 1L << (20 + 15) || largeSegments;
}
@Override
public ChronicleMapBuilder<K, V> largeSegments(boolean largeSegments) {
this.largeSegments = largeSegments;
return this;
}
@Override
public ChronicleMapBuilder<K, V> metaDataBytes(int metaDataBytes) {
if ((metaDataBytes & 0xFF) != metaDataBytes)
throw new IllegalArgumentException("MetaDataBytes must be [0..255] was " + metaDataBytes);
@@ -647,6 +548,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return toString().hashCode();
}
@Override
public ChronicleMapBuilder<K, V> addReplicator(Replicator replicator) {
if (firstReplicator == null) {
firstReplicator = replicator;
@@ -665,6 +567,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return this;
}
@Override
public ChronicleMapBuilder<K, V> setReplicators(Replicator... replicators) {
firstReplicator = null;
this.replicators.clear();
@@ -674,6 +577,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return this;
}
@Override
public ChronicleMapBuilder<K, V> timeProvider(TimeProvider timeProvider) {
this.timeProvider = timeProvider;
return this;
@@ -689,6 +593,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
bytesMarshallerFactory;
}
@Override
public ChronicleMapBuilder<K, V> bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) {
this.bytesMarshallerFactory = bytesMarshallerFactory;
return this;
@@ -701,6 +606,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
objectSerializer;
}
@Override
public ChronicleMapBuilder<K, V> objectSerializer(ObjectSerializer objectSerializer) {
this.objectSerializer = objectSerializer;
return this;
@@ -714,22 +620,13 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return this;
}
@Override
public ChronicleMapBuilder<K, V> keyMarshaller(@NotNull BytesMarshaller<K> keyMarshaller) {
keyBuilder.marshaller(keyMarshaller, null);
return this;
}
/**
* Specifies that key objects, queried with the maps created by this builder, are inherently
* immutable. Keys in {@code ChronicleMap} are not required to be immutable, as in ordinary
* {@link Map} implementations, because they are serialized off-heap. However,
* {@code ChronicleMap} implementations can benefit from the knowledge that keys are not mutated
* between queries. By default, {@code ChronicleMapBuilder} detects immutability automatically
* only for very few standard JDK types (for example, for {@link String}), it is not recommended
* to rely on {@code ChronicleMapBuilder} to be smart enough about this.
*
* @return this builder back
*/
@Override
public ChronicleMapBuilder<K, V> immutableKeys() {
keyBuilder.instancesAreMutable(false);
return this;
@@ -815,6 +712,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return new ConstantValueProvider<K, V>(defaultValue, metaValueWriter, valueWriter);
}
@Override
public ChronicleMap<K, V> create(File file) throws IOException {
for (int i = 0; i < 10; i++) {
if (file.exists() && file.length() > 0) {
@@ -861,6 +759,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
return establishReplication(map);
}
@Override
public ChronicleMap<K, V> create() throws IOException {
VanillaChronicleMap<K, ?, ?, V, ?, ?> map = newMap();
BytesStore bytesStore = new DirectStore(JDKObjectSerializer.INSTANCE,