Alawn/Chronicle-Map
1
0
Fork 0
mirror of https://github.com/OpenHFT/Chronicle-Map.git synced 2021-09-13 01:26:12 +00:00
Code Releases Activity

Merge remote-tracking branch 'origin/master'

Browse Source 
Conflicts:
	src/test/java/net/openhft/chronicle/map/ChronicleMapImportExportTest.java
This commit is contained in:
Rob Austin
2014-12-26 12:55:24 +00:00
parent 6afd73bf30 aa7a83a2a1
commit 95d98675a4
44 changed files with 959 additions and 608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java
+105 -78
View File
@@ -55,6 +55,11 @@ import java.util.concurrent.TimeUnit;
* the corresponding configuration changed. To make an independent configuration, {@linkplain
* #clone} the builder.
*
* <p><a name="low-level-config"></a>There are some "low-level" configurations in this builder,
* that require deep understanding of the Chronicle implementation design to be properly used.
* Know what you do. These configurations are picked up strictly as-is, without extra round-ups,
* adjustments, etc.
*
* @param <K> the type of keys in hash containers, created by this builder
* @param <C> the container type, created by this builder, i. e. {@link ChronicleMap} or {@link
* ChronicleSet}
@@ -83,27 +88,29 @@ public interface ChronicleHashBuilder<K, C extends ChronicleHash,
B minSegments(int minSegments);
/**
* Configures the optimal number of bytes, taken by serialized form of keys, put into hash
* Configures the average number of bytes, taken by serialized form of keys, put into hash
* containers, created by this builder. If key size is always the same, call {@link
* #constantKeySizeBySample(Object)} method instead of this one.
*
* <p>If key size varies moderately, specify the size higher than average, but lower than the
* maximum possible, to minimize average memory overuse. If key size varies in a wide range,
* it's better to use {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it
* directly.
* <p>{@code ChronicleHashBuilder} implementation heuristically chooses
* {@linkplain #actualChunkSize(int) the actual chunk size} based on this configuration, that,
* however, might result to quite high internal fragmentation, i. e. losses because only
* integral number of chunks could be allocated for the entry. If you want to avoid this, you
* should manually configure the actual chunk size in addition to this average key size
* configuration, which is anyway needed.
*
* <p>If key is a boxed primitive type or {@link Byteable} subclass, i. e. if key size is known
* statically, it is automatically accounted and shouldn't be specified by user.
*
* @param keySize number of bytes, taken by serialized form of keys
* @param averageKeySize the average number of bytes, taken by serialized form of keys
* @return this builder back
* @throws IllegalStateException if key size is known statically and shouldn't be configured
* by user
* @throws IllegalArgumentException if the given {@code keySize} is non-positive
* @see #constantKeySizeBySample(Object)
* @see #entrySize(int)
* @see #actualChunkSize(int)
*/
B keySize(int keySize);
B averageKeySize(double averageKeySize);
/**
* Configures the constant number of bytes, taken by serialized form of keys, put into hash
@@ -113,12 +120,12 @@ public interface ChronicleHashBuilder<K, C extends ChronicleHash,
* <p>If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size is
* known statically, it is automatically accounted and this method shouldn't be called.
*
* <p>If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be
* called instead of this one.
* <p>If key size varies, method {@link #averageKeySize(double)} should be called instead of
* this one.
*
* @param sampleKey the sample key
* @return this builder back
* @see #keySize(int)
* @see #averageKeySize(double)
*/
B constantKeySizeBySample(K sampleKey);
@@ -130,104 +137,124 @@ public interface ChronicleHashBuilder<K, C extends ChronicleHash,
* to serialize key (and values, in {@code ChronicleMap} case) (unless they are direct {@link
* Byteable} instances). Serialized key bytes (+ serialized value bytes, in {@code ChronicleMap}
* case) + some metadata bytes comprise "entry space", which {@code ChronicleMap} or {@code
* ChronicleSet} should allocate. So <i>entry size</i> is a minimum allocation portion in the
* hash containers, created by this builder. E. g. if entry size is 100, the created container
* ChronicleSet} should allocate. So <i>chunk size</i> is the minimum allocation portion in the
* hash containers, created by this builder. E. g. if chunk size is 100, the created container
* could only allocate 100, 200, 300... bytes for an entry. If say 150 bytes of entry space are
* required by the entry, 200 bytes will be allocated, 150 used and 50 wasted. To minimize
* memory overuse and improve speed, you should pay decent attention to this configuration.
* required by the entry, 200 bytes will be allocated, 150 used and 50 wasted. This is called
* internal fragmentation.
*
* <p>There are three major patterns of this configuration usage: <ol> <li>Key (and value, in
* {@code ChronicleMap} case) sizes are constant. Configure them via {@link
* #constantKeySizeBySample(Object)} and {@link ChronicleMapBuilder#constantValueSizeBySample(Object)}
* methods, and you will experience no memory waste at all.</li> <li>Key (and/or value size, in
* {@code ChronicleMap} case) varies moderately. Specify them using corresponding methods, or
* specify entry size directly by calling this method, by sizes somewhere between average and
* maximum possible. The idea is to have most (90% or more) entries to fit a single "entry size"
* with moderate memory waste (10-20% on average), rest 10% or less of entries should take 2
* "entry sizes", thus with ~50% memory overuse.</li> <li>Key (and/or value size, in {@code
* ChronicleMap} case) varies in a wide range. Then it's best to use entry size configuration in
* <i>chunk mode</i>. Specify entry size so that most entries should take from 5 to several
* dozens of "chunks". With this approach, average memory waste should be very low.
* <p>To minimize memory overuse and improve speed, you should pay decent attention to this
* configuration. Alternatively, you can just trust the heuristics and doesn't configure
* the chunk size.
*
* <p>However, remember that <ul> <li>Operations with entries that span several "entry sizes"
* are a bit slower, than entries which take a single "entry size". That is why "chunk" approach
* is not recommended, when key (and/or value size) varies moderately.</li> <li>The maximum
* number of chunks could be taken by an entry is {@link #maxEntryOversizeFactor(int)}, of max
* 64 chunks. {@link IllegalArgumentException} is thrown on attempt to insert too large entry,
* compared to the configured or computed entry size.</li> <li>The number of "entries"
* {@linkplain #entries(long) in the whole map} and {@linkplain #actualEntriesPerSegment(long)
* per segment} is actually the number "chunks".</li> </ul>
* <p>Specify chunk size so that most entries would take from 5 to several dozens of chunks.
* However, remember that operations with entries that span several chunks are a bit slower,
* than with entries which take a single chunk. Particularly avoid entries to take more than
* 64 chunks.
*
* <p>Example: if values in your {@code ChronicleMap} are adjacency lists of some social graph,
* where nodes are represented as {@code long} ids, and adjacency lists are serialized in
* efficient manner, for example as {@code long[]} arrays. Typical number of connections is
* 100-300, maximum is 3000. In this case entry size of
* 50 * (8 bytes for each id) = 400 bytes would be a good choice: <pre>{@code
* 100-300, maximum is 3000. In this case chunk size of
* 30 * (8 bytes for each id) = 240 bytes would be a good choice: <pre>{@code
* Map<Long, long[]> socialGraph = ChronicleMapOnHeapUpdatableBuilder
* .of(Long.class, long[].class)
* // given that graph should have of 1 billion nodes, and 150 average adjacency list size
* // => values takes 3 chuncks on average
* .entries(1_000_000_000L * (150 / 50))
* .entrySize(50 * 8)
* .entries(1_000_000_000L)
* .averageValueSize(150 * 8) // 150 is average adjacency list size
* .actualChunkSize(30 * 8) // average 5-6 chunks per entry
* .create();}</pre>
* It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max chunks by
* single entry" limit, and ensures moderate average memory overuse (not more than 20%). </li>
* </ol>
*
* @param entrySize the "chunk size" in bytes
* <p>This is a <a href="#low-level-config">low-level configuration</a>. The configured number
* of bytes is strictly used as-is, without anything like round-up to the multiple of 8 or
* 16, or any other adjustment.
*
* @param actualChunkSize the "chunk size" in bytes
* @return this builder back
* @see #entries(long)
* @see #maxEntryOversizeFactor(int)
* @see #maxChunksPerEntry(int)
*/
B entrySize(int entrySize);
B actualChunkSize(int actualChunkSize);
/**
* Configures how much the actual entry size is allowed to be larger than configured or derived
* {@linkplain #entrySize(int) entry size}.
* Configures how many chunks a single entry, inserted into {@code ChronicleHash}es, created
* by this builder, could take. If you try to insert larger entry, {@link IllegalStateException}
* is fired. This is useful as self-check, that you configured chunk size right and you
* keys (and values, in {@link ChronicleMap} case) take expected number of bytes. For example,
* if {@link #constantKeySizeBySample(Object)} is configured or key size is statically known
* to be constant (boxed primitives, data value generated implementations, {@link Byteable}s,
* etc.), and the same for value objects in {@code ChronicleMap} case, max chunks per entry
* is configured to 1, to ensure keys and values are actually constantly-sized.
*
* @param maxEntryOversizeFactor number of times the actual entry size could oversize the
* configured "entry size" (chunk size, actually)
* @param maxChunksPerEntry how many chunks a single entry could span at most
* @return this builder back
* @throws IllegalArgumentException if the given {@code maxEntryOversizeFactor} is lesser than 1
* @throws IllegalArgumentException if the given {@code maxChunksPerEntry} is lesser than 1
* or greater than 64
* @see #actualChunkSize(int)
*/
B maxEntryOversizeFactor(int maxEntryOversizeFactor);
B maxChunksPerEntry(int maxChunksPerEntry);
/**
* Configures the maximum number of {@linkplain #entrySize(int) "entry size chunks"}, which
* could be taken by the maximum number of entries, inserted into the hash containers, created
* by this builder. If you try to insert more data, {@link IllegalStateException} might be
* thrown, because currently {@link ChronicleMap} and {@link ChronicleSet} doesn't support
* resizing.
* Configures the maximum number of entries, that could be inserted into the hash containers,
* created by this builder. If you try to insert more data, {@link IllegalStateException}
* <i>might</i> be thrown, because currently {@link ChronicleMap} and {@link ChronicleSet}
* don't support resizing.
*
* <ol> <li>If key size (and value size, in {@code ChronicleMap} case) is constant, this number
* is equal to the maximum number of entries (because each entry takes exactly one "entry size"
* memory unit).</li> <li>If key (and/or value, in {@code ChronicleMap} case) size varies
* moderately, you should pass to this method the maximum number of entries + 5-25%, depending
* on your data properties and configured {@linkplain #keySize(int) key}/{@linkplain
* ChronicleMapBuilder#valueSize(int)
* value}/{@linkplain #entrySize(int) entry} sizes.</li> <li>If your data size varies in a wide
* range, pass the maximum number of entries multiplied by average data size and divided by the
* configured "entry size" (i. e. chunk size). See an example in the documentation to {@link
* #entrySize(int)} method.</li> </ol>
*
* <p>You shouldn't put additional margin over the number, computed according the rules above.
* <p><b>You shouldn't put additional margin over the actual maximum number of entries.</b>
* This bad practice was popularized by {@link HashMap#HashMap(int)} and {@link
* HashSet#HashSet(int)} constructors, which accept "capacity", that should be multiplied by
* "load factor" to obtain actual maximum expected number of entries. {@code ChronicleMap} and
* {@code ChronicleSet} don't have a notion of load factor.
* HashSet#HashSet(int)} constructors, which accept <i>capacity</i>, that should be multiplied
* by <i>load factor</i> to obtain the actual maximum expected number of entries.
* {@code ChronicleMap} and {@code ChronicleSet} don't have a notion of load factor.
*
* <p>Default value is 2^20 (~ 1 million).
* <p>Default maximum entries is 2^20 (~ 1 million).
*
* @param entries maximum size of the created maps, in memory allocation units, so-called "entry
* size"
* @param entries maximum size of the maps or sets, created by this builder
* @return this builder back
* @see #entrySize(int)
*/
B entries(long entries);
B actualEntriesPerSegment(long actualEntriesPerSegment);
/**
* Configures the actual maximum number entries, that could be inserted into any single segment
* of the hash containers, created by this builder. Configuring both the actual number of
* entries per segment and {@linkplain #actualSegments(int) actual segments} replaces a single
* {@link #entries(long)} configuration.
*
* <p>This is a <a href="#low-level-config">low-level configuration</a>.
*
* @param entriesPerSegment the actual maximum number entries per segment in the
* hash containers, created by this builder
* @return this builder back
* @see #entries(long)
* @see #actualSegments(int)
*/
B entriesPerSegment(long entriesPerSegment);
/**
* Configures the actual number of chunks, that will be reserved for any single segment of the
* hash containers, created by this builder. This configuration is a lower-level version of
* {@link #entriesPerSegment(long)}. Makes sense only if {@link #actualChunkSize(int)},
* {@link #actualSegments(int)} and {@link #entriesPerSegment(long)} are also configured
* manually.
*
* @param actualChunksPerSegment the actual number of segments, reserved per segment in the
* hash containers, created by this builder
* @return this builder back
*/
B actualChunksPerSegment(long actualChunksPerSegment);
/**
* Configures the actual number of segments in the hash containers, created by this builder.
* With {@linkplain #entriesPerSegment(long) actual number of segments}, this
* configuration replaces a single {@link #entries(long)} call.
*
* <p>This is a <a href="#low-level-config">low-level configuration</a>. The configured number
* is used as-is, without anything like round-up to the closest power of 2.
*
* @param actualSegments the actual number of segments in hash containers, created by
* this builder
* @return this builder back
* @see #minSegments(int)
* @see #entriesPerSegment(long)
*/
B actualSegments(int actualSegments);
/**
src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java
+4
View File
@@ -26,6 +26,10 @@ public interface SizeMarshaller extends Serializable {
int sizeEncodingSize(long size);
int minSizeEncodingSize();
int maxSizeEncodingSize();
void writeSize(Bytes bytes, long size);
long readSize(Bytes bytes);
src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshallers.java
+20
View File
@@ -36,6 +36,16 @@ public final class SizeMarshallers {
return stopBitLength(size);
}
@Override
public int minSizeEncodingSize() {
return 1;
}
@Override
public int maxSizeEncodingSize() {
return 9;
}
@Override
public void writeSize(Bytes bytes, long size) {
bytes.writeStopBit(size);
@@ -65,6 +75,16 @@ public final class SizeMarshallers {
return 0;
}
@Override
public int minSizeEncodingSize() {
return 0;
}
@Override
public int maxSizeEncodingSize() {
return 0;
}
@Override
public void writeSize(Bytes bytes, long size) {
// do nothing
src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java
+10
View File
@@ -69,6 +69,16 @@ public abstract class ByteableMarshaller<E extends Byteable>
return 0;
}
@Override
public int minSizeEncodingSize() {
return 0;
}
@Override
public int maxSizeEncodingSize() {
return 0;
}
@Override
public void writeSize(Bytes bytes, long size) {
// do nothing
src/main/java/net/openhft/chronicle/hash/serialization/internal/DoubleMarshaller.java
+10
View File
@@ -39,6 +39,16 @@ public enum DoubleMarshaller implements BytesInterop<Double>, BytesReader<Double
return 0;
}
@Override
public int minSizeEncodingSize() {
return 0;
}
@Override
public int maxSizeEncodingSize() {
return 0;
}
@Override
public void writeSize(Bytes bytes, long size) {
// do nothing
src/main/java/net/openhft/chronicle/hash/serialization/internal/IntegerMarshaller.java
+10
View File
@@ -36,6 +36,16 @@ public enum IntegerMarshaller
return 0;
}
@Override
public int minSizeEncodingSize() {
return 0;
}
@Override
public int maxSizeEncodingSize() {
return 0;
}
@Override
public void writeSize(Bytes bytes, long size) {
// do nothing
src/main/java/net/openhft/chronicle/hash/serialization/internal/LongMarshaller.java
+10
View File
@@ -38,6 +38,16 @@ public enum LongMarshaller
return 0;
}
@Override
public int minSizeEncodingSize() {
return 0;
}
@Override
public int maxSizeEncodingSize() {
return 0;
}
@Override
public void writeSize(Bytes bytes, long size) {
// do nothing
src/main/java/net/openhft/chronicle/hash/serialization/internal/VoidMarshaller.java
+10
View File
@@ -38,6 +38,16 @@ public enum VoidMarshaller
return 0;
}
@Override
public int minSizeEncodingSize() {
return 0;
}
@Override
public int maxSizeEncodingSize() {
return 0;
}
@Override
public void writeSize(Bytes bytes, long size) {
// do nothing
src/main/java/net/openhft/chronicle/map/Alignment.java
+17
View File
@@ -44,6 +44,11 @@ public enum Alignment {
int alignSize(int size) {
return size;
}
@Override
int alignment() {
return 0;
}
},
/**
@@ -64,6 +69,11 @@ public enum Alignment {
int alignSize(int size) {
return (size + 3) & ~3;
}
@Override
int alignment() {
return 4;
}
},
/**
@@ -84,6 +94,11 @@ public enum Alignment {
int alignSize(int size) {
return (size + 7) & ~7;
}
@Override
int alignment() {
return 8;
}
};
private static final Alignment[] VALUES = values();
@@ -97,4 +112,6 @@ public enum Alignment {
abstract long alignAddr(long addr);
abstract int alignSize(int size);
abstract int alignment();
}
src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java
+399 -215
View File
@@ -43,15 +43,12 @@ import org.slf4j.LoggerFactory;
import java.io.*;
import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import static java.lang.Math.round;
import static net.openhft.chronicle.map.Objects.builderEquals;
import static net.openhft.chronicle.map.VanillaChronicleMap.MAX_ENTRY_OVERSIZE_FACTOR;
import static net.openhft.lang.model.DataValueGenerator.firstPrimitiveFieldType;
/**
@@ -67,21 +64,29 @@ import static net.openhft.lang.model.DataValueGenerator.firstPrimitiveFieldType;
* ChronicleMapBuilder<Key, Value> builder = ChronicleMapBuilder
* .of(Key.class, Value.class)
* .entries(100500);
* <p>
*
* ChronicleMap<Key, Value> map1 = builder.create();
* ChronicleMap<Key, Value> map2 = builder.create();}</pre>
* i. e. created {@code ChronicleMap} instances don't depend on the builder. <p> <p>{@code
* ChronicleMapBuilder} is mutable, see a note in {@link ChronicleHashBuilder} interface
* documentation. <p> <p>Later in this documentation, "ChronicleMap" means "ChronicleMaps, created
* by {@code ChronicleMapBuilder}", unless specified different, because theoretically someone might
* provide {@code ChronicleMap} implementations with completely different properties. <p> <p>{@code
* ChronicleMap} ("ChronicleMaps, created by {@code ChronicleMapBuilder}") currently doesn't support
* resizing. That is why you should <i>always</i> configure {@linkplain #entries(long) number of
* entries} you are going to insert into the created map <i>at most</i>. See {@link #entries(long)}
* method documentation for more information on this. <p> <p>{@code ChronicleMap} allocates memory
* by equally sized chunks. This size is called {@linkplain #entrySize(int) entry size}, you are
* strongly recommended to configure it to achieve least memory consumption and best speed. See
* {@link #entrySize(int)} method documentation for more information on this.
* i. e. created {@code ChronicleMap} instances don't depend on the builder.
*
* <p>{@code ChronicleMapBuilder} is mutable, see a note in {@link ChronicleHashBuilder} interface
* documentation.
*
* <p>Later in this documentation, "ChronicleMap" means "ChronicleMaps, created by {@code
* ChronicleMapBuilder}", unless specified different, because theoretically someone might provide
* {@code ChronicleMap} implementations with completely different properties.
*
* <p>{@code ChronicleMap} ("ChronicleMaps, created by {@code ChronicleMapBuilder}") currently
* doesn't support resizing. That is why you <i>must</i> configure {@linkplain
* #entries(long) number of entries} you are going to insert into the created map <i>at most</i>.
* See {@link #entries(long)} method documentation for more information on this.
*
* <p>If you key or value type is not constantly sized and known to {@code ChronicleHashBuilder},
* i. e. it is not a boxed primitive, data value generated interface, {@link Byteable}, etc. (see
* the complete list TODO insert the link to the complete list),
* you <i>must</i> provide the {@code ChronicleHashBuilder} with some information about you keys
* or values: if they are constantly-sized, call {@link #constantKeySizeBySample(Object)}, otherwise
* {@link ChronicleHashBuilder#averageKeySize(double)} method, accordingly for values.
*
* @param <K> key type of the maps, produced by this builder
* @param <V> value type of the maps, produced by this builder
@@ -93,6 +98,7 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
static final byte UDP_REPLICATION_MODIFICATION_ITERATOR_ID = (byte) 128;
private static final int DEFAULT_KEY_OR_VALUE_SIZE = 120;
private static final long DEFAULT_ENTRIES = 1 << 20;
private static final int MAX_SEGMENTS = (1 << 30);
private static final int MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS = (1 << 20);
private static final Logger LOG =
@@ -101,6 +107,15 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
private static final StringBuilder EMTRY_STRING_BUILDER = new StringBuilder();
private static final double UNDEFINED_DOUBLE_CONFIG = Double.NaN;
private static boolean isDefined(double config) {
return !Double.isNaN(config);
}
private static final boolean strictStateChecks =
Boolean.getBoolean("chronicle.strictStateChecks");
SerializationBuilder<K> keyBuilder;
SerializationBuilder<V> valueBuilder;
@@ -110,15 +125,16 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
private int minSegments = -1;
private int actualSegments = -1;
// used when reading the number of entries per
private long actualEntriesPerSegment = -1L;
private int keySize = 0;
private long entriesPerSegment = -1L;
private long actualChunksPerSegment = -1L;
private double averageKeySize = UNDEFINED_DOUBLE_CONFIG;
private K sampleKey;
private int valueSize = 0;
private double averageValueSize = UNDEFINED_DOUBLE_CONFIG;
private V sampleValue;
private int entrySize = 0;
private int maxEntryOversizeFactor = -1;
private int actualChunkSize = 0;
private int maxChunksPerEntry = -1;
private Alignment alignment = null;
private long entries = 1 << 20; // 1 million by default
private long entries = -1;
private long lockTimeOut = 20000;
private TimeUnit lockTimeOutUnit = TimeUnit.MILLISECONDS;
private int metaDataBytes = 0;
@@ -214,36 +230,37 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
}
/**
* {@inheritDoc} <p> <p>Example: if keys in your map(s) are English words in {@link String}
* form, keys size 10
* (a bit more than average English word length) would be a good choice: <pre>{@code
* {@inheritDoc}
*
* <p>Example: if keys in your map(s) are English words in {@link String} form, average English
* word length is 5.1, configure average key size of 6: <pre>{@code
* ChronicleMap<String, LongValue> wordFrequencies = ChronicleMapBuilder
* .of(String.class, LongValue.class)
* .entries(50000)
* .keySize(10)
* // shouldn't specify valueSize(), because it is statically known
* .averageKeySize(6)
* .create();}</pre>
* (Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded (and
* each character takes 2 bytes on-heap), because default off-heap {@link String} encoding is
* UTF-8 in {@code ChronicleMap}.)
* (Note that 6 is chosen as average key size in bytes despite strings in Java are UTF-16
* encoded (and each character takes 2 bytes on-heap), because default off-heap {@link String}
* encoding is UTF-8 in {@code ChronicleMap}.)
*
* @throws IllegalStateException {@inheritDoc}
* @throws IllegalArgumentException {@inheritDoc}
* @see #constantKeySizeBySample(Object)
* @see #valueSize(int)
* @see #entrySize(int)
* @see #averageValueSize(double)
* @see #actualChunkSize(int)
* @param averageKeySize
*/
@Override
public ChronicleMapBuilder<K, V> keySize(int keySize) {
public ChronicleMapBuilder<K, V> averageKeySize(double averageKeySize) {
checkSizeIsNotStaticallyKnown(keyBuilder);
if (keySize <= 0)
throw new IllegalArgumentException("Key size must be positive");
this.keySize = keySize;
checkAverageSize(averageKeySize, "key");
this.averageKeySize = averageKeySize;
return this;
}
/**
* {@inheritDoc} <p>
* {@inheritDoc}
*
* <p>For example, if your keys are Git commit hashes:<pre>{@code
* Map<byte[], String> gitCommitMessagesByHash =
* ChronicleMapBuilder.of(byte[].class, String.class)
@@ -251,7 +268,7 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
* .immutableKeys()
* .create();}</pre>
*
* @see #keySize(int)
* @see ChronicleHashBuilder#averageKeySize(double)
* @see #constantValueSizeBySample(Object)
*/
@Override
@@ -260,37 +277,49 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
return this;
}
private int keySize() {
return keyOrValueSize(keySize, keyBuilder);
private double averageKeySize() {
return averageKeyOrValueSize(averageKeySize, keyBuilder);
}
/**
* Configures the optimal number of bytes, taken by serialized form of values, put into maps,
* Configures the average number of bytes, taken by serialized form of values, put into maps,
* created by this builder. If value size is always the same, call {@link
* #constantValueSizeBySample(Object)} method instead of this one. <p> <p>If values are of boxed
* primitive type or {@link Byteable} subclass, i. e. if value size is known statically, it is
* automatically accounted and shouldn't be specified by user. <p> <p>If value size varies
* moderately, specify the size higher than average, but lower than the maximum possible, to
* minimize average memory overuse. If value size varies in a wide range, it's better to use
* {@linkplain #entrySize(int) entry size} in "chunk" mode and configure it directly.
* #constantValueSizeBySample(Object)} method instead of this one.
*
* @param valueSize number of bytes, taken by serialized form of values
* <p>{@code ChronicleHashBuilder} implementation heuristically chooses
* {@linkplain #actualChunkSize(int) the actual chunk size} based on this configuration and
* the key size, that, however, might result to quite high internal fragmentation,
* i. e. losses because only integral number of chunks could be allocated for the entry. If you
* want to avoid this, you should manually configure the actual chunk size in addition to this
* average value size configuration, which is anyway needed.
*
* <p>If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is
* known statically, it is automatically accounted and shouldn't be specified by user.
*
* @param averageValueSize number of bytes, taken by serialized form of values
* @return this builder back
* @throws IllegalStateException if value size is known statically and shouldn't be
* configured by user
* @throws IllegalArgumentException if the given {@code valueSize} is non-positive
* @throws IllegalArgumentException if the given {@code averageValueSize} is non-positive
* @see #constantValueSizeBySample(Object)
* @see #keySize(int)
* @see #entrySize(int)
* @see ChronicleHashBuilder#averageKeySize(double)
* @see #actualChunkSize(int)
*/
public ChronicleMapBuilder<K, V> valueSize(int valueSize) {
public ChronicleMapBuilder<K, V> averageValueSize(double averageValueSize) {
checkSizeIsNotStaticallyKnown(valueBuilder);
if (valueSize <= 0)
throw new IllegalArgumentException("Value size must be positive");
this.valueSize = valueSize;
checkAverageSize(averageValueSize, "value");
this.averageValueSize = averageValueSize;
return this;
}
private static void checkAverageSize(double averageSize, String role) {
if (averageSize <= 0 || Double.isNaN(averageSize) ||
Double.isInfinite(averageSize)) {
throw new IllegalArgumentException("Average " + role + " size must be a positive, " +
"finite number");
}
}
private static void checkSizeIsNotStaticallyKnown(SerializationBuilder builder) {
if (builder.sizeIsStaticallyKnown)
throw new IllegalStateException("Size of type " + builder.eClass +
@@ -300,15 +329,17 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
/**
* Configures the constant number of bytes, taken by serialized form of values, put into maps,
* created by this builder. This is done by providing the {@code sampleValue}, all values should
* take the same number of bytes in serialized form, as this sample object. <p> <p>If values are
* of boxed primitive type or {@link Byteable} subclass, i. e. if value size is known
* statically, it is automatically accounted and this method shouldn't be called. <p> <p>If
* value size varies, method {@link #valueSize(int)} or {@link #entrySize(int)} should be called
* instead of this one.
* take the same number of bytes in serialized form, as this sample object.
*
* <p>If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is
* known statically, it is automatically accounted and this method shouldn't be called.
*
* <p>If value size varies, method {@link #averageValueSize(double)} should be called instead of
* this one.
*
* @param sampleValue the sample value
* @return this builder back
* @see #valueSize(int)
* @see #averageValueSize(double)
* @see #constantKeySizeBySample(Object)
*/
public ChronicleMapBuilder<K, V> constantValueSizeBySample(V sampleValue) {
@@ -316,12 +347,12 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
return this;
}
int valueSize() {
return keyOrValueSize(valueSize, valueBuilder);
double averageValueSize() {
return averageKeyOrValueSize(averageValueSize, valueBuilder);
}
private int keyOrValueSize(int configuredSize, SerializationBuilder builder) {
if (configuredSize > 0)
private double averageKeyOrValueSize(double configuredSize, SerializationBuilder builder) {
if (isDefined(configuredSize))
return configuredSize;
if (builder.constantSizeMarshaller())
return builder.pseudoReadConstantSize();
@@ -329,110 +360,195 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
}
/**
* {@inheritDoc} <p> <p>In fully default case you can expect entry size to be about 256 bytes.
* But it is strongly recommended always to configure {@linkplain #keySize(int) key size} and
* {@linkplain #valueSize(int) value size}, if they couldn't be derived statically. <p> <p>If
* entry size is not configured explicitly by calling this method, it is computed based on
* {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size},
* plus {@linkplain #valueSize(int) value size}, plus a few bytes required by implementations,
* with respect to {@linkplain #entryAndValueAlignment(Alignment) alignment}. <p> <p>Note that
* the actual entrySize will be aligned by the configured {@linkplain
* #entryAndValueAlignment(Alignment) entry and value alignment}. I. e. if you set entry size to
* 30, and entry alignment is set to {@link Alignment#OF_4_BYTES}, the actual entry size will be
* 32 (30 aligned to 4 bytes).
* {@inheritDoc}
*
* @throws IllegalStateException is sizes of both keys and values of maps created by this
* builder are statically known, hence entry size shouldn't be
* builder are constant, hence chunk size shouldn't be
* configured by user
* @see #entryAndValueAlignment(Alignment)
* @see #entries(long)
* @see #maxEntryOversizeFactor(int)
* @see #maxChunksPerEntry(int)
*/
@Override
public ChronicleMapBuilder<K, V> entrySize(int entrySize) {
if (keyBuilder.sizeIsStaticallyKnown && valueBuilder.sizeIsStaticallyKnown) {
public ChronicleMapBuilder<K, V> actualChunkSize(int actualChunkSize) {
if (constantlySizedEntries()) {
throw new IllegalStateException("Sizes of key type: " + keyBuilder.eClass + " and " +
"value type: " + valueBuilder.eClass + " are both statically known, " +
"so entry size shouldn't be specified manually");
"value type: " + valueBuilder.eClass + " are both constant, " +
"so chunk size shouldn't be specified manually");
}
if (entrySize <= 0)
throw new IllegalArgumentException("Entry Size must be positive");
this.entrySize = entrySize;
if (actualChunkSize <= 0)
throw new IllegalArgumentException("Chunk size must be positive");
this.actualChunkSize = actualChunkSize;
return this;
}
int entrySize(boolean replicated) {
if (entrySize > 0)
return entryAndValueAlignment().alignSize(entrySize);
int size = metaDataBytes;
int keySize = keySize();
size += keyBuilder.sizeMarshaller().sizeEncodingSize(keySize);
double averageEntrySize(boolean replicated) {
double size = metaDataBytes;
double keySize = averageKeySize();
size += averageSizeEncodingSize(keyBuilder, keySize);
size += keySize;
if (replicated)
size += ReplicatedChronicleMap.ADDITIONAL_ENTRY_BYTES;
int valueSize = valueSize();
size += valueBuilder.sizeMarshaller().sizeEncodingSize(valueSize);
size = entryAndValueAlignment().alignSize(size);
size += valueSize;
// Some cache line heuristics
for (int i = 1; i <= 4; i++) {
int bound = i * 64;
// Not more than 5% oversize.
// DEFAULT_KEY_OR_VALUE_SIZE and this heuristic are specially adjusted to produce
// entry size 256 -- the default value prior to keySize and valueSize -- when key and
// value sizes are both not static nor configured, in both vanilla and replicated modes.
if (size < bound && (bound - size) <= bound / 20) {
size = bound;
break;
}
}
return entryAndValueAlignment().alignSize(size);
}
@Override
public ChronicleMapBuilder<K, V> maxEntryOversizeFactor(int maxEntryOversizeFactor) {
if (maxEntryOversizeFactor < 1)
throw new IllegalArgumentException("maxEntryOversizeFactor should be more than 1, " +
maxEntryOversizeFactor + " given");
this.maxEntryOversizeFactor = maxEntryOversizeFactor;
return this;
}
private void checkMaxEntryOversizeFactorIfReplicated(boolean replicated) {
if (replicated && maxEntryOversizeFactor > MAX_ENTRY_OVERSIZE_FACTOR)
throw new IllegalStateException("As of now Replicated Chornicle Map doesn't support " +
"more than " + MAX_ENTRY_OVERSIZE_FACTOR + " entry oversize factor, " +
maxEntryOversizeFactor + " is configured");
}
int maxEntryOversizeFactor(boolean replicated) {
if (maxEntryOversizeFactor < 0) {
if (keyBuilder.constantSizeMarshaller() && valueBuilder.constantSizeMarshaller()) {
return 1;
} else {
if (replicated) {
return MAX_ENTRY_OVERSIZE_FACTOR;
double valueSize = averageValueSize();
size += averageSizeEncodingSize(valueBuilder, valueSize);
Alignment alignment = valueAlignment();
if (alignment != Alignment.NO_ALIGNMENT) {
if (constantlySizedKeys() && valueBuilder.constantSizeEncodingSizeMarshaller()) {
long constantSizeBeforeAlignment = round(size);
if (constantlySizedValues()) {
// see specialEntrySpaceOffset()
long totalDataSize = constantSizeBeforeAlignment + constantValueSize();
size += alignment.alignAddr(totalDataSize) - totalDataSize;
} else {
return Integer.MAX_VALUE;
if (actualChunkSize > 0) {
size += averageAlignmentAssumingChunkSize(constantSizeBeforeAlignment,
actualChunkSize);
} else {
for (int chunkSize : new int[]{8, 4}) {
double expectedAverageAlignment = averageAlignmentAssumingChunkSize(
constantSizeBeforeAlignment, chunkSize);
if (size + expectedAverageAlignment + valueSize >=
maxDefaultChunksPerAverageEntry(replicated) * chunkSize ||
chunkSize == 4) {
size += expectedAverageAlignment;
break;
}
}
}
}
} else {
if (alignment.alignment() > 0) {
// assume worst case, we always lose most possible bytes for alignment
size += alignment.alignment() - 1;
}
}
}
return maxEntryOversizeFactor;
size += valueSize;
return size;
}
/**
* This is needed, if chunkSize = constant entry size is not aligned, for entry alignment to
* be always the same, we should _misalign_ the first chunk.
*/
int specialEntrySpaceOffset(boolean replicated) {
if (!constantlySizedEntries() || valueAlignment() == Alignment.NO_ALIGNMENT)
return 0;
return (int) (constantValueSize() % valueAlignment().alignment());
}
private long constantValueSize() {
return valueBuilder.pseudoReadConstantSize();
}
private boolean constantlySizedKeys() {
return keyBuilder.constantSizeMarshaller();
}
private static double averageSizeEncodingSize(
SerializationBuilder builder, double averageSize) {
SizeMarshaller sizeMarshaller = builder.sizeMarshaller();
if (averageSize == round(averageSize))
return sizeMarshaller.sizeEncodingSize(round(averageSize));
long lower = (long) averageSize;
long upper = lower + 1;
int lowerEncodingSize = sizeMarshaller.sizeEncodingSize(lower);
int upperEncodingSize = sizeMarshaller.sizeEncodingSize(upper);
if (lowerEncodingSize == upperEncodingSize)
return lowerEncodingSize;
return lower * (upper - averageSize) + upper * (averageSize - lower);
}
private double averageAlignmentAssumingChunkSize(
long constantSizeBeforeAlignment, int chunkSize) {
if (valueAlignment() == Alignment.NO_ALIGNMENT)
return 0.0;
Alignment valueAlignment = valueAlignment();
long firstAlignment = valueAlignment.alignAddr(constantSizeBeforeAlignment) -
constantSizeBeforeAlignment;
int alignment = valueAlignment.alignment();
int gcdOfAlignmentAndChunkSize = greatestCommonDivisor(alignment, chunkSize);
if (gcdOfAlignmentAndChunkSize == alignment)
return firstAlignment;
// assume worst by now because we cannot predict alignment in VanillaCM.entrySize() method
// before allocation
return alignment - gcdOfAlignmentAndChunkSize;
}
static int greatestCommonDivisor(int a, int b) {
if (b == 0) return a;
return greatestCommonDivisor(b, a % b);
}
long chunkSize(boolean replicated) {
if (actualChunkSize > 0)
return actualChunkSize;
double averageEntrySize = averageEntrySize(replicated);
if (constantlySizedEntries())
return round(averageEntrySize);
int maxChunkSize = 1 << 30;
for (long chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) {
if (maxDefaultChunksPerAverageEntry(replicated) * chunkSize > averageEntrySize)
return chunkSize;
}
return maxChunkSize;
}
boolean constantlySizedEntries() {
return constantlySizedKeys() && constantlySizedValues();
}
double averageChunksPerEntry(boolean replicated) {
if (constantlySizedEntries())
return 1.0;
long chunkSize = chunkSize(replicated);
// assuming we always has worst internal fragmentation. This affects total segment
// entry space which is allocated lazily on Linux (main target platform)
// so we can afford this
return (averageEntrySize(replicated) + chunkSize - 1) / chunkSize;
}
private static int maxDefaultChunksPerAverageEntry(boolean replicated) {
return replicated ? 4 : 8;
}
@Override
public ChronicleMapBuilder<K, V> maxChunksPerEntry(int maxChunksPerEntry) {
if (maxChunksPerEntry < 1)
throw new IllegalArgumentException("maxChunksPerEntry should be >= 1, " +
maxChunksPerEntry + " given");
this.maxChunksPerEntry = maxChunksPerEntry;
return this;
}
int maxChunksPerEntry() {
if (maxChunksPerEntry > 0)
return maxChunksPerEntry;
if (constantlySizedEntries()) {
return 1;
} else {
return Integer.MAX_VALUE;
}
}
private boolean constantlySizedValues() {
return valueBuilder.constantSizeMarshaller();
}
/**
* Configures alignment strategy of address in memory of entries and independently of address in
* memory of values within entries in ChronicleMaps, created by this builder. <p> <p>Useful when
* values of the map are updated intensively, particularly fields with volatile access, because
* it doesn't work well if the value crosses cache lines. Also, on some (nowadays rare)
* architectures any misaligned memory access is more expensive than aligned. <p> <p>Note that
* {@linkplain #entrySize(int) entry size} will be aligned according to this alignment. I. e. if
* you set {@code entrySize(20)} and {@link Alignment#OF_8_BYTES}, actual entry size will be 24
* (20 aligned to 8 bytes). <p> <p>If values couldn't reference off-heap memory (i. e. it is not
* {@link Byteable} or "data value generated"), alignment configuration makes no sense and
* forbidden. <p> <p>Default is {@link Alignment#NO_ALIGNMENT} if values couldn't reference
* off-heap memory, otherwise chosen heuristically (configure explicitly for being sure and to
* compare performance in your case).
* memory of values within entries in ChronicleMaps, created by this builder.
*
* <p>Useful when values of the map are updated intensively, particularly fields with volatile
* access, because it doesn't work well if the value crosses cache lines. Also, on some
* (nowadays rare) architectures any misaligned memory access is more expensive than aligned.
*
* <p>If values couldn't reference off-heap memory (i. e. it is not {@link Byteable} or "data
* value generated"), alignment configuration makes no sense and forbidden.
*
* <p>Default is {@link Alignment#NO_ALIGNMENT} if values couldn't reference off-heap memory,
* otherwise chosen heuristically (configure explicitly for being sure and to compare
* performance in your case).
*
* @param alignment the new alignment of the maps constructed by this builder
* @return this {@code ChronicleMapOnHeapUpdatableBuilder} back
@@ -452,7 +568,7 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
"if values might point to off-heap memory");
}
Alignment entryAndValueAlignment() {
Alignment valueAlignment() {
if (alignment != null)
return alignment;
Class firstPrimitiveFieldType = firstPrimitiveFieldType(valueBuilder.eClass);
@@ -463,11 +579,6 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
return Alignment.NO_ALIGNMENT;
}
/**
* Package-private for tests
*/
@Override
public ChronicleMapBuilder<K, V> entries(long entries) {
if (entries <= 0L)
@@ -477,43 +588,80 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
}
long entries() {
if (entries < 0)
return DEFAULT_ENTRIES;
return entries;
}
@Override
public ChronicleMapBuilder<K, V> actualEntriesPerSegment(long actualEntriesPerSegment) {
if (actualEntriesPerSegment <= 0L)
throw new IllegalArgumentException("entries per segment should be positive, " +
actualEntriesPerSegment + " given");
if (tooManyEntriesPerSegment(actualEntriesPerSegment))
throw new IllegalArgumentException("max entries per segment is " +
MultiMapFactory.MAX_CAPACITY + ", " + actualEntriesPerSegment + " given");
this.actualEntriesPerSegment = actualEntriesPerSegment;
public ChronicleMapBuilder<K, V> entriesPerSegment(long entriesPerSegment) {
if (entriesPerSegment <= 0L)
throw new IllegalArgumentException("Entries per segment should be positive, " +
entriesPerSegment + " given");
this.entriesPerSegment = entriesPerSegment;
return this;
}
private boolean tooManyEntriesPerSegment(long entriesPerSegment) {
return entriesPerSegment > MultiMapFactory.MAX_CAPACITY;
long entriesPerSegment(boolean replicated) {
long entriesPerSegment;
if (this.entriesPerSegment > 0L) {
entriesPerSegment = this.entriesPerSegment;
} else {
int actualSegments = actualSegments(replicated);
long totalEntries = totalEntriesIfPoorDistribution(actualSegments);
entriesPerSegment = divideUpper(totalEntries, actualSegments);
}
if (actualChunksPerSegment > 0)
return entriesPerSegment;
double averageChunksPerEntry = averageChunksPerEntry(replicated);
if (entriesPerSegment * averageChunksPerEntry > MultiMapFactory.MAX_CAPACITY)
throw new IllegalStateException("Max chunks per segment is " +
MultiMapFactory.MAX_CAPACITY + " configured entries() and " +
"actualSegments() so that there should be " + entriesPerSegment +
" entries per segment, while average chunks per entry is " +
averageChunksPerEntry);
return entriesPerSegment;
}
long actualEntriesPerSegment() {
if (actualEntriesPerSegment > 0L)
return actualEntriesPerSegment;
int actualSegments = actualSegments();
long actualEntries = totalEntriesIfPoorDistribution(actualSegments);
long actualEntriesPerSegment = divideUpper(actualEntries, actualSegments);
if (tooManyEntriesPerSegment(actualEntriesPerSegment))
throw new IllegalStateException("max entries per segment is " +
MultiMapFactory.MAX_CAPACITY + " configured entries() and " +
"actualSegments() so that there should be " + actualEntriesPerSegment +
" entries per segment");
return actualEntriesPerSegment;
@Override
public ChronicleMapBuilder<K, V> actualChunksPerSegment(long actualChunksPerSegment) {
if (actualChunksPerSegment <= 0)
throw new IllegalArgumentException("Actual chunks per segment should be positive, " +
actualChunksPerSegment + " given");
this.actualChunksPerSegment = actualChunksPerSegment;
return this;
}
private void checkActualChunksPerSegmentIsConfiguredOnlyIfOtherLowLevelConfigsAreManual() {
if (actualChunksPerSegment > 0) {
if (entriesPerSegment <= 0 || (actualChunkSize <= 0 && !constantlySizedEntries()) ||
actualSegments <= 0)
throw new IllegalStateException("Actual chunks per entry could be configured " +
"only if other three low level configs are manual: " +
"entriesPerSegment(), actualSegments() and actualChunkSize(), unless " +
"both keys and value sizes are constant");
}
}
private void checkActualChunksPerSegmentGreaterOrEqualToEntries() {
if (actualChunksPerSegment > 0 && entriesPerSegment > 0 &&
entriesPerSegment > actualChunksPerSegment) {
throw new IllegalStateException("Entries per segment couldn't be greater than " +
"actual chunks per segment. Entries: " + entriesPerSegment + ", " +
"chunks: " + actualChunksPerSegment + " is configured");
}
}
long actualChunksPerSegment(boolean replicated) {
if (actualChunksPerSegment > 0)
return actualChunksPerSegment;
return round(entriesPerSegment(replicated) * averageChunksPerEntry(replicated));
}
private long totalEntriesIfPoorDistribution(int segments) {
if (segments == 1)
return entries;
long entries = this.entries;
return entries();
long entries = entries();
// check if the number of entries is small compared with the number of segments.
long s3 = (long) Math.min(8, segments) * Math.min(32, segments) * Math.min(128, segments);
if (entries * 4 <= s3)
@@ -528,7 +676,7 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
entries *= 1.15;
else
entries *= 1.1;
return Math.min(segments * this.entries,
return Math.min(segments * entries(),
entries + 4 * segments + 8);
}
@@ -544,19 +692,24 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
}
private int estimateSegments() {
return (int) Math.min(Maths.nextPower2(entries / 32, 1), estimateSegmentsBasedOnSize());
return (int) Math.min(Maths.nextPower2(entries() / 32, 1), estimateSegmentsBasedOnSize());
}
//TODO reivew because this heuristic doesn't seem to perform well
private int estimateSegmentsBasedOnSize() {
// based on entries with multimap of 100 bytes.
int segmentsForEntries = estimateSegmentsForEntries(entries);
return entrySize >= 1000000
// the idea is that if values are huge, operations on them (and simply ser/deser)
// could take long time, so we want more segment to minimize probablity that
// two or more concurrent write ops will go to the same segment, and then all but one of
// these threads will wait for long time.
int segmentsForEntries = estimateSegmentsForEntries(entries());
double averageValueSize = averageValueSize();
return averageValueSize >= 1000000
? segmentsForEntries * 16
: entrySize >= 100000
: averageValueSize >= 100000
? segmentsForEntries * 8
: entrySize >= 10000
: averageValueSize >= 10000
? segmentsForEntries * 4
: entrySize >= 1000
: averageValueSize >= 1000
? segmentsForEntries * 2
: segmentsForEntries;
}
@@ -584,36 +737,51 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
return this;
}
int actualSegments() {
int actualSegments(boolean replicated) {
if (actualSegments > 0)
return actualSegments;
if (entriesPerSegment > 0) {
long segments = 1;
for (int i = 0; i < 3; i++) { // try to converge
long totalEntries = totalEntriesIfPoorDistribution((int) segments);
segments = divideUpper(totalEntries, entriesPerSegment);
if (segments > MAX_SEGMENTS)
throw new IllegalStateException();
}
if (minSegments > 0)
segments = Math.max(minSegments, segments);
return (int) segments;
}
long shortMMapSegments = trySegments(MultiMapFactory.I16_MAX_CAPACITY,
MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS);
MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS, replicated);
// TODO why not try I24 multiMap?
if (shortMMapSegments > 0L)
return (int) shortMMapSegments;
long intMMapSegments = trySegments(MultiMapFactory.MAX_CAPACITY, MAX_SEGMENTS);
long intMMapSegments = trySegments(MultiMapFactory.MAX_CAPACITY, MAX_SEGMENTS, replicated);
if (intMMapSegments > 0L)
return (int) intMMapSegments;
throw new IllegalStateException("Max segments is " + MAX_SEGMENTS + ", configured so much" +
" entries() that builder automatically decided to use " +
" entries (" + entries() + ") or average chunks per entry is too high (" +
averageChunksPerEntry(replicated) + ") that builder automatically decided to use " +
(-intMMapSegments) + " segments");
}
private long trySegments(long maxSegmentCapacity, int maxSegments) {
long segments = divideUpper(totalEntriesIfPoorDistribution(minSegments()),
maxSegmentCapacity);
private long trySegments(long maxSegmentCapacity, int maxSegments, boolean replicated) {
long totalChunks = round(totalEntriesIfPoorDistribution(minSegments()) *
averageChunksPerEntry(replicated));
long segments = divideUpper(totalChunks, maxSegmentCapacity);
segments = Maths.nextPower2(Math.max(segments, minSegments()), 1L);
return segments <= maxSegments ? segments : -segments;
}
int segmentHeaderSize() {
int segments = actualSegments();
int segmentHeaderSize(boolean replicated) {
int segments = actualSegments(replicated);
// reduce false sharing unless we have a lot of segments.
return segments <= 16 * 1024 ? 64 : 32;
}
MultiMapFactory multiMapFactory() {
return MultiMapFactory.forCapacity(actualEntriesPerSegment());
MultiMapFactory multiMapFactory(boolean replicated) {
return MultiMapFactory.forCapacity(actualChunksPerSegment(replicated));
}
public ChronicleMapBuilder<K, V> lockTimeOut(long lockTimeOut, TimeUnit unit) {
@@ -703,13 +871,14 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
"name=" + name +
", actualSegments=" + pretty(actualSegments) +
", minSegments=" + pretty(minSegments) +
", actualEntriesPerSegment=" + pretty(actualEntriesPerSegment) +
", keySize=" + pretty(keySize) +
", entriesPerSegment=" + pretty(entriesPerSegment) +
", actualChunksPerSegment=" + pretty(actualChunksPerSegment) +
", averageKeySize=" + pretty(averageKeySize) +
", sampleKeyForConstantSizeComputation=" + pretty(sampleKey) +
", valueSize=" + pretty(valueSize) +
", averageValueSize=" + pretty(averageValueSize) +
", sampleValueForConstantSizeComputation=" + pretty(sampleValue) +
", entrySize=" + pretty(entrySize) +
", entryAndValueAlignment=" + entryAndValueAlignment() +
", actualChunkSize=" + pretty(actualChunkSize) +
", valueAlignment=" + valueAlignment() +
", entries=" + entries() +
", lockTimeOut=" + lockTimeOut + " " + lockTimeOutUnit +
", metaDataBytes=" + metaDataBytes() +
@@ -773,7 +942,7 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
* <p>Example: <pre>{@code Map<Key, Value> map =
* ChronicleMapBuilder.of(Key.class, Value.class)
* .entries(1_000_000)
* .keySize(50).valueSize(200)
* .averageKeySize(50).averageValueSize(200)
* // this class hasn't implemented yet, just for example
* .objectSerializer(new KryoObjectSerializer())
* .create();}</pre>
@@ -997,7 +1166,7 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
}
private void checkPrepareValueBytesOnlyIfConstantValueSize() {
if (prepareValueBytes != null && !valueBuilder.constantSizeMarshaller())
if (prepareValueBytes != null && !constantlySizedValues())
throw new IllegalStateException("Prepare value bytes could be used only if " +
"value size is constant");
}
@@ -1005,11 +1174,12 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
PrepareValueBytesAsWriter<K> prepareValueBytesAsWriter() {
PrepareValueBytes<K, V> prepareValueBytes = this.prepareValueBytes;
if ((prepareValueBytes == null && defaultValueProvider() != null) ||
!valueBuilder.constantSizeMarshaller())
!constantlySizedValues())
return null;
long constantValueSize = constantValueSize();
if (prepareValueBytes == null)
prepareValueBytes = new ZeroOutValueBytes<>(valueSize());
return new PrepareValueBytesAsWriter<>(prepareValueBytes, valueSize());
prepareValueBytes = new ZeroOutValueBytes<>(constantValueSize);
return new PrepareValueBytesAsWriter<>(prepareValueBytes, constantValueSize);
}
/**
@@ -1207,21 +1377,35 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
keyBuilder.objectSerializer(acquireObjectSerializer(JDKObjectSerializer.INSTANCE));
valueBuilder.objectSerializer(acquireObjectSerializer(JDKObjectSerializer.INSTANCE));
long maxSize = (long) entrySize(replicated) * figureBufferAllocationFactor(replicated);
keyBuilder.maxSize(maxSize);
valueBuilder.maxSize(maxSize);
double largeKeySize = averageKeySize();
if (!constantlySizedKeys())
largeKeySize *= figureBufferAllocationFactor();
keyBuilder.maxSize(round(largeKeySize));
double largeValueSize = averageValueSize();
if (!constantlySizedValues())
largeValueSize *= figureBufferAllocationFactor();
valueBuilder.maxSize(round(largeValueSize));
if (sampleKey != null)
keyBuilder.constantSizeBySample(sampleKey);
if (sampleValue != null)
valueBuilder.constantSizeBySample(sampleValue);
stateChecks(replicated);
stateChecks();
}
private void stateChecks(boolean replicated) {
private void stateChecks() {
if (strictStateChecks) {
if (entries < 0)
throw new IllegalStateException("Entries must be specified");
if (!constantlySizedKeys() && !isDefined(averageKeySize))
throw new IllegalStateException("No info about key size");
if (!constantlySizedValues() && !isDefined(averageValueSize))
throw new IllegalStateException("No info about value size");
}
checkAlignmentOnlyIfValuesPossiblyReferenceOffHeap();
checkPrepareValueBytesOnlyIfConstantValueSize();
checkMaxEntryOversizeFactorIfReplicated(replicated);
checkActualChunksPerSegmentIsConfiguredOnlyIfOtherLowLevelConfigsAreManual();
checkActualChunksPerSegmentGreaterOrEqualToEntries();
}
private ChronicleMap<K, V> establishReplication(
@@ -1257,11 +1441,11 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
return map;
}
private int figureBufferAllocationFactor(boolean replicated) {
private int figureBufferAllocationFactor() {
// if expected map size is about 1000, seems rather wasteful to allocate
// key and value serialization buffers each x64 of expected entry size..
return (int) Math.min(Math.max(2L, entries() >> 10),
maxEntryOversizeFactor(replicated));
Math.min(64, maxChunksPerEntry()));
}
}
src/main/java/net/openhft/chronicle/map/MultiMapFactory.java
+1
View File
@@ -74,6 +74,7 @@ enum MultiMapFactory {
public static MultiMapFactory forCapacity(long capacity) {
if (capacity <= ShortShortMultiMap.MAX_CAPACITY)
return I16;
// TODO why not I24?
// if (capacity <= Int24Int24MultiMap.MAX_CAPACITY)
// return I24;
if (capacity <= IntIntMultiMap.MAX_CAPACITY)
src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java
+2 -2
View File
@@ -150,7 +150,7 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
private long bitsPerSegmentInModIterBitSet() {
// min 128 * 8 to prevent false sharing on updating bits from different segments
// TODO this doesn't prevent false sharing. There should be GAPS between per-segment bits
return Maths.nextPower2(entriesPerSegment, 128L * 8L);
return Maths.nextPower2(actualChunksPerSegment, 128L * 8L);
}
@Override
@@ -1157,7 +1157,7 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
// key is not found
if (remote) {
long entrySize = entrySize(keySize, 0);
long pos = alloc(inBlocks(entrySize));
long pos = alloc(inChunks(entrySize));
long offset = offsetFromPos(pos);
clearMetaData(offset);
reuse(entry, offset);
src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java
+2 -1
View File
@@ -81,7 +81,8 @@ class ReplicationHubFindByName<K> implements FindByName {
this.map = (Map) ChronicleMapBuilder
.of(CharSequence.class, MapInstanceConfig.class)
.entrySize(4000)
.averageKeySize(10)
.averageValueSize(4000)
.entries(128)
.eventListener(listener)
.instance()
src/main/java/net/openhft/chronicle/map/SerializationBuilder.java
+15 -4
View File
@@ -274,7 +274,7 @@ final class SerializationBuilder<E> implements Cloneable, Serializable {
}
long constantSize = metaInterop.size(interop, sampleObject);
if (sizeIsStaticallyKnown) {
int expectedConstantSize = pseudoReadConstantSize();
long expectedConstantSize = pseudoReadConstantSize();
if (constantSize != expectedConstantSize) {
throw new IllegalStateException("Although configuring constant size by sample " +
"is not forbidden for types which size we already know statically, they " +
@@ -306,11 +306,22 @@ final class SerializationBuilder<E> implements Cloneable, Serializable {
}
boolean constantSizeMarshaller() {
return sizeMarshaller().sizeEncodingSize(0L) == 0;
try {
return sizeMarshaller().sizeEncodingSize(1L) == 0;
} catch (Exception e) {
// size marshaller thrown likely IllegalArgumentException, that means it doesn't
// expect size 1. marshaller of constant size shouldn't do that, it's body should be
// just `return 0;`
return false;
}
}
int pseudoReadConstantSize() {
return (int) sizeMarshaller().readSize(EMPTY_BYTES);
boolean constantSizeEncodingSizeMarshaller() {
return sizeMarshaller().minSizeEncodingSize() == sizeMarshaller().maxSizeEncodingSize();
}
long pseudoReadConstantSize() {
return sizeMarshaller().readSize(EMPTY_BYTES);
}
public SerializationBuilder<E> sizeMarshaller(SizeMarshaller sizeMarshaller) {
src/main/java/net/openhft/chronicle/map/StatelessMapConfig.java
+2 -1
View File
@@ -19,6 +19,7 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.StatelessClientConfig;
import net.openhft.lang.MemoryUnit;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -35,7 +36,7 @@ class StatelessMapConfig<K, V> implements StatelessClientConfig<ChronicleMap<K,
private final long timeoutMs;
private final String name;
private int tcpBufferSize = 1024 * VanillaChronicleMap.MAX_ENTRY_OVERSIZE_FACTOR;
private int tcpBufferSize = (int) MemoryUnit.KILOBYTES.toBytes(64);
private final AtomicBoolean used;
src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java
+100 -81
View File
@@ -53,6 +53,7 @@ import static java.lang.Long.numberOfTrailingZeros;
import static java.lang.Math.max;
import static java.lang.Thread.currentThread;
import static net.openhft.chronicle.map.Asserts.assertNotNull;
import static net.openhft.chronicle.map.ChronicleMapBuilder.greatestCommonDivisor;
import static net.openhft.lang.MemoryUnit.*;
class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
@@ -62,11 +63,6 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
private static final Logger LOG = LoggerFactory.getLogger(VanillaChronicleMap.class);
/**
* Because DirectBitSet implementations couldn't find more than 64 continuous clear or set
* bits.
*/
static final int MAX_ENTRY_OVERSIZE_FACTOR = 64;
private static final long serialVersionUID = 2L;
final Class<K> kClass;
final String dataFileVersion;
@@ -84,10 +80,13 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
final DefaultValueProvider<K, V> defaultValueProvider;
final PrepareValueBytesAsWriter<K> prepareValueBytesAsWriter;
final int metaDataBytes;
// private final int replicas;
final long entrySize;
final long chunkSize;
final Alignment alignment;
final boolean constantlySizedEntry;
final int worstAlignment;
final int specialEntrySpaceOffset;
final int actualSegments;
final long actualChunksPerSegment;
final long entriesPerSegment;
final MapEventListener<K, V> eventListener;
final BytesMapEventListener bytesEventListener;
@@ -101,7 +100,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
private final HashSplitting hashSplitting;
final Class nativeValueClass;
final MultiMapFactory multiMapFactory;
final int maxEntryOversizeFactor;
final int maxChunksPerEntry;
transient Provider<BytesReader<K>> keyReaderProvider;
transient Provider<KI> keyInteropProvider;
transient Provider<BytesReader<V>> valueReaderProvider;
@@ -152,22 +151,30 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS);
boolean replicated = getClass() == ReplicatedChronicleMap.class;
this.entrySize = builder.entrySize(replicated);
this.alignment = builder.entryAndValueAlignment();
this.chunkSize = builder.chunkSize(replicated);
this.alignment = builder.valueAlignment();
this.constantlySizedEntry = builder.constantlySizedEntries();
int alignment = this.alignment.alignment();
if (this.alignment != Alignment.NO_ALIGNMENT) {
this.worstAlignment = alignment - greatestCommonDivisor((int) chunkSize, alignment);
} else {
this.worstAlignment = 0;
}
this.specialEntrySpaceOffset = builder.specialEntrySpaceOffset(replicated);
this.errorListener = builder.errorListener();
this.putReturnsNull = builder.putReturnsNull();
this.removeReturnsNull = builder.removeReturnsNull();
this.actualSegments = builder.actualSegments();
this.entriesPerSegment = builder.actualEntriesPerSegment();
this.multiMapFactory = builder.multiMapFactory();
this.actualSegments = builder.actualSegments(replicated);
this.actualChunksPerSegment = builder.actualChunksPerSegment(replicated);
this.entriesPerSegment = builder.entriesPerSegment(replicated);
this.multiMapFactory = builder.multiMapFactory(replicated);
this.metaDataBytes = builder.metaDataBytes();
this.eventListener = builder.eventListener();
this.bytesEventListener = builder.bytesEventListener();
this.segmentHeaderSize = builder.segmentHeaderSize();
this.maxEntryOversizeFactor =
builder.maxEntryOversizeFactor(this instanceof ReplicatedChronicleMap);
this.segmentHeaderSize = builder.segmentHeaderSize(replicated);
this.maxChunksPerEntry = builder.maxChunksPerEntry();
hashSplitting = HashSplitting.Splitting.forSegments(actualSegments);
initTransients();
@@ -319,11 +326,11 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
}
private long sizeOfMultiMapBitSet() {
return MultiMapFactory.sizeOfBitSetInBytes(entriesPerSegment);
return MultiMapFactory.sizeOfBitSetInBytes(actualChunksPerSegment);
}
private long sizeOfBitSets() {
return CACHE_LINES.align(BYTES.alignAndConvert(entriesPerSegment, BITS), BYTES);
private long sizeOfSegmentFreeListBitSets() {
return CACHE_LINES.align(BYTES.alignAndConvert(actualChunksPerSegment, BITS), BYTES);
}
private int numberOfBitSets() {
@@ -333,8 +340,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
private long segmentSize() {
long ss = (CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES)
* multiMapsPerSegment())
+ numberOfBitSets() * sizeOfBitSets() // the free list and 0+ dirty lists.
+ sizeOfEntriesInSegment();
// the free list and 0+ dirty lists.
+ numberOfBitSets() * sizeOfSegmentFreeListBitSets()
+ sizeOfEntrySpaceInSegment();
if ((ss & 63L) != 0)
throw new AssertionError();
return breakL1CacheAssociativityContention(ss);
@@ -356,8 +364,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
return 1;
}
private long sizeOfEntriesInSegment() {
return CACHE_LINES.align(entriesPerSegment * entrySize, BYTES);
private long sizeOfEntrySpaceInSegment() {
return CACHE_LINES.align(specialEntrySpaceOffset + actualChunksPerSegment * chunkSize,
BYTES);
}
@Override
@@ -1435,12 +1444,13 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
* multiMapsPerSegment();
final NativeBytes bsBytes = new NativeBytes(ms.objectSerializer(),
start,
start + LONGS.align(BYTES.alignAndConvert(entriesPerSegment, BITS), BYTES),
start + LONGS.align(BYTES.alignAndConvert(actualChunksPerSegment, BITS), BYTES),
null);
freeList = new SingleThreadedDirectBitSet(bsBytes);
start += numberOfBitSets() * sizeOfBitSets();
start += numberOfBitSets() * sizeOfSegmentFreeListBitSets();
start += specialEntrySpaceOffset;
entriesOffset = start - bytes.startAddr();
assert bytes.capacity() >= entriesOffset + entriesPerSegment * entrySize;
assert bytes.capacity() >= entriesOffset + actualChunksPerSegment * chunkSize;
}
final MultiMap hashLookup() {
@@ -1591,7 +1601,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
@Override
public final long offsetFromPos(long pos) {
return entriesOffset + pos * entrySize;
return entriesOffset + pos * chunkSize;
}
final MultiStoreBytes reuse(MultiStoreBytes entry, long offset) {
@@ -1601,19 +1611,26 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
}
long entrySize(long keySize, long valueSize) {
return alignment.alignAddr(metaDataBytes +
long sizeOfEverythingBeforeValue = metaDataBytes +
keySizeMarshaller.sizeEncodingSize(keySize) + keySize +
valueSizeMarshaller.sizeEncodingSize(valueSize)) + valueSize;
valueSizeMarshaller.sizeEncodingSize(valueSize);
if (constantlySizedEntry) {
return alignment.alignAddr(sizeOfEverythingBeforeValue + valueSize);
} else if (worstAlignment > 0) {
return sizeOfEverythingBeforeValue + worstAlignment + valueSize;
} else {
return alignment.alignAddr(sizeOfEverythingBeforeValue) + valueSize;
}
}
final int inBlocks(long sizeInBytes) {
if (sizeInBytes <= entrySize)
final int inChunks(long sizeInBytes) {
if (sizeInBytes <= chunkSize)
return 1;
// int division is MUCH faster than long on Intel CPUs
sizeInBytes -= 1L;
if (sizeInBytes <= Integer.MAX_VALUE)
return (((int) sizeInBytes) / (int) entrySize) + 1;
return (int) (sizeInBytes / entrySize) + 1;
return (((int) sizeInBytes) / (int) chunkSize) + 1;
return (int) (sizeInBytes / chunkSize) + 1;
}
<KB, KBI, MKBI extends MetaBytesInterop<KB, ? super KBI>, RV>
@@ -1890,7 +1907,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
MultiStoreBytes entry, boolean writeDefaultInitialReplicationValues) {
long valueSize = metaElemWriter.size(elemWriter, elem);
long entrySize = entrySize(keySize, valueSize);
long pos = alloc(inBlocks(entrySize));
long pos = alloc(inChunks(entrySize));
segmentState.pos = pos;
long offset = offsetFromPos(pos);
clearMetaData(offset);
@@ -1923,56 +1940,58 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
}
//TODO refactor/optimize
final long alloc(int blocks) {
if (blocks > maxEntryOversizeFactor)
throw new IllegalArgumentException("Entry is too large: requires " + blocks +
" entry size chucks, " + maxEntryOversizeFactor + " is maximum.");
long ret = freeList.setNextNContinuousClearBits(nextPosToSearchFrom, blocks);
if (ret == DirectBitSet.NOT_FOUND || ret + blocks > entriesPerSegment) {
if (ret + blocks > entriesPerSegment)
freeList.clear(ret, ret + blocks);
ret = freeList.setNextNContinuousClearBits(0L, blocks);
if (ret == DirectBitSet.NOT_FOUND || ret + blocks > entriesPerSegment) {
if (ret + blocks > entriesPerSegment)
freeList.clear(ret, ret + blocks);
if (blocks == 1) {
final long alloc(int chunks) {
if (chunks > maxChunksPerEntry)
throw new IllegalArgumentException("Entry is too large: requires " + chunks +
" entry size chucks, " + maxChunksPerEntry + " is maximum.");
long ret = freeList.setNextNContinuousClearBits(nextPosToSearchFrom, chunks);
if (ret == DirectBitSet.NOT_FOUND || ret + chunks > actualChunksPerSegment) {
if (ret != DirectBitSet.NOT_FOUND &&
ret + chunks > actualChunksPerSegment && ret < actualChunksPerSegment)
freeList.clear(ret, actualChunksPerSegment);
ret = freeList.setNextNContinuousClearBits(0L, chunks);
if (ret == DirectBitSet.NOT_FOUND || ret + chunks > actualChunksPerSegment) {
if (ret != DirectBitSet.NOT_FOUND &&
ret + chunks > actualChunksPerSegment && ret < actualChunksPerSegment)
freeList.clear(ret, actualChunksPerSegment);
if (chunks == 1) {
throw new IllegalStateException(
"Segment is full, no free entries found");
} else {
throw new IllegalStateException(
"Segment is full or has no ranges of " + blocks
+ " continuous free blocks"
"Segment is full or has no ranges of " + chunks
+ " continuous free chunks"
);
}
}
updateNextPosToSearchFrom(ret, blocks);
updateNextPosToSearchFrom(ret, chunks);
} else {
// if bit at nextPosToSearchFrom is clear, it was skipped because
// more than 1 block was requested. Don't move nextPosToSearchFrom
// in this case. blocks == 1 clause is just a fast path.
if (blocks == 1 || freeList.isSet(nextPosToSearchFrom)) {
updateNextPosToSearchFrom(ret, blocks);
// more than 1 chunk was requested. Don't move nextPosToSearchFrom
// in this case. chunks == 1 clause is just a fast path.
if (chunks == 1 || freeList.isSet(nextPosToSearchFrom)) {
updateNextPosToSearchFrom(ret, chunks);
}
}
return ret;
}
private void updateNextPosToSearchFrom(long allocated, int blocks) {
if ((nextPosToSearchFrom = allocated + blocks) >= entriesPerSegment)
private void updateNextPosToSearchFrom(long allocated, int chunks) {
if ((nextPosToSearchFrom = allocated + chunks) >= actualChunksPerSegment)
nextPosToSearchFrom = 0L;
}
private boolean realloc(long fromPos, int oldBlocks, int newBlocks) {
if (freeList.allClear(fromPos + oldBlocks, fromPos + newBlocks)) {
freeList.set(fromPos + oldBlocks, fromPos + newBlocks);
private boolean realloc(long fromPos, int oldChunks, int newChunks) {
if (freeList.allClear(fromPos + oldChunks, fromPos + newChunks)) {
freeList.set(fromPos + oldChunks, fromPos + newChunks);
return true;
} else {
return false;
}
}
private void free(long fromPos, int blocks) {
freeList.clear(fromPos, fromPos + blocks);
private void free(long fromPos, int chunks) {
freeList.clear(fromPos, fromPos + chunks);
if (fromPos < nextPosToSearchFrom)
nextPosToSearchFrom = fromPos;
}
@@ -2079,7 +2098,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
if (removeFromMultiMap) {
hashLookup.removePrevPos(searchState);
long entrySizeInBytes = entry.positionAddr() + valueSize - entry.startAddr();
free(pos, inBlocks(entrySizeInBytes));
free(pos, inChunks(entrySizeInBytes));
} else {
hashLookup.removePosition(pos);
}
@@ -2237,7 +2256,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
* Replaces value in existing entry. May cause entry relocation, because there may be not
* enough space for new value in location already allocated for this entry.
*
* @param pos index of the first block occupied by the entry
* @param pos index of the first chunk occupied by the entry
* @param offset relative offset of the entry in Segment bytes (before, i. e.
* including metaData)
* @param entry relative pointer in Segment bytes
@@ -2260,20 +2279,20 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
newValueDoesNotFit:
if (newEntryEndAddr != entryEndAddr) {
long oldEntrySize = entryEndAddr - entryStartAddr;
int oldSizeInBlocks = inBlocks(oldEntrySize);
int newSizeInBlocks = inBlocks(newEntryEndAddr - entryStartAddr);
if (newSizeInBlocks > oldSizeInBlocks) {
if (newSizeInBlocks > maxEntryOversizeFactor) {
int oldSizeInChunks = inChunks(oldEntrySize);
int newSizeInChunks = inChunks(newEntryEndAddr - entryStartAddr);
if (newSizeInChunks > oldSizeInChunks) {
if (newSizeInChunks > maxChunksPerEntry) {
throw new IllegalArgumentException("Value too large: " +
"entry takes " + newSizeInBlocks + " blocks, " +
maxEntryOversizeFactor + " is maximum.");
"entry takes " + newSizeInChunks + " chunks, " +
maxChunksPerEntry + " is maximum.");
}
if (realloc(pos, oldSizeInBlocks, newSizeInBlocks))
if (realloc(pos, oldSizeInChunks, newSizeInChunks))
break newValueDoesNotFit;
// RELOCATION
free(pos, oldSizeInBlocks);
free(pos, oldSizeInChunks);
onRelocation(this, pos);
pos = alloc(newSizeInBlocks);
pos = alloc(newSizeInChunks);
// putValue() is called from put() and replace()
// after successful search by key
searchedHashLookup.replacePrevPos(segmentState.searchState, pos);
@@ -2286,12 +2305,12 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
newEntryStartAddr, valueSizeAddr - entryStartAddr);
entry = reuse(entry, offset);
// END OF RELOCATION
} else if (newSizeInBlocks < oldSizeInBlocks) {
// Freeing extra blocks
freeList.clear(pos + newSizeInBlocks, pos + oldSizeInBlocks);
} else if (newSizeInChunks < oldSizeInChunks) {
// Freeing extra chunks
freeList.clear(pos + newSizeInChunks, pos + oldSizeInChunks);
// Do NOT reset nextPosToSearchFrom, because if value
// once was larger it could easily became oversized again,
// But if these blocks will be taken by that time,
// once was larger it could easily became larger again,
// But if these chunks will be taken by that time,
// this entry will need to be relocated.
}
}
@@ -2356,10 +2375,10 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
manageReplicationBytes(entry, false, false);
long valueSize = valueSizeMarshaller.readSize(entry);
long sizeInBytes = entrySize(keySize, valueSize);
int entrySizeInBlocks = inBlocks(sizeInBytes);
if (!freeList.allSet(pos, pos + entrySizeInBlocks))
int entrySizeInChunks = inChunks(sizeInBytes);
if (!freeList.allSet(pos, pos + entrySizeInChunks))
throw new AssertionError();
pos += entrySizeInBlocks;
pos += entrySizeInChunks;
}
} finally {
readUnlock();
@@ -2503,7 +2522,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
final long entryEndAddr = entry.positionAddr() + valueSize;
if (removeFromMultiMap) {
segment.hashLookup().remove(segmentHash, pos);
segment.free(pos, segment.inBlocks(entryEndAddr - entry.address()));
segment.free(pos, segment.inChunks(entryEndAddr - entry.address()));
} else {
segment.hashLookup().removePosition(pos);
}
src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java
+23 -27
View File
@@ -92,31 +92,38 @@ public final class ChronicleSetBuilder<E>
}
@Override
public ChronicleSetBuilder<E> actualEntriesPerSegment(long actualEntriesPerSegment) {
chronicleMapBuilder.actualEntriesPerSegment(actualEntriesPerSegment);
public ChronicleSetBuilder<E> entriesPerSegment(long entriesPerSegment) {
chronicleMapBuilder.entriesPerSegment(entriesPerSegment);
return this;
}
@Override
public ChronicleSetBuilder<E> actualChunksPerSegment(long actualChunksPerSegment) {
chronicleMapBuilder.actualChunksPerSegment(actualChunksPerSegment);
return this;
}
/**
* {@inheritDoc}
*
* <p>Example: if keys in your set(s) are English words in {@link String} form, keys size 10
* (a bit more than average English word length) would be a good choice: <pre>{@code
* <p>Example: if keys in your set(s) are English words in {@link String} form, average English
* word length is 5.1, configure average key size of 6: <pre>{@code
* ChronicleSet<String> uniqueWords = ChronicleSetBuilder.of(String.class)
* .entries(50000)
* .keySize(10)
* .averageKeySize(6)
* .create();}</pre>
*
* <p>(Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded
* (and each character takes 2 bytes on-heap), because default off-heap {@link String} encoding
* is UTF-8 in {@code ChronicleSet}.)
* <p>(Note that 6 is chosen as average key size in bytes despite strings in Java are UTF-16
* encoded (and each character takes 2 bytes on-heap), because default off-heap {@link String}
* encoding is UTF-8 in {@code ChronicleSet}.)
*
* @see #constantKeySizeBySample(Object)
* @see #entrySize(int)
* @see #actualChunkSize(int)
* @param averageKeySize
*/
@Override
public ChronicleSetBuilder<E> keySize(int keySize) {
chronicleMapBuilder.keySize(keySize);
public ChronicleSetBuilder<E> averageKeySize(double averageKeySize) {
chronicleMapBuilder.averageKeySize(averageKeySize);
return this;
}
@@ -128,7 +135,7 @@ public final class ChronicleSetBuilder<E>
* .constantKeySizeBySample(new byte[20])
* .create();}</pre>
*
* @see #keySize(int)
* @see ChronicleHashBuilder#averageKeySize(double)
*/
@Override
public ChronicleSetBuilder<E> constantKeySizeBySample(E sampleKey) {
@@ -136,26 +143,15 @@ public final class ChronicleSetBuilder<E>
return this;
}
/**
* {@inheritDoc}
*
* <p>In fully default case you can expect entry size to be about 120-130 bytes. But it is
* strongly recommended always to configure {@linkplain #keySize(int) key size}, if they
* couldn't be derived statically.
*
* <p>If entry size is not configured explicitly by calling this method, it is computed based on
* {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size},
* plus a few bytes required by implementations.
*/
@Override
public ChronicleSetBuilder<E> entrySize(int entrySize) {
chronicleMapBuilder.entrySize(entrySize);
public ChronicleSetBuilder<E> actualChunkSize(int actualChunkSize) {
chronicleMapBuilder.actualChunkSize(actualChunkSize);
return this;
}
@Override
public ChronicleSetBuilder<E> maxEntryOversizeFactor(int maxEntryOversizeFactor) {
chronicleMapBuilder.maxEntryOversizeFactor(maxEntryOversizeFactor);
public ChronicleSetBuilder<E> maxChunksPerEntry(int maxChunksPerEntry) {
chronicleMapBuilder.maxChunksPerEntry(maxChunksPerEntry);
return this;
}
src/main/java/net/openhft/chronicle/set/DummyValueMarshaller.java
+10
View File
@@ -64,6 +64,16 @@ enum DummyValueMarshaller
return 0;
}
@Override
public int minSizeEncodingSize() {
return 0;
}
@Override
public int maxSizeEncodingSize() {
return 0;
}
@Override
public void writeSize(Bytes bytes, long size) {
// do nothing