/* * Copyright 2014 Higher Frequency Trading * * http://www.higherfrequencytrading.com * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.map; import net.openhft.chronicle.hash.*; import net.openhft.chronicle.hash.replication.*; import net.openhft.chronicle.hash.serialization.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.MetaBytesWriter; import net.openhft.chronicle.hash.serialization.MetaProvider; import net.openhft.chronicle.set.ChronicleSetBuilder; import net.openhft.lang.Maths; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.BytesStore; import net.openhft.lang.io.DirectStore; import net.openhft.lang.io.serialization.*; import net.openhft.lang.io.serialization.impl.AllocateInstanceObjectFactory; import net.openhft.lang.io.serialization.impl.NewInstanceObjectFactory; import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; import net.openhft.lang.model.Byteable; import net.openhft.lang.threadlocal.Provider; import net.openhft.lang.threadlocal.ThreadLocalCopies; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import static net.openhft.chronicle.map.Objects.builderEquals; /** * {@code AbstractChronicleMapBuilder} manages most of {@link ChronicleMap} configurations; has two * concrete subclasses: {@link ChronicleMapBuilder} should be used to create maps with ordinary * values, {@link OffHeapUpdatableChronicleMapBuilder} -- maps with {@link Byteable} values, which * point directly to off-heap memory; could be used as a classic builder and/or factory. This means * that in addition to the standard builder usage pattern:
{@code
* ChronicleMap map = ChronicleMapBuilder
* .of(Key.class, Value.class)
* .entries(100500)
* // ... other configurations
* .create();}
* one of concrete {@code AbstractChronicleMapBuilder} subclasses, {@link ChronicleMapBuilder} or
* {@link OffHeapUpdatableChronicleMapBuilder}, could be prepared and used to create many similar
* maps: {@code
* OffHeapUpdatableChronicleMapBuilder builder = OffHeapUpdatableChronicleMapBuilder
* .of(Key.class, Value.class)
* .entries(100500);
*
* ChronicleMap map1 = builder.create();
* ChronicleMap map2 = builder.create();}
* i. e. created {@code ChronicleMap} instances don't depend on the builder.
*
* {@code AbstractChronicleMapBuilder} and it's subclasses are mutable, see a note in {@link * ChronicleHashBuilder} interface documentation.
Later in this documentation, * "ChronicleMap" means "ChronicleMaps, created by {@code AbstractChronicleMapBuilder}", unless * specified different, because theoretically someone might provide {@code ChronicleMap} * implementations with completely different properties.
{@code ChronicleMap} * ("ChronicleMaps, created by {@code AbstractChronicleMapBuilder}") currently doesn't support * resizing. That is why you should always configure {@linkplain #entries(long) number of * entries} you are going to insert into the created map at most. See {@link #entries(long)} * method documentation for more information on this.
{@code ChronicleMap} allocates memory * by equally sized chunks. This size is called {@linkplain #entrySize(int) entry size}, you are * strongly recommended to configure it to achieve least memory consumption and best speed. See * {@link #entrySize(int)} method documentation for more information on this.
* * @paramExample: if keys in your map(s) are English words in {@link String} form, keys size 10 * (a bit more than average English word length) would be a good choice:
{@code
* ChronicleMap wordFrequencies = OffHeapUpdatableChronicleMapBuilder
* .of(String.class, LongValue.class)
* .entries(50000)
* .keySize(10)
* // shouldn't specify valueSize(), because it is statically known
* .create();}
* (Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded (and
* each character takes 2 bytes on-heap), because default off-heap {@link String} encoding is
* UTF-8 in {@code ChronicleMap}.)
*
* @see #constantKeySizeBySample(Object)
* @see #entrySize(int)
*/
@Override
public B keySize(int keySize) {
if (keySize <= 0)
throw new IllegalArgumentException("Key size must be positive");
this.keySize = keySize;
return self();
}
/**
* {@inheritDoc}
*
* For example, if your keys are Git commit hashes:
{@code
* Map gitCommitMessagesByHash =
* ChronicleMapBuilder.of(byte[].class, String.class)
* .constantKeySizeBySample(new byte[20])
* .create();}
*
* @see #keySize(int)
*/
@Override
public B constantKeySizeBySample(K sampleKey) {
this.sampleKey = sampleKey;
return self();
}
private int keySize() {
return keyOrValueSize(keySize, keyBuilder);
}
B valueSize(int valueSize) {
if (valueSize <= 0)
throw new IllegalArgumentException("Value size must be positive");
this.valueSize = valueSize;
return self();
}
B constantValueSizeBySample(V sampleValue) {
this.sampleValue = sampleValue;
return self();
}
int valueSize() {
return keyOrValueSize(valueSize, valueBuilder);
}
private int keyOrValueSize(int configuredSize, SerializationBuilder builder) {
if (configuredSize > 0)
return configuredSize;
// this means size is statically known
if (builder.sizeMarshaller().sizeEncodingSize(0L) == 0)
return (int) builder.sizeMarshaller().readSize(EMPTY_BYTES);
return DEFAULT_KEY_OR_VALUE_SIZE;
}
/**
* {@inheritDoc}
*
* In fully default case you can expect entry size to be about 256 bytes. But it is strongly * recommended always to configure {@linkplain #keySize(int) key size} and {@linkplain * #valueSize(int) value size}, if they couldn't be derived statically.
If entry size is * not configured explicitly by calling this method, it is computed based on {@linkplain * #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size}, plus * {@linkplain #valueSize(int) value size}, plus a few bytes required by implementations.
* * @see #entries(long) */ @Override public B entrySize(int entrySize) { if (entrySize <= 0) throw new IllegalArgumentException("Entry Size must be positive"); this.entrySize = entrySize; return self(); } int entrySize(boolean replicated) { if (entrySize > 0) return entryAndValueAlignment().alignSize(entrySize); int size = metaDataBytes; int keySize = keySize(); size += keyBuilder.sizeMarshaller().sizeEncodingSize(keySize); size += keySize; if (replicated) size += ReplicatedChronicleMap.ADDITIONAL_ENTRY_BYTES; int valueSize = valueSize(); size += valueBuilder.sizeMarshaller().sizeEncodingSize(valueSize); size = entryAndValueAlignment().alignSize(size); size += valueSize; // Some cache line heuristics for (int i = 1; i <= 4; i++) { int bound = i * 64; // Not more than 5% oversize. // DEFAULT_KEY_OR_VALUE_SIZE and this heuristic are specially adjusted to produce // entry size 256 -- the default value prior to keySize and valueSize -- when key and // value sizes are both not static nor configured, in both vanilla and replicated modes. if (size < bound && (bound - size) <= bound / 20) { size = bound; break; } } return entryAndValueAlignment().alignSize(size); } B entryAndValueAlignment(Alignment alignment) { this.alignment = alignment; return self(); } Alignment entryAndValueAlignment() { if (alignment != null) return alignment; return offHeapReference(valueBuilder.eClass) ? Alignment.OF_4_BYTES : Alignment.NO_ALIGNMENT; } @Override public B entries(long entries) { if (entries <= 0L) throw new IllegalArgumentException("Entries should be positive, " + entries + " given"); this.entries = entries; return self(); } long entries() { return entries; } @Override public B actualEntriesPerSegment(long actualEntriesPerSegment) { if (actualEntriesPerSegment <= 0L) throw new IllegalArgumentException("entries per segment should be positive, " + actualEntriesPerSegment + " given"); if (tooManyEntriesPerSegment(actualEntriesPerSegment)) throw new IllegalArgumentException("max entries per segment is " + VanillaIntIntMultiMap.MAX_CAPACITY + ", " + actualEntriesPerSegment + " given"); this.actualEntriesPerSegment = actualEntriesPerSegment; return self(); } private boolean tooManyEntriesPerSegment(long entriesPerSegment) { return entriesPerSegment > VanillaIntIntMultiMap.MAX_CAPACITY; } long actualEntriesPerSegment() { if (actualEntriesPerSegment > 0L) return actualEntriesPerSegment; int actualSegments = actualSegments(); long actualEntries = totalEntriesIfPoorDistribution(actualSegments); long actualEntriesPerSegment = divideUpper(actualEntries, actualSegments); if (tooManyEntriesPerSegment(actualEntriesPerSegment)) throw new IllegalStateException("max entries per segment is " + VanillaIntIntMultiMap.MAX_CAPACITY + " configured entries() and " + "actualSegments() so that there should be " + actualEntriesPerSegment + " entries per segment"); return actualEntriesPerSegment; } private long totalEntriesIfPoorDistribution(int segments) { if (segments == 1) return entries; double poorDistEntriesScale = Math.log(segments) * entries; if (segments <= 8) return Math.min(entries * segments, (long) (entries + poorDistEntriesScale * 0.11 + segments * 8)); // 11% was min for tests return (long) (entries + poorDistEntriesScale * 0.11 + 80); // 11% was min for tests } @Override public B minSegments(int minSegments) { checkSegments(minSegments); this.minSegments = minSegments; return self(); } int minSegments() { return Math.max(estimateSegments(), minSegments); } private int estimateSegments() { for (int power = 1; power < 64; power *= 2) { if (entries <= power * power * power) return power; } long size = entrySize * entries; if (size < 20L * 1000 * 1000) // 20 MB return 64; if (size < 200L * 1000 * 1000) // 200 MB return 128; if (size < 800L * 1000 * 1000) // 800 MB return 256; if (size < 2L * 1000 * 1000 * 1000) // 2 GB return 512; return 1024; } @Override public B actualSegments(int actualSegments) { checkSegments(actualSegments); this.actualSegments = actualSegments; return self(); } int actualSegments() { if (actualSegments > 0) return actualSegments; long shortMMapSegments = trySegments(VanillaShortShortMultiMap.MAX_CAPACITY, MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS); if (shortMMapSegments > 0L) return (int) shortMMapSegments; long intMMapSegments = trySegments(VanillaIntIntMultiMap.MAX_CAPACITY, MAX_SEGMENTS); if (intMMapSegments > 0L) return (int) intMMapSegments; throw new IllegalStateException("Max segments is " + MAX_SEGMENTS + ", configured so much" + " entries() that builder automatically decided to use " + (-intMMapSegments) + " segments"); } private long trySegments(long maxSegmentCapacity, int maxSegments) { long segments = divideUpper(totalEntriesIfPoorDistribution(minSegments()), maxSegmentCapacity); segments = Maths.nextPower2(Math.max(segments, minSegments()), 1L); return segments <= maxSegments ? segments : -segments; } private boolean canSupportShortShort() { return entries > (long) minSegments() << 15; } public B lockTimeOut(long lockTimeOut, TimeUnit unit) { this.lockTimeOut = lockTimeOut; lockTimeOutUnit = unit; return self(); } long lockTimeOut(TimeUnit unit) { return unit.convert(lockTimeOut, lockTimeOutUnit); } @Override public B errorListener(ChronicleHashErrorListener errorListener) { this.errorListener = errorListener; return self(); } ChronicleHashErrorListener errorListener() { return errorListener; } /** * Configures if the maps created by this builder should return {@code null} instead of previous * mapped values on {@link ChronicleMap#put(Object, Object) ChornicleMap.put(key, value)} * calls. * *{@link Map#put(Object, Object) Map.put()} returns the previous value, functionality which * is rarely used but fairly cheap for {@link HashMap}. In the case, for an off heap collection, * it has to create a new object and deserialize the data from off-heap memory. It's expensive * for something you probably don't use.
By default, of cause, {@code ChronicleMap} * conforms the general {@code Map} contract and returns the previous mapped value on {@code * put()} calls.
* * @param putReturnsNull {@code true} if you want {@link ChronicleMap#put(Object, Object) * ChronicleMap.put()} to not return the value that was replaced but * instead return {@code null} * @return an instance of the map builder back * @see #removeReturnsNull(boolean) */ public B putReturnsNull(boolean putReturnsNull) { this.putReturnsNull = putReturnsNull; return self(); } boolean putReturnsNull() { return putReturnsNull; } /** * Configures if the maps created by this builder should return {@code null} instead of the last * mapped value on {@link ChronicleMap#remove(Object) ChronicleMap.remove(key)} calls. * *{@link Map#remove(Object) Map.remove()} returns the previous value, functionality which is * rarely used but fairly cheap for {@link HashMap}. In the case, for an off heap collection, it * has to create a new object and deserialize the data from off-heap memory. It's expensive for * something you probably don't use.
* * @param removeReturnsNull {@code true} if you want {@link ChronicleMap#remove(Object) * ChronicleMap.remove()} to not return the value of the removed entry * but instead return {@code null} * @return an instance of the map builder back * @see #putReturnsNull(boolean) */ public B removeReturnsNull(boolean removeReturnsNull) { this.removeReturnsNull = removeReturnsNull; return self(); } boolean removeReturnsNull() { return removeReturnsNull; } @Override public B metaDataBytes(int metaDataBytes) { if ((metaDataBytes & 0xFF) != metaDataBytes) throw new IllegalArgumentException("MetaDataBytes must be [0..255] was " + metaDataBytes); this.metaDataBytes = metaDataBytes; return self(); } int metaDataBytes() { return metaDataBytes; } @Override public String toString() { return "ChronicleMapBuilder{" + "actualSegments=" + pretty(actualSegments) + ", minSegments=" + pretty(minSegments) + ", actualEntriesPerSegment=" + pretty(actualEntriesPerSegment) + ", keySize=" + pretty(keySize) + ", sampleKeyForConstantSizeComputation=" + pretty(sampleKey) + ", valueSize=" + pretty(valueSize) + ", sampleValueForConstantSizeComputation=" + pretty(sampleValue) + ", entrySize=" + pretty(entrySize) + ", entryAndValueAlignment=" + entryAndValueAlignment() + ", entries=" + entries() + ", lockTimeOut=" + lockTimeOut + " " + lockTimeOutUnit + ", metaDataBytes=" + metaDataBytes() + ", errorListener=" + errorListener() + ", putReturnsNull=" + putReturnsNull() + ", removeReturnsNull=" + removeReturnsNull() + ", timeProvider=" + timeProvider() + ", bytesMarshallerFactory=" + pretty(bytesMarshallerFactory) + ", objectSerializer=" + pretty(objectSerializer) + ", keyBuilder=" + keyBuilder + ", valueBuilder=" + valueBuilder + ", eventListener=" + eventListener + ", defaultValue=" + defaultValue + ", defaultValueProvider=" + pretty(defaultValueProvider) + ", prepareValueBytes=" + pretty(prepareValueBytes) + '}'; } @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override public boolean equals(Object o) { return builderEquals(this, o); } @Override public int hashCode() { return toString().hashCode(); } @Override public B timeProvider(TimeProvider timeProvider) { this.timeProvider = timeProvider; return self(); } TimeProvider timeProvider() { return timeProvider; } BytesMarshallerFactory bytesMarshallerFactory() { return bytesMarshallerFactory == null ? bytesMarshallerFactory = new VanillaBytesMarshallerFactory() : bytesMarshallerFactory; } @Override public B bytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) { this.bytesMarshallerFactory = bytesMarshallerFactory; return self(); } ObjectSerializer objectSerializer() { return objectSerializer == null ? objectSerializer = BytesMarshallableSerializer.create( bytesMarshallerFactory(), JDKObjectSerializer.INSTANCE) : objectSerializer; } @Override public B objectSerializer(ObjectSerializer objectSerializer) { this.objectSerializer = objectSerializer; return self(); } /** * {@inheritDoc} * *Example:
*/
@Override
public B keyMarshaller(@NotNull BytesMarshaller Default value deserialization factory is {@link NewInstanceObjectFactory}, which creates a
* new value instance using {@link Class#newInstance()} default constructor. You could provide
* an {@link AllocateInstanceObjectFactory}, which uses {@code Unsafe.allocateInstance(Class)}
* (you might want to do this for better performance or if you don't want to initialize fields),
* or a factory which calls a value class constructor with some arguments, or a factory which
* internally delegates to instance pool or {@link ThreadLocal}, to reduce allocations. This configuration overrides any previous {@link #defaultValueProvider(DefaultValueProvider)}
* configuration to this {@code AbstractChronicleMapBuilder}. This configuration overrides any previous {@link #defaultValue(Object)} configuration to
* this {@code AbstractChronicleMapBuilder}.