Consolidate segment headers to reduce TLB misses. Re-tune the calculated number of segments based on the size of data.

This commit is contained in:
Peter Lawrey
2014-10-31 17:33:10 +00:00
parent ff697fa012
commit bb94e0e547
5 changed files with 274 additions and 296 deletions
@@ -22,8 +22,6 @@ import net.openhft.chronicle.hash.*;
import net.openhft.chronicle.hash.serialization.MetaBytesInterop;
import net.openhft.chronicle.hash.serialization.MetaBytesWriter;
import net.openhft.chronicle.hash.serialization.MetaProvider;
import net.openhft.lang.threadlocal.Provider;
import net.openhft.lang.threadlocal.ThreadLocalCopies;
import net.openhft.chronicle.set.ChronicleSetBuilder;
import net.openhft.lang.Maths;
import net.openhft.lang.io.ByteBufferBytes;
@@ -35,6 +33,8 @@ import net.openhft.lang.io.serialization.impl.AllocateInstanceObjectFactory;
import net.openhft.lang.io.serialization.impl.NewInstanceObjectFactory;
import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory;
import net.openhft.lang.model.Byteable;
import net.openhft.lang.threadlocal.Provider;
import net.openhft.lang.threadlocal.ThreadLocalCopies;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,18 +48,18 @@ import java.util.concurrent.TimeUnit;
import static net.openhft.chronicle.map.Objects.builderEquals;
/**
* {@code AbstractChronicleMapBuilder} manages most of {@link ChronicleMap} configurations; has two
* concrete subclasses: {@link ChronicleMapBuilder} should be used to create maps with ordinary
* values, {@link OffHeapUpdatableChronicleMapBuilder} -- maps with {@link Byteable} values, which
* point directly to off-heap memory; could be used as a classic builder and/or factory. This means
* {@code AbstractChronicleMapBuilder} manages most of {@link ChronicleMap} configurations; has two concrete subclasses:
* {@link ChronicleMapBuilder} should be used to create maps with ordinary values, {@link
* OffHeapUpdatableChronicleMapBuilder} -- maps with {@link Byteable} values, which point directly to off-heap memory;
* could be used as a classic builder and/or factory. This means
* that in addition to the standard builder usage pattern: <pre>{@code
* ChronicleMap<Key, Value> map = ChronicleMapBuilder
* .of(Key.class, Value.class)
* .entries(100500)
* // ... other configurations
* .create();}</pre>
* one of concrete {@code AbstractChronicleMapBuilder} subclasses, {@link ChronicleMapBuilder} or
* {@link OffHeapUpdatableChronicleMapBuilder}, could be prepared and used to create many similar
* one of concrete {@code AbstractChronicleMapBuilder} subclasses, {@link ChronicleMapBuilder} or {@link
* OffHeapUpdatableChronicleMapBuilder}, could be prepared and used to create many similar
* maps: <pre>{@code
* OffHeapUpdatableChronicleMapBuilder<Key, Value> builder = OffHeapUpdatableChronicleMapBuilder
* .of(Key.class, Value.class)
@@ -69,22 +69,22 @@ import static net.openhft.chronicle.map.Objects.builderEquals;
* ChronicleMap<Key, Value> map2 = builder.create();}</pre>
* i. e. created {@code ChronicleMap} instances don't depend on the builder.
*
* <p>{@code AbstractChronicleMapBuilder} and it's subclasses are mutable, see a note in
* {@link ChronicleHashBuilder} interface documentation.
*
* <p>Later in this documentation, "ChronicleMap" means "ChronicleMaps, created by {@code
* AbstractChronicleMapBuilder}", unless specified different, because theoretically someone might
* provide {@code ChronicleMap} implementations with completely different properties.
*
* <p>{@code ChronicleMap} ("ChronicleMaps, created by {@code AbstractChronicleMapBuilder}")
* currently doesn't support resizing. That is why you should <i>always</i> configure {@linkplain
* #entries(long) number of entries} you are going to insert into the created map <i>at most</i>.
* See {@link #entries(long)} method documentation for more information on this.
*
* <p>{@code ChronicleMap} allocates memory by equally sized chunks. This size is called {@linkplain
* #entrySize(int) entry size}, you are strongly recommended to configure it to achieve least memory
* consumption and best speed. See {@link #entrySize(int)} method documentation for more information
* <p>{@code AbstractChronicleMapBuilder} and it's subclasses are mutable, see a note in {@link ChronicleHashBuilder}
* interface documentation.
* </p>
* <p>Later in this documentation, "ChronicleMap" means "ChronicleMaps, created by {@code AbstractChronicleMapBuilder}",
* unless specified different, because theoretically someone might provide {@code ChronicleMap} implementations with
* completely different properties.
* </p>
* <p>{@code ChronicleMap} ("ChronicleMaps, created by {@code AbstractChronicleMapBuilder}") currently doesn't support
* resizing. That is why you should <i>always</i> configure {@linkplain #entries(long) number of entries} you are going
* to insert into the created map <i>at most</i>. See {@link #entries(long)} method documentation for more information
* on this.
* </p>
* <p>{@code ChronicleMap} allocates memory by equally sized chunks. This size is called {@linkplain #entrySize(int)
* entry size}, you are strongly recommended to configure it to achieve least memory consumption and best speed. See
* {@link #entrySize(int)} method documentation for more information on this.
* </p>
*
* @param <K> key type of the maps, produced by this builder
* @param <V> value type of the maps, produced by this builder
@@ -95,13 +95,11 @@ public abstract class AbstractChronicleMapBuilder<K, V,
B extends AbstractChronicleMapBuilder<K, V, B>>
implements Cloneable, ChronicleHashBuilder<K, ChronicleMap<K, V>, B> {
static final short UDP_REPLICATION_MODIFICATION_ITERATOR_ID = 128;
private static final Bytes EMPTY_BYTES = new ByteBufferBytes(ByteBuffer.allocate(0));
private static final int DEFAULT_KEY_OR_VALUE_SIZE = 120;
private static final int MAX_SEGMENTS = (1 << 30);
private static final int MAX_SEGMENTS_TO_CHAISE_COMPACT_MULTI_MAPS = (1 << 20);
static final short UDP_REPLICATION_MODIFICATION_ITERATOR_ID = 128;
private static final Logger LOG =
LoggerFactory.getLogger(AbstractChronicleMapBuilder.class.getName());
@@ -157,6 +155,27 @@ public abstract class AbstractChronicleMapBuilder<K, V,
return roundUp;
}
private static void checkSegments(int segments) {
if (segments <= 0 || segments > MAX_SEGMENTS)
throw new IllegalArgumentException("segments should be positive, " +
segments + " given");
if (segments > MAX_SEGMENTS)
throw new IllegalArgumentException("Max segments is " + MAX_SEGMENTS + ", " +
segments + " given");
}
private static long divideUpper(long dividend, long divisor) {
return ((dividend - 1L) / divisor) + 1L;
}
private static String pretty(int value) {
return value > 0 ? value + "" : "not configured";
}
private static String pretty(Object obj) {
return obj != null ? obj + "" : "not configured";
}
@Override
public B clone() {
try {
@@ -183,9 +202,8 @@ public abstract class AbstractChronicleMapBuilder<K, V,
* .keySize(10)
* // shouldn't specify valueSize(), because it is statically known
* .create();}</pre>
* (Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded (and
* each character takes 2 bytes on-heap), because default off-heap {@link String} encoding is
* UTF-8 in {@code ChronicleMap}.)
* (Note that 10 is chosen as key size in bytes despite strings in Java are UTF-16 encoded (and each character takes
* 2 bytes on-heap), because default off-heap {@link String} encoding is UTF-8 in {@code ChronicleMap}.)
*
* @see #constantKeySizeBySample(Object)
* @see #entrySize(int)
@@ -247,13 +265,11 @@ public abstract class AbstractChronicleMapBuilder<K, V,
/**
* {@inheritDoc}
*
* <p>In fully default case you can expect entry size to be about 256 bytes. But it is strongly
* recommended always to configure {@linkplain #keySize(int) key size} and {@linkplain
* #valueSize(int) value size}, if they couldn't be derived statically.
*
* <p>If entry size is not configured explicitly by calling this method, it is computed based on
* {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size},
* plus {@linkplain #valueSize(int) value size}, plus a few bytes required by implementations.
* <p>In fully default case you can expect entry size to be about 256 bytes. But it is strongly recommended always
* to configure {@linkplain #keySize(int) key size} and {@linkplain #valueSize(int) value size}, if they couldn't be
* derived statically. </p> <p>If entry size is not configured explicitly by calling this method, it is computed
* based on {@linkplain #metaDataBytes(int) meta data bytes}, plus {@linkplain #keySize(int) key size}, plus
* {@linkplain #valueSize(int) value size}, plus a few bytes required by implementations. </p>
*
* @see #entries(long)
*/
@@ -350,8 +366,8 @@ public abstract class AbstractChronicleMapBuilder<K, V,
double poorDistEntriesScale = Math.log(segments) * entries;
if (segments <= 8)
return Math.min(entries * segments,
(long) (entries + poorDistEntriesScale * 0.08 + 64)); // 8% was min for tests
return (long) (entries + poorDistEntriesScale * 0.11 + 80); // 11% was min for tests
(long) (entries + poorDistEntriesScale * 0.11 + segments * 8)); // 11% was min for tests
return (long) (entries + poorDistEntriesScale * 0.11 + 80); // 11% was min for tests
}
@Override
@@ -366,17 +382,20 @@ public abstract class AbstractChronicleMapBuilder<K, V,
}
private int estimateSegments() {
for (int power = 0; power < 7; power++) {
if (entries < (1L << (7 + (2 * power))))
return 1 << power;
for (int power = 1; power < 64; power *= 2) {
if (entries <= power * power * power)
return power;
}
// Heuristic -- number of segments ~= cube root from number of entries seems optimal
int maxSegments = 65536;
for (int segments = 4; segments < maxSegments; segments <<= 1) {
if (((long) segments) * segments * segments >= entries)
return segments;
}
return maxSegments;
long size = entrySize * entries;
if (size < 200L * 1000 * 1000) // 200 MB
return 64;
if (size < 2000L * 1000 * 1000) // 2 GB
return 128;
if (size < 8000L * 1000 * 1000) // 8 GB
return 256;
if (size < 20L * 1000 * 1000 * 1000) // 20 GB
return 512;
return 1024;
}
@Override
@@ -386,15 +405,6 @@ public abstract class AbstractChronicleMapBuilder<K, V,
return self();
}
private static void checkSegments(int segments) {
if (segments <= 0 || segments > MAX_SEGMENTS)
throw new IllegalArgumentException("segments should be positive, " +
segments + " given");
if (segments > MAX_SEGMENTS)
throw new IllegalArgumentException("Max segments is " + MAX_SEGMENTS + ", " +
segments + " given");
}
int actualSegments() {
if (actualSegments > 0)
return actualSegments;
@@ -417,10 +427,6 @@ public abstract class AbstractChronicleMapBuilder<K, V,
return segments <= maxSegments ? segments : -segments;
}
private static long divideUpper(long dividend, long divisor) {
return ((dividend - 1L) / divisor) + 1L;
}
private boolean canSupportShortShort() {
return entries > (long) minSegments() << 15;
}
@@ -446,21 +452,19 @@ public abstract class AbstractChronicleMapBuilder<K, V,
}
/**
* Configures if the maps created by this builder should return {@code null} instead of previous
* mapped values on {@link ChronicleMap#put(Object, Object) ChornicleMap.put(key, value)}
* calls.
* Configures if the maps created by this builder should return {@code null} instead of previous mapped values on
* {@link ChronicleMap#put(Object, Object) ChornicleMap.put(key, value)} calls.
*
* <p>{@link Map#put(Object, Object) Map.put()} returns the previous value, functionality which
* is rarely used but fairly cheap for {@link HashMap}. In the case, for an off heap collection,
* it has to create a new object and deserialize the data from off-heap memory. It's expensive
* for something you probably don't use.
* <p>{@link Map#put(Object, Object) Map.put()} returns the previous value, functionality which is rarely used but
* fairly cheap for {@link HashMap}. In the case, for an off heap collection, it has to create a new object and
* deserialize the data from off-heap memory. It's expensive for something you probably don't use.
* </p>
* <p>By default, of cause, {@code ChronicleMap} conforms the general {@code Map} contract and returns the previous
* mapped value on {@code put()} calls.
* </p>
*
* <p>By default, of cause, {@code ChronicleMap} conforms the general {@code Map} contract and
* returns the previous mapped value on {@code put()} calls.
*
* @param putReturnsNull {@code true} if you want {@link ChronicleMap#put(Object, Object)
* ChronicleMap.put()} to not return the value that was replaced but
* instead return {@code null}
* @param putReturnsNull {@code true} if you want {@link ChronicleMap#put(Object, Object) ChronicleMap.put()} to not
* return the value that was replaced but instead return {@code null}
* @return an instance of the map builder back
* @see #removeReturnsNull(boolean)
*/
@@ -474,17 +478,16 @@ public abstract class AbstractChronicleMapBuilder<K, V,
}
/**
* Configures if the maps created by this builder should return {@code null} instead of the last
* mapped value on {@link ChronicleMap#remove(Object) ChronicleMap.remove(key)} calls.
* Configures if the maps created by this builder should return {@code null} instead of the last mapped value on
* {@link ChronicleMap#remove(Object) ChronicleMap.remove(key)} calls.
*
* <p>{@link Map#remove(Object) Map.remove()} returns the previous value, functionality which is
* rarely used but fairly cheap for {@link HashMap}. In the case, for an off heap collection, it
* has to create a new object and deserialize the data from off-heap memory. It's expensive for
* something you probably don't use.
* <p>{@link Map#remove(Object) Map.remove()} returns the previous value, functionality which is rarely used but
* fairly cheap for {@link HashMap}. In the case, for an off heap collection, it has to create a new object and
* deserialize the data from off-heap memory. It's expensive for something you probably don't use.
* </p>
*
* @param removeReturnsNull {@code true} if you want {@link ChronicleMap#remove(Object)
* ChronicleMap.remove()} to not return the value of the removed entry
* but instead return {@code null}
* @param removeReturnsNull {@code true} if you want {@link ChronicleMap#remove(Object) ChronicleMap.remove()} to
* not return the value of the removed entry but instead return {@code null}
* @return an instance of the map builder back
* @see #putReturnsNull(boolean)
*/
@@ -505,7 +508,6 @@ public abstract class AbstractChronicleMapBuilder<K, V,
return self();
}
int metaDataBytes() {
return metaDataBytes;
}
@@ -540,14 +542,6 @@ public abstract class AbstractChronicleMapBuilder<K, V,
'}';
}
private static String pretty(int value) {
return value > 0 ? value + "" : "not configured";
}
private static String pretty(Object obj) {
return obj != null ? obj + "" : "not configured";
}
@SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
@Override
public boolean equals(Object o) {
@@ -668,19 +662,18 @@ public abstract class AbstractChronicleMapBuilder<K, V,
}
/**
* Configures factory which is used to create a new value instance, if value class is either
* {@link Byteable}, {@link BytesMarshallable} or {@link Externalizable} subclass in maps,
* created by this builder.
* Configures factory which is used to create a new value instance, if value class is either {@link Byteable},
* {@link BytesMarshallable} or {@link Externalizable} subclass in maps, created by this builder.
*
* <p>Default value deserialization factory is {@link NewInstanceObjectFactory}, which creates
* a new value instance using {@link Class#newInstance()} default constructor. You could provide
* an {@link AllocateInstanceObjectFactory}, which uses {@code Unsafe.allocateInstance(Class)}
* (you might want to do this for better performance or if you don't want to initialize fields),
* or a factory which calls a value class constructor with some arguments, or a factory which
* internally delegates to instance pool or {@link ThreadLocal}, to reduce allocations.
* <p>Default value deserialization factory is {@link NewInstanceObjectFactory}, which creates a new value instance
* using {@link Class#newInstance()} default constructor. You could provide an {@link
* AllocateInstanceObjectFactory}, which uses {@code Unsafe.allocateInstance(Class)} (you might want to do this for
* better performance or if you don't want to initialize fields), or a factory which calls a value class constructor
* with some arguments, or a factory which internally delegates to instance pool or {@link ThreadLocal}, to reduce
* allocations.
* </p>
*
* @param valueDeserializationFactory the value factory used to produce instances to deserialize
* data in
* @param valueDeserializationFactory the value factory used to produce instances to deserialize data in
* @return this builder back
*/
public B valueDeserializationFactory(@NotNull ObjectFactory<V> valueDeserializationFactory) {
@@ -698,14 +691,14 @@ public abstract class AbstractChronicleMapBuilder<K, V,
}
/**
* Specifies the value to be put for each key queried in {@link ChronicleMap#acquireUsing
* acquireUsing()} method, if the key is absent in the map, created by this builder.
* Specifies the value to be put for each key queried in {@link ChronicleMap#acquireUsing acquireUsing()} method, if
* the key is absent in the map, created by this builder.
*
* <p>This configuration overrides any previous {@link #defaultValueProvider(
* DefaultValueProvider)} configuration to this {@code AbstractChronicleMapBuilder}.
* <p>This configuration overrides any previous {@link #defaultValueProvider(DefaultValueProvider)} configuration
* to this {@code AbstractChronicleMapBuilder}.
* </p>
*
* @param defaultValue the default value to be put to the map for absent keys during {@code
* acquireUsing()} calls
* @param defaultValue the default value to be put to the map for absent keys during {@code acquireUsing()} calls
* @return this builder object back
*/
public B defaultValue(V defaultValue) {
@@ -718,11 +711,12 @@ public abstract class AbstractChronicleMapBuilder<K, V,
}
/**
* Specifies the function to obtain a value for the key during {@link ChronicleMap#acquireUsing
* acquireUsing()} calls, if the key is absent in the map, created by this builder.
* Specifies the function to obtain a value for the key during {@link ChronicleMap#acquireUsing acquireUsing()}
* calls, if the key is absent in the map, created by this builder.
*
* <p>This configuration overrides any previous {@link #defaultValue(Object)} configuration to
* this {@code AbstractChronicleMapBuilder}.
* <p>This configuration overrides any previous {@link #defaultValue(Object)} configuration to this {@code
* AbstractChronicleMapBuilder}.
* </p>
*
* @param defaultValueProvider the strategy to obtain a default value by the absent key
* @return this builder object back
@@ -876,7 +870,7 @@ public abstract class AbstractChronicleMapBuilder<K, V,
valueBuilder.constantSizeBySample(sampleValue);
}
private ChronicleMap<K, V> establishReplication(VanillaChronicleMap<K, ?, ?, V, ?, ?> map)
private ChronicleMap<K, V> establishReplication(VanillaChronicleMap<K, ?, ?, V, ?, ?> map)
throws IOException {
if (map instanceof ReplicatedChronicleMap) {
ReplicatedChronicleMap result = (ReplicatedChronicleMap) map;
@@ -919,5 +913,11 @@ public abstract class AbstractChronicleMapBuilder<K, V,
public byte identifier() {
return identifier;
}
public int segmentHeaderSize() {
int segments = actualSegments();
// reduce false sharing unless we have alot of segments.
return segments > 8192 ? 16 : segments > 1024 ? 32 : 64;
}
}
@@ -23,13 +23,13 @@ import net.openhft.chronicle.hash.TimeProvider;
import net.openhft.chronicle.hash.serialization.BytesReader;
import net.openhft.chronicle.hash.serialization.MetaBytesInterop;
import net.openhft.chronicle.hash.serialization.MetaBytesWriter;
import net.openhft.lang.threadlocal.ThreadLocalCopies;
import net.openhft.lang.Maths;
import net.openhft.lang.collection.ATSDirectBitSet;
import net.openhft.lang.io.Bytes;
import net.openhft.lang.io.MultiStoreBytes;
import net.openhft.lang.io.NativeBytes;
import net.openhft.lang.model.Byteable;
import net.openhft.lang.threadlocal.ThreadLocalCopies;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,9 +41,7 @@ import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.atomic.AtomicReferenceArray;
import static net.openhft.chronicle.hash.serialization.Hasher.hash;
import static net.openhft.lang.MemoryUnit.BITS;
import static net.openhft.lang.MemoryUnit.BYTES;
import static net.openhft.lang.MemoryUnit.CACHE_LINES;
import static net.openhft.lang.MemoryUnit.*;
import static net.openhft.lang.collection.DirectBitSet.NOT_FOUND;
/**
@@ -96,11 +94,11 @@ class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
Replica, Replica.EntryExternalizable, Replica.EntryResolver<K, V>, Closeable {
// for file, jdbc and UDP replication
public static final int RESERVED_MOD_ITER = 8;
public static final int ADDITIONAL_ENTRY_BYTES = 10;
static final int MAX_UNSIGNED_SHORT = Character.MAX_VALUE;
private static final long serialVersionUID = 0L;
private static final Logger LOG = LoggerFactory.getLogger(ReplicatedChronicleMap.class);
private static final long LAST_UPDATED_HEADER_SIZE = 128L * 8L;
public static final int ADDITIONAL_ENTRY_BYTES = 10;
private final TimeProvider timeProvider;
private final byte localIdentifier;
transient Set<Closeable> closeables;
@@ -127,8 +125,8 @@ class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
}
@Override
Segment createSegment(NativeBytes bytes, int index) {
return new Segment(bytes, index);
VanillaChronicleMap<K, KI, MKI, V, VW, MVW>.Segment createSegment(NativeBytes segmentHeader, NativeBytes bytes, int index) {
return new Segment(segmentHeader, bytes, index);
}
Class segmentType() {
@@ -584,8 +582,8 @@ class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
private volatile MultiMap hashLookupLiveAndDeleted;
private volatile MultiMap hashLookupLiveOnly;
Segment(NativeBytes bytes, int index) {
super(bytes, index);
Segment(NativeBytes segmentHeader, NativeBytes bytes, int index) {
super(segmentHeader, bytes, index);
}
@Override
@@ -1219,8 +1217,6 @@ class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
// the assigned modification iterators
private final ATSDirectBitSet modIterSet;
private final MapEventListener<K, V, ChronicleMap<K, V>> nextListener;
private final AtomicReferenceArray<ModificationIterator> modificationIterators =
new AtomicReferenceArray<ModificationIterator>(127 + RESERVED_MOD_ITER);
public ModificationDelegator(@NotNull final MapEventListener<K, V, ChronicleMap<K, V>> nextListener,
final Bytes bytes) {
@@ -1228,6 +1224,9 @@ class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
modIterSet = new ATSDirectBitSet(bytes);
}
private final AtomicReferenceArray<ModificationIterator> modificationIterators =
new AtomicReferenceArray<ModificationIterator>(127 + RESERVED_MOD_ITER);
public ModificationIterator acquireModificationIterator(
final short remoteIdentifier,
@NotNull final ModificationNotifier modificationNotifier) {
@@ -1322,8 +1321,8 @@ class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
}
/**
* Always throws {@code NotSerializableException} since instances of this class are not
* intended to be serializable.
* Always throws {@code NotSerializableException} since instances of this class are not intended to be
* serializable.
*/
private void writeObject(ObjectOutputStream out) throws IOException {
throw new NotSerializableException(getClass().getCanonicalName());
@@ -41,20 +41,27 @@ import static java.nio.channels.SelectionKey.*;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static net.openhft.chronicle.map.StatelessChronicleMap.EventId.HEARTBEAT;
interface Work {
/**
* @param in the buffer that we will fill up
* @return true when all the work is complete
*/
boolean doWork(@NotNull Bytes in);
}
/**
* Used with a {@see net.openhft.map.ReplicatedSharedHashMap} to send data between the maps using a
* socket connection {@see net.openhft.map.OutSocketReplicator}
* Used with a {@link net.openhft.chronicle.map.ReplicatedChronicleMap} to send data between the maps using a socket
* connection {@link net.openhft.chronicle.map.TcpReplicator}
*
* @author Rob Austin.
*/
public class TcpReplicator extends AbstractChannelReplicator implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName());
private static final int BUFFER_SIZE = 0x100000; // 1MB
public static final int STATELESS_CLIENT = -127;
public static final byte NOT_SET = (byte) HEARTBEAT.ordinal();
private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName());
private static final int BUFFER_SIZE = 0x100000; // 1MB
private final SelectionKey[] selectionKeysStore = new SelectionKey[Byte.MAX_VALUE + 1];
// used to instruct the selector thread to set OP_WRITE on a key correlated by the bit index in the
// bitset
@@ -78,7 +85,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
* the entry, but setting it too large reduces the workable
* space in the buffer.
* @param statelessServerConnector set to NULL if not required
* @throws IOException
* @throws IOException on an io error.
*/
public TcpReplicator(@NotNull final Replica replica,
@NotNull final Replica.EntryExternalizable externalizable,
@@ -271,8 +278,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
/**
* check to see if we have lost connection with the remote node and if we have attempts a
* reconnect.
* check to see if we have lost connection with the remote node and if we have attempts a reconnect.
*
* @param key the key relating to the heartbeat that we are checking
* @param approxTimeOutTime the approximate time in milliseconds
@@ -419,8 +425,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
/**
* this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear
* all the old bootstrap information
* this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear all the old
* bootstrap information
*
* @param key the nio SelectionKey
*/
@@ -432,8 +438,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
/**
* used to exchange identifiers and timestamps and heartbeat intervals between the server and
* client
* used to exchange identifiers and timestamps and heartbeat intervals between the server and client
*
* @param key the SelectionKey relating to the this cha
* @param socketChannel
@@ -504,7 +509,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
if (!attached.hasRemoteHeartbeatInterval) {
final long value = reader.remoteHeartbeatIntervalFromBuffer();
final long value = reader.remoteHeartbeatIntervalFromBuffer();
if (value == Long.MIN_VALUE)
return;
@@ -552,7 +557,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
attached.entryWriter.entriesToBuffer(attached.remoteModificationIterator, key);
try {
final int bytesJustWritten = attached.entryWriter.writeBufferToSocket(socketChannel,
final int bytesJustWritten = attached.entryWriter.writeBufferToSocket(socketChannel,
approxTime);
if (bytesJustWritten > 0)
@@ -600,11 +605,22 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
}
private ServerSocketChannel openServerSocketChannel() throws IOException {
ServerSocketChannel result = null;
try {
result = ServerSocketChannel.open();
} finally {
if (result != null)
closeables.add(result);
}
return result;
}
/**
* sets interestOps to "selector keys",The change to interestOps much be on the same thread as
* the selector. This class, allows via {@link AbstractChannelReplicator
* .KeyInterestUpdater#set(int)} to holds a pending change in interestOps ( via a bitset ),
* this change is processed later on the same thread as the selector
* sets interestOps to "selector keys",The change to interestOps much be on the same thread as the selector. This
* class, allows via {@link AbstractChannelReplicator .KeyInterestUpdater#set(int)} to holds a pending change in
* interestOps ( via a bitset ), this change is processed later on the same thread as the selector
*/
private static class KeyInterestUpdater {
@@ -635,8 +651,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
/**
* @param keyIndex the index of the key that has changed, the list of keys is provided by
* the constructor {@link KeyInterestUpdater(int, SelectionKey[])}
* @param keyIndex the index of the key that has changed, the list of keys is provided by the constructor {@link
* KeyInterestUpdater(int, SelectionKey[])}
*/
public void set(int keyIndex) {
changeOfOpWriteRequired.set(keyIndex);
@@ -703,18 +719,6 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
}
private ServerSocketChannel openServerSocketChannel() throws IOException {
ServerSocketChannel result = null;
try {
result = ServerSocketChannel.open();
} finally {
if (result != null)
closeables.add(result);
}
return result;
}
private class ClientConnector extends AbstractConnector {
private final Details details;
@@ -812,9 +816,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
public boolean hasRemoteHeartbeatInterval;
// true if its socket is a ServerSocket
public boolean isServer; // the frequency the remote node will send a heartbeat
public long remoteHeartbeatInterval = heartBeatIntervalMillis;
public boolean handShakingComplete;
public long remoteHeartbeatInterval = heartBeatIntervalMillis;
boolean isHandShakingComplete() {
return handShakingComplete;
@@ -840,6 +843,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
TcpReplicator.this.opWriteUpdater.set(remoteIdentifier);
}
}
/**
@@ -847,14 +851,13 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
*/
private class TcpSocketChannelEntryWriter {
private final ByteBuffer out;
final ByteBufferBytes in;
private final ByteBuffer out;
private final EntryCallback entryCallback;
private long lastSentTime;
// if uncompletedWork is set ( not null ) , this must be completed before any further work
// is carried out
public Work uncompletedWork;
private long lastSentTime;
private TcpSocketChannelEntryWriter() {
out = ByteBuffer.allocateDirect(replicationConfig.packetSize() + maxEntrySizeBytes);
@@ -890,8 +893,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
/**
* writes all the entries that have changed, to the buffer which will later be written to
* TCP/IP
* writes all the entries that have changed, to the buffer which will later be written to TCP/IP
*
* @param modificationIterator a record of which entries have modification
* @param selectionKey
@@ -969,8 +971,7 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
/**
* used to send an single zero byte if we have not send any data for up to the
* localHeartbeatInterval
* used to send an single zero byte if we have not send any data for up to the localHeartbeatInterval
*/
private void writeHeartbeatToBuffer() {
@@ -987,8 +988,8 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
/**
* removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE
* will get added back in as soon as we have data to write
* removes back in the OP_WRITE from the selector, otherwise it'll spin loop. The OP_WRITE will get added back
* in as soon as we have data to write
*
* @param socketChannel the socketChannel we wish to stop writing to
* @param attached data associated with the socketChannels key
@@ -1023,10 +1024,9 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
*/
class TcpSocketChannelEntryReader {
public static final int HEADROOM = 1024;
public long lastHeartBeatReceived = System.currentTimeMillis();
ByteBuffer in;
ByteBufferBytes out;
public long lastHeartBeatReceived = System.currentTimeMillis();
private long sizeInBytes;
private byte state;
@@ -1218,28 +1218,16 @@ public class TcpReplicator extends AbstractChannelReplicator implements Closeabl
}
}
interface Work {
/**
* @param in the buffer that we will fill up
* @return true when all the work is complete
*/
boolean doWork(@NotNull Bytes in);
}
/**
* @author Rob Austin.
*/
class StatelessServerConnector<K, V> {
private static final Logger LOG = LoggerFactory.getLogger(StatelessServerConnector.class
.getName());
public static final int TRANSACTION_ID_OFFSET = AbstractChannelReplicator.SIZE_OF_SIZE + 1;
public static final boolean MAP_SUPPORTS_BYTES = Boolean.valueOf(System.getProperty
("mapSupportsBytes"));
private static final Logger LOG = LoggerFactory.getLogger(StatelessServerConnector.class
.getName());
private final KeyValueSerializer<K, V> keyValueSerializer;
private final VanillaChronicleMap<K, ?, ?, V, ?, ?> map;
private double maxEntrySizeBytes;
@@ -20,17 +20,15 @@ package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.ChronicleHashErrorListener;
import net.openhft.chronicle.hash.serialization.*;
import net.openhft.lang.threadlocal.Provider;
import net.openhft.lang.threadlocal.ThreadLocalCopies;
import net.openhft.lang.collection.DirectBitSet;
import net.openhft.lang.collection.SingleThreadedDirectBitSet;
import net.openhft.lang.io.*;
import net.openhft.lang.io.serialization.JDKObjectSerializer;
import net.openhft.lang.io.serialization.ObjectFactory;
import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory;
import net.openhft.lang.threadlocal.Provider;
import net.openhft.lang.threadlocal.ThreadLocalCopies;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -50,36 +48,28 @@ import static net.openhft.lang.MemoryUnit.*;
class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
V, VW, MVW extends MetaBytesWriter<V, VW>> extends AbstractMap<K, V>
implements ChronicleMap<K, V>, Serializable {
private static final long serialVersionUID = 2L;
/**
* Because DirectBitSet implementations couldn't find more than 64 continuous clear or set
* bits.
*/
static final int MAX_ENTRY_OVERSIZE_FACTOR = 64;
private static final int SEGMENT_HEADER = 64;
static final ThreadLocal<MultiStoreBytes> tmpBytes = new ThreadLocal<>();
private static final long serialVersionUID = 2L;
final Class<K> kClass;
final SizeMarshaller keySizeMarshaller;
final BytesReader<K> originalKeyReader;
transient Provider<BytesReader<K>> keyReaderProvider;
final KI originalKeyInterop;
transient Provider<KI> keyInteropProvider;
final MKI originalMetaKeyInterop;
final MetaProvider<K, KI, MKI> metaKeyInteropProvider;
final Class<V> vClass;
final SizeMarshaller valueSizeMarshaller;
final BytesReader<V> originalValueReader;
transient Provider<BytesReader<V>> valueReaderProvider;
final VW originalValueWriter;
transient Provider<VW> valueWriterProvider;
final MVW originalMetaValueWriter;
final MetaProvider<V, VW, MVW> metaValueWriterProvider;
final ObjectFactory<V> valueFactory;
final DefaultValueProvider<K, V> defaultValueProvider;
final PrepareValueBytesAsWriter<K> prepareValueBytesAsWriter;
final int metaDataBytes;
// private final int replicas;
final long entrySize;
@@ -91,17 +81,20 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
// rather than as returning the Object can be expensive for something you probably don't use.
final boolean putReturnsNull;
final boolean removeReturnsNull;
private final long lockTimeOutNS;
private final ChronicleHashErrorListener errorListener;
private final int segmentHeaderSize;
private final HashSplitting hashSplitting;
transient Provider<BytesReader<K>> keyReaderProvider;
transient Provider<KI> keyInteropProvider;
transient Provider<BytesReader<V>> valueReaderProvider;
transient Provider<VW> valueWriterProvider;
transient Segment[] segments; // non-final for close()
// non-final for close() and because it is initialized out of constructor
transient BytesStore ms;
transient long headerSize;
transient Set<Map.Entry<K, V>> entrySet;
private final HashSplitting hashSplitting;
public VanillaChronicleMap(AbstractChronicleMapBuilder<K, V, ?> builder) throws IOException {
SerializationBuilder<K> keyBuilder = builder.keyBuilder;
@@ -140,6 +133,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
this.entriesPerSegment = builder.actualEntriesPerSegment();
this.metaDataBytes = builder.metaDataBytes();
this.eventListener = builder.eventListener();
this.segmentHeaderSize = builder.segmentHeaderSize();
hashSplitting = HashSplitting.Splitting.forSegments(actualSegments);
initTransients();
@@ -185,9 +179,17 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
onHeaderCreated();
long offset = getHeaderSize();
long segmentHeaderSize = getSegmentHeaderSize();
long segmentHeadersSize = segments.length * segmentHeaderSize;
Bytes segmentHeaders = ms.bytes(offset, segmentHeadersSize);
offset += segmentHeadersSize;
long segmentSize = segmentSize();
long headerOffset = 0;
for (int i = 0; i < this.segments.length; i++) {
this.segments[i] = createSegment((NativeBytes) ms.bytes(offset, segmentSize), i);
NativeBytes segmentHeader = (NativeBytes) segmentHeaders.bytes(headerOffset, segmentHeaderSize);
NativeBytes segmentData = (NativeBytes) ms.bytes(offset, segmentSize);
this.segments[i] = createSegment(segmentHeader, segmentData, i);
headerOffset += segmentHeaderSize;
offset += segmentSize;
}
return offset;
@@ -214,8 +216,12 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
return headerSize;
}
Segment createSegment(NativeBytes bytes, int index) {
return new Segment(bytes, index);
long getSegmentHeaderSize() {
return segmentHeaderSize;
}
Segment createSegment(NativeBytes segmentHeader, NativeBytes bytes, int index) {
return new Segment(segmentHeader, bytes, index);
}
@Override
@@ -224,7 +230,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
}
long sizeInBytes() {
return getHeaderSize() + segments.length * segmentSize();
return getHeaderSize()
+ segments.length * getSegmentHeaderSize()
+ segments.length * segmentSize();
}
long sizeOfMultiMap() {
@@ -254,8 +262,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
}
long segmentSize() {
long ss = SEGMENT_HEADER
+ (CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES)
long ss = (CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES)
* multiMapsPerSegment())
+ numberOfBitSets() * sizeOfBitSets() // the free list and 0+ dirty lists.
+ sizeOfEntriesInSegment();
@@ -393,7 +400,6 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
return (entrySet != null) ? entrySet : (entrySet = new EntrySet());
}
/**
* @throws NullPointerException if the specified key is null
*/
@@ -412,7 +418,6 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
return removeIfValueIs(key, (V) value) != null;
}
/**
* removes ( if there exists ) an entry from the map, if the {@param key} and {@param
* expectedValue} match that of a maps.entry. If the {@param expectedValue} equals null then (
@@ -446,7 +451,6 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
return oldValue.equals(replaceIfValueIs(key, oldValue, newValue));
}
/**
* @throws NullPointerException if the specified key or value is null
*/
@@ -512,8 +516,6 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
return valueSize;
}
static final ThreadLocal<MultiStoreBytes> tmpBytes = new ThreadLocal<>();
protected MultiStoreBytes acquireTmpBytes() {
MultiStoreBytes b = tmpBytes.get();
if (b == null)
@@ -567,7 +569,12 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
throw new UnsupportedOperationException("todo");
}
long[] segmentSizes() {
long[] sizes = new long[segments.length];
for (int i = 0; i < segments.length; i++)
sizes[i] = segments[i].getSize();
return sizes;
}
// these methods should be package local, not public or private.
class Segment implements SharedSegment {
@@ -582,6 +589,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
static final long LOCK_OFFSET = 0L; // 64-bit
static final long SIZE_OFFSET = LOCK_OFFSET + 8L; // 32-bit
final NativeBytes segmentHeader;
final NativeBytes bytes;
final long entriesOffset;
private final int index;
@@ -593,11 +601,12 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
/**
* @param index the index of this segment held by the map
*/
Segment(NativeBytes bytes, int index) {
Segment(NativeBytes segmentHeader, NativeBytes bytes, int index) {
this.segmentHeader = segmentHeader;
this.bytes = bytes;
this.index = index;
long start = bytes.startAddr() + SEGMENT_HEADER;
long start = bytes.startAddr();
createHashLookups(start);
start += CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES)
* multiMapsPerSegment();
@@ -647,18 +656,18 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
* increments the size by one
*/
void incrementSize() {
this.bytes.addLong(SIZE_OFFSET, 1L);
this.segmentHeader.addLong(SIZE_OFFSET, 1L);
}
void resetSize() {
this.bytes.writeLong(SIZE_OFFSET, 0L);
this.segmentHeader.writeLong(SIZE_OFFSET, 0L);
}
/**
* decrements the size by one
*/
void decrementSize() {
this.bytes.addLong(SIZE_OFFSET, -1L);
this.segmentHeader.addLong(SIZE_OFFSET, -1L);
}
/**
@@ -666,39 +675,39 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
*/
long getSize() {
// any negative value is in error state.
return max(0L, this.bytes.readVolatileLong(SIZE_OFFSET));
return max(0L, this.segmentHeader.readVolatileLong(SIZE_OFFSET));
}
public void readLock() throws IllegalStateException {
while (true) {
final boolean success = bytes.tryRWReadLock(LOCK_OFFSET, lockTimeOutNS);
final boolean success = segmentHeader.tryRWReadLock(LOCK_OFFSET, lockTimeOutNS);
if (success) return;
if (currentThread().isInterrupted()) {
throw new IllegalStateException(new InterruptedException("Unable to obtain lock, interrupted"));
} else {
errorListener.onLockTimeout(bytes.threadIdForLockLong(LOCK_OFFSET));
bytes.resetLockLong(LOCK_OFFSET);
errorListener.onLockTimeout(segmentHeader.threadIdForLockLong(LOCK_OFFSET));
segmentHeader.resetLockLong(LOCK_OFFSET);
}
}
}
public void writeLock() throws IllegalStateException {
while (true) {
final boolean success = bytes.tryRWWriteLock(LOCK_OFFSET, lockTimeOutNS);
final boolean success = segmentHeader.tryRWWriteLock(LOCK_OFFSET, lockTimeOutNS);
if (success) return;
if (currentThread().isInterrupted()) {
throw new IllegalStateException(new InterruptedException("Unable to obtain lock, interrupted"));
} else {
errorListener.onLockTimeout(bytes.threadIdForLockLong(LOCK_OFFSET));
bytes.resetLockLong(LOCK_OFFSET);
errorListener.onLockTimeout(segmentHeader.threadIdForLockLong(LOCK_OFFSET));
segmentHeader.resetLockLong(LOCK_OFFSET);
}
}
}
public void readUnlock() {
try {
bytes.unlockRWReadLock(LOCK_OFFSET);
segmentHeader.unlockRWReadLock(LOCK_OFFSET);
} catch (IllegalMonitorStateException e) {
errorListener.errorOnUnlock(e);
}
@@ -706,7 +715,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
public void writeUnlock() {
try {
bytes.unlockRWWriteLock(LOCK_OFFSET);
segmentHeader.unlockRWWriteLock(LOCK_OFFSET);
} catch (IllegalMonitorStateException e) {
errorListener.errorOnUnlock(e);
}
@@ -991,12 +1000,12 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
/**
* Removes a key (or key-value pair) from the Segment.
*
* <p>The entry will only be removed if {@code expectedValue} equals to {@code null} or the
* value previously corresponding to the specified key.
* <p>The entry will only be removed if {@code expectedValue} equals to {@code null} or the value previously
* corresponding to the specified key.
*
* @param hash2 a hash code related to the {@code keyBytes}
* @return the value of the entry that was removed if the entry corresponding to the {@code
* keyBytes} exists and {@link #removeReturnsNull} is {@code false}, {@code null} otherwise
* @return the value of the entry that was removed if the entry corresponding to the {@code keyBytes} exists and
* {@link #removeReturnsNull} is {@code false}, {@code null} otherwise
*/
V remove(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key,
V expectedValue, long hash2) {
@@ -1052,9 +1061,8 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
}
/**
* Replaces the specified value for the key with the given value. {@code newValue} is set
* only if the existing value corresponding to the specified key is equal to {@code
* expectedValue} or {@code expectedValue == null}.
* Replaces the specified value for the key with the given value. {@code newValue} is set only if the existing
* value corresponding to the specified key is equal to {@code expectedValue} or {@code expectedValue == null}.
*
* @param hash2 a hash code related to the {@code keyBytes}
* @return the replaced value or {@code null} if the value was not replaced
@@ -1130,17 +1138,15 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
}
/**
* Replaces value in existing entry. May cause entry relocation, because there may be not
* enough space for new value in location already allocated for this entry.
* Replaces value in existing entry. May cause entry relocation, because there may be not enough space for new
* value in location already allocated for this entry.
*
* @param pos index of the first block occupied by the entry
* @param offset relative offset of the entry in Segment bytes (before, i. e.
* including metaData)
* @param offset relative offset of the entry in Segment bytes (before, i. e. including metaData)
* @param entry relative pointer in Segment bytes
* @param valueSizePos relative position of value size in entry
* @param entryEndAddr absolute address of the entry end
* @return relative offset of the entry in Segment bytes after putting value (that may cause
* entry relocation)
* @return relative offset of the entry in Segment bytes after putting value (that may cause entry relocation)
*/
long putValue(long pos, long offset, NativeBytes entry, long valueSizePos,
long entryEndAddr, ThreadLocalCopies copies, V value, Bytes valueBytes,
@@ -1472,11 +1478,4 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
return super.setValue(value);
}
}
long[] segmentSizes() {
long[] sizes = new long[segments.length];
for (int i = 0; i < segments.length; i++)
sizes[i] = segments[i].getSize();
return sizes;
}
}