Merge remote-tracking branch 'origin/master'

src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java

@@ -58,7 +58,9 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
     // used when reading the number of entries per
     private int actualEntriesPerSegment = -1;
     private int keySize = 0;
+    private K sampleKey;
     private int valueSize = 0;
+    private V sampleValue;
     private int entrySize = 0;
     private Alignment alignment = Alignment.OF_4_BYTES;
     private long entries = 1 << 20;
@@ -126,18 +128,16 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
 
     private int tryMinSegments(int min, int max) {
         for (int i = min; i < max; i <<= 1) {
-            if (i * i * i >= alignedEntrySize() * 2)
+            if (i * i * i >= entrySize() * 2)
                 return i;
         }
         return max;
     }
 
     /**
-     * Configures the number of bytes, taken by serialized form of keys, put into maps, created by
-     * this builder.
-     *
-     * <p>If key is a boxed primitive type or {@link Byteable} subclass, i. e. if key size is known
-     * statically, it is automatically accounted and shouldn't be specified by user.
+     * Configures the optimal number of bytes, taken by serialized form of keys, put into maps,
+     * created by this builder. If key size is always the same, call
+     * {@link #constantKeySizeBySample(Object)} method instead of this one.
      *
      * <p>If key size varies moderately, specify the size higher than average, but lower than
      * the maximum possible, to minimize average memory overuse. If key size varies in a wide range,
@@ -157,6 +157,7 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
      *
      * @param keySize number of bytes, taken by serialized form of keys
      * @return this {@code ChronicleMapBuilder} back
+     * @see #constantKeySizeBySample(Object)
      * @see #valueSize(int)
      * @see #entrySize()
      */
@@ -167,13 +168,42 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
         return this;
     }
 
+    /**
+     * Configures the constant number of bytes, taken by serialized form of keys, put into maps,
+     * created by this builder. This is done by providing the {@code sampleKey}, all keys should
+     * take the same number of bytes in serialized form, as this sample object.
+     *
+     * <p>For example, if your keys are Git commit hashes:<pre>{@code
+     * Map<byte[], String> gitCommitMessagesByHash =
+     *     ChronicleMapBuilder.of(byte[].class, String.class)
+     *     .constantKeySizeBySample(new byte[20])
+     *     .create();
+     * }</pre>
+     *
+     * <p>If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size
+     * is known statically, it is automatically accounted and this method shouldn't be called.
+     *
+     * <p>If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be
+     * called instead of this one.
+     *
+     * @param sampleKey the sample key
+     * @return this builder back
+     * @see #keySize(int)
+     * @see #constantValueSizeBySample(Object)
+     */
+    public ChronicleMapBuilder<K, V> constantKeySizeBySample(K sampleKey) {
+        this.sampleKey = sampleKey;
+        return this;
+    }
+
     private int keySize() {
         return keyOrValueSize(keySize, keyBuilder);
     }
 
     /**
-     * Configures the number of bytes, taken by serialized form of value, put into maps, created by
-     * this builder.
+     * Configures the optimal number of bytes, taken by serialized form of values, put into maps,
+     * created by this builder. If value size is always the same, call
+     * {@link #constantValueSizeBySample(Object)} method instead of this one.
      *
      * <p>If value is a boxed primitive type or {@link Byteable} subclass, i. e. if value size
      * is known statically, it is automatically accounted and shouldn't be specified by user.
@@ -195,6 +225,27 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
         return this;
     }
 
+    /**
+     * Configures the constant number of bytes, taken by serialized form of values, put into maps,
+     * created by this builder. This is done by providing the {@code sampleValue}, all values should
+     * take the same number of bytes in serialized form, as this sample object.
+     *
+     * <p>If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size
+     * is known statically, it is automatically accounted and this method shouldn't be called.
+     *
+     * <p>If value size varies, method {@link #valueSize(int)} or {@link #entrySize(int)} should be
+     * called instead of this one.
+     *
+     * @param sampleValue the sample value
+     * @return this builder back
+     * @see #valueSize(int)
+     * @see #constantKeySizeBySample(Object)
+     */
+    public ChronicleMapBuilder<K, V> constantValueSizeBySample(V sampleValue) {
+        this.sampleValue = sampleValue;
+        return this;
+    }
+
     private int valueSize() {
         return keyOrValueSize(valueSize, valueBuilder);
     }
@@ -252,9 +303,10 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
      * implementations, with respect to {@linkplain #entryAndValueAlignment() alignment}.
      *
      * <p>There are three major patterns of this configuration usage:
-     * <ul>
-     *     <li>Key and value sizes are known exactly. Just specify them using precisely
-     *     corresponding methods. No memory would be wasted in this case.</li>
+     * <ol>
+     *     <li>Key and value sizes are constant. Configure them via {@link #constantKeySizeBySample(
+     *     Object)} and {@link #constantValueSizeBySample(Object)} methods, and you will experience
+     *     no memory waste at all.</li>
      *     <li>Key and/or value size varies moderately. Specify them using corresponding methods,
      *     or {@linkplain #entrySize(int) specify entry size} directly, by values somewhere between
      *     average and maximum possible. The idea is to have most (90% or more) entries to fit
@@ -265,12 +317,18 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
      *     from 5 to several dozens of "chunks". With this approach, average memory waste should be
      *     very low.
      *
-     *     <p>However, remember that operations with entries that span several "entry sizes"
-     *     are a bit slower, than entries which take a single "entry size" (that is why "chunk"
-     *     approach is not recommended, when key and/or value size varies moderately). Also, note
-     *     that the maximum number of chunks could be taken by an entry is 64.
-     *     {@link IllegalArgumentException} is thrown on attempt to insert too large entry, compared
-     *     to the configured or computed entry size.
+     *     <p>However, remember that
+     *     <ul>
+     *         <li>Operations with entries that span several "entry sizes" are a bit slower, than
+     *         entries which take a single "entry size". That is why "chunk" approach
+     *         is not recommended, when key and/or value size varies moderately.</li>
+     *         <li>The maximum number of chunks could be taken by an entry is 64.
+     *         {@link IllegalArgumentException} is thrown on attempt to insert too large entry,
+     *         compared to the configured or computed entry size.</li>
+     *         <li>The number of "entries" {@linkplain #entries(long) in the whole map} and
+     *         {@linkplain #actualEntriesPerSegment() per segment} is actually the number
+     *         "chunks".</li>
+     *     </ul>
      *
      *     <p>Example: if values in your map are adjacency lists of some social graph, where nodes
      *     are represented as {@code long} ids, and adjacency lists are serialized in efficient
@@ -279,20 +337,23 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
      *     be a good choice: <pre>{@code
      * Map<Long, long[]> socialGraph = ChronicleMapBuilder
      *     .of(Long.class, long[].class)
-     *     .entries(1_000_000_000)
+     *     // given that graph should have of 1 billion nodes, and 150 average adjacency list size
+     *     // => values takes 3 chuncks on average
+     *     .entries(1_000_000_000L * (150 / 50))
      *     .entrySize(50 * 8)
      *     .create();}</pre>
      *     It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max
      *     chunks by single entry" limit, and ensures moderate average memory overuse (not more
      *     than 20%).
      *     </li>
-     * </ul>
+     * </ol>
      *
      * @return size of memory allocation unit (in bytes)
+     * @see #entries(long)
      */
     public int entrySize() {
         if (entrySize > 0)
-            return entrySize;
+            return entryAndValueAlignment().alignSize(entrySize);
         int size = metaDataBytes;
         int keySize = keySize();
         size += keyBuilder.sizeMarshaller().sizeEncodingSize(keySize);
@@ -318,10 +379,6 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
         return entryAndValueAlignment().alignSize(size);
     }
 
-    int alignedEntrySize() {
-        return entryAndValueAlignment().alignSize(entrySize());
-    }
-
     /**
      * Configures alignment strategy of address in memory of entries and independently of address
      * in memory of values within entries in ChronicleMaps, created by this builder.
@@ -622,6 +679,22 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
         return this;
     }
 
+    /**
+     * Specifies that key objects, queried with the maps created by this builder, are inherently
+     * immutable. Keys in {@code ChronicleMap} are not required to be immutable, as in ordinary
+     * {@link Map} implementations, because they are serialized off-heap. However,
+     * {@code ChronicleMap} implementations can benefit from the knowledge that keys are not mutated
+     * between queries. By default, {@code ChronicleMapBuilder} detects immutability automatically
+     * only for very few standard JDK types (for example, for {@link String}), it is not recommended
+     * to rely on {@code ChronicleMapBuilder} to be smart enough about this.
+     *
+     * @return this builder back
+     */
+    public ChronicleMapBuilder<K, V> immutableKeys() {
+        keyBuilder.instancesAreMutable(false);
+        return this;
+    }
+
     public ChronicleMapBuilder<K, V> valueMarshallerAndFactory(
             @NotNull BytesMarshaller<V> valueMarshaller, @NotNull ObjectFactory<V> valueFactory) {
         valueBuilder.marshaller(valueMarshaller, valueFactory);
@@ -777,6 +850,11 @@ public class ChronicleMapBuilder<K, V> implements Cloneable {
         int maxSize = entrySize() * figureBufferAllocationFactor();
         keyBuilder.maxSize(maxSize);
         valueBuilder.maxSize(maxSize);
+
+        if (sampleKey != null)
+            keyBuilder.constantSizeBySample(sampleKey);
+        if (sampleValue != null)
+            valueBuilder.constantSizeBySample(sampleValue);
     }
 
     private ChronicleMap<K, V> establishReplication(ChronicleMap<K, V> map)

src/main/java/net/openhft/chronicle/map/SerializationBuilder.java

@@ -18,6 +18,8 @@ package net.openhft.chronicle.map;
 
 import net.openhft.chronicle.map.serialization.*;
 import net.openhft.chronicle.map.serialization.impl.*;
+import net.openhft.chronicle.map.threadlocal.Provider;
+import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies;
 import net.openhft.lang.io.Bytes;
 import net.openhft.lang.io.serialization.*;
 import net.openhft.lang.io.serialization.impl.*;
@@ -45,28 +47,35 @@ final class SerializationBuilder<E> implements Cloneable {
                 Externalizable.class.isAssignableFrom(c);
     }
 
+    private static boolean instancesAreMutable(Class c) {
+        return c != String.class;
+    }
+
     enum Role {KEY, VALUE}
     private enum CopyingInterop {FROM_MARSHALLER, FROM_WRITER}
 
     private final Serializable bufferIdentity;
     final Class<E> eClass;
+    private boolean instancesAreMutable;
     private SizeMarshaller sizeMarshaller = stopBit();
     private BytesReader<E> reader;
     private Object interop;
     private CopyingInterop copyingInterop = null;
     private MetaBytesInterop<E, ?> metaInterop;
     private MetaProvider<E, ?, ?> metaInteropProvider;
+    private long maxSize;
     private ObjectFactory<E> factory;
 
     @SuppressWarnings("unchecked")
     SerializationBuilder(Class<E> eClass, Role role) {
         this.bufferIdentity = role;
         this.eClass = eClass;
+        instancesAreMutable = instancesAreMutable(eClass);
 
-        ObjectFactory<E> factory = this.factory;
-        if (factory == null && role == Role.VALUE) {
+        ObjectFactory<E> factory = null;
+        if (role == Role.VALUE) {
             factory = concreteClass(eClass) && marshallerUseFactory(eClass) ?
-                    new AllocateInstanceObjectFactory(eClass) :
+                    new NewInstanceObjectFactory<E>(eClass) :
                     NullObjectFactory.INSTANCE;
         }
 
@@ -85,6 +94,9 @@ final class SerializationBuilder<E> implements Cloneable {
             if (marshaller != null)
                 marshaller(marshaller, factory);
         }
+        if (role == Role.VALUE && concreteClass(eClass) && marshallerUseFactory(eClass)) {
+            factory(factory);
+        }
     }
 
     @SuppressWarnings("unchecked")
@@ -123,7 +135,6 @@ final class SerializationBuilder<E> implements Cloneable {
 
     public SerializationBuilder<E> writer(BytesWriter<E> writer) {
         return copyingInterop(CopyingInterop.FROM_WRITER)
-                .sizeMarshaller(stopBit())
                 .setInterop(writer)
                 .metaInterop(CopyingMetaBytesInterop
                         .<E, BytesWriter<E>>forBytesWriter(bufferIdentity));
@@ -133,7 +144,6 @@ final class SerializationBuilder<E> implements Cloneable {
                                               ObjectFactory<E> factory) {
         if (factory == null) factory = this.factory;
         return copyingInterop(CopyingInterop.FROM_MARSHALLER)
-                .sizeMarshaller(stopBit())
                 .reader(BytesReaders.fromBytesMarshaller(marshaller), factory)
                 .setInterop(marshaller)
                 .metaInterop(CopyingMetaBytesInterop
@@ -141,17 +151,58 @@ final class SerializationBuilder<E> implements Cloneable {
     }
 
     public SerializationBuilder<E> maxSize(long maxSize) {
-        boolean mutable = eClass != String.class; // for example
         if (copyingInterop == CopyingInterop.FROM_MARSHALLER) {
+            this.maxSize = maxSize;
             metaInteropProvider(CopyingMetaBytesInterop
-                    .<E, BytesMarshaller<E>>providerForBytesMarshaller(mutable, maxSize));
+                    .<E, BytesMarshaller<E>>providerForBytesMarshaller(
+                            instancesAreMutable, maxSize));
         } else if (copyingInterop == CopyingInterop.FROM_WRITER) {
+            this.maxSize = maxSize;
             metaInteropProvider(CopyingMetaBytesInterop
-                    .<E, BytesWriter<E>>providerForBytesWriter(mutable, maxSize));
+                    .<E, BytesWriter<E>>providerForBytesWriter(instancesAreMutable, maxSize));
         }
         return this;
     }
 
+    public SerializationBuilder<E> constantSizeBySample(E sampleObject) {
+        Object originalInterop = this.interop;
+        Provider interopProvider = Provider.of(originalInterop.getClass());
+        ThreadLocalCopies copies = interopProvider.getCopies(null);
+        Object interop = interopProvider.get(copies, originalInterop);
+        copies = metaInteropProvider.getCopies(copies);
+        MetaBytesWriter metaInterop;
+        if (maxSize > 0) {
+            // this loop is very dumb: tries to find buffer size, sufficient to serialize
+            // the object, by trying to serialize to small buffers first and doubling the buffer
+            // size on _any_ Exception (it may be IllegalArgument, IndexOutOfBounds, IllegalState,
+            // see JLANG-17 issue).
+            // TODO The question is: couldn't this cause JVM crash without throwing an exception?
+            findSufficientSerializationSize:
+            while (true) {
+                MetaProvider metaInteropProvider = this.metaInteropProvider;
+                try {
+                    metaInterop = metaInteropProvider.get(
+                            copies, this.metaInterop, interop, sampleObject);
+                    break findSufficientSerializationSize;
+                } catch (Exception e) {
+                    // assuming nobody need > 512 mb _constant size_ keys/values
+                    if (maxSize > 512 * 1024 * 1024) {
+                        throw e;
+                    }
+                    maxSize(maxSize * 2);
+                }
+            }
+        } else {
+            MetaProvider metaInteropProvider = this.metaInteropProvider;
+            metaInterop = metaInteropProvider.get(copies, this.metaInterop, interop, sampleObject);
+        }
+        long constantSize = metaInterop.size(interop, sampleObject);
+        // set max size once more, because current maxSize could be x64 or x2 larger than needed
+        maxSize(constantSize);
+        sizeMarshaller(SizeMarshallers.constant(constantSize));
+        return this;
+    }
+
     public SerializationBuilder<E> objectSerializer(ObjectSerializer serializer) {
         if (reader == null || interop == null) {
             ObjectFactory<E> factory = this.factory;
@@ -162,6 +213,11 @@ final class SerializationBuilder<E> implements Cloneable {
         return this;
     }
 
+    public SerializationBuilder<E> instancesAreMutable(boolean mutable) {
+        this.instancesAreMutable = mutable;
+        return this;
+    }
+
     public SizeMarshaller sizeMarshaller() {
         return sizeMarshaller;
     }

src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java

@@ -124,7 +124,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, KI>,
         lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS);
 
         //  this.replicas = builder.replicas();
-        this.entrySize = builder.alignedEntrySize();
+        this.entrySize = builder.entrySize();
         this.alignment = builder.entryAndValueAlignment();
 
         this.errorListener = builder.errorListener();

src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java

@@ -34,6 +34,7 @@ public abstract class CopyingMetaBytesInterop<E, W> implements MetaBytesInterop<
 
     final DirectBytesBuffer buffer;
     transient long size;
+    transient long hash;
     transient W writer;
     transient E cur;
 
@@ -49,6 +50,7 @@ public abstract class CopyingMetaBytesInterop<E, W> implements MetaBytesInterop<
             innerWrite(writer, buffer, e);
             buffer.flip();
             size = buffer.remaining();
+            hash = 0L;
         }
     }
 
@@ -66,7 +68,10 @@ public abstract class CopyingMetaBytesInterop<E, W> implements MetaBytesInterop<
 
     @Override
     public long hash(W writer, E e) {
-        return Hasher.hash(buffer.buffer);
+        long h;
+        if ((h = hash) == 0L)
+            return hash = Hasher.hash(buffer.buffer);
+        return h;
     }
 
     @Override

src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java

@@ -47,5 +47,34 @@ public final class SizeMarshallers {
         }
     }
 
+    public static SizeMarshaller constant(long size) {
+        return new ConstantSizeMarshaller(size);
+    }
+
+    private static class ConstantSizeMarshaller implements SizeMarshaller {
+        private static final long serialVersionUID = 0L;
+
+        private final long size;
+
+        private ConstantSizeMarshaller(long size) {
+            this.size = size;
+        }
+
+        @Override
+        public int sizeEncodingSize(long size) {
+            return 0;
+        }
+
+        @Override
+        public void writeSize(Bytes bytes, long size) {
+            // do nothing
+        }
+
+        @Override
+        public long readSize(Bytes bytes) {
+            return size;
+        }
+    }
+
     private SizeMarshallers() {}
 }

src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java

@@ -58,7 +58,7 @@ public enum ByteArrayMarshaller implements BytesInterop<byte[]>, BytesReader<byt
 
     @Override
     public byte[] read(Bytes bytes, long size, byte[] ba) {
-        if (size != ba.length)
+        if (ba == null || size != ba.length)
             ba = new byte[(int) size];
         bytes.read(ba);
         return ba;

src/main/java/net/openhft/chronicle/map/serialization/impl/LongMarshaller.java

@@ -25,7 +25,7 @@ public enum LongMarshaller implements AgileBytesMarshaller<Long> {
 
     @Override
     public long size(Long e) {
-        return 4L;
+        return 8L;
     }
 
     @Override
@@ -55,7 +55,7 @@ public enum LongMarshaller implements AgileBytesMarshaller<Long> {
 
     @Override
     public long readSize(Bytes bytes) {
-        return 4L;
+        return 8L;
     }
 
     @Override