From 7d1e4dd2dfdffb1d7df1c34ecda23b2967be5760 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 7 Oct 2014 00:52:26 +0400 Subject: [PATCH 1/7] HCOLL-156 changed default value object factory from AllocateInstanceObjectFactory to NewInstanceObjectFactory --- .../openhft/chronicle/map/SerializationBuilder.java | 9 ++++++--- .../map/fromdocs/OpenJDKAndHashMapExamplesTest.java | 13 ++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index 6728d638f..afdefe5f6 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -63,10 +63,10 @@ final class SerializationBuilder implements Cloneable { this.bufferIdentity = role; this.eClass = eClass; - ObjectFactory factory = this.factory; - if (factory == null && role == Role.VALUE) { + ObjectFactory factory = null; + if (role == Role.VALUE) { factory = concreteClass(eClass) && marshallerUseFactory(eClass) ? - new AllocateInstanceObjectFactory(eClass) : + new NewInstanceObjectFactory(eClass) : NullObjectFactory.INSTANCE; } @@ -85,6 +85,9 @@ final class SerializationBuilder implements Cloneable { if (marshaller != null) marshaller(marshaller, factory); } + if (role == Role.VALUE && concreteClass(eClass) && marshallerUseFactory(eClass)) { + factory(factory); + } } @SuppressWarnings("unchecked") diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java index dbf61accc..7c90b209e 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java @@ -52,15 +52,11 @@ public class OpenJDKAndHashMapExamplesTest { @Test public void bondExample() throws IOException, InterruptedException { - Class bondVODirectClass = directClassFor(BondVOInterface.class); - NewInstanceObjectFactory bondVODirectObjectFactory = - new NewInstanceObjectFactory(bondVODirectClass); File file = new File(TMP + "/chm-myBondPortfolioCHM-" + System.nanoTime()); file.deleteOnExit(); ChronicleMap chm = ChronicleMapBuilder - .of(String.class, bondVODirectClass) - .valueFactory(bondVODirectObjectFactory) - .entrySize(512) + .of(String.class, directClassFor(BondVOInterface.class)) + .keySize(10) .create(file); @@ -80,9 +76,8 @@ public class OpenJDKAndHashMapExamplesTest { ChronicleMap chmB = ChronicleMapBuilder - .of(String.class, bondVODirectClass) - .valueFactory(bondVODirectObjectFactory) - .entrySize(320) + .of(String.class, directClassFor(BondVOInterface.class)) + .keySize(10) .create(file); // ZERO Copy but creates a new off heap reference each time From 9b27b8a7580946e98eb590280ce132199553c14d Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 7 Oct 2014 00:56:03 +0400 Subject: [PATCH 2/7] Memoize hash computation in CopyingMetaBytesInterop --- .../map/serialization/CopyingMetaBytesInterop.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java b/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java index 1c1922a8f..a8fc3d368 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java @@ -34,6 +34,7 @@ public abstract class CopyingMetaBytesInterop implements MetaBytesInterop< final DirectBytesBuffer buffer; transient long size; + transient long hash; transient W writer; transient E cur; @@ -49,6 +50,7 @@ public abstract class CopyingMetaBytesInterop implements MetaBytesInterop< innerWrite(writer, buffer, e); buffer.flip(); size = buffer.remaining(); + hash = 0L; } } @@ -66,7 +68,10 @@ public abstract class CopyingMetaBytesInterop implements MetaBytesInterop< @Override public long hash(W writer, E e) { - return Hasher.hash(buffer.buffer); + long h; + if ((h = hash) == 0L) + return hash = Hasher.hash(buffer.buffer); + return h; } @Override From fdd5988d6ffcdbc5e415fae4634ee612b41f969d Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 7 Oct 2014 01:25:09 +0400 Subject: [PATCH 3/7] Made keys immutability configurable --- .../chronicle/map/ChronicleMapBuilder.java | 16 ++++++++++++++++ .../chronicle/map/SerializationBuilder.java | 17 ++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index aa33102f8..b0d7da343 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -622,6 +622,22 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Specifies that key objects, queried with the maps created by this builder, are inherently + * immutable. Keys in {@code ChronicleMap} are not required to be immutable, as in ordinary + * {@link Map} implementations, because they are serialized off-heap. However, + * {@code ChronicleMap} implementations can benefit from the knowledge that keys are not mutated + * between queries. By default, {@code ChronicleMapBuilder} detects immutability automatically + * only for very few standard JDK types (for example, for {@link String}), it is not recommended + * to rely on {@code ChronicleMapBuilder} to be smart enough about this. + * + * @return this builder back + */ + public ChronicleMapBuilder immutableKeys() { + keyBuilder.instancesAreMutable(false); + return this; + } + public ChronicleMapBuilder valueMarshallerAndFactory( @NotNull BytesMarshaller valueMarshaller, @NotNull ObjectFactory valueFactory) { valueBuilder.marshaller(valueMarshaller, valueFactory); diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index afdefe5f6..5e57e8f12 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -45,11 +45,16 @@ final class SerializationBuilder implements Cloneable { Externalizable.class.isAssignableFrom(c); } + private static boolean instancesAreMutable(Class c) { + return c != String.class; + } + enum Role {KEY, VALUE} private enum CopyingInterop {FROM_MARSHALLER, FROM_WRITER} private final Serializable bufferIdentity; final Class eClass; + private boolean instancesAreMutable; private SizeMarshaller sizeMarshaller = stopBit(); private BytesReader reader; private Object interop; @@ -62,6 +67,7 @@ final class SerializationBuilder implements Cloneable { SerializationBuilder(Class eClass, Role role) { this.bufferIdentity = role; this.eClass = eClass; + instancesAreMutable = instancesAreMutable(eClass); ObjectFactory factory = null; if (role == Role.VALUE) { @@ -144,13 +150,13 @@ final class SerializationBuilder implements Cloneable { } public SerializationBuilder maxSize(long maxSize) { - boolean mutable = eClass != String.class; // for example if (copyingInterop == CopyingInterop.FROM_MARSHALLER) { metaInteropProvider(CopyingMetaBytesInterop - .>providerForBytesMarshaller(mutable, maxSize)); + .>providerForBytesMarshaller( + instancesAreMutable, maxSize)); } else if (copyingInterop == CopyingInterop.FROM_WRITER) { metaInteropProvider(CopyingMetaBytesInterop - .>providerForBytesWriter(mutable, maxSize)); + .>providerForBytesWriter(instancesAreMutable, maxSize)); } return this; } @@ -165,6 +171,11 @@ final class SerializationBuilder implements Cloneable { return this; } + public SerializationBuilder instancesAreMutable(boolean mutable) { + this.instancesAreMutable = mutable; + return this; + } + public SizeMarshaller sizeMarshaller() { return sizeMarshaller; } From 0cf883eaded2473b8a662ac7aa427a622046d5b1 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 7 Oct 2014 03:26:38 +0400 Subject: [PATCH 4/7] Removed ChronicleMapBuilder.alignedEntrySize() in favor of simply entrySize() --- .../net/openhft/chronicle/map/ChronicleMapBuilder.java | 8 ++------ .../net/openhft/chronicle/map/VanillaChronicleMap.java | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index b0d7da343..c7d4ba894 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -126,7 +126,7 @@ public class ChronicleMapBuilder implements Cloneable { private int tryMinSegments(int min, int max) { for (int i = min; i < max; i <<= 1) { - if (i * i * i >= alignedEntrySize() * 2) + if (i * i * i >= entrySize() * 2) return i; } return max; @@ -292,7 +292,7 @@ public class ChronicleMapBuilder implements Cloneable { */ public int entrySize() { if (entrySize > 0) - return entrySize; + return entryAndValueAlignment().alignSize(entrySize); int size = metaDataBytes; int keySize = keySize(); size += keyBuilder.sizeMarshaller().sizeEncodingSize(keySize); @@ -318,10 +318,6 @@ public class ChronicleMapBuilder implements Cloneable { return entryAndValueAlignment().alignSize(size); } - int alignedEntrySize() { - return entryAndValueAlignment().alignSize(entrySize()); - } - /** * Configures alignment strategy of address in memory of entries and independently of address * in memory of values within entries in ChronicleMaps, created by this builder. diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index c35bad84f..13f445976 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -124,7 +124,7 @@ class VanillaChronicleMap, lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); // this.replicas = builder.replicas(); - this.entrySize = builder.alignedEntrySize(); + this.entrySize = builder.entrySize(); this.alignment = builder.entryAndValueAlignment(); this.errorListener = builder.errorListener(); From b6f00d110b8c4b297a6cb295fcfcfb43e2d5e9cb Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 7 Oct 2014 05:14:32 +0400 Subject: [PATCH 5/7] Fixed a bug in ByteArrayMarshaller --- .../chronicle/map/serialization/impl/ByteArrayMarshaller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java b/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java index 0fa9c4e26..9b84ec7d9 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java @@ -58,7 +58,7 @@ public enum ByteArrayMarshaller implements BytesInterop, BytesReader Date: Tue, 7 Oct 2014 05:15:01 +0400 Subject: [PATCH 6/7] Fixed a bug in LongMarshaller --- .../chronicle/map/serialization/impl/LongMarshaller.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/serialization/impl/LongMarshaller.java b/src/main/java/net/openhft/chronicle/map/serialization/impl/LongMarshaller.java index 2506058e9..0ad365acf 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/impl/LongMarshaller.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/impl/LongMarshaller.java @@ -25,7 +25,7 @@ public enum LongMarshaller implements AgileBytesMarshaller { @Override public long size(Long e) { - return 4L; + return 8L; } @Override @@ -55,7 +55,7 @@ public enum LongMarshaller implements AgileBytesMarshaller { @Override public long readSize(Bytes bytes) { - return 4L; + return 8L; } @Override From c98bd1d5bc889f3b3f048cddc027812b9dbbd427 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 7 Oct 2014 05:27:28 +0400 Subject: [PATCH 7/7] Added ChronicleMapBuilder.constantKey[Value]SizeBySample() methods --- .../chronicle/map/ChronicleMapBuilder.java | 102 +++++++++++--- .../chronicle/map/SerializationBuilder.java | 46 ++++++- .../map/serialization/SizeMarshallers.java | 29 ++++ .../map/ConstantSizeBySampleTest.java | 128 ++++++++++++++++++ 4 files changed, 285 insertions(+), 20 deletions(-) create mode 100644 src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index c7d4ba894..b7aa0831b 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -58,7 +58,9 @@ public class ChronicleMapBuilder implements Cloneable { // used when reading the number of entries per private int actualEntriesPerSegment = -1; private int keySize = 0; + private K sampleKey; private int valueSize = 0; + private V sampleValue; private int entrySize = 0; private Alignment alignment = Alignment.OF_4_BYTES; private long entries = 1 << 20; @@ -133,11 +135,9 @@ public class ChronicleMapBuilder implements Cloneable { } /** - * Configures the number of bytes, taken by serialized form of keys, put into maps, created by - * this builder. - * - *

If key is a boxed primitive type or {@link Byteable} subclass, i. e. if key size is known - * statically, it is automatically accounted and shouldn't be specified by user. + * Configures the optimal number of bytes, taken by serialized form of keys, put into maps, + * created by this builder. If key size is always the same, call + * {@link #constantKeySizeBySample(Object)} method instead of this one. * *

If key size varies moderately, specify the size higher than average, but lower than * the maximum possible, to minimize average memory overuse. If key size varies in a wide range, @@ -157,6 +157,7 @@ public class ChronicleMapBuilder implements Cloneable { * * @param keySize number of bytes, taken by serialized form of keys * @return this {@code ChronicleMapBuilder} back + * @see #constantKeySizeBySample(Object) * @see #valueSize(int) * @see #entrySize() */ @@ -167,13 +168,42 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Configures the constant number of bytes, taken by serialized form of keys, put into maps, + * created by this builder. This is done by providing the {@code sampleKey}, all keys should + * take the same number of bytes in serialized form, as this sample object. + * + *

For example, if your keys are Git commit hashes:

{@code
+     * Map gitCommitMessagesByHash =
+     *     ChronicleMapBuilder.of(byte[].class, String.class)
+     *     .constantKeySizeBySample(new byte[20])
+     *     .create();
+     * }
+ * + *

If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size + * is known statically, it is automatically accounted and this method shouldn't be called. + * + *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be + * called instead of this one. + * + * @param sampleKey the sample key + * @return this builder back + * @see #keySize(int) + * @see #constantValueSizeBySample(Object) + */ + public ChronicleMapBuilder constantKeySizeBySample(K sampleKey) { + this.sampleKey = sampleKey; + return this; + } + private int keySize() { return keyOrValueSize(keySize, keyBuilder); } /** - * Configures the number of bytes, taken by serialized form of value, put into maps, created by - * this builder. + * Configures the optimal number of bytes, taken by serialized form of values, put into maps, + * created by this builder. If value size is always the same, call + * {@link #constantValueSizeBySample(Object)} method instead of this one. * *

If value is a boxed primitive type or {@link Byteable} subclass, i. e. if value size * is known statically, it is automatically accounted and shouldn't be specified by user. @@ -195,6 +225,27 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Configures the constant number of bytes, taken by serialized form of values, put into maps, + * created by this builder. This is done by providing the {@code sampleValue}, all values should + * take the same number of bytes in serialized form, as this sample object. + * + *

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size + * is known statically, it is automatically accounted and this method shouldn't be called. + * + *

If value size varies, method {@link #valueSize(int)} or {@link #entrySize(int)} should be + * called instead of this one. + * + * @param sampleValue the sample value + * @return this builder back + * @see #valueSize(int) + * @see #constantKeySizeBySample(Object) + */ + public ChronicleMapBuilder constantValueSizeBySample(V sampleValue) { + this.sampleValue = sampleValue; + return this; + } + private int valueSize() { return keyOrValueSize(valueSize, valueBuilder); } @@ -252,9 +303,10 @@ public class ChronicleMapBuilder implements Cloneable { * implementations, with respect to {@linkplain #entryAndValueAlignment() alignment}. * *

There are three major patterns of this configuration usage: - *

    - *
  • Key and value sizes are known exactly. Just specify them using precisely - * corresponding methods. No memory would be wasted in this case.
    • + *
      + *
    1. Key and value sizes are constant. Configure them via {@link #constantKeySizeBySample( + * Object)} and {@link #constantValueSizeBySample(Object)} methods, and you will experience + * no memory waste at all.
      2. *
    2. Key and/or value size varies moderately. Specify them using corresponding methods, * or {@linkplain #entrySize(int) specify entry size} directly, by values somewhere between * average and maximum possible. The idea is to have most (90% or more) entries to fit @@ -265,12 +317,18 @@ public class ChronicleMapBuilder implements Cloneable { * from 5 to several dozens of "chunks". With this approach, average memory waste should be * very low. * - *

      However, remember that operations with entries that span several "entry sizes" - * are a bit slower, than entries which take a single "entry size" (that is why "chunk" - * approach is not recommended, when key and/or value size varies moderately). Also, note - * that the maximum number of chunks could be taken by an entry is 64. - * {@link IllegalArgumentException} is thrown on attempt to insert too large entry, compared - * to the configured or computed entry size. + *

      However, remember that + *

        + *
      • Operations with entries that span several "entry sizes" are a bit slower, than + * entries which take a single "entry size". That is why "chunk" approach + * is not recommended, when key and/or value size varies moderately.
        • + *
      • The maximum number of chunks could be taken by an entry is 64. + * {@link IllegalArgumentException} is thrown on attempt to insert too large entry, + * compared to the configured or computed entry size.
        • + *
      • The number of "entries" {@linkplain #entries(long) in the whole map} and + * {@linkplain #actualEntriesPerSegment() per segment} is actually the number + * "chunks".
        • + *
      * *

      Example: if values in your map are adjacency lists of some social graph, where nodes * are represented as {@code long} ids, and adjacency lists are serialized in efficient @@ -279,16 +337,19 @@ public class ChronicleMapBuilder implements Cloneable { * be a good choice: 

      {@code
      * Map socialGraph = ChronicleMapBuilder
      *     .of(Long.class, long[].class)
-     *     .entries(1_000_000_000)
+     *     // given that graph should have of 1 billion nodes, and 150 average adjacency list size
+     *     // => values takes 3 chuncks on average
+     *     .entries(1_000_000_000L * (150 / 50))
      *     .entrySize(50 * 8)
      *     .create();}
      * It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max * chunks by single entry" limit, and ensures moderate average memory overuse (not more * than 20%). *
      3. - *
+ * * * @return size of memory allocation unit (in bytes) + * @see #entries(long) */ public int entrySize() { if (entrySize > 0) @@ -789,6 +850,11 @@ public class ChronicleMapBuilder implements Cloneable { int maxSize = entrySize() * figureBufferAllocationFactor(); keyBuilder.maxSize(maxSize); valueBuilder.maxSize(maxSize); + + if (sampleKey != null) + keyBuilder.constantSizeBySample(sampleKey); + if (sampleValue != null) + valueBuilder.constantSizeBySample(sampleValue); } private ChronicleMap establishReplication(ChronicleMap map) diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index 5e57e8f12..3451c2c0a 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -18,6 +18,8 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.map.serialization.*; import net.openhft.chronicle.map.serialization.impl.*; +import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.*; import net.openhft.lang.io.serialization.impl.*; @@ -61,6 +63,7 @@ final class SerializationBuilder implements Cloneable { private CopyingInterop copyingInterop = null; private MetaBytesInterop metaInterop; private MetaProvider metaInteropProvider; + private long maxSize; private ObjectFactory factory; @SuppressWarnings("unchecked") @@ -132,7 +135,6 @@ final class SerializationBuilder implements Cloneable { public SerializationBuilder writer(BytesWriter writer) { return copyingInterop(CopyingInterop.FROM_WRITER) - .sizeMarshaller(stopBit()) .setInterop(writer) .metaInterop(CopyingMetaBytesInterop .>forBytesWriter(bufferIdentity)); @@ -142,7 +144,6 @@ final class SerializationBuilder implements Cloneable { ObjectFactory factory) { if (factory == null) factory = this.factory; return copyingInterop(CopyingInterop.FROM_MARSHALLER) - .sizeMarshaller(stopBit()) .reader(BytesReaders.fromBytesMarshaller(marshaller), factory) .setInterop(marshaller) .metaInterop(CopyingMetaBytesInterop @@ -151,16 +152,57 @@ final class SerializationBuilder implements Cloneable { public SerializationBuilder maxSize(long maxSize) { if (copyingInterop == CopyingInterop.FROM_MARSHALLER) { + this.maxSize = maxSize; metaInteropProvider(CopyingMetaBytesInterop .>providerForBytesMarshaller( instancesAreMutable, maxSize)); } else if (copyingInterop == CopyingInterop.FROM_WRITER) { + this.maxSize = maxSize; metaInteropProvider(CopyingMetaBytesInterop .>providerForBytesWriter(instancesAreMutable, maxSize)); } return this; } + public SerializationBuilder constantSizeBySample(E sampleObject) { + Object originalInterop = this.interop; + Provider interopProvider = Provider.of(originalInterop.getClass()); + ThreadLocalCopies copies = interopProvider.getCopies(null); + Object interop = interopProvider.get(copies, originalInterop); + copies = metaInteropProvider.getCopies(copies); + MetaBytesWriter metaInterop; + if (maxSize > 0) { + // this loop is very dumb: tries to find buffer size, sufficient to serialize + // the object, by trying to serialize to small buffers first and doubling the buffer + // size on _any_ Exception (it may be IllegalArgument, IndexOutOfBounds, IllegalState, + // see JLANG-17 issue). + // TODO The question is: couldn't this cause JVM crash without throwing an exception? + findSufficientSerializationSize: + while (true) { + MetaProvider metaInteropProvider = this.metaInteropProvider; + try { + metaInterop = metaInteropProvider.get( + copies, this.metaInterop, interop, sampleObject); + break findSufficientSerializationSize; + } catch (Exception e) { + // assuming nobody need > 512 mb _constant size_ keys/values + if (maxSize > 512 * 1024 * 1024) { + throw e; + } + maxSize(maxSize * 2); + } + } + } else { + MetaProvider metaInteropProvider = this.metaInteropProvider; + metaInterop = metaInteropProvider.get(copies, this.metaInterop, interop, sampleObject); + } + long constantSize = metaInterop.size(interop, sampleObject); + // set max size once more, because current maxSize could be x64 or x2 larger than needed + maxSize(constantSize); + sizeMarshaller(SizeMarshallers.constant(constantSize)); + return this; + } + public SerializationBuilder objectSerializer(ObjectSerializer serializer) { if (reader == null || interop == null) { ObjectFactory factory = this.factory; diff --git a/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java b/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java index 6a26114b7..d5fdbc810 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java @@ -47,5 +47,34 @@ public final class SizeMarshallers { } } + public static SizeMarshaller constant(long size) { + return new ConstantSizeMarshaller(size); + } + + private static class ConstantSizeMarshaller implements SizeMarshaller { + private static final long serialVersionUID = 0L; + + private final long size; + + private ConstantSizeMarshaller(long size) { + this.size = size; + } + + @Override + public int sizeEncodingSize(long size) { + return 0; + } + + @Override + public void writeSize(Bytes bytes, long size) { + // do nothing + } + + @Override + public long readSize(Bytes bytes) { + return size; + } + } + private SizeMarshallers() {} } diff --git a/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java b/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java new file mode 100644 index 000000000..46b902b24 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java @@ -0,0 +1,128 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.Ignore; +import org.junit.Test; +import org.testng.Assert; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +public class ConstantSizeBySampleTest { + + @Test + public void testConstantKeys() throws IOException { + ChronicleMap map = ChronicleMapBuilder.of(byte[].class, Long.class) + .constantKeySizeBySample(new byte[8]) + .immutableKeys() + .entries(100) + .create(); + + byte[] zero = ByteBuffer.allocate(8).putLong(0L).array(); + map.put(zero, 0L); + assertEquals(0L, (long) map.get(zero)); + + byte[] one = ByteBuffer.allocate(8).putLong(1L).array(); + map.put(one, 1L); + assertEquals(1L, (long) map.get(one)); + + map.put(one, 0L); + assertEquals(0L, (long) map.get(one)); + } + + @Test + public void testUnexpectedlyLongConstantByteArrayValues() throws IOException { + ChronicleMap map = ChronicleMapBuilder.of(Long.class, byte[].class) + .constantValueSizeBySample(new byte[512 * 1024]) + .entries(100) + .actualSegments(1) + .create(); + + byte[] value = new byte[512 * 1024]; + + value[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } + + static class ExternalizableData implements Externalizable { + byte[] data = new byte[512 * 1024]; + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExternalizableData)) + return false; + return Arrays.equals(((ExternalizableData) obj).data, data); + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.write(data); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + in.read(data = new byte[512 * 1024]); + } + } + + @Test + public void testUnexpectedlyLongConstantExternalizableValues() throws IOException { + ChronicleMap map = + ChronicleMapBuilder.of(Long.class, ExternalizableData.class) + .constantValueSizeBySample(new ExternalizableData()) + .entries(100) + .actualSegments(1) + .create(); + + ExternalizableData value = new ExternalizableData(); + value.data[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } + + static class SerializableData implements Serializable { + byte[] data = new byte[512 * 1024]; + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExternalizableData)) + return false; + return Arrays.equals(((ExternalizableData) obj).data, data); + } + } + + @Test + @Ignore + public void testUnexpectedlyLongConstantSerializableValues() throws IOException { + ChronicleMap map = + ChronicleMapBuilder.of(Long.class, SerializableData.class) + .constantValueSizeBySample(new SerializableData()) + .entries(100) + .actualSegments(1) + .create(); + + SerializableData value = new SerializableData(); + value.data[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } +}