From c98bd1d5bc889f3b3f048cddc027812b9dbbd427 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 7 Oct 2014 05:27:28 +0400 Subject: [PATCH] Added ChronicleMapBuilder.constantKey[Value]SizeBySample() methods --- .../chronicle/map/ChronicleMapBuilder.java | 102 +++++++++++--- .../chronicle/map/SerializationBuilder.java | 46 ++++++- .../map/serialization/SizeMarshallers.java | 29 ++++ .../map/ConstantSizeBySampleTest.java | 128 ++++++++++++++++++ 4 files changed, 285 insertions(+), 20 deletions(-) create mode 100644 src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index c7d4ba894..b7aa0831b 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -58,7 +58,9 @@ public class ChronicleMapBuilder implements Cloneable { // used when reading the number of entries per private int actualEntriesPerSegment = -1; private int keySize = 0; + private K sampleKey; private int valueSize = 0; + private V sampleValue; private int entrySize = 0; private Alignment alignment = Alignment.OF_4_BYTES; private long entries = 1 << 20; @@ -133,11 +135,9 @@ public class ChronicleMapBuilder implements Cloneable { } /** - * Configures the number of bytes, taken by serialized form of keys, put into maps, created by - * this builder. - * - *

If key is a boxed primitive type or {@link Byteable} subclass, i. e. if key size is known - * statically, it is automatically accounted and shouldn't be specified by user. + * Configures the optimal number of bytes, taken by serialized form of keys, put into maps, + * created by this builder. If key size is always the same, call + * {@link #constantKeySizeBySample(Object)} method instead of this one. * *

If key size varies moderately, specify the size higher than average, but lower than * the maximum possible, to minimize average memory overuse. If key size varies in a wide range, @@ -157,6 +157,7 @@ public class ChronicleMapBuilder implements Cloneable { * * @param keySize number of bytes, taken by serialized form of keys * @return this {@code ChronicleMapBuilder} back + * @see #constantKeySizeBySample(Object) * @see #valueSize(int) * @see #entrySize() */ @@ -167,13 +168,42 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Configures the constant number of bytes, taken by serialized form of keys, put into maps, + * created by this builder. This is done by providing the {@code sampleKey}, all keys should + * take the same number of bytes in serialized form, as this sample object. + * + *

For example, if your keys are Git commit hashes:

{@code
+     * Map gitCommitMessagesByHash =
+     *     ChronicleMapBuilder.of(byte[].class, String.class)
+     *     .constantKeySizeBySample(new byte[20])
+     *     .create();
+     * }
+ * + *

If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size + * is known statically, it is automatically accounted and this method shouldn't be called. + * + *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be + * called instead of this one. + * + * @param sampleKey the sample key + * @return this builder back + * @see #keySize(int) + * @see #constantValueSizeBySample(Object) + */ + public ChronicleMapBuilder constantKeySizeBySample(K sampleKey) { + this.sampleKey = sampleKey; + return this; + } + private int keySize() { return keyOrValueSize(keySize, keyBuilder); } /** - * Configures the number of bytes, taken by serialized form of value, put into maps, created by - * this builder. + * Configures the optimal number of bytes, taken by serialized form of values, put into maps, + * created by this builder. If value size is always the same, call + * {@link #constantValueSizeBySample(Object)} method instead of this one. * *

If value is a boxed primitive type or {@link Byteable} subclass, i. e. if value size * is known statically, it is automatically accounted and shouldn't be specified by user. @@ -195,6 +225,27 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Configures the constant number of bytes, taken by serialized form of values, put into maps, + * created by this builder. This is done by providing the {@code sampleValue}, all values should + * take the same number of bytes in serialized form, as this sample object. + * + *

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size + * is known statically, it is automatically accounted and this method shouldn't be called. + * + *

If value size varies, method {@link #valueSize(int)} or {@link #entrySize(int)} should be + * called instead of this one. + * + * @param sampleValue the sample value + * @return this builder back + * @see #valueSize(int) + * @see #constantKeySizeBySample(Object) + */ + public ChronicleMapBuilder constantValueSizeBySample(V sampleValue) { + this.sampleValue = sampleValue; + return this; + } + private int valueSize() { return keyOrValueSize(valueSize, valueBuilder); } @@ -252,9 +303,10 @@ public class ChronicleMapBuilder implements Cloneable { * implementations, with respect to {@linkplain #entryAndValueAlignment() alignment}. * *

There are three major patterns of this configuration usage: - *

    - *
  • Key and value sizes are known exactly. Just specify them using precisely - * corresponding methods. No memory would be wasted in this case.
    • + *
      + *
    1. Key and value sizes are constant. Configure them via {@link #constantKeySizeBySample( + * Object)} and {@link #constantValueSizeBySample(Object)} methods, and you will experience + * no memory waste at all.
      2. *
    2. Key and/or value size varies moderately. Specify them using corresponding methods, * or {@linkplain #entrySize(int) specify entry size} directly, by values somewhere between * average and maximum possible. The idea is to have most (90% or more) entries to fit @@ -265,12 +317,18 @@ public class ChronicleMapBuilder implements Cloneable { * from 5 to several dozens of "chunks". With this approach, average memory waste should be * very low. * - *

      However, remember that operations with entries that span several "entry sizes" - * are a bit slower, than entries which take a single "entry size" (that is why "chunk" - * approach is not recommended, when key and/or value size varies moderately). Also, note - * that the maximum number of chunks could be taken by an entry is 64. - * {@link IllegalArgumentException} is thrown on attempt to insert too large entry, compared - * to the configured or computed entry size. + *

      However, remember that + *

        + *
      • Operations with entries that span several "entry sizes" are a bit slower, than + * entries which take a single "entry size". That is why "chunk" approach + * is not recommended, when key and/or value size varies moderately.
        • + *
      • The maximum number of chunks could be taken by an entry is 64. + * {@link IllegalArgumentException} is thrown on attempt to insert too large entry, + * compared to the configured or computed entry size.
        • + *
      • The number of "entries" {@linkplain #entries(long) in the whole map} and + * {@linkplain #actualEntriesPerSegment() per segment} is actually the number + * "chunks".
        • + *
      * *

      Example: if values in your map are adjacency lists of some social graph, where nodes * are represented as {@code long} ids, and adjacency lists are serialized in efficient @@ -279,16 +337,19 @@ public class ChronicleMapBuilder implements Cloneable { * be a good choice: 

      {@code
      * Map socialGraph = ChronicleMapBuilder
      *     .of(Long.class, long[].class)
-     *     .entries(1_000_000_000)
+     *     // given that graph should have of 1 billion nodes, and 150 average adjacency list size
+     *     // => values takes 3 chuncks on average
+     *     .entries(1_000_000_000L * (150 / 50))
      *     .entrySize(50 * 8)
      *     .create();}
      * It is minimum possible (because 3000 friends / 50 friends = 60 is close to 64 "max * chunks by single entry" limit, and ensures moderate average memory overuse (not more * than 20%). *
      3. - *
+ * * * @return size of memory allocation unit (in bytes) + * @see #entries(long) */ public int entrySize() { if (entrySize > 0) @@ -789,6 +850,11 @@ public class ChronicleMapBuilder implements Cloneable { int maxSize = entrySize() * figureBufferAllocationFactor(); keyBuilder.maxSize(maxSize); valueBuilder.maxSize(maxSize); + + if (sampleKey != null) + keyBuilder.constantSizeBySample(sampleKey); + if (sampleValue != null) + valueBuilder.constantSizeBySample(sampleValue); } private ChronicleMap establishReplication(ChronicleMap map) diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index 5e57e8f12..3451c2c0a 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -18,6 +18,8 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.map.serialization.*; import net.openhft.chronicle.map.serialization.impl.*; +import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.*; import net.openhft.lang.io.serialization.impl.*; @@ -61,6 +63,7 @@ final class SerializationBuilder implements Cloneable { private CopyingInterop copyingInterop = null; private MetaBytesInterop metaInterop; private MetaProvider metaInteropProvider; + private long maxSize; private ObjectFactory factory; @SuppressWarnings("unchecked") @@ -132,7 +135,6 @@ final class SerializationBuilder implements Cloneable { public SerializationBuilder writer(BytesWriter writer) { return copyingInterop(CopyingInterop.FROM_WRITER) - .sizeMarshaller(stopBit()) .setInterop(writer) .metaInterop(CopyingMetaBytesInterop .>forBytesWriter(bufferIdentity)); @@ -142,7 +144,6 @@ final class SerializationBuilder implements Cloneable { ObjectFactory factory) { if (factory == null) factory = this.factory; return copyingInterop(CopyingInterop.FROM_MARSHALLER) - .sizeMarshaller(stopBit()) .reader(BytesReaders.fromBytesMarshaller(marshaller), factory) .setInterop(marshaller) .metaInterop(CopyingMetaBytesInterop @@ -151,16 +152,57 @@ final class SerializationBuilder implements Cloneable { public SerializationBuilder maxSize(long maxSize) { if (copyingInterop == CopyingInterop.FROM_MARSHALLER) { + this.maxSize = maxSize; metaInteropProvider(CopyingMetaBytesInterop .>providerForBytesMarshaller( instancesAreMutable, maxSize)); } else if (copyingInterop == CopyingInterop.FROM_WRITER) { + this.maxSize = maxSize; metaInteropProvider(CopyingMetaBytesInterop .>providerForBytesWriter(instancesAreMutable, maxSize)); } return this; } + public SerializationBuilder constantSizeBySample(E sampleObject) { + Object originalInterop = this.interop; + Provider interopProvider = Provider.of(originalInterop.getClass()); + ThreadLocalCopies copies = interopProvider.getCopies(null); + Object interop = interopProvider.get(copies, originalInterop); + copies = metaInteropProvider.getCopies(copies); + MetaBytesWriter metaInterop; + if (maxSize > 0) { + // this loop is very dumb: tries to find buffer size, sufficient to serialize + // the object, by trying to serialize to small buffers first and doubling the buffer + // size on _any_ Exception (it may be IllegalArgument, IndexOutOfBounds, IllegalState, + // see JLANG-17 issue). + // TODO The question is: couldn't this cause JVM crash without throwing an exception? + findSufficientSerializationSize: + while (true) { + MetaProvider metaInteropProvider = this.metaInteropProvider; + try { + metaInterop = metaInteropProvider.get( + copies, this.metaInterop, interop, sampleObject); + break findSufficientSerializationSize; + } catch (Exception e) { + // assuming nobody need > 512 mb _constant size_ keys/values + if (maxSize > 512 * 1024 * 1024) { + throw e; + } + maxSize(maxSize * 2); + } + } + } else { + MetaProvider metaInteropProvider = this.metaInteropProvider; + metaInterop = metaInteropProvider.get(copies, this.metaInterop, interop, sampleObject); + } + long constantSize = metaInterop.size(interop, sampleObject); + // set max size once more, because current maxSize could be x64 or x2 larger than needed + maxSize(constantSize); + sizeMarshaller(SizeMarshallers.constant(constantSize)); + return this; + } + public SerializationBuilder objectSerializer(ObjectSerializer serializer) { if (reader == null || interop == null) { ObjectFactory factory = this.factory; diff --git a/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java b/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java index 6a26114b7..d5fdbc810 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java @@ -47,5 +47,34 @@ public final class SizeMarshallers { } } + public static SizeMarshaller constant(long size) { + return new ConstantSizeMarshaller(size); + } + + private static class ConstantSizeMarshaller implements SizeMarshaller { + private static final long serialVersionUID = 0L; + + private final long size; + + private ConstantSizeMarshaller(long size) { + this.size = size; + } + + @Override + public int sizeEncodingSize(long size) { + return 0; + } + + @Override + public void writeSize(Bytes bytes, long size) { + // do nothing + } + + @Override + public long readSize(Bytes bytes) { + return size; + } + } + private SizeMarshallers() {} } diff --git a/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java b/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java new file mode 100644 index 000000000..46b902b24 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java @@ -0,0 +1,128 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.Ignore; +import org.junit.Test; +import org.testng.Assert; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +public class ConstantSizeBySampleTest { + + @Test + public void testConstantKeys() throws IOException { + ChronicleMap map = ChronicleMapBuilder.of(byte[].class, Long.class) + .constantKeySizeBySample(new byte[8]) + .immutableKeys() + .entries(100) + .create(); + + byte[] zero = ByteBuffer.allocate(8).putLong(0L).array(); + map.put(zero, 0L); + assertEquals(0L, (long) map.get(zero)); + + byte[] one = ByteBuffer.allocate(8).putLong(1L).array(); + map.put(one, 1L); + assertEquals(1L, (long) map.get(one)); + + map.put(one, 0L); + assertEquals(0L, (long) map.get(one)); + } + + @Test + public void testUnexpectedlyLongConstantByteArrayValues() throws IOException { + ChronicleMap map = ChronicleMapBuilder.of(Long.class, byte[].class) + .constantValueSizeBySample(new byte[512 * 1024]) + .entries(100) + .actualSegments(1) + .create(); + + byte[] value = new byte[512 * 1024]; + + value[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } + + static class ExternalizableData implements Externalizable { + byte[] data = new byte[512 * 1024]; + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExternalizableData)) + return false; + return Arrays.equals(((ExternalizableData) obj).data, data); + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.write(data); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + in.read(data = new byte[512 * 1024]); + } + } + + @Test + public void testUnexpectedlyLongConstantExternalizableValues() throws IOException { + ChronicleMap map = + ChronicleMapBuilder.of(Long.class, ExternalizableData.class) + .constantValueSizeBySample(new ExternalizableData()) + .entries(100) + .actualSegments(1) + .create(); + + ExternalizableData value = new ExternalizableData(); + value.data[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } + + static class SerializableData implements Serializable { + byte[] data = new byte[512 * 1024]; + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExternalizableData)) + return false; + return Arrays.equals(((ExternalizableData) obj).data, data); + } + } + + @Test + @Ignore + public void testUnexpectedlyLongConstantSerializableValues() throws IOException { + ChronicleMap map = + ChronicleMapBuilder.of(Long.class, SerializableData.class) + .constantValueSizeBySample(new SerializableData()) + .entries(100) + .actualSegments(1) + .create(); + + SerializableData value = new SerializableData(); + value.data[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } +}