diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index aa33102f8..b7aa0831b 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -58,7 +58,9 @@ public class ChronicleMapBuilder implements Cloneable { // used when reading the number of entries per private int actualEntriesPerSegment = -1; private int keySize = 0; + private K sampleKey; private int valueSize = 0; + private V sampleValue; private int entrySize = 0; private Alignment alignment = Alignment.OF_4_BYTES; private long entries = 1 << 20; @@ -126,18 +128,16 @@ public class ChronicleMapBuilder implements Cloneable { private int tryMinSegments(int min, int max) { for (int i = min; i < max; i <<= 1) { - if (i * i * i >= alignedEntrySize() * 2) + if (i * i * i >= entrySize() * 2) return i; } return max; } /** - * Configures the number of bytes, taken by serialized form of keys, put into maps, created by - * this builder. - * - *

If key is a boxed primitive type or {@link Byteable} subclass, i. e. if key size is known - * statically, it is automatically accounted and shouldn't be specified by user. + * Configures the optimal number of bytes, taken by serialized form of keys, put into maps, + * created by this builder. If key size is always the same, call + * {@link #constantKeySizeBySample(Object)} method instead of this one. * *

If key size varies moderately, specify the size higher than average, but lower than * the maximum possible, to minimize average memory overuse. If key size varies in a wide range, @@ -157,6 +157,7 @@ public class ChronicleMapBuilder implements Cloneable { * * @param keySize number of bytes, taken by serialized form of keys * @return this {@code ChronicleMapBuilder} back + * @see #constantKeySizeBySample(Object) * @see #valueSize(int) * @see #entrySize() */ @@ -167,13 +168,42 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Configures the constant number of bytes, taken by serialized form of keys, put into maps, + * created by this builder. This is done by providing the {@code sampleKey}, all keys should + * take the same number of bytes in serialized form, as this sample object. + * + *

For example, if your keys are Git commit hashes:

{@code
+     * Map gitCommitMessagesByHash =
+     *     ChronicleMapBuilder.of(byte[].class, String.class)
+     *     .constantKeySizeBySample(new byte[20])
+     *     .create();
+     * }
+ * + *

If keys are of boxed primitive type or {@link Byteable} subclass, i. e. if key size + * is known statically, it is automatically accounted and this method shouldn't be called. + * + *

If key size varies, method {@link #keySize(int)} or {@link #entrySize(int)} should be + * called instead of this one. + * + * @param sampleKey the sample key + * @return this builder back + * @see #keySize(int) + * @see #constantValueSizeBySample(Object) + */ + public ChronicleMapBuilder constantKeySizeBySample(K sampleKey) { + this.sampleKey = sampleKey; + return this; + } + private int keySize() { return keyOrValueSize(keySize, keyBuilder); } /** - * Configures the number of bytes, taken by serialized form of value, put into maps, created by - * this builder. + * Configures the optimal number of bytes, taken by serialized form of values, put into maps, + * created by this builder. If value size is always the same, call + * {@link #constantValueSizeBySample(Object)} method instead of this one. * *

If value is a boxed primitive type or {@link Byteable} subclass, i. e. if value size * is known statically, it is automatically accounted and shouldn't be specified by user. @@ -195,6 +225,27 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Configures the constant number of bytes, taken by serialized form of values, put into maps, + * created by this builder. This is done by providing the {@code sampleValue}, all values should + * take the same number of bytes in serialized form, as this sample object. + * + *

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size + * is known statically, it is automatically accounted and this method shouldn't be called. + * + *

If value size varies, method {@link #valueSize(int)} or {@link #entrySize(int)} should be + * called instead of this one. + * + * @param sampleValue the sample value + * @return this builder back + * @see #valueSize(int) + * @see #constantKeySizeBySample(Object) + */ + public ChronicleMapBuilder constantValueSizeBySample(V sampleValue) { + this.sampleValue = sampleValue; + return this; + } + private int valueSize() { return keyOrValueSize(valueSize, valueBuilder); } @@ -252,9 +303,10 @@ public class ChronicleMapBuilder implements Cloneable { * implementations, with respect to {@linkplain #entryAndValueAlignment() alignment}. * *

There are three major patterns of this configuration usage: - *

+ * * * @return size of memory allocation unit (in bytes) + * @see #entries(long) */ public int entrySize() { if (entrySize > 0) - return entrySize; + return entryAndValueAlignment().alignSize(entrySize); int size = metaDataBytes; int keySize = keySize(); size += keyBuilder.sizeMarshaller().sizeEncodingSize(keySize); @@ -318,10 +379,6 @@ public class ChronicleMapBuilder implements Cloneable { return entryAndValueAlignment().alignSize(size); } - int alignedEntrySize() { - return entryAndValueAlignment().alignSize(entrySize()); - } - /** * Configures alignment strategy of address in memory of entries and independently of address * in memory of values within entries in ChronicleMaps, created by this builder. @@ -622,6 +679,22 @@ public class ChronicleMapBuilder implements Cloneable { return this; } + /** + * Specifies that key objects, queried with the maps created by this builder, are inherently + * immutable. Keys in {@code ChronicleMap} are not required to be immutable, as in ordinary + * {@link Map} implementations, because they are serialized off-heap. However, + * {@code ChronicleMap} implementations can benefit from the knowledge that keys are not mutated + * between queries. By default, {@code ChronicleMapBuilder} detects immutability automatically + * only for very few standard JDK types (for example, for {@link String}), it is not recommended + * to rely on {@code ChronicleMapBuilder} to be smart enough about this. + * + * @return this builder back + */ + public ChronicleMapBuilder immutableKeys() { + keyBuilder.instancesAreMutable(false); + return this; + } + public ChronicleMapBuilder valueMarshallerAndFactory( @NotNull BytesMarshaller valueMarshaller, @NotNull ObjectFactory valueFactory) { valueBuilder.marshaller(valueMarshaller, valueFactory); @@ -777,6 +850,11 @@ public class ChronicleMapBuilder implements Cloneable { int maxSize = entrySize() * figureBufferAllocationFactor(); keyBuilder.maxSize(maxSize); valueBuilder.maxSize(maxSize); + + if (sampleKey != null) + keyBuilder.constantSizeBySample(sampleKey); + if (sampleValue != null) + valueBuilder.constantSizeBySample(sampleValue); } private ChronicleMap establishReplication(ChronicleMap map) diff --git a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java index 6728d638f..3451c2c0a 100644 --- a/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/SerializationBuilder.java @@ -18,6 +18,8 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.map.serialization.*; import net.openhft.chronicle.map.serialization.impl.*; +import net.openhft.chronicle.map.threadlocal.Provider; +import net.openhft.chronicle.map.threadlocal.ThreadLocalCopies; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.serialization.*; import net.openhft.lang.io.serialization.impl.*; @@ -45,28 +47,35 @@ final class SerializationBuilder implements Cloneable { Externalizable.class.isAssignableFrom(c); } + private static boolean instancesAreMutable(Class c) { + return c != String.class; + } + enum Role {KEY, VALUE} private enum CopyingInterop {FROM_MARSHALLER, FROM_WRITER} private final Serializable bufferIdentity; final Class eClass; + private boolean instancesAreMutable; private SizeMarshaller sizeMarshaller = stopBit(); private BytesReader reader; private Object interop; private CopyingInterop copyingInterop = null; private MetaBytesInterop metaInterop; private MetaProvider metaInteropProvider; + private long maxSize; private ObjectFactory factory; @SuppressWarnings("unchecked") SerializationBuilder(Class eClass, Role role) { this.bufferIdentity = role; this.eClass = eClass; + instancesAreMutable = instancesAreMutable(eClass); - ObjectFactory factory = this.factory; - if (factory == null && role == Role.VALUE) { + ObjectFactory factory = null; + if (role == Role.VALUE) { factory = concreteClass(eClass) && marshallerUseFactory(eClass) ? - new AllocateInstanceObjectFactory(eClass) : + new NewInstanceObjectFactory(eClass) : NullObjectFactory.INSTANCE; } @@ -85,6 +94,9 @@ final class SerializationBuilder implements Cloneable { if (marshaller != null) marshaller(marshaller, factory); } + if (role == Role.VALUE && concreteClass(eClass) && marshallerUseFactory(eClass)) { + factory(factory); + } } @SuppressWarnings("unchecked") @@ -123,7 +135,6 @@ final class SerializationBuilder implements Cloneable { public SerializationBuilder writer(BytesWriter writer) { return copyingInterop(CopyingInterop.FROM_WRITER) - .sizeMarshaller(stopBit()) .setInterop(writer) .metaInterop(CopyingMetaBytesInterop .>forBytesWriter(bufferIdentity)); @@ -133,7 +144,6 @@ final class SerializationBuilder implements Cloneable { ObjectFactory factory) { if (factory == null) factory = this.factory; return copyingInterop(CopyingInterop.FROM_MARSHALLER) - .sizeMarshaller(stopBit()) .reader(BytesReaders.fromBytesMarshaller(marshaller), factory) .setInterop(marshaller) .metaInterop(CopyingMetaBytesInterop @@ -141,17 +151,58 @@ final class SerializationBuilder implements Cloneable { } public SerializationBuilder maxSize(long maxSize) { - boolean mutable = eClass != String.class; // for example if (copyingInterop == CopyingInterop.FROM_MARSHALLER) { + this.maxSize = maxSize; metaInteropProvider(CopyingMetaBytesInterop - .>providerForBytesMarshaller(mutable, maxSize)); + .>providerForBytesMarshaller( + instancesAreMutable, maxSize)); } else if (copyingInterop == CopyingInterop.FROM_WRITER) { + this.maxSize = maxSize; metaInteropProvider(CopyingMetaBytesInterop - .>providerForBytesWriter(mutable, maxSize)); + .>providerForBytesWriter(instancesAreMutable, maxSize)); } return this; } + public SerializationBuilder constantSizeBySample(E sampleObject) { + Object originalInterop = this.interop; + Provider interopProvider = Provider.of(originalInterop.getClass()); + ThreadLocalCopies copies = interopProvider.getCopies(null); + Object interop = interopProvider.get(copies, originalInterop); + copies = metaInteropProvider.getCopies(copies); + MetaBytesWriter metaInterop; + if (maxSize > 0) { + // this loop is very dumb: tries to find buffer size, sufficient to serialize + // the object, by trying to serialize to small buffers first and doubling the buffer + // size on _any_ Exception (it may be IllegalArgument, IndexOutOfBounds, IllegalState, + // see JLANG-17 issue). + // TODO The question is: couldn't this cause JVM crash without throwing an exception? + findSufficientSerializationSize: + while (true) { + MetaProvider metaInteropProvider = this.metaInteropProvider; + try { + metaInterop = metaInteropProvider.get( + copies, this.metaInterop, interop, sampleObject); + break findSufficientSerializationSize; + } catch (Exception e) { + // assuming nobody need > 512 mb _constant size_ keys/values + if (maxSize > 512 * 1024 * 1024) { + throw e; + } + maxSize(maxSize * 2); + } + } + } else { + MetaProvider metaInteropProvider = this.metaInteropProvider; + metaInterop = metaInteropProvider.get(copies, this.metaInterop, interop, sampleObject); + } + long constantSize = metaInterop.size(interop, sampleObject); + // set max size once more, because current maxSize could be x64 or x2 larger than needed + maxSize(constantSize); + sizeMarshaller(SizeMarshallers.constant(constantSize)); + return this; + } + public SerializationBuilder objectSerializer(ObjectSerializer serializer) { if (reader == null || interop == null) { ObjectFactory factory = this.factory; @@ -162,6 +213,11 @@ final class SerializationBuilder implements Cloneable { return this; } + public SerializationBuilder instancesAreMutable(boolean mutable) { + this.instancesAreMutable = mutable; + return this; + } + public SizeMarshaller sizeMarshaller() { return sizeMarshaller; } diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index c35bad84f..13f445976 100644 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -124,7 +124,7 @@ class VanillaChronicleMap, lockTimeOutNS = builder.lockTimeOut(TimeUnit.NANOSECONDS); // this.replicas = builder.replicas(); - this.entrySize = builder.alignedEntrySize(); + this.entrySize = builder.entrySize(); this.alignment = builder.entryAndValueAlignment(); this.errorListener = builder.errorListener(); diff --git a/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java b/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java index 1c1922a8f..a8fc3d368 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/CopyingMetaBytesInterop.java @@ -34,6 +34,7 @@ public abstract class CopyingMetaBytesInterop implements MetaBytesInterop< final DirectBytesBuffer buffer; transient long size; + transient long hash; transient W writer; transient E cur; @@ -49,6 +50,7 @@ public abstract class CopyingMetaBytesInterop implements MetaBytesInterop< innerWrite(writer, buffer, e); buffer.flip(); size = buffer.remaining(); + hash = 0L; } } @@ -66,7 +68,10 @@ public abstract class CopyingMetaBytesInterop implements MetaBytesInterop< @Override public long hash(W writer, E e) { - return Hasher.hash(buffer.buffer); + long h; + if ((h = hash) == 0L) + return hash = Hasher.hash(buffer.buffer); + return h; } @Override diff --git a/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java b/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java index 6a26114b7..d5fdbc810 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/SizeMarshallers.java @@ -47,5 +47,34 @@ public final class SizeMarshallers { } } + public static SizeMarshaller constant(long size) { + return new ConstantSizeMarshaller(size); + } + + private static class ConstantSizeMarshaller implements SizeMarshaller { + private static final long serialVersionUID = 0L; + + private final long size; + + private ConstantSizeMarshaller(long size) { + this.size = size; + } + + @Override + public int sizeEncodingSize(long size) { + return 0; + } + + @Override + public void writeSize(Bytes bytes, long size) { + // do nothing + } + + @Override + public long readSize(Bytes bytes) { + return size; + } + } + private SizeMarshallers() {} } diff --git a/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java b/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java index 0fa9c4e26..9b84ec7d9 100644 --- a/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java +++ b/src/main/java/net/openhft/chronicle/map/serialization/impl/ByteArrayMarshaller.java @@ -58,7 +58,7 @@ public enum ByteArrayMarshaller implements BytesInterop, BytesReader { @Override public long size(Long e) { - return 4L; + return 8L; } @Override @@ -55,7 +55,7 @@ public enum LongMarshaller implements AgileBytesMarshaller { @Override public long readSize(Bytes bytes) { - return 4L; + return 8L; } @Override diff --git a/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java b/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java new file mode 100644 index 000000000..46b902b24 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ConstantSizeBySampleTest.java @@ -0,0 +1,128 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.junit.Ignore; +import org.junit.Test; +import org.testng.Assert; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +public class ConstantSizeBySampleTest { + + @Test + public void testConstantKeys() throws IOException { + ChronicleMap map = ChronicleMapBuilder.of(byte[].class, Long.class) + .constantKeySizeBySample(new byte[8]) + .immutableKeys() + .entries(100) + .create(); + + byte[] zero = ByteBuffer.allocate(8).putLong(0L).array(); + map.put(zero, 0L); + assertEquals(0L, (long) map.get(zero)); + + byte[] one = ByteBuffer.allocate(8).putLong(1L).array(); + map.put(one, 1L); + assertEquals(1L, (long) map.get(one)); + + map.put(one, 0L); + assertEquals(0L, (long) map.get(one)); + } + + @Test + public void testUnexpectedlyLongConstantByteArrayValues() throws IOException { + ChronicleMap map = ChronicleMapBuilder.of(Long.class, byte[].class) + .constantValueSizeBySample(new byte[512 * 1024]) + .entries(100) + .actualSegments(1) + .create(); + + byte[] value = new byte[512 * 1024]; + + value[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } + + static class ExternalizableData implements Externalizable { + byte[] data = new byte[512 * 1024]; + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExternalizableData)) + return false; + return Arrays.equals(((ExternalizableData) obj).data, data); + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.write(data); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + in.read(data = new byte[512 * 1024]); + } + } + + @Test + public void testUnexpectedlyLongConstantExternalizableValues() throws IOException { + ChronicleMap map = + ChronicleMapBuilder.of(Long.class, ExternalizableData.class) + .constantValueSizeBySample(new ExternalizableData()) + .entries(100) + .actualSegments(1) + .create(); + + ExternalizableData value = new ExternalizableData(); + value.data[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } + + static class SerializableData implements Serializable { + byte[] data = new byte[512 * 1024]; + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExternalizableData)) + return false; + return Arrays.equals(((ExternalizableData) obj).data, data); + } + } + + @Test + @Ignore + public void testUnexpectedlyLongConstantSerializableValues() throws IOException { + ChronicleMap map = + ChronicleMapBuilder.of(Long.class, SerializableData.class) + .constantValueSizeBySample(new SerializableData()) + .entries(100) + .actualSegments(1) + .create(); + + SerializableData value = new SerializableData(); + value.data[42] = 1; + map.put(1L, value); + Assert.assertEquals(map.get(1L), value); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java index dbf61accc..7c90b209e 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/OpenJDKAndHashMapExamplesTest.java @@ -52,15 +52,11 @@ public class OpenJDKAndHashMapExamplesTest { @Test public void bondExample() throws IOException, InterruptedException { - Class bondVODirectClass = directClassFor(BondVOInterface.class); - NewInstanceObjectFactory bondVODirectObjectFactory = - new NewInstanceObjectFactory(bondVODirectClass); File file = new File(TMP + "/chm-myBondPortfolioCHM-" + System.nanoTime()); file.deleteOnExit(); ChronicleMap chm = ChronicleMapBuilder - .of(String.class, bondVODirectClass) - .valueFactory(bondVODirectObjectFactory) - .entrySize(512) + .of(String.class, directClassFor(BondVOInterface.class)) + .keySize(10) .create(file); @@ -80,9 +76,8 @@ public class OpenJDKAndHashMapExamplesTest { ChronicleMap chmB = ChronicleMapBuilder - .of(String.class, bondVODirectClass) - .valueFactory(bondVODirectObjectFactory) - .entrySize(320) + .of(String.class, directClassFor(BondVOInterface.class)) + .keySize(10) .create(file); // ZERO Copy but creates a new off heap reference each time