Add tutorial section about custom serialization, add custom serialization tests, improve Javadocs in net.openhft.chronicle.hash.serialization package

This commit is contained in:
leventov
2016-03-19 16:47:26 +07:00
parent 182cf784db
commit 0ec0eed5eb
26 changed files with 1710 additions and 229 deletions
@@ -18,14 +18,26 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.bytes.ReadBytesMarshallable;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* External version on {@link ReadBytesMarshallable}.
* Deserializer of objects from bytes, pairing {@link BytesWriter}, without accepting additional
* information about serialized objects.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#byteswriter-and-bytesreader">{@code
* BytesWriter} and {@code BytesReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> type of objects deserialized
* @see BytesWriter
* @see ChronicleHashBuilder#keyMarshallers(BytesReader, BytesWriter)
* @see ChronicleMapBuilder#valueMarshallers(BytesReader, BytesWriter)
*/
public interface BytesReader<T> extends Marshallable {
@@ -45,5 +57,5 @@ public interface BytesReader<T> extends Marshallable {
* @return the object read from the bytes, either reused or newly created
*/
@NotNull
T read(Bytes in, T using);
T read(Bytes in, @Nullable T using);
}
@@ -18,14 +18,24 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.bytes.WriteBytesMarshallable;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
/**
* External version of {@link WriteBytesMarshallable}.
* Serializer of objects to bytes, pairing {@link BytesReader}.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#byteswriter-and-bytesreader">{@code
* BytesWriter} and {@code BytesReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of objects serialized
* @see BytesReader
* @see ChronicleHashBuilder#keyMarshallers(BytesReader, BytesWriter)
* @see ChronicleMapBuilder#valueMarshallers(BytesReader, BytesWriter)
*/
public interface BytesWriter<T> extends Marshallable {
@@ -16,13 +16,52 @@
package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.hash.ChronicleHash;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
import java.nio.ByteBuffer;
/**
* Strategy of accessing flat byte contents of objects, using a reusable {@link Data} object, cached
* inside the {@code DataAccess} instance. This strategy interface is suitable for types of objects,
* that are already in fact sequences of bytes ({@code byte[]}, {@link ByteBuffer}, etc.) and
* shouldn't be serialized, and allows to avoid extra data copying. Accessed bytes should be later
* readable by some {@link BytesReader}.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#dataaccess-and-sizedreader">{@link
* DataAccess} and {@code SizedReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of objects accessed
* @see ChronicleHashBuilder#keyReaderAndDataAccess(SizedReader, DataAccess)
* @see ChronicleMapBuilder#valueReaderAndDataAccess(SizedReader, DataAccess)
*/
public interface DataAccess<T> extends StatefulCopyable<DataAccess<T>>, Marshallable {
/**
* Obtain {@code Data} accessor to the bytes of the given object. Typically a {@code Data}
* instance should be cached in a field of this {@code DataAccess}, so always the same object
* is returned.
*
* @param instance the object to access bytes (serialized form) of
* @return an accessor to the bytes of the given object
*/
Data<T> getData(@NotNull T instance);
/**
* Clear references to the accessed object (provided in {@link #getData(Object)} method) in
* caching fields of this {@code DataAccess} or the cached {@link Data} object, returned from
* {@link #getData(Object)}.
*
* {@code DataAccess} is cached itself in thread-local variables for {@link ChronicleHash}
* instances, this method prevents leaking of accessed objects (they are not eligible for
* garbage collection while there are some references).
*/
void uninit();
}
@@ -59,11 +59,31 @@ import static net.openhft.chronicle.hash.serialization.StatefulCopyable.copyIfNe
public final class ListMarshaller<T>
implements BytesReader<List<T>>, BytesWriter<List<T>>, StatefulCopyable<ListMarshaller<T>> {
/**
* Returns a {@code ListMarshaller} which uses the given list elements' serializers.
*
* @param elementReader list elements' reader
* @param elementWriter list elements' writer
* @param <T> type of list elements
* @return a {@code ListMarshaller} which uses the given list elements' serializers
*/
public static <T> ListMarshaller<T> of(
BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
return new ListMarshaller<>(elementReader, elementWriter);
}
/**
* Returns a {@code ListMarshaller} which uses the given marshaller as both reader and writer of
* list elements. Example: <pre><code>
* ChronicleMap
* .of(String.class,{@code (Class<List<Integer>>)} ((Class) List.class))
* .valueMarshaller(ListMarshaller.of(IntegerMarshaller.INSTANCE))
* ...</code></pre>
* @param elementMarshaller list elements' marshaller
* @param <T> type of list elements
* @param <M> type of list elements' marshaller
* @return a {@code ListMarshaller} which uses the given list elements' marshaller
*/
public static <T, M extends BytesReader<T> & BytesWriter<? super T>> ListMarshaller<T> of(
M elementMarshaller) {
return of(elementMarshaller, elementMarshaller);
@@ -73,6 +93,15 @@ public final class ListMarshaller<T>
private BytesReader<T> elementReader;
private BytesWriter<? super T> elementWriter;
/**
* Constructs a {@code ListMarshaller} with the given list elements' serializers.
*
* <p>Use static factory {@link #of(BytesReader, BytesWriter)} instead of this constructor
* directly.
*
* @param elementReader list elements' reader
* @param elementWriter list elements' writer
*/
public ListMarshaller(BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
this.elementReader = elementReader;
this.elementWriter = elementWriter;
@@ -60,6 +60,14 @@ public final class MapMarshaller<K, V> implements BytesReader<Map<K, V>>, BytesW
private transient Deque<K> orderedKeys;
private transient Deque<V> orderedValues;
/**
* Constructs a {@code MapMarshaller} with the given map keys' and values' serializers.
*
* @param keyReader map keys' reader
* @param keyWriter map keys' writer
* @param valueReader map values' reader
* @param valueWriter map values' writer
*/
public MapMarshaller(
BytesReader<K> keyReader, BytesWriter<? super K> keyWriter,
BytesReader<V> valueReader, BytesWriter<? super V> valueWriter) {
@@ -61,11 +61,31 @@ import static net.openhft.chronicle.hash.serialization.StatefulCopyable.copyIfNe
public final class SetMarshaller<T>
implements BytesReader<Set<T>>, BytesWriter<Set<T>>, StatefulCopyable<SetMarshaller<T>> {
/**
* Returns a {@code SetMarshaller} which uses the given set elements' serializers.
*
* @param elementReader set elements' reader
* @param elementWriter set elements' writer
* @param <T> type of set elements
* @return a {@code SetMarshaller} which uses the given set elements' serializers
*/
public static <T> SetMarshaller<T> of(
BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
return new SetMarshaller<>(elementReader, elementWriter);
}
/**
* Returns a {@code SetMarshaller} which uses the given marshaller as both reader and writer of
* set elements. Example: <pre><code>
* ChronicleMap
* .of(String.class,{@code (Class<Set<Integer>>)} ((Class) Set.class))
* .valueMarshaller(SetMarshaller.of(IntegerMarshaller.INSTANCE))
* ...</code></pre>
* @param elementMarshaller set elements' marshaller
* @param <T> type of set elements
* @param <M> type of set elements' marshaller
* @return a {@code SetMarshaller} which uses the given set elements' marshaller
*/
public static <T, M extends BytesReader<T> & BytesWriter<? super T>> SetMarshaller<T> of(
M elementMarshaller) {
return of(elementMarshaller, elementMarshaller);
@@ -78,6 +98,15 @@ public final class SetMarshaller<T>
/** Cache field */
private transient Deque<T> orderedElements;
/**
* Constructs a {@code SetMarshaller} with the given set elements' serializers.
*
* <p>Use static factory {@link #of(BytesReader, BytesWriter)} instead of this constructor
* directly.
*
* @param elementReader set elements' reader
* @param elementWriter set elements' writer
*/
public SetMarshaller(BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
this.elementReader = elementReader;
this.elementWriter = elementWriter;
@@ -17,8 +17,10 @@
package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.hash.serialization.impl.ConstantSizeMarshaller;
import net.openhft.chronicle.hash.serialization.impl.StopBitSizeMarshaller;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
/**
@@ -26,7 +28,7 @@ import net.openhft.chronicle.wire.Marshallable;
* the same bytes store). Normal number storing (4 bytes for {@code int}, 8 bytes for {@code long}
* is rather wasteful for marshalling purposes, when the number (size) to store is usually very
* small, and 1-2 bytes could be enough to encode it (this is how {@link #stopBit()} marshaller
* work). Also, this interface allows to generalize storing constantly-sized and variable-sized
* works). Also, this interface allows to generalize storing constantly-sized and variable-sized
* blocks of data. Constantly-sized don't require to store the size actually, the corresponding
* {@link #constant} {@code SizeMarshaller} consumes 0 bytes.
*
@@ -34,13 +36,34 @@ import net.openhft.chronicle.wire.Marshallable;
* example each {@link #constant} {@code SizeMarshaller} is able to "store" only a single specific
* value (it's constant size). If the marshaller is fed with the size it is not able store, it could
* throw an {@code IllegalArgumentException}.
*
* @see ChronicleHashBuilder#keySizeMarshaller(SizeMarshaller)
* @see ChronicleMapBuilder#valueSizeMarshaller(SizeMarshaller)
*/
public interface SizeMarshaller extends Marshallable {
/**
* Returns a {@code SizeMarshaller} which stores number using so-called <a
* href="https://github.com/OpenHFT/RFC/blob/master/Stop-Bit-Encoding/Stop-Bit-Encoding-0.1.md#signed-integers"
* >stop bit encoding</a>. This marshaller is used as the default marshaller for keys' and
* values' sizes in Chronicle Map, unless Chronicle Map could figure out (or hinted in {@code
* ChronicleMapBuilder}) that keys or values are constant-sized, in which case {@link
* #constant(long)} {@code SizeMarshaller} is used.
*/
static SizeMarshaller stopBit() {
return StopBitSizeMarshaller.INSTANCE;
}
/**
* Returns a {@code SizeMarshaller}, that consumes 0 bytes and is above to "store" only a single
* constant number, provided to this factory method. This size marshaller is used in Chronicle
* Map to serialize keys' or values' sizes, when Chronicle Map could figure out (or was hinted
* in {@code ChronicleMapBuilder}) that keys or values are constant-sized.
*
* @param constantSize the single constant size the returned {@code SizeMarshaller} is able to
* "store"
* @return a {@code SizeMarshaller}, that is able to "store" only the given constant size
*/
static SizeMarshaller constant(long constantSize) {
return new ConstantSizeMarshaller(constantSize);
}
@@ -18,57 +18,31 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.ChronicleHash;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* Deserializer of objects from bytes, mirroring the {@link SizedWriter}, i. e. assuming the length
* Deserializer of objects from bytes, pairing {@link SizedWriter}, i. e. assuming the length
* of the serialized form isn't written in the beginning of the serialized form itself, but managed
* by {@link ChronicleHash} implementation and passed to the reading methods.
*
* <p>Implementation example:<pre><code>
* class LongPair { long first, second; }
*
* enum LongPairArrayReader
* implements{@code SizedReader<LongPair[]>, EnumMarshallable<LongPairArrayReader>} {
* INSTANCE;
*
* {@literal @}Override
* {@literal @}NotNull
* public LongPair[]{@literal read(@}NotNull Bytes in, long size,
* {@literal @}Nullable LongPair[] using) {
* if (size{@code >} Integer.MAX_VALUE * 16L)
* throw new IllegalStateException("LongPair[] size couldn't be " + (size / 16L));
* int resLen = (int) (size / 16L);
* LongPair[] res;
* if (using != null) {
* if (using.length == resLen) {
* res = using;
* } else {
* res = Arrays.copyOf(using, resLen);
* }
* } else {
* res = new LongPair[resLen];
* }
* for (int i = 0; i{@code <} resLen; i++) {
* LongPair pair = res[i];
* if (pair == null)
* res[i] = pair = new LongPair();
* pair.first = in.readLong();
* pair.second = in.readLong();
* }
* return res;
* }
*
* {@literal @}Override
* public LongPairArrayReader readResolve() {
* return INSTANCE;
* }
* }</code></pre>
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#sizedwriter-and-sizedreader">{@code
* SizedWriter} and {@code SizedReader}</a>,
* <a href="https://github.com/OpenHFT/Chronicle-Map#dataaccess-and-sizedreader">{@link DataAccess}
* and {@code SizedReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of the object deserialized
* @see SizedWriter
* @see ChronicleHashBuilder#keyMarshallers(SizedReader, SizedWriter)
* @see ChronicleHashBuilder#keyReaderAndDataAccess(SizedReader, DataAccess)
* @see ChronicleMapBuilder#valueMarshallers(SizedReader, SizedWriter)
* @see ChronicleMapBuilder#valueReaderAndDataAccess(SizedReader, DataAccess)
*/
public interface SizedReader<T> extends Marshallable {
@@ -18,44 +18,27 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.ChronicleHash;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
/**
* Serializer (object to bytes writer) which 1) knows the length of serialized
* Serializer of objects to bytes, pairing {@link SizedReader}, which knows the length of serialized
* form of any object before actual serialization 2) doesn't include that length in the serialized
* form itself, assuming it will be passed by the {@link ChronicleHash} into {@link
* SizedReader#read} deserialization method.
*
* <p>Implementation example: <pre><code>
* class LongPair { long first, second; }
*
* enum LongPairArrayWriter
* implements{@code SizedWriter<LongPair[]>, EnumMarshallable<LongPairArrayWriter>} {
* INSTANCE;
*
* {@literal @}Override
* public long{@literal size(@}NotNull LongPair[] toWrite) {
* return toWrite.length * 16L;
* }
*
* {@literal @}Override
* public void{@literal write(@}NotNull Bytes out, long size,
* {@literal @}NotNull LongPair[] toWrite) {
* for (LongPair pair : toWrite) {
* out.writeLong(pair.first);
* out.writeLong(pair.second);
* }
* }
*
* {@literal @}Override
* public LongPairArrayWriter readResolve() {
* return INSTANCE;
* }
* }</code></pre>
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#sizedwriter-and-sizedreader">{@code
* SizedWriter} and {@code SizedReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of the object marshalled
* @see SizedReader
* @see ChronicleHashBuilder#keyMarshallers(SizedReader, SizedWriter)
* @see ChronicleMapBuilder#valueMarshallers(SizedReader, SizedWriter)
*/
public interface SizedWriter<T> extends Marshallable {
@@ -16,36 +16,20 @@
package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.hash.serialization.impl.StringBytesReader;
import net.openhft.chronicle.map.ChronicleMap;
import net.openhft.chronicle.set.ChronicleSet;
/**
* Stateful implementations of marshaller interfaces ({@link SizedReader}, {@link SizedWriter},
* {@link BytesReader}, {@link BytesWriter}, {@link DataAccess}), configured for {@link
* {@link Cloneable}-like interface used by Chronicle Map to populate stateful serializer
* implementations for each isolated site (thread + Chronicle Map instance + objects domain).
* Stateful implementations of serialization interfaces ({@link SizedReader}, {@link SizedWriter},
* {@link BytesReader}, {@link BytesWriter} or {@link DataAccess}), configured for {@link
* ChronicleMap} or {@link ChronicleSet} in builder, should implement the {@code StatefulCopyable}
* interface. The marshaller instance is populated for each site by {@link #copy()} method.
* interface.
*
* <p>For example, see {@link StringBytesReader} implementation: <pre><code>
* public class StringBytesReader
* implements{@code BytesReader<String>, StatefulCopyable<StringBytesReader>} {
*
* private final StringBuilder sb = new StringBuilder();
*
* {@literal @}NotNull
* {@literal @}Override
* public String read(Bytes in,{@literal @}Nullable String using) {
* sb.setLength(0);
* in.readUtf8(sb);
* return sb.toString();
* }
*
* {@literal @}Override
* public StringBytesReader copy() {
* return new StringBytesReader();
* }
* }
* </code></pre>
* <p>See <a href="https://github.com/OpenHFT/Chronicle-Map#understanding-statefulcopyable">
* Understanding {@code StatefulCopyable}</a> section in the Chronicle Map tutorial for more info
* on how to implement and use this interface properly, and for examples.
*
* @param <T> the type of marshaller, implementing this interface
*/
@@ -88,14 +88,14 @@ public class ByteArrayDataAccess extends AbstractData<byte[]> implements DataAcc
return new ByteArrayDataAccess();
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
// no fields to write
}
@Override
public void readMarshallable(@NotNull WireIn wireIn) {
// no fields to read
initTransients();
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
// no fields to write
}
}
@@ -17,31 +17,30 @@
package net.openhft.chronicle.hash.serialization.impl;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.core.io.IORuntimeException;
import net.openhft.chronicle.hash.serialization.SizedReader;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public enum ByteArraySizedReader
implements SizedReader<byte[]>, EnumMarshallable<ByteArraySizedReader> {
INSTANCE;
@NotNull
@Override
public byte[] read(@NotNull Bytes in, long size, @Nullable byte[] using) {
int resLen = resLen(size);
if (using == null || resLen != using.length)
using = new byte[resLen];
if (size < 0L || size > (long) Integer.MAX_VALUE) {
throw new IORuntimeException("byte[] size should be non-negative int, " +
size + " given. Memory corruption?");
}
int arrayLength = (int) size;
if (using == null || arrayLength != using.length)
using = new byte[arrayLength];
in.read(using);
return using;
}
private int resLen(long size) {
if (size < 0L || size > (long) Integer.MAX_VALUE)
throw new IllegalArgumentException("byte[] size should be non-negative int, " +
size + " given. Memory corruption?");
return (int) size;
}
@Override
public ByteArraySizedReader readResolve() {
return INSTANCE;
@@ -15,8 +15,13 @@
*/
/**
* The package contains interfaces for serializing objects and values between Java heap and {@link
* net.openhft.chronicle.bytes.Bytes} or {@link net.openhft.chronicle.bytes.BytesStore}.
* The package contains interfaces for serializing objects between Java heap and {@link
* net.openhft.chronicle.bytes.Bytes} or {@link net.openhft.chronicle.bytes.BytesStore}, used by
* Chronicle Map to store objects off-heap, and read them back from off-heap memory to on-heap
* objects.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serializers">Custom serializers
* </a> section in the Chronicle Map tutorial for more information.
*
* <p>Reading methods in the interfaces in this package could use {@link
* net.openhft.chronicle.bytes.StreamingDataInput} as the "input" parameter type and {@link