Add tutorial section about custom serialization, add custom serialization tests, improve Javadocs in net.openhft.chronicle.hash.serialization package

This commit is contained in:
leventov
2016-03-19 16:47:26 +07:00
parent 182cf784db
commit 0ec0eed5eb
26 changed files with 1710 additions and 229 deletions
@@ -18,14 +18,26 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.bytes.ReadBytesMarshallable;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* External version on {@link ReadBytesMarshallable}.
* Deserializer of objects from bytes, pairing {@link BytesWriter}, without accepting additional
* information about serialized objects.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#byteswriter-and-bytesreader">{@code
* BytesWriter} and {@code BytesReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> type of objects deserialized
* @see BytesWriter
* @see ChronicleHashBuilder#keyMarshallers(BytesReader, BytesWriter)
* @see ChronicleMapBuilder#valueMarshallers(BytesReader, BytesWriter)
*/
public interface BytesReader<T> extends Marshallable {
@@ -45,5 +57,5 @@ public interface BytesReader<T> extends Marshallable {
* @return the object read from the bytes, either reused or newly created
*/
@NotNull
T read(Bytes in, T using);
T read(Bytes in, @Nullable T using);
}
@@ -18,14 +18,24 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.bytes.WriteBytesMarshallable;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
/**
* External version of {@link WriteBytesMarshallable}.
* Serializer of objects to bytes, pairing {@link BytesReader}.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#byteswriter-and-bytesreader">{@code
* BytesWriter} and {@code BytesReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of objects serialized
* @see BytesReader
* @see ChronicleHashBuilder#keyMarshallers(BytesReader, BytesWriter)
* @see ChronicleMapBuilder#valueMarshallers(BytesReader, BytesWriter)
*/
public interface BytesWriter<T> extends Marshallable {
@@ -16,13 +16,52 @@
package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.hash.ChronicleHash;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
import java.nio.ByteBuffer;
/**
* Strategy of accessing flat byte contents of objects, using a reusable {@link Data} object, cached
* inside the {@code DataAccess} instance. This strategy interface is suitable for types of objects,
* that are already in fact sequences of bytes ({@code byte[]}, {@link ByteBuffer}, etc.) and
* shouldn't be serialized, and allows to avoid extra data copying. Accessed bytes should be later
* readable by some {@link BytesReader}.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#dataaccess-and-sizedreader">{@link
* DataAccess} and {@code SizedReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of objects accessed
* @see ChronicleHashBuilder#keyReaderAndDataAccess(SizedReader, DataAccess)
* @see ChronicleMapBuilder#valueReaderAndDataAccess(SizedReader, DataAccess)
*/
public interface DataAccess<T> extends StatefulCopyable<DataAccess<T>>, Marshallable {
/**
* Obtain {@code Data} accessor to the bytes of the given object. Typically a {@code Data}
* instance should be cached in a field of this {@code DataAccess}, so always the same object
* is returned.
*
* @param instance the object to access bytes (serialized form) of
* @return an accessor to the bytes of the given object
*/
Data<T> getData(@NotNull T instance);
/**
* Clear references to the accessed object (provided in {@link #getData(Object)} method) in
* caching fields of this {@code DataAccess} or the cached {@link Data} object, returned from
* {@link #getData(Object)}.
*
* {@code DataAccess} is cached itself in thread-local variables for {@link ChronicleHash}
* instances, this method prevents leaking of accessed objects (they are not eligible for
* garbage collection while there are some references).
*/
void uninit();
}
@@ -59,11 +59,31 @@ import static net.openhft.chronicle.hash.serialization.StatefulCopyable.copyIfNe
public final class ListMarshaller<T>
implements BytesReader<List<T>>, BytesWriter<List<T>>, StatefulCopyable<ListMarshaller<T>> {
/**
* Returns a {@code ListMarshaller} which uses the given list elements' serializers.
*
* @param elementReader list elements' reader
* @param elementWriter list elements' writer
* @param <T> type of list elements
* @return a {@code ListMarshaller} which uses the given list elements' serializers
*/
public static <T> ListMarshaller<T> of(
BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
return new ListMarshaller<>(elementReader, elementWriter);
}
/**
* Returns a {@code ListMarshaller} which uses the given marshaller as both reader and writer of
* list elements. Example: <pre><code>
* ChronicleMap
* .of(String.class,{@code (Class<List<Integer>>)} ((Class) List.class))
* .valueMarshaller(ListMarshaller.of(IntegerMarshaller.INSTANCE))
* ...</code></pre>
* @param elementMarshaller list elements' marshaller
* @param <T> type of list elements
* @param <M> type of list elements' marshaller
* @return a {@code ListMarshaller} which uses the given list elements' marshaller
*/
public static <T, M extends BytesReader<T> & BytesWriter<? super T>> ListMarshaller<T> of(
M elementMarshaller) {
return of(elementMarshaller, elementMarshaller);
@@ -73,6 +93,15 @@ public final class ListMarshaller<T>
private BytesReader<T> elementReader;
private BytesWriter<? super T> elementWriter;
/**
* Constructs a {@code ListMarshaller} with the given list elements' serializers.
*
* <p>Use static factory {@link #of(BytesReader, BytesWriter)} instead of this constructor
* directly.
*
* @param elementReader list elements' reader
* @param elementWriter list elements' writer
*/
public ListMarshaller(BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
this.elementReader = elementReader;
this.elementWriter = elementWriter;
@@ -60,6 +60,14 @@ public final class MapMarshaller<K, V> implements BytesReader<Map<K, V>>, BytesW
private transient Deque<K> orderedKeys;
private transient Deque<V> orderedValues;
/**
* Constructs a {@code MapMarshaller} with the given map keys' and values' serializers.
*
* @param keyReader map keys' reader
* @param keyWriter map keys' writer
* @param valueReader map values' reader
* @param valueWriter map values' writer
*/
public MapMarshaller(
BytesReader<K> keyReader, BytesWriter<? super K> keyWriter,
BytesReader<V> valueReader, BytesWriter<? super V> valueWriter) {
@@ -61,11 +61,31 @@ import static net.openhft.chronicle.hash.serialization.StatefulCopyable.copyIfNe
public final class SetMarshaller<T>
implements BytesReader<Set<T>>, BytesWriter<Set<T>>, StatefulCopyable<SetMarshaller<T>> {
/**
* Returns a {@code SetMarshaller} which uses the given set elements' serializers.
*
* @param elementReader set elements' reader
* @param elementWriter set elements' writer
* @param <T> type of set elements
* @return a {@code SetMarshaller} which uses the given set elements' serializers
*/
public static <T> SetMarshaller<T> of(
BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
return new SetMarshaller<>(elementReader, elementWriter);
}
/**
* Returns a {@code SetMarshaller} which uses the given marshaller as both reader and writer of
* set elements. Example: <pre><code>
* ChronicleMap
* .of(String.class,{@code (Class<Set<Integer>>)} ((Class) Set.class))
* .valueMarshaller(SetMarshaller.of(IntegerMarshaller.INSTANCE))
* ...</code></pre>
* @param elementMarshaller set elements' marshaller
* @param <T> type of set elements
* @param <M> type of set elements' marshaller
* @return a {@code SetMarshaller} which uses the given set elements' marshaller
*/
public static <T, M extends BytesReader<T> & BytesWriter<? super T>> SetMarshaller<T> of(
M elementMarshaller) {
return of(elementMarshaller, elementMarshaller);
@@ -78,6 +98,15 @@ public final class SetMarshaller<T>
/** Cache field */
private transient Deque<T> orderedElements;
/**
* Constructs a {@code SetMarshaller} with the given set elements' serializers.
*
* <p>Use static factory {@link #of(BytesReader, BytesWriter)} instead of this constructor
* directly.
*
* @param elementReader set elements' reader
* @param elementWriter set elements' writer
*/
public SetMarshaller(BytesReader<T> elementReader, BytesWriter<? super T> elementWriter) {
this.elementReader = elementReader;
this.elementWriter = elementWriter;
@@ -17,8 +17,10 @@
package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.hash.serialization.impl.ConstantSizeMarshaller;
import net.openhft.chronicle.hash.serialization.impl.StopBitSizeMarshaller;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
/**
@@ -26,7 +28,7 @@ import net.openhft.chronicle.wire.Marshallable;
* the same bytes store). Normal number storing (4 bytes for {@code int}, 8 bytes for {@code long}
* is rather wasteful for marshalling purposes, when the number (size) to store is usually very
* small, and 1-2 bytes could be enough to encode it (this is how {@link #stopBit()} marshaller
* work). Also, this interface allows to generalize storing constantly-sized and variable-sized
* works). Also, this interface allows to generalize storing constantly-sized and variable-sized
* blocks of data. Constantly-sized don't require to store the size actually, the corresponding
* {@link #constant} {@code SizeMarshaller} consumes 0 bytes.
*
@@ -34,13 +36,34 @@ import net.openhft.chronicle.wire.Marshallable;
* example each {@link #constant} {@code SizeMarshaller} is able to "store" only a single specific
* value (it's constant size). If the marshaller is fed with the size it is not able store, it could
* throw an {@code IllegalArgumentException}.
*
* @see ChronicleHashBuilder#keySizeMarshaller(SizeMarshaller)
* @see ChronicleMapBuilder#valueSizeMarshaller(SizeMarshaller)
*/
public interface SizeMarshaller extends Marshallable {
/**
* Returns a {@code SizeMarshaller} which stores number using so-called <a
* href="https://github.com/OpenHFT/RFC/blob/master/Stop-Bit-Encoding/Stop-Bit-Encoding-0.1.md#signed-integers"
* >stop bit encoding</a>. This marshaller is used as the default marshaller for keys' and
* values' sizes in Chronicle Map, unless Chronicle Map could figure out (or hinted in {@code
* ChronicleMapBuilder}) that keys or values are constant-sized, in which case {@link
* #constant(long)} {@code SizeMarshaller} is used.
*/
static SizeMarshaller stopBit() {
return StopBitSizeMarshaller.INSTANCE;
}
/**
* Returns a {@code SizeMarshaller}, that consumes 0 bytes and is above to "store" only a single
* constant number, provided to this factory method. This size marshaller is used in Chronicle
* Map to serialize keys' or values' sizes, when Chronicle Map could figure out (or was hinted
* in {@code ChronicleMapBuilder}) that keys or values are constant-sized.
*
* @param constantSize the single constant size the returned {@code SizeMarshaller} is able to
* "store"
* @return a {@code SizeMarshaller}, that is able to "store" only the given constant size
*/
static SizeMarshaller constant(long constantSize) {
return new ConstantSizeMarshaller(constantSize);
}
@@ -18,57 +18,31 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.ChronicleHash;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* Deserializer of objects from bytes, mirroring the {@link SizedWriter}, i. e. assuming the length
* Deserializer of objects from bytes, pairing {@link SizedWriter}, i. e. assuming the length
* of the serialized form isn't written in the beginning of the serialized form itself, but managed
* by {@link ChronicleHash} implementation and passed to the reading methods.
*
* <p>Implementation example:<pre><code>
* class LongPair { long first, second; }
*
* enum LongPairArrayReader
* implements{@code SizedReader<LongPair[]>, EnumMarshallable<LongPairArrayReader>} {
* INSTANCE;
*
* {@literal @}Override
* {@literal @}NotNull
* public LongPair[]{@literal read(@}NotNull Bytes in, long size,
* {@literal @}Nullable LongPair[] using) {
* if (size{@code >} Integer.MAX_VALUE * 16L)
* throw new IllegalStateException("LongPair[] size couldn't be " + (size / 16L));
* int resLen = (int) (size / 16L);
* LongPair[] res;
* if (using != null) {
* if (using.length == resLen) {
* res = using;
* } else {
* res = Arrays.copyOf(using, resLen);
* }
* } else {
* res = new LongPair[resLen];
* }
* for (int i = 0; i{@code <} resLen; i++) {
* LongPair pair = res[i];
* if (pair == null)
* res[i] = pair = new LongPair();
* pair.first = in.readLong();
* pair.second = in.readLong();
* }
* return res;
* }
*
* {@literal @}Override
* public LongPairArrayReader readResolve() {
* return INSTANCE;
* }
* }</code></pre>
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#sizedwriter-and-sizedreader">{@code
* SizedWriter} and {@code SizedReader}</a>,
* <a href="https://github.com/OpenHFT/Chronicle-Map#dataaccess-and-sizedreader">{@link DataAccess}
* and {@code SizedReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of the object deserialized
* @see SizedWriter
* @see ChronicleHashBuilder#keyMarshallers(SizedReader, SizedWriter)
* @see ChronicleHashBuilder#keyReaderAndDataAccess(SizedReader, DataAccess)
* @see ChronicleMapBuilder#valueMarshallers(SizedReader, SizedWriter)
* @see ChronicleMapBuilder#valueReaderAndDataAccess(SizedReader, DataAccess)
*/
public interface SizedReader<T> extends Marshallable {
@@ -18,44 +18,27 @@ package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.ChronicleHash;
import net.openhft.chronicle.hash.ChronicleHashBuilder;
import net.openhft.chronicle.map.ChronicleMapBuilder;
import net.openhft.chronicle.wire.Marshallable;
import org.jetbrains.annotations.NotNull;
/**
* Serializer (object to bytes writer) which 1) knows the length of serialized
* Serializer of objects to bytes, pairing {@link SizedReader}, which knows the length of serialized
* form of any object before actual serialization 2) doesn't include that length in the serialized
* form itself, assuming it will be passed by the {@link ChronicleHash} into {@link
* SizedReader#read} deserialization method.
*
* <p>Implementation example: <pre><code>
* class LongPair { long first, second; }
*
* enum LongPairArrayWriter
* implements{@code SizedWriter<LongPair[]>, EnumMarshallable<LongPairArrayWriter>} {
* INSTANCE;
*
* {@literal @}Override
* public long{@literal size(@}NotNull LongPair[] toWrite) {
* return toWrite.length * 16L;
* }
*
* {@literal @}Override
* public void{@literal write(@}NotNull Bytes out, long size,
* {@literal @}NotNull LongPair[] toWrite) {
* for (LongPair pair : toWrite) {
* out.writeLong(pair.first);
* out.writeLong(pair.second);
* }
* }
*
* {@literal @}Override
* public LongPairArrayWriter readResolve() {
* return INSTANCE;
* }
* }</code></pre>
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#sizedwriter-and-sizedreader">{@code
* SizedWriter} and {@code SizedReader}</a> and
* <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serialization-checklist">custom
* serialization checklist</a> sections in the Chronicle Map tutorial for more information on this
* interface, how to implement and use it properly.
*
* @param <T> the type of the object marshalled
* @see SizedReader
* @see ChronicleHashBuilder#keyMarshallers(SizedReader, SizedWriter)
* @see ChronicleMapBuilder#valueMarshallers(SizedReader, SizedWriter)
*/
public interface SizedWriter<T> extends Marshallable {
@@ -16,36 +16,20 @@
package net.openhft.chronicle.hash.serialization;
import net.openhft.chronicle.hash.serialization.impl.StringBytesReader;
import net.openhft.chronicle.map.ChronicleMap;
import net.openhft.chronicle.set.ChronicleSet;
/**
* Stateful implementations of marshaller interfaces ({@link SizedReader}, {@link SizedWriter},
* {@link BytesReader}, {@link BytesWriter}, {@link DataAccess}), configured for {@link
* {@link Cloneable}-like interface used by Chronicle Map to populate stateful serializer
* implementations for each isolated site (thread + Chronicle Map instance + objects domain).
* Stateful implementations of serialization interfaces ({@link SizedReader}, {@link SizedWriter},
* {@link BytesReader}, {@link BytesWriter} or {@link DataAccess}), configured for {@link
* ChronicleMap} or {@link ChronicleSet} in builder, should implement the {@code StatefulCopyable}
* interface. The marshaller instance is populated for each site by {@link #copy()} method.
* interface.
*
* <p>For example, see {@link StringBytesReader} implementation: <pre><code>
* public class StringBytesReader
* implements{@code BytesReader<String>, StatefulCopyable<StringBytesReader>} {
*
* private final StringBuilder sb = new StringBuilder();
*
* {@literal @}NotNull
* {@literal @}Override
* public String read(Bytes in,{@literal @}Nullable String using) {
* sb.setLength(0);
* in.readUtf8(sb);
* return sb.toString();
* }
*
* {@literal @}Override
* public StringBytesReader copy() {
* return new StringBytesReader();
* }
* }
* </code></pre>
* <p>See <a href="https://github.com/OpenHFT/Chronicle-Map#understanding-statefulcopyable">
* Understanding {@code StatefulCopyable}</a> section in the Chronicle Map tutorial for more info
* on how to implement and use this interface properly, and for examples.
*
* @param <T> the type of marshaller, implementing this interface
*/
@@ -88,14 +88,14 @@ public class ByteArrayDataAccess extends AbstractData<byte[]> implements DataAcc
return new ByteArrayDataAccess();
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
// no fields to write
}
@Override
public void readMarshallable(@NotNull WireIn wireIn) {
// no fields to read
initTransients();
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
// no fields to write
}
}
@@ -17,31 +17,30 @@
package net.openhft.chronicle.hash.serialization.impl;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.core.io.IORuntimeException;
import net.openhft.chronicle.hash.serialization.SizedReader;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
public enum ByteArraySizedReader
implements SizedReader<byte[]>, EnumMarshallable<ByteArraySizedReader> {
INSTANCE;
@NotNull
@Override
public byte[] read(@NotNull Bytes in, long size, @Nullable byte[] using) {
int resLen = resLen(size);
if (using == null || resLen != using.length)
using = new byte[resLen];
if (size < 0L || size > (long) Integer.MAX_VALUE) {
throw new IORuntimeException("byte[] size should be non-negative int, " +
size + " given. Memory corruption?");
}
int arrayLength = (int) size;
if (using == null || arrayLength != using.length)
using = new byte[arrayLength];
in.read(using);
return using;
}
private int resLen(long size) {
if (size < 0L || size > (long) Integer.MAX_VALUE)
throw new IllegalArgumentException("byte[] size should be non-negative int, " +
size + " given. Memory corruption?");
return (int) size;
}
@Override
public ByteArraySizedReader readResolve() {
return INSTANCE;
@@ -15,8 +15,13 @@
*/
/**
* The package contains interfaces for serializing objects and values between Java heap and {@link
* net.openhft.chronicle.bytes.Bytes} or {@link net.openhft.chronicle.bytes.BytesStore}.
* The package contains interfaces for serializing objects between Java heap and {@link
* net.openhft.chronicle.bytes.Bytes} or {@link net.openhft.chronicle.bytes.BytesStore}, used by
* Chronicle Map to store objects off-heap, and read them back from off-heap memory to on-heap
* objects.
*
* <p>Read <a href="https://github.com/OpenHFT/Chronicle-Map#custom-serializers">Custom serializers
* </a> section in the Chronicle Map tutorial for more information.
*
* <p>Reading methods in the interfaces in this package could use {@link
* net.openhft.chronicle.bytes.StreamingDataInput} as the "input" parameter type and {@link
@@ -40,8 +40,8 @@ public class DefaultValueTest {
defaultValue.add(42);
try (ChronicleMap<String, List<Integer>> map = ChronicleMap
.of(String.class, (Class<List<Integer>>) ((Class) List.class))
.entries(3)
.valueMarshaller(ListMarshaller.of(IntegerMarshaller.INSTANCE))
.entries(3)
.averageKey("a").averageValue(Arrays.asList(1, 2))
.defaultValueProvider(absentEntry ->
absentEntry.context().wrapValueAsData(defaultValue))
@@ -29,7 +29,7 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
public class ComplexSerializationTest {
public class PointListSerializationTest {
static class A {
String str_;
@@ -0,0 +1,85 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.core.util.ReadResolvable;
import net.openhft.chronicle.hash.impl.util.Objects;
import net.openhft.chronicle.hash.serialization.BytesReader;
import net.openhft.chronicle.hash.serialization.BytesWriter;
import net.openhft.chronicle.wire.WireIn;
import net.openhft.chronicle.wire.WireOut;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Arrays;
public enum CharSequenceArrayBytesMarshaller
implements BytesWriter<CharSequence[]>, BytesReader<CharSequence[]>,
ReadResolvable<CharSequenceArrayBytesMarshaller> {
INSTANCE;
@Override
public void write(Bytes out, @NotNull CharSequence[] toWrite) {
// Note that the *array length*, not the *serialization size* is written in the beginning,
// so BytesWriter + BytesReader is the most suitable pair of serialization interfaces
// to implement for CharSequence[] type
out.writeInt(toWrite.length);
for (CharSequence cs : toWrite) {
// Assume elements non-null for simplicity
Objects.requireNonNull(cs);
out.writeUtf8(cs);
}
}
@NotNull
@Override
public CharSequence[] read(Bytes in, @Nullable CharSequence[] using) {
int len = in.readInt();
if (using == null)
using = new CharSequence[len];
if (using.length != len)
using = Arrays.copyOf(using, len);
for (int i = 0; i < len; i++) {
CharSequence cs = using[i];
if (cs instanceof StringBuilder) {
in.readUtf8((StringBuilder) cs);
} else {
StringBuilder sb = new StringBuilder(0);
in.readUtf8(sb);
using[i] = sb;
}
}
return using;
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
// no fields to write
}
@Override
public void readMarshallable(@NotNull WireIn wireIn) {
// no fields to read
}
@Override
public CharSequenceArrayBytesMarshaller readResolve() {
return INSTANCE;
}
}
@@ -0,0 +1,40 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.map.ChronicleMap;
import org.junit.Assert;
import org.junit.Test;
public class CharSequenceArraySerializationTest {
@Test
public void charSequenceArraySerializationTest() {
try (ChronicleMap<String, CharSequence[]> map = ChronicleMap
.of(String.class, CharSequence[].class)
.averageKey("fruits")
.valueMarshaller(CharSequenceArrayBytesMarshaller.INSTANCE)
.averageValue(new CharSequence[]{"banana", "pineapple"})
.entries(2)
.create()) {
map.put("fruits", new CharSequence[]{"banana", "pineapple"});
map.put("vegetables", new CharSequence[] {"carrot", "potato"});
Assert.assertEquals(2, map.get("fruits").length);
Assert.assertEquals(2, map.get("vegetables").length);
}
}
}
@@ -0,0 +1,147 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.core.io.IORuntimeException;
import net.openhft.chronicle.hash.serialization.BytesReader;
import net.openhft.chronicle.hash.serialization.StatefulCopyable;
import net.openhft.chronicle.wire.WireIn;
import net.openhft.chronicle.wire.WireOut;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
public final class CharSequenceCustomEncodingBytesReader
implements BytesReader<CharSequence>,
StatefulCopyable<CharSequenceCustomEncodingBytesReader> {
// config fields, non-final because read in readMarshallable()
private Charset charset;
private int inputBufferSize;
// cache fields
private transient CharsetDecoder charsetDecoder;
private transient ByteBuffer inputBuffer;
private transient CharBuffer outputBuffer;
public CharSequenceCustomEncodingBytesReader(Charset charset, int inputBufferSize) {
this.charset = charset;
this.inputBufferSize = inputBufferSize;
initTransients();
}
private void initTransients() {
charsetDecoder = charset.newDecoder();
inputBuffer = ByteBuffer.allocate(inputBufferSize);
int outputBufferSize = (int) (inputBufferSize * charsetDecoder.averageCharsPerByte());
outputBuffer = CharBuffer.allocate(outputBufferSize);
}
@NotNull
@Override
public CharSequence read(Bytes in, @Nullable CharSequence using) {
long csLengthAsLong = in.readStopBit();
if (csLengthAsLong > Integer.MAX_VALUE) {
throw new IORuntimeException("cs len shouldn't be more than " + Integer.MAX_VALUE +
", " + csLengthAsLong + " read");
}
int csLength = (int) csLengthAsLong;
StringBuilder sb;
if (using instanceof StringBuilder) {
sb = (StringBuilder) using;
sb.setLength(0);
sb.ensureCapacity(csLength);
} else {
sb = new StringBuilder(csLength);
}
int remainingBytes = in.readInt();
charsetDecoder.reset();
inputBuffer.clear();
outputBuffer.clear();
boolean endOfInput = false;
// this loop inspired by the CharsetDecoder.decode(ByteBuffer) implementation
while (true) {
if (!endOfInput) {
int inputChunkSize = Math.min(inputBuffer.remaining(), remainingBytes);
inputBuffer.limit(inputBuffer.position() + inputChunkSize);
in.read(inputBuffer);
inputBuffer.flip();
remainingBytes -= inputChunkSize;
endOfInput = remainingBytes == 0;
}
CoderResult cr = inputBuffer.hasRemaining() ?
charsetDecoder.decode(inputBuffer, outputBuffer, endOfInput) :
CoderResult.UNDERFLOW;
if (cr.isUnderflow() && endOfInput)
cr = charsetDecoder.flush(outputBuffer);
if (cr.isUnderflow()) {
if (endOfInput) {
break;
} else {
inputBuffer.compact();
continue;
}
}
if (cr.isOverflow()) {
outputBuffer.flip();
sb.append(outputBuffer);
outputBuffer.clear();
continue;
}
try {
cr.throwException();
} catch (CharacterCodingException e) {
throw new IORuntimeException(e);
}
}
outputBuffer.flip();
sb.append(outputBuffer);
return sb;
}
@Override
public void readMarshallable(@NotNull WireIn wireIn) throws IORuntimeException {
charset = (Charset) wireIn.read(() -> "charset").object();
inputBufferSize = wireIn.read(() -> "inputBufferSize").int32();
initTransients();
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
wireOut.write(() -> "charset").object(charset);
wireOut.write(() -> "inputBufferSize").int32(inputBufferSize);
}
@Override
public CharSequenceCustomEncodingBytesReader copy() {
return new CharSequenceCustomEncodingBytesReader(charset, inputBufferSize);
}
}
@@ -0,0 +1,142 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.core.io.IORuntimeException;
import net.openhft.chronicle.hash.serialization.BytesWriter;
import net.openhft.chronicle.hash.serialization.StatefulCopyable;
import net.openhft.chronicle.wire.WireIn;
import net.openhft.chronicle.wire.WireOut;
import org.jetbrains.annotations.NotNull;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
public final class CharSequenceCustomEncodingBytesWriter
implements BytesWriter<CharSequence>,
StatefulCopyable<CharSequenceCustomEncodingBytesWriter> {
// config fields, non-final because read in readMarshallable()
private Charset charset;
private int inputBufferSize;
// cache fields
private transient CharsetEncoder charsetEncoder;
private transient CharBuffer inputBuffer;
private transient ByteBuffer outputBuffer;
public CharSequenceCustomEncodingBytesWriter(Charset charset, int inputBufferSize) {
this.charset = charset;
this.inputBufferSize = inputBufferSize;
initTransients();
}
private void initTransients() {
charsetEncoder = charset.newEncoder();
inputBuffer = CharBuffer.allocate(inputBufferSize);
int outputBufferSize = (int) (inputBufferSize * charsetEncoder.averageBytesPerChar());
outputBuffer = ByteBuffer.allocate(outputBufferSize);
}
@Override
public void write(Bytes out, @NotNull CharSequence cs) {
// Write the actual cs length for accurate StringBuilder.ensureCapacity() while reading
out.writeStopBit(cs.length());
long encodedSizePos = out.writePosition();
out.writeSkip(4);
charsetEncoder.reset();
inputBuffer.clear();
outputBuffer.clear();
int csPos = 0;
boolean endOfInput = false;
// this loop inspired by the CharsetEncoder.encode(CharBuffer) implementation
while (true) {
if (!endOfInput) {
int nextCsPos = Math.min(csPos + inputBuffer.remaining(), cs.length());
append(inputBuffer, cs, csPos, nextCsPos);
inputBuffer.flip();
endOfInput = nextCsPos == cs.length();
csPos = nextCsPos;
}
CoderResult cr = inputBuffer.hasRemaining() ?
charsetEncoder.encode(inputBuffer, outputBuffer, endOfInput) :
CoderResult.UNDERFLOW;
if (cr.isUnderflow() && endOfInput)
cr = charsetEncoder.flush(outputBuffer);
if (cr.isUnderflow()) {
if (endOfInput) {
break;
} else {
inputBuffer.compact();
continue;
}
}
if (cr.isOverflow()) {
outputBuffer.flip();
out.write(outputBuffer);
outputBuffer.clear();
continue;
}
try {
cr.throwException();
} catch (CharacterCodingException e) {
throw new IORuntimeException(e);
}
}
outputBuffer.flip();
out.write(outputBuffer);
out.writeInt(encodedSizePos, (int) (out.writePosition() - encodedSizePos - 4));
}
/**
* Need this method because {@link CharBuffer#append(CharSequence, int, int)} produces garbage
*/
private void append(CharBuffer charBuffer, CharSequence cs, int start, int end) {
for (int i = start; i < end; i++) {
charBuffer.put(cs.charAt(i));
}
}
@Override
public void readMarshallable(@NotNull WireIn wireIn) {
charset = (Charset) wireIn.read(() -> "charset").object();
inputBufferSize = wireIn.read(() -> "inputBufferSize").int32();
initTransients();
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
wireOut.write(() -> "charset").object(charset);
wireOut.write(() -> "inputBufferSize").int32(inputBufferSize);
}
@Override
public CharSequenceCustomEncodingBytesWriter copy() {
return new CharSequenceCustomEncodingBytesWriter(charset, inputBufferSize);
}
}
@@ -0,0 +1,74 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.map.ChronicleMap;
import org.junit.Assert;
import org.junit.Test;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import static org.junit.Assert.assertNotNull;
public class CustomCharSequenceEncodingTest {
@Test
public void customCharSequenceEncodingTest() {
Charset charset = StandardCharsets.UTF_8;
int charBufferSize = 4;
int bytesBufferSize = 8;
CharSequenceCustomEncodingBytesWriter writer =
new CharSequenceCustomEncodingBytesWriter(charset, charBufferSize);
CharSequenceCustomEncodingBytesReader reader =
new CharSequenceCustomEncodingBytesReader(charset, bytesBufferSize);
try (ChronicleMap<String, CharSequence> map = ChronicleMap
.of(String.class, CharSequence.class)
.valueMarshallers(reader, writer)
.averageKey("Russian")
.averageValue("Всем нравится субботний вечерок")
.entries(10)
.create()) {
map.put("Russian", "Всем нравится субботний вечерок");
map.put("", "Quick brown fox jumps over the lazy dog");
}
}
@Test
public void gbkCharSequenceEncodingTest() {
Charset charset = Charset.forName("GBK");
int charBufferSize = 100;
int bytesBufferSize = 200;
CharSequenceCustomEncodingBytesWriter writer =
new CharSequenceCustomEncodingBytesWriter(charset, charBufferSize);
CharSequenceCustomEncodingBytesReader reader =
new CharSequenceCustomEncodingBytesReader(charset, bytesBufferSize);
try (ChronicleMap<String, CharSequence> englishToChinese = ChronicleMap
.of(String.class, CharSequence.class)
.valueMarshallers(reader, writer)
.averageKey("hello")
.averageValue("你好")
.entries(10)
.create()) {
englishToChinese.put("hello", "你好");
englishToChinese.put("bye", "再见");
Assert.assertEquals("你好", englishToChinese.get("hello").toString());
Assert.assertEquals("再见", englishToChinese.get("bye").toString());
}
}
}
@@ -1,59 +0,0 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.serialization.SizedReader;
import net.openhft.chronicle.hash.serialization.impl.EnumMarshallable;
import org.jetbrains.annotations.NotNull;
import java.util.Arrays;
enum LongPairArrayReader implements SizedReader<LongPair[]>, EnumMarshallable<LongPairArrayReader> {
INSTANCE;
@NotNull
@Override
public LongPair[] read(@NotNull Bytes in, long size, LongPair[] using) {
if (size > Integer.MAX_VALUE * 16L)
throw new IllegalStateException("LongPair[] size couldn't be " + (size / 16L));
int resLen = (int) (size / 16L);
LongPair[] res;
if (using != null) {
if (using.length == resLen) {
res = using;
} else {
res = Arrays.copyOf(using, resLen);
}
} else {
res = new LongPair[resLen];
}
for (int i = 0; i < resLen; i++) {
LongPair pair = res[i];
if (pair == null)
res[i] = pair = new LongPair();
pair.first = in.readLong();
pair.second = in.readLong();
}
return res;
}
@Override
public LongPairArrayReader readResolve() {
return INSTANCE;
}
}
@@ -1,44 +0,0 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.hash.serialization.SizedWriter;
import net.openhft.chronicle.hash.serialization.impl.EnumMarshallable;
import org.jetbrains.annotations.NotNull;
enum LongPairArrayWriter implements SizedWriter<LongPair[]>, EnumMarshallable<LongPairArrayWriter> {
INSTANCE;
@Override
public long size(@NotNull LongPair[] toWrite) {
return toWrite.length * 16L;
}
@Override
public void write(@NotNull Bytes out, long size, @NotNull LongPair[] toWrite) {
for (LongPair pair : toWrite) {
out.writeLong(pair.first);
out.writeLong(pair.second);
}
}
@Override
public LongPairArrayWriter readResolve() {
return INSTANCE;
}
}
@@ -16,6 +16,14 @@
package net.openhft.chronicle.map.fromdocs;
public final class LongPair {
long first, second;
public final class Point {
public static Point of(double x, double y) {
Point p = new Point();
p.x = x;
p.y = y;
return p;
}
double x, y;
}
@@ -0,0 +1,102 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.core.io.IORuntimeException;
import net.openhft.chronicle.core.util.ReadResolvable;
import net.openhft.chronicle.hash.serialization.SizedReader;
import net.openhft.chronicle.hash.serialization.SizedWriter;
import net.openhft.chronicle.wire.WireIn;
import net.openhft.chronicle.wire.WireOut;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.List;
public enum PointListSizedMarshaller
implements SizedReader<List<Point>>, SizedWriter<List<Point>>,
ReadResolvable<PointListSizedMarshaller> {
INSTANCE;
/** A point takes 16 bytes in serialized form: 8 bytes for both x and y value */
private static final long ELEMENT_SIZE = 16;
@Override
public long size(@NotNull List<Point> toWrite) {
return toWrite.size() * ELEMENT_SIZE;
}
@Override
public void write(Bytes out, long size, @NotNull List<Point> toWrite) {
toWrite.forEach(point -> {
out.writeDouble(point.x);
out.writeDouble(point.y);
});
}
@NotNull
@Override
public List<Point> read(@NotNull Bytes in, long size, List<Point> using) {
if (size % ELEMENT_SIZE != 0) {
throw new IORuntimeException("Bytes size should be a multiple of " + ELEMENT_SIZE +
", " + size + " read");
}
long listSizeAsLong = size / ELEMENT_SIZE;
if (listSizeAsLong > Integer.MAX_VALUE) {
throw new IORuntimeException("List size couldn't be more than " + Integer.MAX_VALUE +
", " + listSizeAsLong + " read");
}
int listSize = (int) listSizeAsLong;
if (using == null) {
using = new ArrayList<>(listSize);
for (int i = 0; i < listSize; i++) {
using.add(null);
}
} else if (using.size() < listSize) {
while (using.size() < listSize) {
using.add(null);
}
} else if (using.size() > listSize) {
using.subList(listSize, using.size()).clear();
}
for (int i = 0; i < listSize; i++) {
Point point = using.get(i);
if (point == null)
using.set(i, point = new Point());
point.x = in.readDouble();
point.y = in.readDouble();
}
return using;
}
@Override
public void writeMarshallable(@NotNull WireOut wireOut) {
// no fields to write
}
@Override
public void readMarshallable(@NotNull WireIn wireIn) {
// no fields to read
}
@Override
public PointListSizedMarshaller readResolve() {
return INSTANCE;
}
}
@@ -0,0 +1,46 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.fromdocs;
import net.openhft.chronicle.map.ChronicleMap;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
import static java.util.Arrays.asList;
import static net.openhft.chronicle.map.fromdocs.Point.of;
public class PointSerializationTest {
@Test
public void pointSerializationTest() {
try (ChronicleMap<String, List<Point>> objects = ChronicleMap
.of(String.class, (Class<List<Point>>) (Class) List.class)
.averageKey("range")
.valueMarshaller(PointListSizedMarshaller.INSTANCE)
.averageValue(asList(of(0, 0), of(1, 1)))
.entries(10)
.create()) {
objects.put("range", asList(of(0, 0), of(1, 1)));
objects.put("square", asList(of(0, 0), of(0, 100), of(100, 100), of(100, 0)));
Assert.assertEquals(2, objects.get("range").size());
Assert.assertEquals(4, objects.get("square").size());
}
}
}