diff --git a/docs/CM_Tutorial_Bytes.adoc b/docs/CM_Tutorial_Bytes.adoc new file mode 100755 index 000000000..ea4515c69 --- /dev/null +++ b/docs/CM_Tutorial_Bytes.adoc @@ -0,0 +1,411 @@ += `BytesWriter` and `BytesReader` +Neil Clifford +:toc: macro +:toclevels: 1 +:css-signature: demo +:toc-placement: macro +:icons: font + +toc::[] + +This pair of interfaces is configured via `ChronicleMapBuilder.keyMarshallers()` or +`valueMarshallers()` for the key or value type of the map, respectively. + +This pair of interfaces is the most suitable, if the size of the serialized form is not known +in advance, i. e. the easiest way to compute the size of the serialized form of some object of the +type, is performing serialization itself and looking at the number of written bytes. + +This pair of interfaces is the least efficient and the simplest to implement, so it should also be +used when efficiency is not the top priority, or when gains of using other pairs of interfaces +(which are more complicated to implement) are marginal. + +Basically you should implement two serialization methods: + + - `void write(Bytes out, @NotNull T toWrite)` from `BytesWriter` interface, which writes the given + `toWrite` instance of the serialized type to the given `out` bytes sink. + - `T read(Bytes in, @Nullable T using)` from `BytesReader` interface, which reads the serialized + object into the given `using` instance (if the serialized type is reusable, the `using` object is + not `null`, and suitable for reusing for this particular serialized object), or a newly created + instance. The returned object contains the serialized data; it may be identical or not identical to + the passed `using` instance. + +For example, here is the implementation of `BytesWriter` and `BytesReader` for `CharSequence[]` +value type (array of CharSequences): + +```java +public final class CharSequenceArrayBytesMarshaller + implements BytesWriter, BytesReader, + ReadResolvable { + + static final CharSequenceArrayBytesMarshaller INSTANCE = new CharSequenceArrayBytesMarshaller(); + + private CharSequenceArrayBytesMarshaller() {} + + @Override + public void write(Bytes out, @NotNull CharSequence[] toWrite) { + out.writeInt(toWrite.length); + for (CharSequence cs : toWrite) { + // Assume elements non-null for simplicity + Objects.requireNonNull(cs); + out.writeUtf8(cs); + } + } + + @NotNull + @Override + public CharSequence[] read(Bytes in, @Nullable CharSequence[] using) { + int len = in.readInt(); + if (using == null) + using = new CharSequence[len]; + if (using.length != len) + using = Arrays.copyOf(using, len); + for (int i = 0; i < len; i++) { + CharSequence cs = using[i]; + if (cs instanceof StringBuilder) { + in.readUtf8((StringBuilder) cs); + } else { + StringBuilder sb = new StringBuilder(0); + in.readUtf8(sb); + using[i] = sb; + } + } + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public CharSequenceArrayBytesMarshaller readResolve() { + return INSTANCE; + } +} +``` + +Usage example: + +```java +try (ChronicleMap map = ChronicleMap + .of(String.class, CharSequence[].class) + .averageKey("fruits") + .valueMarshaller(CharSequenceArrayBytesMarshaller.INSTANCE) + .averageValue(new CharSequence[]{"banana", "pineapple"}) + .entries(2) + .create()) { + map.put("fruits", new CharSequence[]{"banana", "pineapple"}); + map.put("vegetables", new CharSequence[] {"carrot", "potato"}); + Assert.assertEquals(2, map.get("fruits").length); + Assert.assertEquals(2, map.get("vegetables").length); +} +``` + +The total size of serialization form for some `CharSequence[]` array is 4 bytes for storing the +array length, plus the sum of sizes of all CharSequences, in UTF-8 encoding. Computing this size +without actual encoding has comparable computational cost with performing actual encoding, that +makes `CharSequence[]` type to meet the second criteria (see above) which makes `BytesWriter` and +`BytesReader` the most suitable pair of serialization interfaces to implement for the type. + +Note how `read()` implementation attempts to reuse not only the array object, but also the elements, +minimizing the amount of produced garbage. This is a recommended practice. + +Some additional notes: + + - If the reader or writer interface implementation is not configurable and doesn't have + per-instance cache or state fields, i. e. it doesn't have instance fields at all, there is + a convention to make such implementation classes `final`, give them a `private` constructor and + expose a single `INSTANCE` constant - a sole instance of this implementation in the JVM. + - *Don't* make marshaller class `enum`, because there are some issues with `enum` serialization/ + deserialization. + - For such no-state implementations, don't forget to implement `ReadResolvable` + interface and return `INSTANCE`, otherwise you have no guarantee that `INSTANCE` constant is the + only alive instance of this implementation in the JVM. + - If *both* writer and reader interface implementations have no fields, it might be a good idea + to merge them into a single type, in order to keep writing and reading logic together. + +=== Custom `CharSequence` encoding + +Another example shows how to serialize `CharSequence`s using custom encoding (rather than UTF-8): + +Writer: + +```java +public final class CharSequenceCustomEncodingBytesWriter + implements BytesWriter, + StatefulCopyable { + + // config fields, non-final because read in readMarshallable() + private Charset charset; + private int inputBufferSize; + + // cache fields + private transient CharsetEncoder charsetEncoder; + private transient CharBuffer inputBuffer; + private transient ByteBuffer outputBuffer; + + public CharSequenceCustomEncodingBytesWriter(Charset charset, int inputBufferSize) { + this.charset = charset; + this.inputBufferSize = inputBufferSize; + initTransients(); + } + + private void initTransients() { + charsetEncoder = charset.newEncoder(); + inputBuffer = CharBuffer.allocate(inputBufferSize); + int outputBufferSize = (int) (inputBufferSize * charsetEncoder.averageBytesPerChar()); + outputBuffer = ByteBuffer.allocate(outputBufferSize); + } + + @Override + public void write(Bytes out, @NotNull CharSequence cs) { + // Write the actual cs length for accurate StringBuilder.ensureCapacity() while reading + out.writeStopBit(cs.length()); + long encodedSizePos = out.writePosition(); + out.writeSkip(4); + charsetEncoder.reset(); + inputBuffer.clear(); + outputBuffer.clear(); + int csPos = 0; + boolean endOfInput = false; + // this loop inspired by the CharsetEncoder.encode(CharBuffer) implementation + while (true) { + if (!endOfInput) { + int nextCsPos = Math.min(csPos + inputBuffer.remaining(), cs.length()); + append(inputBuffer, cs, csPos, nextCsPos); + inputBuffer.flip(); + endOfInput = nextCsPos == cs.length(); + csPos = nextCsPos; + } + + CoderResult cr = inputBuffer.hasRemaining() ? + charsetEncoder.encode(inputBuffer, outputBuffer, endOfInput) : + CoderResult.UNDERFLOW; + + if (cr.isUnderflow() && endOfInput) + cr = charsetEncoder.flush(outputBuffer); + + if (cr.isUnderflow()) { + if (endOfInput) { + break; + } else { + inputBuffer.compact(); + continue; + } + } + + if (cr.isOverflow()) { + outputBuffer.flip(); + writeOutputBuffer(out); + outputBuffer.clear(); + continue; + } + + try { + cr.throwException(); + } catch (CharacterCodingException e) { + throw new IORuntimeException(e); + } + } + outputBuffer.flip(); + writeOutputBuffer(out); + + out.writeInt(encodedSizePos, (int) (out.writePosition() - encodedSizePos - 4)); + } + + private void writeOutputBuffer(Bytes out) { + int remaining = outputBuffer.remaining(); + out.write(out.writePosition(), outputBuffer, 0, remaining); + out.writeSkip(remaining); + } + + /** + * Need this method because {@link CharBuffer#append(CharSequence, int, int)} produces garbage + */ + private static void append(CharBuffer charBuffer, CharSequence cs, int start, int end) { + for (int i = start; i < end; i++) { + charBuffer.put(cs.charAt(i)); + } + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + charset = (Charset) wireIn.read(() -> "charset").object(); + inputBufferSize = wireIn.read(() -> "inputBufferSize").int32(); + initTransients(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + wireOut.write(() -> "charset").object(charset); + wireOut.write(() -> "inputBufferSize").int32(inputBufferSize); + } + + @Override + public CharSequenceCustomEncodingBytesWriter copy() { + return new CharSequenceCustomEncodingBytesWriter(charset, inputBufferSize); + } +} +``` + +Reader: + +```java +public final class CharSequenceCustomEncodingBytesReader + implements BytesReader, + StatefulCopyable { + + // config fields, non-final because read in readMarshallable() + private Charset charset; + private int inputBufferSize; + + // cache fields + private transient CharsetDecoder charsetDecoder; + private transient ByteBuffer inputBuffer; + private transient CharBuffer outputBuffer; + + public CharSequenceCustomEncodingBytesReader(Charset charset, int inputBufferSize) { + this.charset = charset; + this.inputBufferSize = inputBufferSize; + initTransients(); + } + + private void initTransients() { + charsetDecoder = charset.newDecoder(); + inputBuffer = ByteBuffer.allocate(inputBufferSize); + int outputBufferSize = (int) (inputBufferSize * charsetDecoder.averageCharsPerByte()); + outputBuffer = CharBuffer.allocate(outputBufferSize); + } + + @NotNull + @Override + public CharSequence read(Bytes in, @Nullable CharSequence using) { + long csLengthAsLong = in.readStopBit(); + if (csLengthAsLong > Integer.MAX_VALUE) { + throw new IORuntimeException("cs len shouldn't be more than " + Integer.MAX_VALUE + + ", " + csLengthAsLong + " read"); + } + int csLength = (int) csLengthAsLong; + StringBuilder sb; + if (using instanceof StringBuilder) { + sb = (StringBuilder) using; + sb.setLength(0); + sb.ensureCapacity(csLength); + } else { + sb = new StringBuilder(csLength); + } + + int remainingBytes = in.readInt(); + charsetDecoder.reset(); + inputBuffer.clear(); + outputBuffer.clear(); + boolean endOfInput = false; + // this loop inspired by the CharsetDecoder.decode(ByteBuffer) implementation + while (true) { + if (!endOfInput) { + int inputChunkSize = Math.min(inputBuffer.remaining(), remainingBytes); + inputBuffer.limit(inputBuffer.position() + inputChunkSize); + in.read(inputBuffer); + inputBuffer.flip(); + remainingBytes -= inputChunkSize; + endOfInput = remainingBytes == 0; + } + + CoderResult cr = inputBuffer.hasRemaining() ? + charsetDecoder.decode(inputBuffer, outputBuffer, endOfInput) : + CoderResult.UNDERFLOW; + + if (cr.isUnderflow() && endOfInput) + cr = charsetDecoder.flush(outputBuffer); + + if (cr.isUnderflow()) { + if (endOfInput) { + break; + } else { + inputBuffer.compact(); + continue; + } + } + + if (cr.isOverflow()) { + outputBuffer.flip(); + sb.append(outputBuffer); + outputBuffer.clear(); + continue; + } + + try { + cr.throwException(); + } catch (CharacterCodingException e) { + throw new IORuntimeException(e); + } + } + outputBuffer.flip(); + sb.append(outputBuffer); + + return sb; + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) throws IORuntimeException { + charset = (Charset) wireIn.read(() -> "charset").object(); + inputBufferSize = wireIn.read(() -> "inputBufferSize").int32(); + initTransients(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + wireOut.write(() -> "charset").object(charset); + wireOut.write(() -> "inputBufferSize").int32(inputBufferSize); + } + + @Override + public CharSequenceCustomEncodingBytesReader copy() { + return new CharSequenceCustomEncodingBytesReader(charset, inputBufferSize); + } +} +``` + +Usage example: + +```java +Charset charset = Charset.forName("GBK"); +int charBufferSize = 100; +int bytesBufferSize = 200; +CharSequenceCustomEncodingBytesWriter writer = + new CharSequenceCustomEncodingBytesWriter(charset, charBufferSize); +CharSequenceCustomEncodingBytesReader reader = + new CharSequenceCustomEncodingBytesReader(charset, bytesBufferSize); +try (ChronicleMap englishToChinese = ChronicleMap + .of(String.class, CharSequence.class) + .valueMarshallers(reader, writer) + .averageKey("hello") + .averageValue("你好") + .entries(10) + .create()) { + englishToChinese.put("hello", "你好"); + englishToChinese.put("bye", "再见"); + + Assert.assertEquals("你好", englishToChinese.get("hello").toString()); + Assert.assertEquals("再见", englishToChinese.get("bye").toString()); +} +``` + +Some notes on this case of custom serialization: + + - Both `CharSequenceCustomEncodingBytesWriter` and `CharSequenceCustomEncodingBytesReader` have + configurations (charset and input buffer size), hence they are implemented as normal classes rather + than classes with `private` constructors and a single `INSTANCE`. + - Both writer and reader classes have some "cache" fields, their contents are mutated during + writing and reading. That is why they have to implement `StatefulCopyable` interface. See + [Understanding `StatefulCopyable`](#understanding-statefulcopyable) section for more infromation on + this. + + +''' +<> diff --git a/docs/CM_Tutorial_DataAccess.adoc b/docs/CM_Tutorial_DataAccess.adoc new file mode 100755 index 000000000..72caacbdf --- /dev/null +++ b/docs/CM_Tutorial_DataAccess.adoc @@ -0,0 +1,162 @@ += `DataAccess` and `SizedReader` +Neil Clifford +:toc: macro +:toclevels: 1 +:css-signature: demo +:toc-placement: macro +:icons: font + +toc::[] + +This pair of interfaces is configured via `ChronicleMapBuilder.keyReaderAndDataAccess()` or +`valueReaderAndDataAccess()` for the key, or value type, of the map, respectively. + +The reader part, `SizedReader`, is the same as in <> pair, so `DataAccess` is an "advanced" interface to +replace `SizedWriter`. + +The main method in `DataAccess` is `Data getData(@NotNull T instance)`, it returns a `Data` +accessor which is used to write "serialized" form of the instance to off-heap memory. `Data.size()` +on the returned `Data` object is used for the same purpose as `SizedWriter.size()` method in +<> pair interfaces. `Data.writeTo()` +is used instead of `SizedWriter.write()`. + +`DataAccess` assumes that the `Data` object, returned from the `getData()` method is cached in some way. That is why it also has `uninit()` method to clear references to the serialized object after query operation to a Chronicle Map is complete to prevent memory leaks. This, in turn, implies that `DataAccess` implementation is stateful. Therefore, `DataAccess` is made a subinterface of +`StatefulCopyable` to force all `DataAccess` implementations to implement `StatefulCopyable` as well. See [Understanding `StatefulCopyable`](#understanding-statefulcopyable) for more infromation +on this. If your `DataAccess` implementation is not actually stateful, it is free to return `this` from the `StatefulCopyable.copy()` method. + +The `DataAccess` interface is primarily intended for "serializing" objects that are already sequences of bytes, and in fact doesn't require serialization, like `byte[]`, `ByteBuffer`, arrays of Java +primitives. For such types of objects, `DataAccess` allows bypassing of the intermediate buffering, copying +data directly from objects to Chronicle Map's off-heap memory. + +For example, look at the `DataAccess` implementation for `byte[]`: + +```java +public final class ByteArrayDataAccess extends AbstractData implements DataAccess { + + /** Cache field */ + private transient HeapBytesStore bs; + + /** State field */ + private transient byte[] array; + + public ByteArrayDataAccess() { + initTransients(); + } + + private void initTransients() { + bs = HeapBytesStore.uninitialized(); + } + + @Override + public RandomDataInput bytes() { + return bs; + } + + @Override + public long offset() { + return bs.start(); + } + + @Override + public long size() { + return bs.capacity(); + } + + @Override + public byte[] get() { + return array; + } + + @Override + public byte[] getUsing(@Nullable byte[] using) { + if (using == null || using.length != array.length) + using = new byte[array.length]; + System.arraycopy(array, 0, using, 0, array.length); + return using; + } + + @Override + public Data getData(@NotNull byte[] instance) { + array = instance; + bs.init(instance); + return this; + } + + @Override + public void uninit() { + array = null; + bs.uninit(); + } + + @Override + public DataAccess copy() { + return new ByteArrayDataAccess(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + initTransients(); + } +} +``` + +Note that the `getData()` method returns `this`, and the `DataAccess` implementation implements `Data` interface as well. This is recommended practice, because it reduces the number of objects involved (hence pointer chasing), and keeps `DataAccess`, and `Data` logic together. + +`Data` interface puts constraints on `equals()`, `hashCode()`, and `toString()` implementations. This is why `ByteArrayDataAccess` subclasses `AbstractData`, and inherits proper implementations from it. +It is OK for serializer strategy implementation to have `equals()`, `hashCode()`, and `toString()` from a very different domain, because those methods are never called on serializers inside Chronicle Map. + +The easiest way to implement `equals()`, `hashCode()`, and `toString()` is to extend the `AbstractData` class. If it is not possible (the `Data` implementation already extends some other class), do this +by delegating to `dataEquals()`, `dataHashCode()`, and `dataToString()` default methods, provided in the `Data` interface. + +Corresponding `SizedReader` for `byte[]`: + +```java +public final class ByteArraySizedReader + implements SizedReader, Marshallable, ReadResolvable { + + public static final ByteArraySizedReader INSTANCE = new ByteArraySizedReader(); + + private ByteArraySizedReader() {} + + @NotNull + @Override + public byte[] read(@NotNull Bytes in, long size, @Nullable byte[] using) { + if (size < 0L || size > (long) Integer.MAX_VALUE) { + throw new IORuntimeException("byte[] size should be non-negative int, " + + size + " given. Memory corruption?"); + } + int arrayLength = (int) size; + if (using == null || arrayLength != using.length) + using = new byte[arrayLength]; + in.read(using); + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public ByteArraySizedReader readResolve() { + return INSTANCE; + } +} +``` + +NOTE: No special coding is required to use this pair of serializers with `byte[]` keys of values in Chronicle Map. It is used as the default, if you configure `byte[]` key, or value type. + + +''' +<> \ No newline at end of file diff --git a/docs/CM_Tutorial_Sized.adoc b/docs/CM_Tutorial_Sized.adoc new file mode 100755 index 000000000..3d5d21e68 --- /dev/null +++ b/docs/CM_Tutorial_Sized.adoc @@ -0,0 +1,159 @@ += `SizedWriter` and `SizedReader` +Neil Clifford +:toc: macro +:toclevels: 1 +:css-signature: demo +:toc-placement: macro +:icons: font + +toc::[] + +This pair of interfaces is configured via `ChronicleMapBuilder.keyMarshallers()` or +`valueMarshallers()` for the key or value type of the map, respectively (overloaded methods, those +for [`BytesWriter` and `BytesReader`](#byteswriter-and-bytesreader) interfaces have the same names). + +The main two methods to implement in `SizedWriter` interface: + + - `long size(@NotNull T toWrite);` returns the number of bytes, which is written by the subsequent + `write()` method given the same `toWrite` instance of the serialized type. + - `void write(Bytes out, long size, @NotNull T toWrite);` writes the given `toWrite` instance to + the given `out` bytes sink. Additionally `size` is provided, which is the value computed for the + same `toWrite` instance by calling `size()` method. Hence, when `SizedWriter` is used internally, + `size()` method is always called before `write()`, caching logic in `SizedWriter` implementation + may rely on this. This method should advance `writePosition()` of the given out `Bytes` exactly by + the given `size` number of bytes. + +`SizedReader`'s `T read(Bytes in, long size, @Nullable T using);` method is similar to the +corresponding `read()` method in `BytesReader` interface, except the size of the serialized object +is provided. + +This pair of interfaces is suitable, if + + - The serialized form of the type effectively includes the size of the rest of the serialized form + in the beginning, + - But the type is not a plain sequence of bytes like `byte[]` or `ByteBuffer` (for those types + [`DataAccess` and `SizedReader`](#dataaccess-and-sizedreader) pair of interfaces is a much better + choice). + +Examples of such types include lists and arrays of constant-sized elements. + +Compared to `BytesWriter` and `BytesReader`, this pair of interfaces allows to share the information +about the serialized form between serialization logic and the Chronicle Map, saving a few bytes by +storing the serialization size only once rather than twice. This pair of interfaces is not much more +difficult to implement, but the gains are also not big. + +Example: serializing lists of simple `Point` structures: + +```java +public final class Point { + + public static Point of(double x, double y) { + Point p = new Point(); + p.x = x; + p.y = y; + return p; + } + + double x, y; +} +``` + +Serializer implementation: + +```java +public final class PointListSizedMarshaller + implements SizedReader>, SizedWriter>, + ReadResolvable { + + static final PointListSizedMarshaller INSTANCE = new PointListSizedMarshaller(); + + private PointListSizedMarshaller() {} + + /** A point takes 16 bytes in serialized form: 8 bytes for both x and y value */ + private static final long ELEMENT_SIZE = 16; + + @Override + public long size(@NotNull List toWrite) { + return toWrite.size() * ELEMENT_SIZE; + } + + @Override + public void write(Bytes out, long size, @NotNull List toWrite) { + toWrite.forEach(point -> { + out.writeDouble(point.x); + out.writeDouble(point.y); + }); + } + + @NotNull + @Override + public List read(@NotNull Bytes in, long size, List using) { + if (size % ELEMENT_SIZE != 0) { + throw new IORuntimeException("Bytes size should be a multiple of " + ELEMENT_SIZE + + ", " + size + " read"); + } + long listSizeAsLong = size / ELEMENT_SIZE; + if (listSizeAsLong > Integer.MAX_VALUE) { + throw new IORuntimeException("List size couldn't be more than " + Integer.MAX_VALUE + + ", " + listSizeAsLong + " read"); + } + int listSize = (int) listSizeAsLong; + if (using == null) { + using = new ArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + using.add(null); + } + } else if (using.size() < listSize) { + while (using.size() < listSize) { + using.add(null); + } + } else if (using.size() > listSize) { + using.subList(listSize, using.size()).clear(); + } + for (int i = 0; i < listSize; i++) { + Point point = using.get(i); + if (point == null) + using.set(i, point = new Point()); + point.x = in.readDouble(); + point.y = in.readDouble(); + } + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public PointListSizedMarshaller readResolve() { + return INSTANCE; + } +} +``` + +Usage example: + +```java +try (ChronicleMap> objects = ChronicleMap + .of(String.class, (Class>) (Class) List.class) + .averageKey("range") + .valueMarshaller(PointListSizedMarshaller.INSTANCE) + .averageValue(asList(of(0, 0), of(1, 1))) + .entries(10) + .create()) { + objects.put("range", asList(of(0, 0), of(1, 1))); + objects.put("square", asList(of(0, 0), of(0, 100), of(100, 100), of(100, 0))); + + Assert.assertEquals(2, objects.get("range").size()); + Assert.assertEquals(4, objects.get("square").size()); +} +``` + +''' +<> \ No newline at end of file