From 0ec0eed5eb2dded54ff6e6523b2d923aa0949f11 Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 19 Mar 2016 16:47:26 +0700 Subject: [PATCH] Add tutorial section about custom serialization, add custom serialization tests, improve Javadocs in net.openhft.chronicle.hash.serialization package --- README.md | 865 +++++++++++++++++- .../hash/serialization/BytesReader.java | 16 +- .../hash/serialization/BytesWriter.java | 12 +- .../hash/serialization/DataAccess.java | 39 + .../hash/serialization/ListMarshaller.java | 29 + .../hash/serialization/MapMarshaller.java | 8 + .../hash/serialization/SetMarshaller.java | 29 + .../hash/serialization/SizeMarshaller.java | 25 +- .../hash/serialization/SizedReader.java | 54 +- .../hash/serialization/SizedWriter.java | 37 +- .../hash/serialization/StatefulCopyable.java | 32 +- .../impl/ByteArrayDataAccess.java | 10 +- .../impl/ByteArraySizedReader.java | 19 +- .../hash/serialization/package-info.java | 9 +- .../chronicle/map/DefaultValueTest.java | 2 +- ...t.java => PointListSerializationTest.java} | 2 +- .../CharSequenceArrayBytesMarshaller.java | 85 ++ .../CharSequenceArraySerializationTest.java | 40 + ...CharSequenceCustomEncodingBytesReader.java | 147 +++ ...CharSequenceCustomEncodingBytesWriter.java | 142 +++ .../CustomCharSequenceEncodingTest.java | 74 ++ .../map/fromdocs/LongPairArrayReader.java | 59 -- .../map/fromdocs/LongPairArrayWriter.java | 44 - .../fromdocs/{LongPair.java => Point.java} | 12 +- .../fromdocs/PointListSizedMarshaller.java | 102 +++ .../map/fromdocs/PointSerializationTest.java | 46 + 26 files changed, 1710 insertions(+), 229 deletions(-) rename src/test/java/net/openhft/chronicle/map/example/{ComplexSerializationTest.java => PointListSerializationTest.java} (98%) create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArrayBytesMarshaller.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArraySerializationTest.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesReader.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesWriter.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/CustomCharSequenceEncodingTest.java delete mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayReader.java delete mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayWriter.java rename src/test/java/net/openhft/chronicle/map/fromdocs/{LongPair.java => Point.java} (80%) create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/PointListSizedMarshaller.java create mode 100644 src/test/java/net/openhft/chronicle/map/fromdocs/PointSerializationTest.java diff --git a/README.md b/README.md index 1e823a594..3dd309147 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,9 @@ Chronicle Map doesn't support - [Download the library](#download-the-library) - [Create a `ChronicleMap` Instance](#create-a-chroniclemap-instance) - [Key and Value Types](#key-and-value-types) + - [Custom serializers](#custom-serializers) + - [Custom `CharSequence` encoding](#custom-charsequence-encoding) + - [Custom serialization checklist](#custom-serialization-checklist) - [`ChronicleMap` instance usage patterns](#chroniclemap-instance-usage-patterns) - [Single-key queries](#single-key-queries) - [Multi-key queries](#multi-key-queries) @@ -303,15 +306,28 @@ accessed concurrently the same way as e. g. `ConcurrentHashMap`. Either key or value type of `ChronicleMap` could be: - - Boxed primitive: `Integer`, `Long`, `Double`, etc. - - `String` or `CharSequence` - - Array of Java primitives, e. g. `byte[]`, `char[]` or `int[]` - - Any type implementing `BytesMarshallable` from [Chronicle Bytes]( - https://github.com/OpenHFT/Chronicle-Bytes) - - Any [value interface](https://github.com/OpenHFT/Chronicle-Values) - - Any Java type implementing `Serializable` or `Externalizable` interface - - *Any other type*, if `.keyMarshaller()` or `.valueMarshaller()` (for the key or value type - respectively) is additionally configured in the `ChronicleMapBuilder`. + - Types with best possible out of the box support: + - Any [value interface](https://github.com/OpenHFT/Chronicle-Values) + - Any class implementing [`Byteable`]( + http://openhft.github.io/Chronicle-Bytes/apidocs/net/openhft/chronicle/bytes/Byteable.html) + interface from [Chronicle Bytes](https://github.com/OpenHFT/Chronicle-Bytes) + - Any class implementing [`BytesMarshallable`]( + http://openhft.github.io/Chronicle-Bytes/apidocs/net/openhft/chronicle/bytes/BytesMarshallable.html) + interface from Chronicle Bytes. The implementation class should have a public no-arg constructor. + - `byte[]` and `ByteBuffer` + - `CharSequence`, `String` and `StringBuilder`. Note that these char sequence types are + serialized using UTF-8 encoding by default. If you need a different encoding, refer to the + example in the [custom `CharSequence` encoding](#custom-charsequence-encoding) section. + - `Integer`, `Long` and `Double` + + - Types supported out of the box, but not particularly efficiently. You might want to implement + more efficient [custom serializers](#custom-serializers) for them: + - Any class implementing `java.io.Externalizable`. The implementation class should have a public + no-arg constructor. + - Any type implementing `java.io.Serializable`, including boxed primitive types (except listed + above) and array types + + - Any other type, if [custom serializers](#custom-serializers) are provided. **Prefer [value interfaces](https://github.com/OpenHFT/Chronicle-Values).** They don't generate garbage and have close to zero serialization/deserialization costs. Prefer them even to boxed @@ -353,6 +369,835 @@ ChronicleSet uuids = .create(); ``` +#### Custom serializers + +Chronicle Map allows to configure custom marshallers for key or value types which are not supported +out of the box, or serialize supported types like `String` in some custom way (i. e. in encoding, +different from UTF-8), or serialize supported types more efficiently than it is done by default. + +There are three pairs of serialization interfaces, only one of them should be implemented and +provided to the `ChronicleMapBuilder` for the key or value type: + +##### `BytesWriter` and `BytesReader` + +This pair of interfaces is configured via `ChronicleMapBuilder.keyMarshallers()` or +`valueMarshallers()` for the key or value type of the map, respectively. + +This pair of interfaces is the most suitable, if the size of the serialized form is not known +in advance, i. e. the easiest way to compute the size of the serialized form of some object of the +type, is performing serialization itself and looking at the number of written bytes. + +This pair of interfaces is the least efficient and the simplest to implement, so it should also be +used when efficiency is not the top priority, or when gains of using other pairs of interfaces +(which are more complicated to implement) are marginal. + +Basically you should implement two serialization methods: + + - `void write(Bytes out, @NotNull T toWrite)` from `BytesWriter` interface, which writes the given + `toWrite` instance of the serialized type to the given `out` bytes sink. + - `T read(Bytes in, @Nullable T using)` from `BytesReader` interface, which reads the serialized + object into the given `using` instance (if the serialized type is reusable, the `using` object is + not `null`, and suitable for reusing for this particular serialized object), or a newly created + instance. The returned object contains the serialized data; it may be identical or not identical to + the passed `using` instance. + +For example, here is the implementation of `BytesWriter` and `BytesReader` for `CharSequence[]` +value type (array of CharSequences): + +```java +public enum CharSequenceArrayBytesMarshaller + implements BytesWriter, BytesReader, + ReadResolvable { + + INSTANCE; + + @Override + public void write(Bytes out, @NotNull CharSequence[] toWrite) { + out.writeInt(toWrite.length); + for (CharSequence cs : toWrite) { + // Assume elements non-null for simplicity + Objects.requireNonNull(cs); + out.writeUtf8(cs); + } + } + + @NotNull + @Override + public CharSequence[] read(Bytes in, @Nullable CharSequence[] using) { + int len = in.readInt(); + if (using == null) + using = new CharSequence[len]; + if (using.length != len) + using = Arrays.copyOf(using, len); + for (int i = 0; i < len; i++) { + CharSequence cs = using[i]; + if (cs instanceof StringBuilder) { + in.readUtf8((StringBuilder) cs); + } else { + StringBuilder sb = new StringBuilder(0); + in.readUtf8(sb); + using[i] = sb; + } + } + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public CharSequenceArrayBytesMarshaller readResolve() { + return INSTANCE; + } +} +``` + +Usage example: + +```java +try (ChronicleMap map = ChronicleMap + .of(String.class, CharSequence[].class) + .averageKey("fruits") + .valueMarshaller(CharSequenceArrayBytesMarshaller.INSTANCE) + .averageValue(new CharSequence[]{"banana", "pineapple"}) + .entries(2) + .create()) { + map.put("fruits", new CharSequence[]{"banana", "pineapple"}); + map.put("vegetables", new CharSequence[] {"carrot", "potato"}); + Assert.assertEquals(2, map.get("fruits").length); + Assert.assertEquals(2, map.get("vegetables").length); +} +``` + +The total size of serialization form for some `CharSequence[]` array is 4 bytes for storing the +array length, plus the sum of sizes of all CharSequences, in UTF-8 encoding. Computing this size +without actual encoding has comparable computational cost with performing actual encoding, that +makes `CharSequence[]` type to meet the second criteria (see above) which makes `BytesWriter` and +`BytesReader` the most suitable pair of serialization interfaces to implement for the type. + +Note how `read()` implementation attempts to reuse not only the array object, but also the elements, +minimizing the amount of produced garbage. This is a recommended practice. + +Some additional notes: + + - If the reader or writer interface implementation is not configurable and doesn't have + per-instance cache or state fields, i. e. it doesn't have instance fields at all, there is + a convention to make such implementations `enum`s with a single `INSTANCE` constant. + - For `enum` serialization interface implementations, don't forget to implement `ReadResolvable` + interface and return `INSTANCE`, otherwise you have no guarantee that `INSTANCE` constant is the + only alive instance of this implementation in the JVM. + - If *both* writer and reader interface implementations have no fields, it might be a good idea + to merge them into a single `enum` type, in order to keep writing and reading logic together. + +###### Custom `CharSequence` encoding + +Another example shows how to serialize `CharSequence`s using custom encoding (rather than UTF-8): + +Writer: + +```java +public final class CharSequenceCustomEncodingBytesWriter + implements BytesWriter, + StatefulCopyable { + + // config fields, non-final because read in readMarshallable() + private Charset charset; + private int inputBufferSize; + + // cache fields + private transient CharsetEncoder charsetEncoder; + private transient CharBuffer inputBuffer; + private transient ByteBuffer outputBuffer; + + public CharSequenceCustomEncodingBytesWriter(Charset charset, int inputBufferSize) { + this.charset = charset; + this.inputBufferSize = inputBufferSize; + initTransients(); + } + + private void initTransients() { + charsetEncoder = charset.newEncoder(); + inputBuffer = CharBuffer.allocate(inputBufferSize); + int outputBufferSize = (int) (inputBufferSize * charsetEncoder.averageBytesPerChar()); + outputBuffer = ByteBuffer.allocate(outputBufferSize); + } + + @Override + public void write(Bytes out, @NotNull CharSequence cs) { + // Write the actual cs length for accurate StringBuilder.ensureCapacity() while reading + out.writeStopBit(cs.length()); + long encodedSizePos = out.writePosition(); + out.writeSkip(4); + charsetEncoder.reset(); + inputBuffer.clear(); + outputBuffer.clear(); + int csPos = 0; + boolean endOfInput = false; + // this loop inspired by the CharsetEncoder.encode(CharBuffer) implementation + while (true) { + if (!endOfInput) { + int nextCsPos = Math.min(csPos + inputBuffer.remaining(), cs.length()); + append(inputBuffer, cs, csPos, nextCsPos); + inputBuffer.flip(); + endOfInput = nextCsPos == cs.length(); + csPos = nextCsPos; + } + + CoderResult cr = inputBuffer.hasRemaining() ? + charsetEncoder.encode(inputBuffer, outputBuffer, endOfInput) : + CoderResult.UNDERFLOW; + + if (cr.isUnderflow() && endOfInput) + cr = charsetEncoder.flush(outputBuffer); + + if (cr.isUnderflow()) { + if (endOfInput) { + break; + } else { + inputBuffer.compact(); + continue; + } + } + + if (cr.isOverflow()) { + outputBuffer.flip(); + out.write(outputBuffer); + outputBuffer.clear(); + continue; + } + + try { + cr.throwException(); + } catch (CharacterCodingException e) { + throw new IORuntimeException(e); + } + } + outputBuffer.flip(); + out.write(outputBuffer); + + out.writeInt(encodedSizePos, (int) (out.writePosition() - encodedSizePos - 4)); + } + + /** + * Need this method because {@link CharBuffer#append(CharSequence, int, int)} produces garbage + */ + private void append(CharBuffer charBuffer, CharSequence cs, int start, int end) { + for (int i = start; i < end; i++) { + charBuffer.put(cs.charAt(i)); + } + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + charset = (Charset) wireIn.read(() -> "charset").object(); + inputBufferSize = wireIn.read(() -> "inputBufferSize").int32(); + initTransients(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + wireOut.write(() -> "charset").object(charset); + wireOut.write(() -> "inputBufferSize").int32(inputBufferSize); + } + + @Override + public CharSequenceCustomEncodingBytesWriter copy() { + return new CharSequenceCustomEncodingBytesWriter(charset, inputBufferSize); + } +} +``` + +Reader: + +```java +public final class CharSequenceCustomEncodingBytesReader + implements BytesReader, + StatefulCopyable { + + // config fields, non-final because read in readMarshallable() + private Charset charset; + private int inputBufferSize; + + // cache fields + private transient CharsetDecoder charsetDecoder; + private transient ByteBuffer inputBuffer; + private transient CharBuffer outputBuffer; + + public CharSequenceCustomEncodingBytesReader(Charset charset, int inputBufferSize) { + this.charset = charset; + this.inputBufferSize = inputBufferSize; + initTransients(); + } + + private void initTransients() { + charsetDecoder = charset.newDecoder(); + inputBuffer = ByteBuffer.allocate(inputBufferSize); + int outputBufferSize = (int) (inputBufferSize * charsetDecoder.averageCharsPerByte()); + outputBuffer = CharBuffer.allocate(outputBufferSize); + } + + @NotNull + @Override + public CharSequence read(Bytes in, @Nullable CharSequence using) { + long csLengthAsLong = in.readStopBit(); + if (csLengthAsLong > Integer.MAX_VALUE) { + throw new IORuntimeException("cs len shouldn't be more than " + Integer.MAX_VALUE + + ", " + csLengthAsLong + " read"); + } + int csLength = (int) csLengthAsLong; + StringBuilder sb; + if (using instanceof StringBuilder) { + sb = (StringBuilder) using; + sb.setLength(0); + sb.ensureCapacity(csLength); + } else { + sb = new StringBuilder(csLength); + } + + int remainingBytes = in.readInt(); + charsetDecoder.reset(); + inputBuffer.clear(); + outputBuffer.clear(); + boolean endOfInput = false; + // this loop inspired by the CharsetDecoder.decode(ByteBuffer) implementation + while (true) { + if (!endOfInput) { + int inputChunkSize = Math.min(inputBuffer.remaining(), remainingBytes); + inputBuffer.limit(inputBuffer.position() + inputChunkSize); + in.read(inputBuffer); + inputBuffer.flip(); + remainingBytes -= inputChunkSize; + endOfInput = remainingBytes == 0; + } + + CoderResult cr = inputBuffer.hasRemaining() ? + charsetDecoder.decode(inputBuffer, outputBuffer, endOfInput) : + CoderResult.UNDERFLOW; + + if (cr.isUnderflow() && endOfInput) + cr = charsetDecoder.flush(outputBuffer); + + if (cr.isUnderflow()) { + if (endOfInput) { + break; + } else { + inputBuffer.compact(); + continue; + } + } + + if (cr.isOverflow()) { + outputBuffer.flip(); + sb.append(outputBuffer); + outputBuffer.clear(); + continue; + } + + try { + cr.throwException(); + } catch (CharacterCodingException e) { + throw new IORuntimeException(e); + } + } + outputBuffer.flip(); + sb.append(outputBuffer); + + return sb; + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) throws IORuntimeException { + charset = (Charset) wireIn.read(() -> "charset").object(); + inputBufferSize = wireIn.read(() -> "inputBufferSize").int32(); + initTransients(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + wireOut.write(() -> "charset").object(charset); + wireOut.write(() -> "inputBufferSize").int32(inputBufferSize); + } + + @Override + public CharSequenceCustomEncodingBytesReader copy() { + return new CharSequenceCustomEncodingBytesReader(charset, inputBufferSize); + } +} +``` + +Usage example: + +```java +Charset charset = Charset.forName("GBK"); +int charBufferSize = 100; +int bytesBufferSize = 200; +CharSequenceCustomEncodingBytesWriter writer = + new CharSequenceCustomEncodingBytesWriter(charset, charBufferSize); +CharSequenceCustomEncodingBytesReader reader = + new CharSequenceCustomEncodingBytesReader(charset, bytesBufferSize); +try (ChronicleMap englishToChinese = ChronicleMap + .of(String.class, CharSequence.class) + .valueMarshallers(reader, writer) + .averageKey("hello") + .averageValue("你好") + .entries(10) + .create()) { + englishToChinese.put("hello", "你好"); + englishToChinese.put("bye", "再见"); + + Assert.assertEquals("你好", englishToChinese.get("hello").toString()); + Assert.assertEquals("再见", englishToChinese.get("bye").toString()); +} +``` + +Some notes on this case of custom serialization: + + - Both `CharSequenceCustomEncodingBytesWriter` and `CharSequenceCustomEncodingBytesReader` have + configurations (charset and input buffer size), hence they are implemented as normal classes rather + than enums with a single `INSTANCE` constant. + - Both writer and reader classes have some "cache" fields, their contents are mutated during + writing and reading. That is why they have to implement `StatefulCopyable` interface. See + [Understanding `StatefulCopyable`](#understanding-statefulcopyable) section for more infromation on + this. + +##### `SizedWriter` and `SizedReader` + +This pair of interfaces is configured via `ChronicleMapBuilder.keyMarshallers()` or +`valueMarshallers()` for the key or value type of the map, respectively (overloaded methods, those +for [`BytesWriter` and `BytesReader`](#byteswriter-and-bytesreader) interfaces have the same names). + +The main two methods to implement in `SizedWriter` interface: + + - `long size(@NotNull T toWrite);` returns the number of bytes, which is written by the subsequent + `write()` method given the same `toWrite` instance of the serialized type. + - `void write(Bytes out, long size, @NotNull T toWrite);` writes the given `toWrite` instance to + the given `out` bytes sink. Additionally `size` is provided, which is the value computed for the + same `toWrite` instance by calling `size()` method. Hence, when `SizedWriter` is used internally, + `size()` method is always called before `write()`, caching logic in `SizedWriter` implementation + may rely on this. This method should advance `writePosition()` of the given out `Bytes` exactly by + the given `size` number of bytes. + +`SizedReader`'s `T read(Bytes in, long size, @Nullable T using);` method is similar to the +corresponding `read()` method in `BytesReader` interface, except the size of the serialized object +is provided. + +This pair of interfaces is suitable, if + + - The serialized form of the type effectively includes the size of the rest of the serialized form + in the beginning, + - But the type is not a plain sequence of bytes like `byte[]` or `ByteBuffer` (for those types + [`DataAccess` and `SizedReader`](#dataaccess-and-sizedreader) pair of interfaces is a much better + choice). + +Examples of such types include lists and arrays of constant-sized elements. + +Compared to `BytesWriter` and `BytesReader`, this pair of interfaces allows to share the information +about the serialized form between serialization logic and the Chronicle Map, saving a few bytes by +storing the serialization size only once rather than twice. This pair of interfaces is not much more +difficult to implement, but the gains are also not big. + +Example: serializing lists of simple `Point` structures: + +```java +public final class Point { + + public static Point of(double x, double y) { + Point p = new Point(); + p.x = x; + p.y = y; + return p; + } + + double x, y; +} +``` + +Serializer implementation: + +```java +public enum PointListSizedMarshaller + implements SizedReader>, SizedWriter>, + ReadResolvable { + + INSTANCE; + + /** A point takes 16 bytes in serialized form: 8 bytes for both x and y value */ + private static final long ELEMENT_SIZE = 16; + + @Override + public long size(@NotNull List toWrite) { + return toWrite.size() * ELEMENT_SIZE; + } + + @Override + public void write(Bytes out, long size, @NotNull List toWrite) { + toWrite.forEach(point -> { + out.writeDouble(point.x); + out.writeDouble(point.y); + }); + } + + @NotNull + @Override + public List read(@NotNull Bytes in, long size, List using) { + if (size % ELEMENT_SIZE != 0) { + throw new IORuntimeException("Bytes size should be a multiple of " + ELEMENT_SIZE + + ", " + size + " read"); + } + long listSizeAsLong = size / ELEMENT_SIZE; + if (listSizeAsLong > Integer.MAX_VALUE) { + throw new IORuntimeException("List size couldn't be more than " + Integer.MAX_VALUE + + ", " + listSizeAsLong + " read"); + } + int listSize = (int) listSizeAsLong; + if (using == null) { + using = new ArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + using.add(null); + } + } else if (using.size() < listSize) { + while (using.size() < listSize) { + using.add(null); + } + } else if (using.size() > listSize) { + using.subList(listSize, using.size()).clear(); + } + for (int i = 0; i < listSize; i++) { + Point point = using.get(i); + if (point == null) + using.set(i, point = new Point()); + point.x = in.readDouble(); + point.y = in.readDouble(); + } + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public PointListSizedMarshaller readResolve() { + return INSTANCE; + } +} +``` + +Usage example: + +```java +try (ChronicleMap> objects = ChronicleMap + .of(String.class, (Class>) (Class) List.class) + .averageKey("range") + .valueMarshaller(PointListSizedMarshaller.INSTANCE) + .averageValue(asList(of(0, 0), of(1, 1))) + .entries(10) + .create()) { + objects.put("range", asList(of(0, 0), of(1, 1))); + objects.put("square", asList(of(0, 0), of(0, 100), of(100, 100), of(100, 0))); + + Assert.assertEquals(2, objects.get("range").size()); + Assert.assertEquals(4, objects.get("square").size()); +} +``` + +##### `DataAccess` and `SizedReader` + +This pair of interfaces is configured via `ChronicleMapBuilder.keyReaderAndDataAccess()` or +`valueReaderAndDataAccess()` for the key or value type of the map, respectively. + +The reader part, `SizedReader`, is the same as in [`SizedWriter` and +`SizedReader`](#sizedwriter-and-sizedreader) pair, so `DataAccess` is an "advanced" interface to +replace `SizedWriter`. + +The main method in `DataAccess` is `Data getData(@NotNull T instance)`, it returns a `Data` +accessor which is used to write "serialized" form of the instance to off-heap memory. `Data.size()` +on the returned `Data` object is used for the same purpose as `SizedWriter.size()` method in +[`SizedWriter` and `SizedReader`](#sizedwriter-and-sizedreader) pair interfaces. `Data.writeTo()` +is used instead of `SizedWriter.write()`. + +`DataAccess` assumes that the `Data` object, returned from the `getData()` method is cached in some +way, that is why it also has `uninit()` method to clear references to the serialized object after +query operation to a Chronicle Map is over (to prevent memory leaks). This, in its turn, implies +that `DataAccess` implementation is stateful, therefore `DataAccess` is made a subinterface of +`StatefulCopyable` to force all `DataAccess` implementations to implement `StatefulCopyable` as +well. See [Understanding `StatefulCopyable`](#understanding-statefulcopyable) for more infromation +on this. If your `DataAccess` implementation is not actually stateful, it is free to return `this` +from `StatefulCopyable.copy()` method. + +`DataAccess` interface is primarily intended for "serializing" objects that are already sequences +of bytes and in fact doesn't require serialization, like `byte[]`, `ByteBuffer`, arrays of Java +primitives. For such types of objects, `DataAccess` allows to bypass intermediate buffering, copying +data directly from objects to Chronicle Map's off-heap memory. + +For example, look at the `DataAccess` implementation for `byte[]`: + +```java +public final class ByteArrayDataAccess extends AbstractData implements DataAccess { + + /** Cache field */ + private transient HeapBytesStore bs; + + /** State field */ + private transient byte[] array; + + public ByteArrayDataAccess() { + initTransients(); + } + + private void initTransients() { + bs = HeapBytesStore.uninitialized(); + } + + @Override + public RandomDataInput bytes() { + return bs; + } + + @Override + public long offset() { + return bs.start(); + } + + @Override + public long size() { + return bs.capacity(); + } + + @Override + public byte[] get() { + return array; + } + + @Override + public byte[] getUsing(@Nullable byte[] using) { + if (using == null || using.length != array.length) + using = new byte[array.length]; + System.arraycopy(array, 0, using, 0, array.length); + return using; + } + + @Override + public Data getData(@NotNull byte[] instance) { + array = instance; + bs.init(instance); + return this; + } + + @Override + public void uninit() { + array = null; + bs.uninit(); + } + + @Override + public DataAccess copy() { + return new ByteArrayDataAccess(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + initTransients(); + } +} +``` + +Note that `getData()` method returns `this`, and the `DataAccess` implementation implements `Data` +interface as well. This is a recommended practice, because it reduces the number of objects involved +(hence pointer chasing), and keeps `DataAccess` and `Data` logic together. + +`Data` interface puts constrains on `equals()`, `hashCode()` and `toString()` implementations, this +is why `ByteArrayDataAccess` subclasses `AbstractData` and inherits proper implementations from it. +This is OK to serializer strategy implementation to have `equals()`, `hashCode()` and `toString()` +from a very different domain, because those methods are never called on serializers inside Chronicle +Map. + +The easiest way to implement `equals()`, `hashCode()` and `toString()` is to extend `AbstractData` +class, if it is not possible (the `Data` implementation already extends some other class), do this +by delegating to `dataEquals()`, `dataHashCode()` and `dataToString()` default methods, provided +right in `Data` interface. + +Corresponding `SizedReader` for `byte[]`: + +```java +public enum ByteArraySizedReader + implements SizedReader, Marshallable, ReadResolvable { + + INSTANCE; + + @NotNull + @Override + public byte[] read(@NotNull Bytes in, long size, @Nullable byte[] using) { + if (size < 0L || size > (long) Integer.MAX_VALUE) { + throw new IORuntimeException("byte[] size should be non-negative int, " + + size + " given. Memory corruption?"); + } + int arrayLength = (int) size; + if (using == null || arrayLength != using.length) + using = new byte[arrayLength]; + in.read(using); + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public ByteArraySizedReader readResolve() { + return INSTANCE; + } +} +``` + +(Nothing is required to use this pair of serializers with `byte[]` keys of values in Chronicle Map, +it is used by default, if you configure `byte[]` key or value type.) + +##### Understanding `StatefulCopyable` + +**Problems:** + + 1. Sometimes on writing, reading or in other methods, defined in serialization interfaces, it is + needed to operate with intermediate objects, holding some writing/reading state. + + In the simplest and most common case, those objects are of some kind of buffers between serialized + or deserialized instances and output or input `Bytes`: for example, see writer and reader + implementations in the [custom `CharSequence` encoding](#custom-charsequence-encoding) section. + + To avoid producing a lot of garbage, those intermediate objects should be cached. + + 2. `Data` object, returned from `getData()` method of `DataAccess` serialization interface, should + be cached in order to avoid producing garbage. + + 3. `SizedWriter` and `DataAccess` serialization interfaces, and the `Data` object, returned from + `DataAccess.getData()` method, since it is cached (see the previous point) have several methods: + `size()` and `write()` in `SizedWriter`, `getData()` and `uninit()` in `DataAccess`, many methods + in `Data`. It is usually essential to save some computation results between calls to those methods + from inside Chronicle Map implementation, while working some key or value during some query to + Chronicle Map, because otherwise expensive computations are performed several times, that is very + inefficient. + +All these problems require to access some mutable, context-dependant fields from within serializer +interface implementations. But, only a *single instance* of serializer implementation is configured +for a Chronicle Map (either by `keyMarshallers()` or `valueMarshallers()` or +`keyReaderAndDataAccess()` or `valueReaderAndDataAccess()` method in `ChronicleMapBuilder`). On the +other hand, `ChronicleMap` is a `ConcurrentMap`, i. e. serializer implementations could be accessed +from multiple threads concurrently. Moreover, a single `ChronicleMapBuilder` (with a single pair +of serializer instances configured for keys and values) could be used to construct many independent +`ChronicleMap`s, which could also be accessed concurrently. + +Inefficient and fragile solution: a single instance of serialization interface, static `ThreadLocal` +fields in serializer implementation class. + +It is inefficient, because `ThreadLocal`s are accessed each time a serializer interface method is +called, that is much slower than accessing vanilla object fields. + +It is fragile, because if you need `ThreadLocal` fields for preserving some state between calls to +multiple methods in serializer interfaces over a single query to a Chronicle Map (the 3rd point in +the problems list above), you should consider that a single serializer instance could be used for +both keys and values of the same Chronicle Map (calls to some methods of serializer interface for +serializing the key and the value over a Chronicle Map query are interspersed in unspecified order). +Plus, you should consider that a single serializer instance could be used to access multiple +independent Chronicle Map instances in the same thread, calls to serializers within maps could be +interspersed via [contexts access](#working-with-an-entry-within-a-context-section). So, +`ThreadLocal` fields should be isolated not just per accessing thread, but also per serialized +object domain (is the serialized object a key or a value in Chronicle Map), and per accessing +Chronicle Map instance, that is hard to implement correctly. + +**Recommended solution: `StatefulCopyable`.** A serializer implementation has ordinary instance +fields for caching anything and preserving state between calls to different methods. It implements +`StatefulCopyable` interface with a single method `copy()`, that is like `Object.clone()`, but +copies only configuration fields of the serializer instance, not cache or state fields. + +Call to `copy()` method at any point of a serializer instance lifetime should return an instance of +the same serializer implementation in the state, exactly equal to the state of the current instance +at the moment after construction and initial configuration. I. e. with "empty" cache and state. +As a consequence, `copy()` method is transitive, i. e. `serializer.copy().copy().copy()` should +return an object in exactly the same state, as after a single `copy()` call. + +Chronicle Map implementation recognizes that configured `BytesWriter`, `BytesReader`, `SizedWriter`, +`SizedReader` or `DataAccess` instance implements `StatefulCopyable`, and "populates" it internally +via `copy()` for each thread, Chronicle Map instance and serialized object domain (keys or values). + +See examples of implementing this interface in the [custom `CharSequence` +encoding](#custom-charsequence-encoding) section above. + +It is allowed to return `this` from the `copy()` method, if with some configurations the +serializer implementation doesn't have state. Typically this is the case when serializer is +configured with sub-serializers which might be `StatefulCopyable` or not, for example +[`ListMarshaller`](src\main\java\net\openhft\chronicle\hash\serialization\ListMarshaller.java) +class. + +##### Custom serialization checklist + + 1. Choose the most suitable pair of serialization interfaces: [`BytesWriter` and + `BytesReader`](#byteswriter-and-bytesreader), [`SizedWriter` and + `SizedReader`](#sizedwriter-and-sizedreader) or [`DataAccess` and + `SizedReader`](#dataaccess-and-sizedreader). Recommendations on which pair to choose are given in + the linked sections, describing each pair. + 2. If implementation of the writer or reader part is configuration-less, make it a `enum` with + a single `INSTANCE` constant. Implement `ReadResolvable` and return `INSTANCE` from `readResolve()` + method. + 3. If both the writer and reader are configuration-less, merge them into a single `-Marshaller` + `enum` implementation. + 4. Make best effort in reusing `using` objects on the reader side (`BytesReader` or `SizedReader`), + including nesting objects. + 5. Make best effort in caching intermediate serialization results on writer side while working with + some object, e. g. try not to make expensive computations in both `size()` and `write()` methods + of `SizedWriter` implementation, but rather lazily compute them and cache in an serializer instance + field. + 6. Make best effort in reusing intermediate objects, used for reading or writing. Store them in + instance fields of serializer implementation. + 7. If a serializer implementation is stateful or have cache fields, implement `StatefulCopyable`. + See [Understanding `StatefulCopyable`](#understanding-statefulcopyable) section for more info. + 8. Implement `writeMarshallable()` and `readMarshallable()` by writing and reading configuration + fields (but not state or cache fields) of the serializer instance one-by-one, using the given + `WireOut`/`WireIn` object. See [Custom `CharSequence` encoding](#custom-charsequence-encoding) + section for some non-trivial example of implementing these methods. See also [Wire tutorial]( + https://github.com/OpenHFT/Chronicle-Wire#using-wire). + 9. *Don't forget to initialize transient/cache/state fileds of the instance in the end of + `readMarshallable()` implementation.* This is needed, because fefore calling `readMarshallable()`, + Wire framework creates a serializer instance by means of `Unsafe.allocateInstance()` rather than + calling any constructor. + 10. If implementing `DataAccess`, consider implementation to be `Data` also, and return `this` from + `getData()` method. + 11. Don't forget to implement `equals()`, `hashCode()` and `toString()` in `Data` implementation, + returned from `DataAccess.getData()` method, regardless if this is actually the same `DataAccess` + object, or a separate object. + 12. Except `DataAccess` which is also a `Data`, serializers shouldn't override Object's `equals()`, + `hashCode()` and `toString()` (these methods are never called on serializers inside Chronicle Map + library); they shouldn't implement `Serializable` or `Externalizable` (but have to implement + `net.openhft.chronicle.wire.Marshallable`); shouldn't implement `Cloneable` (but have to implement + `StatefulCopyable`, if they are stateful or have cache fields). + 13. After implementing custom serializers, don't forget to actually apply them to + `ChronicleMapBuilder` by `keyMarshallers()`, `keyReaderAndDataAccess()`, `valueMarshallers()` or + `valueReaderAndDataAccess()` methods. + ### `ChronicleMap` instance usage patterns #### Single-key queries @@ -475,7 +1320,7 @@ at most - corrupt the `ChronicleMap` memory. For accessing the `ChronicleMap` value memory directly use the following technique: -##### Working with the entry within a context section +##### Working with an entry within a context section ```java try (ExternalMapQueryContext c = diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java b/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java index d05d9b568..58e1a14ca 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/BytesReader.java @@ -18,14 +18,26 @@ package net.openhft.chronicle.hash.serialization; import net.openhft.chronicle.bytes.Bytes; import net.openhft.chronicle.bytes.ReadBytesMarshallable; +import net.openhft.chronicle.hash.ChronicleHashBuilder; +import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.chronicle.wire.Marshallable; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; /** - * External version on {@link ReadBytesMarshallable}. + * Deserializer of objects from bytes, pairing {@link BytesWriter}, without accepting additional + * information about serialized objects. + * + *

Read {@code + * BytesWriter} and {@code BytesReader} and + * custom + * serialization checklist sections in the Chronicle Map tutorial for more information on this + * interface, how to implement and use it properly. * * @param type of objects deserialized * @see BytesWriter + * @see ChronicleHashBuilder#keyMarshallers(BytesReader, BytesWriter) + * @see ChronicleMapBuilder#valueMarshallers(BytesReader, BytesWriter) */ public interface BytesReader extends Marshallable { @@ -45,5 +57,5 @@ public interface BytesReader extends Marshallable { * @return the object read from the bytes, either reused or newly created */ @NotNull - T read(Bytes in, T using); + T read(Bytes in, @Nullable T using); } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java b/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java index 7586e050c..66fa8bfbd 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/BytesWriter.java @@ -18,14 +18,24 @@ package net.openhft.chronicle.hash.serialization; import net.openhft.chronicle.bytes.Bytes; import net.openhft.chronicle.bytes.WriteBytesMarshallable; +import net.openhft.chronicle.hash.ChronicleHashBuilder; +import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.chronicle.wire.Marshallable; import org.jetbrains.annotations.NotNull; /** - * External version of {@link WriteBytesMarshallable}. + * Serializer of objects to bytes, pairing {@link BytesReader}. + * + *

Read {@code + * BytesWriter} and {@code BytesReader} and + * custom + * serialization checklist sections in the Chronicle Map tutorial for more information on this + * interface, how to implement and use it properly. * * @param the type of objects serialized * @see BytesReader + * @see ChronicleHashBuilder#keyMarshallers(BytesReader, BytesWriter) + * @see ChronicleMapBuilder#valueMarshallers(BytesReader, BytesWriter) */ public interface BytesWriter extends Marshallable { diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/DataAccess.java b/src/main/java/net/openhft/chronicle/hash/serialization/DataAccess.java index 6488ceb52..6baf9079f 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/DataAccess.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/DataAccess.java @@ -16,13 +16,52 @@ package net.openhft.chronicle.hash.serialization; +import net.openhft.chronicle.hash.ChronicleHash; +import net.openhft.chronicle.hash.ChronicleHashBuilder; import net.openhft.chronicle.hash.Data; +import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.chronicle.wire.Marshallable; import org.jetbrains.annotations.NotNull; +import java.nio.ByteBuffer; + +/** + * Strategy of accessing flat byte contents of objects, using a reusable {@link Data} object, cached + * inside the {@code DataAccess} instance. This strategy interface is suitable for types of objects, + * that are already in fact sequences of bytes ({@code byte[]}, {@link ByteBuffer}, etc.) and + * shouldn't be serialized, and allows to avoid extra data copying. Accessed bytes should be later + * readable by some {@link BytesReader}. + * + *

Read {@link + * DataAccess} and {@code SizedReader} and + * custom + * serialization checklist sections in the Chronicle Map tutorial for more information on this + * interface, how to implement and use it properly. + * + * @param the type of objects accessed + * @see ChronicleHashBuilder#keyReaderAndDataAccess(SizedReader, DataAccess) + * @see ChronicleMapBuilder#valueReaderAndDataAccess(SizedReader, DataAccess) + */ public interface DataAccess extends StatefulCopyable>, Marshallable { + /** + * Obtain {@code Data} accessor to the bytes of the given object. Typically a {@code Data} + * instance should be cached in a field of this {@code DataAccess}, so always the same object + * is returned. + * + * @param instance the object to access bytes (serialized form) of + * @return an accessor to the bytes of the given object + */ Data getData(@NotNull T instance); + /** + * Clear references to the accessed object (provided in {@link #getData(Object)} method) in + * caching fields of this {@code DataAccess} or the cached {@link Data} object, returned from + * {@link #getData(Object)}. + * + * {@code DataAccess} is cached itself in thread-local variables for {@link ChronicleHash} + * instances, this method prevents leaking of accessed objects (they are not eligible for + * garbage collection while there are some references). + */ void uninit(); } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/ListMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/ListMarshaller.java index 10794059a..79bb08584 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/ListMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/ListMarshaller.java @@ -59,11 +59,31 @@ import static net.openhft.chronicle.hash.serialization.StatefulCopyable.copyIfNe public final class ListMarshaller implements BytesReader>, BytesWriter>, StatefulCopyable> { + /** + * Returns a {@code ListMarshaller} which uses the given list elements' serializers. + * + * @param elementReader list elements' reader + * @param elementWriter list elements' writer + * @param type of list elements + * @return a {@code ListMarshaller} which uses the given list elements' serializers + */ public static ListMarshaller of( BytesReader elementReader, BytesWriter elementWriter) { return new ListMarshaller<>(elementReader, elementWriter); } + /** + * Returns a {@code ListMarshaller} which uses the given marshaller as both reader and writer of + * list elements. Example:


+     * ChronicleMap
+     *     .of(String.class,{@code (Class>)} ((Class) List.class))
+     *     .valueMarshaller(ListMarshaller.of(IntegerMarshaller.INSTANCE))
+     *     ...
+ * @param elementMarshaller list elements' marshaller + * @param type of list elements + * @param type of list elements' marshaller + * @return a {@code ListMarshaller} which uses the given list elements' marshaller + */ public static & BytesWriter> ListMarshaller of( M elementMarshaller) { return of(elementMarshaller, elementMarshaller); @@ -73,6 +93,15 @@ public final class ListMarshaller private BytesReader elementReader; private BytesWriter elementWriter; + /** + * Constructs a {@code ListMarshaller} with the given list elements' serializers. + * + *

Use static factory {@link #of(BytesReader, BytesWriter)} instead of this constructor + * directly. + * + * @param elementReader list elements' reader + * @param elementWriter list elements' writer + */ public ListMarshaller(BytesReader elementReader, BytesWriter elementWriter) { this.elementReader = elementReader; this.elementWriter = elementWriter; diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/MapMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/MapMarshaller.java index 5e6ec62e3..f08facf58 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/MapMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/MapMarshaller.java @@ -60,6 +60,14 @@ public final class MapMarshaller implements BytesReader>, BytesW private transient Deque orderedKeys; private transient Deque orderedValues; + /** + * Constructs a {@code MapMarshaller} with the given map keys' and values' serializers. + * + * @param keyReader map keys' reader + * @param keyWriter map keys' writer + * @param valueReader map values' reader + * @param valueWriter map values' writer + */ public MapMarshaller( BytesReader keyReader, BytesWriter keyWriter, BytesReader valueReader, BytesWriter valueWriter) { diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/SetMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/SetMarshaller.java index 79883a390..7b3d0a4e3 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/SetMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/SetMarshaller.java @@ -61,11 +61,31 @@ import static net.openhft.chronicle.hash.serialization.StatefulCopyable.copyIfNe public final class SetMarshaller implements BytesReader>, BytesWriter>, StatefulCopyable> { + /** + * Returns a {@code SetMarshaller} which uses the given set elements' serializers. + * + * @param elementReader set elements' reader + * @param elementWriter set elements' writer + * @param type of set elements + * @return a {@code SetMarshaller} which uses the given set elements' serializers + */ public static SetMarshaller of( BytesReader elementReader, BytesWriter elementWriter) { return new SetMarshaller<>(elementReader, elementWriter); } + /** + * Returns a {@code SetMarshaller} which uses the given marshaller as both reader and writer of + * set elements. Example:


+     * ChronicleMap
+     *     .of(String.class,{@code (Class>)} ((Class) Set.class))
+     *     .valueMarshaller(SetMarshaller.of(IntegerMarshaller.INSTANCE))
+     *     ...
+ * @param elementMarshaller set elements' marshaller + * @param type of set elements + * @param type of set elements' marshaller + * @return a {@code SetMarshaller} which uses the given set elements' marshaller + */ public static & BytesWriter> SetMarshaller of( M elementMarshaller) { return of(elementMarshaller, elementMarshaller); @@ -78,6 +98,15 @@ public final class SetMarshaller /** Cache field */ private transient Deque orderedElements; + /** + * Constructs a {@code SetMarshaller} with the given set elements' serializers. + * + *

Use static factory {@link #of(BytesReader, BytesWriter)} instead of this constructor + * directly. + * + * @param elementReader set elements' reader + * @param elementWriter set elements' writer + */ public SetMarshaller(BytesReader elementReader, BytesWriter elementWriter) { this.elementReader = elementReader; this.elementWriter = elementWriter; diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java index 9e340be22..3e9012739 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/SizeMarshaller.java @@ -17,8 +17,10 @@ package net.openhft.chronicle.hash.serialization; import net.openhft.chronicle.bytes.Bytes; +import net.openhft.chronicle.hash.ChronicleHashBuilder; import net.openhft.chronicle.hash.serialization.impl.ConstantSizeMarshaller; import net.openhft.chronicle.hash.serialization.impl.StopBitSizeMarshaller; +import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.chronicle.wire.Marshallable; /** @@ -26,7 +28,7 @@ import net.openhft.chronicle.wire.Marshallable; * the same bytes store). Normal number storing (4 bytes for {@code int}, 8 bytes for {@code long} * is rather wasteful for marshalling purposes, when the number (size) to store is usually very * small, and 1-2 bytes could be enough to encode it (this is how {@link #stopBit()} marshaller - * work). Also, this interface allows to generalize storing constantly-sized and variable-sized + * works). Also, this interface allows to generalize storing constantly-sized and variable-sized * blocks of data. Constantly-sized don't require to store the size actually, the corresponding * {@link #constant} {@code SizeMarshaller} consumes 0 bytes. * @@ -34,13 +36,34 @@ import net.openhft.chronicle.wire.Marshallable; * example each {@link #constant} {@code SizeMarshaller} is able to "store" only a single specific * value (it's constant size). If the marshaller is fed with the size it is not able store, it could * throw an {@code IllegalArgumentException}. + * + * @see ChronicleHashBuilder#keySizeMarshaller(SizeMarshaller) + * @see ChronicleMapBuilder#valueSizeMarshaller(SizeMarshaller) */ public interface SizeMarshaller extends Marshallable { + /** + * Returns a {@code SizeMarshaller} which stores number using so-called stop bit encoding. This marshaller is used as the default marshaller for keys' and + * values' sizes in Chronicle Map, unless Chronicle Map could figure out (or hinted in {@code + * ChronicleMapBuilder}) that keys or values are constant-sized, in which case {@link + * #constant(long)} {@code SizeMarshaller} is used. + */ static SizeMarshaller stopBit() { return StopBitSizeMarshaller.INSTANCE; } + /** + * Returns a {@code SizeMarshaller}, that consumes 0 bytes and is above to "store" only a single + * constant number, provided to this factory method. This size marshaller is used in Chronicle + * Map to serialize keys' or values' sizes, when Chronicle Map could figure out (or was hinted + * in {@code ChronicleMapBuilder}) that keys or values are constant-sized. + * + * @param constantSize the single constant size the returned {@code SizeMarshaller} is able to + * "store" + * @return a {@code SizeMarshaller}, that is able to "store" only the given constant size + */ static SizeMarshaller constant(long constantSize) { return new ConstantSizeMarshaller(constantSize); } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/SizedReader.java b/src/main/java/net/openhft/chronicle/hash/serialization/SizedReader.java index 66072c0ab..f8d9d8f55 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/SizedReader.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/SizedReader.java @@ -18,57 +18,31 @@ package net.openhft.chronicle.hash.serialization; import net.openhft.chronicle.bytes.Bytes; import net.openhft.chronicle.hash.ChronicleHash; +import net.openhft.chronicle.hash.ChronicleHashBuilder; +import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.chronicle.wire.Marshallable; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** - * Deserializer of objects from bytes, mirroring the {@link SizedWriter}, i. e. assuming the length + * Deserializer of objects from bytes, pairing {@link SizedWriter}, i. e. assuming the length * of the serialized form isn't written in the beginning of the serialized form itself, but managed * by {@link ChronicleHash} implementation and passed to the reading methods. * - *

Implementation example:


- * class LongPair { long first, second; }
- * 
- * enum LongPairArrayReader
- *         implements{@code SizedReader, EnumMarshallable} {
- *     INSTANCE;
- *
- *    {@literal @}Override
- *    {@literal @}NotNull
- *     public LongPair[]{@literal read(@}NotNull Bytes in, long size,
- *            {@literal @}Nullable LongPair[] using) {
- *         if (size{@code >} Integer.MAX_VALUE * 16L)
- *             throw new IllegalStateException("LongPair[] size couldn't be " + (size / 16L));
- *         int resLen = (int) (size / 16L);
- *         LongPair[] res;
- *         if (using != null) {
- *             if (using.length == resLen) {
- *                 res = using;
- *             } else {
- *                 res = Arrays.copyOf(using, resLen);
- *             }
- *         } else {
- *             res = new LongPair[resLen];
- *         }
- *         for (int i = 0; i{@code <} resLen; i++) {
- *             LongPair pair = res[i];
- *             if (pair == null)
- *                 res[i] = pair = new LongPair();
- *             pair.first = in.readLong();
- *             pair.second = in.readLong();
- *         }
- *         return res;
- *     }
- *
- *    {@literal @}Override
- *     public LongPairArrayReader readResolve() {
- *         return INSTANCE;
- *     }
- * }
+ *

Read {@code + * SizedWriter} and {@code SizedReader}, + * {@link DataAccess} + * and {@code SizedReader} and + * custom + * serialization checklist sections in the Chronicle Map tutorial for more information on this + * interface, how to implement and use it properly. * * @param the type of the object deserialized * @see SizedWriter + * @see ChronicleHashBuilder#keyMarshallers(SizedReader, SizedWriter) + * @see ChronicleHashBuilder#keyReaderAndDataAccess(SizedReader, DataAccess) + * @see ChronicleMapBuilder#valueMarshallers(SizedReader, SizedWriter) + * @see ChronicleMapBuilder#valueReaderAndDataAccess(SizedReader, DataAccess) */ public interface SizedReader extends Marshallable { diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/SizedWriter.java b/src/main/java/net/openhft/chronicle/hash/serialization/SizedWriter.java index f277610e8..67925b82f 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/SizedWriter.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/SizedWriter.java @@ -18,44 +18,27 @@ package net.openhft.chronicle.hash.serialization; import net.openhft.chronicle.bytes.Bytes; import net.openhft.chronicle.hash.ChronicleHash; +import net.openhft.chronicle.hash.ChronicleHashBuilder; +import net.openhft.chronicle.map.ChronicleMapBuilder; import net.openhft.chronicle.wire.Marshallable; import org.jetbrains.annotations.NotNull; /** - * Serializer (object to bytes writer) which 1) knows the length of serialized + * Serializer of objects to bytes, pairing {@link SizedReader}, which knows the length of serialized * form of any object before actual serialization 2) doesn't include that length in the serialized * form itself, assuming it will be passed by the {@link ChronicleHash} into {@link * SizedReader#read} deserialization method. * - *

Implementation example:


- * class LongPair { long first, second; }
- *
- * enum LongPairArrayWriter
- *         implements{@code SizedWriter, EnumMarshallable} {
- *     INSTANCE;
- *
- *    {@literal @}Override
- *     public long{@literal size(@}NotNull LongPair[] toWrite) {
- *         return toWrite.length * 16L;
- *     }
- *
- *    {@literal @}Override
- *     public void{@literal write(@}NotNull Bytes out, long size,
- *            {@literal @}NotNull LongPair[] toWrite) {
- *         for (LongPair pair : toWrite) {
- *             out.writeLong(pair.first);
- *             out.writeLong(pair.second);
- *         }
- *     }
- *
- *    {@literal @}Override
- *     public LongPairArrayWriter readResolve() {
- *         return INSTANCE;
- *     }
- * }
+ *

Read {@code + * SizedWriter} and {@code SizedReader} and + * custom + * serialization checklist sections in the Chronicle Map tutorial for more information on this + * interface, how to implement and use it properly. * * @param the type of the object marshalled * @see SizedReader + * @see ChronicleHashBuilder#keyMarshallers(SizedReader, SizedWriter) + * @see ChronicleMapBuilder#valueMarshallers(SizedReader, SizedWriter) */ public interface SizedWriter extends Marshallable { diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/StatefulCopyable.java b/src/main/java/net/openhft/chronicle/hash/serialization/StatefulCopyable.java index b38af1688..b7d1806c2 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/StatefulCopyable.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/StatefulCopyable.java @@ -16,36 +16,20 @@ package net.openhft.chronicle.hash.serialization; -import net.openhft.chronicle.hash.serialization.impl.StringBytesReader; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.set.ChronicleSet; /** - * Stateful implementations of marshaller interfaces ({@link SizedReader}, {@link SizedWriter}, - * {@link BytesReader}, {@link BytesWriter}, {@link DataAccess}), configured for {@link + * {@link Cloneable}-like interface used by Chronicle Map to populate stateful serializer + * implementations for each isolated site (thread + Chronicle Map instance + objects domain). + * Stateful implementations of serialization interfaces ({@link SizedReader}, {@link SizedWriter}, + * {@link BytesReader}, {@link BytesWriter} or {@link DataAccess}), configured for {@link * ChronicleMap} or {@link ChronicleSet} in builder, should implement the {@code StatefulCopyable} - * interface. The marshaller instance is populated for each site by {@link #copy()} method. + * interface. * - *

For example, see {@link StringBytesReader} implementation:


- * public class StringBytesReader
- *         implements{@code BytesReader, StatefulCopyable} {
- *
- *     private final StringBuilder sb = new StringBuilder();
- *
- *    {@literal @}NotNull
- *    {@literal @}Override
- *     public String read(Bytes in,{@literal @}Nullable String using) {
- *         sb.setLength(0);
- *         in.readUtf8(sb);
- *         return sb.toString();
- *     }
- *
- *    {@literal @}Override
- *     public StringBytesReader copy() {
- *         return new StringBytesReader();
- *     }
- * }
- * 
+ *

See + * Understanding {@code StatefulCopyable} section in the Chronicle Map tutorial for more info + * on how to implement and use this interface properly, and for examples. * * @param the type of marshaller, implementing this interface */ diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayDataAccess.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayDataAccess.java index d9fb9aecb..753f9b14f 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayDataAccess.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArrayDataAccess.java @@ -88,14 +88,14 @@ public class ByteArrayDataAccess extends AbstractData implements DataAcc return new ByteArrayDataAccess(); } + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + @Override public void readMarshallable(@NotNull WireIn wireIn) { // no fields to read initTransients(); } - - @Override - public void writeMarshallable(@NotNull WireOut wireOut) { - // no fields to write - } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArraySizedReader.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArraySizedReader.java index 9a71cfd98..7c70c5320 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArraySizedReader.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/ByteArraySizedReader.java @@ -17,31 +17,30 @@ package net.openhft.chronicle.hash.serialization.impl; import net.openhft.chronicle.bytes.Bytes; +import net.openhft.chronicle.core.io.IORuntimeException; import net.openhft.chronicle.hash.serialization.SizedReader; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; public enum ByteArraySizedReader implements SizedReader, EnumMarshallable { + INSTANCE; @NotNull @Override public byte[] read(@NotNull Bytes in, long size, @Nullable byte[] using) { - int resLen = resLen(size); - if (using == null || resLen != using.length) - using = new byte[resLen]; + if (size < 0L || size > (long) Integer.MAX_VALUE) { + throw new IORuntimeException("byte[] size should be non-negative int, " + + size + " given. Memory corruption?"); + } + int arrayLength = (int) size; + if (using == null || arrayLength != using.length) + using = new byte[arrayLength]; in.read(using); return using; } - private int resLen(long size) { - if (size < 0L || size > (long) Integer.MAX_VALUE) - throw new IllegalArgumentException("byte[] size should be non-negative int, " + - size + " given. Memory corruption?"); - return (int) size; - } - @Override public ByteArraySizedReader readResolve() { return INSTANCE; diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/package-info.java b/src/main/java/net/openhft/chronicle/hash/serialization/package-info.java index 9455a319d..30531c96b 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/package-info.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/package-info.java @@ -15,8 +15,13 @@ */ /** - * The package contains interfaces for serializing objects and values between Java heap and {@link - * net.openhft.chronicle.bytes.Bytes} or {@link net.openhft.chronicle.bytes.BytesStore}. + * The package contains interfaces for serializing objects between Java heap and {@link + * net.openhft.chronicle.bytes.Bytes} or {@link net.openhft.chronicle.bytes.BytesStore}, used by + * Chronicle Map to store objects off-heap, and read them back from off-heap memory to on-heap + * objects. + * + *

Read Custom serializers + * section in the Chronicle Map tutorial for more information. * *

Reading methods in the interfaces in this package could use {@link * net.openhft.chronicle.bytes.StreamingDataInput} as the "input" parameter type and {@link diff --git a/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java b/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java index 2c75232be..ea50551cf 100644 --- a/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java +++ b/src/test/java/net/openhft/chronicle/map/DefaultValueTest.java @@ -40,8 +40,8 @@ public class DefaultValueTest { defaultValue.add(42); try (ChronicleMap> map = ChronicleMap .of(String.class, (Class>) ((Class) List.class)) - .entries(3) .valueMarshaller(ListMarshaller.of(IntegerMarshaller.INSTANCE)) + .entries(3) .averageKey("a").averageValue(Arrays.asList(1, 2)) .defaultValueProvider(absentEntry -> absentEntry.context().wrapValueAsData(defaultValue)) diff --git a/src/test/java/net/openhft/chronicle/map/example/ComplexSerializationTest.java b/src/test/java/net/openhft/chronicle/map/example/PointListSerializationTest.java similarity index 98% rename from src/test/java/net/openhft/chronicle/map/example/ComplexSerializationTest.java rename to src/test/java/net/openhft/chronicle/map/example/PointListSerializationTest.java index e4a0de87d..2009b99e7 100644 --- a/src/test/java/net/openhft/chronicle/map/example/ComplexSerializationTest.java +++ b/src/test/java/net/openhft/chronicle/map/example/PointListSerializationTest.java @@ -29,7 +29,7 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; -public class ComplexSerializationTest { +public class PointListSerializationTest { static class A { String str_; diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArrayBytesMarshaller.java b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArrayBytesMarshaller.java new file mode 100644 index 000000000..000d6176f --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArrayBytesMarshaller.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.bytes.Bytes; +import net.openhft.chronicle.core.util.ReadResolvable; +import net.openhft.chronicle.hash.impl.util.Objects; +import net.openhft.chronicle.hash.serialization.BytesReader; +import net.openhft.chronicle.hash.serialization.BytesWriter; +import net.openhft.chronicle.wire.WireIn; +import net.openhft.chronicle.wire.WireOut; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Arrays; + +public enum CharSequenceArrayBytesMarshaller + implements BytesWriter, BytesReader, + ReadResolvable { + + INSTANCE; + + @Override + public void write(Bytes out, @NotNull CharSequence[] toWrite) { + // Note that the *array length*, not the *serialization size* is written in the beginning, + // so BytesWriter + BytesReader is the most suitable pair of serialization interfaces + // to implement for CharSequence[] type + out.writeInt(toWrite.length); + for (CharSequence cs : toWrite) { + // Assume elements non-null for simplicity + Objects.requireNonNull(cs); + out.writeUtf8(cs); + } + } + + @NotNull + @Override + public CharSequence[] read(Bytes in, @Nullable CharSequence[] using) { + int len = in.readInt(); + if (using == null) + using = new CharSequence[len]; + if (using.length != len) + using = Arrays.copyOf(using, len); + for (int i = 0; i < len; i++) { + CharSequence cs = using[i]; + if (cs instanceof StringBuilder) { + in.readUtf8((StringBuilder) cs); + } else { + StringBuilder sb = new StringBuilder(0); + in.readUtf8(sb); + using[i] = sb; + } + } + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public CharSequenceArrayBytesMarshaller readResolve() { + return INSTANCE; + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArraySerializationTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArraySerializationTest.java new file mode 100644 index 000000000..52331b165 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceArraySerializationTest.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.map.ChronicleMap; +import org.junit.Assert; +import org.junit.Test; + +public class CharSequenceArraySerializationTest { + + @Test + public void charSequenceArraySerializationTest() { + try (ChronicleMap map = ChronicleMap + .of(String.class, CharSequence[].class) + .averageKey("fruits") + .valueMarshaller(CharSequenceArrayBytesMarshaller.INSTANCE) + .averageValue(new CharSequence[]{"banana", "pineapple"}) + .entries(2) + .create()) { + map.put("fruits", new CharSequence[]{"banana", "pineapple"}); + map.put("vegetables", new CharSequence[] {"carrot", "potato"}); + Assert.assertEquals(2, map.get("fruits").length); + Assert.assertEquals(2, map.get("vegetables").length); + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesReader.java b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesReader.java new file mode 100644 index 000000000..9a2182721 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesReader.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.bytes.Bytes; +import net.openhft.chronicle.core.io.IORuntimeException; +import net.openhft.chronicle.hash.serialization.BytesReader; +import net.openhft.chronicle.hash.serialization.StatefulCopyable; +import net.openhft.chronicle.wire.WireIn; +import net.openhft.chronicle.wire.WireOut; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; + +public final class CharSequenceCustomEncodingBytesReader + implements BytesReader, + StatefulCopyable { + + // config fields, non-final because read in readMarshallable() + private Charset charset; + private int inputBufferSize; + + // cache fields + private transient CharsetDecoder charsetDecoder; + private transient ByteBuffer inputBuffer; + private transient CharBuffer outputBuffer; + + public CharSequenceCustomEncodingBytesReader(Charset charset, int inputBufferSize) { + this.charset = charset; + this.inputBufferSize = inputBufferSize; + initTransients(); + } + + private void initTransients() { + charsetDecoder = charset.newDecoder(); + inputBuffer = ByteBuffer.allocate(inputBufferSize); + int outputBufferSize = (int) (inputBufferSize * charsetDecoder.averageCharsPerByte()); + outputBuffer = CharBuffer.allocate(outputBufferSize); + } + + @NotNull + @Override + public CharSequence read(Bytes in, @Nullable CharSequence using) { + long csLengthAsLong = in.readStopBit(); + if (csLengthAsLong > Integer.MAX_VALUE) { + throw new IORuntimeException("cs len shouldn't be more than " + Integer.MAX_VALUE + + ", " + csLengthAsLong + " read"); + } + int csLength = (int) csLengthAsLong; + StringBuilder sb; + if (using instanceof StringBuilder) { + sb = (StringBuilder) using; + sb.setLength(0); + sb.ensureCapacity(csLength); + } else { + sb = new StringBuilder(csLength); + } + + int remainingBytes = in.readInt(); + charsetDecoder.reset(); + inputBuffer.clear(); + outputBuffer.clear(); + boolean endOfInput = false; + // this loop inspired by the CharsetDecoder.decode(ByteBuffer) implementation + while (true) { + if (!endOfInput) { + int inputChunkSize = Math.min(inputBuffer.remaining(), remainingBytes); + inputBuffer.limit(inputBuffer.position() + inputChunkSize); + in.read(inputBuffer); + inputBuffer.flip(); + remainingBytes -= inputChunkSize; + endOfInput = remainingBytes == 0; + } + + CoderResult cr = inputBuffer.hasRemaining() ? + charsetDecoder.decode(inputBuffer, outputBuffer, endOfInput) : + CoderResult.UNDERFLOW; + + if (cr.isUnderflow() && endOfInput) + cr = charsetDecoder.flush(outputBuffer); + + if (cr.isUnderflow()) { + if (endOfInput) { + break; + } else { + inputBuffer.compact(); + continue; + } + } + + if (cr.isOverflow()) { + outputBuffer.flip(); + sb.append(outputBuffer); + outputBuffer.clear(); + continue; + } + + try { + cr.throwException(); + } catch (CharacterCodingException e) { + throw new IORuntimeException(e); + } + } + outputBuffer.flip(); + sb.append(outputBuffer); + + return sb; + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) throws IORuntimeException { + charset = (Charset) wireIn.read(() -> "charset").object(); + inputBufferSize = wireIn.read(() -> "inputBufferSize").int32(); + initTransients(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + wireOut.write(() -> "charset").object(charset); + wireOut.write(() -> "inputBufferSize").int32(inputBufferSize); + } + + @Override + public CharSequenceCustomEncodingBytesReader copy() { + return new CharSequenceCustomEncodingBytesReader(charset, inputBufferSize); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesWriter.java b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesWriter.java new file mode 100644 index 000000000..aa25f96bc --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/CharSequenceCustomEncodingBytesWriter.java @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.bytes.Bytes; +import net.openhft.chronicle.core.io.IORuntimeException; +import net.openhft.chronicle.hash.serialization.BytesWriter; +import net.openhft.chronicle.hash.serialization.StatefulCopyable; +import net.openhft.chronicle.wire.WireIn; +import net.openhft.chronicle.wire.WireOut; +import org.jetbrains.annotations.NotNull; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; + +public final class CharSequenceCustomEncodingBytesWriter + implements BytesWriter, + StatefulCopyable { + + // config fields, non-final because read in readMarshallable() + private Charset charset; + private int inputBufferSize; + + // cache fields + private transient CharsetEncoder charsetEncoder; + private transient CharBuffer inputBuffer; + private transient ByteBuffer outputBuffer; + + public CharSequenceCustomEncodingBytesWriter(Charset charset, int inputBufferSize) { + this.charset = charset; + this.inputBufferSize = inputBufferSize; + initTransients(); + } + + private void initTransients() { + charsetEncoder = charset.newEncoder(); + inputBuffer = CharBuffer.allocate(inputBufferSize); + int outputBufferSize = (int) (inputBufferSize * charsetEncoder.averageBytesPerChar()); + outputBuffer = ByteBuffer.allocate(outputBufferSize); + } + + @Override + public void write(Bytes out, @NotNull CharSequence cs) { + // Write the actual cs length for accurate StringBuilder.ensureCapacity() while reading + out.writeStopBit(cs.length()); + long encodedSizePos = out.writePosition(); + out.writeSkip(4); + charsetEncoder.reset(); + inputBuffer.clear(); + outputBuffer.clear(); + int csPos = 0; + boolean endOfInput = false; + // this loop inspired by the CharsetEncoder.encode(CharBuffer) implementation + while (true) { + if (!endOfInput) { + int nextCsPos = Math.min(csPos + inputBuffer.remaining(), cs.length()); + append(inputBuffer, cs, csPos, nextCsPos); + inputBuffer.flip(); + endOfInput = nextCsPos == cs.length(); + csPos = nextCsPos; + } + + CoderResult cr = inputBuffer.hasRemaining() ? + charsetEncoder.encode(inputBuffer, outputBuffer, endOfInput) : + CoderResult.UNDERFLOW; + + if (cr.isUnderflow() && endOfInput) + cr = charsetEncoder.flush(outputBuffer); + + if (cr.isUnderflow()) { + if (endOfInput) { + break; + } else { + inputBuffer.compact(); + continue; + } + } + + if (cr.isOverflow()) { + outputBuffer.flip(); + out.write(outputBuffer); + outputBuffer.clear(); + continue; + } + + try { + cr.throwException(); + } catch (CharacterCodingException e) { + throw new IORuntimeException(e); + } + } + outputBuffer.flip(); + out.write(outputBuffer); + + out.writeInt(encodedSizePos, (int) (out.writePosition() - encodedSizePos - 4)); + } + + /** + * Need this method because {@link CharBuffer#append(CharSequence, int, int)} produces garbage + */ + private void append(CharBuffer charBuffer, CharSequence cs, int start, int end) { + for (int i = start; i < end; i++) { + charBuffer.put(cs.charAt(i)); + } + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + charset = (Charset) wireIn.read(() -> "charset").object(); + inputBufferSize = wireIn.read(() -> "inputBufferSize").int32(); + initTransients(); + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + wireOut.write(() -> "charset").object(charset); + wireOut.write(() -> "inputBufferSize").int32(inputBufferSize); + } + + @Override + public CharSequenceCustomEncodingBytesWriter copy() { + return new CharSequenceCustomEncodingBytesWriter(charset, inputBufferSize); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/CustomCharSequenceEncodingTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/CustomCharSequenceEncodingTest.java new file mode 100644 index 000000000..4720596b1 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/CustomCharSequenceEncodingTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.map.ChronicleMap; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertNotNull; + +public class CustomCharSequenceEncodingTest { + + @Test + public void customCharSequenceEncodingTest() { + Charset charset = StandardCharsets.UTF_8; + int charBufferSize = 4; + int bytesBufferSize = 8; + CharSequenceCustomEncodingBytesWriter writer = + new CharSequenceCustomEncodingBytesWriter(charset, charBufferSize); + CharSequenceCustomEncodingBytesReader reader = + new CharSequenceCustomEncodingBytesReader(charset, bytesBufferSize); + try (ChronicleMap map = ChronicleMap + .of(String.class, CharSequence.class) + .valueMarshallers(reader, writer) + .averageKey("Russian") + .averageValue("Всем нравится субботний вечерок") + .entries(10) + .create()) { + map.put("Russian", "Всем нравится субботний вечерок"); + map.put("", "Quick brown fox jumps over the lazy dog"); + } + } + + @Test + public void gbkCharSequenceEncodingTest() { + Charset charset = Charset.forName("GBK"); + int charBufferSize = 100; + int bytesBufferSize = 200; + CharSequenceCustomEncodingBytesWriter writer = + new CharSequenceCustomEncodingBytesWriter(charset, charBufferSize); + CharSequenceCustomEncodingBytesReader reader = + new CharSequenceCustomEncodingBytesReader(charset, bytesBufferSize); + try (ChronicleMap englishToChinese = ChronicleMap + .of(String.class, CharSequence.class) + .valueMarshallers(reader, writer) + .averageKey("hello") + .averageValue("你好") + .entries(10) + .create()) { + englishToChinese.put("hello", "你好"); + englishToChinese.put("bye", "再见"); + + Assert.assertEquals("你好", englishToChinese.get("hello").toString()); + Assert.assertEquals("再见", englishToChinese.get("bye").toString()); + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayReader.java b/src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayReader.java deleted file mode 100644 index aa9db288a..000000000 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayReader.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map.fromdocs; - -import net.openhft.chronicle.bytes.Bytes; -import net.openhft.chronicle.hash.serialization.SizedReader; -import net.openhft.chronicle.hash.serialization.impl.EnumMarshallable; -import org.jetbrains.annotations.NotNull; - -import java.util.Arrays; - -enum LongPairArrayReader implements SizedReader, EnumMarshallable { - INSTANCE; - - @NotNull - @Override - public LongPair[] read(@NotNull Bytes in, long size, LongPair[] using) { - if (size > Integer.MAX_VALUE * 16L) - throw new IllegalStateException("LongPair[] size couldn't be " + (size / 16L)); - int resLen = (int) (size / 16L); - LongPair[] res; - if (using != null) { - if (using.length == resLen) { - res = using; - } else { - res = Arrays.copyOf(using, resLen); - } - } else { - res = new LongPair[resLen]; - } - for (int i = 0; i < resLen; i++) { - LongPair pair = res[i]; - if (pair == null) - res[i] = pair = new LongPair(); - pair.first = in.readLong(); - pair.second = in.readLong(); - } - return res; - } - - @Override - public LongPairArrayReader readResolve() { - return INSTANCE; - } -} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayWriter.java b/src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayWriter.java deleted file mode 100644 index 7685c9e3e..000000000 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/LongPairArrayWriter.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map.fromdocs; - -import net.openhft.chronicle.bytes.Bytes; -import net.openhft.chronicle.hash.serialization.SizedWriter; -import net.openhft.chronicle.hash.serialization.impl.EnumMarshallable; -import org.jetbrains.annotations.NotNull; - -enum LongPairArrayWriter implements SizedWriter, EnumMarshallable { - INSTANCE; - - @Override - public long size(@NotNull LongPair[] toWrite) { - return toWrite.length * 16L; - } - - @Override - public void write(@NotNull Bytes out, long size, @NotNull LongPair[] toWrite) { - for (LongPair pair : toWrite) { - out.writeLong(pair.first); - out.writeLong(pair.second); - } - } - - @Override - public LongPairArrayWriter readResolve() { - return INSTANCE; - } -} diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/LongPair.java b/src/test/java/net/openhft/chronicle/map/fromdocs/Point.java similarity index 80% rename from src/test/java/net/openhft/chronicle/map/fromdocs/LongPair.java rename to src/test/java/net/openhft/chronicle/map/fromdocs/Point.java index 1203ba6ed..66dbeb107 100644 --- a/src/test/java/net/openhft/chronicle/map/fromdocs/LongPair.java +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/Point.java @@ -16,6 +16,14 @@ package net.openhft.chronicle.map.fromdocs; -public final class LongPair { - long first, second; +public final class Point { + + public static Point of(double x, double y) { + Point p = new Point(); + p.x = x; + p.y = y; + return p; + } + + double x, y; } diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/PointListSizedMarshaller.java b/src/test/java/net/openhft/chronicle/map/fromdocs/PointListSizedMarshaller.java new file mode 100644 index 000000000..dd2a2553f --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/PointListSizedMarshaller.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.bytes.Bytes; +import net.openhft.chronicle.core.io.IORuntimeException; +import net.openhft.chronicle.core.util.ReadResolvable; +import net.openhft.chronicle.hash.serialization.SizedReader; +import net.openhft.chronicle.hash.serialization.SizedWriter; +import net.openhft.chronicle.wire.WireIn; +import net.openhft.chronicle.wire.WireOut; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +public enum PointListSizedMarshaller + implements SizedReader>, SizedWriter>, + ReadResolvable { + + INSTANCE; + + /** A point takes 16 bytes in serialized form: 8 bytes for both x and y value */ + private static final long ELEMENT_SIZE = 16; + + @Override + public long size(@NotNull List toWrite) { + return toWrite.size() * ELEMENT_SIZE; + } + + @Override + public void write(Bytes out, long size, @NotNull List toWrite) { + toWrite.forEach(point -> { + out.writeDouble(point.x); + out.writeDouble(point.y); + }); + } + + @NotNull + @Override + public List read(@NotNull Bytes in, long size, List using) { + if (size % ELEMENT_SIZE != 0) { + throw new IORuntimeException("Bytes size should be a multiple of " + ELEMENT_SIZE + + ", " + size + " read"); + } + long listSizeAsLong = size / ELEMENT_SIZE; + if (listSizeAsLong > Integer.MAX_VALUE) { + throw new IORuntimeException("List size couldn't be more than " + Integer.MAX_VALUE + + ", " + listSizeAsLong + " read"); + } + int listSize = (int) listSizeAsLong; + if (using == null) { + using = new ArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + using.add(null); + } + } else if (using.size() < listSize) { + while (using.size() < listSize) { + using.add(null); + } + } else if (using.size() > listSize) { + using.subList(listSize, using.size()).clear(); + } + for (int i = 0; i < listSize; i++) { + Point point = using.get(i); + if (point == null) + using.set(i, point = new Point()); + point.x = in.readDouble(); + point.y = in.readDouble(); + } + return using; + } + + @Override + public void writeMarshallable(@NotNull WireOut wireOut) { + // no fields to write + } + + @Override + public void readMarshallable(@NotNull WireIn wireIn) { + // no fields to read + } + + @Override + public PointListSizedMarshaller readResolve() { + return INSTANCE; + } +} \ No newline at end of file diff --git a/src/test/java/net/openhft/chronicle/map/fromdocs/PointSerializationTest.java b/src/test/java/net/openhft/chronicle/map/fromdocs/PointSerializationTest.java new file mode 100644 index 000000000..6db477752 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/fromdocs/PointSerializationTest.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map.fromdocs; + +import net.openhft.chronicle.map.ChronicleMap; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +import static java.util.Arrays.asList; +import static net.openhft.chronicle.map.fromdocs.Point.of; + +public class PointSerializationTest { + + @Test + public void pointSerializationTest() { + try (ChronicleMap> objects = ChronicleMap + .of(String.class, (Class>) (Class) List.class) + .averageKey("range") + .valueMarshaller(PointListSizedMarshaller.INSTANCE) + .averageValue(asList(of(0, 0), of(1, 1))) + .entries(10) + .create()) { + objects.put("range", asList(of(0, 0), of(1, 1))); + objects.put("square", asList(of(0, 0), of(0, 100), of(100, 100), of(100, 0))); + + Assert.assertEquals(2, objects.get("range").size()); + Assert.assertEquals(4, objects.get("square").size()); + } + } +}