From c935c4b05f771b0ea370c9ea32da0fa9ce4715ab Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 13 Dec 2014 15:19:34 +0700 Subject: [PATCH] HCOLL-171 HCOLL-206 HCOLL-247 improved hashing algorithm (now murmur3), implemented ByteBufferInterop, some improvements in existing marshallers --- .../chronicle/hash/hashing/Access.java | 140 +++++++++++++ .../chronicle/hash/hashing/Accesses.java | 69 +++++++ .../hash/hashing/ByteBufferAccess.java | 64 ++++++ .../chronicle/hash/hashing/BytesAccess.java | 66 ++++++ .../chronicle/hash/hashing/Hasher.java | 194 ++++++++++++++++++ .../chronicle/hash/hashing/Primitives.java | 34 +++ .../chronicle/hash/hashing/UnsafeAccess.java | 93 +++++++++ .../chronicle/hash/serialization/Hasher.java | 83 -------- .../internal/ByteArrayMarshaller.java | 9 +- .../internal/ByteBufferMarshaller.java | 95 +++++++++ .../internal/ByteableMarshaller.java | 3 + .../internal/BytesBytesInterop.java | 8 +- .../internal/CharArrayMarshaller.java | 6 +- .../internal/CopyingMetaBytesInterop.java | 3 +- .../internal/DoubleMarshaller.java | 2 +- .../internal/IntegerMarshaller.java | 1 + .../internal/LongMarshaller.java | 2 +- .../internal/VoidMarshaller.java | 3 +- .../chronicle/map/ReplicatedChronicleMap.java | 2 +- .../chronicle/map/SerializationBuilder.java | 4 + .../chronicle/map/VanillaChronicleMap.java | 2 +- .../chronicle/map/CHMUseCasesTest.java | 3 - .../net/openhft/chronicle/map/MarkTest.java | 89 ++++++++ 23 files changed, 876 insertions(+), 99 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/hash/hashing/Access.java create mode 100644 src/main/java/net/openhft/chronicle/hash/hashing/Accesses.java create mode 100644 src/main/java/net/openhft/chronicle/hash/hashing/ByteBufferAccess.java create mode 100644 src/main/java/net/openhft/chronicle/hash/hashing/BytesAccess.java create mode 100644 src/main/java/net/openhft/chronicle/hash/hashing/Hasher.java create mode 100644 src/main/java/net/openhft/chronicle/hash/hashing/Primitives.java create mode 100644 src/main/java/net/openhft/chronicle/hash/hashing/UnsafeAccess.java delete mode 100644 src/main/java/net/openhft/chronicle/hash/serialization/Hasher.java create mode 100644 src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java create mode 100644 src/test/java/net/openhft/chronicle/map/MarkTest.java diff --git a/src/main/java/net/openhft/chronicle/hash/hashing/Access.java b/src/main/java/net/openhft/chronicle/hash/hashing/Access.java new file mode 100644 index 000000000..44088b816 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/hashing/Access.java @@ -0,0 +1,140 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.hashing; + +import java.nio.ByteOrder; + +/** + * Strategy of reading bytes, defines the abstraction of {@code T} class instances as ordered byte + * sequence. All {@code getXXX(input, offset)} should be consistent to each other in terms of + * ordered byte sequence each {@code T} instance represents. For example, if some {@code + * Access} implementation returns {@link ByteOrder#LITTLE_ENDIAN} on {@link #byteOrder(Object) + * byteOrder(input)} call, the following expressions should always have the same value: + * + * + *

{@code getXXX(input, offset)} methods could throw unchecked exceptions when requested bytes + * range is outside of the bounds of the byte sequence, represented by the given {@code input}. + * However, they could omit checks for better performance. + * + *

{@code Access} API is designed for inputs, that actually represent byte sequences that lay + * continuously in memory. Theoretically {@code Access} strategy could be implemented for + * non-continuous byte sequences, or abstractions which aren't actually present in memory as they + * are accessed, but this should be awkward, and hashing using such {@code Access} is expected to + * be slow. + * + * @param the type of the object to access + */ +public interface Access { + + /** + * Reads {@code [offset, offset + 7]} bytes of the byte sequence represented by the given + * {@code input} as a single {@code long} value. + * + * @param input the object to access + * @param offset offset to the first byte to read within the byte sequence represented + * by the given object + * @return eight bytes as a {@code long} value, in {@linkplain #byteOrder(Object) the expected + * order} + */ + long getLong(T input, long offset); + + /** + * Shortcut for {@code getInt(input, offset) & 0xFFFFFFFFL}. Could be implemented more + * efficiently. + * + * @param input the object to access + * @param offset offset to the first byte to read within the byte sequence represented + * by the given object + * @return four bytes as an unsigned int value, in {@linkplain #byteOrder(Object) the expected + * order} + */ + long getUnsignedInt(T input, long offset); + + /** + * Reads {@code [offset, offset + 3]} bytes of the byte sequence represented by the given + * {@code input} as a single {@code int} value. + * + * @param input the object to access + * @param offset offset to the first byte to read within the byte sequence represented + * by the given object + * @return four bytes as an {@code int} value, in {@linkplain #byteOrder(Object) the expected + * order} + */ + int getInt(T input, long offset); + + /** + * Shortcut for {@code getShort(input, offset) & 0xFFFF}. Could be implemented more + * efficiently. + * + * @param input the object to access + * @param offset offset to the first byte to read within the byte sequence represented + * by the given object + * @return two bytes as an unsigned short value, in {@linkplain #byteOrder(Object) the expected + * order} + */ + int getUnsignedShort(T input, long offset); + + /** + * Reads {@code [offset, offset + 1]} bytes of the byte sequence represented by the given + * {@code input} as a single {@code short} value, returned widened to {@code int}. + * + * @param input the object to access + * @param offset offset to the first byte to read within the byte sequence represented + * by the given object + * @return two bytes as a {@code short} value, in {@linkplain #byteOrder(Object) the expected + * order}, widened to {@code int} + */ + int getShort(T input, long offset); + + /** + * Shortcut for {@code getByte(input, offset) & 0xFF}. Could be implemented more efficiently. + * + * @param input the object to access + * @param offset offset to the byte to read within the byte sequence represented + * by the given object + * @return a byte by the given {@code offset}, interpreted as unsigned + */ + int getUnsignedByte(T input, long offset); + + /** + * Reads a single byte at the given {@code offset} in the byte sequence represented by the given + * {@code input}, returned widened to {@code int}. + * + * @param input the object to access + * @param offset offset to the byte to read within the byte sequence represented + * by the given object + * @return a byte by the given {@code offset}, widened to {@code int} + */ + int getByte(T input, long offset); + + /** + * The byte order in which all multi-byte {@code getXXX()} reads from the given {@code input} + * are performed. + * + * @param input the accessed object + * @return the byte order of all multi-byte reads from the given {@code input} + */ + ByteOrder byteOrder(T input); +} diff --git a/src/main/java/net/openhft/chronicle/hash/hashing/Accesses.java b/src/main/java/net/openhft/chronicle/hash/hashing/Accesses.java new file mode 100644 index 000000000..d9b71abd5 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/hashing/Accesses.java @@ -0,0 +1,69 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.hashing; + +import net.openhft.lang.io.Bytes; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Static methods returning useful {@link Access} implementations. + */ +public final class Accesses { + + /** + * Returns the {@code Access} delegating {@code getXXX(input, offset)} methods to {@code + * sun.misc.Unsafe.getXXX(input, offset)}. + * + *

Usage example: 

{@code
+     * class Pair {
+     *     long first, second;
+     *
+     *     static final long pairDataOffset =
+     *         theUnsafe.objectFieldOffset(Pair.class.getDeclaredField("first"));
+     *
+     *     static long hashPair(Pair pair, LongHashFunction hashFunction) {
+     *         return hashFunction.hash(pair, Accesses.unsafe(), pairDataOffset, 16L);
+     *     }
+     * }}
+ * + *

{@code null} is a valid input, on accepting {@code null} {@code Unsafe} just interprets + * the given offset as a wild memory address. + * + * @param the type of objects to access + * @return the unsafe memory {@code Access} + */ + public static Access unsafe() { + return (Access) UnsafeAccess.INSTANCE; + } + + /** + * Returns the {@code Access} to any {@link ByteBuffer}. + * + * @return the {@code Access} to {@link ByteBuffer}s + */ + public static Access toByteBuffer() { + return ByteBufferAccess.INSTANCE; + } + + public static Access toBytes() { + return BytesAccess.INSTANCE; + } + + private Accesses() {} +} diff --git a/src/main/java/net/openhft/chronicle/hash/hashing/ByteBufferAccess.java b/src/main/java/net/openhft/chronicle/hash/hashing/ByteBufferAccess.java new file mode 100644 index 000000000..703111dff --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/hashing/ByteBufferAccess.java @@ -0,0 +1,64 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.hashing; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +enum ByteBufferAccess implements Access { + INSTANCE; + + @Override + public long getLong(ByteBuffer input, long offset) { + return input.getLong((int) offset); + } + + @Override + public long getUnsignedInt(ByteBuffer input, long offset) { + return Primitives.unsignedInt(getInt(input, offset)); + } + + @Override + public int getInt(ByteBuffer input, long offset) { + return input.getInt((int) offset); + } + + @Override + public int getUnsignedShort(ByteBuffer input, long offset) { + return Primitives.unsignedShort(getShort(input, offset)); + } + + @Override + public int getShort(ByteBuffer input, long offset) { + return input.getShort((int) offset); + } + + @Override + public int getUnsignedByte(ByteBuffer input, long offset) { + return Primitives.unsignedByte(getByte(input, offset)); + } + + @Override + public int getByte(ByteBuffer input, long offset) { + return input.get((int) offset); + } + + @Override + public ByteOrder byteOrder(ByteBuffer input) { + return input.order(); + } +} diff --git a/src/main/java/net/openhft/chronicle/hash/hashing/BytesAccess.java b/src/main/java/net/openhft/chronicle/hash/hashing/BytesAccess.java new file mode 100644 index 000000000..4e67b40f8 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/hashing/BytesAccess.java @@ -0,0 +1,66 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.hashing; + +import net.openhft.lang.io.Bytes; + +import java.nio.ByteOrder; + +enum BytesAccess implements Access { + INSTANCE; + + @Override + public long getLong(Bytes input, long offset) { + return input.readLong(offset); + } + + @Override + public long getUnsignedInt(Bytes input, long offset) { + return input.readUnsignedInt(offset); + } + + @Override + public int getInt(Bytes input, long offset) { + return input.readInt(offset); + } + + @Override + public int getUnsignedShort(Bytes input, long offset) { + return input.readUnsignedShort(offset); + } + + @Override + public int getShort(Bytes input, long offset) { + return input.readShort(offset); + } + + @Override + public int getUnsignedByte(Bytes input, long offset) { + return input.readUnsignedByte(offset); + } + + @Override + public int getByte(Bytes input, long offset) { + return input.readByte(offset); + } + + @Override + public ByteOrder byteOrder(Bytes input) { + return input.byteOrder(); + } + +} diff --git a/src/main/java/net/openhft/chronicle/hash/hashing/Hasher.java b/src/main/java/net/openhft/chronicle/hash/hashing/Hasher.java new file mode 100644 index 000000000..a7c97393c --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/hashing/Hasher.java @@ -0,0 +1,194 @@ +/* + * Copyright 2014 Higher Frequency Trading + * + * http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.hashing; + +import net.openhft.lang.io.Bytes; + +import static net.openhft.chronicle.hash.hashing.UnsafeAccess.BYTE_BASE; +import static net.openhft.chronicle.hash.hashing.UnsafeAccess.CHAR_BASE; + +public final class Hasher { + + // MurmurHash3, derived from + // https://github.com/google/guava/blob/fa95e381e665d8ee9639543b99ed38020c8de5ef + // /guava/src/com/google/common/hash/Murmur3_128HashFunction.java + + + private static final long C1 = 0x87c37b91114253d5L; + private static final long C2 = 0x4cf5ad432745937fL; + + /** + * Returns the hash code for {@code len} continuous bytes of the given {@code input} object, + * starting from the given offset. The abstraction of input as ordered byte sequence and + * "offset within the input" is defined by the given {@code access} strategy. + * + *

This method doesn't promise to throw a {@code RuntimeException} if {@code + * [off, off + len - 1]} subsequence exceeds the bounds of the bytes sequence, defined by {@code + * access} strategy for the given {@code input}, so use this method with caution. + * + * @param input the object to read bytes from + * @param access access which defines the abstraction of the given input + * as ordered byte sequence + * @param offset offset to the first byte of the subsequence to hash + * @param length length of the subsequence to hash + * @param the type of the input + * @return hash code for the specified bytes subsequence + */ + public static long hash(T input, Access access, long offset, long length) { + assert offset >= 0L && length >= 0L; + + long h1 = 0L; + long h2 = 0L; + long remaining = length; + while (remaining >= 16L) { + long k1 = access.getLong(input, offset); + long k2 = access.getLong(input, offset); + offset += 16L; + remaining -= 16L; + h1 ^= mixK1(k1); + + h1 = Long.rotateLeft(h1, 27); + h1 += h2; + h1 = h1 * 5L + 0x52dce729L; + + h2 ^= mixK2(k2); + + h2 = Long.rotateLeft(h2, 31); + h2 += h1; + h2 = h2 * 5L + 0x38495ab5L; + } + + if (remaining > 0L) { + long k1 = 0L; + long k2 = 0L; + switch ((int) remaining) { + case 15: + k2 ^= ((long) access.getUnsignedByte(input, offset + 14L)) << 48;// fall through + case 14: + k2 ^= ((long) access.getUnsignedByte(input, offset + 13L)) << 40;// fall through + case 13: + k2 ^= ((long) access.getUnsignedByte(input, offset + 12L)) << 32;// fall through + case 12: + k2 ^= ((long) access.getUnsignedByte(input, offset + 11L)) << 24;// fall through + case 11: + k2 ^= ((long) access.getUnsignedByte(input, offset + 10L)) << 16;// fall through + case 10: + k2 ^= ((long) access.getUnsignedByte(input, offset + 9L)) << 8; // fall through + case 9: + k2 ^= ((long) access.getUnsignedByte(input, offset + 8L)); // fall through + case 8: + k1 ^= access.getLong(input, offset); + break; + case 7: + k1 ^= ((long) access.getUnsignedByte(input, offset + 6L)) << 48; // fall through + case 6: + k1 ^= ((long) access.getUnsignedByte(input, offset + 5L)) << 40; // fall through + case 5: + k1 ^= ((long) access.getUnsignedByte(input, offset + 4L)) << 32; // fall through + case 4: + k1 ^= access.getUnsignedInt(input, offset); + break; + case 3: + k1 ^= ((long) access.getUnsignedByte(input, offset + 2L)) << 16; // fall through + case 2: + k1 ^= ((long) access.getUnsignedByte(input, offset + 1L)) << 8; // fall through + case 1: + k1 ^= ((long) access.getUnsignedByte(input, offset)); + case 0: + break; + default: + throw new AssertionError("Should never get here."); + } + h1 ^= mixK1(k1); + h2 ^= mixK2(k2); + } + + return finalize(length, h1, h2); + } + + private static long finalize(long length, long h1, long h2) { + h1 ^= length; + h2 ^= length; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + return h1; + } + + + private static long fmix64(long k) { + k ^= k >>> 33; + k *= 0xff51afd7ed558ccdL; + k ^= k >>> 33; + k *= 0xc4ceb9fe1a85ec53L; + k ^= k >>> 33; + return k; + } + + private static long mixK1(long k1) { + k1 *= C1; + k1 = Long.rotateLeft(k1, 31); + k1 *= C2; + return k1; + } + + private static long mixK2(long k2) { + k2 *= C2; + k2 = Long.rotateLeft(k2, 33); + k2 *= C1; + return k2; + } + + public static long hash(Bytes bytes) { + return hash(bytes, bytes.position(), bytes.limit()); + } + + public static long hash(Bytes bytes, long offset, long limit) { + return hash(bytes, Accesses.toBytes(), offset, limit - offset); + } + + public static long hash(byte[] array) { + return hash(array, Accesses.unsafe(), BYTE_BASE, (long) array.length); + } + + public static long hash(char[] array) { + return hash(array, Accesses.unsafe(), CHAR_BASE, array.length * 2L); + } + + public static long hash(int value) { + long k1 = Primitives.unsignedInt(value); + long h1 = 0L ^ mixK1(k1); + long h2 = 0L; + return finalize(4L, h1, h2); + } + + public static long hash(long value) { + long k1 = value; + long h1 = 0L ^ mixK1(k1); + long h2 = 0L; + return finalize(8L, h1, h2); + } + + private Hasher() {} +} diff --git a/src/main/java/net/openhft/chronicle/hash/hashing/Primitives.java b/src/main/java/net/openhft/chronicle/hash/hashing/Primitives.java new file mode 100644 index 000000000..ba1ac75a9 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/hashing/Primitives.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.hashing; + +final class Primitives { + + private Primitives() {} + + static long unsignedInt(int i) { + return i & 0xFFFFFFFFL; + } + + static int unsignedShort(int s) { + return s & 0xFFFF; + } + + static int unsignedByte(int b) { + return b & 0xFF; + } +} diff --git a/src/main/java/net/openhft/chronicle/hash/hashing/UnsafeAccess.java b/src/main/java/net/openhft/chronicle/hash/hashing/UnsafeAccess.java new file mode 100644 index 000000000..36d54a1cb --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/hashing/UnsafeAccess.java @@ -0,0 +1,93 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.hashing; + +import sun.misc.Unsafe; + +import java.lang.reflect.Field; +import java.nio.ByteOrder; + +import static net.openhft.chronicle.hash.hashing.Primitives.unsignedByte; +import static net.openhft.chronicle.hash.hashing.Primitives.unsignedInt; +import static net.openhft.chronicle.hash.hashing.Primitives.unsignedShort; + +enum UnsafeAccess implements Access { + INSTANCE; + + static final Unsafe UNSAFE; + static final long BYTE_BASE; + static final long CHAR_BASE; + static final long SHORT_BASE; + static final long INT_BASE; + static final long LONG_BASE; + + static { + try { + Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe"); + theUnsafe.setAccessible(true); + UNSAFE = (Unsafe) theUnsafe.get(null); + BYTE_BASE = UNSAFE.arrayBaseOffset(byte[].class); + CHAR_BASE = UNSAFE.arrayBaseOffset(char[].class); + SHORT_BASE = UNSAFE.arrayBaseOffset(short[].class); + INT_BASE = UNSAFE.arrayBaseOffset(int[].class); + LONG_BASE = UNSAFE.arrayBaseOffset(long[].class); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Override + public long getLong(Object input, long offset) { + return UNSAFE.getLong(input, offset); + } + + @Override + public long getUnsignedInt(Object input, long offset) { + return unsignedInt(getInt(input, offset)); + } + + @Override + public int getInt(Object input, long offset) { + return UNSAFE.getInt(input, offset); + } + + @Override + public int getUnsignedShort(Object input, long offset) { + return unsignedShort(getShort(input, offset)); + } + + @Override + public int getShort(Object input, long offset) { + return UNSAFE.getShort(input, offset); + } + + @Override + public int getUnsignedByte(Object input, long offset) { + return unsignedByte(getByte(input, offset)); + } + + @Override + public int getByte(Object input, long offset) { + return UNSAFE.getByte(input, offset); + } + + @Override + public ByteOrder byteOrder(Object input) { + return ByteOrder.nativeOrder(); + } + +} diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/Hasher.java b/src/main/java/net/openhft/chronicle/hash/serialization/Hasher.java deleted file mode 100644 index 21eeb1d30..000000000 --- a/src/main/java/net/openhft/chronicle/hash/serialization/Hasher.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.hash.serialization; - -import net.openhft.lang.Maths; -import net.openhft.lang.io.Bytes; -import net.openhft.lang.io.NativeBytes; -import sun.misc.Unsafe; - -import static java.nio.ByteOrder.LITTLE_ENDIAN; -import static java.nio.ByteOrder.nativeOrder; - -public enum Hasher { - ; - - private static final long LONG_LEVEL_PRIME_MULTIPLE = 0x9ddfea08eb382d69L; - private static final short SHORT_LEVEL_PRIME_MULTIPLE = 0x404f; - private static final byte BYTE_LEVEL_PRIME_MULTIPLE = 0x57; - - private static final int INT_HASH_LOW_SHORT_MULTIPLE = - nativeOrder() == LITTLE_ENDIAN ? SHORT_LEVEL_PRIME_MULTIPLE : 1; - private static final int INT_HASH_HIGH_SHORT_MULTIPLE = - nativeOrder() == LITTLE_ENDIAN ? 1 : SHORT_LEVEL_PRIME_MULTIPLE; - - public static long hash(Bytes bytes) { - return hash(bytes, bytes.position(), bytes.limit()); - } - - public static long hash(Bytes bytes, long offset, long limit) { - if (limit - offset == 8) - return Maths.hash(bytes.readLong(offset)); - long h = 0; - long i = offset; - for (; i < limit - 7; i += 8) - h = LONG_LEVEL_PRIME_MULTIPLE * h + bytes.readLong(i); - for (; i < limit - 1; i += 2) - h = SHORT_LEVEL_PRIME_MULTIPLE * h + bytes.readShort(i); - if (i < limit) - h = BYTE_LEVEL_PRIME_MULTIPLE * h + bytes.readByte(i); - return hash(h); - } - - public static long hash(Object array, int length) { - Unsafe unsafe = NativeBytes.UNSAFE; - long base = unsafe.arrayBaseOffset(array.getClass()); - long scale = unsafe.arrayIndexScale(array.getClass()); - long h = 0; - long i = 0; - long limit = ((long) length) * scale; - for (; i < limit - 7; i += 8) - h = LONG_LEVEL_PRIME_MULTIPLE * h + unsafe.getLong(array, base + i); - for (; i < limit - 1; i += 2) - h = SHORT_LEVEL_PRIME_MULTIPLE * h + unsafe.getShort(array, base + i); - if (i < limit) - h = BYTE_LEVEL_PRIME_MULTIPLE * h + unsafe.getByte(array, base + i); - return hash(h); - } - - public static long hash(int value) { - return hash(((long) (((short) (value >>> 16)) * INT_HASH_HIGH_SHORT_MULTIPLE)) + - ((short) value) * INT_HASH_LOW_SHORT_MULTIPLE); - } - - public static long hash(long value) { - return Maths.hash(value); - } -} diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteArrayMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteArrayMarshaller.java index b77e524bb..cb1084c2b 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteArrayMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteArrayMarshaller.java @@ -20,7 +20,7 @@ package net.openhft.chronicle.hash.serialization.internal; import net.openhft.chronicle.hash.serialization.BytesInterop; import net.openhft.chronicle.hash.serialization.BytesReader; -import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.chronicle.hash.hashing.Hasher; import net.openhft.lang.io.Bytes; public enum ByteArrayMarshaller implements BytesInterop, BytesReader { @@ -33,6 +33,8 @@ public enum ByteArrayMarshaller implements BytesInterop, BytesReader, BytesReader, BytesReader (long) Integer.MAX_VALUE) + throw new IllegalArgumentException("byte[] size should be non-negative int, " + + size + " given. Memory corruption?"); if (toReuse == null || size != toReuse.length) toReuse = new byte[(int) size]; bytes.read(toReuse); diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java new file mode 100644 index 000000000..faffb7876 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java @@ -0,0 +1,95 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.serialization.internal; + +import net.openhft.chronicle.hash.hashing.Accesses; +import net.openhft.chronicle.hash.hashing.Hasher; +import net.openhft.chronicle.hash.serialization.BytesInterop; +import net.openhft.chronicle.hash.serialization.BytesReader; +import net.openhft.lang.io.Bytes; +import org.jetbrains.annotations.Nullable; + +import java.nio.ByteBuffer; + +public enum ByteBufferMarshaller implements BytesInterop, BytesReader { + INSTANCE; + + @Override + public boolean startsWith(Bytes bytes, ByteBuffer bb) { + int inputRemaining = bb.remaining(); + if(bytes.capacity() - bytes.position() < (long) inputRemaining) + return false; + + long pos = bytes.position(); + int inputPos = bb.position(); + + int i; + for(i = 0; i < inputRemaining - 7; i += 8) { + if (bytes.readLong(pos + (long) i) != bb.getLong(inputPos + i)) + return false; + } + for(; i < inputRemaining - 3; i += 4) { + if (bytes.readInt(pos + (long) i) != bb.getInt(inputPos + i)) + return false; + } + for(; i < inputRemaining; i++) { + if (bytes.readByte(pos + (long) i) != bb.get(inputPos + i)) + return false; + } + + return true; + } + + @Override + public long hash(ByteBuffer bb) { + return Hasher.hash(bb, Accesses.toByteBuffer(), 0L, (long) bb.remaining()); + } + + @Override + public ByteBuffer read(Bytes bytes, long size) { + return read(bytes, size, null); + } + + @Override + public ByteBuffer read(Bytes bytes, long size, @Nullable ByteBuffer toReuse) { + if (size < 0L || size > (long) Integer.MAX_VALUE) + throw new IllegalArgumentException("ByteBuffer size should be non-negative int, " + + size + " given. Memory corruption?"); + int intSize = (int) size; + if (toReuse == null || toReuse.capacity() < intSize) { + toReuse = ByteBuffer.allocate(intSize); + } else { + toReuse.position(0); + toReuse.limit(intSize); + } + bytes.read(toReuse); + toReuse.flip(); + return toReuse; + } + + @Override + public long size(ByteBuffer bb) { + return (long) bb.remaining(); + } + + @Override + public void write(Bytes bytes, ByteBuffer bb) { + int position = bb.position(); + bytes.write(bb); + bb.position(position); + } +} diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java index ed8a20cde..bb39475c7 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java @@ -18,6 +18,7 @@ package net.openhft.chronicle.hash.serialization.internal; +import net.openhft.chronicle.hash.hashing.Hasher; import net.openhft.chronicle.hash.serialization.*; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.MultiStoreBytes; @@ -75,6 +76,8 @@ public abstract class ByteableMarshaller @Override public boolean startsWith(Bytes bytes, E e) { + if (bytes.capacity() - bytes.position() < size) + return false; Bytes input = e.bytes(); long pos = bytes.position(), inputPos = e.offset(); diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java index ae25852d8..c8cb5ea3d 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java @@ -17,7 +17,7 @@ package net.openhft.chronicle.hash.serialization.internal; import net.openhft.chronicle.hash.serialization.BytesInterop; -import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.chronicle.hash.hashing.Hasher; import net.openhft.lang.io.Bytes; public enum BytesBytesInterop implements BytesInterop { @@ -25,7 +25,11 @@ public enum BytesBytesInterop implements BytesInterop { @Override public boolean startsWith(Bytes bytes, Bytes e) { - return bytes.startsWith(e); + long limit = bytes.limit(); + bytes.limit(bytes.capacity()); + boolean result = bytes.startsWith(e); + bytes.limit(limit); + return result; } @Override diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharArrayMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharArrayMarshaller.java index 1a5d6ea0f..748575067 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharArrayMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharArrayMarshaller.java @@ -20,7 +20,7 @@ package net.openhft.chronicle.hash.serialization.internal; import net.openhft.chronicle.hash.serialization.BytesInterop; import net.openhft.chronicle.hash.serialization.BytesReader; -import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.chronicle.hash.hashing.Hasher; import net.openhft.lang.io.Bytes; public enum CharArrayMarshaller implements BytesInterop, BytesReader { @@ -28,6 +28,8 @@ public enum CharArrayMarshaller implements BytesInterop, BytesReader, BytesReader implements Cloneable, Serializable { } else if (eClass == char[].class) { reader((BytesReader) CharArrayMarshaller.INSTANCE); interop((BytesInterop) CharArrayMarshaller.INSTANCE); + } else if (eClass == ByteBuffer.class) { + reader((BytesReader) ByteBufferMarshaller.INSTANCE); + writer((BytesWriter) ByteBufferMarshaller.INSTANCE); } else if (concreteClass(eClass)) { BytesMarshaller marshaller = chooseMarshaller(eClass, eClass); if (marshaller != null) diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index fe5a0c584..ec03878d0 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -29,7 +29,7 @@ import com.thoughtworks.xstream.io.json.JettisonMappedXmlDriver; import net.openhft.chronicle.hash.ChronicleHashErrorListener; import net.openhft.chronicle.hash.serialization.BytesInterop; import net.openhft.chronicle.hash.serialization.BytesReader; -import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.chronicle.hash.hashing.Hasher; import net.openhft.chronicle.hash.serialization.SizeMarshaller; import net.openhft.chronicle.hash.serialization.internal.*; import net.openhft.lang.collection.DirectBitSet; diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index daf577fef..4f8569770 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -626,14 +626,11 @@ public class CHMUseCasesTest { } } - @Ignore("HCOLL-247 automatically register ByteBufferMarshaller") @Test public void testByteBufferByteBufferDefaultKeyValueMarshaller() throws ExecutionException, InterruptedException { try (ChronicleMap map = ChronicleMapBuilder .of(ByteBuffer.class, ByteBuffer.class) - // .keyMarshaller(ByteBufferMarshaller.INSTANCE) - // .valueMarshaller(ByteBufferMarshaller.INSTANCE) .keySize(8) .valueSize(8) .maxEntryOversizeFactor(1) diff --git a/src/test/java/net/openhft/chronicle/map/MarkTest.java b/src/test/java/net/openhft/chronicle/map/MarkTest.java new file mode 100644 index 000000000..c54b7fe16 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/MarkTest.java @@ -0,0 +1,89 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.MemoryUnit; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; + +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class MarkTest { + + static int ENTRIES = 25_000_000; + + @Test + public void inMemoryTest() { + test(new Function, ChronicleMap>() { + @Override + public ChronicleMap apply( + ChronicleMapBuilder builder) { + return builder.create(); + } + }); + } + + @Test + public void persistedTest() { + final File db = Paths.get(System.getProperty("java.io.tmpdir"), "mark").toFile(); + try { + test(new Function, + ChronicleMap>() { + @Override + public ChronicleMap apply( + ChronicleMapBuilder builder) { + try { + return builder.createPersistedTo(db); + } catch (IOException e) { + throw new AssertionError(e); + } + } + }); + System.out.println(MemoryUnit.BYTES.toMegabytes(db.length()) + " MB"); + assertTrue("ChronicleMap of 25 million int-int entries should be lesser than 400MB", + db.length() < MemoryUnit.MEGABYTES.toBytes(400)); + } finally { + db.delete(); + } + } + + private static void test(Function, + ChronicleMap> createMap) { + long ms = System.currentTimeMillis(); + double entriesWithMargin = ENTRIES * 1.1; + long actualEntriesPerSegment = (1L << 17) / 3L; + try (ChronicleMap map = createMap.apply(ChronicleMapBuilder + .of(Integer.class, Integer.class) + .actualSegments((int) (entriesWithMargin / actualEntriesPerSegment)) + .actualEntriesPerSegment(actualEntriesPerSegment) + .putReturnsNull(true) + .removeReturnsNull(true))) { + + Random r = ThreadLocalRandom.current(); + for (int i = 0; i < ENTRIES; i++) { + map.put(r.nextInt(), r.nextInt()); + } + } + System.out.println(System.currentTimeMillis() - ms); + } +}