From 40a3080ac98cdef60a56738adfd87a3534d4d69a Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 28 Nov 2014 00:05:31 +0700 Subject: [PATCH] HCOLL-185 refactored VanillaChronicleMap and ReplicatedChronicleMap; split BytesMapEventListener from MapEventListener; many bug fixes and optimizations; Stateless client queries without serialization/deserialization not working yet --- .../internal/BytesBytesInterop.java | 45 + .../serialization/internal/MetaProvider.java | 2 +- .../map/AbstractChannelReplicator.java | 8 +- .../map/AbstractChronicleMapBuilder.java | 31 +- .../net/openhft/chronicle/map/Asserts.java | 28 + .../chronicle/map/BytesMapEventListener.java | 56 + .../chronicle/map/ChannelProvider.java | 12 +- .../chronicle/map/ChronicleMapBuilder.java | 27 +- .../chronicle/map/ChronicleMapBuilderI.java | 6 +- .../chronicle/map/GetValueInterops.java | 27 + .../map/InstanceOrBytesToInstance.java | 24 + .../chronicle/map/Int24Int24MultiMap.java | 12 + .../openhft/chronicle/map/IntIntMultiMap.java | 12 + .../chronicle/map/MapEventListener.java | 94 +- .../chronicle/map/MapEventListeners.java | 153 -- .../net/openhft/chronicle/map/MultiMap.java | 4 + .../net/openhft/chronicle/map/ReadValue.java | 29 + .../net/openhft/chronicle/map/Replica.java | 7 +- .../chronicle/map/ReplicatedChronicleMap.java | 1494 +++++------- .../map/ReplicationHubFindByName.java | 14 +- .../openhft/chronicle/map/SharedSegment.java | 13 +- .../chronicle/map/ShortShortMultiMap.java | 13 + .../openhft/chronicle/map/TcpReplicator.java | 213 +- .../openhft/chronicle/map/UdpReplicator.java | 4 +- .../chronicle/map/VanillaChronicleMap.java | 2025 ++++++++++++----- .../chronicle/map/CHMMetaDataTest.java | 28 +- ...tListenerTestWithTCPSocketReplication.java | 9 +- .../map/TimeBasedReplicationTest.java | 14 +- 28 files changed, 2527 insertions(+), 1877 deletions(-) create mode 100644 src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java create mode 100644 src/main/java/net/openhft/chronicle/map/Asserts.java create mode 100644 src/main/java/net/openhft/chronicle/map/BytesMapEventListener.java create mode 100644 src/main/java/net/openhft/chronicle/map/GetValueInterops.java create mode 100644 src/main/java/net/openhft/chronicle/map/InstanceOrBytesToInstance.java delete mode 100644 src/main/java/net/openhft/chronicle/map/MapEventListeners.java create mode 100644 src/main/java/net/openhft/chronicle/map/ReadValue.java diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java new file mode 100644 index 000000000..ae25852d8 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/BytesBytesInterop.java @@ -0,0 +1,45 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.hash.serialization.internal; + +import net.openhft.chronicle.hash.serialization.BytesInterop; +import net.openhft.chronicle.hash.serialization.Hasher; +import net.openhft.lang.io.Bytes; + +public enum BytesBytesInterop implements BytesInterop { + INSTANCE; + + @Override + public boolean startsWith(Bytes bytes, Bytes e) { + return bytes.startsWith(e); + } + + @Override + public long hash(Bytes e) { + return Hasher.hash(e); + } + + @Override + public long size(Bytes e) { + return e.remaining(); + } + + @Override + public void write(Bytes bytes, Bytes e) { + bytes.write(e); + } +} diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaProvider.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaProvider.java index 5a12019f9..fb99e3639 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaProvider.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaProvider.java @@ -22,7 +22,7 @@ import net.openhft.lang.threadlocal.ThreadLocalCopies; import java.io.Serializable; -public interface MetaProvider> extends Serializable { +public interface MetaProvider> extends Serializable { MW get(ThreadLocalCopies copies, MW originalMetaWriter, W writer, E e); diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java index 52d4a57ab..ae54163b2 100644 --- a/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChannelReplicator.java @@ -22,6 +22,7 @@ import net.openhft.chronicle.hash.replication.ThrottlingConfig; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; import net.openhft.lang.thread.NamedThreadFactory; +import net.openhft.lang.threadlocal.ThreadLocalCopies; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -69,6 +70,9 @@ abstract class AbstractChannelReplicator implements Closeable { volatile boolean isClosed = false; + ThreadLocalCopies copies; + VanillaChronicleMap.SegmentState segmentState; + AbstractChannelReplicator(String name, ThrottlingConfig throttlingConfig, int maxEntrySizeBytes) throws IOException { @@ -211,6 +215,8 @@ abstract class AbstractChannelReplicator implements Closeable { @Override public void run() { try { + copies = VanillaChronicleMap.SegmentState.getCopies(null); + segmentState = VanillaChronicleMap.SegmentState.get(copies); processEvent(); } catch (Exception e) { LOG.error("", e); @@ -227,7 +233,7 @@ abstract class AbstractChannelReplicator implements Closeable { executorService.shutdown(); closeables.closeQuietly(); executorService.shutdownNow(); - + segmentState.close(); try { // we HAVE to be sure we have terminated before calling close() on the diff --git a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java index c8081859c..c656baa11 100755 --- a/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/AbstractChronicleMapBuilder.java @@ -136,15 +136,15 @@ abstract class AbstractChronicleMapBuilder> eventListener = - MapEventListeners.nop(); + private MapEventListener eventListener = null; + private BytesMapEventListener bytesEventListener = null; private V defaultValue = null; private DefaultValueProvider defaultValueProvider = null; private PrepareValueBytes prepareValueBytes = null; private SingleChronicleHashReplication singleHashReplication = null; private InetSocketAddress[] pushToAddresses; - public boolean bootstapOnlyLocalEntries = false; + public boolean bootstrapOnlyLocalEntries = false; AbstractChronicleMapBuilder(Class keyClass, Class valueClass) { keyBuilder = new SerializationBuilder(keyClass, SerializationBuilder.Role.KEY); @@ -680,15 +680,24 @@ abstract class AbstractChronicleMapBuilder> eventListener) { + public B eventListener(MapEventListener eventListener) { this.eventListener = eventListener; return self(); } - MapEventListener> eventListener() { + MapEventListener eventListener() { return eventListener; } + public B bytesEventListener(BytesMapEventListener eventListener) { + this.bytesEventListener = eventListener; + return self(); + } + + BytesMapEventListener bytesEventListener() { + return bytesEventListener; + } + /** * Specifies the value to be put for each key queried in {@link ChronicleMap#acquireUsing acquireUsing()} * method, if the key is absent in the map, created by this builder. @@ -859,7 +868,7 @@ abstract class AbstractChronicleMapBuilder map = new ReplicatedChronicleMap, - V, Object, MetaBytesWriter>(this, identifier); + V, Object, MetaBytesInterop>(this, identifier); BytesStore bytesStore = new DirectStore(JDKZObjectSerializer.INSTANCE, map.sizeInBytes(), true); map.createMappedStoreAndSegments(bytesStore); @@ -895,7 +904,7 @@ abstract class AbstractChronicleMapBuilder>) constructor.newInstance((Object) statelessClients); + eventListener = (MapEventListener) constructor.newInstance((Object) statelessClients); } catch (ClassNotFoundException e) { LoggerFactory.getLogger(getClass().getName()).warn("Chronicle Enterprise not found in the class path"); } catch (Exception e) { @@ -924,10 +933,10 @@ abstract class AbstractChronicleMapBuilder, - V, Object, MetaBytesWriter>(this, identifier); + V, Object, MetaBytesInterop>(this, identifier); } else { return new VanillaChronicleMap, - V, Object, MetaBytesWriter>(this); + V, Object, MetaBytesInterop>(this); } } @@ -998,8 +1007,8 @@ abstract class AbstractChronicleMapBuilder bootstapOnlyLocalEntries(boolean bootstapOnlyLocalEntries) { - this.bootstapOnlyLocalEntries = bootstapOnlyLocalEntries; + public ChronicleMapBuilderI bootstrapOnlyLocalEntries(boolean bootstapOnlyLocalEntries) { + this.bootstrapOnlyLocalEntries = bootstapOnlyLocalEntries; return this; } diff --git a/src/main/java/net/openhft/chronicle/map/Asserts.java b/src/main/java/net/openhft/chronicle/map/Asserts.java new file mode 100644 index 000000000..fcb71f197 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/Asserts.java @@ -0,0 +1,28 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import org.jetbrains.annotations.NotNull; + +final class Asserts { + + public static void assertNotNull(@NotNull Object obj) { + assert obj != null; + } + + private Asserts() {} +} diff --git a/src/main/java/net/openhft/chronicle/map/BytesMapEventListener.java b/src/main/java/net/openhft/chronicle/map/BytesMapEventListener.java new file mode 100644 index 000000000..5843ae8bc --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/BytesMapEventListener.java @@ -0,0 +1,56 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.chronicle.hash.serialization.SizeMarshaller; +import net.openhft.lang.io.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; + +/** + *
    + *
  • Methods must not update {@code entry} state ({@linkplain Bytes#position(long) position}, + * {@linkplain Bytes#limit() limit}).
    • + *
  • Methods must not update entry bytes, except in metadata area
    • + *
  • From {@code metaDataPos} offset metadata area starts in the given {@code entry}, listener + * should be itself aware of it's length (see {@link ChronicleMapBuilder#metaDataBytes()}).
    • + *
  • From {@code keyPos} offset key area in the given {@code entry}, serialized key size + * (using {@linkplain ChronicleMapBuilder#keySizeMarshaller(SizeMarshaller)}), directly followed + * by the serialized key itself.
    • + *
  • From {@code valuePos} offset value area in the given {@code entry}, serialized value size + * (using {@linkplain ChronicleMapBuilder#keySizeMarshaller(SizeMarshaller)}), then (optionally) + * alignment (see {@link OffHeapUpdatableChronicleMapBuilder#entryAndValueAlignment(Alignment)} + * ), followed by the serialized value itself.
    • + *
+ */ +public abstract class BytesMapEventListener implements Serializable { + private static final long serialVersionUID = 0L; + + public void onGetFound(Bytes entry, long metaDataPos, long keyPos, long valuePos) { + // do nothing + } + + public void onPut(Bytes entry, long metaDataPos, long keyPos, long valuePos, boolean added) { + // do nothing + } + + public void onRemove(Bytes entry, long metaDataPos, long keyPos, long valuePos) { + // do nothing + } +} diff --git a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java index d07e76a4d..e57ab02b0 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java @@ -26,6 +26,7 @@ import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.ByteBufferBytes; import net.openhft.lang.io.Bytes; +import net.openhft.lang.threadlocal.ThreadLocalCopies; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,7 +109,9 @@ final class ChannelProvider implements Closeable { } @Override - public void readExternalEntry(@NotNull Bytes source) { + public void readExternalEntry( + @NotNull ThreadLocalCopies copies, + @NotNull VanillaChronicleMap.SegmentState segmentState, @NotNull Bytes source) { channelDataLock.readLock().lock(); try { final int chronicleId = (int) source.readStopBit(); @@ -116,7 +119,8 @@ final class ChannelProvider implements Closeable { // this channel is has not currently been created so it updates will be ignored if (channelEntryExternalizables[chronicleId] != null) - channelEntryExternalizables[chronicleId].readExternalEntry(source); + channelEntryExternalizables[chronicleId] + .readExternalEntry(copies, segmentState, source); } else LOG.info("skipped entry with chronicleId=" + chronicleId + ", "); } finally { @@ -484,7 +488,9 @@ final class ChannelProvider implements Closeable { } @Override - public void readExternalEntry(@NotNull Bytes source) { + public void readExternalEntry( + @NotNull ThreadLocalCopies copies, + @NotNull VanillaChronicleMap.SegmentState segmentState, @NotNull Bytes source) { messageHandler.onMessage(source); } }; diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index d956e13db..ce4326803 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -492,27 +492,34 @@ public final class ChronicleMapBuilder implements ChronicleMapBuilderI eventListener(MapEventListener> eventListener) { + public ChronicleMapBuilder eventListener(MapEventListener eventListener) { delegate.eventListener(eventListener); return this; } + @Override + public ChronicleMapBuilderI bytesEventListener(BytesMapEventListener eventListener) { + delegate.bytesEventListener(eventListener); + return this; + } + + @Override public ChronicleMap createPersistedTo(File file) throws IOException { return delegate.createPersistedTo(file); } /** - * @param bootstapOnlyLocalEntries if set to true - when a new node joins a TCP replication grid, the new - * node will be populated with data, only for the nodes that created that - * data. Otherwise, all the nodes will publish all the data they have ( - * potentially swamping the new node with duplicates ) however this does - * guarantee that all the data is replicated over to the new node, and is - * useful especially in the case that the originating node is not - * currently running. + * @param bootstrapOnlyLocalEntries if set to true - when a new node joins a TCP replication grid, the new + * node will be populated with data, only for the nodes that created that + * data. Otherwise, all the nodes will publish all the data they have ( + * potentially swamping the new node with duplicates ) however this does + * guarantee that all the data is replicated over to the new node, and is + * useful especially in the case that the originating node is not + * currently running. */ - public ChronicleMapBuilder bootstapOnlyLocalEntries(boolean bootstapOnlyLocalEntries) { - delegate.bootstapOnlyLocalEntries(bootstapOnlyLocalEntries); + public ChronicleMapBuilder bootstrapOnlyLocalEntries(boolean bootstrapOnlyLocalEntries) { + delegate.bootstrapOnlyLocalEntries(bootstrapOnlyLocalEntries); return this; } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java index bfb940eb9..617bb3144 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderI.java @@ -290,7 +290,9 @@ public interface ChronicleMapBuilderI extends Serializable { ChronicleMapBuilderI clone(); - ChronicleMapBuilderI eventListener(MapEventListener> eventListener); + ChronicleMapBuilderI eventListener(MapEventListener eventListener); + + ChronicleMapBuilderI bytesEventListener(BytesMapEventListener eventListener); ChronicleMapBuilderI putReturnsNull(boolean b); @@ -307,7 +309,7 @@ public interface ChronicleMapBuilderI extends Serializable { * useful especially in the case that the originating node is not * currently running. */ - ChronicleMapBuilderI bootstapOnlyLocalEntries(boolean bootstapOnlyLocalEntries); + ChronicleMapBuilderI bootstrapOnlyLocalEntries(boolean bootstapOnlyLocalEntries); String name(); diff --git a/src/main/java/net/openhft/chronicle/map/GetValueInterops.java b/src/main/java/net/openhft/chronicle/map/GetValueInterops.java new file mode 100644 index 000000000..d0801b4d6 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/GetValueInterops.java @@ -0,0 +1,27 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.chronicle.hash.serialization.internal.MetaBytesInterop; +import net.openhft.lang.threadlocal.ThreadLocalCopies; +import org.jetbrains.annotations.NotNull; + +interface GetValueInterops> { + MVBI getMetaValueInterop(@NotNull ThreadLocalCopies copies, VBI valueInterop, VB value); + + VBI getValueInterop(@NotNull ThreadLocalCopies copies); +} diff --git a/src/main/java/net/openhft/chronicle/map/InstanceOrBytesToInstance.java b/src/main/java/net/openhft/chronicle/map/InstanceOrBytesToInstance.java new file mode 100644 index 000000000..5f192033a --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/InstanceOrBytesToInstance.java @@ -0,0 +1,24 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.threadlocal.ThreadLocalCopies; +import org.jetbrains.annotations.NotNull; + +interface InstanceOrBytesToInstance { + T toInstance(@NotNull ThreadLocalCopies copies, TB instanceOrBytes, long size); +} diff --git a/src/main/java/net/openhft/chronicle/map/Int24Int24MultiMap.java b/src/main/java/net/openhft/chronicle/map/Int24Int24MultiMap.java index f0255a4e0..76e9e1e82 100644 --- a/src/main/java/net/openhft/chronicle/map/Int24Int24MultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/Int24Int24MultiMap.java @@ -120,6 +120,12 @@ class Int24Int24MultiMap implements MultiMap { return (pos -= ENTRY_SIZE) >= 0L ? pos : pos + capacityInBytes; } + @Override + public void putPosition(long value) { + checkValueForPut(value); + positions.set(value); + } + @Override public void put(long key, long value) { key = maskUnsetKey(key); @@ -134,6 +140,12 @@ class Int24Int24MultiMap implements MultiMap { } } + @Override + public void removePosition(long value) { + checkValueForRemove(value); + positions.clear(value); + } + @Override public void remove(long key, long value) { key = maskUnsetKey(key); diff --git a/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java b/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java index e441a14f8..29c1ceb55 100644 --- a/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/IntIntMultiMap.java @@ -121,6 +121,12 @@ class IntIntMultiMap implements MultiMap { return (pos - ENTRY_SIZE) & capacityMask2; } + @Override + public void putPosition(long value) { + checkValueForPut(value); + positions.set(value); + } + @Override public void put(long key, long value) { key = maskUnsetKey(key); @@ -135,6 +141,12 @@ class IntIntMultiMap implements MultiMap { } } + @Override + public void removePosition(long value) { + checkValueForRemove(value); + positions.clear(value); + } + @Override public void remove(long key, long value) { key = maskUnsetKey(key); diff --git a/src/main/java/net/openhft/chronicle/map/MapEventListener.java b/src/main/java/net/openhft/chronicle/map/MapEventListener.java index 9c0cc1fae..72b8e2679 100644 --- a/src/main/java/net/openhft/chronicle/map/MapEventListener.java +++ b/src/main/java/net/openhft/chronicle/map/MapEventListener.java @@ -19,8 +19,9 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.hash.ChronicleHashBuilder; -import net.openhft.lang.io.Bytes; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.Serializable; @@ -34,8 +35,7 @@ import java.io.Serializable; *

To configure {@code MapEventListener} for {@code ChronicleMap}, use {@link * AbstractChronicleMapBuilder#eventListener(MapEventListener)} method. * - *

{@link MapEventListeners} uninstantiable class contains several logging implementations and the default - * {@linkplain MapEventListeners#nop() no-op implementation}. + *

See {@link #logging(String)} implementation. * *

All these calls are synchronous while a {@code ChronicleMap} lock is held so make them as quick as * possible. @@ -45,33 +45,61 @@ import java.io.Serializable; * * @param key type of the maps, trackable by this event listener * @param value type of the maps, trackable by this event listener - * @param {@code ChronicleMap} subtype, trackable by this event listener * @see AbstractChronicleMapBuilder#eventListener(MapEventListener) - * @see MapEventListeners */ -public abstract class MapEventListener> - implements Serializable { +public abstract class MapEventListener implements Serializable { private static final long serialVersionUID = 0L; + /** + * Returns the map event listener, which logs strings like + * "{@linkplain ChronicleMap#file() map file} opType key value", where opType is either "get", + * "put" or "remove", to the logger provided by SLF4J. + * + * @param the map key type + * @param the map value type + * @return the logging event listener + */ + public static MapEventListener logging(String prefix) { + return new LoggingMapEventListener(prefix); + } + + private static class LoggingMapEventListener extends MapEventListener { + private static final long serialVersionUID = 0L; + public final static Logger LOGGER = LoggerFactory.getLogger(LoggingMapEventListener.class); + + private final String prefix; + + private LoggingMapEventListener(String prefix) { + this.prefix = prefix; + } + + @Override + public void onGetFound(Object key, Object value) { + LOGGER.info("get {} => {}", prefix, key, value); + } + + @Override + public void onPut(Object key, Object value, Object replacedValue) { + LOGGER.info("{} put {} => {}", prefix, key, value); + } + + @Override + public void onRemove(Object key, Object value) { + LOGGER.info("{} remove {} was {}", prefix, key, value); + } + } + /** * This method is called if the key is found in the map during {@link ChronicleMap#get get}, {@link * ChronicleMap#getUsing getUsing} or {@link ChronicleMap#acquireUsing acquireUsing} method call. * - * @param map the accessed map - * @param entry bytes of the entry - * @param metaDataBytes length of {@link ChronicleHashBuilder#metaDataBytes(int) meta data} for this map * @param key the key looked up - * @param value the value found for the key + * @param foundValue the value found for the key */ - public void onGetFound(M map, Bytes entry, int metaDataBytes, K key, V value) { + public void onGetFound(K key, V foundValue) { // do nothing } - void onPut(M map, Bytes entry, int metaDataBytes, boolean added, K key, V replacedValue, V value, - long pos, SharedSegment segment) { - onPut(map, entry, metaDataBytes, added, key, value, replacedValue); - } - /** * This method is called whenever a new value is put for the key in the map during calls of such methods * as {@link ChronicleMap#put put}, {@link ChronicleMap#putIfAbsent putIfAbsent}, {@link @@ -81,41 +109,23 @@ public abstract class MapEventListener> * *

This method is called when put is already happened. * - * @param map the accessed map - * @param entry bytes of the entry (with the value already written to) - * @param metaDataBytes length of {@link net.openhft.chronicle.hash.ChronicleHashBuilder#metaDataBytes(int) - * meta data} for this map - * @param added {@code true} is the key was absent in the map before current {@code ChronicleMap} - * method call, led to putting a value and inherently calling this listener method * @param key the key the given value is put for - * @param value the value which is now associated with the given key - * @param replacedValue the value which was replaced by {@code value}, is NULL if {@code added} is true + * @param newValue the value which is now associated with the given key + * @param replacedValue the value which was replaced by {@code newValue}, {@code null} + * if the key was absent in the map before current {@code ChronicleMap} */ - public void onPut(M map, Bytes entry, int metaDataBytes, boolean added, K key, V value, - @Nullable V replacedValue) { + public void onPut(K key, V newValue, @Nullable V replacedValue) { // do nothing } - void onRemove(M map, Bytes entry, int metaDataBytes, K key, V value, - long pos, SharedSegment segment) { - onRemove(map, entry, metaDataBytes, key, value); - } - /** - * This is called when an entry is removed. Misses, i. e. when {@code map.remove(key)} is called, but key - * is already absent in the map, are not notified. + * This is called when an entry is removed. Misses, i. e. when {@code map.remove(key)} + * is called, but key is already absent in the map, are not notified. * - * @param map accessed - * @param entry bytes of the entry just removed, but any data is not yet erased - * @param metaDataBytes length of {@link ChronicleHashBuilder#metaDataBytes(int) meta data} for this map * @param key the key removed from the map * @param value the value which was associated with the given key */ - public void onRemove(M map, Bytes entry, int metaDataBytes, K key, V value) { - // do nothing - } - - void onRelocation(long pos, SharedSegment segment) { + public void onRemove(K key, V value) { // do nothing } } diff --git a/src/main/java/net/openhft/chronicle/map/MapEventListeners.java b/src/main/java/net/openhft/chronicle/map/MapEventListeners.java deleted file mode 100644 index 754c54f9c..000000000 --- a/src/main/java/net/openhft/chronicle/map/MapEventListeners.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2014 Higher Frequency Trading - * - * http://www.higherfrequencytrading.com - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.openhft.chronicle.map; - - -import net.openhft.lang.io.Bytes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Methods returning useful implementations of {@link MapEventListener}. - */ -public final class MapEventListeners { - - /** - * Factories a more flexible than public static instances. - * We can add some configuration-on-the-first-call in the future. - */ - - static final MapEventListener NOP = new MapEventListener() { - private static final long serialVersionUID = 0L; - }; - private static final MapEventListener BYTES_LOGGING = new MapEventListener() { - private static final long serialVersionUID = 0L; - public final Logger LOGGER = LoggerFactory.getLogger(getClass()); - - @Override - public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, - Object key, Object value) { - logOperation(map, entry, metaDataBytes, " get "); - } - - private void logOperation(ChronicleMap map, Bytes entry, int metaDataBytes, String oper) { - StringBuilder sb = new StringBuilder(); - sb.append(map.file()).append(oper); - if (metaDataBytes > 0) { - sb.append("Meta: "); - entry.toString(sb, 0L, 0L, metaDataBytes); - sb.append(" | "); - } - Bytes slice = entry.slice(metaDataBytes, entry.limit() - metaDataBytes); - VanillaChronicleMap vanillaMap = (VanillaChronicleMap) map; - long keySize = vanillaMap.keySizeMarshaller.readSize(slice); - slice.toString(sb, slice.position(), 0L, slice.position() + keySize); - slice.position(slice.position() + keySize); - long valueSize = vanillaMap.valueSizeMarshaller.readSize(slice); - vanillaMap.alignment.alignPositionAddr(slice); - sb.append(" = "); - slice.toString(sb, slice.position(), 0L, slice.position() + valueSize); - LOGGER.info(sb.toString()); - } - - @Override - public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, boolean added, - Object key, Object replacedValue, Object value, long pos, SharedSegment segment) { - logOperation(map, entry, metaDataBytes, added ? " +put " : " put "); - } - - @Override - public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, - Object key, Object value, long pos, SharedSegment segment) { - logOperation(map, entry, metaDataBytes, " remove "); - } - }; - private static final MapEventListener KEY_VALUE_LOGGING = new MapEventListener() { - private static final long serialVersionUID = 0L; - public final Logger LOGGER = LoggerFactory.getLogger(getClass()); - - @Override - public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, - Object key, Object value) { - LOGGER.info("{} get {} => {}", map.file(), key, value); - } - - @Override - public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, boolean added, - Object key, Object replacedValue, Object value, long pos, SharedSegment segment) { - LOGGER.info("{} put {} => {}", map.file(), key, value); - } - - @Override - public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, - Object key, Object value, long pos, SharedSegment segment) { - LOGGER.info("{} remove {} was {}", map.file(), key, value); - } - }; - - private MapEventListeners() { - } - - /** - * Returns the no-op event listener, i. e. all it's method implementations have empty bodies. - * - * @param the map key type - * @param the map value type - * @param the {@code ChronicleMap} subclass, or {@code ChronicleMap} type itself. - * Typically this type should be just {@code ChronicleMap}. - * @return the no-op event listener - */ - @SuppressWarnings("unchecked") - public static > MapEventListener nop() { - return NOP; - } - - /** - * Returns the map event listener, which logs strings like - * "Meta: [meta bytes] | [key bytes] = [value bytes]", where bytes are meta, key and value - * areas from {@link Bytes} passed to {@code MapEventListener} methods, printed in ASCII - * coding. - * - * @param the map key type - * @param the map value type - * @param the {@code ChronicleMap} subclass, or {@code ChronicleMap} type itself. - * Typically this type should be just {@code ChronicleMap}. - * @return the bytes logging event listener - */ - @SuppressWarnings("unchecked") - public static > MapEventListener bytesLogging() { - return BYTES_LOGGING; - } - - /** - * Returns the map event listener, which logs strings like - * "{@linkplain ChronicleMap#file() map file} opType key value", where opType is either "get", - * "put" or "remove", to the logger provided by SLF4J. - * - * @param the map key type - * @param the map value type - * @param the {@code ChronicleMap} subclass, or {@code ChronicleMap} type itself. - * Typically this type should be just {@code ChronicleMap}. - * @return the logging event listener - */ - @SuppressWarnings("unchecked") - public static > MapEventListener keyValueLogging() { - return KEY_VALUE_LOGGING; - } -} diff --git a/src/main/java/net/openhft/chronicle/map/MultiMap.java b/src/main/java/net/openhft/chronicle/map/MultiMap.java index 60d013d1d..4275ac1c8 100644 --- a/src/main/java/net/openhft/chronicle/map/MultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/MultiMap.java @@ -36,6 +36,8 @@ interface MultiMap { */ void put(long key, long value); + void putPosition(long value); + /** * Remove a key/value pair. * @@ -45,6 +47,8 @@ interface MultiMap { */ void remove(long key, long value); + void removePosition(long value); + void replace(long key, long oldValue, long newValue); /** diff --git a/src/main/java/net/openhft/chronicle/map/ReadValue.java b/src/main/java/net/openhft/chronicle/map/ReadValue.java new file mode 100644 index 000000000..8d925ce37 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/ReadValue.java @@ -0,0 +1,29 @@ +/* + * Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.chronicle.map; + +import net.openhft.lang.io.Bytes; +import net.openhft.lang.threadlocal.ThreadLocalCopies; +import org.jetbrains.annotations.NotNull; + +/** + * @param result type, map value type or Bytes + */ +interface ReadValue { + RV readValue(@NotNull ThreadLocalCopies copies, Bytes entry, RV usingValue, long valueSize); + RV readNull(); +} diff --git a/src/main/java/net/openhft/chronicle/map/Replica.java b/src/main/java/net/openhft/chronicle/map/Replica.java index 0ca766f4b..172077921 100644 --- a/src/main/java/net/openhft/chronicle/map/Replica.java +++ b/src/main/java/net/openhft/chronicle/map/Replica.java @@ -19,6 +19,7 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.Bytes; +import net.openhft.lang.threadlocal.ThreadLocalCopies; import org.jetbrains.annotations.NotNull; import java.io.Closeable; @@ -140,9 +141,13 @@ interface Replica extends Closeable { * is typically called when we receive a remote replication event, this event could originate from * either a remote {@code put(K key, V value)} or {@code remove(Object key)} * + * @param copies + * @param segmentState * @param source bytes to read an entry from */ - void readExternalEntry(@NotNull Bytes source); + void readExternalEntry( + @NotNull ThreadLocalCopies copies, + @NotNull VanillaChronicleMap.SegmentState segmentState, @NotNull Bytes source); } diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 4dfb2a062..e3e3f038f 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -20,7 +20,10 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.hash.replication.TimeProvider; +import net.openhft.chronicle.hash.serialization.BytesInterop; import net.openhft.chronicle.hash.serialization.BytesReader; +import net.openhft.chronicle.hash.serialization.internal.BytesBytesInterop; +import net.openhft.chronicle.hash.serialization.internal.DelegatingMetaBytesInterop; import net.openhft.chronicle.hash.serialization.internal.MetaBytesInterop; import net.openhft.chronicle.hash.serialization.internal.MetaBytesWriter; import net.openhft.lang.Maths; @@ -28,9 +31,9 @@ import net.openhft.lang.collection.ATSDirectBitSet; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.MultiStoreBytes; import net.openhft.lang.io.NativeBytes; -import net.openhft.lang.model.Byteable; import net.openhft.lang.threadlocal.ThreadLocalCopies; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,11 +82,10 @@ import static net.openhft.lang.collection.DirectBitSet.NOT_FOUND; * @param the entries key type * @param the entries value type */ -final class ReplicatedChronicleMap, - V, VW, MVW extends MetaBytesWriter> - extends VanillaChronicleMap - implements ChronicleMap, - Replica, Replica.EntryExternalizable, Replica.EntryResolver, Closeable { +final class ReplicatedChronicleMap, + V, VI, MVI extends MetaBytesInterop> + extends VanillaChronicleMap + implements Replica, Replica.EntryExternalizable, Replica.EntryResolver { // for file, jdbc and UDP replication public static final int RESERVED_MOD_ITER = 8; public static final int ADDITIONAL_ENTRY_BYTES = 10; @@ -95,9 +97,10 @@ final class ReplicatedChronicleMap, transient Set closeables; private transient Bytes identifierUpdatedBytes; - private transient ModificationDelegator modificationDelegator; + private transient ATSDirectBitSet modIterSet; + private transient AtomicReferenceArray modificationIterators; private transient long startOfModificationIterators; - private boolean bootstapOnlyLocalEntries; + private boolean bootstrapOnlyLocalEntries; public ReplicatedChronicleMap(@NotNull AbstractChronicleMapBuilder builder, byte identifier) @@ -106,7 +109,7 @@ final class ReplicatedChronicleMap, this.timeProvider = builder.timeProvider(); this.localIdentifier = identifier; - this.bootstapOnlyLocalEntries = builder.bootstapOnlyLocalEntries; + this.bootstrapOnlyLocalEntries = builder.bootstrapOnlyLocalEntries; if (localIdentifier == -1) { throw new IllegalStateException("localIdentifier should not be -1"); @@ -119,7 +122,8 @@ final class ReplicatedChronicleMap, } @Override - VanillaChronicleMap.Segment createSegment(NativeBytes segmentHeader, NativeBytes bytes, int index) { + VanillaChronicleMap.Segment createSegment( + NativeBytes segmentHeader, NativeBytes bytes, int index) { return new Segment(segmentHeader, bytes, index); } @@ -130,6 +134,8 @@ final class ReplicatedChronicleMap, @Override void initTransients() { super.initTransients(); + modificationIterators = + new AtomicReferenceArray(127 + RESERVED_MOD_ITER); closeables = new CopyOnWriteArraySet(); } @@ -144,11 +150,6 @@ final class ReplicatedChronicleMap, return Maths.nextPower2(entriesPerSegment, 128L * 8L); } - @Override - int multiMapsPerSegment() { - return 2; - } - @Override long getHeaderSize() { return super.getHeaderSize() + LAST_UPDATED_HEADER_SIZE + @@ -177,7 +178,6 @@ final class ReplicatedChronicleMap, @Override void onHeaderCreated() { - long offset = super.getHeaderSize(); identifierUpdatedBytes = ms.bytes(offset, LAST_UPDATED_HEADER_SIZE).zeroOut(); @@ -185,44 +185,36 @@ final class ReplicatedChronicleMap, Bytes modDelBytes = ms.bytes(offset, assignedModIterBitSetSizeInBytes()).zeroOut(); offset += assignedModIterBitSetSizeInBytes(); - startOfModificationIterators = offset; + modIterSet = new ATSDirectBitSet(modDelBytes); + } - modificationDelegator = new ModificationDelegator(attachedEventListener(), modDelBytes); + /** + * @param segmentNum a unique index of the segment + * @return the segment associated with the {@code segmentNum} + */ + private Segment segment(int segmentNum) { + return (Segment) segments[segmentNum]; + } + + private long currentTime() { + return timeProvider.currentTimeMillis(); } @Override - MapEventListener> eventListener() { - return modificationDelegator; + void putBytes(ThreadLocalCopies copies, SegmentState segmentState, Bytes key, long keySize, + GetRemoteBytesValueInterops getRemoteBytesValueInterops, MultiStoreBytes value, + boolean replaceIfPresent, ReadValue readValue) { + put(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, key, keySize, keyBytesToInstance, + getRemoteBytesValueInterops, value, valueBytesToInstance, + replaceIfPresent, readValue, false, localIdentifier, currentTime());; } /** - * when a remote node receives and event it can't use the {@link ReplicatedChronicleMap#eventListener() } - * as this will trigger the modification iterators to send out another replication event. So instead with - * call this method iterators - * - * @return the event listener added by the user - */ - MapEventListener> attachedEventListener() { - return eventListener == null ? MapEventListeners.NOP : eventListener; - } - - - /** - * @return true if the user has added an event listerner - */ - boolean hasAttachedEventListeners() { - return eventListener != null; - } - - @Override - public V put(K key, V value) { - return put0(key, value, true, localIdentifier, timeProvider.currentTimeMillis()); - } - - /** - * Used in conjunction with map replication, all put events that originate from a remote node will be - * processed using this method. + * Used in conjunction with map replication, all put events that originate from a remote node + * will be processed using this method. * * @param key key with which the specified value is to be associated * @param value value to be associated with the specified key @@ -237,8 +229,8 @@ final class ReplicatedChronicleMap, } @Override - public V putIfAbsent(@net.openhft.lang.model.constraints.NotNull K key, V value) { - return put0(key, value, false, localIdentifier, timeProvider.currentTimeMillis()); + V put0(@NotNull K key, V value, boolean replaceIfPresent) { + return put0(key, value, replaceIfPresent, localIdentifier, currentTime()); } /** @@ -254,87 +246,56 @@ final class ReplicatedChronicleMap, checkKey(key); checkValue(value); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); + KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); copies = metaKeyInteropProvider.getCopies(copies); - MKI metaKeyWriter = - metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); - long hash = metaKeyWriter.hash(keyWriter, key); - int segmentNum = getSegment(hash); - long segmentHash = segmentHash(hash); - return segment(segmentNum).put(copies, metaKeyWriter, keyWriter, key, value, segmentHash, - replaceIfPresent, identifier, timeStamp); - } - - /** - * @param segmentNum a unique index of the segment - * @return the segment associated with the {@code segmentNum} - */ - private Segment segment(int segmentNum) { - return (Segment) segments[segmentNum]; + MKI metaKeyInterop = + metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + long keySize = metaKeyInterop.size(keyInterop, key); + return put(copies, null, metaKeyInterop, keyInterop, key, keySize, keyIdentity(), + this, value, valueIdentity(), replaceIfPresent, this, putReturnsNull, + identifier, timeStamp); } @Override - T lookupUsing(K key, V usingValue, LockType lockType, boolean mustReuseValue) { - checkKey(key); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); - copies = metaKeyInteropProvider.getCopies(copies); - MKI metaKeyWriter = - metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); - long hash = metaKeyWriter.hash(keyWriter, key); - int segmentNum = getSegment(hash); - long segmentHash = segmentHash(hash); - - boolean create = lockType == LockType.WRITE_LOCK; - Segment segment = segment(segmentNum); - - MutableLockedEntry lock = (lockType == LockType.WRITE_LOCK) - ? segment.writeLock() - : segment.readLock(); - - boolean dirtyReplicationIfKeyPresent = lock instanceof VanillaChronicleMap.Segment.NativeWriteLocked; - - V v = segment.acquireWithoutLock(copies, metaKeyWriter, keyWriter, key, - usingValue, - segmentHash, create, create ? timeProvider.currentTimeMillis() : 0L, - dirtyReplicationIfKeyPresent); - - checkReallyUsingValue(usingValue, mustReuseValue, v); - - lock.copies = copies; - lock.metaKeyInterop = metaKeyWriter; - lock.keyInterop = keyWriter; - lock.segmentHash = segmentHash; - - if (!isNativeValueClass && lockType == LockType.WRITE_LOCK && v == null) - v = usingValue; - - lock.value(v); - lock.key(key); - - return (T) lock; + , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV put(ThreadLocalCopies copies, SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + boolean replaceIfPresent, ReadValue readValue, boolean resultUnused) { + return put(copies, segmentState, metaKeyInterop, keyInterop, key, keySize, toKey, + getValueInterops, value, toValue, replaceIfPresent, readValue, resultUnused, + localIdentifier, currentTime()); } - @Override - public V remove(final Object key) { - return removeIfValueIs(key, null, localIdentifier, timeProvider.currentTimeMillis()); + public void clear() { + // we have to make sure that every calls notifies on remove, + // so that the replicators can pick it up + for (K k : keySet()) { + ReplicatedChronicleMap.this.remove(k); + } } - /** - * Used in conjunction with map replication, all remove events that originate from a remote node will be - * processed using this method. - * - * @param key key with which the specified value is associated - * @param value value expected to be associated with the specified key - * @param identifier a unique identifier for a replicating node - * @param timeStamp timestamp in milliseconds, when the remove event originally occurred - * @return {@code true} if the entry was removed - * @see #remove(Object, Object) - */ - V remove(K key, V value, byte identifier, long timeStamp) { - assert identifier > 0; - return removeIfValueIs(key, value, identifier, timeStamp); + private , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV put(ThreadLocalCopies copies, SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + boolean replaceIfPresent, ReadValue readValue, boolean resultUnused, + byte identifier, long timeStamp) { + long hash = metaKeyInterop.hash(keyInterop, key); + int segmentNum = getSegment(hash); + long segmentHash = segmentHash(hash); + return segment(segmentNum).put(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + getValueInterops, value, toValue, + segmentHash, replaceIfPresent, readValue, resultUnused, + identifier, timeStamp); } void addCloseable(Closeable closeable) { @@ -350,9 +311,7 @@ final class ReplicatedChronicleMap, LOG.error("", e); } } - super.close(); - } @Override @@ -361,52 +320,65 @@ final class ReplicatedChronicleMap, } @Override - public Replica.ModificationIterator acquireModificationIterator - (byte remoteIdentifier, - @NotNull final ModificationNotifier modificationNotifier) { + public Replica.ModificationIterator acquireModificationIterator( + byte remoteIdentifier, @NotNull final ModificationNotifier modificationNotifier) { + ModificationIterator modificationIterator = modificationIterators.get(remoteIdentifier); + if (modificationIterator != null) + return modificationIterator; - return modificationDelegator - .acquireModificationIterator(remoteIdentifier, modificationNotifier); + synchronized (modificationIterators) { + modificationIterator = modificationIterators.get(remoteIdentifier); + + if (modificationIterator != null) + return modificationIterator; + + final Bytes bytes = ms.bytes(startOfModificationIterators + + (modIterBitSetSizeInBytes() * remoteIdentifier), + modIterBitSetSizeInBytes()); + + final ModificationIterator newModificationIterator = new ModificationIterator( + bytes, modificationNotifier); + + modificationIterators.set(remoteIdentifier, newModificationIterator); + modIterSet.set(remoteIdentifier); + return newModificationIterator; + } } @Override - public boolean remove(@NotNull final Object key, final Object value) { - if (value == null) - return false; // CHM compatibility; I would throw NPE - return removeIfValueIs(key, (V) value, - localIdentifier, timeProvider.currentTimeMillis()) != null; - } - - private V removeIfValueIs(final Object k, final V expectedValue, - final byte identifier, final long timestamp) { - checkKey(k); - K key = (K) k; - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); - copies = metaKeyInteropProvider.getCopies(copies); - MKI metaKeyWriter = - metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); - long hash = metaKeyWriter.hash(keyWriter, key); - int segmentNum = getSegment(hash); - long segmentHash = segmentHash(hash); - return segment(segmentNum).remove(copies, metaKeyWriter, keyWriter, key, expectedValue, - segmentHash, timestamp, identifier); + void onPut(VanillaChronicleMap.Segment segment, long pos) { + for (long next = modIterSet.nextSetBit(0L); next > 0L; + next = modIterSet.nextSetBit(next + 1L)) { + try { + modificationIterators.get((int) next).onPut(pos, segment); + } catch (Exception e) { + LOG.error("", e); + } + } } @Override - V replaceIfValueIs(@NotNull final K key, final V existingValue, final V newValue) { - checkKey(key); - checkValue(newValue); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); - copies = metaKeyInteropProvider.getCopies(copies); - MKI metaKeyWriter = - metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); - long hash = metaKeyWriter.hash(keyWriter, key); - int segmentNum = getSegment(hash); - long segmentHash = segmentHash(hash); - return segment(segmentNum).replace(copies, metaKeyWriter, keyWriter, key, existingValue, - newValue, segmentHash, timeProvider.currentTimeMillis()); + void onRemove(VanillaChronicleMap.Segment segment, long pos) { + for (long next = modIterSet.nextSetBit(0L); next > 0L; + next = modIterSet.nextSetBit(next + 1L)) { + try { + modificationIterators.get((int) next).onRemove(pos, segment); + } catch (Exception e) { + LOG.error("", e); + } + } + } + + @Override + void onRelocation(VanillaChronicleMap.Segment segment, long pos) { + for (long next = modIterSet.nextSetBit(0L); next > 0L; + next = modIterSet.nextSetBit(next + 1L)) { + try { + modificationIterators.get((int) next).onRelocation(pos, segment); + } catch (Exception e) { + LOG.error("", e); + } + } } /** @@ -492,7 +464,9 @@ final class ReplicatedChronicleMap, * especially when being used in a multi threaded context. */ @Override - public void readExternalEntry(@NotNull Bytes source) { + public void readExternalEntry( + @NotNull ThreadLocalCopies copies, @NotNull SegmentState segmentState, + @NotNull Bytes source) { final long keySize = keySizeMarshaller.readSize(source); final long valueSize = valueSizeMarshaller.readSize(source); @@ -535,7 +509,8 @@ final class ReplicatedChronicleMap, ); } - segment(segmentNum).remoteRemove(source, segmentHash, timeStamp, remoteIdentifier); + segment(segmentNum).remoteRemove(copies, segmentState, + source, segmentHash, timeStamp, remoteIdentifier); setLastModificationTime(remoteIdentifier, timeStamp); return; } @@ -549,8 +524,8 @@ final class ReplicatedChronicleMap, final long valuePosition = keyLimit; final long valueLimit = valuePosition + valueSize; - segment(segmentNum).remotePut(source, segmentHash, remoteIdentifier, timeStamp, - valuePosition, valueLimit); + segment(segmentNum).remotePut(copies, segmentState, source, + keySize, valueSize, segmentHash, remoteIdentifier, timeStamp); setLastModificationTime(remoteIdentifier, timeStamp); if (debugEnabled) { @@ -562,9 +537,8 @@ final class ReplicatedChronicleMap, } - @NotNull @Override - public Set> entrySet() { + Set> newEntrySet() { return new EntrySet(); } @@ -622,27 +596,14 @@ final class ReplicatedChronicleMap, } } - class Segment extends VanillaChronicleMap.Segment { - - private volatile MultiMap hashLookupLiveAndDeleted; - private volatile MultiMap hashLookupLiveOnly; + class Segment extends VanillaChronicleMap.Segment { Segment(NativeBytes segmentHeader, NativeBytes bytes, int index) { super(segmentHeader, bytes, index); } @Override - void createHashLookups(long start) { - hashLookupLiveAndDeleted = createMultiMap(start); - start += CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES); - hashLookupLiveOnly = createMultiMap(start); - } - - public MultiMap getHashLookup() { - return hashLookupLiveOnly; - } - - private long entrySize(long keySize, long valueSize) { + long entrySize(long keySize, long valueSize) { long result = alignment.alignAddr(metaDataBytes + keySizeMarshaller.sizeEncodingSize(keySize) + keySize + ADDITIONAL_ENTRY_BYTES + valueSizeMarshaller.sizeEncodingSize(valueSize)) + valueSize; @@ -657,382 +618,319 @@ final class ReplicatedChronicleMap, return result; } - V acquireWithoutLock(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V usingValue, - long hash2, boolean create, long timestamp, final boolean - dirtyReplicationIfKeyPresent) { - long keySize = metaKeyInterop.size(keyInterop, key); - MultiStoreBytes entry = acquireTmpBytes(); - long offset = searchKey(keyInterop, metaKeyInterop, key, keySize, hash2, entry, - hashLookupLiveOnly); - - if (offset >= 0L) { - - V v = onKeyPresentOnAcquire(copies, key, usingValue, offset, entry); - - // require for acquireUsingLocked() when working with native - if (dirtyReplicationIfKeyPresent) - modificationDelegator.dirty(ReplicatedChronicleMap.this, posFromOffset(offset), this); - - return v; - - } else { - if (!create) - return null; - if (defaultValueProvider != null) { - V defaultValue = defaultValueProvider.get(key); - copies = valueWriterProvider.getCopies(copies); - VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = metaValueWriterProvider.getCopies(copies); - MetaBytesWriter metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, defaultValue); - - offset = putEntry(metaKeyInterop, keyInterop, key, keySize, hash2, - localIdentifier, timestamp, hashLookupLiveOnly, - metaValueWriter, valueWriter, defaultValue); - } else { - offset = putEntry(metaKeyInterop, keyInterop, key, keySize, hash2, - localIdentifier, timestamp, hashLookupLiveOnly, - prepareValueBytesAsWriter, null, key); - } - return onKeyAbsentOnAcquire(copies, key, keySize, usingValue, offset); - } - - } - @Override - void skipReplicationBytes(Bytes bytes) { - bytes.skip(ADDITIONAL_ENTRY_BYTES); - } + , RV> + RV acquireWithoutLock( + @Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, + long hash2, boolean create) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); + SegmentState localSegmentState = segmentState; + try { + MultiStoreBytes entry = null; + MultiMap hashLookup = hashLookup(); + hashLookup.startSearch(hash2); + for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { + long offset = offsetFromPos(pos); + if (entry == null) { + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); + localSegmentState = SegmentState.get(copies); + } + entry = localSegmentState.tmpBytes; + } + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) + continue; + // key is found + entry.skip(keySize); + entry.skip(ADDITIONAL_ENTRY_BYTES - 1L); + boolean isDeleted = entry.readBoolean(); + if (isDeleted) { + if (!create) + return readValue.readNull(); + long valueSizePos = entry.position(); + + entry.position(valueSizePos - ADDITIONAL_ENTRY_BYTES); + // todo theoretically, currentTime() call should be outside locking + // because locking might take time > 1 ms. but making current time a param + // of acquireWithoutLock would complicate code much, requiring putting + // lookupUsing() back into ReplicatedChMap, and other methods + entry.writeLong(currentTime()); + entry.writeByte(localIdentifier); + // deleted flag + entry.writeBoolean(false); + + long prevValueSize = readValueSize(entry); + long valueAddr = entry.positionAddr(); + long entryEndAddr = valueAddr + prevValueSize; + + // todo add api which doesn't require key instance + K keyInstance = toKey.toInstance(copies, key, keySize); + MetaBytesWriter metaElemWriter; + Object elemWriter; + Object elem; + if (defaultValueProvider != null) { + V defaultValue = defaultValueProvider.get(keyInstance); + elem = defaultValue; + VI valueInterop = + valueInteropProvider.get(copies, originalValueInterop); + elemWriter = valueInterop; + metaElemWriter = metaValueInteropProvider.get( + copies, originalMetaValueInterop, valueInterop, defaultValue); + } else if (prepareValueBytesAsWriter != null) { + elem = keyInstance; + elemWriter = null; + metaElemWriter = prepareValueBytesAsWriter; + } else { + throw defaultValueOrPrepareBytesShouldBeSpecified(); + } + putValue(pos, offset, entry, valueSizePos, entryEndAddr, + copies, localSegmentState, metaElemWriter, elemWriter, elem, + metaElemWriter.size(elemWriter, elem), hashLookup); + pos = localSegmentState.pos; + + incrementSize(); + hashLookup.putPosition(pos); + + + entry.position(valueSizePos); + long valueSize = readValueSize(entry); + long valuePos = entry.position(); + RV v = readValue.readValue(copies, entry, usingValue, valueSize); + + // put callbacks + onPut(this, pos); + if (bytesEventListener != null) { + bytesEventListener.onPut(entry, 0L, metaDataBytes, valuePos, true); + } + if (eventListener != null) { + V valueInstance = toValue.toInstance(copies, v, valueSize); + eventListener.onPut(keyInstance, valueInstance, null); + } + + return v; + } else { + return readValueAndNotifyGet(copies, key, keySize, toKey, + readValue, usingValue, toValue, entry); + } + } + if (!create) + return readValue.readNull(); + if (entry == null) { + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); // after this, copies is not null + localSegmentState = SegmentState.get(copies); + } + entry = localSegmentState.tmpBytes; + } + return createEntryOnAcquire(copies, localSegmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + readValue, usingValue, toValue, entry); + } finally { + if (segmentState == null && localSegmentState != null) + localSegmentState.close(); + } + } /** * called from a remote node as part of replication */ - private void remoteRemove(Bytes keyBytes, long hash2, - final long timestamp, final byte identifier) { - - - writeLock(); + private void remoteRemove( + @NotNull ThreadLocalCopies copies, @NotNull SegmentState segmentState, + Bytes keyBytes, long hash2, final long timestamp, final byte identifier) { + writeLock(null); try { - long keySize = keyBytes.remaining(); - hashLookupLiveAndDeleted.startSearch(hash2); - for (long pos; (pos = hashLookupLiveAndDeleted.nextPos()) >= 0L; ) { - long offset = offsetFromPos(pos); - NativeBytes entry = entry(offset); - if (!remoteKeyEquals(keyBytes, keySize, entry)) - continue; - - // key is found - - long keyPosition = entry.position(); - entry.skip(keySize); - - long timeStampPos = entry.position(); - - if (shouldIgnore(entry, timestamp, identifier)) { - return; - } - - // read the is deleted flag - boolean wasDeleted = entry.readBoolean(); - - if (!wasDeleted) { - hashLookupLiveOnly.remove(hash2, pos); - decrementSize(); - } - - entry.position(timeStampPos); - entry.writeLong(timestamp); - assert identifier > 0; - entry.writeByte(identifier); - - // was deleted - entry.writeBoolean(true); - - long valuePosition = entry.position(); - - if (hasAttachedEventListeners() && !wasDeleted) { - entry.position(keyPosition); - K key = null; - ThreadLocalCopies keyCopies = null; - - if (hasAttachedEventListeners()) { - keyCopies = keyInteropProvider.getCopies(null); - BytesReader keyReader = keyReaderProvider.get(keyCopies, originalKeyReader); - key = keyReader.read(entry, keySize); - } - - entry.position(valuePosition); - ThreadLocalCopies valueCopies = valueReaderProvider.getCopies(keyCopies); - BytesReader valueReader = valueReaderProvider.get(valueCopies, - originalValueReader); - V value = valueReader.read(entry, readValueSize(entry)); - - notifyRemoteRemove(offset, key, value, pos); - } - - return; - } - - // key is not found - if (LOG.isDebugEnabled()) + Boolean removed = (Boolean) removeWithoutLock(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, keyBytes, keyBytes.remaining(), + keyBytesToInstance, null, null, null, hash2, true, + timestamp, identifier, true, true); + if (!removed && LOG.isDebugEnabled()) { LOG.debug("Segment.remoteRemove() : key=" + keyBytes.toString().trim() + - " was not found"); + " was not found (or the remote update is late)"); + } } finally { writeUnlock(); } } - - void notifyRemoteRemove(long offset, K key, V value, final long pos) { - if (eventListener() != MapEventListeners.NOP) { - MultiStoreBytes b = acquireTmpBytes(); - b.storePositionAndSize(bytes, offset, entrySize); - attachedEventListener().onRemove(ReplicatedChronicleMap.this, b, metaDataBytes, - key, value, pos, this); - } - } - - - void notifyRemotePut(long offset, boolean added, final long pos, long keySize, final Bytes inBytes, - final long valueSize, final long valuePosition, - final long keyPosition, final V replacedValue, BytesReader valueReader, - BytesReader keyReader) { - if (eventListener() != MapEventListeners.NOP) { - final MultiStoreBytes b = acquireTmpBytes(); - b.storePositionAndSize(bytes, offset, entrySize); - - inBytes.limit(inBytes.capacity()); - - // deserialize the key - inBytes.position(keyPosition); - final K key = keyReader.read(inBytes, keySize); - - // deserialize the value - inBytes.limit(inBytes.capacity()); - inBytes.position(valuePosition); - final V value = valueReader.read(inBytes, valueSize); - - attachedEventListener().onPut(ReplicatedChronicleMap.this, b, metaDataBytes, - added, key, replacedValue, value, pos, this); - } - } - - /** * called from a remote node when it wishes to propagate a remove event */ - void remotePut(@NotNull final Bytes inBytes, long hash2, - final byte identifier, final long timestamp, - long valuePos, long valueLimit) { + void remotePut(@NotNull ThreadLocalCopies copies, @NotNull SegmentState segmentState, + @NotNull final Bytes entry, long keySize, long valueSize, long hash2, + final byte identifier, final long timestamp) { + GetRemoteBytesValueInterops getRemoteBytesValueInterops = + segmentState.getRemoteBytesValueInterops; + MultiStoreBytes value = getRemoteBytesValueInterops.getValueBytes( + entry, entry.position() + keySize); + getRemoteBytesValueInterops.valueSize(valueSize); + ReadValueToBytes readValueToLazyBytes = segmentState.readValueToLazyBytes; + readValueToLazyBytes.valueSizeMarshaller(valueSizeMarshaller); - - BytesReader valueReader = null; - BytesReader keyReader = null; - - if (hasAttachedEventListeners()) { - ThreadLocalCopies keyCopies = keyReaderProvider.getCopies(null); - keyReader = keyReaderProvider.get(keyCopies, originalKeyReader); - - ThreadLocalCopies valueCopies = valueReaderProvider.getCopies(keyCopies); - valueReader = valueReaderProvider.get(valueCopies, originalValueReader); - } - - writeLock(); + writeLock(null); try { - // inBytes position and limit correspond to the key - final long keySize = inBytes.remaining(); - V replacedValue = null; + putWithoutLock(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, entry, keySize, keyBytesToInstance, + getRemoteBytesValueInterops, value, valueBytesToInstance, + hash2, true, readValueToLazyBytes, true, identifier, timestamp, + true); + } finally { + writeUnlock(); + } + } + , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV put(@Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + long hash2, boolean replaceIfPresent, + ReadValue readValue, boolean resultUnused, + byte identifier, long timeStamp) { + writeLock(null); + try { + return putWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + getValueInterops, value, toValue, + hash2, replaceIfPresent, readValue, resultUnused, + identifier, timeStamp, false); + } finally { + if (segmentState != null) + segmentState.close(); + writeUnlock(); + } + } - hashLookupLiveAndDeleted.startSearch(hash2); - for (long pos; (pos = hashLookupLiveAndDeleted.nextPos()) >= 0L; ) { + @Override + , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV putWithoutLock( + @Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, long hash2, + boolean replaceIfPresent, ReadValue readValue, boolean resultUnused) { + return putWithoutLock(copies, segmentState, metaKeyInterop, keyInterop, key, keySize, + toKey, getValueInterops, value, toValue, hash2, replaceIfPresent, readValue, + resultUnused, localIdentifier, currentTime(), false); + } + private , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV putWithoutLock( + @Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + long hash2, boolean replaceIfPresent, + ReadValue readValue, boolean resultUnused, + byte identifier, long timestamp, boolean remote) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); + SegmentState localSegmentState = segmentState; + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); // copies is not null now + localSegmentState = SegmentState.get(copies); + } + try { + MultiMap hashLookup = hashLookup(); + hashLookup.startSearch(hash2); + MultiStoreBytes entry = localSegmentState.tmpBytes; + for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { long offset = offsetFromPos(pos); - NativeBytes entry = entry(offset); - long keyPosition = inBytes.position(); - if (!remoteKeyEquals(inBytes, keySize, entry)) + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) continue; - // key is found entry.skip(keySize); - final long timeStampPos = entry.positionAddr(); + final long timeStampPosAddr = entry.positionAddr(); if (shouldIgnore(entry, timestamp, identifier)) { - return; + // the following assert should be enabled, but TimeBasedReplicationTest + // intentionally violates the explained invariant, => assertion fails. + // todo do something with this + + // we should ignore only external remote updates + // which don't use put and remove results + // assert resultUnused; + return null; } - boolean wasDeleted = entry.readBoolean(); - entry.positionAddr(timeStampPos); - entry.writeLong(timestamp); - assert identifier > 0; - entry.writeByte(identifier); + boolean isDeleted = entry.readBoolean(); - // deleted flag - entry.writeBoolean(false); + if (replaceIfPresent || isDeleted) { + entry.positionAddr(timeStampPosAddr); + entry.writeLong(timestamp); + entry.writeByte(identifier); + // deleted flag + entry.writeBoolean(false); - long valueSizePos = entry.position(); - long replacedValueSize = readValueSize(entry); - long entryEndAddr = entry.positionAddr() + replacedValueSize; - - // write the value - inBytes.limit(valueLimit); - inBytes.position(valuePos); - - // if we have event listeners attached we have to read back the old value - if (hasAttachedEventListeners()) - replacedValue = valueReader.read(entry, replacedValueSize, null); - - putValue(pos, offset, entry, valueSizePos, entryEndAddr, null, null, inBytes, - hashLookupLiveAndDeleted); - - if (wasDeleted) { - // remove() would have got rid of this so we have to add it back in - hashLookupLiveOnly.put(hash2, pos); - incrementSize(); + RV prevValue = replaceValueAndNotifyPut(copies, localSegmentState, + key, keySize, toKey, + getValueInterops, value, toValue, + entry, pos, offset, hashLookup, readValue, + resultUnused, isDeleted, remote); + // for DRY (reusing replaceValueAndNotifyPut() method), + // size is updated AFTER callbacks are called. + // however this shouldn't be an issue because exclusive segment lock + // is still held + if (isDeleted) { + incrementSize(); + hashLookup.putPosition(localSegmentState.pos); + } + if (resultUnused) + return null; + return isDeleted ? readValue.readNull() : prevValue; + } else { + long valueSize = readValueSize(entry); + return resultUnused ? null : + readValue.readValue(copies, entry, null, valueSize); } - - if (hasAttachedEventListeners()) { - - // this is just like notifyPut, but it won't trigger the modification iterators - notifyRemotePut(offset, - false, - posFromOffset(offset), - keySize, - inBytes, - valueLimit - valuePos, - valuePos, - keyPosition, - replacedValue, - valueReader, - keyReader - ); - } - - return; } - // key is not found - long valueSize = valueLimit - valuePos; - long pos = alloc(inBlocks(entrySize(keySize, valueSize))); - long offset = offsetFromPos(pos); - clearMetaData(offset); - NativeBytes entry = entry(offset); - - keySizeMarshaller.writeSize(entry, keySize); - - // write the key - long kPosition = inBytes.position(); - entry.write(inBytes); - + VBI valueInterop = getValueInterops.getValueInterop(copies); + MVBI metaValueInterop = + getValueInterops.getMetaValueInterop(copies, valueInterop, value); + long valueSize = metaValueInterop.size(valueInterop, value); + putEntry(localSegmentState, metaKeyInterop, keyInterop, key, keySize, + metaValueInterop, valueInterop, value, entry, false); + entry.position(localSegmentState.valueSizePos - ADDITIONAL_ENTRY_BYTES); entry.writeLong(timestamp); entry.writeByte(identifier); entry.writeBoolean(false); - valueSizeMarshaller.writeSize(entry, valueSize); - alignment.alignPositionAddr(entry); - - // write the value - inBytes.limit(valueLimit); - inBytes.position(valuePos); - - entry.write(inBytes); - - hashLookupLiveAndDeleted.putAfterFailedSearch(pos); - hashLookupLiveOnly.put(hash2, pos); - - incrementSize(); - - // this is just like notifyPut, but it won't trigger the modification iterators - notifyRemotePut(offset, true, posFromOffset(offset), keySize, inBytes, valueSize, valuePos, - kPosition, null, valueReader, keyReader); + // put callbacks + if (!remote) + onPut(this, localSegmentState.pos); + if (bytesEventListener != null) + bytesEventListener.onPut(entry, 0L, metaDataBytes, + localSegmentState.valueSizePos, true); + if (eventListener != null) + eventListener.onPut(toKey.toInstance(copies, key, keySize), + toValue.toInstance(copies, value, valueSize), null); + return resultUnused ? null : readValue.readNull(); } finally { - writeUnlock(); + if (segmentState == null && localSegmentState != null) + localSegmentState.close(); } } - private boolean remoteKeyEquals(Bytes keyBytes, long keySize, Bytes entry) { - return keySize == keySizeMarshaller.readSize(entry) && entry.startsWith(keyBytes); - } - - @Override - V putWithoutLock(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V value, long hash2, boolean replaceIfPresent) { - return putWithoutLock(copies, metaKeyInterop, keyInterop, key, value, hash2, replaceIfPresent, - localIdentifier, timeProvider.currentTimeMillis()); - } - - V put(ThreadLocalCopies copies, MKI metaKeyWriter, KI keyWriter, - K key, V value, long hash2, boolean replaceIfPresent, - final byte identifier, final long timestamp) { - writeLock(); - try { - return putWithoutLock(copies, metaKeyWriter, keyWriter, key, value, hash2, replaceIfPresent, identifier, timestamp); - } finally { - writeUnlock(); - } - } - - private V putWithoutLock(ThreadLocalCopies copies, MKI metaKeyWriter, KI keyWriter, K key, V value, long hash2, boolean replaceIfPresent, byte identifier, long timestamp) { - MultiMap hashLookup = hashLookupLiveAndDeleted; - long keySize = metaKeyWriter.size(keyWriter, key); - hashLookup.startSearch(hash2); - for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { - long offset = offsetFromPos(pos); - MultiStoreBytes entry = entry(offset); - if (!keyEquals(keyWriter, metaKeyWriter, key, keySize, entry)) - continue; - // key is found - entry.skip(keySize); - - final long timeStampPos = entry.positionAddr(); - - if (shouldIgnore(entry, timestamp, identifier)) - return null; - - boolean wasDeleted = entry.readBoolean(); - - // if wasDeleted==true then we even if replaceIfPresent==false we can treat it - // the same way as replaceIfPresent true - if (replaceIfPresent || wasDeleted) { - - entry.positionAddr(timeStampPos); - entry.writeLong(timestamp); - entry.writeByte(identifier); - // deleted flag - entry.writeBoolean(false); - - final V prevValue = replaceValueOnPut(copies, key, value, entry, pos, - offset, !wasDeleted && !putReturnsNull, hashLookup); - - if (wasDeleted) { - // remove() would have got rid of this so we have to add it back in - hashLookupLiveOnly.put(hash2, pos); - incrementSize(); - return null; - } else { - return prevValue; - } - } else { - // we wont replaceIfPresent and the entry is not deleted - return putReturnsNull ? null : readValue(copies, entry, null); - } - } - - // key is not found - long offset = putEntry(copies, metaKeyWriter, keyWriter, key, keySize, hash2, value, - identifier, timestamp, hashLookup); - incrementSize(); - notifyPut(offset, true, key, value, null, posFromOffset(offset)); - return null; - } - - /** - * Used only with replication, its sometimes possible to receive an old ( or stale update ) from a - * remote map. This method is used to determine if we should ignore such updates.

We can reject - * put() and removes() when comparing times stamps with remote systems + * Used only with replication, its sometimes possible to receive an old ( or stale update ) + * from a remote map. This method is used to determine if we should ignore such updates. + * + *

We can reject put() and removes() when comparing times stamps with remote systems * * @param entry the maps entry * @param timestamp the time the entry was created or updated @@ -1059,220 +957,249 @@ final class ReplicatedChronicleMap, } - /** - * Puts entry. If {@code value} implements {@link Byteable} interface and {@code usingValue} is {@code - * true}, the value is backed with the bytes of this entry. - * - * @param hash2 a hash of the {@code keyBytes}. Caller was searching for the key in the - * {@code searchedHashLookup} using this hash. - * @param value the value to put - * @param identifier the identifier of the outer CHM node - * @param timestamp the timestamp when the entry was put (this could be later if it was a - * remote put) this method is called only from usual put or acquire - * @param searchedHashLookup the hash lookup that used to find the entry based on the key - * @return offset of the written entry in the Segment bytes - * @see VanillaChronicleMap.Segment#putEntry - */ - private long putEntry(ThreadLocalCopies copies, - MKI metaKeyInterop, KI keyInterop, K key, long keySize, long hash2, - V value, - int identifier, long timestamp, MultiMap searchedHashLookup) { - copies = valueWriterProvider.getCopies(copies); - VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = metaValueWriterProvider.getCopies(copies); - MetaBytesWriter metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, value); - - return putEntry(metaKeyInterop, keyInterop, key, keySize, hash2, - identifier, timestamp, searchedHashLookup, metaValueWriter, valueWriter, value); - } - - private long putEntry( - MKI metaKeyInterop, KI keyInterop, K key, long keySize, long hash2, - final int identifier, final long timestamp, MultiMap searchedHashLookup, - MetaBytesWriter metaElemWriter, EW elemWriter, E elem) { - long valueSize = metaElemWriter.size(elemWriter, elem); - long entrySize = entrySize(keySize, valueSize); - long pos = alloc(inBlocks(entrySize)); - long offset = offsetFromPos(pos); - clearMetaData(offset); - NativeBytes entry = entry(offset); - - keySizeMarshaller.writeSize(entry, keySize); - metaKeyInterop.write(keyInterop, entry, key); - - entry.writeLong(timestamp); - entry.writeByte(identifier); - entry.writeBoolean(false); - - valueSizeMarshaller.writeSize(entry, valueSize); - alignment.alignPositionAddr(entry); - metaElemWriter.write(elemWriter, entry, elem); - - // we have to add it both to the live - if (searchedHashLookup == hashLookupLiveAndDeleted) { - hashLookupLiveAndDeleted.putAfterFailedSearch(pos); - hashLookupLiveOnly.put(hash2, pos); + @Override + void manageReplicationBytes(Bytes entry, boolean writeDefaultInitialReplicationValues, + boolean remove) { + if (!writeDefaultInitialReplicationValues) { + entry.skip(ADDITIONAL_ENTRY_BYTES); } else { - hashLookupLiveOnly.putAfterFailedSearch(pos); - hashLookupLiveAndDeleted.put(hash2, pos); + entry.writeLong(currentTime()); + entry.writeByte(localIdentifier); + entry.writeBoolean(remove); } + } - return offset; + @Override + , + VB, VBI, MVBI extends MetaBytesInterop> + Object remove(@Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB expectedValue, + InstanceOrBytesToInstance toValue, + long hash2, boolean resultUnused) { + writeLock(null); + try { + return removeWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + getValueInterops, expectedValue, toValue, + hash2, resultUnused, currentTime(), localIdentifier, false, + expectedValue != null); + } finally { + if (segmentState != null) + segmentState.close(); + writeUnlock(); + } + } + + @Override + boolean isDeleted(Bytes entry, long keySize) { + return entry.readBoolean(entry.position() + keySize + ADDITIONAL_ENTRY_BYTES - 1L); + } + + @Override + , + VB, VBI, MVBI extends MetaBytesInterop> + Object removeWithoutLock( + @Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB expectedValue, + InstanceOrBytesToInstance toValue, + long hash2, boolean resultUnused) { + return removeWithoutLock(copies, segmentState, metaKeyInterop, keyInterop, key, keySize, + toKey, getValueInterops, expectedValue, toValue, hash2, resultUnused, + currentTime(), localIdentifier, false, expectedValue != null); } /** - * @see VanillaChronicleMap.Segment#remove + * - if expectedValue is not null, returns Boolean.TRUE (removed) or + * Boolean.FALSE (entry not found), regardless the expectedValue object is Bytes instance + * (RPC call) or the value instance + * - if expectedValue is null: + * - if resultUnused is false, null or removed value is returned + * - if resultUnused is true, null is always returned */ - V remove(ThreadLocalCopies copies, MKI metaKeyWriter, KI keyWriter, - K key, V expectedValue, long hash2, - final long timestamp, final byte identifier) { + , + VB, VBI, MVBI extends MetaBytesInterop> + Object removeWithoutLock( + @Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB expectedValue, + InstanceOrBytesToInstance toValue, + long hash2, boolean resultUnused, + long timestamp, byte identifier, boolean remote, boolean booleanResult) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); assert identifier > 0; - writeLock(); + expectedValueNotNullImpliesBooleanResult(expectedValue, booleanResult); + SegmentState localSegmentState = segmentState; try { - long keySize = metaKeyWriter.size(keyWriter, key); - final MultiMap multiMap = hashLookupLiveAndDeleted; - multiMap.startSearch(hash2); - for (long pos; (pos = multiMap.nextPos()) >= 0L; ) { + MultiMap hashLookup = hashLookup(); + hashLookup.startSearch(hash2); + MultiStoreBytes entry = null; + for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { long offset = offsetFromPos(pos); - MultiStoreBytes entry = entry(offset); - if (!keyEquals(keyWriter, metaKeyWriter, key, keySize, entry)) + if (entry == null) { + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); // copies is not null now + localSegmentState = SegmentState.get(copies); + } + entry = localSegmentState.tmpBytes; + } + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) continue; // key is found entry.skip(keySize); - long timeStampPos = entry.position(); + long timestampPos = entry.position(); if (shouldIgnore(entry, timestamp, identifier)) { - return null; + // the following assert should be enabled, but TimeBasedReplicationTest + // intentionally violates the explained invariant, => assertion fails. + // todo do something with this + + // we should ignore only remote updates + // which don't use remove, put results + // assert expectedValue != null || resultUnused; + return booleanResult ? Boolean.FALSE : null; } - // skip the is deleted flag - final boolean wasDeleted = entry.readBoolean(); - if (wasDeleted) { - // this caters for the case when the entry in not in our hashLookupLiveOnly - // map but maybe in our hashLookupLiveAndDeleted, - // so we have to send the deleted notification - entry.position(timeStampPos); + boolean isDeleted = entry.readBoolean(); + if (isDeleted) { + entry.position(timestampPos); entry.writeLong(timestamp); entry.writeByte(identifier); - // was deleted is already true - entry.skip(1); - - notifyRemoved(offset, key, null, pos); - return null; + break; // to "key not found" location } + long valueSizePos = entry.position(); long valueSize = readValueSize(entry); - V valueRemoved = expectedValue != null || !removeReturnsNull - ? readValue(copies, entry, null, valueSize) : null; + long valuePos = entry.position(); - if (expectedValue != null && !expectedValue.equals(valueRemoved)) - return null; + // check the value assigned for the key is that we expect + if (expectedValue != null) { + VBI valueInterop = getValueInterops.getValueInterop(copies); + MVBI metaValueInterop = getValueInterops.getMetaValueInterop( + copies, valueInterop, expectedValue); + if (metaValueInterop.size(valueInterop, expectedValue) != valueSize) + return Boolean.FALSE; + if (!metaValueInterop.startsWith(valueInterop, entry, expectedValue)) + return Boolean.FALSE; + } - hashLookupLiveOnly.remove(hash2, pos); - decrementSize(); - - entry.position(timeStampPos); + entry.position(timestampPos); entry.writeLong(timestamp); entry.writeByte(identifier); entry.writeBoolean(true); + entry.position(valuePos); - notifyRemoved(offset, key, valueRemoved, pos); - return valueRemoved; + return removeEntry(copies, key, keySize, toKey, expectedValue, toValue, + resultUnused, hashLookup, entry, pos, valueSizePos, valueSize, + remote, false, booleanResult); } // key is not found - return null; + return booleanResult ? Boolean.FALSE : null; } finally { - writeUnlock(); + if (segmentState == null && localSegmentState != null) + localSegmentState.close(); } } - @Override - MultiMap containsKeyHashLookup() { - return hashLookupLiveOnly; - } - - /** - * @see VanillaChronicleMap.Segment#replace - */ - V replace(ThreadLocalCopies copies, MKI metaKeyWriter, KI keyWriter, - K key, V expectedValue, V newValue, long hash2, long timestamp) { - writeLock(); + , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + Object replace( + @Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getExpectedValueInterops, VB expectedValue, + GetValueInterops getNewValueInterops, VB newValue, + ReadValue readValue, InstanceOrBytesToInstance toValue, + long hash2) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); + long timestamp = currentTime(); + byte identifier = localIdentifier; + writeLock(null); try { - long keySize = metaKeyWriter.size(keyWriter, key); - hashLookupLiveOnly.startSearch(hash2); - for (long pos; (pos = hashLookupLiveOnly.nextPos()) >= 0L; ) { + MultiMap hashLookup = hashLookup(); + hashLookup.startSearch(hash2); + MultiStoreBytes entry = null; + for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { long offset = offsetFromPos(pos); - MultiStoreBytes entry = entry(offset); - if (!keyEquals(keyWriter, metaKeyWriter, key, keySize, entry)) + if (entry == null) { + if (segmentState == null) { + copies = SegmentState.getCopies(copies); + segmentState = SegmentState.get(copies); + } + entry = segmentState.tmpBytes; + } + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) continue; // key is found entry.skip(keySize); - if (shouldIgnore(entry, timestamp, localIdentifier)) + long timestampPos = entry.position(); + if (shouldIgnore(entry, timestamp, identifier)) return null; - // skip the is deleted flag - entry.skip(1); + boolean isDeleted = entry.readBoolean(); + if (isDeleted) + break; - return onKeyPresentOnReplace(copies, key, expectedValue, newValue, pos, offset, - entry, hashLookupLiveOnly); + Object result = onKeyPresentOnReplace(copies, segmentState, key, keySize, toKey, + getExpectedValueInterops, expectedValue, getNewValueInterops, newValue, + readValue, toValue, pos, offset, entry, hashLookup); + if (result != Boolean.FALSE) { + entry.position(timestampPos); + entry.writeLong(timestamp); + entry.writeByte(identifier); + } + return result; } // key is not found return null; } finally { + if (segmentState != null) + segmentState.close(); writeUnlock(); } } - @Override - void replacePosInHashLookupOnRelocation(MultiMap searchedHashLookup, - long prevPos, long pos) { - searchedHashLookup.replacePrevPos(pos); - long hash = searchedHashLookup.getSearchHash(); - MultiMap anotherLookup = searchedHashLookup == hashLookupLiveAndDeleted ? - hashLookupLiveOnly : hashLookupLiveAndDeleted; - anotherLookup.replace(hash, prevPos, pos); - } - - /** - * @param timeStamp - * @param callback - * @param bootstapOnlyLocalEntries if true - only entries that have been create from this node will be - * published during a bootstrap + * @param bootstrapOnlyLocalEntries if true - only entries that have been create from + * this node will be published during a bootstrap */ public void dirtyEntries(final long timeStamp, final ModificationIterator.EntryModifiableCallback callback, - final boolean bootstapOnlyLocalEntries) { + final boolean bootstrapOnlyLocalEntries) { - this.readLock(); - try { + readLock(null); + ThreadLocalCopies copies = SegmentState.getCopies(null); + try (SegmentState segmentState = SegmentState.get(copies)) { final int index = Segment.this.getIndex(); - hashLookupLiveAndDeleted.forEach(new MultiMap.EntryConsumer() { + final MultiStoreBytes tmpBytes = segmentState.tmpBytes; + hashLookup().forEach(new MultiMap.EntryConsumer() { @Override public void accept(long hash, long pos) { - final NativeBytes entry = entry(offsetFromPos(pos)); + final NativeBytes entry = reuse(tmpBytes, offsetFromPos(pos)); long keySize = keySizeMarshaller.readSize(entry); entry.skip(keySize); final long entryTimestamp = entry.readLong(); - if (entryTimestamp >= timeStamp && (!bootstapOnlyLocalEntries || + if (entryTimestamp >= timeStamp && (!bootstrapOnlyLocalEntries || entry.readByte() == ReplicatedChronicleMap.this.identifier())) callback.set(index, pos); } }); - } finally { readUnlock(); } } @Override - public Entry getEntry(long pos) { - MultiStoreBytes entry = acquireTmpBytes(); - long offset = offsetFromPos(pos) + metaDataBytes; - entry.setBytesOffset(bytes, offset); + public Entry getEntry(@NotNull SegmentState segmentState, long pos) { + Bytes entry = reuse(segmentState.tmpBytes, offsetFromPos(pos)); long keySize = keySizeMarshaller.readSize(entry); ThreadLocalCopies copies = keyReaderProvider.getCopies(null); @@ -1290,15 +1217,19 @@ final class ReplicatedChronicleMap, } @Override - void clear() { - // we have to make sure that every calls notifies on remove, - // so that the replicators can pick it up - for (K k : keySet()) { - ReplicatedChronicleMap.this.remove(k); - } + boolean isDeleted(long pos) { + bytes.position(offsetFromPos(pos) + metaDataBytes); + long keySize = keySizeMarshaller.readSize(bytes); + return bytes.readBoolean(bytes.position() + keySize + ADDITIONAL_ENTRY_BYTES - 1L); } } + @Override + void shouldNotBeCalledFromReplicatedChronicleMap(String method) { + throw new AssertionError(method + "() method should not be called by " + + "ReplicatedChronicleMap instance"); + } + class TimestampTrackingEntry extends SimpleEntry { private static final long serialVersionUID = 0L; @@ -1311,21 +1242,26 @@ final class ReplicatedChronicleMap, @Override public V setValue(V value) { - long newTimestamp = timestamp = timeProvider.currentTimeMillis(); + long newTimestamp = timestamp = currentTime(); put(getKey(), value, localIdentifier, newTimestamp); return super.setValue(value); } } - class EntryIterator extends VanillaChronicleMap.EntryIterator { + class EntryIterator extends VanillaChronicleMap.EntryIterator { + + @Override + public Entry next() { + return super.next(); + } + @Override void removePresent(VanillaChronicleMap.Segment seg, long pos) { @SuppressWarnings("unchecked") Segment segment = (Segment) seg; final long offset = segment.offsetFromPos(pos); - final NativeBytes entry = segment.entry(offset); - final long limit = entry.limit(); + final NativeBytes entry = segment.reuse(this.entry, offset); final long keySize = keySizeMarshaller.readSize(entry); long keyPosition = entry.position(); @@ -1337,13 +1273,13 @@ final class ReplicatedChronicleMap, // Check that it is still the entry with the same key K key = returnedEntry.getKey(); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); - KI keyWriter = keyInteropProvider.get(copies, originalKeyInterop); + KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); copies = metaKeyInteropProvider.getCopies(copies); - MKI metaKeyWriter = - metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyWriter, key); - long returnedKeySize = metaKeyWriter.size(keyWriter, key); + MKI metaKeyInterop = + metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + long returnedKeySize = metaKeyInterop.size(keyInterop, key); if (returnedKeySize != keySize || - !metaKeyWriter.startsWith(keyWriter, entry, key)) { + !metaKeyInterop.startsWith(keyInterop, entry, key)) { // The case: // 1. iterator.next() - thread 1 // 2. map.put() which cause relocation of the key, returned above - thread 2 @@ -1354,21 +1290,12 @@ final class ReplicatedChronicleMap, return; } } - final long segmentHash = segmentHash(hash(entry, keyPosition, keyPosition + keySize)); - segment.hashLookupLiveOnly.remove(segmentHash, pos); - segment.decrementSize(); - - entry.skip(keySize); - entry.writeLong(timeProvider.currentTimeMillis()); - entry.writeByte(localIdentifier); - entry.writeBoolean(true); - - segment.notifyRemoved(offset, returnedEntry.getKey(), returnedEntry.getValue(), pos); + removePresent(segment, pos, entry, keySize, 0L, false); } } - class EntrySet extends VanillaChronicleMap.EntrySet { + class EntrySet extends VanillaChronicleMap.EntrySet { @NotNull @Override public Iterator> iterator() { @@ -1376,146 +1303,6 @@ final class ReplicatedChronicleMap, } } - /** - * receive an update from the map, via the MapEventListener and delegates the changes to the currently - * active modification iterators - */ - class ModificationDelegator extends MapEventListener> { - private static final long serialVersionUID = 0L; - - // the assigned modification iterators - private final ATSDirectBitSet modIterSet; - private final MapEventListener> nextListener; - - public ModificationDelegator(@NotNull final MapEventListener> nextListener, - final Bytes bytes) { - this.nextListener = nextListener; - modIterSet = new ATSDirectBitSet(bytes); - } - - private final AtomicReferenceArray modificationIterators = - new AtomicReferenceArray(127 + RESERVED_MOD_ITER); - - public ModificationIterator acquireModificationIterator( - final short remoteIdentifier, - @NotNull final ModificationNotifier modificationNotifier) { - - final ModificationIterator modificationIterator = modificationIterators.get(remoteIdentifier); - - if (modificationIterator != null) - return modificationIterator; - - synchronized (modificationIterators) { - final ModificationIterator modificationIterator0 = modificationIterators.get(remoteIdentifier); - - if (modificationIterator0 != null) - return modificationIterator0; - - final Bytes bytes = ms.bytes(startOfModificationIterators + (modIterBitSetSizeInBytes() - * remoteIdentifier), - modIterBitSetSizeInBytes()); - - final ModificationIterator newModificationIterator = new ModificationIterator( - bytes, modificationNotifier); - - modificationIterators.set(remoteIdentifier, newModificationIterator); - modIterSet.set(remoteIdentifier); - return newModificationIterator; - } - } - - @Override - public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, - boolean added, K key, V replacedValue, V value, long pos, SharedSegment segment) { - - assert ReplicatedChronicleMap.this == map : - "ModificationIterator.onPut() is called from outside of the parent map"; - - try { - nextListener.onPut(map, entry, metaDataBytes, added, key, replacedValue, value, pos, segment); - } catch (Exception e) { - LOG.error("", e); - } - - dirty(map, pos, segment); - } - - /** - * used to dirty the entry for replication - */ - void dirty(ChronicleMap map, long pos, SharedSegment segment) { - for (long next = modIterSet.nextSetBit(0); next > 0; next = modIterSet.nextSetBit(next + 1)) { - try { - final ModificationIterator modificationIterator = modificationIterators.get((int) next); - modificationIterator.onPut(map, null, 0, - false, null, null, null, pos, segment); - } catch (Exception e) { - LOG.error("", e); - } - } - } - - @Override - public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, - K key, V value, long pos, SharedSegment segment) { - assert ReplicatedChronicleMap.this == map : - "ModificationIterator.onRemove() is called from outside of the parent map"; - - try { - nextListener.onRemove(map, entry, metaDataBytes, key, value, pos, segment); - } catch (Exception e) { - LOG.error("", e); - } - - for (long next = modIterSet.nextSetBit(0); next > 0; next = modIterSet.nextSetBit(next + 1)) { - try { - final ModificationIterator modificationIterator = modificationIterators.get((int) next); - modificationIterator.onRemove(map, entry, metaDataBytes, key, value, pos, segment); - } catch (Exception e) { - LOG.error("", e); - } - } - - } - - @Override - void onRelocation(long pos, SharedSegment segment) { - for (long next = modIterSet.nextSetBit(0); next > 0; next = modIterSet.nextSetBit(next + 1)) { - try { - final ModificationIterator modificationIterator = modificationIterators.get((int) next); - modificationIterator.onRelocation(pos, segment); - } catch (Exception e) { - LOG.error("", e); - } - } - } - - @Override - public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, - K key, V value) { - nextListener.onGetFound(map, entry, metaDataBytes, key, value); - } - - /** - * Always throws {@code NotSerializableException} since instances of this class are not intended to be - * serializable. - */ - private void writeObject(ObjectOutputStream out) throws IOException { - throw new NotSerializableException(getClass().getCanonicalName()); - } - - /** - * Always throws {@code NotSerializableException} since instances of this class are not intended to be - * serializable. - */ - private void readObject(ObjectInputStream in) - throws IOException, ClassNotFoundException { - throw new NotSerializableException(getClass().getCanonicalName()); - } - - - } - /** *

Once a change occurs to a map, map replication requires that these changes are picked up by another * thread, this class provides an iterator like interface to poll for such changes.

In most cases @@ -1528,27 +1315,27 @@ final class ReplicatedChronicleMap, * * @author Rob Austin. */ - class ModificationIterator extends MapEventListener> - implements Replica.ModificationIterator { - private static final long serialVersionUID = 0L; - + class ModificationIterator implements Replica.ModificationIterator { private final ModificationNotifier modificationNotifier; private final ATSDirectBitSet changes; private final int segmentIndexShift; private final long posMask; - private final EntryModifiableCallback entryModifiableCallback = new EntryModifiableCallback(); + private final EntryModifiableCallback entryModifiableCallback = + new EntryModifiableCallback(); // records the current position of the cursor in the bitset private long position = -1L; + // todo get rid of this + private final MultiStoreBytes tmpBytes = new MultiStoreBytes(); + /** * @param bytes the back the bitset, used to mark which entries have changed * @param modificationNotifier called when ever there is a change applied */ public ModificationIterator(@NotNull final Bytes bytes, @NotNull final ModificationNotifier modificationNotifier) { - this.modificationNotifier = modificationNotifier; long bitsPerSegment = bitsPerSegmentInModIterBitSet(); segmentIndexShift = Long.numberOfTrailingZeros(bitsPerSegment); @@ -1567,58 +1354,41 @@ final class ReplicatedChronicleMap, return (((long) segmentIndex) << segmentIndexShift) | pos; } - @Override - public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, - boolean added, K key, V replacedValue, V value, long pos, SharedSegment segment) { - - assert ReplicatedChronicleMap.this == map : - "ModificationIterator.onPut() is called from outside of the parent map"; - - + public void onPut(long pos, SharedSegment segment) { changes.set(combine(segment.getIndex(), pos)); modificationNotifier.onChange(); } - @Override - public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, - K key, V value, long pos, SharedSegment segment) { - assert ReplicatedChronicleMap.this == map : - "ModificationIterator.onRemove() is called from outside of the parent map"; - - + public void onRemove(long pos, SharedSegment segment) { changes.set(combine(segment.getIndex(), pos)); modificationNotifier.onChange(); - } /** * Ensures that garbage in the old entry's location won't be broadcast as changed entry. */ - @Override void onRelocation(long pos, SharedSegment segment) { changes.clear(combine(segment.getIndex(), pos)); - // don't call nextListener.onRelocation(), - // because no one event listener else overrides this method. } - /** * you can continue to poll hasNext() until data becomes available. If are are in the middle of * processing an entry via {@code nextEntry}, hasNext will return true until the bit is cleared * * @return true if there is an entry */ + @Override public boolean hasNext() { final long position = this.position; return changes.nextSetBit(position == NOT_FOUND ? 0L : position) != NOT_FOUND || (position > 0L && changes.nextSetBit(0L) != NOT_FOUND); } - /** * @param entryCallback call this to get an entry, this class will take care of the locking * @return true if an entry was processed */ + @Override public boolean nextEntry(@NotNull final EntryCallback entryCallback, final int chronicleId) { long position = this.position; while (true) { @@ -1634,15 +1404,17 @@ final class ReplicatedChronicleMap, } this.position = position; - final SharedSegment segment = segment((int) (position >>> segmentIndexShift)); - segment.readLock(); + final VanillaChronicleMap.Segment segment = + segment((int) (position >>> segmentIndexShift)); + segment.readLock(null); try { if (changes.clearIfSet(position)) { entryCallback.onBeforeEntry(); final long segmentPos = position & posMask; - final NativeBytes entry = segment.entry(segment.offsetFromPos(segmentPos)); + final NativeBytes entry = + segment.reuse(tmpBytes, segment.offsetFromPos(segmentPos)); // if the entry should be ignored, we'll move the next entry final boolean success = entryCallback.onEntry(entry, chronicleId); @@ -1663,31 +1435,14 @@ final class ReplicatedChronicleMap, @Override public void dirtyEntries(long fromTimeStamp) { - // iterate over all the segments and mark bit in the modification iterator // that correspond to entries with an older timestamp for (final Segment segment : (Segment[]) segments) { - segment.dirtyEntries(fromTimeStamp, entryModifiableCallback, bootstapOnlyLocalEntries); + segment.dirtyEntries(fromTimeStamp, entryModifiableCallback, + bootstrapOnlyLocalEntries); } } - /** - * Always throws {@code NotSerializableException} since instances of this class are not intended to be - * serializable. - */ - private void writeObject(ObjectOutputStream out) throws IOException { - throw new NotSerializableException(getClass().getCanonicalName()); - } - - /** - * Always throws {@code NotSerializableException} since instances of this class are not intended to be - * serializable. - */ - private void readObject(ObjectInputStream in) - throws IOException, ClassNotFoundException { - throw new NotSerializableException(getClass().getCanonicalName()); - } - /** * details about when a modification to an entry was made */ @@ -1705,5 +1460,4 @@ final class ReplicatedChronicleMap, } } } - } \ No newline at end of file diff --git a/src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java b/src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java index 8f404c1a9..c8998354f 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java @@ -64,17 +64,15 @@ class ReplicationHubFindByName implements FindByName { this.replicationHub = replicationHub; ReplicationChannel channel = replicationHub.createChannel((short) MAP_BY_NAME_CHANNEL); - final MapEventListener> listener = new - MapEventListener>() { + final MapEventListener listener = new + MapEventListener() { // creates a map based on the details that are sent to the map of builders @Override - void onPut(ChronicleMap map, Bytes - entry, int metaDataBytes, boolean added, CharSequence key, - ChronicleMapBuilderWithChannelId replaced, ChronicleMapBuilderWithChannelId - value, long pos, SharedSegment segment) { - super.onPut(map, entry, metaDataBytes, added, key, replaced, value, pos, segment); - + public void onPut(CharSequence key, ChronicleMapBuilderWithChannelId value, + ChronicleMapBuilderWithChannelId replacedValue) { + super.onPut(key, value, replacedValue); + boolean added = replacedValue == null; if (!added || value == null) return; diff --git a/src/main/java/net/openhft/chronicle/map/SharedSegment.java b/src/main/java/net/openhft/chronicle/map/SharedSegment.java index b0fd84c40..eb294a940 100644 --- a/src/main/java/net/openhft/chronicle/map/SharedSegment.java +++ b/src/main/java/net/openhft/chronicle/map/SharedSegment.java @@ -19,6 +19,8 @@ package net.openhft.chronicle.map; import net.openhft.lang.io.NativeBytes; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.Map; @@ -28,18 +30,19 @@ import java.util.Map; */ interface SharedSegment { - AutoCloseable readLock(); + /** + * if passed segmentState is null, null is returned instead of readLock + */ + AutoCloseable readLock(@Nullable VanillaChronicleMap.SegmentState segmentState); - AutoCloseable writeLock(); + AutoCloseable writeLock(@Nullable VanillaChronicleMap.SegmentState segmentState); - Map.Entry getEntry(long pos); + Map.Entry getEntry(@NotNull VanillaChronicleMap.SegmentState segmentState, long pos); void readUnlock(); void writeUnlock(); - NativeBytes entry(long pos); - int getIndex(); long offsetFromPos(long pos); diff --git a/src/main/java/net/openhft/chronicle/map/ShortShortMultiMap.java b/src/main/java/net/openhft/chronicle/map/ShortShortMultiMap.java index f5ce4b7f5..325028e97 100644 --- a/src/main/java/net/openhft/chronicle/map/ShortShortMultiMap.java +++ b/src/main/java/net/openhft/chronicle/map/ShortShortMultiMap.java @@ -120,6 +120,12 @@ class ShortShortMultiMap implements MultiMap { return (long) (entry & 0xFFFF); } + @Override + public void putPosition(long value) { + checkValueForPut(value); + positions.set(value); + } + @Override public void put(long key, long value) { key = maskUnsetKey(key); @@ -134,6 +140,12 @@ class ShortShortMultiMap implements MultiMap { } } + @Override + public void removePosition(long value) { + checkValueForRemove(value); + positions.clear(value); + } + @Override public void remove(long key, long value) { key = maskUnsetKey(key); @@ -240,6 +252,7 @@ class ShortShortMultiMap implements MultiMap { bytes.writeInt(searchPos, entry(searchHash, value)); } + @Override public long getSearchHash() { return searchHash; } diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 718bfde30..1baeb0d5a 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -1050,7 +1050,6 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * * @param socketChannel the socket to publish the buffer to * @param approxTime an approximation of the current time in millis - * @param key the selectors key * @throws IOException */ private int writeBufferToSocket(@NotNull final SocketChannel socketChannel, @@ -1282,7 +1281,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable } } else - externalizable.readExternalEntry(out); + externalizable.readExternalEntry(copies, segmentState, out); out.limit(limit); @@ -1506,41 +1505,35 @@ class StatelessServerConnector { } private Work removeWithValue(Bytes reader, Bytes writer, final long sizeLocation) { - if (MAP_SUPPORTS_BYTES) { - map.removeWithValue(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K key = readKey(reader, local); - final V readValue = readValue(reader, local); - - final boolean result; - try { - result = map.remove(key, readValue); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); + try { + if (MAP_SUPPORTS_BYTES) { + writer.writeBoolean(map.removeWithValue(reader)); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K key = readKey(reader, local); + final V readValue = readValue(reader, local); + writer.writeBoolean(map.remove(key, readValue)); } - writer.writeBoolean(result); + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; } private Work replaceWithOldAndNew(Bytes reader, Bytes writer, final long sizeLocation) { - if (MAP_SUPPORTS_BYTES) { - map.replaceWithOldAndNew(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K key = readKey(reader, local); - final V oldValue = readValue(reader, local); - final V newValue = readValue(reader, local); - boolean replaced; - - try { - replaced = map.replace(key, oldValue, newValue); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); + try { + if (MAP_SUPPORTS_BYTES) { + map.replaceWithOldAndNew(reader, writer); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K key = readKey(reader, local); + final V oldValue = readValue(reader, local); + final V newValue = readValue(reader, local); + writer.writeBoolean(map.replace(key, oldValue, newValue)); } - writer.writeBoolean(replaced); + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; @@ -1621,52 +1614,46 @@ class StatelessServerConnector { } private Work containsKey(Bytes reader, Bytes writer, final long sizeLocation) { - if (MAP_SUPPORTS_BYTES) { - map.containsKey(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K k = readKey(reader, local); - - try { + try { + if (MAP_SUPPORTS_BYTES) { + writer.writeBoolean(map.containsKey(reader)); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K k = readKey(reader, local); writer.writeBoolean(map.containsKey(k)); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); } + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; } private Work containsValue(Bytes reader, Bytes writer, final long sizeLocation) { + // todo optimize -- eliminate + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final V v = readValue(reader, local); - if (MAP_SUPPORTS_BYTES) { - map.containsValue(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final V v = readValue(reader, local); - - try { - writer.writeBoolean(map.containsValue(v)); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); - } + try { + writer.writeBoolean(map.containsValue(v)); + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; } private Work get(Bytes reader, Bytes writer, final long sizeLocation) { - if (MAP_SUPPORTS_BYTES) { - map.get(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K k = readKey(reader, local); - - try { + try { + if (MAP_SUPPORTS_BYTES) { + map.get(reader, writer); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K k = readKey(reader, local); writeValue(map.get(k), writer, local); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); } + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; @@ -1688,18 +1675,17 @@ class StatelessServerConnector { } private Work put(Bytes reader, Bytes writer, final long sizeLocation) { - if (MAP_SUPPORTS_BYTES) { - map.put(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K k = readKey(reader, local); - final V v = readValue(reader, local); - - try { + try { + if (MAP_SUPPORTS_BYTES) { + map.put(reader, writer); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K k = readKey(reader, local); + final V v = readValue(reader, local); writeValue(map.put(k, v), writer, local); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); } + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; @@ -1707,7 +1693,7 @@ class StatelessServerConnector { private Work remove(Bytes reader) { if (MAP_SUPPORTS_BYTES) { - map.removeWithBytes(reader); + map.removeKeyAsBytes(reader); } else { final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); final K key = readKey(reader, local); @@ -1719,40 +1705,34 @@ class StatelessServerConnector { private Work remove(Bytes reader, Bytes writer, final long sizeLocation) { - if (MAP_SUPPORTS_BYTES) { - map.remove(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K key = readKey(reader, local); - - try { + try { + if (MAP_SUPPORTS_BYTES) { + map.remove(reader, writer); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K key = readKey(reader, local); if (LOG.isDebugEnabled()) LOG.debug("removing entry key=" + key); writeValue(map.remove(key), writer, local); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); } - + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } - writeSizeAndFlags(sizeLocation, false, writer); return null; } private Work putAll(Bytes reader, Bytes writer, final long sizeLocation) { - - if (MAP_SUPPORTS_BYTES) { - map.putAll(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final Map m = readEntries(reader, local); - - try { + try { + if (MAP_SUPPORTS_BYTES) { + map.putAll(reader); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final Map m = readEntries(reader, local); map.putAll(m); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); } - + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; @@ -1924,23 +1904,17 @@ class StatelessServerConnector { private Work putIfAbsent(Bytes reader, Bytes writer, final long sizeLocation) { - - if (MAP_SUPPORTS_BYTES) { - map.putIfAbsent(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K key = readKey(reader, local); - final V v = readValue(reader, local); - - final V result; - - try { - result = map.putIfAbsent(key, v); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); + try { + if (MAP_SUPPORTS_BYTES) { + map.putIfAbsent(reader, writer); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K key = readKey(reader, local); + final V v = readValue(reader, local); + writeValue(map.putIfAbsent(key, v), writer, local); } - - writeValue(result, writer, local); + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } writeSizeAndFlags(sizeLocation, false, writer); return null; @@ -1948,23 +1922,18 @@ class StatelessServerConnector { private Work replace(Bytes reader, Bytes writer, final long sizeLocation) { - - if (MAP_SUPPORTS_BYTES) { - map.replaceKV(reader, writer); - } else { - final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); - final K k = readKey(reader, local); - final V v = readValue(reader, local); - final V replaced; - - try { - replaced = map.replace(k, v); - } catch (Throwable e) { - return sendException(writer, sizeLocation, e); + try { + if (MAP_SUPPORTS_BYTES) { + map.replaceKV(reader, writer); + } else { + final ThreadLocalCopies local = keyValueSerializer.threadLocalCopies(); + final K k = readKey(reader, local); + final V v = readValue(reader, local); + writeValue(map.replace(k, v), writer, local); } - writeValue(replaced, writer, local); + } catch (Throwable e) { + return sendException(writer, sizeLocation, e); } - writeSizeAndFlags(sizeLocation, false, writer); return null; } diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java index 5eab17b22..d535ceb3b 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java @@ -60,7 +60,7 @@ final class UdpReplicator extends UdpChannelReplicator implements Replica.Modifi start(); } - private static class UdpSocketChannelEntryReader implements EntryReader { + private class UdpSocketChannelEntryReader implements EntryReader { private final Replica.EntryExternalizable externalizable; private final ByteBuffer in; @@ -113,7 +113,7 @@ final class UdpReplicator extends UdpChannelReplicator implements Replica.Modifi if (out.remaining() != size) return; - externalizable.readExternalEntry(out); + externalizable.readExternalEntry(copies, segmentState, out); } } diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index b1c0f99f3..9176e5da1 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -27,23 +27,23 @@ import com.thoughtworks.xstream.io.HierarchicalStreamReader; import com.thoughtworks.xstream.io.HierarchicalStreamWriter; import com.thoughtworks.xstream.io.json.JettisonMappedXmlDriver; import net.openhft.chronicle.hash.ChronicleHashErrorListener; +import net.openhft.chronicle.hash.serialization.BytesInterop; import net.openhft.chronicle.hash.serialization.BytesReader; import net.openhft.chronicle.hash.serialization.Hasher; import net.openhft.chronicle.hash.serialization.SizeMarshaller; -import net.openhft.chronicle.hash.serialization.internal.MetaBytesInterop; -import net.openhft.chronicle.hash.serialization.internal.MetaBytesWriter; -import net.openhft.chronicle.hash.serialization.internal.MetaProvider; +import net.openhft.chronicle.hash.serialization.internal.*; import net.openhft.lang.collection.DirectBitSet; import net.openhft.lang.collection.SingleThreadedDirectBitSet; import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.JDKZObjectSerializer; import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; -import net.openhft.lang.model.constraints.Nullable; import net.openhft.lang.threadlocal.Provider; +import net.openhft.lang.threadlocal.StatefulCopyable; import net.openhft.lang.threadlocal.ThreadLocalCopies; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.jetbrains.annotations.Nullable; import java.io.*; import java.lang.reflect.Array; @@ -61,11 +61,13 @@ import static java.lang.Long.numberOfTrailingZeros; import static java.lang.Math.max; import static java.lang.Thread.currentThread; import static java.nio.ByteBuffer.allocateDirect; +import static net.openhft.chronicle.map.Asserts.assertNotNull; import static net.openhft.lang.MemoryUnit.*; -class VanillaChronicleMap, - V, VW, MVW extends MetaBytesWriter> extends AbstractMap - implements ChronicleMap, Serializable { +class VanillaChronicleMap, + V, VI, MVI extends MetaBytesInterop> extends AbstractMap + implements ChronicleMap, Serializable, ReadValue, InstanceOrBytesToInstance, + GetValueInterops { private static final Logger LOG = LoggerFactory.getLogger(VanillaChronicleMap.class); @@ -74,7 +76,6 @@ class VanillaChronicleMap, * bits. */ static final int MAX_ENTRY_OVERSIZE_FACTOR = 64; - static final ThreadLocal tmpBytes = new ThreadLocal<>(); private static final long serialVersionUID = 2L; final Class kClass; final SizeMarshaller keySizeMarshaller; @@ -85,9 +86,9 @@ class VanillaChronicleMap, final Class vClass; final SizeMarshaller valueSizeMarshaller; final BytesReader originalValueReader; - final VW originalValueWriter; - final MVW originalMetaValueWriter; - final MetaProvider metaValueWriterProvider; + final VI originalValueInterop; + final MVI originalMetaValueInterop; + final MetaProvider metaValueInteropProvider; final DefaultValueProvider defaultValueProvider; final PrepareValueBytesAsWriter prepareValueBytesAsWriter; final int metaDataBytes; @@ -96,7 +97,8 @@ class VanillaChronicleMap, final Alignment alignment; final int actualSegments; final long entriesPerSegment; - transient MapEventListener> eventListener; + final MapEventListener eventListener; + final BytesMapEventListener bytesEventListener; // if set the ReturnsNull fields will cause some functions to return NULL // rather than as returning the Object can be expensive for something you probably don't use. final boolean putReturnsNull; @@ -110,12 +112,12 @@ class VanillaChronicleMap, transient Provider> keyReaderProvider; transient Provider keyInteropProvider; transient Provider> valueReaderProvider; - transient Provider valueWriterProvider; + transient Provider valueInteropProvider; transient Segment[] segments; // non-final for close() // non-final for close() and because it is initialized out of constructor transient BytesStore ms; transient long headerSize; - transient Set> entrySet; + private transient Set> entrySet; private transient SerializationBuilder keyBuilder; private transient SerializationBuilder valueBuilder; @@ -138,9 +140,9 @@ class VanillaChronicleMap, valueSizeMarshaller = valueBuilder.sizeMarshaller(); originalValueReader = valueBuilder.reader(); - originalValueWriter = (VW) valueBuilder.interop(); - originalMetaValueWriter = (MVW) valueBuilder.metaInterop(); - metaValueWriterProvider = (MetaProvider) valueBuilder.metaInteropProvider(); + originalValueInterop = (VI) valueBuilder.interop(); + originalMetaValueInterop = (MVI) valueBuilder.metaInterop(); + metaValueInteropProvider = (MetaProvider) valueBuilder.metaInteropProvider(); defaultValueProvider = builder.defaultValueProvider(); prepareValueBytesAsWriter = builder.prepareValueBytesAsWriter(); @@ -159,17 +161,18 @@ class VanillaChronicleMap, this.multiMapFactory = builder.multiMapFactory(); this.metaDataBytes = builder.metaDataBytes(); this.eventListener = builder.eventListener(); + this.bytesEventListener = builder.bytesEventListener(); this.segmentHeaderSize = builder.segmentHeaderSize(); hashSplitting = HashSplitting.Splitting.forSegments(actualSegments); initTransients(); } - long segmentHash(long hash) { + final long segmentHash(long hash) { return hashSplitting.segmentHash(hash); } - int getSegment(long hash) { + final int getSegment(long hash) { return hashSplitting.segmentIndex(hash); } @@ -178,7 +181,7 @@ class VanillaChronicleMap, keyInteropProvider = Provider.of((Class) originalKeyInterop.getClass()); valueReaderProvider = Provider.of((Class) originalValueReader.getClass()); - valueWriterProvider = Provider.of((Class) originalValueWriter.getClass()); + valueInteropProvider = Provider.of((Class) originalValueInterop.getClass()); if (defaultValueProvider instanceof ConstantValueProvider) { ConstantValueProvider constantValueProvider = @@ -199,7 +202,7 @@ class VanillaChronicleMap, return Segment.class; } - long createMappedStoreAndSegments(BytesStore bytesStore) throws IOException { + final long createMappedStoreAndSegments(BytesStore bytesStore) throws IOException { this.ms = bytesStore; onHeaderCreated(); @@ -212,7 +215,8 @@ class VanillaChronicleMap, long segmentSize = segmentSize(); long headerOffset = 0; for (int i = 0; i < this.segments.length; i++) { - NativeBytes segmentHeader = (NativeBytes) segmentHeaders.bytes(headerOffset, segmentHeaderSize); + NativeBytes segmentHeader = + (NativeBytes) segmentHeaders.bytes(headerOffset, segmentHeaderSize); NativeBytes segmentData = (NativeBytes) ms.bytes(offset, segmentSize); this.segments[i] = createSegment(segmentHeader, segmentData, i); headerOffset += segmentHeaderSize; @@ -221,7 +225,7 @@ class VanillaChronicleMap, return offset; } - long createMappedStoreAndSegments(File file) throws IOException { + final long createMappedStoreAndSegments(File file) throws IOException { return createMappedStoreAndSegments(new MappedStore(file, FileChannel.MapMode.READ_WRITE, sizeInBytes(), JDKZObjectSerializer.INSTANCE)); } @@ -242,7 +246,7 @@ class VanillaChronicleMap, return headerSize; } - long getSegmentHeaderSize() { + final long getSegmentHeaderSize() { return segmentHeaderSize; } @@ -255,31 +259,29 @@ class VanillaChronicleMap, return ms.file(); } - long sizeInBytes() { + final long sizeInBytes() { return getHeaderSize() + segments.length * getSegmentHeaderSize() + segments.length * segmentSize(); } - long sizeOfMultiMap() { + private long sizeOfMultiMap() { return multiMapFactory.sizeInBytes(entriesPerSegment); } - long sizeOfMultiMapBitSet() { + private long sizeOfMultiMapBitSet() { return MultiMapFactory.sizeOfBitSetInBytes(entriesPerSegment); } - long sizeOfBitSets() { + private long sizeOfBitSets() { return CACHE_LINES.align(BYTES.alignAndConvert(entriesPerSegment, BITS), BYTES); } - int numberOfBitSets() { - return 1; // for free list - // + (replicas > 0 ? 1 : 0) // deleted set - // + replicas; // to notify each replica of a change. + private int numberOfBitSets() { + return 1; } - long segmentSize() { + private long segmentSize() { long ss = (CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES) * multiMapsPerSegment()) + numberOfBitSets() * sizeOfBitSets() // the free list and 0+ dirty lists. @@ -301,7 +303,7 @@ class VanillaChronicleMap, return segmentSize; } - int multiMapsPerSegment() { + private int multiMapsPerSegment() { return 1; } @@ -318,11 +320,7 @@ class VanillaChronicleMap, ms = null; } - MapEventListener> eventListener() { - return eventListener == null ? MapEventListeners.NOP : eventListener; - } - - void checkKey(Object key) { + final void checkKey(Object key) { if (!kClass.isInstance(key)) { // key.getClass will cause NPE exactly as needed throw new ClassCastException("Key must be a " + kClass.getName() + @@ -330,25 +328,136 @@ class VanillaChronicleMap, } } - void checkValue(Object value) { + final void checkValue(Object value) { if (vClass != Void.class && !vClass.isInstance(value)) { throw new ClassCastException("Value must be a " + vClass.getName() + " but was a " + value.getClass()); } } + final void put(Bytes entry, Bytes output) { + put(entry, output, true); + } + + final void putIfAbsent(Bytes entry, Bytes output) { + put(entry, output, false); + } + + private void put(Bytes entry, Bytes output, boolean replaceIfPresent) { + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); + ReadValueToOutputBytes readValueToOutputBytes = segmentState.readValueToOutputBytes; + readValueToOutputBytes.reuse(valueSizeMarshaller, output); + long keySize = keySizeMarshaller.readSize(entry); + GetRemoteBytesValueInterops getRemoteBytesValueInterops = + segmentState.getRemoteBytesValueInterops; + MultiStoreBytes value = getRemoteBytesValueInterops.getValueBytes( + entry, entry.position() + keySize); + getRemoteBytesValueInterops.valueSizeMarshaller(valueSizeMarshaller); + putBytes(copies, segmentState, entry, keySize, getRemoteBytesValueInterops, value, + replaceIfPresent, readValueToOutputBytes); + } + + void putBytes(ThreadLocalCopies copies, SegmentState segmentState, Bytes key, long keySize, + GetRemoteBytesValueInterops getRemoteBytesValueInterops, MultiStoreBytes value, + boolean replaceIfPresent, ReadValue readValue) { + put(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, key, keySize, keyBytesToInstance, + getRemoteBytesValueInterops, value, valueBytesToInstance, + replaceIfPresent, readValue, false); + } + + final void put(Bytes entry) { + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); + long keySize = keySizeMarshaller.readSize(entry); + GetRemoteBytesValueInterops getRemoteBytesValueInterops = + segmentState.getRemoteBytesValueInterops; + MultiStoreBytes value = getRemoteBytesValueInterops.getValueBytes( + entry, entry.position() + keySize); + getRemoteBytesValueInterops.valueSizeMarshaller(valueSizeMarshaller); + ReadValueToBytes readValueToLazyBytes = segmentState.readValueToLazyBytes; + readValueToLazyBytes.valueSizeMarshaller(valueSizeMarshaller); + put(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, entry, keySize, keyBytesToInstance, + getRemoteBytesValueInterops, value, valueBytesToInstance, + true, readValueToLazyBytes, true); + } + + static abstract class ReadValueToBytes implements ReadValue { + private SizeMarshaller valueSizeMarshaller; + abstract Bytes bytes(long valueSize); + + void valueSizeMarshaller(SizeMarshaller valueSizeMarshaller) { + this.valueSizeMarshaller = valueSizeMarshaller; + } + + @Override + public Bytes readValue(@NotNull ThreadLocalCopies copies, Bytes entry, Bytes usingBytes, + long valueSize) { + usingBytes = bytes(valueSize); + usingBytes.writeBoolean(false); + valueSizeMarshaller.writeSize(usingBytes, valueSize); + usingBytes.write(entry, entry.position(), valueSize); + return usingBytes; + } + } + + private static class ReadValueToOutputBytes extends ReadValueToBytes { + private Bytes outputBytes; + + void reuse(SizeMarshaller valueSizeMarshaller, Bytes outputBytes) { + valueSizeMarshaller(valueSizeMarshaller); + this.outputBytes = outputBytes; + } + + @Override + Bytes bytes(long valueSize) { + return outputBytes; + } + + @Override + public Bytes readNull() { + outputBytes.writeBoolean(true); + return null; + } + } + + private static class ReadValueToLazyBytes extends ReadValueToBytes { + DirectBytes lazyBytes; + @Override + Bytes bytes(long valueSize) { + if (lazyBytes != null) { + if (lazyBytes.capacity() < valueSize) { + DirectStore store = (DirectStore) lazyBytes.store(); + store.resize(valueSize, false); + lazyBytes = store.bytes(); + } + lazyBytes.clear(); + return lazyBytes; + } + return lazyBytes = new DirectStore(valueSize).bytes(); + } + + @Override + public Bytes readNull() { + return null; + } + } + @Override - public V put(K key, V value) { + public final V put(K key, V value) { return put0(key, value, true); } @Override - public V putIfAbsent(K key, V value) { + public final V putIfAbsent(@NotNull K key, V value) { return put0(key, value, false); } - private V put0(@net.openhft.lang.model.constraints.NotNull K key, V value, - boolean replaceIfPresent) { + V put0(@NotNull K key, V value, boolean replaceIfPresent) { checkKey(key); checkValue(value); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); @@ -356,32 +465,85 @@ class VanillaChronicleMap, copies = metaKeyInteropProvider.getCopies(copies); MKI metaKeyInterop = metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + long keySize = metaKeyInterop.size(keyInterop, key); + return put(copies, null, metaKeyInterop, keyInterop, key, keySize, keyIdentity(), + this, value, valueIdentity(), replaceIfPresent, this, putReturnsNull); + } + + , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV put(ThreadLocalCopies copies, SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + boolean replaceIfPresent, ReadValue readValue, boolean resultUnused) { long hash = metaKeyInterop.hash(keyInterop, key); int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); - return segments[segmentNum].put(copies, metaKeyInterop, keyInterop, key, value, segmentHash, - replaceIfPresent); + return segments[segmentNum].put(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + getValueInterops, value, toValue, + segmentHash, replaceIfPresent, readValue, resultUnused); } enum LockType {READ_LOCK, WRITE_LOCK} @Override - public V get(Object key) { - - try (Context entry = lookupUsing((K) key, null, - LockType.READ_LOCK, false)) { - return entry.value(); - } + @SuppressWarnings("unchecked") + public final V get(Object key) { + checkKey(key); + return lookupUsing((K) key, null, false); } @Override - public V getUsing(K key, V usingValue) { + public final V getUsing(K key, V usingValue) { + checkKey(key); + return lookupUsing(key, usingValue, false); + } - try (Context entry = lookupUsing(key, usingValue, - LockType.READ_LOCK, false)) { - return entry.value(); - } + @Override + public final V acquireUsing(@NotNull K key, V usingValue) { + checkKey(key); + return lookupUsing(key, usingValue, true); + } + + final void get(Bytes key, Bytes output) { + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); + ReadValueToOutputBytes readValueToOutputBytes = segmentState.readValueToOutputBytes; + readValueToOutputBytes.reuse(valueSizeMarshaller, output); + lookupUsing(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, key, keySizeMarshaller.readSize(key), + keyBytesToInstance, readValueToOutputBytes, null, outputValueBytesToInstance, false); + } + + private V lookupUsing(K key, V usingValue, boolean create) { + ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); + copies = metaKeyInteropProvider.getCopies(copies); + MKI metaKeyInterop = + metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + long keySize = metaKeyInterop.size(keyInterop, key); + return lookupUsing(copies, null, metaKeyInterop, keyInterop, key, keySize, keyIdentity(), + this, usingValue, valueIdentity(), create); + } + + private , RV> + RV lookupUsing(ThreadLocalCopies copies, SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, + boolean create) { + long hash = metaKeyInterop.hash(keyInterop, key); + int segmentNum = getSegment(hash); + long segmentHash = segmentHash(hash); + Segment segment = segments[segmentNum]; + return segment.acquire(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + readValue, usingValue, toValue, segmentHash, create); } @Override @@ -426,7 +588,8 @@ class VanillaChronicleMap, } @Override - public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + public V get(long timeout, @NotNull TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { return v; } }; @@ -458,7 +621,8 @@ class VanillaChronicleMap, } @Override - public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + public V get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { return v; } }; @@ -490,7 +654,8 @@ class VanillaChronicleMap, } @Override - public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + public V get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { return v; } }; @@ -675,7 +840,8 @@ class VanillaChronicleMap, } @Override - public Object unmarshal(HierarchicalStreamReader reader, UnmarshallingContext unmarshallingContext) { + public Object unmarshal(HierarchicalStreamReader reader, + UnmarshallingContext unmarshallingContext) { String value = reader.getValue(); buffer.clear(); append(value); @@ -734,52 +900,39 @@ class VanillaChronicleMap, @NotNull @Override - public WriteContext acquireUsingLocked(@NotNull K key, @NotNull V usingValue) { - return this.lookupUsing(key, usingValue, LockType.WRITE_LOCK, true); + public final WriteContext acquireUsingLocked(@NotNull K key, @NotNull V usingValue) { + return lookupUsing(key, usingValue, LockType.WRITE_LOCK, true); } @NotNull @Override - public ReadContext getUsingLocked(@NotNull K key, @NotNull V usingValue) { - return this.lookupUsing(key, usingValue, LockType.READ_LOCK, true); + public final ReadContext getUsingLocked(@NotNull K key, @NotNull V usingValue) { + return lookupUsing(key, usingValue, LockType.READ_LOCK, true); } - - @Override - public V acquireUsing(@NotNull K key, V usingValue) { - V value = getUsing(key, usingValue); - - if (value != null) - return value; - - try (WriteContext kvContext = - lookupUsing(key, usingValue, LockType.WRITE_LOCK, false)) { - - if (!isNativeValueClass) - kvContext.dontPutOnClose(); - - return kvContext.value(); - } - } - - - T lookupUsing(K key, V usingValue, LockType lockType, boolean mustReuseValue) { + private T lookupUsing( + K key, V usingValue, LockType lockType, boolean mustReuseValue) { checkKey(key); - ThreadLocalCopies copies = keyInteropProvider.getCopies(null); + + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); + KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); - copies = metaKeyInteropProvider.getCopies(copies); MKI metaKeyInterop = metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + long keySize = metaKeyInterop.size(keyInterop, key); long hash = metaKeyInterop.hash(keyInterop, key); int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); Segment segment = segments[segmentNum]; - MutableLockedEntry lock = (lockType == LockType.WRITE_LOCK) - ? segment.writeLock() - : segment.readLock(); + MutableLockedEntry lock = (lockType == LockType.WRITE_LOCK) + ? segment.writeLock(segmentState) + : segment.readLock(segmentState); - V v = segment.acquireWithoutLock(copies, metaKeyInterop, keyInterop, key, usingValue, + V v = segment.acquireWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, keyIdentity(), + this, usingValue, valueIdentity(), segmentHash, lockType == LockType.WRITE_LOCK); checkReallyUsingValue(usingValue, mustReuseValue, v); @@ -787,6 +940,7 @@ class VanillaChronicleMap, lock.copies = copies; lock.metaKeyInterop = metaKeyInterop; lock.keyInterop = keyInterop; + lock.keySize = keySize; lock.segmentHash = segmentHash; if (!isNativeValueClass && lockType == LockType.WRITE_LOCK && v == null) @@ -796,61 +950,204 @@ class VanillaChronicleMap, lock.key(key); return (T) lock; - } - void checkReallyUsingValue(V usingValue, boolean mustReuseValue, V returnedValue) { + private void checkReallyUsingValue(V usingValue, boolean mustReuseValue, V returnedValue) { if (mustReuseValue && returnedValue != null && returnedValue != usingValue) - throw new IllegalArgumentException("acquireUsingLocked/getUsingLocked MUST reuse given " + - "values. Given value" + usingValue + " cannot be reused to read " + returnedValue); + throw new IllegalArgumentException( + "acquireUsingLocked/getUsingLocked MUST reuse given " + + "values. Given value" + usingValue + + " cannot be reused to read " + returnedValue); } @Override - public boolean containsKey(final Object k) { + public final Object toInstance(@NotNull ThreadLocalCopies copies, Object instance, long size) { + return instance; + } + + @SuppressWarnings("unchecked") + final InstanceOrBytesToInstance keyIdentity() { + return this; + } + + @SuppressWarnings("unchecked") + final InstanceOrBytesToInstance valueIdentity() { + return this; + } + + transient InstanceOrBytesToInstance keyBytesToInstance = + new InstanceOrBytesToInstance() { + @Override + public K toInstance(@NotNull ThreadLocalCopies copies, Bytes keyBytes, long size) { + assertNotNull(copies); + BytesReader keyReader = keyReaderProvider.get(copies, originalKeyReader); + return keyReader.read(keyBytes, size); + } + }; + + transient InstanceOrBytesToInstance valueBytesToInstance = + new InstanceOrBytesToInstance() { + @Override + public V toInstance(@NotNull ThreadLocalCopies copies, Bytes valueBytes, long size) { + return readValue(copies, valueBytes, null, size); + } + }; + + private transient InstanceOrBytesToInstance outputValueBytesToInstance = + new InstanceOrBytesToInstance() { + @Override + public V toInstance(@NotNull ThreadLocalCopies copies, Bytes outputBytes, long size) { + outputBytes.position(outputBytes.position() - size); + return readValue(copies, outputBytes, null, size); + } + }; + + @Override + public final boolean containsKey(final Object k) { checkKey(k); + @SuppressWarnings("unchecked") K key = (K) k; ThreadLocalCopies copies = keyInteropProvider.getCopies(null); KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); copies = metaKeyInteropProvider.getCopies(copies); MKI metaKeyInterop = metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + return containsKey(copies, metaKeyInterop, keyInterop, key, + metaKeyInterop.size(keyInterop, key)); + } + + final boolean containsKey(Bytes key) { + return containsKey(null, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, + key, keySizeMarshaller.readSize(key)); + } + + + private > + boolean containsKey(ThreadLocalCopies copies, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize) { long hash = metaKeyInterop.hash(keyInterop, key); int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); - return segments[segmentNum].containsKey(keyInterop, metaKeyInterop, key, segmentHash); + Segment segment = segments[segmentNum]; + return segment.containsKey(copies, metaKeyInterop, keyInterop, key, keySize, segmentHash); } @Override public void clear() { - for (Segment segment : segments) + for (Segment segment : segments) { segment.clear(); + } } @NotNull @Override - public Set> entrySet() { - return (entrySet != null) ? entrySet : (entrySet = new EntrySet()); + public final Set> entrySet() { + return (entrySet != null) ? entrySet : (entrySet = newEntrySet()); } - - /** - * @throws NullPointerException if the specified key is null - */ - @Override - public V remove(final Object key) { - return removeIfValueIs(key, null); + Set> newEntrySet() { + return new EntrySet(); } /** * @throws NullPointerException if the specified key is null */ @Override - public boolean remove(final Object key, final Object value) { + @SuppressWarnings("unchecked") + public final V remove(final Object key) { + return (V) removeIfValueIs(key, null); + } + + /** + * @throws NullPointerException if the specified key is null + */ + @Override + @SuppressWarnings("unchecked") + public final boolean remove(@NotNull final Object key, final Object value) { if (value == null) return false; // CHM compatibility; I would throw NPE - return removeIfValueIs(key, (V) value) != null; + return (Boolean) removeIfValueIs(key, (V) value); } + @Override + public final VI getValueInterop(@NotNull ThreadLocalCopies copies) { + assertNotNull(copies); + return valueInteropProvider.get(copies, originalValueInterop); + } + + static class GetRemoteBytesValueInterops + implements GetValueInterops, + MetaBytesInterop { + private static final long serialVersionUID = 0L; + private final MultiStoreBytes valueBytes = new MultiStoreBytes(); + + private Bytes remoteEntryBytes; + private long offset; + private SizeMarshaller valueSizeMarshaller; + + private long valueSize; + + MultiStoreBytes getValueBytes(Bytes remoteEntryBytes, long offset) { + this.remoteEntryBytes = remoteEntryBytes; + this.offset = offset; + this.valueSizeMarshaller = null; + valueSize = -1L; + return valueBytes; + } + + void valueSize(long valueSize) { + this.valueSize = valueSize; + } + + void valueSizeMarshaller(SizeMarshaller valueSizeMarshaller) { + this.valueSizeMarshaller = valueSizeMarshaller; + } + + @Override + public GetRemoteBytesValueInterops getMetaValueInterop( + @NotNull ThreadLocalCopies copies, Void valueInterop, MultiStoreBytes value) { + value.setBytesOffset(remoteEntryBytes, offset); + if (valueSize == -1L) + valueSize = valueSizeMarshaller.readSize(value); + value.limit(value.position() + valueSize); + return this; + } + + @Override + public Void getValueInterop(@NotNull ThreadLocalCopies copies) { + return null; + } + + @Override + public boolean startsWith(Void interop, Bytes bytes, Bytes value) { + return bytes.startsWith(value); + } + + @Override + public long hash(Void interop, Bytes value) { + return BytesBytesInterop.INSTANCE.hash(value); + } + + @Override + public long size(Void writer, Bytes value) { + return valueSize; + } + + @Override + public void write(Void writer, Bytes bytes, Bytes value) { + bytes.write(value); + } + } + + @Override + public final MVI getMetaValueInterop( + @NotNull ThreadLocalCopies copies, VI valueInterop, V value) { + assertNotNull(copies); + return metaValueInteropProvider.get( + copies, originalMetaValueInterop, valueInterop, value); + } /** * removes ( if there exists ) an entry from the map, if the {@param key} and {@param @@ -861,67 +1158,106 @@ class VanillaChronicleMap, * @param expectedValue null if not required * @return true if and entry was removed */ - V removeIfValueIs(final Object k, final V expectedValue) { + private Object removeIfValueIs(final Object k, final V expectedValue) { checkKey(k); + @SuppressWarnings("unchecked") K key = (K) k; ThreadLocalCopies copies = keyInteropProvider.getCopies(null); KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); copies = metaKeyInteropProvider.getCopies(copies); MKI metaKeyInterop = metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + long keySize = metaKeyInterop.size(keyInterop, key); + return remove(copies, null, metaKeyInterop, keyInterop, key, keySize, + keyIdentity(), this, expectedValue, valueIdentity(), removeReturnsNull); + } + + final void removeKeyAsBytes(Bytes key) { + remove(null, null, DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, key, keySizeMarshaller.readSize(key), + keyBytesToInstance, null, null, null, true); + } + + // for boolean remove(Object key, Object value); + final boolean removeWithValue(Bytes entry) { + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); + long keySize = keySizeMarshaller.readSize(entry); + GetRemoteBytesValueInterops getRemoveBytesValueInterops = + segmentState.getRemoteBytesValueInterops; + MultiStoreBytes value = getRemoveBytesValueInterops.getValueBytes( + entry, entry.position() + keySize); + getRemoveBytesValueInterops.valueSizeMarshaller(valueSizeMarshaller); + return (Boolean) remove(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, entry, keySize, keyBytesToInstance, + getRemoveBytesValueInterops, value, valueBytesToInstance, false); + } + + + private , + VB, VBI, MVBI extends MetaBytesInterop> + Object remove(ThreadLocalCopies copies, SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB expectedValue, + InstanceOrBytesToInstance toValue, + boolean resultUnused) { long hash = metaKeyInterop.hash(keyInterop, key); int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); - return segments[segmentNum].remove(copies, metaKeyInterop, keyInterop, key, expectedValue, - segmentHash); + Segment segment = segments[segmentNum]; + return segment.remove(copies, segmentState, metaKeyInterop, keyInterop, key, keySize, + toKey, getValueInterops, expectedValue, + toValue, segmentHash, resultUnused); } /** * @throws NullPointerException if any of the arguments are null */ @Override - public boolean replace(final K key, final V oldValue, final V newValue) { + public final boolean replace(@NotNull K key, @NotNull V oldValue, @NotNull V newValue) { checkValue(oldValue); - return oldValue.equals(replaceIfValueIs(key, oldValue, newValue)); + return (Boolean) replaceIfValueIs(key, oldValue, newValue); } - /** * @throws NullPointerException if the specified key or value is null */ @Override - public V replace(final K key, final V value) { - return replaceIfValueIs(key, null, value); + @SuppressWarnings("unchecked") + public final V replace(@NotNull final K key, @NotNull final V value) { + return (V) replaceIfValueIs(key, null, value); } @Override - public long longSize() { + public final long longSize() { long result = 0L; for (final Segment segment : this.segments) { - result += segment.getSize(); + result += segment.size(); } return result; } @Override - public int size() { + public final int size() { long size = longSize(); return size > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) size; } /** - * replace the value in a map, only if the existing entry equals {@param existingValue} + * replace the value in a map, only if the existing entry equals {@param expectedValue} * * @param key the key into the map - * @param existingValue the expected existing value in the map ( could be null when we don't - * wish to do this check ) + * @param expectedValue the expected existing value in the map + * ( could be null when we don't wish to do this check ) * @param newValue the new value you wish to store in the map * @return the value that was replaced */ - V replaceIfValueIs(@net.openhft.lang.model.constraints.NotNull final K key, - final V existingValue, final V newValue) { + private Object replaceIfValueIs(@NotNull final K key, + final V expectedValue, final V newValue) { checkKey(key); checkValue(newValue); ThreadLocalCopies copies = keyInteropProvider.getCopies(null); @@ -929,100 +1265,253 @@ class VanillaChronicleMap, copies = metaKeyInteropProvider.getCopies(copies); MKI metaKeyInterop = metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); + long keySize = metaKeyInterop.size(keyInterop, key); + return replace(copies, null, + metaKeyInterop, keyInterop, key, keySize, keyIdentity(), + this, expectedValue, this, newValue, this, valueIdentity()); + } + + final void replaceKV(Bytes keyAndNewValue, Bytes output) { + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); + ReadValueToOutputBytes readValueToOutputBytes = segmentState.readValueToOutputBytes; + readValueToOutputBytes.reuse(valueSizeMarshaller, output); + + long keySize = keySizeMarshaller.readSize(keyAndNewValue); + GetRemoteBytesValueInterops getRemoteBytesValueInterops = + segmentState.getRemoteBytesValueInterops; + MultiStoreBytes value = getRemoteBytesValueInterops.getValueBytes( + keyAndNewValue, keyAndNewValue.position() + keySize); + getRemoteBytesValueInterops.valueSizeMarshaller(valueSizeMarshaller); + replace(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, keyAndNewValue, keySize, keyBytesToInstance, + null, null, getRemoteBytesValueInterops, value, + readValueToOutputBytes, valueBytesToInstance); + } + + // for boolean replace(K key, V oldValue, V newValue); + final void replaceWithOldAndNew(Bytes entryAndNewValue, Bytes output) { + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); + + long keySize = keySizeMarshaller.readSize(entryAndNewValue); + long keyPosition = entryAndNewValue.position(); + GetRemoteBytesValueInterops getExpectedValueInterops = + segmentState.getRemoteBytesValueInterops; + MultiStoreBytes expectedValue = getExpectedValueInterops.getValueBytes( + entryAndNewValue, keyPosition + keySize); + getExpectedValueInterops.valueSizeMarshaller(valueSizeMarshaller); + + entryAndNewValue.position(keyPosition + keySize); + long expectedValueSize = valueSizeMarshaller.readSize(entryAndNewValue); + GetRemoteBytesValueInterops getNewValueInterops = segmentState.getRemoteBytesValueInterops2; + MultiStoreBytes newValue = getNewValueInterops.getValueBytes(entryAndNewValue, + entryAndNewValue.position() + expectedValueSize); + getNewValueInterops.valueSizeMarshaller(valueSizeMarshaller); + + entryAndNewValue.position(keyPosition); + + output.writeBoolean((Boolean) replace(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, entryAndNewValue, keySize, keyBytesToInstance, + getExpectedValueInterops, expectedValue, getNewValueInterops, newValue, + null, valueBytesToInstance)); + } + + private , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + Object replace( + ThreadLocalCopies copies, SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getExpectedValueInterops, VB existingValue, + GetValueInterops getNewValueInterops, VB newValue, + ReadValue readValue, InstanceOrBytesToInstance toValue) { long hash = metaKeyInterop.hash(keyInterop, key); int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); - return segments[segmentNum].replace(copies, metaKeyInterop, keyInterop, key, existingValue, - newValue, segmentHash); + return segments[segmentNum].replace(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + getExpectedValueInterops, existingValue, getNewValueInterops, newValue, + readValue, toValue, segmentHash); } /** * For testing */ - void checkConsistency() { + final void checkConsistency() { for (Segment segment : segments) { segment.checkConsistency(); } } - long readValueSize(Bytes entry) { + final long readValueSize(Bytes entry) { long valueSize = valueSizeMarshaller.readSize(entry); alignment.alignPositionAddr(entry); return valueSize; } - protected MultiStoreBytes acquireTmpBytes() { - MultiStoreBytes b = tmpBytes.get(); - if (b == null) - tmpBytes.set(b = new MultiStoreBytes()); - return b; + final V readValue(@NotNull ThreadLocalCopies copies, MultiStoreBytes entry, V value) { + return readValue(copies, entry, value, readValueSize(entry)); } - void replace(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); + @Override + public final V readValue( + @NotNull ThreadLocalCopies copies, Bytes entry, V value, long valueSize) { + assertNotNull(copies); + BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); + return valueReader.read(entry, valueSize, value); } - void isEmpty(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); + @Override + public final V readNull() { + return null; } - void containsKey(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); + final void putAll(Bytes entries) { + long numberOfEntries = entries.readStopBit(); + long entryPosition = entries.position(); + while (numberOfEntries --> 0) { + long keySize = keySizeMarshaller.readSize(entries); + entries.skip(keySize); + long valueSize = valueSizeMarshaller.readSize(entries); + long nextEntryPosition = entries.position() + valueSize; + entries.position(entryPosition); + put(entries); + entryPosition = nextEntryPosition; + } } - void containsValue(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - void get(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - void putAll(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - void put(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - void putIfAbsent(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - void replaceKV(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - // for boolean replace(K key, V oldValue, V newValue); - void replaceWithOldAndNew(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - // for boolean remove(Object key, Object value); - void removeWithValue(Bytes reader, Bytes writer) { - throw new UnsupportedOperationException("todo"); - } - - public void putAll(Bytes reader) { - throw new UnsupportedOperationException("todo"); - } - - long[] segmentSizes() { + /** + * For testing + */ + final long[] segmentSizes() { long[] sizes = new long[segments.length]; for (int i = 0; i < segments.length; i++) - sizes[i] = segments[i].getSize(); + sizes[i] = segments[i].size(); return sizes; } - public void put(Bytes reader) { - throw new UnsupportedOperationException("todo"); + void onRemove(Segment segment, long pos) { + // do nothing } - public void removeWithBytes(Bytes reader) { - throw new UnsupportedOperationException("todo"); + void onPut(Segment segment, long pos) { + // do nothing } + void onRelocation(Segment segment, long pos) { + // do nothing + } + + static final class SegmentState implements StatefulCopyable, AutoCloseable { + private static final Provider segmentStateProvider = + Provider.of(SegmentState.class); + private static final SegmentState originalSegmentState = new SegmentState(0); + + static @NotNull ThreadLocalCopies getCopies(ThreadLocalCopies copies) { + return segmentStateProvider.getCopies(copies); + } + + static SegmentState get(@NotNull ThreadLocalCopies copies) { + assertNotNull(copies); + return segmentStateProvider.get(copies, originalSegmentState).get(); + } + + // inner state + private final int depth; + private boolean used; + private SegmentState next; + + // segment state (reusable objects/fields) + private final ReadLocked readLocked = new ReadLocked(this); + private final NativeWriteLocked nativeWriteLocked = new NativeWriteLocked(this); + private final HeapWriteLocked heapWriteLocked = new HeapWriteLocked(this); + + final MultiStoreBytes tmpBytes = new MultiStoreBytes(); + final GetRemoteBytesValueInterops getRemoteBytesValueInterops = + new GetRemoteBytesValueInterops(); + final GetRemoteBytesValueInterops getRemoteBytesValueInterops2 = + new GetRemoteBytesValueInterops(); + final ReadValueToOutputBytes readValueToOutputBytes = new ReadValueToOutputBytes(); + final ReadValueToBytes readValueToLazyBytes = new ReadValueToLazyBytes(); + + long pos; + long valueSizePos; + + private SegmentState(int depth) { + if (depth > (1 << 16)) + throw new IllegalStateException("More than " + (1 << 16) + + " nested ChronicleMap contexts are not supported. Very probable that you " + + "simply forgot to close context somewhere (recommended to use " + + "try-with-resources statement). " + + "Otherwise this is a ChronicleMap bug, please report with this " + + "stack trace on https://github.com/OpenHFT/Chronicle-Map/issues"); + this.depth = depth; + } + + private SegmentState get() { + if (!used) { + used = true; + return this; + } + if (next != null) + return next.get(); + return next = new SegmentState(depth + 1); + } + + @Override + public Object stateIdentity() { + return SegmentState.class; + } + + @Override + public SegmentState copy() { + return new SegmentState(0); + } + + @Override + public void close() { + used = false; + } + + , + V, VI, MVI extends MetaBytesInterop> + ReadLocked readLocked(VanillaChronicleMap.Segment segment) { + readLocked.segment = segment; + readLocked.map = segment.map(); + return readLocked; + } + + , + V, VI, MVI extends MetaBytesInterop> + WriteLocked nativeWriteLocked(VanillaChronicleMap.Segment segment) { + nativeWriteLocked.segment = segment; + nativeWriteLocked.map = segment.map(); + return nativeWriteLocked; + } + + , + V, VI, MVI extends MetaBytesInterop> + WriteLocked heapWriteLocked(VanillaChronicleMap.Segment segment) { + heapWriteLocked.segment = segment; + heapWriteLocked.map = segment.map(); + return heapWriteLocked; + } + } + + static void segmentStateNotNullImpliesCopiesNotNull( + @Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState) { + assert copies != null || segmentState == null; // segmentState != null -> copies != null + } + + static void expectedValueNotNullImpliesBooleanResult( + @Nullable Object expectedValue, boolean booleanResult) { + assert booleanResult || expectedValue == null; + } + + // these methods should be package local, not public or private. class Segment implements SharedSegment { /* @@ -1044,15 +1533,9 @@ class VanillaChronicleMap, private MultiMap hashLookup; private long nextPosToSearchFrom = 0L; - final ThreadLocal> readUnlockTL = new ThreadLocal<>(); - - /** - * close for native values does not call put on close - */ - final WriteLocked writeUnlock = isNativeValueClass - ? new NativeWriteLocked() - : new HeapWriteLocked(); - + private VanillaChronicleMap map() { + return VanillaChronicleMap.this; + } /** * @param index the index of this segment held by the map @@ -1063,7 +1546,7 @@ class VanillaChronicleMap, this.index = index; long start = bytes.startAddr(); - createHashLookups(start); + hashLookup = createMultiMap(start); start += CACHE_LINES.align(sizeOfMultiMap() + sizeOfMultiMapBitSet(), BYTES) * multiMapsPerSegment(); final NativeBytes bsBytes = new NativeBytes(ms.objectSerializer(), @@ -1076,16 +1559,11 @@ class VanillaChronicleMap, assert bytes.capacity() >= entriesOffset + entriesPerSegment * entrySize; } - void createHashLookups(long start) { - hashLookup = createMultiMap(start); - } - - public MultiMap getHashLookup() { + final MultiMap hashLookup() { return hashLookup; } - - MultiMap createMultiMap(long start) { + private MultiMap createMultiMap(long start) { final NativeBytes multiMapBytes = new NativeBytes(new VanillaBytesMarshallerFactory(), start, start = start + sizeOfMultiMap(), null); @@ -1097,7 +1575,7 @@ class VanillaChronicleMap, return multiMapFactory.create(multiMapBytes, sizeOfMultiMapBitSetBytes); } - public int getIndex() { + public final int getIndex() { return index; } @@ -1109,41 +1587,44 @@ class VanillaChronicleMap, /** * increments the size by one */ - void incrementSize() { + final void incrementSize() { this.segmentHeader.addLong(SIZE_OFFSET, 1L); } - void resetSize() { + private void resetSize() { this.segmentHeader.writeLong(SIZE_OFFSET, 0L); } /** * decrements the size by one */ - void decrementSize() { + final void decrementSize() { this.segmentHeader.addLong(SIZE_OFFSET, -1L); } /** * reads the the number of entries in this segment */ - long getSize() { + private long size() { // any negative value is in error state. return max(0L, this.segmentHeader.readVolatileLong(SIZE_OFFSET)); } - - public ReadLocked readLock() throws IllegalStateException { + @Override + public final ReadLocked readLock( + @Nullable SegmentState segmentState) { while (true) { final boolean success = segmentHeader.tryRWReadLock(LOCK_OFFSET, lockTimeOutNS); if (success) { - ReadLocked readLocked = readUnlockTL.get(); - if (readLocked == null) - readUnlockTL.set(readLocked = new TLReadLocked()); - return readLocked; + if (segmentState != null) { + return segmentState.readLocked(this); + } else { + return null; + } } if (currentThread().isInterrupted()) { - throw new IllegalStateException(new InterruptedException("Unable to obtain lock, interrupted")); + throw new IllegalStateException( + new InterruptedException("Unable to obtain lock, interrupted")); } else { errorListener.onLockTimeout(segmentHeader.threadIdForLockLong(LOCK_OFFSET)); segmentHeader.resetLockLong(LOCK_OFFSET); @@ -1151,13 +1632,26 @@ class VanillaChronicleMap, } } - public WriteLocked writeLock() throws IllegalStateException { + @Override + public final WriteLocked writeLock( + @Nullable SegmentState segmentState) { while (true) { final boolean success = segmentHeader.tryRWWriteLock(LOCK_OFFSET, lockTimeOutNS); - if (success) return writeUnlock; + if (success) { + if (segmentState != null) { + if (isNativeValueClass) { + return segmentState.nativeWriteLocked(this); + } else { + return segmentState.heapWriteLocked(this); + } + } else { + return null; + } + } if (currentThread().isInterrupted()) { - throw new IllegalStateException(new InterruptedException("Unable to obtain lock, interrupted")); + throw new IllegalStateException( + new InterruptedException("Unable to obtain lock, interrupted")); } else { errorListener.onLockTimeout(segmentHeader.threadIdForLockLong(LOCK_OFFSET)); segmentHeader.resetLockLong(LOCK_OFFSET); @@ -1165,7 +1659,8 @@ class VanillaChronicleMap, } } - public void readUnlock() { + @Override + public final void readUnlock() { try { segmentHeader.unlockRWReadLock(LOCK_OFFSET); } catch (IllegalMonitorStateException e) { @@ -1173,7 +1668,8 @@ class VanillaChronicleMap, } } - public void writeUnlock() { + @Override + public final void writeUnlock() { try { segmentHeader.unlockRWWriteLock(LOCK_OFFSET); } catch (IllegalMonitorStateException e) { @@ -1181,38 +1677,24 @@ class VanillaChronicleMap, } } - public long offsetFromPos(long pos) { + @Override + public final long offsetFromPos(long pos) { return entriesOffset + pos * entrySize; } - //TODO get rid of this function, integral division is slow - long posFromOffset(long offset) { - return (offset - entriesOffset) / entrySize; - } - - - public MultiStoreBytes entry(long offset) { - return reuse(acquireTmpBytes(), offset); - } - - private MultiStoreBytes reuse(MultiStoreBytes entry, long offset) { - offset += metaDataBytes; + final MultiStoreBytes reuse(MultiStoreBytes entry, long offset) { entry.setBytesOffset(bytes, offset); + entry.position(metaDataBytes); return entry; } - long entryStartAddr(long offset) { - // entry.address() points to "needed" start addr + metaDataBytes - return bytes.startAddr() + offset; - } - - private long entrySize(long keySize, long valueSize) { + long entrySize(long keySize, long valueSize) { return alignment.alignAddr(metaDataBytes + keySizeMarshaller.sizeEncodingSize(keySize) + keySize + valueSizeMarshaller.sizeEncodingSize(valueSize)) + valueSize; } - int inBlocks(long sizeInBytes) { + final int inBlocks(long sizeInBytes) { if (sizeInBytes <= entrySize) return 1; // int division is MUCH faster than long on Intel CPUs @@ -1222,172 +1704,323 @@ class VanillaChronicleMap, return (int) (sizeInBytes / entrySize) + 1; } - V acquireWithoutLock(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V usingValue, - long hash2, boolean create) { - long keySize = metaKeyInterop.size(keyInterop, key); - MultiStoreBytes entry = acquireTmpBytes(); - long offset = searchKey(keyInterop, metaKeyInterop, key, keySize, hash2, entry, - hashLookup); - if (offset >= 0L) { - return onKeyPresentOnAcquire(copies, key, usingValue, offset, entry); - } else { - if (!create) - return null; - if (defaultValueProvider != null) { - V defaultValue = defaultValueProvider.get(key); - copies = valueWriterProvider.getCopies(copies); - VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = metaValueWriterProvider.getCopies(copies); - MetaBytesWriter metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, defaultValue); - - offset = putEntry(metaKeyInterop, keyInterop, key, keySize, - metaValueWriter, valueWriter, defaultValue); - } else { - if (prepareValueBytesAsWriter != null) { - offset = putEntry(metaKeyInterop, keyInterop, key, keySize, - prepareValueBytesAsWriter, null, key); - } else { - throw new IllegalStateException("To call acquire*() methods, " + - "you should specify either default value, default value provider " + - "or prepareBytes strategy during map building"); - } - } - return onKeyAbsentOnAcquire(copies, key, keySize, usingValue, offset); - } - - } - - long searchKey(KI keyInterop, MKI metaKeyInterop, K key, long keySize, long hash2, - MultiStoreBytes entry, MultiMap hashLookup) { - hashLookup.startSearch(hash2); - for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { - long offset = offsetFromPos(pos); - reuse(entry, offset); - if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) - continue; - // key is found - entry.skip(keySize); - return offset; - } - // key is not found - return -1L; - } - - V onKeyPresentOnAcquire(ThreadLocalCopies copies, K key, V usingValue, long offset, - MultiStoreBytes entry) { - skipReplicationBytes(entry); - V v = readValue(copies, entry, usingValue); - notifyGet(offset, key, v); - return v; - } - - V onKeyAbsentOnAcquire(ThreadLocalCopies copies, K key, long keySize, V usingValue, - long offset) { - incrementSize(); - MultiStoreBytes entry = entry(offset); - entry.skip(keySizeMarshaller.sizeEncodingSize(keySize) + keySize); - skipReplicationBytes(entry); - V v = readValue(copies, entry, usingValue); - notifyPut(offset, true, key, v, null, posFromOffset(offset)); - notifyGet(offset, key, v); - return v; - } - - void skipReplicationBytes(Bytes bytes) { - // don't skip - } - - V put(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V value, - long hash2, boolean replaceIfPresent) { - writeLock(); + private , RV> + RV acquire( + @Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, + long hash2, boolean create) { + readLock(null); try { - return putWithoutLock(copies, metaKeyInterop, keyInterop, key, value, hash2, replaceIfPresent); + return acquireWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + readValue, usingValue, toValue, hash2, create); } finally { + if (segmentState != null) + segmentState.close(); + readUnlock(); + } + } + + , RV> + RV acquireWithoutLock( + @Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, + long hash2, boolean create) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); + SegmentState localSegmentState = segmentState; + try { + MultiStoreBytes entry = null; + MultiMap hashLookup = this.hashLookup; + hashLookup.startSearch(hash2); + for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { + long offset = offsetFromPos(pos); + if (entry == null) { + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); + localSegmentState = SegmentState.get(copies); + } + entry = localSegmentState.tmpBytes; + } + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) + continue; + // key is found + entry.skip(keySize); + return readValueAndNotifyGet(copies, key, keySize, toKey, + readValue, usingValue, toValue, entry); + } + // key is not found + if (!create) + return readValue.readNull(); + if (entry == null) { + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); // after this, copies is not null + localSegmentState = SegmentState.get(copies); + } + entry = localSegmentState.tmpBytes; + } + return createEntryOnAcquire(copies, localSegmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + readValue, usingValue, toValue, entry); + } finally { + if (segmentState == null && localSegmentState != null) + localSegmentState.close(); + } + } + + final , RV> + RV createEntryOnAcquire( + ThreadLocalCopies copies, @NotNull SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, + MultiStoreBytes entry) { + long valueSize; + // omit copies assignment because it is 100% non-null (see above) + // todo add api which doesn't require key instance for assigning default value bytes + K keyInstance = toKey.toInstance(copies, key, keySize); + if (defaultValueProvider != null) { + V defaultValue = defaultValueProvider.get(keyInstance); + VI valueInterop = valueInteropProvider.get(copies, originalValueInterop); + MetaBytesWriter metaValueInterop = metaValueInteropProvider.get( + copies, originalMetaValueInterop, valueInterop, defaultValue); + + valueSize = putEntry(segmentState, + metaKeyInterop, keyInterop, key, keySize, + metaValueInterop, valueInterop, defaultValue, entry, true); + } else if (prepareValueBytesAsWriter != null) { + valueSize = putEntry(segmentState, + metaKeyInterop, keyInterop, key, keySize, + prepareValueBytesAsWriter, null, keyInstance, entry, true); + } else { + throw defaultValueOrPrepareBytesShouldBeSpecified(); + } + + entry.positionAddr(entry.positionAddr() - valueSize); + RV v = readValue.readValue(copies, entry, usingValue, valueSize); + + // put callbacks + onPut(this, segmentState.pos); + if (bytesEventListener != null) { + long keyPos = metaDataBytes; + bytesEventListener.onPut(entry, 0L, keyPos, segmentState.valueSizePos, true); + } + if (eventListener != null) { + eventListener.onPut(toKey.toInstance(copies, key, keySize), + toValue.toInstance(copies, v, valueSize), null); + } + + return v; + } + + final IllegalStateException defaultValueOrPrepareBytesShouldBeSpecified() { + return new IllegalStateException("To call acquire*() methods, " + + "you should specify either default value, default value provider " + + "or prepareBytes strategy during map building"); + } + + final RV readValueAndNotifyGet( + ThreadLocalCopies copies, + KB key, long keySize, InstanceOrBytesToInstance toKey, + ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, + MultiStoreBytes entry) { + long valueSize = readValueSize(entry); + long valuePos = entry.position(); + RV v = readValue.readValue(copies, entry, usingValue, valueSize); + + // get callbacks + if (bytesEventListener != null) { + bytesEventListener.onGetFound(entry, 0L, metaDataBytes, valuePos); + } + if (eventListener != null) { + eventListener.onGetFound(toKey.toInstance(copies, key, keySize), + toValue.toInstance(copies, v, valueSize)); + } + + return v; + } + + private , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV put(@Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + long hash2, boolean replaceIfPresent, + ReadValue readValue, boolean resultUnused) { + shouldNotBeCalledFromReplicatedChronicleMap("Segment.put"); + writeLock(null); + try { + return putWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + getValueInterops, value, toValue, + hash2, replaceIfPresent, readValue, resultUnused); + } finally { + if (segmentState != null) + segmentState.close(); writeUnlock(); } } - V putWithoutLock(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V value, long hash2, boolean replaceIfPresent) { - long keySize = metaKeyInterop.size(keyInterop, key); - hashLookup.startSearch(hash2); - for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { - long offset = offsetFromPos(pos); - MultiStoreBytes entry = entry(offset); - if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) - continue; - // key is found - entry.skip(keySize); - if (replaceIfPresent) { - return replaceValueOnPut(copies, key, value, entry, pos, offset, - !putReturnsNull, hashLookup); - } else { - return putReturnsNull ? null : readValue(copies, entry, null); + , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + RV putWithoutLock( + @Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + long hash2, boolean replaceIfPresent, + ReadValue readValue, boolean resultUnused) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); + SegmentState localSegmentState = segmentState; + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); // copies is not null now + localSegmentState = SegmentState.get(copies); + } + try { + hashLookup.startSearch(hash2); + MultiStoreBytes entry = localSegmentState.tmpBytes; + for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { + long offset = offsetFromPos(pos); + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) + continue; + // key is found + entry.skip(keySize); + if (replaceIfPresent) { + return replaceValueAndNotifyPut(copies, localSegmentState, + key, keySize, toKey, + getValueInterops, value, toValue, + entry, pos, offset, hashLookup, readValue, resultUnused, + false, false); + } else { + long valueSize = readValueSize(entry); + return resultUnused ? null : + readValue.readValue(copies, entry, null, valueSize); + } } + // key is not found + VBI valueInterop = getValueInterops.getValueInterop(copies); + MVBI metaValueInterop = + getValueInterops.getMetaValueInterop(copies, valueInterop, value); + long valueSize = metaValueInterop.size(valueInterop, value); + putEntry(localSegmentState, metaKeyInterop, keyInterop, key, keySize, + metaValueInterop, valueInterop, value, entry, false); + + // put callbacks + onPut(this, localSegmentState.pos); + if (bytesEventListener != null) + bytesEventListener.onPut(entry, 0L, metaDataBytes, + localSegmentState.valueSizePos, true); + if (eventListener != null) + eventListener.onPut(toKey.toInstance(copies, key, keySize), + toValue.toInstance(copies, value, valueSize), null); + + return resultUnused ? null : readValue.readNull(); + } finally { + if (segmentState == null && localSegmentState != null) + localSegmentState.close(); } - // key is not found - long offset = putEntry(copies, metaKeyInterop, keyInterop, key, keySize, value); - incrementSize(); - notifyPut(offset, true, key, value, null, posFromOffset(offset)); - return null; } - V replaceValueOnPut(ThreadLocalCopies copies, K key, V value, MultiStoreBytes entry, long pos, - long offset, boolean readPrevValue, MultiMap searchedHashLookup) { + final > + RV replaceValueAndNotifyPut( + ThreadLocalCopies copies, SegmentState segmentState, + KB key, long keySize, InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB value, + InstanceOrBytesToInstance toValue, + MultiStoreBytes entry, long pos, long offset, MultiMap searchedHashLookup, + ReadValue readValue, boolean resultUnused, + boolean entryIsDeleted, boolean remote) { + VBI valueInterop = getValueInterops.getValueInterop(copies); + MVBI metaValueInterop = getValueInterops.getMetaValueInterop( + copies, valueInterop, value); + long valueSize = metaValueInterop.size(valueInterop, value); + long valueSizePos = entry.position(); - long valueSize = readValueSize(entry); - long entryEndAddr = entry.positionAddr() + valueSize; - V prevValue = null; - if (readPrevValue) - prevValue = readValue(copies, entry, null, valueSize); + long prevValueSize = readValueSize(entry); + long valueAddr = entry.positionAddr(); + long entryEndAddr = valueAddr + prevValueSize; + + RV prevValue = null; + V prevValueInstance = null; + if (!resultUnused && !entryIsDeleted) + prevValue = readValue.readValue(copies, entry, null, prevValueSize); + // todo optimize -- prevValue could be read twice + if (eventListener != null && !entryIsDeleted) { + entry.positionAddr(valueAddr); + prevValueInstance = readValue(copies, entry, null, prevValueSize); + } // putValue may relocate entry and change offset - offset = putValue(pos, offset, entry, valueSizePos, entryEndAddr, copies, value, null, - searchedHashLookup); - notifyPut(offset, false, key, value, prevValue, posFromOffset(offset)); - return prevValue; + putValue(pos, offset, entry, valueSizePos, entryEndAddr, copies, segmentState, + metaValueInterop, valueInterop, value, valueSize, searchedHashLookup); + + // put callbacks + if (!remote) + onPut(this, segmentState.pos); + if (bytesEventListener != null) + bytesEventListener.onPut(entry, 0L, metaDataBytes, valueSizePos, false); + if (eventListener != null) { + eventListener.onPut(toKey.toInstance(copies, key, keySize), + toValue.toInstance(copies, value, valueSize), prevValueInstance); + } + + return resultUnused ? null : prevValue; } - private long putEntry(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, - long keySize, V value) { - copies = valueWriterProvider.getCopies(copies); - VW valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = metaValueWriterProvider.getCopies(copies); - MetaBytesWriter metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, value); - - return putEntry(metaKeyInterop, keyInterop, key, keySize, - metaValueWriter, valueWriter, value); - } - - private long putEntry(MKI metaKeyInterop, KI keyInterop, K key, long keySize, - MetaBytesWriter metaElemWriter, EW elemWriter, - E elem) { + /** + * Returns value size, writes the entry (key, value, sizes) to the entry, after this + * method call entry positioned after value bytes written (i. e. at the end of entry), + * sets entry position (in segment) and value size position in the given segmentState + */ + final , E, EW> + long putEntry(SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + MetaBytesWriter metaElemWriter, EW elemWriter, E elem, + MultiStoreBytes entry, boolean writeDefaultInitialReplicationValues) { long valueSize = metaElemWriter.size(elemWriter, elem); long entrySize = entrySize(keySize, valueSize); long pos = alloc(inBlocks(entrySize)); + segmentState.pos = pos; long offset = offsetFromPos(pos); clearMetaData(offset); - NativeBytes entry = entry(offset); + reuse(entry, offset); keySizeMarshaller.writeSize(entry, keySize); metaKeyInterop.write(keyInterop, entry, key); + + manageReplicationBytes(entry, writeDefaultInitialReplicationValues, false); + + segmentState.valueSizePos = entry.position(); valueSizeMarshaller.writeSize(entry, valueSize); alignment.alignPositionAddr(entry); metaElemWriter.write(elemWriter, entry, elem); hashLookup.putAfterFailedSearch(pos); + incrementSize(); - return offset; + return valueSize; } - void clearMetaData(long offset) { + void manageReplicationBytes(Bytes entry, boolean writeDefaultInitialReplicationValues, + boolean remove) { + // do nothing + } + + private void clearMetaData(long offset) { if (metaDataBytes > 0) bytes.zeroOut(offset, offset + metaDataBytes); } //TODO refactor/optimize - long alloc(int blocks) { + private long alloc(int blocks) { if (blocks > MAX_ENTRY_OVERSIZE_FACTOR) throw new IllegalArgumentException("Entry is too large: requires " + blocks + " entry size chucks, " + MAX_ENTRY_OVERSIZE_FACTOR + " is maximum."); @@ -1421,6 +2054,10 @@ class VanillaChronicleMap, return ret; } + private void setBlocks(long pos, int blocks) { + freeList.set(pos, pos + blocks); + } + private void updateNextPosToSearchFrom(long allocated, int blocks) { if ((nextPosToSearchFrom = allocated + blocks) >= entriesPerSegment) nextPosToSearchFrom = 0L; @@ -1435,92 +2072,178 @@ class VanillaChronicleMap, } } - void free(long fromPos, int blocks) { + private void free(long fromPos, int blocks) { freeList.clear(fromPos, fromPos + blocks); if (fromPos < nextPosToSearchFrom) nextPosToSearchFrom = fromPos; } - V readValue(ThreadLocalCopies copies, MultiStoreBytes entry, V value) { - return readValue(copies, entry, value, readValueSize(entry)); - } - - V readValue(ThreadLocalCopies copies, NativeBytes entry, V value, long valueSize) { - copies = valueReaderProvider.getCopies(copies); - BytesReader valueReader = valueReaderProvider.get(copies, originalValueReader); - return valueReader.read(entry, valueSize, value); - } - - boolean keyEquals(KI keyInterop, MKI metaKeyInterop, K key, - long keySize, Bytes entry) { + final > + boolean keyEquals(KBI keyInterop, MKBI metaKeyInterop, KB key, long keySize, Bytes entry) { return keySize == keySizeMarshaller.readSize(entry) && metaKeyInterop.startsWith(keyInterop, entry, key); } - /** - * Removes a key (or key-value pair) from the Segment.

The entry will only be - * removed if {@code expectedValue} equals to {@code null} or the value previously - * corresponding to the specified key. - * - * @param hash2 a hash code related to the {@code keyBytes} - * @return the value of the entry that was removed if the entry corresponding to the {@code - * keyBytes} exists and {@link #removeReturnsNull} is {@code false}, {@code null} otherwise - */ - V remove(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, - V expectedValue, long hash2) { - writeLock(); + , + VB, VBI, MVBI extends MetaBytesInterop> + Object remove(@Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB expectedValue, + InstanceOrBytesToInstance toValue, + long hash2, boolean resultUnused) { + writeLock(null); try { - return removeWithoutLock(copies, metaKeyInterop, keyInterop, key, expectedValue, hash2); + return removeWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key, keySize, toKey, + getValueInterops, expectedValue, toValue, + hash2, resultUnused); } finally { + if (segmentState != null) + segmentState.close(); writeUnlock(); } } - private V removeWithoutLock(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, V expectedValue, long hash2) { - long keySize = metaKeyInterop.size(keyInterop, key); - hashLookup.startSearch(hash2); - for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { - long offset = offsetFromPos(pos); - MultiStoreBytes entry = entry(offset); - if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) - continue; - // key is found - entry.skip(keySize); - long valueSize = readValueSize(entry); - long entryEndAddr = entry.positionAddr() + valueSize; - V valueRemoved = expectedValue != null || !removeReturnsNull - ? readValue(copies, entry, null, valueSize) : null; - if (expectedValue != null && !expectedValue.equals(valueRemoved)) - return null; - hashLookup.removePrevPos(); - decrementSize(); - free(pos, inBlocks(entryEndAddr - entryStartAddr(offset))); - notifyRemoved(offset, key, valueRemoved, pos); - return valueRemoved; + /** + * - if expectedValue is not null, returns Boolean.TRUE (removed) or + * Boolean.FALSE (entry not found), regardless the expectedValue object is Bytes instance + * (RPC call) or the value instance + * - if expectedValue is null: + * - if resultUnused is false, null or removed value is returned + * - if resultUnused is true, null is always returned + */ + , + VB, VBI, MVBI extends MetaBytesInterop> + Object removeWithoutLock( + @Nullable ThreadLocalCopies copies, final @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getValueInterops, VB expectedValue, + InstanceOrBytesToInstance toValue, + long hash2, boolean resultUnused) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); + SegmentState localSegmentState = segmentState; + try { + MultiMap hashLookup = hashLookup(); + hashLookup.startSearch(hash2); + MultiStoreBytes entry = null; + for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { + long offset = offsetFromPos(pos); + if (entry == null) { + if (localSegmentState == null) { + copies = SegmentState.getCopies(copies); // copies is not null now + localSegmentState = SegmentState.get(copies); + } + entry = localSegmentState.tmpBytes; + } + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) + continue; + // key is found + entry.skip(keySize); + long valueSizePos = entry.position(); + long valueSize = readValueSize(entry); + + // check the value assigned for the key is that we expect + if (expectedValue != null) { + VBI valueInterop = getValueInterops.getValueInterop(copies); + MVBI metaValueInterop = getValueInterops.getMetaValueInterop( + copies, valueInterop, expectedValue); + if (metaValueInterop.size(valueInterop, expectedValue) != valueSize) + return Boolean.FALSE; + if (!metaValueInterop.startsWith(valueInterop, entry, expectedValue)) + return Boolean.FALSE; + } + return removeEntry(copies, key, keySize, toKey, expectedValue, toValue, + resultUnused, hashLookup, entry, pos, valueSizePos, valueSize, + false, true, expectedValue != null); + } + // key is not found + return expectedValue == null ? null : Boolean.FALSE; + } finally { + if (segmentState == null && localSegmentState != null) + localSegmentState.close(); } - // key is not found - return null; } - boolean containsKey(KI keyInterop, MKI metaKeyInterop, K key, long hash2) { - readLock(); + final Object removeEntry( + ThreadLocalCopies copies, + KB key, long keySize, InstanceOrBytesToInstance toKey, + VB expectedValue, InstanceOrBytesToInstance toValue, + boolean resultUnused, MultiMap hashLookup, MultiStoreBytes entry, long pos, + long valueSizePos, long valueSize, boolean remote, boolean removeFromMultiMap, + boolean booleanResult) { + // get the removed value, if needed + V removedValue = null; + if ((!booleanResult && !resultUnused) || eventListener != null) { + if (expectedValue != null) + removedValue = toValue.toInstance(copies, expectedValue, valueSize); + if (removedValue == null) { + // todo reuse some value + removedValue = readValue(copies, entry, null, valueSize); + entry.position(entry.position() - valueSize); + } + } + + // update segment state + if (removeFromMultiMap) { + hashLookup.removePrevPos(); + } else { + hashLookup.removePosition(pos); + } + decrementSize(); + long entrySizeInBytes = entry.positionAddr() + valueSize - entry.startAddr(); + free(pos, inBlocks(entrySizeInBytes)); + + // remove callbacks + if (!remote) + onRemove(this, pos); + if (bytesEventListener != null) + bytesEventListener.onRemove(entry, 0L, metaDataBytes, valueSizePos); + if (eventListener != null) { + V removedValueForEventListener = removedValue != null ? removedValue : + readValue(copies, entry, null); + eventListener.onRemove(toKey.toInstance(copies, key, keySize), + removedValueForEventListener); + } + + return booleanResult ? Boolean.TRUE : removedValue; + } + + private > + boolean containsKey(ThreadLocalCopies copies, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, long hash2) { + readLock(null); + SegmentState segmentState = null; try { - long keySize = metaKeyInterop.size(keyInterop, key); - MultiMap hashLookup = containsKeyHashLookup(); + MultiStoreBytes entry = null; + MultiMap hashLookup = hashLookup(); hashLookup.startSearch(hash2); for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { - Bytes entry = entry(offsetFromPos(pos)); - if (keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) - return true; + long offset = offsetFromPos(pos); + if (entry == null) { + copies = SegmentState.getCopies(copies); + segmentState = SegmentState.get(copies); + entry = segmentState.tmpBytes; + } + reuse(entry, offset); + if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) + continue; + if (isDeleted(entry, keySize)) + continue; + return true; } return false; } finally { + if (segmentState != null) + segmentState.close(); readUnlock(); } } - MultiMap containsKeyHashLookup() { - return hashLookup; + boolean isDeleted(Bytes entry, long keySize) { + return false; } /** @@ -1531,75 +2254,96 @@ class VanillaChronicleMap, * @param hash2 a hash code related to the {@code keyBytes} * @return the replaced value or {@code null} if the value was not replaced */ - V replace(ThreadLocalCopies copies, MKI metaKeyInterop, KI keyInterop, K key, - V expectedValue, V newValue, long hash2) { - writeLock(); + , + RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> + Object replace( + @Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState, + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, + InstanceOrBytesToInstance toKey, + GetValueInterops getExpectedValueInterops, VB expectedValue, + GetValueInterops getNewValueInterops, VB newValue, + ReadValue readValue, InstanceOrBytesToInstance toValue, + long hash2) { + segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); + writeLock(null); try { - long keySize = metaKeyInterop.size(keyInterop, key); hashLookup.startSearch(hash2); + MultiStoreBytes entry = null; for (long pos; (pos = hashLookup.nextPos()) >= 0L; ) { long offset = offsetFromPos(pos); - MultiStoreBytes entry = entry(offset); + if (entry == null) { + if (segmentState == null) { + copies = SegmentState.getCopies(copies); + segmentState = SegmentState.get(copies); + } + entry = segmentState.tmpBytes; + } + reuse(entry, offset); if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry)) continue; // key is found entry.skip(keySize); - return onKeyPresentOnReplace(copies, key, expectedValue, newValue, - pos, offset, entry, hashLookup); + + return onKeyPresentOnReplace(copies, segmentState, key, keySize, toKey, + getExpectedValueInterops, expectedValue, getNewValueInterops, newValue, + readValue, toValue, pos, offset, entry, hashLookup); } // key is not found - return null; + return expectedValue == null ? null : Boolean.FALSE; } finally { + if (segmentState != null) + segmentState.close(); writeUnlock(); } } - V onKeyPresentOnReplace(ThreadLocalCopies copies, K key, V expectedValue, V newValue, - long pos, long offset, MultiStoreBytes entry, - MultiMap searchedHashLookup) { + final > + Object onKeyPresentOnReplace( + ThreadLocalCopies copies, @NotNull SegmentState segmentState, + KB key, long keySize, InstanceOrBytesToInstance toKey, + GetValueInterops getExpectedValueInterops, VB expectedValue, + GetValueInterops getNewValueInterops, VB newValue, + ReadValue readValue, InstanceOrBytesToInstance toValue, + long pos, long offset, MultiStoreBytes entry, MultiMap searchedHashLookup) { long valueSizePos = entry.position(); long valueSize = readValueSize(entry); long entryEndAddr = entry.positionAddr() + valueSize; - V valueRead = readValue(copies, entry, null, valueSize); - if (valueRead == null) - return null; - if (expectedValue == null || expectedValue.equals(valueRead)) { - // putValue may relocate entry and change offset - offset = putValue(pos, offset, entry, valueSizePos, entryEndAddr, copies, newValue, - null, searchedHashLookup); - notifyPut(offset, false, key, newValue, valueRead, posFromOffset(offset)); - return valueRead; - } - return null; - } + RV prevValue; - - void notifyPut(long offset, boolean added, K key, V value, @Nullable final V replacedValue, - final long pos) { - if (eventListener() != MapEventListeners.NOP) { - MultiStoreBytes b = acquireTmpBytes(); - b.storePositionAndSize(bytes, offset, entrySize); - eventListener().onPut(VanillaChronicleMap.this, b, metaDataBytes, - added, key, replacedValue, value, pos, this); + if (expectedValue != null) { + // check the value assigned for the key is that we expect + VBI valueInterop = getExpectedValueInterops.getValueInterop(copies); + MVBI metaValueInterop = getExpectedValueInterops.getMetaValueInterop( + copies, valueInterop, expectedValue); + if (metaValueInterop.size(valueInterop, expectedValue) != valueSize) + return Boolean.FALSE; + if (!metaValueInterop.startsWith(valueInterop, entry, expectedValue)) + return Boolean.FALSE; + prevValue = expectedValue; + } else { + // todo reuse value + prevValue = readValue.readValue(copies, entry, null, valueSize); } - } - void notifyGet(long offset, K key, V value) { - if (eventListener() != MapEventListeners.NOP) { - MultiStoreBytes b = acquireTmpBytes(); - b.storePositionAndSize(bytes, offset, entrySize); - eventListener().onGetFound(VanillaChronicleMap.this, b, metaDataBytes, - key, value); - } - } + VBI valueInterop = getNewValueInterops.getValueInterop(copies); + MVBI metaValueInterop = getNewValueInterops.getMetaValueInterop( + copies, valueInterop, newValue); + long newValueSize = metaValueInterop.size(valueInterop, newValue); + putValue(pos, offset, entry, valueSizePos, entryEndAddr, copies, segmentState, + metaValueInterop, valueInterop, newValue, newValueSize, searchedHashLookup); - void notifyRemoved(long offset, K key, V value, final long pos) { - if (eventListener() != MapEventListeners.NOP) { - MultiStoreBytes b = acquireTmpBytes(); - b.storePositionAndSize(bytes, offset, entrySize); - eventListener().onRemove(VanillaChronicleMap.this, b, metaDataBytes, - key, value, pos, this); + // put callbacks + onPut(this, segmentState.pos); + if (bytesEventListener != null) { + long keyPos = metaDataBytes; + bytesEventListener.onPut(entry, 0L, keyPos, segmentState.valueSizePos, true); } + if (eventListener != null) + eventListener.onPut(toKey.toInstance(copies, key, keySize), + toValue.toInstance(copies, newValue, newValueSize), + toValue.toInstance(copies, prevValue, valueSize)); + + return expectedValue == null ? prevValue : Boolean.TRUE; } /** @@ -1615,33 +2359,21 @@ class VanillaChronicleMap, * @return relative offset of the entry in Segment bytes after putting value (that may cause * entry relocation) */ - long putValue(long pos, long offset, NativeBytes entry, long valueSizePos, - long entryEndAddr, ThreadLocalCopies copies, V value, Bytes valueBytes, - MultiMap searchedHashLookup) { + final long putValue( + long pos, long offset, MultiStoreBytes entry, long valueSizePos, long entryEndAddr, + ThreadLocalCopies copies, SegmentState segmentState, + MetaBytesWriter metaElemWriter, EW elemWriter, E newElem, + long newElemSize, + MultiMap searchedHashLookup) { long valueSizeAddr = entry.address() + valueSizePos; - long newValueSize; - VW valueWriter = null; - MetaBytesWriter metaValueWriter = null; - if (valueBytes != null) { - newValueSize = valueBytes.remaining(); - } else { - copies = valueWriterProvider.getCopies(copies); - valueWriter = valueWriterProvider.get(copies, originalValueWriter); - copies = metaValueWriterProvider.getCopies(copies); - metaValueWriter = metaValueWriterProvider.get( - copies, originalMetaValueWriter, valueWriter, value); - newValueSize = metaValueWriter.size(valueWriter, value); - } long newValueAddr = alignment.alignAddr( - valueSizeAddr + valueSizeMarshaller.sizeEncodingSize(newValueSize)); - long newEntryEndAddr = newValueAddr + newValueSize; - // Fast check before counting "sizes in blocks" that include - // integral division + valueSizeAddr + valueSizeMarshaller.sizeEncodingSize(newElemSize)); + long newEntryEndAddr = newValueAddr + newElemSize; + long entryStartAddr = entry.address(); + long oldEntrySize = entryEndAddr - entryStartAddr; + int oldSizeInBlocks = inBlocks(oldEntrySize); newValueDoesNotFit: if (newEntryEndAddr != entryEndAddr) { - long entryStartAddr = entryStartAddr(offset); - long oldEntrySize = entryEndAddr - entryStartAddr; - int oldSizeInBlocks = inBlocks(oldEntrySize); int newSizeInBlocks = inBlocks(newEntryEndAddr - entryStartAddr); if (newSizeInBlocks > oldSizeInBlocks) { if (newSizeInBlocks > MAX_ENTRY_OVERSIZE_FACTOR) { @@ -1653,20 +2385,19 @@ class VanillaChronicleMap, break newValueDoesNotFit; // RELOCATION free(pos, oldSizeInBlocks); - eventListener().onRelocation(pos, this); - long prevPos = pos; + onRelocation(this, pos); pos = alloc(newSizeInBlocks); // putValue() is called from put() and replace() // after successful search by key - replacePosInHashLookupOnRelocation(searchedHashLookup, prevPos, pos); + searchedHashLookup.replacePrevPos(pos); offset = offsetFromPos(pos); // Moving metadata, key size and key. // Don't want to fiddle with pseudo-buffers for this, // since we already have all absolute addresses. - long newEntryStartAddr = entryStartAddr(offset); + long newEntryStartAddr = entry.address(); NativeBytes.UNSAFE.copyMemory(entryStartAddr, newEntryStartAddr, valueSizeAddr - entryStartAddr); - entry = entry(offset); + entry = reuse(entry, offset); // END OF RELOCATION } else if (newSizeInBlocks < oldSizeInBlocks) { // Freeing extra blocks @@ -1676,26 +2407,22 @@ class VanillaChronicleMap, // But if these blocks will be taken by that time, // this entry will need to be relocated. } + } else { + // todo investivate when we are able to avoid this assignment + setBlocks(pos, oldSizeInBlocks); } // Common code for all cases entry.position(valueSizePos); - valueSizeMarshaller.writeSize(entry, newValueSize); + valueSizeMarshaller.writeSize(entry, newElemSize); alignment.alignPositionAddr(entry); - if (valueBytes != null) { - entry.write(valueBytes); - } else { - metaValueWriter.write(valueWriter, entry, value); - } + metaElemWriter.write(elemWriter, entry, newElem); + segmentState.pos = pos; return offset; } - void replacePosInHashLookupOnRelocation(MultiMap searchedHashLookup, - long prevPos, long pos) { - searchedHashLookup.replacePrevPos(pos); - } - - void clear() { - writeLock(); + private void clear() { + shouldNotBeCalledFromReplicatedChronicleMap("Segment.clear"); + writeLock(null); try { hashLookup.clear(); freeList.clear(); @@ -1706,38 +2433,44 @@ class VanillaChronicleMap, } } - public Entry getEntry(long pos) { - bytes.position(offsetFromPos(pos) + metaDataBytes); + public Entry getEntry(@NotNull SegmentState segmentState, long pos) { + Bytes entry = reuse(segmentState.tmpBytes, offsetFromPos(pos)); - long keySize = keySizeMarshaller.readSize(bytes); + long keySize = keySizeMarshaller.readSize(entry); ThreadLocalCopies copies = keyReaderProvider.getCopies(null); - K key = keyReaderProvider.get(copies, originalKeyReader).read(bytes, keySize); + K key = keyReaderProvider.get(copies, originalKeyReader).read(entry, keySize); - long valueSize = valueSizeMarshaller.readSize(bytes); - alignment.alignPositionAddr(bytes); + long valueSize = valueSizeMarshaller.readSize(entry); + alignment.alignPositionAddr(entry); copies = valueReaderProvider.getCopies(copies); - V value = valueReaderProvider.get(copies, originalValueReader).read(bytes, valueSize); + V value = valueReaderProvider.get(copies, originalValueReader).read(entry, valueSize); return new WriteThroughEntry(key, value); } + + boolean isDeleted(long pos) { + return false; + } + /** * Check there is no garbage in freeList. */ - void checkConsistency() { - readLock(); - try { - MultiMap hashLookup = checkConsistencyHashLookup(); + private void checkConsistency() { + readLock(null); + try (SegmentState segmentState = SegmentState.get(SegmentState.getCopies(null))) { + MultiStoreBytes entry = segmentState.tmpBytes; + MultiMap hashLookup = hashLookup(); for (long pos = 0L; (pos = freeList.nextSetBit(pos)) >= 0L; ) { PosPresentOnce check = new PosPresentOnce(pos); hashLookup.forEach(check); if (check.count != 1) throw new AssertionError(); long offset = offsetFromPos(pos); - Bytes entry = entry(offset); + reuse(entry, offset); long keySize = keySizeMarshaller.readSize(entry); entry.skip(keySize); - afterKeyHookOnCheckConsistency(entry); + manageReplicationBytes(entry, false, false); long valueSize = valueSizeMarshaller.readSize(entry); long sizeInBytes = entrySize(keySize, valueSize); int entrySizeInBlocks = inBlocks(sizeInBytes); @@ -1750,15 +2483,7 @@ class VanillaChronicleMap, } } - void afterKeyHookOnCheckConsistency(Bytes entry) { - // no-op - } - - MultiMap checkConsistencyHashLookup() { - return hashLookup; - } - - class PosPresentOnce implements MultiMap.EntryConsumer { + private class PosPresentOnce implements MultiMap.EntryConsumer { long pos; int count = 0; @@ -1771,57 +2496,10 @@ class VanillaChronicleMap, if (this.pos == pos) count++; } } + } - class NativeWriteLocked extends VanillaChronicleMap.WriteLocked { - @Override - public void close() { - Segment.this.writeUnlock(); - } - - @Override - public void dontPutOnClose() { - throw new IllegalStateException( - "This method is not supported for native value classes"); - } - - @Override - public void removeEntry() { - removeWithoutLock(copies, metaKeyInterop, keyInterop, key(), null, segmentHash); - } - } - - class HeapWriteLocked extends VanillaChronicleMap.WriteLocked { - - private boolean putOnClose = true; - - @Override - public void close() { - if (putOnClose) - putWithoutLock(copies, metaKeyInterop, keyInterop, key(), value(), - segmentHash, true); - - putOnClose = true; - Segment.this.writeUnlock(); - } - - @Override - public void dontPutOnClose() { - putOnClose = false; - } - - @Override - public void removeEntry() { - putOnClose = false; - removeWithoutLock(copies, metaKeyInterop, keyInterop, key(), null, segmentHash); - } - } - - class TLReadLocked extends VanillaChronicleMap.ReadLocked { - @Override - public void close() { - Segment.this.readUnlock(); - } - } + void shouldNotBeCalledFromReplicatedChronicleMap(String method) { + // do nothing } class EntryIterator implements Iterator> { @@ -1831,13 +2509,16 @@ class VanillaChronicleMap, private int nextSeg; private long nextPos; - public EntryIterator() { + // todo get rid of this + MultiStoreBytes entry = new MultiStoreBytes(); + + EntryIterator() { advance(nextSeg = segments.length - 1, nextPos = -1L); } private boolean advance(int segIndex, long pos) { while (segIndex >= 0) { - pos = segments[segIndex].getHashLookup().getPositions().nextSetBit(pos + 1L); + pos = segments[segIndex].hashLookup().getPositions().nextSetBit(pos + 1L); if (pos >= 0L) { nextSeg = segIndex; nextPos = pos; @@ -1853,7 +2534,7 @@ class VanillaChronicleMap, } @Override - public boolean hasNext() { + public final boolean hasNext() { return nextSeg >= 0; } @@ -1865,31 +2546,34 @@ class VanillaChronicleMap, if (segIndex < 0) throw new NoSuchElementException(); final Segment segment = segments[segIndex]; + ThreadLocalCopies copies = SegmentState.getCopies(null); + SegmentState segmentState = SegmentState.get(copies); try { - segment.readLock(); - if (segment.getHashLookup().getPositions().isClear(pos)) { + segment.readLock(null); + if (segment.hashLookup().getPositions().isClear(pos)) { // the pos was removed after the previous advance advance(segIndex, pos); continue; } advance(returnedSeg = segIndex, returnedPos = pos); - return returnedEntry = segment.getEntry(pos); + return returnedEntry = segment.getEntry(segmentState, pos); } finally { + segmentState.close(); segment.readUnlock(); } } } @Override - public void remove() { + public final void remove() { int segIndex = returnedSeg; if (segIndex < 0) throw new IllegalStateException(); final Segment segment = segments[segIndex]; final long pos = returnedPos; try { - segment.writeLock(); - if (segment.getHashLookup().getPositions().isClear(pos)) { + segment.writeLock(null); + if (segment.hashLookup().getPositions().isClear(pos)) { // The case: // 1. iterator.next() - thread 1 // 2. map.put() which cause relocation of the key, returned in above - thread 2 @@ -1897,7 +2581,7 @@ class VanillaChronicleMap, // 3. iterator.remove() - thread 1 segment.writeUnlock(); // not re-entrant. VanillaChronicleMap.this.remove(returnedEntry.getKey()); - segment.writeLock(); + segment.writeLock(null); } else { removePresent(segment, pos); } @@ -1918,19 +2602,36 @@ class VanillaChronicleMap, // of returnedEntry.getKey(), but it seems rather wasteful to workaround so rare // case. final long offset = segment.offsetFromPos(pos); - final NativeBytes entry = segment.entry(offset); + final NativeBytes entry = segment.reuse(this.entry, offset); final long keySize = keySizeMarshaller.readSize(entry); long position = entry.position(); final long segmentHash = segmentHash(Hasher.hash(entry, position, position + keySize)); + removePresent(segment, pos, entry, keySize, segmentHash, true); + } + + final void removePresent(Segment segment, long pos, NativeBytes entry, long keySize, + long segmentHash, boolean removeFromMultiMap) { entry.skip(keySize); + segment.manageReplicationBytes(entry, true, true); + long valueSizePos = entry.position(); long valueSize = readValueSize(entry); final long entryEndAddr = entry.positionAddr() + valueSize; - segment.getHashLookup().remove(segmentHash, pos); + if (removeFromMultiMap) { + segment.hashLookup().remove(segmentHash, pos); + } else { + segment.hashLookup().removePosition(pos); + } segment.decrementSize(); - segment.free(pos, segment.inBlocks(entryEndAddr - segment.entryStartAddr(offset))); - segment.notifyRemoved(offset, returnedEntry.getKey(), returnedEntry.getValue(), pos); + segment.free(pos, segment.inBlocks(entryEndAddr - entry.address())); + + // remove callbacks + onRemove(segment, pos); + if (bytesEventListener != null) + bytesEventListener.onRemove(entry, 0L, metaDataBytes, valueSizePos); + if (eventListener != null) + eventListener.onRemove(returnedEntry.getKey(), returnedEntry.getValue()); } } @@ -1940,7 +2641,7 @@ class VanillaChronicleMap, return new EntryIterator(); } - public boolean contains(Object o) { + public final boolean contains(Object o) { if (!(o instanceof Map.Entry)) return false; Map.Entry e = (Map.Entry) o; @@ -1954,7 +2655,7 @@ class VanillaChronicleMap, } } - public boolean remove(Object o) { + public final boolean remove(Object o) { if (!(o instanceof Map.Entry)) return false; Map.Entry e = (Map.Entry) o; @@ -1969,20 +2670,20 @@ class VanillaChronicleMap, } } - public int size() { + public final int size() { return VanillaChronicleMap.this.size(); } - public boolean isEmpty() { + public final boolean isEmpty() { return VanillaChronicleMap.this.isEmpty(); } - public void clear() { + public final void clear() { VanillaChronicleMap.this.clear(); } } - final class WriteThroughEntry extends SimpleEntry { + private class WriteThroughEntry extends SimpleEntry { private static final long serialVersionUID = 0L; WriteThroughEntry(K key, V value) { @@ -1996,38 +2697,53 @@ class VanillaChronicleMap, } } - static abstract class MutableLockedEntry implements Context { + private static abstract class MutableLockedEntry< + K, KI, MKI extends MetaBytesInterop, + V, VI, MVI extends MetaBytesInterop> implements Context { + final SegmentState segmentState; + + VanillaChronicleMap map; + VanillaChronicleMap.Segment segment; ThreadLocalCopies copies; MKI metaKeyInterop; KI keyInterop; + long keySize; long segmentHash; - private K key; private V value; - public K key() { + MutableLockedEntry(SegmentState segmentState) { + this.segmentState = segmentState; + } + + public final K key() { return key; } - void key(K key) { + final void key(K key) { this.key = key; } - public V value() { + public final V value() { return value; } void value(V value) { this.value = value; } - } - static abstract class ReadLocked extends MutableLockedEntry implements ReadContext { + private static class ReadLocked, + V, VI, MVI extends MetaBytesInterop> + extends MutableLockedEntry + implements ReadContext { - boolean present; + private boolean present; + + ReadLocked(SegmentState segmentState) { + super(segmentState); + } void value(V value) { this.present = value != null; @@ -2041,10 +2757,85 @@ class VanillaChronicleMap, public boolean present() { return present; } + + @Override + public void close() { + segmentState.close(); + segment.readUnlock(); + } } - static abstract class WriteLocked extends MutableLockedEntry implements WriteContext { + private static abstract class WriteLocked, + V, VI, MVI extends MetaBytesInterop> + extends MutableLockedEntry + implements WriteContext { + + WriteLocked(SegmentState segmentState) { + super(segmentState); + } + } + + private static class NativeWriteLocked, + V, VI, MVI extends MetaBytesInterop> + extends WriteLocked { + + NativeWriteLocked(SegmentState segmentState) { + super(segmentState); + } + + @Override + public void close() { + segmentState.close(); + segment.writeUnlock(); + } + + @Override + public void dontPutOnClose() { + throw new IllegalStateException( + "This method is not supported for native value classes"); + } + + @Override + public void removeEntry() { + segment.removeWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key(), keySize, map.keyIdentity(), + map, null, map.valueIdentity(), segmentHash, true); + } + } + + private static class HeapWriteLocked, + V, VI, MVI extends MetaBytesInterop> + extends WriteLocked { + + private boolean putOnClose = true; + + HeapWriteLocked(SegmentState segmentState) { + super(segmentState); + } + + @Override + public void close() { + if (putOnClose) { + segment.putWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key(), keySize, map.keyIdentity(), + map, value(), map.valueIdentity(), segmentHash, true, map, true); + } + putOnClose = true; + segmentState.close(); + segment.writeUnlock(); + } + + @Override + public void dontPutOnClose() { + putOnClose = false; + } + + @Override + public void removeEntry() { + putOnClose = false; + segment.removeWithoutLock(copies, segmentState, + metaKeyInterop, keyInterop, key(), keySize, map.keyIdentity(), + map, null, map.valueIdentity(), segmentHash, true); + } } } - diff --git a/src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java b/src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java index 65f699a45..75b05edc7 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMMetaDataTest.java @@ -33,11 +33,10 @@ public class CHMMetaDataTest { @Test public void testAccessTimes() throws IOException { File file = new File(TMP, "testAccessTimes"); - MapEventListener> listener = + BytesMapEventListener listener = new StringStringMapEventListener(new AtomicLong(1)); ChronicleMap map = ChronicleMapBuilder.of(String.class, String.class) - .metaDataBytes(8) - .eventListener(listener).create(); + .metaDataBytes(8).bytesEventListener(listener).create(); try { map.put("a", "aye"); @@ -63,7 +62,7 @@ public class CHMMetaDataTest { } private static class StringStringMapEventListener - extends MapEventListener> { + extends BytesMapEventListener { private static final long serialVersionUID = 0L; private final AtomicLong timeStamps; @@ -73,27 +72,20 @@ public class CHMMetaDataTest { } @Override - public void onGetFound(ChronicleMap map, Bytes entry, int metaDataBytes, - String key, String value) { - assertEquals(8, metaDataBytes); - entry.writeLong(0, timeStamps.incrementAndGet()); + public void onGetFound(Bytes entry, long metaDataPos, long keyPos, long valuePos) { + entry.writeLong(metaDataPos, timeStamps.incrementAndGet()); } @Override - public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, - boolean added, String key, String replacedValue, String value, long pos, - SharedSegment segment) { - assertEquals(8, metaDataBytes); + public void onPut(Bytes entry, long metaDataPos, long keyPos, long valuePos, boolean added) { if (added) - assertEquals(0, entry.readLong()); - entry.writeLong(0, timeStamps.incrementAndGet()); + assertEquals(0, entry.readLong(metaDataPos)); + entry.writeLong(metaDataPos, timeStamps.incrementAndGet()); } @Override - public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, - String key, String value, long pos, SharedSegment segment) { - assertEquals(8, metaDataBytes); - System.out.println("Removed " + key + "/" + value + " with ts of " + entry.readLong(0)); + public void onRemove(Bytes entry, long metaDataPos, long keyPos, long valuePos) { + System.out.println("Removed entry with ts of " + entry.readLong(metaDataPos));; } } } diff --git a/src/test/java/net/openhft/chronicle/map/EventListenerTestWithTCPSocketReplication.java b/src/test/java/net/openhft/chronicle/map/EventListenerTestWithTCPSocketReplication.java index 1959daa06..ef8430724 100644 --- a/src/test/java/net/openhft/chronicle/map/EventListenerTestWithTCPSocketReplication.java +++ b/src/test/java/net/openhft/chronicle/map/EventListenerTestWithTCPSocketReplication.java @@ -50,13 +50,12 @@ public class EventListenerTestWithTCPSocketReplication { final AtomicBoolean wasRemoved = new AtomicBoolean(false); final AtomicReference valueRemoved = new AtomicReference(); - final MapEventListener> eventListener = new - MapEventListener>() { + final MapEventListener eventListener = new + MapEventListener() { private static final long serialVersionUID = 1L; @Override - public void onPut(ChronicleMap map, Bytes entry, int metaDataBytes, boolean added, Integer key, - CharSequence value, CharSequence replacedValue) { + public void onPut(Integer key, CharSequence value, CharSequence replacedValue) { putWasCalled.getAndSet(true); keyRef.set(key); valueRef.set(value); @@ -64,7 +63,7 @@ public class EventListenerTestWithTCPSocketReplication { } @Override - public void onRemove(ChronicleMap map, Bytes entry, int metaDataBytes, Integer key, CharSequence value) { + public void onRemove(Integer key, CharSequence value) { wasRemoved.set(true); valueRemoved.set(value); } diff --git a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java index bae6ed43f..315750da6 100644 --- a/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/TimeBasedReplicationTest.java @@ -247,18 +247,20 @@ public class TimeBasedReplicationTest extends JSR166TestCase { current(timeProvider); // we do a put at the current time - map.put("key-1", "value-1"); + String key1 = "key-1"; + map.put(key1, "value-1"); assertEquals(1, map.size()); - assertEquals("value-1", map.get("key-1")); + assertEquals("value-1", map.get(key1)); - // now test assume that we receive a late update to the map, the following update should be ignored - final long late = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(5); - assertEquals(null, map.remove("key-1", "value-2", IDENTIFIER, late)); + // now test assume that we receive a late update to the map, + // the following update should be ignored + late(timeProvider); + assertFalse(map.remove(key1, "value-2")); // we'll now flip the time back to the current in order to do the read the result current(timeProvider); assertEquals(1, map.size()); - assertEquals("value-1", map.get("key-1")); + assertEquals("value-1", map.get(key1)); assertTrue(map.containsValue("value-1")); assertFalse(map.containsValue("value-2")); }