Alawn/Chronicle-Map
1
0
Fork 0
mirror of https://github.com/OpenHFT/Chronicle-Map.git synced 2021-09-13 01:26:12 +00:00
Code Releases Activity

HCOLL-343 HCOLL-227 Remove old entries from ReplicatedChronicleMap

Browse Source
This commit is contained in:
leventov
2015-08-23 00:01:44 +03:00
parent a3a45e1681
commit 904ecca036
40 changed files with 945 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/net/openhft/chronicle/hash/ChronicleHash.java
+22 -1
View File
@@ -80,8 +80,22 @@ public interface ChronicleHash<K, E extends HashEntry<K>, SC extends HashSegment
*/
@NotNull EQC queryContext(Data<K> key);
/**
* Returns the context of the segment with the given index. Segments are indexed from 0 to
* {@link #segments()}{@code - 1}.
*
* @see HashSegmentContext
*/
SC segmentContext(int segmentIndex);
/**
* Returns the number of segments in this {@code ChronicleHash}.
*
* @see ChronicleHashBuilder#minSegments(int)
* @see ChronicleHashBuilder#actualSegments(int)
*/
int segments();
/**
* Checks the given predicate on each entry in this {@code ChronicleHash} until all entries
* have been processed or the predicate returns {@code false} for some entry, or throws
@@ -133,4 +147,11 @@ public interface ChronicleHash<K, E extends HashEntry<K>, SC extends HashSegment
*/
@Override
void close();
/**
* Tells whether or not this {@code ChronicleHash} (on-heap instance) is open.
*
* @return {@code true} is {@link #close()} is not yet called
*/
boolean isOpen();
}
src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilder.java
+21
View File
@@ -305,6 +305,27 @@ public interface ChronicleHashBuilder<K, H extends ChronicleHash<K, ?, ?, ?>,
*/
B timeProvider(TimeProvider timeProvider);
/**
* Configures timeout after which entries, marked as removed in the Chronicle Hash, constructed
* by this builder, are allowed to be completely removed from the data structure. In replicated
* Chronicle nodes, when {@code remove()} on the key is called, the corresponding entry
* is not immediately erased from the data structure, to let the distributed system eventually
* converge on some value for this key (or converge on the fact, that this key is removed).
* Chronicle Hash watch in runtime after the entries, and if one is removed and not updated
* in anyway for this {@code removedEntryCleanupTimeout}, Chronicle is allowed to remove this
* entry completely from the data structure. This timeout should depend on your distributed
* system topology, and typical replication latencies, that should be determined experimentally.
*
* <p>Default timeout is 1 second.
*
* @param removedEntryCleanupTimeout timeout, after which stale removed entries could be erased
* from Chronicle Hash data structure completely
* @param unit time unit, in which the timeout is given
* @return this builder back
* @throws IllegalArgumentException is the specified timeout is less than 1 millisecond
*/
B removedEntryCleanupTimeout(long removedEntryCleanupTimeout, TimeUnit unit);
/**
* Configures a {@link BytesMarshallerFactory} to be used with {@link
* BytesMarshallableSerializer}, which is a default {@link #objectSerializer ObjectSerializer},
src/main/java/net/openhft/chronicle/hash/ReplicatedHashSegmentContext.java
+30
View File
@@ -0,0 +1,30 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.hash;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import java.util.function.Consumer;
import java.util.function.Predicate;
public interface ReplicatedHashSegmentContext<K, E extends HashEntry<K>>
extends HashSegmentContext<K, E> {
void forEachSegmentReplicableEntry(Consumer<? super ReplicableEntry> action);
boolean forEachSegmentReplicableEntryWhile(Predicate<? super ReplicableEntry> predicate);
}
src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java
+16 -1
View File
@@ -114,6 +114,8 @@ public abstract class VanillaChronicleHash<K, KI, MKI extends MetaBytesInterop<K
public transient CompactOffHeapLinearHashTable hashLookup;
protected transient boolean closed = false;
public VanillaChronicleHash(
ChronicleMapBuilder<K, ?> builder, boolean replicated) {
// Version
@@ -262,13 +264,21 @@ public abstract class VanillaChronicleHash<K, KI, MKI extends MetaBytesInterop<K
}
@Override
public void close() {
public synchronized void close() {
if (closed)
return;
if (ms == null)
return;
bytes.release();
bytes = null;
ms.free();
ms = null;
closed = true;
}
@Override
public boolean isOpen() {
return !closed;
}
public final void checkKey(Object key) {
@@ -324,4 +334,9 @@ public abstract class VanillaChronicleHash<K, KI, MKI extends MetaBytesInterop<K
long size = longSize();
return size > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) size;
}
@Override
public int segments() {
return actualSegments;
}
}
src/main/java/net/openhft/chronicle/hash/impl/stage/entry/SegmentStages.java
+22 -12
View File
@@ -373,8 +373,7 @@ public abstract class SegmentStages implements SegmentLock {
entrySpaceOffset = 0;
}
//TODO refactor/optimize
public long alloc(int chunks) {
public long allocReturnCode(int chunks) {
VanillaChronicleHash<?, ?, ?, ?, ?, ?> h = hh.h();
if (chunks > h.maxChunksPerEntry)
throw new IllegalArgumentException("Entry is too large: requires " + chunks +
@@ -382,23 +381,17 @@ public abstract class SegmentStages implements SegmentLock {
long ret = freeList.setNextNContinuousClearBits(nextPosToSearchFrom(), chunks);
if (ret == DirectBitSet.NOT_FOUND || ret + chunks > h.actualChunksPerSegment) {
if (ret != DirectBitSet.NOT_FOUND &&
ret + chunks > h.actualChunksPerSegment && ret < h.actualChunksPerSegment)
ret + chunks > h.actualChunksPerSegment && ret < h.actualChunksPerSegment) {
freeList.clear(ret, h.actualChunksPerSegment);
}
ret = freeList.setNextNContinuousClearBits(0L, chunks);
if (ret == DirectBitSet.NOT_FOUND || ret + chunks > h.actualChunksPerSegment) {
if (ret != DirectBitSet.NOT_FOUND &&
ret + chunks > h.actualChunksPerSegment &&
ret < h.actualChunksPerSegment)
ret < h.actualChunksPerSegment) {
freeList.clear(ret, h.actualChunksPerSegment);
if (chunks == 1) {
throw new IllegalStateException(
"Segment is full, no free entries found");
} else {
throw new IllegalStateException(
"Segment is full or has no ranges of " + chunks
+ " continuous free chunks"
);
}
return -1;
}
updateNextPosToSearchFrom(ret, chunks);
} else {
@@ -412,6 +405,23 @@ public abstract class SegmentStages implements SegmentLock {
return ret;
}
public long alloc(int chunks) {
long ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
if (chunks == 1) {
throw new IllegalStateException(
"Segment is full, no free entries found");
} else {
throw new IllegalStateException(
"Segment is full or has no ranges of " + chunks
+ " continuous free chunks"
);
}
}
}
public void free(long fromPos, int chunks) {
freeList.clear(fromPos, fromPos + chunks);
if (fromPos < nextPosToSearchFrom())
src/main/java/net/openhft/chronicle/hash/impl/stage/hash/CheckOnEachPublicOperation.java
+4
View File
@@ -16,6 +16,7 @@
package net.openhft.chronicle.hash.impl.stage.hash;
import net.openhft.chronicle.hash.impl.VanillaChronicleHashHolder;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
@@ -23,8 +24,11 @@ import net.openhft.sg.Staged;
public abstract class CheckOnEachPublicOperation {
@StageRef OwnerThreadHolder holder;
@StageRef VanillaChronicleHashHolder<?, ?, ?> hh;
public void checkOnEachPublicOperation() {
holder.checkAccessingFromOwnerThread();
if (!hh.h().isOpen())
throw new IllegalStateException("Access to Chronicle Hash after close()");
}
}
src/main/java/net/openhft/chronicle/hash/impl/stage/iter/HashSegmentIteration.java
+17 -9
View File
@@ -17,6 +17,7 @@
package net.openhft.chronicle.hash.impl.stage.iter;
import net.openhft.chronicle.hash.HashEntry;
import net.openhft.chronicle.hash.HashSegmentContext;
import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable;
import net.openhft.chronicle.hash.impl.VanillaChronicleHashHolder;
import net.openhft.chronicle.hash.impl.stage.entry.HashEntryStages;
@@ -30,7 +31,8 @@ import java.util.function.Consumer;
import java.util.function.Predicate;
@Staged
public abstract class HashSegmentIteration<K, E extends HashEntry<K>> implements HashEntry<K> {
public abstract class HashSegmentIteration<K, E extends HashEntry<K>>
implements HashEntry<K>, HashSegmentContext<K, E> {
@StageRef public SegmentStages s;
@StageRef HashEntryStages<K> e;
@@ -50,6 +52,7 @@ public abstract class HashSegmentIteration<K, E extends HashEntry<K>> implements
this.entryRemovedOnThisIteration = entryRemovedOnThisIteration;
}
@Override
public boolean forEachSegmentEntryWhile(Predicate<? super E> action) {
s.innerUpdateLock.lock();
try {
@@ -88,6 +91,7 @@ public abstract class HashSegmentIteration<K, E extends HashEntry<K>> implements
}
}
@Override
public void forEachSegmentEntry(Consumer<? super E> action) {
forEachSegmentEntryWhile(e -> {
action.accept(e);
@@ -103,18 +107,22 @@ public abstract class HashSegmentIteration<K, E extends HashEntry<K>> implements
@Override
public void doRemove() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
initEntryRemovedOnThisIteration(true);
s.innerWriteLock.lock();
try {
// this condition mean -- some other entry taken place of the removed one
if (hh.h().hashLookup.remove(s.segmentBase, hlp.hashLookupPos) != hlp.hashLookupPos) {
// if so, should make step back, to compensate step forward on the next iteration,
// to consume the shifted entry
hlp.setHashLookupPos(hh.h().hashLookup.stepBack(hlp.hashLookupPos));
}
e.innerRemoveEntryExceptHashLookupUpdate();
iterationRemove();
} finally {
s.innerWriteLock.unlock();
}
initEntryRemovedOnThisIteration(true);
}
public void iterationRemove() {
// this condition mean -- some other entry taken place of the removed one
if (hh.h().hashLookup.remove(s.segmentBase, hlp.hashLookupPos) != hlp.hashLookupPos) {
// if so, should make step back, to compensate step forward on the next iteration,
// to consume the shifted entry
hlp.setHashLookupPos(hh.h().hashLookup.stepBack(hlp.hashLookupPos));
}
e.innerRemoveEntryExceptHashLookupUpdate();
}
}
src/main/java/net/openhft/chronicle/hash/impl/stage/query/HashQuery.java
+1 -1
View File
@@ -80,7 +80,7 @@ public abstract class HashQuery<K> implements HashEntry<K> {
public void doRemove() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
s.innerUpdateLock.lock();
if (entryPresent()) {
if (ks.searchStatePresent()) {
// TODO optimize: if shift-deletion is trivial, updateLock.lock()
s.innerWriteLock.lock();
hashLookupSearch.remove();
src/main/java/net/openhft/chronicle/hash/impl/stage/replication/ReplicableEntryDelegating.java
+59
View File
@@ -0,0 +1,59 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.hash.impl.stage.replication;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
public interface ReplicableEntryDelegating extends ReplicableEntry {
ReplicableEntry d();
@Override
default byte originIdentifier() {
return d().originIdentifier();
}
@Override
default long originTimestamp() {
return d().originTimestamp();
}
@Override
default void updateOrigin(byte newIdentifier, long newTimestamp) {
d().updateOrigin(newIdentifier, newTimestamp);
}
@Override
default void dropChanged() {
d().dropChanged();
}
@Override
default void raiseChanged() {
d().raiseChanged();
}
@Override
default boolean isChanged() {
return d().isChanged();
}
@Override
default void doRemoveCompletely() {
d().doRemoveCompletely();
}
}
src/main/java/net/openhft/chronicle/hash/replication/ReplicableEntry.java
+20
View File
@@ -86,4 +86,24 @@ public interface ReplicableEntry {
* Propagate the entry, schedule it to be replicated over to remote Chronicle nodes.
*/
void raiseChanged();
/**
* Check is the entry is scheduled to be replicated to the remote Chronicle nodes, to which
* the connection is currently established.
*
* @return {@code true} is the entry is "dirty" locally, i. e. should be replicated to remote
* Chronicle nodes, {@code false} otherwise
*/
boolean isChanged();
/**
* Completely remove the entry from the Chronicle hash, not just mark is as deleted, what
* {@link MapEntry#doRemove()} does for Replicated Chronicle hash. After calling this method
* the entry won't be replicated over to remote Chronicle nodes, as if {@link #dropChanged()}
* was called, and incoming entry replication event with the key if this entry will be
* processed, as the entry have never been present in the Chronicle hash. After calling this
* method any subsequent call of any method of {@code ReplicableEntry} class throws {@code
* IllegalStateException}.
*/
void doRemoveCompletely();
}
src/main/java/net/openhft/chronicle/hash/replication/TimeProvider.java
+24
View File
@@ -22,6 +22,7 @@ import net.openhft.chronicle.set.ChronicleSet;
import java.io.ObjectStreamException;
import java.io.Serializable;
import java.util.concurrent.TimeUnit;
/**
* TimeProvider was aims to possibly later provide an optimization to
@@ -44,14 +45,37 @@ public abstract class TimeProvider implements Serializable {
public abstract long currentTime();
/**
* {@code timeProvider.scale(System.currentTimeMillis(), TimeUnit.MILLISECONDS)} should
* result to {@link #currentTime()}.
*/
public abstract long scale(long time, TimeUnit unit);
/**
* {@code timeProvider.unscale(timeProvider.currentTime(), TimeUnit.NANOSECONDS)} should result
* to something close to {@code System.nanoTime()}.
*/
public abstract long unscale(long time, TimeUnit toUnit);
private static class System extends TimeProvider {
private static final long serialVersionUID = 1L;
private static final long SCALE = 1000;
@Override
public long currentTime() {
return java.lang.System.currentTimeMillis() * SCALE;
}
@Override
public long scale(long time, TimeUnit unit) {
return TimeUnit.MILLISECONDS.convert(time, unit) * SCALE;
}
@Override
public long unscale(long time, TimeUnit toUnit) {
return toUnit.convert(time, TimeUnit.MILLISECONDS) / SCALE;
}
@Override
public boolean equals(Object o) {
return o != null && o.getClass() == getClass();
src/main/java/net/openhft/chronicle/map/AbstractChronicleMap.java
+1 -3
View File
@@ -76,8 +76,6 @@ interface AbstractChronicleMap<K, V> extends ChronicleMap<K, V>, Serializable {
}
}
int actualSegments();
@Override
default boolean containsValue(Object value) {
return !forEachEntryWhile(c -> !c.value().equals(c.context().wrapValueAsData((V) value)));
@@ -257,7 +255,7 @@ interface AbstractChronicleMap<K, V> extends ChronicleMap<K, V>, Serializable {
@Override
default boolean forEachEntryWhile(final Predicate<? super MapEntry<K, V>> action) {
boolean interrupt = false;
for (int i = actualSegments() - 1; i >= 0; i--) {
for (int i = segments() - 1; i >= 0; i--) {
try (MapSegmentContext<K, V, ?> c = segmentContext(i)) {
if (!c.forEachSegmentEntryWhile(action)) {
interrupt = true;
src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java
+67 -2
View File
@@ -37,6 +37,7 @@ import net.openhft.lang.io.serialization.impl.NewInstanceObjectFactory;
import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory;
import net.openhft.lang.model.Byteable;
import net.openhft.lang.model.DataValueClasses;
import net.openhft.lang.thread.NamedThreadFactory;
import net.openhft.lang.threadlocal.Provider;
import net.openhft.lang.threadlocal.ThreadLocalCopies;
import org.jetbrains.annotations.NotNull;
@@ -48,8 +49,11 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -159,6 +163,9 @@ public final class ChronicleMapBuilder<K, V> implements
// replication
private TimeProvider timeProvider = TimeProvider.SYSTEM;
long cleanupTimeout = 1;
TimeUnit cleanupTimeoutUnit = TimeUnit.SECONDS;
private BytesMarshallerFactory bytesMarshallerFactory;
private ObjectSerializer objectSerializer;
private V defaultValue = null;
@@ -1056,6 +1063,18 @@ public final class ChronicleMapBuilder<K, V> implements
return this;
}
@Override
public ChronicleMapBuilder<K, V> removedEntryCleanupTimeout(
long removedEntryCleanupTimeout, TimeUnit unit) {
if (unit.toMillis(removedEntryCleanupTimeout) < 1) {
throw new IllegalArgumentException("timeout should be >= 1 millisecond, " +
removedEntryCleanupTimeout + " " + unit + " is given");
}
cleanupTimeout = removedEntryCleanupTimeout;
cleanupTimeoutUnit = unit;
return this;
}
TimeProvider timeProvider() {
return timeProvider;
}
@@ -1548,10 +1567,12 @@ public final class ChronicleMapBuilder<K, V> implements
SingleChronicleHashReplication singleHashReplication,
ReplicationChannel channel) throws IOException {
if (map instanceof ReplicatedChronicleMap) {
if (singleHashReplication != null && channel != null) {
if (singleHashReplication != null && channel != null)
throw new AssertionError("Only one non-null replication should be passed");
}
ReplicatedChronicleMap result = (ReplicatedChronicleMap) map;
establishCleanupThread(result);
List<Replicator> replicators = new ArrayList<>(2);
if (singleHashReplication != null) {
if (singleHashReplication.tcpTransportAndNetwork() != null)
@@ -1577,6 +1598,50 @@ public final class ChronicleMapBuilder<K, V> implements
return map;
}
private void establishCleanupThread(ReplicatedChronicleMap map) {
map.globalStateLock();
try {
if (!map.globalMutableState.isCurrentlyCleanupIterated()) {
OldDeletedEntriesCleanup cleanup = new OldDeletedEntriesCleanup(map,
timeProvider.scale(cleanupTimeout, cleanupTimeoutUnit));
NamedThreadFactory threadFactory =
new NamedThreadFactory("cleanup thread for map persisted at " + map.file());
ExecutorService executor = Executors.newSingleThreadExecutor(threadFactory);
executor.submit(cleanup);
map.addCloseable(cleanup);
// WARNING this relies on the fact that ReplicatedChronicleMap closes in the same
// order, i. e. OldDeletedEntriesCleanup instance first
map.addCloseable(() -> {
executor.shutdown();
try {
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
LOG.error("", e);
}
});
map.addCloseable(() -> {
map.globalStateLock();
try {
map.globalMutableState.setCurrentlyCleanupIterated(false);
} finally {
map.globalStateUnlock();
}
});
map.globalStateLock();
try {
if (!map.globalMutableState.isCurrentlyCleanupIterated())
map.globalMutableState.setCurrentlyCleanupIterated(true);
} finally {
map.globalStateUnlock();
}
}
} finally {
map.globalStateUnlock();
}
}
private long bufferSize(SerializationBuilder builder, double averageSize, boolean replicated) {
if (builder.constantSizeMarshaller())
return round(ceil(averageSize));
src/main/java/net/openhft/chronicle/map/ChronicleMapIterator.java
+1 -1
View File
@@ -36,7 +36,7 @@ abstract class ChronicleMapIterator<K, V, E> implements Iterator<E>, Consumer<Ma
ChronicleMapIterator(AbstractChronicleMap<K, V> map) {
this.map = map;
segmentIndex = map.actualSegments() - 1;
segmentIndex = map.segments() - 1;
}
private void checkSingleThreaded() {
src/main/java/net/openhft/chronicle/map/OldDeletedEntriesCleanup.java
+111
View File
@@ -0,0 +1,111 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map;
import net.openhft.chronicle.core.io.Closeable;
import net.openhft.chronicle.hash.ReplicatedHashSegmentContext;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.hash.replication.TimeProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
public class OldDeletedEntriesCleanup implements Runnable, Closeable, Predicate<ReplicableEntry> {
private static final Logger LOG = LoggerFactory.getLogger(OldDeletedEntriesCleanup.class);
private static final int PRIME_1 = 1_000_003, PRIME_2 = 999_983;
private final ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map;
private final long timeout;
private final int step;
private volatile boolean shutdown;
private long prevSegment0ScanStart = -1;
private long removedCompletely;
public OldDeletedEntriesCleanup(ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map, long timeout) {
this.map = map;
this.timeout = timeout;
// step primality guarantees perfect segment permutation
step = map.segments() != PRIME_1 ? PRIME_1 : PRIME_2;
}
@Override
public void run() {
while (!shutdown) {
int segmentIndex = map.globalMutableState.getCurrentCleanupSegmentIndex();
TimeProvider timeProvider = map.timeProvider;
if (segmentIndex == 0 && prevSegment0ScanStart >= 0) {
long currentTime = timeProvider.currentTime();
long mapScanTime = currentTime - prevSegment0ScanStart;
LOG.debug("Old deleted entries scan time: {}", mapScanTime);
if (mapScanTime < timeout) {
long timeToSleep =
timeProvider.unscale(timeout - mapScanTime, TimeUnit.MILLISECONDS);
if (timeToSleep > 0) {
try {
Thread.sleep(timeToSleep);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
} else {
timeToSleep = timeProvider.unscale(timeout - mapScanTime,
TimeUnit.NANOSECONDS);
try {
Thread.sleep(0, (int) timeToSleep);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}
}
try (MapSegmentContext<?, ?, ?> context = map.segmentContext(segmentIndex)) {
if (segmentIndex == 0)
prevSegment0ScanStart = timeProvider.currentTime();
removedCompletely = 0;
if (((ReplicatedHashSegmentContext<?, ?>) context)
.forEachSegmentReplicableEntryWhile(this)) {
segmentIndex = (segmentIndex + step) % map.segments();
map.globalMutableState.setCurrentCleanupSegmentIndex(segmentIndex);
}
}
LOG.debug("Removed {} old deleted entries in the segment {}",
removedCompletely, segmentIndex);
}
}
@Override
public boolean test(ReplicableEntry e) {
if (shutdown)
return false;
if (e instanceof MapAbsentEntry) {
long deleteTimeout = map.timeProvider.currentTime() - e.originTimestamp();
if (deleteTimeout > timeout && !e.isChanged()) {
e.doRemoveCompletely();
removedCompletely++;
}
}
return true;
}
@Override
public void close() {
shutdown = true;
}
}
src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java
+51 -8
View File
@@ -16,6 +16,7 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.ReplicatedHashSegmentContext;
import net.openhft.chronicle.hash.replication.AbstractReplication;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.hash.replication.TimeProvider;
@@ -28,6 +29,7 @@ import net.openhft.lang.Maths;
import net.openhft.lang.collection.ATSDirectBitSet;
import net.openhft.lang.collection.SingleThreadedDirectBitSet;
import net.openhft.lang.io.Bytes;
import net.openhft.lang.model.DataValueClasses;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,16 +95,21 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
private static final Logger LOG = LoggerFactory.getLogger(ReplicatedChronicleMap.class);
private static final long LAST_UPDATED_HEADER_SIZE = 128L * 8L;
private static final int SIZE_OF_BOOTSTRAP_TIME_STAMP = 8;
private static final long GLOBAL_MUTABLE_STATE_SIZE = 64; // a cache line
public final TimeProvider timeProvider;
private final byte localIdentifier;
transient Set<Closeable> closeables;
private transient Bytes identifierUpdatedBytes;
transient Bytes globalMutableStateBytes;
transient ReplicatedChronicleMapGlobalMutableState globalMutableState;
private transient ATSDirectBitSet modIterSet;
private transient AtomicReferenceArray<ModificationIterator> modificationIterators;
private transient long startOfModificationIterators;
private boolean bootstrapOnlyLocalEntries;
public long cleanupTimeout;
public transient MapRemoteOperations<K, V, R> remoteOperations;
transient CompiledReplicatedMapQueryContext<K, KI, MKI, V, VI, MVI, R, ?> remoteOpContext;
@@ -118,6 +125,8 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
this.localIdentifier = replication.identifier();
this.bootstrapOnlyLocalEntries = replication.bootstrapOnlyLocalEntries();
cleanupTimeout = timeProvider.scale(builder.cleanupTimeout, builder.cleanupTimeoutUnit);
if (localIdentifier == -1) {
throw new IllegalStateException("localIdentifier should not be -1");
}
@@ -181,7 +190,7 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
@Override
public long mapHeaderInnerSize() {
return super.mapHeaderInnerSize() + LAST_UPDATED_HEADER_SIZE +
return super.mapHeaderInnerSize() + GLOBAL_MUTABLE_STATE_SIZE + LAST_UPDATED_HEADER_SIZE +
(modIterBitSetSizeInBytes() * (128 + RESERVED_MOD_ITER)) +
assignedModIterBitSetSizeInBytes();
}
@@ -208,6 +217,12 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
public void onHeaderCreated() {
long offset = super.mapHeaderInnerSize();
globalMutableStateBytes = ms.bytes(offset, GLOBAL_MUTABLE_STATE_SIZE);
globalMutableState =
DataValueClasses.newDirectReference(ReplicatedChronicleMapGlobalMutableState.class);
globalMutableState.bytes(globalMutableStateBytes, 8);
offset += GLOBAL_MUTABLE_STATE_SIZE;
identifierUpdatedBytes = ms.bytes(offset, LAST_UPDATED_HEADER_SIZE);
offset += LAST_UPDATED_HEADER_SIZE;
@@ -217,6 +232,18 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
modIterSet = new ATSDirectBitSet(modDelBytes);
}
void globalStateLock() {
try {
globalMutableStateBytes.busyLockLong(0);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
void globalStateUnlock() {
globalMutableStateBytes.unlockLong(0);
}
@Override
public void clear() {
forEachEntry(c -> c.context().remove(c));
@@ -227,7 +254,9 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
}
@Override
public void close() {
public synchronized void close() {
if (closed)
return;
for (Closeable closeable : closeables) {
try {
closeable.close();
@@ -304,6 +333,17 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
}
}
public boolean isChanged(long segmentIndex, long pos) {
for (long next = modIterSet.nextSetBit(0L); next > 0L;
next = modIterSet.nextSetBit(next + 1L)) {
ModificationIterator modificationIterator =
acquireModificationIterator((byte) next);
if (modificationIterator.isChanged(segmentIndex, pos))
return true;
}
return false;
}
@Override
public boolean identifierCheck(@NotNull ReplicableEntry entry, int chronicleId) {
return entry.originIdentifier() == localIdentifier;
@@ -551,6 +591,10 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
return changesForUpdates.clearIfSet(combine(segmentIndex, pos));
}
boolean isChanged(long segmentIndex, long pos) {
return changesForUpdates.isSet(combine(segmentIndex, pos));
}
/**
* you can continue to poll hasNext() until data becomes available. If are are in the middle
* of processing an entry via {@code nextEntry}, hasNext will return true until the bit is
@@ -648,23 +692,22 @@ public class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? sup
for (int i = 0; i < actualSegments; i++) {
final int segmentIndex = i;
c.initTheSegmentIndex(segmentIndex);
c.forEachReplicableEntry(entry -> {
c.forEachSegmentReplicableEntry(e -> {
if (debugEnabled) {
LOG.debug("Bootstrap entry: id {}, key {}, value {}", localIdentifier,
c.key(), c.value());
}
MapReplicableEntry re = (MapReplicableEntry) entry;
// Bizarrely the next line line cause NPE in JDT compiler
//assert re.originTimestamp() > 0L;
if (debugEnabled) {
LOG.debug("Bootstrap decision: bs ts: {}, entry ts: {}, " +
"entry id: {}, local id: {}",
fromTimeStamp, re.originTimestamp(),
re.originIdentifier(), localIdentifier);
fromTimeStamp, e.originTimestamp(),
e.originIdentifier(), localIdentifier);
}
if (re.originTimestamp() >= fromTimeStamp &&
if (e.originTimestamp() >= fromTimeStamp &&
(!bootstrapOnlyLocalEntries ||
re.originIdentifier() == localIdentifier)) {
e.originIdentifier() == localIdentifier)) {
raiseChange(segmentIndex, c.pos(), c.timestamp());
}
});
src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMapGlobalMutableState.java
+30
View File
@@ -0,0 +1,30 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map;
import net.openhft.lang.model.Byteable;
interface ReplicatedChronicleMapGlobalMutableState extends Byteable {
int getCurrentCleanupSegmentIndex();
void setCurrentCleanupSegmentIndex(int currentCleanupSegmentIndex);
boolean isCurrentlyCleanupIterated();
void setCurrentlyCleanupIterated(boolean isCurrentlyCleanupIterated);
}
src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java
+10
View File
@@ -409,6 +409,11 @@ class StatelessChronicleMap<K, V> implements ChronicleMap<K, V>, Closeable, Clon
}
@Override
public boolean isOpen() {
return closeables != null;
}
/**
* the transaction id are generated as unique timestamps
*
@@ -558,6 +563,11 @@ class StatelessChronicleMap<K, V> implements ChronicleMap<K, V>, Closeable, Clon
throw new UnsupportedOperationException();
}
@Override
public int segments() {
throw new UnsupportedOperationException();
}
public V get(Object key) {
return fetchObject(vClass, GET, (K) key);
}
src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java
-5
View File
@@ -199,11 +199,6 @@ public class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super
return (V) value;
}
@Override
public int actualSegments() {
return actualSegments;
}
@Override
public V newValueInstance() {
if (vClass == CharSequence.class || vClass == StringBuilder.class)
src/main/java/net/openhft/chronicle/map/impl/MapQueryContext.java
+2 -4
View File
@@ -33,10 +33,7 @@ import net.openhft.chronicle.map.impl.stage.data.bytes.InputSecondValueBytesData
import net.openhft.chronicle.map.impl.stage.data.instance.InputValueInstanceData;
import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceData;
import net.openhft.chronicle.map.impl.stage.input.MapInputBytesValues;
import net.openhft.chronicle.map.impl.stage.map.MapEntryOperationsDelegation;
import net.openhft.chronicle.map.impl.stage.map.ValueBytesInterop;
import net.openhft.chronicle.map.impl.stage.map.VanillaChronicleMapHolderImpl;
import net.openhft.chronicle.map.impl.stage.map.WrappedValueInstanceValueHolder;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.query.*;
import net.openhft.chronicle.map.impl.stage.ret.DefaultReturnValue;
import net.openhft.chronicle.map.impl.stage.ret.UsingReturnValue;
@@ -65,6 +62,7 @@ import net.openhft.sg.Staged;
WrappedValueInstanceValueHolder.class,
MapQuery.class,
MapAbsent.class,
DefaultValue.class,
ValueBytesInterop.class,
MapAbsentHolder.class,
src/main/java/net/openhft/chronicle/map/impl/ReplicatedIterationContextInterface.java
-6
View File
@@ -16,14 +16,8 @@
package net.openhft.chronicle.map.impl;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import java.util.function.Consumer;
public interface ReplicatedIterationContextInterface<K, V, R>
extends IterationContextInterface<K, V, R> {
void forEachReplicableEntry(Consumer<? super ReplicableEntry> action);
void readExistingEntry(long pos);
}
src/main/java/net/openhft/chronicle/map/impl/ReplicatedMapIterationContext.java
+8 -4
View File
@@ -26,11 +26,11 @@ import net.openhft.chronicle.map.impl.stage.data.bytes.EntryValueBytesData;
import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceData;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.chronicle.map.impl.stage.iter.IterationCheckOnEachPublicOperation;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapAbsentDelegatingForIteration;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapEntryDelegating;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapSegmentIteration;
import net.openhft.chronicle.map.impl.stage.map.MapEntryOperationsDelegation;
import net.openhft.chronicle.map.impl.stage.map.ReplicatedChronicleMapHolderImpl;
import net.openhft.chronicle.map.impl.stage.map.ValueBytesInterop;
import net.openhft.chronicle.map.impl.stage.map.WrappedValueInstanceValueHolder;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.query.ReplicatedMapAbsent;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.sg.Context;
import net.openhft.sg.Staged;
@@ -56,6 +56,7 @@ import net.openhft.sg.Staged;
MapEntryOperationsDelegation.class,
ReplicationUpdate.class,
DefaultValue.class,
}, nested = {
ReadLock.class,
UpdateLock.class,
@@ -67,6 +68,9 @@ import net.openhft.sg.Staged;
WrappedValueInstanceData.class,
DummyValueZeroData.class,
ReplicatedMapAbsentDelegatingForIteration.class,
ReplicatedMapEntryDelegating.class
})
public class ReplicatedMapIterationContext {
}
src/main/java/net/openhft/chronicle/map/impl/ReplicatedMapQueryContext.java
+3 -5
View File
@@ -35,10 +35,7 @@ import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceDa
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.chronicle.map.impl.stage.input.MapInputBytesValues;
import net.openhft.chronicle.map.impl.stage.input.ReplicatedInput;
import net.openhft.chronicle.map.impl.stage.map.MapEntryOperationsDelegation;
import net.openhft.chronicle.map.impl.stage.map.ReplicatedChronicleMapHolderImpl;
import net.openhft.chronicle.map.impl.stage.map.ValueBytesInterop;
import net.openhft.chronicle.map.impl.stage.map.WrappedValueInstanceValueHolder;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.query.*;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.chronicle.map.impl.stage.ret.BytesReturnValue;
@@ -58,7 +55,7 @@ import net.openhft.sg.Staged;
ReplicatedChronicleMapHolderImpl.class,
KeyBytesInterop.class,
QuerySegmentStages.class,
ReplicatedQuerySegmentStages.class,
KeySearch.class,
HashLookupSearch.class,
QueryHashLookupPos.class,
@@ -71,6 +68,7 @@ import net.openhft.sg.Staged;
WrappedValueInstanceValueHolder.class,
ReplicatedMapQuery.class,
ReplicatedMapAbsent.class,
DefaultValue.class,
ValueBytesInterop.class,
ReplicatedMapAbsentHolder.class,
ReplicationUpdate.class,
src/main/java/net/openhft/chronicle/map/impl/stage/entry/ReplicatedMapEntryStages.java
+16 -1
View File
@@ -18,18 +18,20 @@ package net.openhft.chronicle.map.impl.stage.entry;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.replication.TimeProvider;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.impl.ReplicatedChronicleMapHolder;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.chronicle.map.replication.MapReplicableEntry;
import net.openhft.sg.Stage;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
import static net.openhft.chronicle.map.ReplicatedChronicleMap.ADDITIONAL_ENTRY_BYTES;
@Staged
public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K, V>
implements MapReplicableEntry<K, V> {
implements MapReplicableEntry<K, V>, MapAbsentEntry<K, V> {
@StageRef ReplicatedChronicleMapHolder<?, ?, ?, ?, ?, ?, ?> mh;
@StageRef ReplicationUpdate ru;
@@ -116,6 +118,13 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
ru.raiseChange();
}
@Override
public boolean isChanged() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
s.innerReadLock.lock();
return ru.changed();
}
public void updatedReplicationStateOnPresentEntry() {
if (!ru.replicationUpdateInit()) {
s.innerWriteLock.lock();
@@ -158,4 +167,10 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
long sizeOfEverythingBeforeValue(long keySize, long valueSize) {
return super.sizeOfEverythingBeforeValue(keySize, valueSize) + ADDITIONAL_ENTRY_BYTES;
}
@NotNull
@Override
public Data<K> absentKey() {
return key();
}
}
src/main/java/net/openhft/chronicle/map/impl/stage/iter/ReplicatedMapAbsentDelegatingForIteration.java
+63
View File
@@ -0,0 +1,63 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.iter;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.replication.ReplicableEntryDelegating;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.MapContext;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
@Staged
public class ReplicatedMapAbsentDelegatingForIteration<K, V>
implements MapAbsentEntry<K, V>, ReplicableEntryDelegating {
@StageRef ReplicatedMapSegmentIteration<K, V, ?> delegate;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@NotNull
@Override
public MapContext<K, V, ?> context() {
return delegate.context();
}
@Override
public void doInsert(Data<V> value) {
delegate.doInsert(value);
}
@NotNull
@Override
public Data<V> defaultValue() {
return delegate.defaultValue();
}
@NotNull
@Override
public Data<K> absentKey() {
return delegate.absentKey();
}
@Override
public ReplicableEntry d() {
return e;
}
}
src/main/java/net/openhft/chronicle/map/impl/stage/iter/ReplicatedMapEntryDelegating.java
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.iter;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.replication.ReplicableEntryDelegating;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapContext;
import net.openhft.chronicle.map.MapEntry;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
@Staged
public class ReplicatedMapEntryDelegating<K, V>
implements MapEntry<K, V>, ReplicableEntryDelegating {
@StageRef ReplicatedMapSegmentIteration<K, V, ?> delegate;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@NotNull
@Override
public MapContext<K, V, ?> context() {
return delegate.context();
}
@NotNull
@Override
public Data<K> key() {
return delegate.key();
}
@NotNull
@Override
public Data<V> value() {
return delegate.value();
}
@Override
public void doReplaceValue(Data<V> newValue) {
delegate.doReplaceValue(newValue);
}
@Override
public void doRemove() {
delegate.doRemove();
}
@Override
public ReplicableEntry d() {
return e;
}
}
src/main/java/net/openhft/chronicle/map/impl/stage/iter/ReplicatedMapSegmentIteration.java
+59 -7
View File
@@ -17,9 +17,12 @@
package net.openhft.chronicle.map.impl.stage.iter;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.ReplicatedHashSegmentContext;
import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable;
import net.openhft.chronicle.hash.impl.VanillaChronicleHashHolder;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.MapEntry;
import net.openhft.chronicle.map.impl.ReplicatedIterationContextInterface;
import net.openhft.chronicle.map.impl.stage.data.DummyValueZeroData;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
@@ -28,15 +31,19 @@ import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import java.util.function.Consumer;
import java.util.function.Predicate;
@Staged
public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentIteration<K, V, R>
implements ReplicatedIterationContextInterface<K, V, R>, ReplicableEntry {
implements ReplicatedIterationContextInterface<K, V, R>, ReplicableEntry,
ReplicatedHashSegmentContext<K, MapEntry<K, V>>, MapAbsentEntry<K, V> {
@StageRef VanillaChronicleHashHolder<?, ?, ?> hh;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@StageRef ReplicationUpdate<K> ru;
@StageRef DummyValueZeroData<V> dummyValue;
@StageRef ReplicatedMapAbsentDelegatingForIteration<K, V> absentEntryDelegating;
@StageRef ReplicatedMapEntryDelegating<K, V> entryDelegating;
@Override
public boolean entryIsPresent() {
@@ -56,12 +63,14 @@ public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentI
}
@Override
public void forEachReplicableEntry(Consumer<? super ReplicableEntry> action) {
public boolean forEachSegmentReplicableEntryWhile(
Predicate<? super ReplicableEntry> predicate) {
s.innerUpdateLock.lock();
try {
long entries = s.entries();
if (entries == 0)
return;
return true;
boolean interrupted = false;
long startPos = 0L;
CompactOffHeapLinearHashTable hashLookup = hh.h().hashLookup;
while (!hashLookup.empty(hashLookup.readEntry(s.segmentBase, startPos))) {
@@ -73,22 +82,36 @@ public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentI
long entry = hashLookup.readEntry(s.segmentBase, hlp.hashLookupPos);
if (!hashLookup.empty(entry)) {
e.readExistingEntry(hashLookup.value(entry));
action.accept(this);
if (--entries == 0)
ReplicableEntry e = !this.e.entryDeleted() ? entryDelegating :
absentEntryDelegating;
initEntryRemovedOnThisIteration(false);
if (!predicate.test(e)) {
interrupted = true;
break;
} else {
if (--entries == 0)
break;
}
}
} while (hlp.hashLookupPos != startPos);
return !interrupted;
} finally {
s.innerReadLock.unlock();
initEntryRemovedOnThisIteration(false);
}
}
@Override
public void forEachSegmentReplicableEntry(Consumer<? super ReplicableEntry> action) {
forEachSegmentReplicableEntryWhile(e -> {
action.accept(e);
return true;
});
}
@Override
public void doRemove() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
initEntryRemovedOnThisIteration(true);
try {
if (e.valueSize > dummyValue.size())
e.innerDefaultReplaceValue(dummyValue);
@@ -99,5 +122,34 @@ public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentI
} finally {
s.innerWriteLock.unlock();
}
initEntryRemovedOnThisIteration(true);
}
@Override
public void doRemoveCompletely() {
boolean wasDeleted = e.entryDeleted();
super.doRemove();
if (wasDeleted)
s.deleted(s.deleted() - 1);
}
@Override
public void doInsert(Data<V> value) {
checkOnEachPublicOperation.checkOnEachPublicOperation();
if (e.entryDeleted()) {
try {
e.innerDefaultReplaceValue(value);
s.deleted(s.deleted() - 1);
s.incrementModCount();
e.writeEntryPresent();
ru.updateChange();
e.updatedReplicationStateOnPresentEntry();
} finally {
s.innerWriteLock.unlock();
}
} else {
throw new IllegalStateException(
"Entry is present in the map when doInsert() is called");
}
}
}
src/main/java/net/openhft/chronicle/map/impl/stage/map/DefaultValue.java
+45
View File
@@ -0,0 +1,45 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.map;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.hash.CheckOnEachPublicOperation;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.impl.VanillaChronicleMapHolder;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
@Staged
public abstract class DefaultValue<K, V> implements MapAbsentEntry<K, V> {
@StageRef CheckOnEachPublicOperation checkOnEachPublicOperation;
@StageRef VanillaChronicleMapHolder<K, ?, ?, V, ?, ?, ?> mh;
@NotNull
@Override
public Data<V> defaultValue() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
if (mh.m().constantValueProvider == null) {
throw new IllegalStateException("to call acquireUsing(), " +
"or defaultValue() on AbsentEntry, you should configure " +
"ChronicleMapBuilder.defaultValue(), or use one of the 'known' value types: " +
"boxed primitives, or so-called data-value-generated interface as a value");
}
return context().wrapValueAsData(mh.m().constantValueProvider.defaultValue());
}
}
src/main/java/net/openhft/chronicle/map/impl/stage/query/MapAbsent.java
+1 -14
View File
@@ -32,7 +32,7 @@ import org.jetbrains.annotations.NotNull;
import static net.openhft.chronicle.hash.impl.stage.query.KeySearch.SearchState.PRESENT;
@Staged
public class MapAbsent<K, V> implements MapAbsentEntry<K, V> {
public abstract class MapAbsent<K, V> implements MapAbsentEntry<K, V> {
@StageRef public KeySearch<K> ks;
@StageRef MapQuery<K, V, ?> q;
@@ -81,17 +81,4 @@ public class MapAbsent<K, V> implements MapAbsentEntry<K, V> {
"Entry is present in the map when doInsert() is called");
}
}
@NotNull
@Override
public Data<V> defaultValue() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
if (mh.m().constantValueProvider == null) {
throw new IllegalStateException("to call acquireUsing(), " +
"or defaultValue() on AbsentEntry, you should configure " +
"ChronicleMapBuilder.defaultValue(), or use one of the 'known' value types: " +
"boxed primitives, or so-called data-value-generated interface as a value");
}
return context().wrapValueAsData(mh.m().constantValueProvider.defaultValue());
}
}
src/main/java/net/openhft/chronicle/map/impl/stage/query/ReplicatedMapAbsent.java
+1 -1
View File
@@ -25,7 +25,7 @@ import net.openhft.sg.Staged;
import static net.openhft.chronicle.hash.impl.stage.query.KeySearch.SearchState.PRESENT;
@Staged
public class ReplicatedMapAbsent<K, V> extends MapAbsent<K, V> {
public abstract class ReplicatedMapAbsent<K, V> extends MapAbsent<K, V> {
@StageRef MapQuery<K, V, ?> q;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
src/main/java/net/openhft/chronicle/map/impl/stage/query/ReplicatedMapQuery.java
+12 -1
View File
@@ -17,6 +17,7 @@
package net.openhft.chronicle.map.impl.stage.query;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.impl.stage.data.DummyValueZeroData;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
@@ -28,9 +29,11 @@ import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.Nullable;
import static net.openhft.chronicle.hash.impl.stage.query.KeySearch.SearchState.DELETED;
@Staged
public abstract class ReplicatedMapQuery<K, V, R> extends MapQuery<K, V, R>
implements MapRemoteQueryContext<K, V, R> {
implements MapRemoteQueryContext<K, V, R>, ReplicableEntry {
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@StageRef ReplicationUpdate ru;
@@ -83,6 +86,14 @@ public abstract class ReplicatedMapQuery<K, V, R> extends MapQuery<K, V, R>
}
}
@Override
public void doRemoveCompletely() {
boolean wasDeleted = e.entryDeleted();
super.doRemove();
if (wasDeleted)
s.deleted(s.deleted() - 1L);
}
@Override
public void doReplaceValue(Data<V> newValue) {
super.doReplaceValue(newValue);
src/main/java/net/openhft/chronicle/map/impl/stage/query/ReplicatedQuerySegmentStages.java
+71
View File
@@ -0,0 +1,71 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.query;
import net.openhft.chronicle.hash.ReplicatedHashSegmentContext;
import net.openhft.chronicle.hash.impl.stage.query.QuerySegmentStages;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.MapSegmentContext;
import net.openhft.chronicle.map.ReplicatedChronicleMap;
import net.openhft.chronicle.map.impl.ReplicatedChronicleMapHolder;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import java.util.function.Consumer;
@Staged
public abstract class ReplicatedQuerySegmentStages extends QuerySegmentStages {
@StageRef
ReplicatedChronicleMapHolder<?, ?, ?, ?, ?, ?, ?> mh;
final Consumer<ReplicableEntry> cleanupAction = e -> {
ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map = mh.m();
if (e instanceof MapAbsentEntry) {
long deleteTimeout = map.timeProvider.currentTime() - e.originTimestamp();
if (deleteTimeout > map.cleanupTimeout && !e.isChanged())
e.doRemoveCompletely();
}
};
@Override
public long alloc(int chunks) {
long ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map = mh.m();
MapSegmentContext<?, ?, ?> sc = map.segmentContext(segmentIndex);
((ReplicatedHashSegmentContext<?, ?>) sc).forEachSegmentReplicableEntry(cleanupAction);
ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
if (chunks == 1) {
throw new IllegalStateException(
"Segment is full, no free entries found");
} else {
throw new IllegalStateException(
"Segment is full or has no ranges of " + chunks
+ " continuous free chunks"
);
}
}
}
}
}
src/main/java/net/openhft/chronicle/map/impl/stage/replication/ReplicationUpdate.java
+4
View File
@@ -62,6 +62,10 @@ public abstract class ReplicationUpdate<K> implements RemoteOperationContext<K>
mh.m().raiseChange(s.segmentIndex, e.pos, e.timestamp());
}
public boolean changed() {
return mh.m().isChanged(s.segmentIndex, e.pos);
}
@Override
public long remoteTimestamp() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
src/main/java/net/openhft/chronicle/set/ChronicleSetBuilder.java
+8
View File
@@ -37,6 +37,7 @@ import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
/**
* {@code ChronicleSetBuilder} manages the whole set of {@link ChronicleSet} configurations, could
@@ -189,6 +190,13 @@ public final class ChronicleSetBuilder<K>
return this;
}
@Override
public ChronicleSetBuilder<K> removedEntryCleanupTimeout(
long removedEntryCleanupTimeout, TimeUnit unit) {
chronicleMapBuilder.removedEntryCleanupTimeout(removedEntryCleanupTimeout, unit);
return this;
}
@Override
public ChronicleSetBuilder<K> bytesMarshallerFactory(
BytesMarshallerFactory bytesMarshallerFactory) {
src/main/java/net/openhft/chronicle/set/SetFromMap.java
+10
View File
@@ -144,6 +144,11 @@ class SetFromMap<E> extends AbstractSet<E>
throw new UnsupportedOperationException();
}
@Override
public int segments() {
return m.segments();
}
@Override
public boolean forEachEntryWhile(Predicate<? super SetEntry<E>> predicate) {
throw new UnsupportedOperationException();
@@ -163,4 +168,9 @@ class SetFromMap<E> extends AbstractSet<E>
public void close() {
m.close();
}
@Override
public boolean isOpen() {
return m.isOpen();
}
}