HCOLL-343 HCOLL-227 Remove old entries from ReplicatedChronicleMap

This commit is contained in:
leventov
2015-08-23 00:01:44 +03:00
parent a3a45e1681
commit 904ecca036
40 changed files with 945 additions and 90 deletions
@@ -33,10 +33,7 @@ import net.openhft.chronicle.map.impl.stage.data.bytes.InputSecondValueBytesData
import net.openhft.chronicle.map.impl.stage.data.instance.InputValueInstanceData;
import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceData;
import net.openhft.chronicle.map.impl.stage.input.MapInputBytesValues;
import net.openhft.chronicle.map.impl.stage.map.MapEntryOperationsDelegation;
import net.openhft.chronicle.map.impl.stage.map.ValueBytesInterop;
import net.openhft.chronicle.map.impl.stage.map.VanillaChronicleMapHolderImpl;
import net.openhft.chronicle.map.impl.stage.map.WrappedValueInstanceValueHolder;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.query.*;
import net.openhft.chronicle.map.impl.stage.ret.DefaultReturnValue;
import net.openhft.chronicle.map.impl.stage.ret.UsingReturnValue;
@@ -65,6 +62,7 @@ import net.openhft.sg.Staged;
WrappedValueInstanceValueHolder.class,
MapQuery.class,
MapAbsent.class,
DefaultValue.class,
ValueBytesInterop.class,
MapAbsentHolder.class,
@@ -16,14 +16,8 @@
package net.openhft.chronicle.map.impl;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import java.util.function.Consumer;
public interface ReplicatedIterationContextInterface<K, V, R>
extends IterationContextInterface<K, V, R> {
void forEachReplicableEntry(Consumer<? super ReplicableEntry> action);
void readExistingEntry(long pos);
}
@@ -26,11 +26,11 @@ import net.openhft.chronicle.map.impl.stage.data.bytes.EntryValueBytesData;
import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceData;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.chronicle.map.impl.stage.iter.IterationCheckOnEachPublicOperation;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapAbsentDelegatingForIteration;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapEntryDelegating;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapSegmentIteration;
import net.openhft.chronicle.map.impl.stage.map.MapEntryOperationsDelegation;
import net.openhft.chronicle.map.impl.stage.map.ReplicatedChronicleMapHolderImpl;
import net.openhft.chronicle.map.impl.stage.map.ValueBytesInterop;
import net.openhft.chronicle.map.impl.stage.map.WrappedValueInstanceValueHolder;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.query.ReplicatedMapAbsent;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.sg.Context;
import net.openhft.sg.Staged;
@@ -56,6 +56,7 @@ import net.openhft.sg.Staged;
MapEntryOperationsDelegation.class,
ReplicationUpdate.class,
DefaultValue.class,
}, nested = {
ReadLock.class,
UpdateLock.class,
@@ -67,6 +68,9 @@ import net.openhft.sg.Staged;
WrappedValueInstanceData.class,
DummyValueZeroData.class,
ReplicatedMapAbsentDelegatingForIteration.class,
ReplicatedMapEntryDelegating.class
})
public class ReplicatedMapIterationContext {
}
@@ -35,10 +35,7 @@ import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceDa
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.chronicle.map.impl.stage.input.MapInputBytesValues;
import net.openhft.chronicle.map.impl.stage.input.ReplicatedInput;
import net.openhft.chronicle.map.impl.stage.map.MapEntryOperationsDelegation;
import net.openhft.chronicle.map.impl.stage.map.ReplicatedChronicleMapHolderImpl;
import net.openhft.chronicle.map.impl.stage.map.ValueBytesInterop;
import net.openhft.chronicle.map.impl.stage.map.WrappedValueInstanceValueHolder;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.query.*;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.chronicle.map.impl.stage.ret.BytesReturnValue;
@@ -58,7 +55,7 @@ import net.openhft.sg.Staged;
ReplicatedChronicleMapHolderImpl.class,
KeyBytesInterop.class,
QuerySegmentStages.class,
ReplicatedQuerySegmentStages.class,
KeySearch.class,
HashLookupSearch.class,
QueryHashLookupPos.class,
@@ -71,6 +68,7 @@ import net.openhft.sg.Staged;
WrappedValueInstanceValueHolder.class,
ReplicatedMapQuery.class,
ReplicatedMapAbsent.class,
DefaultValue.class,
ValueBytesInterop.class,
ReplicatedMapAbsentHolder.class,
ReplicationUpdate.class,
@@ -18,18 +18,20 @@ package net.openhft.chronicle.map.impl.stage.entry;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.replication.TimeProvider;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.impl.ReplicatedChronicleMapHolder;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.chronicle.map.replication.MapReplicableEntry;
import net.openhft.sg.Stage;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
import static net.openhft.chronicle.map.ReplicatedChronicleMap.ADDITIONAL_ENTRY_BYTES;
@Staged
public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K, V>
implements MapReplicableEntry<K, V> {
implements MapReplicableEntry<K, V>, MapAbsentEntry<K, V> {
@StageRef ReplicatedChronicleMapHolder<?, ?, ?, ?, ?, ?, ?> mh;
@StageRef ReplicationUpdate ru;
@@ -116,6 +118,13 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
ru.raiseChange();
}
@Override
public boolean isChanged() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
s.innerReadLock.lock();
return ru.changed();
}
public void updatedReplicationStateOnPresentEntry() {
if (!ru.replicationUpdateInit()) {
s.innerWriteLock.lock();
@@ -158,4 +167,10 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
long sizeOfEverythingBeforeValue(long keySize, long valueSize) {
return super.sizeOfEverythingBeforeValue(keySize, valueSize) + ADDITIONAL_ENTRY_BYTES;
}
@NotNull
@Override
public Data<K> absentKey() {
return key();
}
}
@@ -0,0 +1,63 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.iter;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.replication.ReplicableEntryDelegating;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.MapContext;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
@Staged
public class ReplicatedMapAbsentDelegatingForIteration<K, V>
implements MapAbsentEntry<K, V>, ReplicableEntryDelegating {
@StageRef ReplicatedMapSegmentIteration<K, V, ?> delegate;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@NotNull
@Override
public MapContext<K, V, ?> context() {
return delegate.context();
}
@Override
public void doInsert(Data<V> value) {
delegate.doInsert(value);
}
@NotNull
@Override
public Data<V> defaultValue() {
return delegate.defaultValue();
}
@NotNull
@Override
public Data<K> absentKey() {
return delegate.absentKey();
}
@Override
public ReplicableEntry d() {
return e;
}
}
@@ -0,0 +1,68 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.iter;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.replication.ReplicableEntryDelegating;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapContext;
import net.openhft.chronicle.map.MapEntry;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
@Staged
public class ReplicatedMapEntryDelegating<K, V>
implements MapEntry<K, V>, ReplicableEntryDelegating {
@StageRef ReplicatedMapSegmentIteration<K, V, ?> delegate;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@NotNull
@Override
public MapContext<K, V, ?> context() {
return delegate.context();
}
@NotNull
@Override
public Data<K> key() {
return delegate.key();
}
@NotNull
@Override
public Data<V> value() {
return delegate.value();
}
@Override
public void doReplaceValue(Data<V> newValue) {
delegate.doReplaceValue(newValue);
}
@Override
public void doRemove() {
delegate.doRemove();
}
@Override
public ReplicableEntry d() {
return e;
}
}
@@ -17,9 +17,12 @@
package net.openhft.chronicle.map.impl.stage.iter;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.ReplicatedHashSegmentContext;
import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable;
import net.openhft.chronicle.hash.impl.VanillaChronicleHashHolder;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.MapEntry;
import net.openhft.chronicle.map.impl.ReplicatedIterationContextInterface;
import net.openhft.chronicle.map.impl.stage.data.DummyValueZeroData;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
@@ -28,15 +31,19 @@ import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import java.util.function.Consumer;
import java.util.function.Predicate;
@Staged
public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentIteration<K, V, R>
implements ReplicatedIterationContextInterface<K, V, R>, ReplicableEntry {
implements ReplicatedIterationContextInterface<K, V, R>, ReplicableEntry,
ReplicatedHashSegmentContext<K, MapEntry<K, V>>, MapAbsentEntry<K, V> {
@StageRef VanillaChronicleHashHolder<?, ?, ?> hh;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@StageRef ReplicationUpdate<K> ru;
@StageRef DummyValueZeroData<V> dummyValue;
@StageRef ReplicatedMapAbsentDelegatingForIteration<K, V> absentEntryDelegating;
@StageRef ReplicatedMapEntryDelegating<K, V> entryDelegating;
@Override
public boolean entryIsPresent() {
@@ -56,12 +63,14 @@ public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentI
}
@Override
public void forEachReplicableEntry(Consumer<? super ReplicableEntry> action) {
public boolean forEachSegmentReplicableEntryWhile(
Predicate<? super ReplicableEntry> predicate) {
s.innerUpdateLock.lock();
try {
long entries = s.entries();
if (entries == 0)
return;
return true;
boolean interrupted = false;
long startPos = 0L;
CompactOffHeapLinearHashTable hashLookup = hh.h().hashLookup;
while (!hashLookup.empty(hashLookup.readEntry(s.segmentBase, startPos))) {
@@ -73,22 +82,36 @@ public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentI
long entry = hashLookup.readEntry(s.segmentBase, hlp.hashLookupPos);
if (!hashLookup.empty(entry)) {
e.readExistingEntry(hashLookup.value(entry));
action.accept(this);
if (--entries == 0)
ReplicableEntry e = !this.e.entryDeleted() ? entryDelegating :
absentEntryDelegating;
initEntryRemovedOnThisIteration(false);
if (!predicate.test(e)) {
interrupted = true;
break;
} else {
if (--entries == 0)
break;
}
}
} while (hlp.hashLookupPos != startPos);
return !interrupted;
} finally {
s.innerReadLock.unlock();
initEntryRemovedOnThisIteration(false);
}
}
@Override
public void forEachSegmentReplicableEntry(Consumer<? super ReplicableEntry> action) {
forEachSegmentReplicableEntryWhile(e -> {
action.accept(e);
return true;
});
}
@Override
public void doRemove() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
initEntryRemovedOnThisIteration(true);
try {
if (e.valueSize > dummyValue.size())
e.innerDefaultReplaceValue(dummyValue);
@@ -99,5 +122,34 @@ public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentI
} finally {
s.innerWriteLock.unlock();
}
initEntryRemovedOnThisIteration(true);
}
@Override
public void doRemoveCompletely() {
boolean wasDeleted = e.entryDeleted();
super.doRemove();
if (wasDeleted)
s.deleted(s.deleted() - 1);
}
@Override
public void doInsert(Data<V> value) {
checkOnEachPublicOperation.checkOnEachPublicOperation();
if (e.entryDeleted()) {
try {
e.innerDefaultReplaceValue(value);
s.deleted(s.deleted() - 1);
s.incrementModCount();
e.writeEntryPresent();
ru.updateChange();
e.updatedReplicationStateOnPresentEntry();
} finally {
s.innerWriteLock.unlock();
}
} else {
throw new IllegalStateException(
"Entry is present in the map when doInsert() is called");
}
}
}
@@ -0,0 +1,45 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.map;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.hash.CheckOnEachPublicOperation;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.impl.VanillaChronicleMapHolder;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
@Staged
public abstract class DefaultValue<K, V> implements MapAbsentEntry<K, V> {
@StageRef CheckOnEachPublicOperation checkOnEachPublicOperation;
@StageRef VanillaChronicleMapHolder<K, ?, ?, V, ?, ?, ?> mh;
@NotNull
@Override
public Data<V> defaultValue() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
if (mh.m().constantValueProvider == null) {
throw new IllegalStateException("to call acquireUsing(), " +
"or defaultValue() on AbsentEntry, you should configure " +
"ChronicleMapBuilder.defaultValue(), or use one of the 'known' value types: " +
"boxed primitives, or so-called data-value-generated interface as a value");
}
return context().wrapValueAsData(mh.m().constantValueProvider.defaultValue());
}
}
@@ -32,7 +32,7 @@ import org.jetbrains.annotations.NotNull;
import static net.openhft.chronicle.hash.impl.stage.query.KeySearch.SearchState.PRESENT;
@Staged
public class MapAbsent<K, V> implements MapAbsentEntry<K, V> {
public abstract class MapAbsent<K, V> implements MapAbsentEntry<K, V> {
@StageRef public KeySearch<K> ks;
@StageRef MapQuery<K, V, ?> q;
@@ -81,17 +81,4 @@ public class MapAbsent<K, V> implements MapAbsentEntry<K, V> {
"Entry is present in the map when doInsert() is called");
}
}
@NotNull
@Override
public Data<V> defaultValue() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
if (mh.m().constantValueProvider == null) {
throw new IllegalStateException("to call acquireUsing(), " +
"or defaultValue() on AbsentEntry, you should configure " +
"ChronicleMapBuilder.defaultValue(), or use one of the 'known' value types: " +
"boxed primitives, or so-called data-value-generated interface as a value");
}
return context().wrapValueAsData(mh.m().constantValueProvider.defaultValue());
}
}
@@ -25,7 +25,7 @@ import net.openhft.sg.Staged;
import static net.openhft.chronicle.hash.impl.stage.query.KeySearch.SearchState.PRESENT;
@Staged
public class ReplicatedMapAbsent<K, V> extends MapAbsent<K, V> {
public abstract class ReplicatedMapAbsent<K, V> extends MapAbsent<K, V> {
@StageRef MapQuery<K, V, ?> q;
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@@ -17,6 +17,7 @@
package net.openhft.chronicle.map.impl.stage.query;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.impl.stage.data.DummyValueZeroData;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
@@ -28,9 +29,11 @@ import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.Nullable;
import static net.openhft.chronicle.hash.impl.stage.query.KeySearch.SearchState.DELETED;
@Staged
public abstract class ReplicatedMapQuery<K, V, R> extends MapQuery<K, V, R>
implements MapRemoteQueryContext<K, V, R> {
implements MapRemoteQueryContext<K, V, R>, ReplicableEntry {
@StageRef ReplicatedMapEntryStages<K, V, ?> e;
@StageRef ReplicationUpdate ru;
@@ -83,6 +86,14 @@ public abstract class ReplicatedMapQuery<K, V, R> extends MapQuery<K, V, R>
}
}
@Override
public void doRemoveCompletely() {
boolean wasDeleted = e.entryDeleted();
super.doRemove();
if (wasDeleted)
s.deleted(s.deleted() - 1L);
}
@Override
public void doReplaceValue(Data<V> newValue) {
super.doReplaceValue(newValue);
@@ -0,0 +1,71 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.query;
import net.openhft.chronicle.hash.ReplicatedHashSegmentContext;
import net.openhft.chronicle.hash.impl.stage.query.QuerySegmentStages;
import net.openhft.chronicle.hash.replication.ReplicableEntry;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.MapSegmentContext;
import net.openhft.chronicle.map.ReplicatedChronicleMap;
import net.openhft.chronicle.map.impl.ReplicatedChronicleMapHolder;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import java.util.function.Consumer;
@Staged
public abstract class ReplicatedQuerySegmentStages extends QuerySegmentStages {
@StageRef
ReplicatedChronicleMapHolder<?, ?, ?, ?, ?, ?, ?> mh;
final Consumer<ReplicableEntry> cleanupAction = e -> {
ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map = mh.m();
if (e instanceof MapAbsentEntry) {
long deleteTimeout = map.timeProvider.currentTime() - e.originTimestamp();
if (deleteTimeout > map.cleanupTimeout && !e.isChanged())
e.doRemoveCompletely();
}
};
@Override
public long alloc(int chunks) {
long ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map = mh.m();
MapSegmentContext<?, ?, ?> sc = map.segmentContext(segmentIndex);
((ReplicatedHashSegmentContext<?, ?>) sc).forEachSegmentReplicableEntry(cleanupAction);
ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
if (chunks == 1) {
throw new IllegalStateException(
"Segment is full, no free entries found");
} else {
throw new IllegalStateException(
"Segment is full or has no ranges of " + chunks
+ " continuous free chunks"
);
}
}
}
}
}
@@ -62,6 +62,10 @@ public abstract class ReplicationUpdate<K> implements RemoteOperationContext<K>
mh.m().raiseChange(s.segmentIndex, e.pos, e.timestamp());
}
public boolean changed() {
return mh.m().isChanged(s.segmentIndex, e.pos);
}
@Override
public long remoteTimestamp() {
checkOnEachPublicOperation.checkOnEachPublicOperation();