HCOLL-378 #36 Add ChronicleHashBuilder.recoverPersistedTo() and ChronicleHashInstanceBuilder.recover() functionality

This commit is contained in:
leventov
2016-01-04 23:24:39 +03:00
parent b4c97ced36
commit 97e4c12de1
31 changed files with 1026 additions and 121 deletions
@@ -19,8 +19,13 @@ package net.openhft.chronicle.map.impl;
import net.openhft.chronicle.map.MapEntry;
import net.openhft.chronicle.map.MapSegmentContext;
import java.util.List;
import java.util.Set;
public interface IterationContext<K, V, R> extends MapEntry<K, V>, MapSegmentContext<K, V, R> {
long pos();
void initSegmentIndex(int segmentIndex);
void recoverSegments();
}
@@ -21,9 +21,7 @@ import net.openhft.chronicle.hash.impl.stage.entry.*;
import net.openhft.chronicle.hash.impl.stage.hash.KeyBytesInterop;
import net.openhft.chronicle.hash.impl.stage.hash.LogHolder;
import net.openhft.chronicle.hash.impl.stage.hash.OwnerThreadHolder;
import net.openhft.chronicle.hash.impl.stage.iter.IterationAlloc;
import net.openhft.chronicle.hash.impl.stage.iter.IterationKeyHashCode;
import net.openhft.chronicle.hash.impl.stage.iter.IterationSegmentStages;
import net.openhft.chronicle.hash.impl.stage.iter.*;
import net.openhft.chronicle.hash.impl.stage.query.KeySearch;
import net.openhft.chronicle.map.impl.stage.data.bytes.EntryValueBytesData;
import net.openhft.chronicle.map.impl.stage.data.bytes.WrappedValueBytesData;
@@ -61,6 +59,9 @@ import net.openhft.sg.Staged;
MapEntryOperationsDelegation.class,
IterationAlloc.class,
TierRecovery.class,
SegmentsRecovery.class,
}, nested = {
ReadLock.class,
UpdateLock.class,
@@ -24,16 +24,14 @@ import net.openhft.chronicle.hash.impl.stage.hash.OwnerThreadHolder;
import net.openhft.chronicle.hash.impl.stage.iter.IterationAlloc;
import net.openhft.chronicle.hash.impl.stage.iter.IterationKeyHashCode;
import net.openhft.chronicle.hash.impl.stage.iter.IterationSegmentStages;
import net.openhft.chronicle.hash.impl.stage.iter.SegmentsRecovery;
import net.openhft.chronicle.hash.impl.stage.query.KeySearch;
import net.openhft.chronicle.map.impl.stage.data.DummyValueZeroData;
import net.openhft.chronicle.map.impl.stage.data.bytes.EntryValueBytesData;
import net.openhft.chronicle.map.impl.stage.data.bytes.WrappedValueBytesData;
import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceDataHolder;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.chronicle.map.impl.stage.iter.IterationCheckOnEachPublicOperation;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapAbsentDelegatingForIteration;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapEntryDelegating;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapSegmentIteration;
import net.openhft.chronicle.map.impl.stage.iter.*;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.sg.Context;
@@ -68,6 +66,9 @@ import net.openhft.sg.Staged;
DefaultValue.class,
IterationAlloc.class,
ReplicatedTierRecovery.class,
SegmentsRecovery.class,
}, nested = {
ReadLock.class,
UpdateLock.class,
@@ -112,7 +112,7 @@ public abstract class MapEntryStages<K, V> extends HashEntryStages<K>
}
@Override
protected long entryEnd() {
public long entryEnd() {
return valueOffset + valueSize;
}
@@ -26,6 +26,11 @@ import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
import org.jetbrains.annotations.NotNull;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Staged
public abstract class MapSegmentIteration<K, V, R> extends HashSegmentIteration<K, MapEntry<K, V>>
implements MapEntry<K, V>, IterationContext<K, V, R> {
@@ -48,4 +53,15 @@ public abstract class MapSegmentIteration<K, V, R> extends HashSegmentIteration<
public WrappedValueInstanceDataHolderAccess<K, V, ?> context() {
return wrappedValueInstanceDataHolderAccess;
}
//
// @Override
// public void recover(List<Set<Long>> usedTierIndexesBySegment) {
// for (int visitedSegmentIndex = 0; visitedSegmentIndex < usedTierIndexesBySegment.size();
// visitedSegmentIndex++) {
// Set<Long> intersection = new HashSet<>(
// usedTierIndexesBySegment.get(visitedSegmentIndex));
// intersection.retainAll(usedTierIndexes);
// if (Collections.dis)
// }
// }
}
@@ -0,0 +1,99 @@
/*
* Copyright (C) 2015 higherfrequencytrading.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map.impl.stage.iter;
import net.openhft.chronicle.algo.bitset.ReusableBitSet;
import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable;
import net.openhft.chronicle.hash.impl.VanillaChronicleHash;
import net.openhft.chronicle.hash.impl.stage.entry.SegmentStages;
import net.openhft.chronicle.hash.impl.stage.hash.LogHolder;
import net.openhft.chronicle.hash.impl.stage.iter.TierRecovery;
import net.openhft.chronicle.map.ReplicatedChronicleMap;
import net.openhft.chronicle.map.impl.ReplicatedChronicleMapHolder;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
@Staged
public class ReplicatedTierRecovery extends TierRecovery {
@StageRef ReplicatedChronicleMapHolder<?, ?, ?> rh;
@StageRef SegmentStages s;
@StageRef ReplicatedMapEntryStages<?, ?> e;
@StageRef LogHolder lh;
@Override
public void removeDuplicatesInSegment() {
super.removeDuplicatesInSegment();
recoverTierDeleted();
cleanupModificationIterationBits();
}
private void recoverTierDeleted() {
VanillaChronicleHash<?, ?, ?, ?> h = rh.h();
CompactOffHeapLinearHashTable hl = h.hashLookup;
long hlAddr = s.tierBaseAddr;
long deleted = 0;
long hlPos = 0;
do {
long hlEntry = hl.readEntry(hlAddr, hlPos);
if (!hl.empty(hlEntry)) {
e.readExistingEntry(hl.value(hlEntry));
if (e.entryDeleted()) {
deleted++;
}
}
hlPos = hl.step(hlPos);
} while (hlPos != 0);
if (s.tierDeleted() != deleted) {
lh.LOG.error("wrong deleted counter for tier with index {}, stored: {}, should be: {}",
s.tierIndex, s.tierDeleted(), deleted);
s.tierDeleted(deleted);
}
}
private void cleanupModificationIterationBits() {
ReplicatedChronicleMap<?, ?, ?> m = rh.m();
ReplicatedChronicleMap<?, ?, ?>.ModificationIterator[] its =
m.acquireAllModificationIterators();
ReusableBitSet freeList = s.freeList;
for (long pos = 0;;) {
long nextPos = freeList.nextSetBit(pos);
if (nextPos > pos) {
for (ReplicatedChronicleMap<?, ?, ?>.ModificationIterator it : its) {
it.clearRange0(s.tierIndex, pos, nextPos);
}
}
if (nextPos > 0) {
e.readExistingEntry(nextPos);
if (e.entrySizeInChunks > 1) {
for (ReplicatedChronicleMap<?, ?, ?>.ModificationIterator it : its) {
it.clearRange0(s.tierIndex, nextPos + 1, nextPos + e.entrySizeInChunks);
}
}
pos = nextPos + e.entrySizeInChunks;
} else {
if (pos < m.actualChunksPerSegmentTier) {
for (ReplicatedChronicleMap<?, ?, ?>.ModificationIterator it : its) {
it.clearRange0(s.tierIndex, pos, m.actualChunksPerSegmentTier);
}
}
}
}
}
}