HCOLL-360 Add segment tiers

This commit is contained in:
leventov
2015-09-17 20:58:52 +03:00
parent 4606b64a02
commit 2eed65e80e
39 changed files with 1182 additions and 285 deletions
@@ -21,6 +21,8 @@ import net.openhft.chronicle.hash.impl.stage.entry.*;
import net.openhft.chronicle.hash.impl.stage.hash.KeyBytesInterop;
import net.openhft.chronicle.hash.impl.stage.hash.OwnerThreadHolder;
import net.openhft.chronicle.hash.impl.stage.hash.ThreadLocalCopiesHolder;
import net.openhft.chronicle.hash.impl.stage.iter.IterationSegmentStages;
import net.openhft.chronicle.hash.impl.stage.query.KeySearch;
import net.openhft.chronicle.map.impl.stage.data.bytes.EntryValueBytesData;
import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceData;
import net.openhft.chronicle.map.impl.stage.entry.MapEntryStages;
@@ -43,10 +45,12 @@ import net.openhft.sg.Staged;
MapSegmentIteration.class,
KeyBytesInterop.class,
SegmentStages.class,
IterationSegmentStages.class,
HashLookupPos.class,
IterationCheckOnEachPublicOperation.class,
AllocatedChunks.class,
KeySearch.class,
HashLookupSearch.class,
WrappedValueInstanceValueHolder.class,
MapEntryStages.class,
@@ -51,7 +51,7 @@ import net.openhft.sg.Staged;
KeyBytesInterop.class,
QuerySegmentStages.class,
KeySearch.class,
HashLookupSearch.class,
QueryHashLookupSearch.class,
QueryHashLookupPos.class,
QueryCheckOnEachPublicOperation.class,
@@ -41,6 +41,4 @@ public interface QueryContextInterface<K, V, R> extends ExternalMapQueryContext<
void initSegmentIndex(int segmentIndex);
boolean segmentIndexInit();
void clear();
}
@@ -21,6 +21,8 @@ import net.openhft.chronicle.hash.impl.stage.entry.*;
import net.openhft.chronicle.hash.impl.stage.hash.KeyBytesInterop;
import net.openhft.chronicle.hash.impl.stage.hash.OwnerThreadHolder;
import net.openhft.chronicle.hash.impl.stage.hash.ThreadLocalCopiesHolder;
import net.openhft.chronicle.hash.impl.stage.iter.IterationSegmentStages;
import net.openhft.chronicle.hash.impl.stage.query.KeySearch;
import net.openhft.chronicle.map.impl.stage.data.DummyValueZeroData;
import net.openhft.chronicle.map.impl.stage.data.bytes.EntryValueBytesData;
import net.openhft.chronicle.map.impl.stage.data.instance.WrappedValueInstanceData;
@@ -30,7 +32,6 @@ import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapAbsentDelegatingFo
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapEntryDelegating;
import net.openhft.chronicle.map.impl.stage.iter.ReplicatedMapSegmentIteration;
import net.openhft.chronicle.map.impl.stage.map.*;
import net.openhft.chronicle.map.impl.stage.query.ReplicatedMapAbsent;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.sg.Context;
import net.openhft.sg.Staged;
@@ -45,10 +46,12 @@ import net.openhft.sg.Staged;
ReplicatedMapSegmentIteration.class,
KeyBytesInterop.class,
SegmentStages.class,
IterationSegmentStages.class,
HashLookupPos.class,
IterationCheckOnEachPublicOperation.class,
AllocatedChunks.class,
KeySearch.class,
HashLookupSearch.class,
WrappedValueInstanceValueHolder.class,
ReplicatedMapEntryStages.class,
@@ -27,7 +27,10 @@ import net.openhft.chronicle.hash.impl.stage.hash.LogHolder;
import net.openhft.chronicle.hash.impl.stage.hash.OwnerThreadHolder;
import net.openhft.chronicle.hash.impl.stage.hash.ThreadLocalCopiesHolder;
import net.openhft.chronicle.hash.impl.stage.input.HashInputBytes;
import net.openhft.chronicle.hash.impl.stage.query.*;
import net.openhft.chronicle.hash.impl.stage.query.KeySearch;
import net.openhft.chronicle.hash.impl.stage.query.QueryHashLookupPos;
import net.openhft.chronicle.hash.impl.stage.query.QueryHashLookupSearch;
import net.openhft.chronicle.hash.impl.stage.query.SearchAllocatedChunks;
import net.openhft.chronicle.map.impl.stage.data.DummyValueZeroData;
import net.openhft.chronicle.map.impl.stage.data.bytes.*;
import net.openhft.chronicle.map.impl.stage.data.instance.InputValueInstanceData;
@@ -57,7 +60,7 @@ import net.openhft.sg.Staged;
KeyBytesInterop.class,
ReplicatedQuerySegmentStages.class,
KeySearch.class,
HashLookupSearch.class,
QueryHashLookupSearch.class,
QueryHashLookupPos.class,
QueryCheckOnEachPublicOperation.class,
@@ -18,6 +18,7 @@ package net.openhft.chronicle.map.impl.stage.data.bytes;
import net.openhft.chronicle.bytes.RandomDataInput;
import net.openhft.chronicle.hash.AbstractData;
import net.openhft.chronicle.hash.impl.stage.entry.SegmentStages;
import net.openhft.chronicle.hash.impl.stage.hash.CheckOnEachPublicOperation;
import net.openhft.chronicle.map.impl.stage.entry.MapEntryStages;
import net.openhft.chronicle.map.impl.stage.map.ValueBytesInterop;
@@ -29,6 +30,7 @@ import net.openhft.sg.Staged;
public class EntryValueBytesData<V> extends AbstractData<V> {
@StageRef ValueBytesInterop<V, ?, ?> vi;
@StageRef SegmentStages s;
@StageRef MapEntryStages<?, V> entry;
@StageRef CheckOnEachPublicOperation checkOnEachPublicOperation;
@@ -43,7 +45,7 @@ public class EntryValueBytesData<V> extends AbstractData<V> {
@Override
public RandomDataInput bytes() {
checkOnEachPublicOperation.checkOnEachPublicOperation();
return entry.entryBS;
return s.segmentBS;
}
@Override
@@ -71,7 +73,7 @@ public class EntryValueBytesData<V> extends AbstractData<V> {
}
private V innerGetUsing(V usingValue) {
entry.entryBytes.position(entry.valueOffset);
return vi.valueReader.read(entry.entryBytes, size(), usingValue);
s.segmentBytes.position(entry.valueOffset);
return vi.valueReader.read(s.segmentBytes, size(), usingValue);
}
}
@@ -17,8 +17,10 @@
package net.openhft.chronicle.map.impl.stage.entry;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable;
import net.openhft.chronicle.hash.impl.stage.entry.AllocatedChunks;
import net.openhft.chronicle.hash.impl.stage.entry.HashEntryStages;
import net.openhft.chronicle.hash.impl.stage.query.KeySearch;
import net.openhft.chronicle.map.MapEntry;
import net.openhft.chronicle.map.VanillaChronicleMap;
import net.openhft.chronicle.map.impl.VanillaChronicleMapHolder;
@@ -34,6 +36,7 @@ public abstract class MapEntryStages<K, V> extends HashEntryStages<K>
@StageRef public VanillaChronicleMapHolder<?, ?, ?, ?, ?, ?, ?> mh;
@StageRef public AllocatedChunks allocatedChunks;
@StageRef KeySearch<?> ks;
long countValueSizeOffset() {
return keyEnd();
@@ -50,20 +53,20 @@ public abstract class MapEntryStages<K, V> extends HashEntryStages<K>
@Stage("ValueSize")
private void countValueOffset() {
mh.m().alignment.alignPositionAddr(entryBytes);
valueOffset = entryBytes.position();
mh.m().alignment.alignPositionAddr(s.segmentBytes);
valueOffset = s.segmentBytes.position();
}
void initValueSize(long valueSize) {
this.valueSize = valueSize;
entryBytes.position(valueSizeOffset);
mh.m().valueSizeMarshaller.writeSize(entryBytes, valueSize);
s.segmentBytes.position(valueSizeOffset);
mh.m().valueSizeMarshaller.writeSize(s.segmentBytes, valueSize);
countValueOffset();
}
void initValueSize() {
entryBytes.position(valueSizeOffset);
valueSize = mh.m().readValueSize(entryBytes);
s.segmentBytes.position(valueSizeOffset);
valueSize = mh.m().readValueSize(s.segmentBytes);
countValueOffset();
}
@@ -73,13 +76,13 @@ public abstract class MapEntryStages<K, V> extends HashEntryStages<K>
}
public void initValue(Data<?> value) {
entryBytes.position(valueSizeOffset);
s.segmentBytes.position(valueSizeOffset);
initValueSize(value.size());
writeValue(value);
}
public void writeValue(Data<?> value) {
value.writeTo(entryBS, valueOffset);
value.writeTo(s.segmentBS, valueOffset);
}
public void initValue_WithoutSize(
@@ -138,15 +141,15 @@ public abstract class MapEntryStages<K, V> extends HashEntryStages<K>
"entry takes " + newSizeInChunks + " chunks, " +
m.maxChunksPerEntry + " is maximum.");
}
if (s.freeList.allClear(pos + entrySizeInChunks, pos + newSizeInChunks)) {
s.freeList.set(pos + entrySizeInChunks, pos + newSizeInChunks);
if (s.freeList.isRangeClear(pos + entrySizeInChunks, pos + newSizeInChunks)) {
s.freeList.setRange(pos + entrySizeInChunks, pos + newSizeInChunks);
break newValueDoesNotFit;
}
relocation(newValue, newSizeOfEverythingBeforeValue);
return;
} else if (newSizeInChunks < entrySizeInChunks) {
// Freeing extra chunks
s.freeList.clear(pos + newSizeInChunks, pos + entrySizeInChunks);
s.freeList.clearRange(pos + newSizeInChunks, pos + entrySizeInChunks);
// Do NOT reset nextPosToSearchFrom, because if value
// once was larger it could easily became larger again,
// But if these chunks will be taken by that time,
@@ -188,8 +191,30 @@ public abstract class MapEntryStages<K, V> extends HashEntryStages<K>
s.innerWriteLock.lock();
s.free(pos, entrySizeInChunks);
long entrySize = innerEntrySize(newSizeOfEverythingBeforeValue, newValue.size());
allocatedChunks.initEntryAndKeyCopying(entrySize, valueSizeOffset - keySizeOffset);
writeValueAndPutPos(newValue);
// need to copy, because in initEntryAndKeyCopying(), in alloc(), nextTier() called ->
// hashLookupPos cleared, as a dependant
long oldHashLookupPos = hlp.hashLookupPos;
long oldHashLookupAddr = s.segmentBaseAddr;
boolean tierHasChanged =
allocatedChunks.initEntryAndKeyCopying(entrySize, valueSizeOffset - keySizeOffset);
if (tierHasChanged) {
// implicitly inits key search, locating hashLookupPos on the empty slot
assert ks.searchStateAbsent();
}
initValue(newValue);
freeExtraAllocatedChunks();
CompactOffHeapLinearHashTable hl = hh.h().hashLookup;
long oldEntry = hl.readEntry(oldHashLookupAddr, oldHashLookupPos);
hl.checkValueForPut(pos);
hl.writeEntryVolatile(s.segmentBaseAddr, hlp.hashLookupPos,
oldEntry, hl.key(oldEntry), pos);
if (tierHasChanged)
hl.remove(oldHashLookupAddr, oldHashLookupPos);
}
public final long entrySize(long keySize, long valueSize) {
@@ -43,9 +43,9 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
}
void updateReplicationState(long timestamp, byte identifier) {
entryBytes.position(replicationBytesOffset);
entryBytes.writeLong(timestamp);
entryBytes.writeByte(identifier);
s.segmentBytes.position(replicationBytesOffset);
s.segmentBytes.writeLong(timestamp);
s.segmentBytes.writeByte(identifier);
}
private long timestampOffset() {
@@ -53,7 +53,7 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
}
public long timestamp() {
return entryBS.readLong(replicationBytesOffset);
return s.segmentBS.readLong(replicationBytesOffset);
}
private long identifierOffset() {
@@ -61,7 +61,7 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
}
byte identifier() {
return entryBS.readByte(identifierOffset());
return s.segmentBS.readByte(identifierOffset());
}
private long entryDeletedOffset() {
@@ -69,15 +69,15 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
}
public boolean entryDeleted() {
return entryBS.readBoolean(entryDeletedOffset());
return s.segmentBS.readBoolean(entryDeletedOffset());
}
public void writeEntryPresent() {
entryBS.writeBoolean(entryDeletedOffset(), false);
s.segmentBS.writeBoolean(entryDeletedOffset(), false);
}
public void writeEntryDeleted() {
entryBS.writeBoolean(entryDeletedOffset(), true);
s.segmentBS.writeBoolean(entryDeletedOffset(), true);
}
@Override
@@ -148,8 +148,9 @@ public abstract class ReplicatedMapEntryStages<K, V, T> extends MapEntryStages<K
@Override
protected void relocation(Data<V> newValue, long newSizeOfEverythingBeforeValue) {
long oldPos = pos;
long oldTierIndex = s.tierIndex;
super.relocation(newValue, newSizeOfEverythingBeforeValue);
ru.moveChange(oldPos, pos);
ru.moveChange(oldTierIndex, oldPos, pos);
}
private boolean testTimeStampInSensibleRange() {
@@ -73,13 +73,13 @@ public abstract class ReplicatedMapSegmentIteration<K, V, R> extends MapSegmentI
boolean interrupted = false;
long startPos = 0L;
CompactOffHeapLinearHashTable hashLookup = hh.h().hashLookup;
while (!hashLookup.empty(hashLookup.readEntry(s.segmentBase, startPos))) {
while (!hashLookup.empty(hashLookup.readEntry(s.segmentBaseAddr, startPos))) {
startPos = hashLookup.step(startPos);
}
hlp.initHashLookupPos(startPos);
do {
hlp.setHashLookupPos(hashLookup.step(hlp.hashLookupPos));
long entry = hashLookup.readEntry(s.segmentBase, hlp.hashLookupPos);
long entry = hashLookup.readEntry(s.segmentBaseAddr, hlp.hashLookupPos);
if (!hashLookup.empty(entry)) {
e.readExistingEntry(hashLookup.value(entry));
ReplicableEntry e = !this.e.entryDeleted() ? entryDelegating :
@@ -19,7 +19,8 @@ package net.openhft.chronicle.map.impl.stage.query;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.entry.SegmentStages;
import net.openhft.chronicle.hash.impl.stage.hash.CheckOnEachPublicOperation;
import net.openhft.chronicle.hash.impl.stage.query.HashLookupSearch;
import net.openhft.chronicle.hash.impl.stage.entry.HashLookupSearch;
import net.openhft.chronicle.hash.impl.stage.query.HashQuery.EntryPresence;
import net.openhft.chronicle.hash.impl.stage.query.KeySearch;
import net.openhft.chronicle.map.MapAbsentEntry;
import net.openhft.chronicle.map.MapContext;
@@ -76,6 +77,7 @@ public abstract class MapAbsent<K, V> implements MapAbsentEntry<K, V> {
}
s.incrementModCount();
ks.setSearchState(PRESENT);
q.initPresenceOfEntry(EntryPresence.PRESENT);
} else {
throw new IllegalStateException(
"Entry is present in the map when doInsert() is called");
@@ -92,6 +92,7 @@ public abstract class MapQuery<K, V, R> extends HashQuery<K>
e.innerDefaultReplaceValue(newValue);
s.incrementModCount();
ks.setSearchState(PRESENT);
initPresenceOfEntry(EntryPresence.PRESENT);
} else {
throw new IllegalStateException(
"Entry is absent in the map when doReplaceValue() is called");
@@ -17,7 +17,7 @@
package net.openhft.chronicle.map.impl.stage.query;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.query.HashLookupSearch;
import net.openhft.chronicle.hash.impl.stage.entry.HashLookupSearch;
import net.openhft.chronicle.map.impl.stage.entry.MapEntryStages;
import net.openhft.sg.StageRef;
import net.openhft.sg.Staged;
@@ -44,9 +44,9 @@ public abstract class QueryMapEntryStages<K, V> extends MapEntryStages<K, V> {
} else {
newSizeInChunks = entrySizeInChunks;
}
if (pos + newSizeInChunks < s.freeList.size() &&
s.freeList.allClear(pos, pos + newSizeInChunks)) {
s.freeList.set(pos, pos + newSizeInChunks);
if (pos + newSizeInChunks < s.freeList.logicalSize() &&
s.freeList.isRangeClear(pos, pos + newSizeInChunks)) {
s.freeList.setRange(pos, pos + newSizeInChunks);
s.innerWriteLock.lock();
allocatedChunks.incrementSegmentEntriesIfNeeded();
if (newValueSizeIsDifferent) {
@@ -17,6 +17,7 @@
package net.openhft.chronicle.map.impl.stage.query;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.impl.stage.query.HashQuery.EntryPresence;
import net.openhft.chronicle.map.impl.stage.entry.ReplicatedMapEntryStages;
import net.openhft.chronicle.map.impl.stage.replication.ReplicationUpdate;
import net.openhft.sg.StageRef;
@@ -38,6 +39,7 @@ public abstract class ReplicatedMapAbsent<K, V> extends MapAbsent<K, V> {
if (!ks.searchStatePresent()) {
putEntry(value);
ks.setSearchState(PRESENT);
q.initPresenceOfEntry(EntryPresence.PRESENT);
} else {
e.innerDefaultReplaceValue(value);
s.deleted(s.deleted() - 1);
@@ -45,25 +45,20 @@ public abstract class ReplicatedQuerySegmentStages extends QuerySegmentStages {
@Override
public long alloc(int chunks) {
long ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map = mh.m();
MapSegmentContext<?, ?, ?> sc = map.segmentContext(segmentIndex);
((ReplicatedHashSegmentContext<?, ?>) sc).forEachSegmentReplicableEntry(cleanupAction);
ret = allocReturnCode(chunks);
while (true) {
long ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
if (chunks == 1) {
throw new IllegalStateException(
"Segment is full, no free entries found");
ReplicatedChronicleMap<?, ?, ?, ?, ?, ?, ?> map = mh.m();
MapSegmentContext<?, ?, ?> sc = map.segmentContext(segmentIndex);
((ReplicatedHashSegmentContext<?, ?>) sc)
.forEachSegmentReplicableEntry(cleanupAction);
ret = allocReturnCode(chunks);
if (ret >= 0) {
return ret;
} else {
throw new IllegalStateException(
"Segment is full or has no ranges of " + chunks
+ " continuous free chunks"
);
nextTier();
}
}
}
@@ -45,11 +45,11 @@ public abstract class ReplicationUpdate<K> implements RemoteOperationContext<K>
}
public void dropChange() {
mh.m().dropChange(s.segmentIndex, e.pos);
mh.m().dropChange(s.tierIndex, e.pos);
}
public void moveChange(long oldPos, long newPos) {
mh.m().moveChange(s.segmentIndex, oldPos, newPos, e.timestamp());
public void moveChange(long oldTierIndex, long oldPos, long newPos) {
mh.m().moveChange(oldTierIndex, oldPos, s.tierIndex, newPos, e.timestamp());
}
public void updateChange() {
@@ -59,11 +59,11 @@ public abstract class ReplicationUpdate<K> implements RemoteOperationContext<K>
}
public void raiseChange() {
mh.m().raiseChange(s.segmentIndex, e.pos, e.timestamp());
mh.m().raiseChange(s.tierIndex, e.pos, e.timestamp());
}
public boolean changed() {
return mh.m().isChanged(s.segmentIndex, e.pos);
return mh.m().isChanged(s.tierIndex, e.pos);
}
@Override