readValue, boolean resultUnused,
byte identifier, long timestamp, boolean remote) {
segmentState.identifier = identifier;
segmentState.timestamp = timestamp;
MultiMap hashLookup = hashLookup();
// System.out.println("Before put size: " + hashLookup.size());
SearchState searchState = segmentState.searchState;
hashLookup.startSearch(hash2, searchState);
MultiStoreBytes entry = segmentState.tmpBytes;
for (long pos; (pos = hashLookup.nextPos(searchState)) >= 0L; ) {
long offset = offsetFromPos(pos);
reuse(entry, offset);
if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry))
continue;
// key is found
entry.skip(keySize);
final long timeStampPosAddr = entry.positionAddr();
if (shouldIgnore(entry, timestamp, identifier)) {
// the following assert should be enabled, but TimeBasedReplicationTest
// intentionally violates the explained invariant, => assertion fails.
// todo do something with this
// we should ignore only external remote updates
// which don't use put and remove results
// assert resultUnused;
return null;
}
boolean isDeleted = entry.readBoolean();
if (replaceIfPresent || isDeleted) {
entry.positionAddr(timeStampPosAddr);
long replacedTimestamp = entry.readLong(entry.position());
entry.writeLong(timestamp);
final byte replacedIdentifier = entry.readByte(entry.position());
entry.writeByte(identifier);
// deleted flag
entry.writeBoolean(false);
RV prevValue = replaceValueAndNotifyPut(copies, segmentState,
key, keySize, toKey,
getValueInterops, value, toValue,
entry, pos, hashLookup, readValue,
resultUnused, isDeleted, remote,
replacedIdentifier, replacedTimestamp);
if (resultUnused)
return null;
return isDeleted ? readValue.readNull() : prevValue;
} else {
long valueSize = readValueSize(entry);
return resultUnused ? null :
readValue.readValue(copies, entry, null, valueSize);
}
}
// key is not found
VBI valueInterop = getValueInterops.getValueInterop(copies);
MVBI metaValueInterop =
getValueInterops.getMetaValueInterop(copies, valueInterop, value);
long valueSize = metaValueInterop.size(valueInterop, value);
putEntry(segmentState, metaKeyInterop, keyInterop, key, keySize,
metaValueInterop, valueInterop, value, entry, false);
entry.position(segmentState.valueSizePos - ADDITIONAL_ENTRY_BYTES);
entry.writeLong(timestamp);
entry.writeByte(identifier);
entry.writeBoolean(false);
// put callbacks
onPutMaybeRemote(segmentState.pos, remote);
if (bytesEventListener != null) {
byte replacedIdentifier = 0;
long replacedTimestamp = 0;
bytesEventListener.onPut(entry, 0L, metaDataBytes,
segmentState.valueSizePos, true, remote, true, identifier,
replacedIdentifier, timestamp,
replacedTimestamp, this);
}
if (eventListener != null && eventListener.isActive()) {
byte replacedIdentifier = 0;
long replacedTimestamp = 0;
final K key1 = toKey.toInstance(copies, key, keySize);
final V newValue = eventListener.usesValue() ? toValue.toInstance(copies, value, valueSize) : null;
writeUnlock();
try {
eventListener.onPut(key1,
newValue, null, remote, true,
true, identifier, replacedIdentifier, timestamp,
replacedTimestamp);
} finally {
writeLock();
}
}
return resultUnused ? null : readValue.readNull();
}
@Override
void replaceValueDeletedCallback(
SegmentState segmentState, MultiMap hashLookup, long pos, boolean isDeleted) {
if (isDeleted) {
incrementSize();
// if they are NOT equal, it means the entry was relocated in putValue(),
// hence position is already set
if (pos == segmentState.pos) {
hashLookup.putPosition(segmentState.pos);
} else {
assert hashLookup.getPositions().isSet(segmentState.pos);
}
}
}
/**
* Used only with replication, its sometimes possible to receive an old ( or stale update )
* from a remote map. This method is used to determine if we should ignore such updates.
* We can reject put() and removes() when comparing times stamps with remote
* systems
*
* @param entry the maps entry
* @param timestamp the time the entry was created or updated
* @param identifier the unique identifier relating to this map
* @return true if the entry should not be processed
*/
private boolean shouldIgnore(@NotNull final Bytes entry, final long timestamp,
final byte identifier) {
final long lastModifiedTimeStamp = entry.readLong();
// if the readTimeStamp is newer then we'll reject this put()
// or they are the same and have a larger id
if (lastModifiedTimeStamp < timestamp) {
entry.skip(1); // skip the byte used for the identifier
return false;
}
if (lastModifiedTimeStamp > timestamp)
return true;
// check the identifier
return entry.readByte() > identifier;
}
@Override
void manageReplicationBytes(
SegmentState segmentState, Bytes entry,
boolean writeDefaultInitialReplicationValues, boolean remove) {
if (!writeDefaultInitialReplicationValues) {
entry.skip(ADDITIONAL_ENTRY_BYTES);
} else {
segmentState.timestamp = currentTime();
entry.writeLong(segmentState.timestamp);
entry.writeByte(localIdentifier);
entry.writeBoolean(remove);
}
}
@Override
,
RV, VB extends RV, VBI, MVBI extends MetaBytesInterop super VB, ? super VBI>>
Object remove(@Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState,
MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize,
InstanceOrBytesToInstance toKey,
GetValueInterops getValueInterops, VB expectedValue,
InstanceOrBytesToInstance toValue,
long hash2, ReadValue readValue, boolean resultUnused) {
segmentStateNotNullImpliesCopiesNotNull(copies, segmentState);
if (segmentState == null) {
copies = SegmentState.getCopies(copies);
segmentState = SegmentState.get(copies);
}
writeLock();
try {
return removeWithoutLock(copies, segmentState,
metaKeyInterop, keyInterop, key, keySize, toKey,
getValueInterops, expectedValue, toValue,
hash2, readValue, resultUnused, currentTime(), localIdentifier, false,
expectedValue != null);
} finally {
segmentState.close();
writeUnlock();
}
}
@Override
boolean isDeleted(Bytes entry, long keySize) {
return entry.readBoolean(entry.position() + keySize + ADDITIONAL_ENTRY_BYTES - 1L);
}
@Override
,
RV, VB extends RV, VBI, MVBI extends MetaBytesInterop super VB, ? super VBI>>
Object removeWithoutLock(
@Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState,
MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize,
InstanceOrBytesToInstance toKey,
GetValueInterops getValueInterops, VB expectedValue,
InstanceOrBytesToInstance toValue,
long hash2, ReadValue readValue, boolean resultUnused) {
return removeWithoutLock(copies, segmentState, metaKeyInterop, keyInterop, key, keySize,
toKey, getValueInterops, expectedValue, toValue, hash2, readValue, resultUnused,
currentTime(), localIdentifier, false, expectedValue != null);
}
/**
* - if expectedValue is not null, returns Boolean.TRUE (removed) or Boolean.FALSE (entry
* not found), regardless the expectedValue object is Bytes instance (RPC call) or the value
* instance - if expectedValue is null: - if resultUnused is false, null or removed value is
* returned - if resultUnused is true, null is always returned
*/
,
RV, VB extends RV, VBI, MVBI extends MetaBytesInterop super VB, ? super VBI>>
Object removeWithoutLock(
@NotNull ThreadLocalCopies copies, @NotNull SegmentState segmentState,
MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize,
InstanceOrBytesToInstance toKey,
GetValueInterops getValueInterops, VB expectedValue,
InstanceOrBytesToInstance toValue,
long hash2, ReadValue readValue, boolean resultUnused,
long timestamp, byte identifier, boolean remote, boolean booleanResult) {
segmentState.identifier = identifier;
segmentState.timestamp = timestamp;
assert identifier > 0;
expectedValueNotNullImpliesBooleanResult(expectedValue, booleanResult);
MultiMap hashLookup = hashLookup();
SearchState searchState = segmentState.searchState;
hashLookup.startSearch(hash2, searchState);
MultiStoreBytes entry = segmentState.tmpBytes;
returnNothing:
{
for (long pos; (pos = hashLookup.nextPos(searchState)) >= 0L; ) {
long offset = offsetFromPos(pos);
reuse(entry, offset);
if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry))
continue;
// key is found
entry.skip(keySize);
long timestampPos = entry.position();
long replacedTimestamp = entry.readLong(timestampPos);
byte replacedIdentifier = entry.readByte(timestampPos + 8);
if (shouldIgnore(entry, timestamp, identifier)) {
// the following assert should be enabled, but TimeBasedReplicationTest
// intentionally violates the explained invariant, => assertion fails.
// todo do something with this
// we should ignore only remote updates
// which don't use remove, put results
// assert booleanResult || resultUnused;
return booleanResult ? Boolean.FALSE : null;
}
boolean isDeleted = entry.readBoolean();
if (isDeleted) {
if (expectedValue != null)
return Boolean.FALSE;
entry.position(timestampPos);
entry.writeLong(timestamp);
entry.writeByte(identifier);
onRemoveMaybeRemote(pos, remote);
break returnNothing;
}
long valueSizePos = entry.position();
long valueSize = readValueSize(entry);
long valuePos = entry.position();
// check the value assigned for the key is that we expect
if (expectedValue != null) {
VBI valueInterop = getValueInterops.getValueInterop(copies);
MVBI metaValueInterop = getValueInterops.getMetaValueInterop(
copies, valueInterop, expectedValue);
if (metaValueInterop.size(valueInterop, expectedValue) != valueSize)
return Boolean.FALSE;
if (!metaValueInterop.startsWith(valueInterop, entry, expectedValue))
return Boolean.FALSE;
}
entry.position(timestampPos);
entry.writeLong(timestamp);
entry.writeByte(identifier);
entry.writeBoolean(true);
entry.position(valuePos);
return removeEntry(copies, segmentState, key, keySize, toKey, toValue, readValue,
resultUnused, hashLookup, entry, pos, valueSizePos,
valueSize, remote, false, booleanResult,
replacedIdentifier, replacedTimestamp);
}
// key is not found
if (remote) {
long minEncodableValueSize = valueSizeMarshaller.minEncodableSize();
long entrySize = entrySize(keySize, minEncodableValueSize);
int allocatedChunks = inChunks(entrySize);
long pos = alloc(allocatedChunks);
long offset = offsetFromPos(pos);
clearMetaData(offset);
reuse(entry, offset);
keySizeMarshaller.writeSize(entry, keySize);
metaKeyInterop.write(keyInterop, entry, key);
entry.writeLong(timestamp);
entry.writeByte(identifier);
entry.writeBoolean(true);
valueSizeMarshaller.writeSize(entry, minEncodableValueSize);
alignment.alignPositionAddr(entry);
entry.skip(minEncodableValueSize);
freeExtraAllocatedChunks(pos, allocatedChunks, entry);
hashLookup.putAfterFailedSearch(searchState, pos);
hashLookup.removePosition(pos);
// listener.onRemove() might be missing here
// depends on onRemove() semantics
}
}
// return nothing
if (booleanResult) {
return Boolean.FALSE;
} else {
return resultUnused ? null : readValue.readNull();
}
}
,
RV, VB extends RV, VBI, MVBI extends MetaBytesInterop>
Object replace(
@Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState,
MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize,
InstanceOrBytesToInstance toKey,
GetValueInterops getExpectedValueInterops, VB expectedValue,
GetValueInterops getNewValueInterops, VB newValue,
ReadValue readValue, InstanceOrBytesToInstance super RV, V> toValue,
long hash2) {
segmentStateNotNullImpliesCopiesNotNull(copies, segmentState);
if (segmentState == null) {
copies = SegmentState.getCopies(copies);
segmentState = SegmentState.get(copies);
}
long timestamp = currentTime();
byte identifier = localIdentifier;
writeLock();
segmentState.identifier = identifier;
segmentState.timestamp = timestamp;
try {
MultiMap hashLookup = hashLookup();
SearchState searchState = segmentState.searchState;
hashLookup.startSearch(hash2, searchState);
MultiStoreBytes entry = segmentState.tmpBytes;
for (long pos; (pos = hashLookup.nextPos(searchState)) >= 0L; ) {
long offset = offsetFromPos(pos);
reuse(entry, offset);
if (!keyEquals(keyInterop, metaKeyInterop, key, keySize, entry))
continue;
// key is found
entry.skip(keySize);
long timestampPos = entry.position();
long replacedTimestamp = entry.readLong();
byte replacedIdentifier = entry.readByte();
entry.position(timestampPos);
if (shouldIgnore(entry, timestamp, identifier)) {
LOG.error("Trying to replace a value for key={} on the node with id={} " +
"at time={} (current time), but the entry is updated " +
"by node with id={} at time={}. Time is not monotonic " +
"across nodes!?",
key, identifier, timestamp, entry.readByte(entry.position() - 1),
entry.readLong(entry.position() - ADDITIONAL_ENTRY_BYTES + 1));
return readValue.readNull();
}
boolean isDeleted = entry.readBoolean();
if (isDeleted)
break;
Object result = onKeyPresentOnReplace(copies, segmentState, key, keySize, toKey,
getExpectedValueInterops, expectedValue, getNewValueInterops, newValue,
readValue, toValue, pos, entry, hashLookup,
replacedIdentifier, replacedTimestamp);
return result;
}
// key is not found
return expectedValue == null ? readValue.readNull() : Boolean.FALSE;
} finally {
segmentState.close();
writeUnlock();
}
}
@Override
void updateReplicationBytesOnKeyPresentOnReplace(
Bytes entry, long timestampPos, long timestamp, byte identifier) {
entry.position(timestampPos);
entry.writeLong(timestamp);
entry.writeByte(identifier);
}
public void dirtyEntries(final long timeStamp,
final EntryModifiableCallback callback,
final boolean bootstrapOnlyLocalEntries) {
readLock(null);
ThreadLocalCopies copies = SegmentState.getCopies(null);
try (SegmentState segmentState = SegmentState.get(copies)) {
final int index = Segment.this.getIndex();
final MultiStoreBytes tmpBytes = segmentState.tmpBytes;
hashLookup().forEach(new EntryConsumer() {
@Override
public void accept(long hash, long pos) {
final Bytes entry = reuse(tmpBytes, offsetFromPos(pos));
long keySize = keySizeMarshaller.readSize(entry);
entry.skip(keySize);
final long entryTimestamp = entry.readLong();
if (entryTimestamp >= timeStamp &&
(!bootstrapOnlyLocalEntries || entry.readByte() == ReplicatedChronicleMap.this.identifier()))
callback.set(index, pos);
}
});
} finally {
readUnlock();
}
}
@Override
public Entry getEntry(@NotNull SegmentState segmentState, long pos) {
Bytes entry = reuse(segmentState.tmpBytes, offsetFromPos(pos));
long keySize = keySizeMarshaller.readSize(entry);
ThreadLocalCopies copies = keyReaderProvider.getCopies(null);
K key = keyReaderProvider.get(copies, originalKeyReader).read(entry, keySize);
long timestamp = entry.readLong();
entry.skip(2L); // identifier and isDeleted flag
long valueSize = valueSizeMarshaller.readSize(entry);
alignment.alignPositionAddr(entry);
copies = valueReaderProvider.getCopies(copies);
V value = valueReaderProvider.get(copies, originalValueReader).read(entry, valueSize);
return new TimestampTrackingEntry(key, value, timestamp);
}
/**
* it is assumed that when calling this code the segment lock is in place
*
* @param pos the position of in the segment
* @return the timestamp of the entry when it was written
*/
public long timeStamp(long pos) {
final Bytes entry = reuse(timestampBytes, offsetFromPos(pos));
final long keySize = keySizeMarshaller.readSize(entry);
entry.skip(keySize);
return entry.readLong();
}
}
class TimestampTrackingEntry extends SimpleEntry {
private static final long serialVersionUID = 0L;
transient long timestamp;
public TimestampTrackingEntry(K key, V value, long timestamp) {
super(key, value);
this.timestamp = timestamp;
}
@Override
public V setValue(V value) {
long newTimestamp = timestamp = currentTime();
put(getKey(), value, localIdentifier, newTimestamp);
return super.setValue(value);
}
}
/**
* Once a change occurs to a map, map replication requires that these changes are picked up
* by another thread, this class provides an iterator like interface to poll for such changes.
*
In most cases the thread that adds data to the node is unlikely to be the same thread
* that replicates the data over to the other nodes, so data will have to be marshaled between
* the main thread storing data to the map, and the thread running the replication.
One
* way to perform this marshalling, would be to pipe the data into a queue. However, This class
* takes another approach. It uses a bit set, and marks bits which correspond to the indexes of
* the entries that have changed. It then provides an iterator like interface to poll for such
* changes.
*
* @author Rob Austin.
*/
class ModificationIterator implements Replica.ModificationIterator {
private final DirectBitSet changes;
private final int segmentIndexShift;
private final long posMask;
private final EntryModifiableCallback entryModifiableCallback =
new EntryModifiableCallback();
// todo get rid of this
private final MultiStoreBytes tmpBytes = new MultiStoreBytes();
volatile Thread thread = null;
private ModificationNotifier modificationNotifier;
// when a bootstrap is send this is the time stamp that the client will bootstrap up to
// if it is set as ZERO then the onPut() will set it to the current time, once the
// consumer has cycled through the bit set the timestamp will be set back to zero.
private AtomicLong bootStrapTimeStamp = new AtomicLong();
private long lastBootStrapTimeStamp = currentTime();
// records the current position of the cursor in the bitset
private volatile long position = -1L;
/**
* @param bytes the back the bitset, used to mark which entries have changed
* @param modificationNotifier called when ever there is a change applied
*/
public ModificationIterator(@NotNull final Bytes bytes,
@NotNull final ModificationNotifier modificationNotifier) {
setModificationNotifier(modificationNotifier);
long bitsPerSegment = bitsPerSegmentInModIterBitSet();
segmentIndexShift = Long.numberOfTrailingZeros(bitsPerSegment);
posMask = bitsPerSegment - 1L;
changes = ATSDirectBitSet.wrap(bytes);
}
public ModificationIterator(@NotNull final Bytes bytes) {
long bitsPerSegment = bitsPerSegmentInModIterBitSet();
segmentIndexShift = Long.numberOfTrailingZeros(bitsPerSegment);
posMask = bitsPerSegment - 1L;
changes = ATSDirectBitSet.wrap(bytes);
}
public void setModificationNotifier(ModificationNotifier modificationNotifier) {
this.modificationNotifier = modificationNotifier;
}
/**
* used to merge multiple segments and positions into a single index used by the bit map
*
* @param segmentIndex the index of the maps segment
* @param pos the position within this {@code segmentIndex}
* @return and index the has combined the {@code segmentIndex} and {@code pos} into a
* single value
*/
private long combine(int segmentIndex, long pos) {
return (((long) segmentIndex) << segmentIndexShift) | pos;
}
public void onPut(long pos, SharedSegment segment) {
final long combine = combine(segment.getIndex(), pos);
changes.set(combine);
// todo improve this - use the timestamp from the entry its self
bootStrapTimeStamp.compareAndSet(0, timestamp(pos, segment));
if (modificationNotifier != null)
modificationNotifier.onChange();
}
private long timestamp(long pos, SharedSegment segment) {
long timeStamp = segment.timeStamp(pos);
// this is not true for bootstraping of replicaiton events
// assert timeStamp > currentTime() - TimeUnit.SECONDS.toMillis(1) &&
// timeStamp <= currentTime() : "timeStamp=" + timeStamp + ", " +
// "currentTime=" + currentTime();
return timeStamp;
}
public void onRemove(long pos, SharedSegment segment) {
changes.set(combine(segment.getIndex(), pos));
// todo improve this - use the timestamp from the entry its self
bootStrapTimeStamp.compareAndSet(0, timestamp(pos, segment));
if (modificationNotifier != null)
modificationNotifier.onChange();
}
/**
* Ensures that garbage in the old entry's location won't be broadcast as changed entry.
*/
void onRelocation(long pos, SharedSegment segment) {
changes.clear(combine(segment.getIndex(), pos));
}
/**
* you can continue to poll hasNext() until data becomes available. If are are in the middle
* of processing an entry via {@code nextEntry}, hasNext will return true until the bit is
* cleared
*
* @return true if there is an entry
*/
@Override
public boolean hasNext() {
final long position = this.position;
return changes.nextSetBit(position + 1) != NOT_FOUND ||
(position >= 0L && changes.nextSetBit(0L) != NOT_FOUND);
}
/**
* @param entryCallback call this to get an entry, this class will take care of the locking
* @return true if an entry was processed
*/
@Override
public boolean nextEntry(@NotNull final EntryCallback entryCallback, final int chronicleId) {
Thread t = Thread.currentThread();
if (thread == null)
thread = t;
else if (thread != t)
throw new AssertionError("Used by " + thread + " and " + t);
long position = this.position;
while (true) {
long oldPosition = position;
position = changes.nextSetBit(oldPosition + 1L);
if (position == NOT_FOUND) {
if (oldPosition == NOT_FOUND) {
this.position = NOT_FOUND;
return false;
}
bootStrapTimeStamp.set(0);
continue;
}
this.position = position;
final VanillaChronicleMap.Segment segment =
segment((int) (position >>> segmentIndexShift));
segment.readLock(null);
try {
if (changes.get(position)) {
entryCallback.onBeforeEntry();
final long segmentPos = position & posMask;
long offset = segment.offsetFromPos(segmentPos);
final Bytes entry = segment.reuse2(tmpBytes, offset);
// it may not be successful if the buffer can not be re-sized so we will
// process it later, by NOT clearing the changes.clear(position)
final boolean success = entryCallback.onEntry(entry,
chronicleId,
bootStrapTimeStamp());
entryCallback.onAfterEntry();
if (success)
changes.clear(position);
return success;
}
// if the position was already cleared by another thread
// while we were trying to obtain segment lock (for example, in onRelocation()),
// go to pick up next (next iteration in while (true) loop)
} finally {
segment.readUnlock();
}
}
}
/**
* @return the timestamp that the remote client should bootstrap from when there has been a
* disconnection, this time maybe later than the message time as event are not send in
* chronological order from the bit set.
*/
private long bootStrapTimeStamp() {
final long timeStamp = bootStrapTimeStamp.get();
long result = (timeStamp == 0) ? this.lastBootStrapTimeStamp : timeStamp;
this.lastBootStrapTimeStamp = result;
return result;
}
@Override
public void dirtyEntries(long fromTimeStamp) {
// iterate over all the segments and mark bit in the modification iterator
// that correspond to entries with an older timestamp
for (final Segment segment : (Segment[]) segments) {
segment.dirtyEntries(fromTimeStamp, entryModifiableCallback,
bootstrapOnlyLocalEntries);
}
}
/**
* details about when a modification to an entry was made
*/
class EntryModifiableCallback {
/**
* set the bit related to {@code segment} and {@code pos}
*
* @param segmentIndex the segment relating to the bit to set
* @param pos the position relating to the bit to set
*/
public void set(int segmentIndex, long pos) {
final long combine = combine(segmentIndex, pos);
changes.set(combine);
}
}
}
}