Merge remote-tracking branch 'origin/master'

This commit is contained in:
Rob Austin
2014-12-30 10:08:08 +00:00
15 changed files with 190 additions and 74 deletions
@@ -26,6 +26,8 @@ public interface SizeMarshaller extends Serializable {
int sizeEncodingSize(long size);
long minEncodableSize();
int minSizeEncodingSize();
int maxSizeEncodingSize();
@@ -36,6 +36,11 @@ public final class SizeMarshallers {
return stopBitLength(size);
}
@Override
public long minEncodableSize() {
return 0L;
}
@Override
public int minSizeEncodingSize() {
return 1;
@@ -75,6 +80,11 @@ public final class SizeMarshallers {
return 0;
}
@Override
public long minEncodableSize() {
return size;
}
@Override
public int minSizeEncodingSize() {
return 0;
@@ -71,6 +71,11 @@ public abstract class ByteableMarshaller<E extends Byteable>
return 0;
}
@Override
public long minEncodableSize() {
return size;
}
@Override
public int minSizeEncodingSize() {
return 0;
@@ -39,6 +39,12 @@ public enum DoubleMarshaller implements BytesInterop<Double>, BytesReader<Double
return 0;
}
@Override
public long minEncodableSize() {
return 8L;
}
@Override
public int minSizeEncodingSize() {
return 0;
@@ -38,6 +38,11 @@ public enum IntegerMarshaller
return 0;
}
@Override
public long minEncodableSize() {
return 4L;
}
@Override
public int minSizeEncodingSize() {
return 0;
@@ -38,6 +38,11 @@ public enum LongMarshaller
return 0;
}
@Override
public long minEncodableSize() {
return 8L;
}
@Override
public int minSizeEncodingSize() {
return 0;
@@ -37,6 +37,11 @@ public enum VoidMarshaller
return 0;
}
@Override
public long minEncodableSize() {
return 0L;
}
@Override
public int minSizeEncodingSize() {
return 0;
@@ -47,7 +47,7 @@ public enum Alignment {
@Override
int alignment() {
return 0;
return 1;
}
},
@@ -439,37 +439,34 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
size += averageSizeEncodingSize(valueBuilder, valueSize);
Alignment alignment = valueAlignment();
int worstAlignment;
if (alignment != Alignment.NO_ALIGNMENT) {
if (constantlySizedKeys() && valueBuilder.constantSizeEncodingSizeMarshaller()) {
long constantSizeBeforeAlignment = round(size);
if (constantlySizedValues()) {
// see specialEntrySpaceOffset()
long totalDataSize = constantSizeBeforeAlignment + constantValueSize();
worstAlignment = (int) (alignment.alignAddr(totalDataSize) - totalDataSize);
} else {
determineAlignment:
if (actualChunkSize > 0) {
worstAlignment = worstAlignmentAssumingChunkSize(constantSizeBeforeAlignment,
actualChunkSize);
} else {
int chunkSize = 8;
worstAlignment = worstAlignmentAssumingChunkSize(
constantSizeBeforeAlignment, chunkSize);
if (size + worstAlignment + valueSize >=
maxDefaultChunksPerAverageEntry(replicated) * chunkSize) {
break determineAlignment;
}
chunkSize = 4;
worstAlignment = worstAlignmentAssumingChunkSize(
constantSizeBeforeAlignment, chunkSize);
}
}
if (alignment != Alignment.NO_ALIGNMENT &&
constantlySizedKeys() && valueBuilder.constantSizeEncodingSizeMarshaller()) {
long constantSizeBeforeAlignment = round(size);
if (constantlySizedValues()) {
// see specialEntrySpaceOffset()
long totalDataSize = constantSizeBeforeAlignment + constantValueSize();
worstAlignment = (int) (alignment.alignAddr(totalDataSize) - totalDataSize);
} else {
// assume worst case, we always lose most possible bytes for alignment
worstAlignment = alignment.alignment() - 1;
determineAlignment:
if (actualChunkSize > 0) {
worstAlignment = worstAlignmentAssumingChunkSize(constantSizeBeforeAlignment,
actualChunkSize);
} else {
int chunkSize = 8;
worstAlignment = worstAlignmentAssumingChunkSize(
constantSizeBeforeAlignment, chunkSize);
if (size + worstAlignment + valueSize >=
maxDefaultChunksPerAverageEntry(replicated) * chunkSize) {
break determineAlignment;
}
chunkSize = 4;
worstAlignment = worstAlignmentAssumingChunkSize(
constantSizeBeforeAlignment, chunkSize);
}
}
} else {
worstAlignment = 0;
// assume worst case, we always lose most possible bytes for alignment
worstAlignment = alignment.alignment() - 1;
}
size += worstAlignment;
size += valueSize;
@@ -481,7 +478,7 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
* always the same, we should _misalign_ the first chunk.
*/
int specialEntrySpaceOffset(boolean replicated) {
if (!constantlySizedEntries() || valueAlignment() == Alignment.NO_ALIGNMENT)
if (!constantlySizedEntries())
return 0;
return (int) (constantValueSize() % valueAlignment().alignment());
}
@@ -510,8 +507,6 @@ public final class ChronicleMapBuilder<K, V> implements Cloneable,
private int worstAlignmentAssumingChunkSize(
long constantSizeBeforeAlignment, int chunkSize) {
if (valueAlignment() == Alignment.NO_ALIGNMENT)
return 0;
Alignment valueAlignment = valueAlignment();
long firstAlignment = valueAlignment.alignAddr(constantSizeBeforeAlignment) -
constantSizeBeforeAlignment;
@@ -80,8 +80,8 @@ enum MultiMapFactory {
throw new IllegalArgumentException("Capacity " + capacity + " not supported");
}
public static long sizeOfBitSetInBytes(long minCapacity) {
return BYTES.convert(LONGS.align(multiMapCapacity(minCapacity), BITS), BITS);
public static long sizeOfBitSetInBytes(long actualChunksPerSegment) {
return LONGS.align(BYTES.alignAndConvert(actualChunksPerSegment, BITS), BYTES);
}
public static ATSDirectBitSet newPositions(long capacity) {
@@ -683,19 +683,8 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
}
@Override
long entrySize(long keySize, long valueSize) {
long result = alignment.alignAddr(metaDataBytes +
keySizeMarshaller.sizeEncodingSize(keySize) + keySize + ADDITIONAL_ENTRY_BYTES +
valueSizeMarshaller.sizeEncodingSize(valueSize)) + valueSize;
// replication enforces that the entry size will never be larger than an unsigned short
if (result > Integer.MAX_VALUE)
throw new IllegalStateException("ENTRY WRITE_BUFFER_SIZE TOO LARGE : Replicated " +
"ChronicleMap's" +
" are restricted to an " +
"entry size of " + Integer.MAX_VALUE + ", " +
"your entry size=" + result);
return result;
long sizeOfEverythingBeforeValue(long keySize, long valueSize) {
return super.sizeOfEverythingBeforeValue(keySize, valueSize) + ADDITIONAL_ENTRY_BYTES;
}
@Override
@@ -735,7 +724,9 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
// deleted flag
entry.writeBoolean(false);
long prevValueSize = readValueSize(entry);
long prevValueSize = valueSizeMarshaller.readSize(entry);
long sizeOfEverythingBeforeValue = entry.position();
alignment.alignPositionAddr(entry);
long valueAddr = entry.positionAddr();
long entryEndAddr = valueAddr + prevValueSize;
@@ -758,9 +749,9 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
} else {
throw defaultValueOrPrepareBytesShouldBeSpecified();
}
putValue(pos, offset, entry, valueSizePos, entryEndAddr,
segmentState, metaElemWriter, elemWriter, elem,
metaElemWriter.size(elemWriter, elem), hashLookup);
putValue(pos, offset, entry, valueSizePos, entryEndAddr, segmentState,
metaElemWriter, elemWriter, elem, metaElemWriter.size(elemWriter, elem),
hashLookup, sizeOfEverythingBeforeValue);
pos = segmentState.pos;
incrementSize();
@@ -953,7 +944,13 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
// is still held
if (isDeleted) {
incrementSize();
hashLookup.putPosition(segmentState.pos);
// if they are NOT equal, it means the entry was relocated in putValue(),
// hence position is already set
if (pos == segmentState.pos) {
hashLookup.putPosition(segmentState.pos);
} else {
assert hashLookup.getPositions().isSet(segmentState.pos);
}
}
if (resultUnused)
return null;
@@ -1157,8 +1154,10 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
}
// key is not found
if (remote) {
long entrySize = entrySize(keySize, 0);
long pos = alloc(inChunks(entrySize));
long minEncodableValueSize = valueSizeMarshaller.minEncodableSize();
long entrySize = entrySize(keySize, minEncodableValueSize);
int allocatedChunks = inChunks(entrySize);
long pos = alloc(allocatedChunks);
long offset = offsetFromPos(pos);
clearMetaData(offset);
reuse(entry, offset);
@@ -1170,6 +1169,12 @@ final class ReplicatedChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? supe
entry.writeByte(identifier);
entry.writeBoolean(true);
valueSizeMarshaller.writeSize(entry, minEncodableValueSize);
alignment.alignPositionAddr(entry);
entry.skip(minEncodableValueSize);
freeExtraAllocatedChunks(pos, allocatedChunks, entry);
hashLookup.putAfterFailedSearch(searchState, pos);
hashLookup.removePosition(pos);
}
@@ -47,7 +47,6 @@ import java.lang.reflect.Array;
import java.nio.channels.FileChannel;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static java.lang.Long.numberOfTrailingZeros;
import static java.lang.Math.max;
@@ -1409,6 +1408,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
assert booleanResult || expectedValue == null;
}
static final boolean checkSegmentMultiMapsAndBitSetsConsistencyOnBootstrap = Boolean.getBoolean(
"chronicleMap.checkSegmentMultiMapsAndBitSetsConsistencyOnBootstrap");
// these methods should be package local, not public or private.
class Segment implements SharedSegment {
/*
@@ -1421,7 +1423,6 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
*/
static final long LOCK_OFFSET = 0L; // 64-bit
static final long SIZE_OFFSET = LOCK_OFFSET + 8L; // 32-bit
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
final NativeBytes segmentHeader;
final NativeBytes bytes;
final long entriesOffset;
@@ -1448,13 +1449,15 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
* multiMapsPerSegment();
final NativeBytes bsBytes = new NativeBytes(ms.objectSerializer(),
start,
start + LONGS.align(BYTES.alignAndConvert(actualChunksPerSegment, BITS), BYTES),
start + MultiMapFactory.sizeOfBitSetInBytes(actualChunksPerSegment),
null);
freeList = new SingleThreadedDirectBitSet(bsBytes);
start += numberOfBitSets() * sizeOfSegmentFreeListBitSets();
start += specialEntrySpaceOffset;
entriesOffset = start - bytes.startAddr();
assert bytes.capacity() >= entriesOffset + actualChunksPerSegment * chunkSize;
if (checkSegmentMultiMapsAndBitSetsConsistencyOnBootstrap)
checkMultiMapsAndBitSetsConsistency();
}
final MultiMap hashLookup() {
@@ -1473,6 +1476,30 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
return multiMapFactory.create(multiMapBytes, sizeOfMultiMapBitSetBytes);
}
void checkMultiMapsAndBitSetsConsistency() {
MultiMap hashLookup = hashLookup();
final DirectBitSet positions = hashLookup.getPositions();
class EntryChecker implements MultiMap.EntryConsumer {
long size = 0;
@Override
public void accept(long key, long value) {
if (positions.isSet(value))
size++;
if (freeList.isClear(value))
throw new IllegalStateException("Position " + value + " is present in " +
"multiMap but available in the free chunk list");
}
}
EntryChecker entryChecker = new EntryChecker();
hashLookup.forEach(entryChecker);
if (size() != entryChecker.size || entryChecker.size != positions.cardinality()) {
throw new IllegalStateException("Segment inconsistent: " +
"size by Segment counter: " + size() +
", size by multiMap records present in bit set: " + entryChecker.size +
", size by multiMap bit set cardinality: " + positions.cardinality());
}
}
public final int getIndex() {
return index;
}
@@ -1614,10 +1641,12 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
return entry;
}
long entrySize(long keySize, long valueSize) {
long sizeOfEverythingBeforeValue = metaDataBytes +
keySizeMarshaller.sizeEncodingSize(keySize) + keySize +
valueSizeMarshaller.sizeEncodingSize(valueSize);
final long entrySize(long keySize, long valueSize) {
long sizeOfEverythingBeforeValue = sizeOfEverythingBeforeValue(keySize, valueSize);
return innerEntrySize(sizeOfEverythingBeforeValue, valueSize);
}
private long innerEntrySize(long sizeOfEverythingBeforeValue, long valueSize) {
if (constantlySizedEntry) {
return alignment.alignAddr(sizeOfEverythingBeforeValue + valueSize);
} else if (couldNotDetermineAlignmentBeforeAllocation) {
@@ -1627,6 +1656,12 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
}
}
long sizeOfEverythingBeforeValue(long keySize, long valueSize) {
return metaDataBytes +
keySizeMarshaller.sizeEncodingSize(keySize) + keySize +
valueSizeMarshaller.sizeEncodingSize(valueSize);
}
final int inChunks(long sizeInBytes) {
if (sizeInBytes <= chunkSize)
return 1;
@@ -1862,7 +1897,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
long valueSize = metaValueInterop.size(valueInterop, value);
long valueSizePos = entry.position();
long prevValueSize = readValueSize(entry);
long prevValueSize = valueSizeMarshaller.readSize(entry);
long sizeOfEverythingBeforeValue = entry.position();
alignment.alignPositionAddr(entry);
long valueAddr = entry.positionAddr();
long entryEndAddr = valueAddr + prevValueSize;
@@ -1878,7 +1915,8 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
// putValue may relocate entry and change offset
putValue(pos, offset, entry, valueSizePos, entryEndAddr, segmentState,
metaValueInterop, valueInterop, value, valueSize, searchedHashLookup);
metaValueInterop, valueInterop, value, valueSize, searchedHashLookup,
sizeOfEverythingBeforeValue);
// put callbacks
onPutMaybeRemote(segmentState.pos, remote);
@@ -1912,7 +1950,8 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
MultiStoreBytes entry, boolean writeDefaultInitialReplicationValues) {
long valueSize = metaElemWriter.size(elemWriter, elem);
long entrySize = entrySize(keySize, valueSize);
long pos = alloc(inChunks(entrySize));
int allocatedChunks = inChunks(entrySize);
long pos = alloc(allocatedChunks);
segmentState.pos = pos;
long offset = offsetFromPos(pos);
clearMetaData(offset);
@@ -1928,12 +1967,22 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
alignment.alignPositionAddr(entry);
metaElemWriter.write(elemWriter, entry, elem);
freeExtraAllocatedChunks(pos, allocatedChunks, entry);
hashLookup.putAfterFailedSearch(segmentState.searchState, pos);
incrementSize();
return valueSize;
}
final void freeExtraAllocatedChunks(long pos, int allocatedChunks, Bytes entry) {
int actuallyUsedChunks;
if (!constantlySizedEntry && couldNotDetermineAlignmentBeforeAllocation && // fast path
(actuallyUsedChunks = inChunks(entry.position())) < allocatedChunks) {
free(pos + actuallyUsedChunks, allocatedChunks - actuallyUsedChunks);
}
}
void manageReplicationBytes(Bytes entry, boolean writeDefaultInitialReplicationValues,
boolean remove) {
// do nothing
@@ -2217,7 +2266,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
ReadValue<RV> readValue, InstanceOrBytesToInstance<? super RV, V> toValue,
long pos, long offset, MultiStoreBytes entry, MultiMap searchedHashLookup) {
long valueSizePos = entry.position();
long valueSize = readValueSize(entry);
long valueSize = valueSizeMarshaller.readSize(entry);
long sizeOfEverythingBeforeValue = entry.position();
alignment.alignPositionAddr(entry);
long entryEndAddr = entry.positionAddr() + valueSize;
RV prevValue;
@@ -2241,7 +2292,8 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
copies, valueInterop, newValue);
long newValueSize = metaValueInterop.size(valueInterop, newValue);
putValue(pos, offset, entry, valueSizePos, entryEndAddr, segmentState,
metaValueInterop, valueInterop, newValue, newValueSize, searchedHashLookup);
metaValueInterop, valueInterop, newValue, newValueSize, searchedHashLookup,
sizeOfEverythingBeforeValue);
// put callbacks
onPut(this, segmentState.pos);
@@ -2275,7 +2327,7 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
SegmentState segmentState,
MetaBytesWriter<E, ? super EW> metaElemWriter, EW elemWriter, E newElem,
long newElemSize,
MultiMap searchedHashLookup) {
MultiMap searchedHashLookup, long sizeOfEverythingBeforeValue) {
long entryStartAddr = entry.address();
long valueSizeAddr = entryStartAddr + valueSizePos;
long newValueAddr = alignment.alignAddr(
@@ -2297,7 +2349,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
// RELOCATION
free(pos, oldSizeInChunks);
onRelocation(this, pos);
pos = alloc(newSizeInChunks);
int allocatedChunks =
inChunks(innerEntrySize(sizeOfEverythingBeforeValue, newElemSize));
pos = alloc(allocatedChunks);
// putValue() is called from put() and replace()
// after successful search by key
searchedHashLookup.replacePrevPos(segmentState.searchState, pos);
@@ -2309,6 +2363,15 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
NativeBytes.UNSAFE.copyMemory(entryStartAddr,
newEntryStartAddr, valueSizeAddr - entryStartAddr);
entry = reuse(entry, offset);
entry.position(valueSizePos);
valueSizeMarshaller.writeSize(entry, newElemSize);
alignment.alignPositionAddr(entry);
metaElemWriter.write(elemWriter, entry, newElem);
freeExtraAllocatedChunks(pos, allocatedChunks, entry);
segmentState.pos = pos;
return offset;
// END OF RELOCATION
} else if (newSizeInChunks < oldSizeInChunks) {
// Freeing extra chunks
@@ -2748,7 +2811,9 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
entry.skip(keySize);
segment.manageReplicationBytes(entry, true, false);
long valueSizePos = entry.position();
long valueSize = map.readValueSize(entry);
long valueSize = map.valueSizeMarshaller.readSize(entry);
long sizeOfEverythingBeforeValue = entry.position();
map.alignment.alignPositionAddr(entry);
long entryEndAddr = entry.positionAddr() + valueSize;
VI valueInterop = map.valueInteropProvider.get(copies, map.originalValueInterop);
V value = value();
@@ -2756,7 +2821,8 @@ class VanillaChronicleMap<K, KI, MKI extends MetaBytesInterop<K, ? super KI>,
.get(copies, map.originalMetaValueInterop, valueInterop, value);
long newValueSize = metaValueInterop.size(valueInterop, value);
segment.putValue(pos, offset, entry, valueSizePos, entryEndAddr, segmentState,
metaValueInterop, valueInterop, value, newValueSize, segment.hashLookup());
metaValueInterop, valueInterop, value, newValueSize, segment.hashLookup(),
sizeOfEverythingBeforeValue);
map.onPut(segment, segmentState.pos);
}
putOnClose = true;
@@ -64,6 +64,11 @@ enum DummyValueMarshaller
return 0;
}
@Override
public long minEncodableSize() {
return 0L;
}
@Override
public int minSizeEncodingSize() {
return 0;
@@ -77,9 +77,12 @@ public class AbstractChronicleMapConverter<K, V> implements Converter {
@Override
public Object unmarshal(HierarchicalStreamReader reader,
UnmarshallingContext context) {
reader.moveDown();
// empty map
if ("[\"\"]".equals(reader.getValue()))
return null;
if (!"cmap".equals(reader.getNodeName()))
throw new ConversionException("root node should be cmap, was " + reader.getNodeName());
throw new ConversionException("should be under 'cmap' node");
reader.moveDown();
while (reader.hasMoreChildren()) {
reader.moveDown();
@@ -53,8 +53,12 @@ public class TCPSocketReplicationTest {
ChronicleMapBuilder<Integer, CharSequence> map1Builder =
newTcpSocketShmBuilder(Integer.class, CharSequence.class,
(byte) 1, port, new InetSocketAddress("localhost", port + 1));
map1 = map1Builder.averageValueSize(10).create();
map2 = newTcpSocketShmIntString((byte) 2, port + 1);
map1 = map1Builder.entries(Builder.SIZE).averageValueSize(10).create();
ChronicleMapBuilder<Integer, CharSequence> map2Builder =
newTcpSocketShmBuilder(Integer.class, CharSequence.class,
(byte) 2, port + 1);
map2 = map2Builder.entries(Builder.SIZE).averageValueSize(10).create();
s_port += 2;
}