diff --git a/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java b/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java index 520301e75..af15f7d9f 100644 --- a/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java +++ b/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java @@ -71,84 +71,113 @@ public abstract class VanillaChronicleHash globalMutableStateLockTryAcquireOperation = - TryAcquireOperations.lock(); - static final - AcquisitionStrategy - globalMutableStateLockAcquisitionStrategy = - AcquisitionStrategies.spinLoopOrFail(2, TimeUnit.SECONDS); - private static final long GLOBAL_MUTABLE_STATE_LOCK_OFFSET = 0L; - private static final long GLOBAL_MUTABLE_STATE_VALUE_OFFSET = 8L; + + ///////////////////////////////////////////////// + private String dataFileVersion; + ///////////////////////////////////////////////// // If the hash was created in the first place, or read from disk public transient boolean createdOrInMemory; + ///////////////////////////////////////////////// // Key Data model public Class keyClass; public SizeMarshaller keySizeMarshaller; public SizedReader keyReader; public DataAccess keyDataAccess; + ///////////////////////////////////////////////// public boolean checksumEntries; + ///////////////////////////////////////////////// // Concurrency (number of segments), memory management and dependent fields public int actualSegments; public HashSplitting hashSplitting; + public long chunkSize; public int maxChunksPerEntry; public long actualChunksPerSegmentTier; + + ///////////////////////////////////////////////// + // Precomputed offsets and sizes for fast Context init + int segmentHeaderSize; + public int tierHashLookupValueBits; public int tierHashLookupKeyBits; public int tierHashLookupSlotSize; public long tierHashLookupCapacity; public long maxEntriesPerHashLookup; + long tierHashLookupInnerSize; public long tierHashLookupOuterSize; + public long tierFreeListInnerSize; public long tierFreeListOuterSize; - public int tierEntrySpaceInnerOffset; - public long tierSize; - public long tiersInBulk; - public transient List tierBulkOffsets; - public transient long headerSize; - public transient long segmentHeadersOffset; - ///////////////////////////////////////////////// - // Miscellaneous fields - public transient CompactOffHeapLinearHashTable hashLookup; - public transient Identity identity; - protected int log2TiersInBulk; - ///////////////////////////////////////////////// - // Bytes Store (essentially, the base address) and serialization-dependent offsets - protected transient BytesStore bs; - ///////////////////////////////////////////////// - // Precomputed offsets and sizes for fast Context init - int segmentHeaderSize; - long tierHashLookupInnerSize; + long tierEntrySpaceInnerSize; + public int tierEntrySpaceInnerOffset; long tierEntrySpaceOuterSize; + + public long tierSize; + long maxExtraTiers; long tierBulkSizeInBytes; long tierBulkInnerOffsetToTiers; - transient long segmentsOffset; - ///////////////////////////////////////////////// - private String dataFileVersion; + public long tiersInBulk; + protected int log2TiersInBulk; + ///////////////////////////////////////////////// // Resources private transient File file; private transient RandomAccessFile raf; - - // --- End of instance fields --- private transient ChronicleHashResources resources; private transient Cleaner cleaner; + + ///////////////////////////////////////////////// + // Bytes Store (essentially, the base address) and serialization-dependent offsets + protected transient BytesStore bs; + + public static class TierBulkData { + public final BytesStore bytesStore; + public final long offset; + + public TierBulkData(BytesStore bytesStore, long offset) { + this.bytesStore = bytesStore; + this.offset = offset; + } + + public TierBulkData(TierBulkData data, long offset) { + this.bytesStore = data.bytesStore; + this.offset = offset; + } + } + + public transient List tierBulkOffsets; + + public transient long headerSize; + public transient long segmentHeadersOffset; + transient long segmentsOffset; + + ///////////////////////////////////////////////// + // Miscellaneous fields + public transient CompactOffHeapLinearHashTable hashLookup; + private transient VanillaGlobalMutableState globalMutableState; + /** + * {@link ChronicleHashCloseOnExitHook} needs to use {@code VanillaChronicleHash}es as + * WeakHashMap keys, but with identity comparison, not Map's equals() and hashCode(). + */ + public class Identity { + public VanillaChronicleHash hash() { + return VanillaChronicleHash.this; + } + } + + public transient Identity identity; + + // --- End of instance fields --- + + public VanillaChronicleHash(ChronicleMapBuilder builder) { // Version dataFileVersion = BuildVersion.version(); @@ -209,20 +238,6 @@ public abstract class VanillaChronicleHash globalMutableStateLockTryAcquireOperation = + TryAcquireOperations.lock(); + static final + AcquisitionStrategy + globalMutableStateLockAcquisitionStrategy = + AcquisitionStrategies.spinLoopOrFail(2, TimeUnit.SECONDS); + + private static final long GLOBAL_MUTABLE_STATE_LOCK_OFFSET = 0L; + private static final long GLOBAL_MUTABLE_STATE_VALUE_OFFSET = 8L; + private long globalMutableStateAddress() { return bsAddress() + headerSize; } @@ -998,29 +1045,4 @@ public abstract class VanillaChronicleHash> allContexts() { return Collections.unmodifiableList(resources.contexts()); } - - public static class TierBulkData { - public final BytesStore bytesStore; - public final long offset; - - public TierBulkData(BytesStore bytesStore, long offset) { - this.bytesStore = bytesStore; - this.offset = offset; - } - - public TierBulkData(TierBulkData data, long offset) { - this.bytesStore = data.bytesStore; - this.offset = offset; - } - } - - /** - * {@link ChronicleHashCloseOnExitHook} needs to use {@code VanillaChronicleHash}es as - * WeakHashMap keys, but with identity comparison, not Map's equals() and hashCode(). - */ - public class Identity { - public VanillaChronicleHash hash() { - return VanillaChronicleHash.this; - } - } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/impl/SerializationBuilder.java b/src/main/java/net/openhft/chronicle/hash/serialization/impl/SerializationBuilder.java index 3e5d0780c..6bfd6c418 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/impl/SerializationBuilder.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/impl/SerializationBuilder.java @@ -143,7 +143,7 @@ public final class SerializationBuilder implements Cloneable { try { return (Byteable) OS.memory().allocateInstance(tClass); } catch (InstantiationException e) { - throw new AssertionError(e); + throw new IllegalStateException(e); } } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index fa99e0e87..b847805f7 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -152,6 +152,7 @@ public final class ChronicleMapBuilder implements private static final double UNDEFINED_DOUBLE_CONFIG = Double.NaN; private static final ConcurrentHashMap fileLockingControl = new ConcurrentHashMap<>(128); + private static int MAX_BOOTSTRAPPING_HEADER_SIZE = (int) MemoryUnit.KILOBYTES.toBytes(16); private static final Logger chronicleMapLogger = LoggerFactory.getLogger(ChronicleMap.class); private static final ChronicleHashCorruption.Listener defaultChronicleMapCorruptionListener = corruption -> { @@ -161,7 +162,8 @@ public final class ChronicleMapBuilder implements chronicleMapLogger.error(corruption.message()); } }; - private static int MAX_BOOTSTRAPPING_HEADER_SIZE = (int) MemoryUnit.KILOBYTES.toBytes(16); + + private String name; SerializationBuilder keyBuilder; SerializationBuilder valueBuilder; K averageKey; @@ -175,6 +177,7 @@ public final class ChronicleMapBuilder implements long cleanupTimeout = 1; TimeUnit cleanupTimeoutUnit = TimeUnit.MINUTES; boolean cleanupRemovedEntries = true; + ////////////////////////////// // Configuration fields DefaultValueProvider defaultValueProvider = DefaultSpi.defaultValueProvider(); @@ -182,7 +185,6 @@ public final class ChronicleMapBuilder implements MapMethods methods = DefaultSpi.mapMethods(); MapEntryOperations entryOperations = mapEntryOperations(); MapRemoteOperations remoteOperations = mapRemoteOperations(); - private String name; // not final because of cloning private ChronicleMapBuilderPrivateAPI privateAPI = new ChronicleMapBuilderPrivateAPI<>(this); @@ -418,52 +420,6 @@ public final class ChronicleMapBuilder implements writeFully(fileChannel, SIZE_WORD_OFFSET, headerBuffer); } - /** - * @return ByteBuffer, in [position, limit) range the self bootstrapping header is read - */ - private static ByteBuffer readSelfBootstrappingHeader( - File file, RandomAccessFile raf, int headerSize, boolean recover, - ChronicleHashCorruption.Listener corruptionListener, - ChronicleHashCorruptionImpl corruption) throws IOException { - if (raf.length() < headerSize + SELF_BOOTSTRAPPING_HEADER_OFFSET) { - throw throwRecoveryOrReturnIOException(file, - "The file is shorter than the header size: " + headerSize + - ", file size: " + raf.length(), recover); - } - FileChannel fileChannel = raf.getChannel(); - ByteBuffer headerBuffer = ByteBuffer.allocate( - SELF_BOOTSTRAPPING_HEADER_OFFSET + headerSize); - headerBuffer.order(LITTLE_ENDIAN); - readFully(fileChannel, 0, headerBuffer); - if (headerBuffer.remaining() > 0) { - throw throwRecoveryOrReturnIOException(file, "Unable to read the header fully, " + - headerBuffer.remaining() + " is remaining to read, likely the file was " + - "truncated", recover); - } - int sizeWord = headerBuffer.getInt(SIZE_WORD_OFFSET); - if (!SizePrefixedBlob.isReady(sizeWord)) { - if (recover) { - report(corruptionListener, corruption, -1, () -> - format("file={}: size-prefixed blob readiness bit is set to NOT_COMPLETE", - file) - ); - // the bit will be overwritten to READY in the end of recovery procedure, so nothing - // to fix right here - } else { - throw new IOException("file=" + file + ": sizeWord is not ready: " + sizeWord); - } - } - headerBuffer.position(SELF_BOOTSTRAPPING_HEADER_OFFSET); - return headerBuffer; - } - - private static boolean checkSumSelfBootstrappingHeader( - ByteBuffer headerBuffer, int headerSize) { - long checkSum = headerChecksum(headerBuffer, headerSize); - long storedChecksum = headerBuffer.getLong(HEADER_OFFSET); - return storedChecksum == checkSum; - } - @Override public ChronicleMapBuilder clone() { try { @@ -1709,6 +1665,52 @@ public final class ChronicleMapBuilder implements } } + /** + * @return ByteBuffer, in [position, limit) range the self bootstrapping header is read + */ + private static ByteBuffer readSelfBootstrappingHeader( + File file, RandomAccessFile raf, int headerSize, boolean recover, + ChronicleHashCorruption.Listener corruptionListener, + ChronicleHashCorruptionImpl corruption) throws IOException { + if (raf.length() < headerSize + SELF_BOOTSTRAPPING_HEADER_OFFSET) { + throw throwRecoveryOrReturnIOException(file, + "The file is shorter than the header size: " + headerSize + + ", file size: " + raf.length(), recover); + } + FileChannel fileChannel = raf.getChannel(); + ByteBuffer headerBuffer = ByteBuffer.allocate( + SELF_BOOTSTRAPPING_HEADER_OFFSET + headerSize); + headerBuffer.order(LITTLE_ENDIAN); + readFully(fileChannel, 0, headerBuffer); + if (headerBuffer.remaining() > 0) { + throw throwRecoveryOrReturnIOException(file, "Unable to read the header fully, " + + headerBuffer.remaining() + " is remaining to read, likely the file was " + + "truncated", recover); + } + int sizeWord = headerBuffer.getInt(SIZE_WORD_OFFSET); + if (!SizePrefixedBlob.isReady(sizeWord)) { + if (recover) { + report(corruptionListener, corruption, -1, () -> + format("file={}: size-prefixed blob readiness bit is set to NOT_COMPLETE", + file) + ); + // the bit will be overwritten to READY in the end of recovery procedure, so nothing + // to fix right here + } else { + throw new IOException("file=" + file + ": sizeWord is not ready: " + sizeWord); + } + } + headerBuffer.position(SELF_BOOTSTRAPPING_HEADER_OFFSET); + return headerBuffer; + } + + private static boolean checkSumSelfBootstrappingHeader( + ByteBuffer headerBuffer, int headerSize) { + long checkSum = headerChecksum(headerBuffer, headerSize); + long storedChecksum = headerBuffer.getLong(HEADER_OFFSET); + return storedChecksum == checkSum; + } + private VanillaChronicleMap createWithNewFile( VanillaChronicleMap map, File file, RandomAccessFile raf, ChronicleHashResources resources, ByteBuffer headerBuffer, int headerSize) @@ -1749,7 +1751,6 @@ public final class ChronicleMapBuilder implements Bytes headerBytes = Bytes.wrapForRead(headerBuffer); headerBytes.readPosition(headerBuffer.position()); headerBytes.readLimit(headerBuffer.limit()); - System.out.println(headerBytes); Wire wire = new TextWire(headerBytes); VanillaChronicleMap map = wire.getValueIn().typedMarshallable(); map.initBeforeMapping(file, raf, headerBuffer.limit(), recover); diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 99f8b0198..30545578b 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -56,29 +56,20 @@ public class VanillaChronicleMap ExternalMapQueryContext> implements AbstractChronicleMap { + ///////////////////////////////////////////////// + // Value Data model + Class valueClass; public SizeMarshaller valueSizeMarshaller; public SizedReader valueReader; public DataAccess valueDataAccess; + public boolean constantlySizedEntry; + ///////////////////////////////////////////////// // Memory management and dependent fields public int alignment; public int worstAlignment; - public transient boolean couldNotDetermineAlignmentBeforeAllocation; - /** @see net.openhft.chronicle.set.SetFromMap */ - public transient ChronicleSet chronicleSet; - public transient MapEntryOperations entryOperations; - public transient MapMethods methods; - public transient DefaultValueProvider defaultValueProvider; - ///////////////////////////////////////////////// - // Value Data model - Class valueClass; - ///////////////////////////////////////////////// - // Behavior - transient boolean putReturnsNull; - transient boolean removeReturnsNull; - transient Set> entrySet; - transient ThreadLocal cxt; + ///////////////////////////////////////////////// private transient String name; /** @@ -87,7 +78,25 @@ public class VanillaChronicleMap * initOwnTransients(). */ private transient String identityString; + + public transient boolean couldNotDetermineAlignmentBeforeAllocation; + + ///////////////////////////////////////////////// + // Behavior + transient boolean putReturnsNull; + transient boolean removeReturnsNull; + + transient Set> entrySet; + + /** @see net.openhft.chronicle.set.SetFromMap */ + public transient ChronicleSet chronicleSet; + + public transient MapEntryOperations entryOperations; + public transient MapMethods methods; private transient boolean defaultEntryOperationsAndMethods; + public transient DefaultValueProvider defaultValueProvider; + + transient ThreadLocal cxt; public VanillaChronicleMap(ChronicleMapBuilder builder) throws IOException { super(builder); @@ -107,10 +116,6 @@ public class VanillaChronicleMap initTransients(); } - public static long alignAddr(long addr, long alignment) { - return (addr + alignment - 1) & ~(alignment - 1L); - } - @Override protected void readMarshallableFields(@NotNull WireIn wireIn) { super.readMarshallableFields(wireIn); @@ -282,7 +287,7 @@ public class VanillaChronicleMap ", identityHashCode=" + System.identityHashCode(this) + "}"; } - + @Override public void clear() { forEachEntry(c -> c.context().remove(c)); @@ -301,6 +306,10 @@ public class VanillaChronicleMap entry.readSkip(skip); } + public static long alignAddr(long addr, long alignment) { + return (addr + alignment - 1) & ~(alignment - 1L); + } + final ChainingInterface q() { ThreadLocal cxt = this.cxt; if (cxt == null)