Add ChronicleHash.offHeapMemoryUsed() method; Centralized management of all direct memory and file resources of ChronicleMap in one Cleaner

src/main/java/net/openhft/chronicle/hash/ChronicleHash.java

@@ -52,6 +52,16 @@ public interface ChronicleHash<K, E extends HashEntry<K>, SC extends HashSegment
      */
     long longSize();
 
+    /**
+     * Returns the amount of off-heap memory (in bytes), allocated by this {@code ChronicleHash} or
+     * shared with with other ChronicleHashes, persisting to the same {@link #file()}.
+     *
+     * <p>After {@link #close()} this method returns 0.
+     *
+     * @return the amount of off-heap memory, used by this {@code ChronicleHash} (in bytes)
+     */
+    long offHeapMemoryUsed();
+
     /**
      * @return the class of {@code <K>}
      */

src/main/java/net/openhft/chronicle/hash/impl/ChronicleHashResourceReleaser.java

@@ -0,0 +1,59 @@
+/*
+ *      Copyright (C) 2012, 2016  higherfrequencytrading.com
+ *      Copyright (C) 2016 Roman Leventov
+ *
+ *      This program is free software: you can redistribute it and/or modify
+ *      it under the terms of the GNU Lesser General Public License as published by
+ *      the Free Software Foundation, either version 3 of the License.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU Lesser General Public License for more details.
+ *
+ *      You should have received a copy of the GNU Lesser General Public License
+ *      along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.openhft.chronicle.hash.impl;
+
+import net.openhft.chronicle.hash.impl.util.Throwables;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class ChronicleHashResourceReleaser implements Runnable {
+
+    List<MemoryResource> memoryResources = new ArrayList<>(1);
+
+    public final synchronized void addMemoryResource(long address, long size) {
+        if (memoryResources == null)
+            throw new IllegalStateException("Already released");
+        memoryResources.add(new MemoryResource(address, size));
+    }
+
+    public final synchronized long totalMemory() {
+        if (memoryResources == null)
+            return 0L;
+        long totalMemory = 0L;
+        for (MemoryResource memoryResource : memoryResources) {
+            totalMemory += memoryResource.size;
+        }
+        return totalMemory;
+    }
+
+    abstract Throwable doRelease();
+
+    public final synchronized void releaseManually() {
+        if (memoryResources == null)
+            return; // Already released
+        Throwable thrown = null;
+        try {
+            thrown = doRelease();
+        } finally {
+            memoryResources = null;
+        }
+        if (thrown != null)
+            throw Throwables.propagate(thrown);
+    }
+}

src/main/java/net/openhft/chronicle/hash/impl/InMemoryChronicleHashResourceReleaser.java

@@ -0,0 +1,71 @@
+/*
+ *      Copyright (C) 2012, 2016  higherfrequencytrading.com
+ *      Copyright (C) 2016 Roman Leventov
+ *
+ *      This program is free software: you can redistribute it and/or modify
+ *      it under the terms of the GNU Lesser General Public License as published by
+ *      the Free Software Foundation, either version 3 of the License.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU Lesser General Public License for more details.
+ *
+ *      You should have received a copy of the GNU Lesser General Public License
+ *      along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.openhft.chronicle.hash.impl;
+
+import net.openhft.chronicle.core.OS;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class InMemoryChronicleHashResourceReleaser extends ChronicleHashResourceReleaser {
+
+    private static final Logger LOG =
+            LoggerFactory.getLogger(PersistedChronicleHashResourceReleaser.class);
+
+    public InMemoryChronicleHashResourceReleaser() {}
+
+    @Override
+    public synchronized void run() {
+        if (memoryResources == null)
+            return; // Already released
+        // All Throwables are catched because this class is used as Runnable for sun.misc.Cleaner,
+        // hence must not fail
+        Throwable thrown = null;
+        try {
+            thrown = doRelease();
+        } finally {
+            memoryResources = null;
+        }
+        if (thrown != null) {
+            try {
+                LOG.error("Error on releasing memory of a Chronicle Map:", thrown);
+            } catch (Throwable t) {
+                // This may occur if Releaser is run in a shutdown hook, and the log service has
+                // already been shut down. Try to fall back to printStackTrace().
+                thrown.addSuppressed(t);
+                thrown.printStackTrace();
+            }
+        }
+    }
+
+    @Override
+    Throwable doRelease() {
+        Throwable thrown = null;
+        for (MemoryResource allocation : memoryResources) {
+            try {
+                OS.memory().freeMemory(allocation.address, allocation.size);
+            } catch (Throwable t) {
+                if (thrown == null) {
+                    thrown = t;
+                } else {
+                    thrown.addSuppressed(t);
+                }
+            }
+        }
+        return thrown;
+    }
+}

src/main/java/net/openhft/chronicle/hash/impl/MemoryResource.java

@@ -17,21 +17,12 @@
 
 package net.openhft.chronicle.hash.impl;
 
-import net.openhft.chronicle.core.ReferenceCounted;
+final class MemoryResource {
+    final long address;
+    final long size;
 
-public enum DummyReferenceCounted implements ReferenceCounted {
-    DUMMY_REFERENCE_COUNTED;
-
-    @Override
-    public void reserve() throws IllegalStateException {
-    }
-
-    @Override
-    public void release() throws IllegalStateException {
-    }
-
-    @Override
-    public long refCount() {
-        return 0;
+    MemoryResource(long address, long size) {
+        this.address = address;
+        this.size = size;
     }
 }

src/main/java/net/openhft/chronicle/hash/impl/PersistedChronicleHashResourceReleaser.java

@@ -0,0 +1,89 @@
+/*
+ *      Copyright (C) 2012, 2016  higherfrequencytrading.com
+ *      Copyright (C) 2016 Roman Leventov
+ *
+ *      This program is free software: you can redistribute it and/or modify
+ *      it under the terms of the GNU Lesser General Public License as published by
+ *      the Free Software Foundation, either version 3 of the License.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU Lesser General Public License for more details.
+ *
+ *      You should have received a copy of the GNU Lesser General Public License
+ *      along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.openhft.chronicle.hash.impl;
+
+import net.openhft.chronicle.core.OS;
+import net.openhft.chronicle.hash.impl.util.CanonicalRandomAccessFiles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+
+public final class PersistedChronicleHashResourceReleaser
+        extends ChronicleHashResourceReleaser {
+    private static final Logger LOG =
+            LoggerFactory.getLogger(PersistedChronicleHashResourceReleaser.class);
+
+    private final File file;
+
+    public PersistedChronicleHashResourceReleaser(File file) {
+        this.file = file;
+    }
+
+    @Override
+    public synchronized void run() {
+        if (memoryResources == null)
+            return; // Already released
+        // All Throwables are catched because this class is used as Runnable for sun.misc.Cleaner,
+        // hence must not fail
+        Throwable thrown = null;
+        try {
+            thrown = doRelease();
+        } finally {
+            memoryResources = null;
+        }
+        if (thrown != null) {
+            try {
+                LOG.error("Error on releasing memory and RAF of a Chronicle Map at file=" + file,
+                        thrown);
+            } catch (Throwable t) {
+                // This may occur if Releaser is run in a shutdown hook, and the log service has
+                // already been shut down. Try to fall back to printStackTrace().
+                thrown.addSuppressed(t);
+                thrown.printStackTrace();
+            }
+        }
+    }
+
+    @Override
+    Throwable doRelease() {
+        Throwable thrown = null;
+        for (MemoryResource mapping : memoryResources) {
+            try {
+                OS.unmap(mapping.address, mapping.size);
+            } catch (Throwable t) {
+                if (thrown == null) {
+                    thrown = t;
+                } else {
+                    thrown.addSuppressed(t);
+                }
+            }
+        }
+        try {
+            CanonicalRandomAccessFiles.release(file);
+        } catch (Throwable t) {
+            if (thrown == null) {
+                thrown = t;
+            } else {
+                thrown.addSuppressed(t);
+            }
+        }
+        return thrown;
+    }
+}

src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java

@@ -22,11 +22,11 @@ import net.openhft.chronicle.bytes.BytesStore;
 import net.openhft.chronicle.bytes.MappedBytesStoreFactory;
 import net.openhft.chronicle.bytes.NativeBytesStore;
 import net.openhft.chronicle.core.Maths;
+import net.openhft.chronicle.core.Memory;
 import net.openhft.chronicle.core.OS;
 import net.openhft.chronicle.hash.*;
 import net.openhft.chronicle.hash.impl.stage.hash.ChainingInterface;
 import net.openhft.chronicle.hash.impl.util.BuildVersion;
-import net.openhft.chronicle.hash.impl.util.CanonicalRandomAccessFiles;
 import net.openhft.chronicle.hash.impl.util.jna.PosixMsync;
 import net.openhft.chronicle.hash.impl.util.jna.WindowsMsync;
 import net.openhft.chronicle.hash.serialization.DataAccess;
@@ -57,10 +57,8 @@ import static java.lang.Math.max;
 import static java.nio.channels.FileChannel.MapMode.READ_WRITE;
 import static net.openhft.chronicle.algo.MemoryUnit.*;
 import static net.openhft.chronicle.algo.bytes.Access.nativeAccess;
-import static net.openhft.chronicle.bytes.NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity;
 import static net.openhft.chronicle.core.OS.pageAlign;
 import static net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable.*;
-import static net.openhft.chronicle.hash.impl.DummyReferenceCounted.DUMMY_REFERENCE_COUNTED;
 
 public abstract class VanillaChronicleHash<K,
         C extends HashEntry<K>, SC extends HashSegmentContext<K, ?>,
@@ -131,7 +129,8 @@ public abstract class VanillaChronicleHash<K,
     // Bytes Store (essentially, the base address) and serialization-dependent offsets
     private transient File file;
     private transient RandomAccessFile raf;
-    private transient Cleaner rafCleaner;
+    private transient ChronicleHashResourceReleaser resourceReleaser;
+    private transient Cleaner cleaner;
     protected transient BytesStore bs;
 
     public static class TierBulkData {
@@ -438,7 +437,14 @@ public abstract class VanillaChronicleHash<K,
         return CACHE_LINES.align(headerSize, BYTES);
     }
 
-    public final void createMappedStoreAndSegments(BytesStore bytesStore) throws IOException {
+    public final void createInMemoryStoreAndSegments(
+            ChronicleHashResourceReleaser resourceReleaser) throws IOException {
+        this.resourceReleaser = resourceReleaser;
+        BytesStore bytesStore = nativeBytesStoreWithFixedCapacity(sizeInBytesWithoutTiers());
+        createStoreAndSegments(bytesStore);
+    }
+
+    private void createStoreAndSegments(BytesStore bytesStore) throws IOException {
         initBytesStoreAndHeadersViews(bytesStore);
         initOffsetsAndBulks();
     }
@@ -479,32 +485,19 @@ public abstract class VanillaChronicleHash<K,
         onHeaderCreated();
     }
 
-    public void registerRafReleaser() {
-        this.rafCleaner = Cleaner.create(this, new RafReleaser(file));
+    public void registerCleaner() {
+        this.cleaner = Cleaner.create(this, resourceReleaser);
     }
 
-    private static class RafReleaser implements Runnable {
-        final File file;
-
-        private RafReleaser(File file) {
-            this.file = file;
-        }
-
-        @Override
-        public void run() {
-            try {
-                CanonicalRandomAccessFiles.release(file);
-            } catch (IOException e) {
-                LOG.error("error on releasing RAF for file {}: {}", file, e);
-            }
-        }
+    public final void createMappedStoreAndSegments(ChronicleHashResourceReleaser resourceReleaser)
+            throws IOException {
+        this.resourceReleaser = resourceReleaser;
+        createStoreAndSegments(map(dataStoreSize(), 0));
     }
 
-    public final void createMappedStoreAndSegments() throws IOException {
-        createMappedStoreAndSegments(map(dataStoreSize(), 0));
-    }
-
-    public final void basicRecover() throws IOException {
+    public final void basicRecover(ChronicleHashResourceReleaser resourceReleaser)
+            throws IOException {
+        this.resourceReleaser = resourceReleaser;
         long segmentHeadersOffset = globalMutableState().getSegmentHeadersOffset();
         if (segmentHeadersOffset <= 0 || segmentHeadersOffset % 4096 != 0 ||
                 segmentHeadersOffset > GIGABYTES.toBytes(1)) {
@@ -681,16 +674,10 @@ public abstract class VanillaChronicleHash<K,
             }
         }
 
-        bs.release();
-        assert bs.refCount() == 0;
-        for (TierBulkData bulkData : tierBulkOffsets) {
-            if (bulkData.bytesStore != bs) {
-                bulkData.bytesStore.release();
-                assert bulkData.bytesStore.refCount() == 0;
-            }
-        }
-        if (rafCleaner != null)
-            rafCleaner.clean();
+        resourceReleaser.releaseManually();
+        // Releases nothing after resourceReleaser.releaseManually(), only removes the cleaner
+        // from the internal linked list of all cleaners.
+        cleaner.clean();
     }
 
     @Override
@@ -777,6 +764,16 @@ public abstract class VanillaChronicleHash<K,
                 globalMutableStateAddress() + GLOBAL_MUTABLE_STATE_LOCK_OFFSET);
     }
 
+    /** For tests */
+    public boolean hasExtraTierBulks() {
+        return globalMutableState.getAllocatedExtraTierBulks() > 0;
+    }
+
+    @Override
+    public long offHeapMemoryUsed() {
+        return resourceReleaser.totalMemory();
+    }
+
     public long allocateTier() {
         globalMutableStateLock();
         try {
@@ -992,8 +989,8 @@ public abstract class VanillaChronicleHash<K,
             raf.setLength(minFileSize);
         }
         long address = OS.map(fileChannel, READ_WRITE, mappingOffsetInFile, mapSize);
-        OS.Unmapper unmapper = new OS.Unmapper(address, mapSize, DUMMY_REFERENCE_COUNTED);
-        return new NativeBytesStore(address, mapSize, unmapper, false);
+        resourceReleaser.addMemoryResource(address, mapSize);
+        return new NativeBytesStore(address, mapSize, null, false);
     }
 
     private long bulkOffset(int bulkIndex) {
@@ -1001,11 +998,17 @@ public abstract class VanillaChronicleHash<K,
     }
 
     private void allocateTierBulks(int upToBulkIndex) {
-        int firstBulkToMapIndex = tierBulkOffsets.size();
-        int bulksToMap = upToBulkIndex + 1 - firstBulkToMapIndex;
-        long mapSize = bulksToMap * tierBulkSizeInBytes;
-        BytesStore extraStore = lazyNativeBytesStoreWithFixedCapacity(mapSize);
-        appendBulkData(firstBulkToMapIndex, upToBulkIndex, extraStore, 0);
+        int firstBulkToAllocateIndex = tierBulkOffsets.size();
+        int bulksToAllocate = upToBulkIndex + 1 - firstBulkToAllocateIndex;
+        long allocationSize = bulksToAllocate * tierBulkSizeInBytes;
+        BytesStore extraStore = nativeBytesStoreWithFixedCapacity(allocationSize);
+        appendBulkData(firstBulkToAllocateIndex, upToBulkIndex, extraStore, 0);
+    }
+
+    private BytesStore nativeBytesStoreWithFixedCapacity(long capacity) {
+        long address = OS.memory().allocate(capacity);
+        resourceReleaser.addMemoryResource(address, capacity);
+        return new NativeBytesStore<>(address, capacity, null, false);
     }
 
     private void appendBulkData(int firstBulkToMapIndex, int upToBulkIndex, BytesStore extraStore,

src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java

@@ -22,15 +22,15 @@ import net.openhft.chronicle.algo.hashing.LongHashFunction;
 import net.openhft.chronicle.bytes.Byteable;
 import net.openhft.chronicle.bytes.Bytes;
 import net.openhft.chronicle.bytes.BytesStore;
+import net.openhft.chronicle.bytes.NativeBytesStore;
 import net.openhft.chronicle.core.Jvm;
 import net.openhft.chronicle.core.OS;
 import net.openhft.chronicle.hash.ChronicleHashBuilder;
 import net.openhft.chronicle.hash.ChronicleHashRecoveryFailedException;
-import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable;
-import net.openhft.chronicle.hash.impl.SizePrefixedBlob;
-import net.openhft.chronicle.hash.impl.VanillaChronicleHash;
+import net.openhft.chronicle.hash.impl.*;
 import net.openhft.chronicle.hash.impl.stage.entry.ChecksumStrategy;
 import net.openhft.chronicle.hash.impl.util.CanonicalRandomAccessFiles;
+import net.openhft.chronicle.hash.impl.util.Throwables;
 import net.openhft.chronicle.hash.impl.util.math.PoissonDistribution;
 import net.openhft.chronicle.hash.serialization.*;
 import net.openhft.chronicle.hash.serialization.impl.SerializationBuilder;
@@ -60,7 +60,6 @@ import java.util.concurrent.TimeUnit;
 import static java.lang.Double.isNaN;
 import static java.lang.Math.round;
 import static java.nio.ByteOrder.LITTLE_ENDIAN;
-import static net.openhft.chronicle.bytes.NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity;
 import static net.openhft.chronicle.core.Maths.*;
 import static net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable.*;
 import static net.openhft.chronicle.hash.impl.SizePrefixedBlob.*;
@@ -215,7 +214,7 @@ public final class ChronicleMapBuilder<K, V> implements
     /**
      * When Chronicle Maps are created using {@link #createPersistedTo(File)} or
      * {@link #recoverPersistedTo(File, boolean)} or {@link #createOrRecoverPersistedTo(File)}
-     * methods, file lock on the Chronicle Map's lock is acquired, that shouldn't be done from
+     * methods, file lock on the Chronicle Map's file is acquired, that shouldn't be done from
      * concurrent threads within the same JVM process. So creation of Chronicle Maps
      * persisted to the same File should be synchronized across JVM's threads. Simple way would be
      * to synchronize on some static (lock) object, but would serialize all Chronicle Maps creations
@@ -1472,46 +1471,57 @@ public final class ChronicleMapBuilder<K, V> implements
             //noinspection ResultOfMethodCallIgnored
             file.createNewFile();
         }
-        VanillaChronicleMap<K, V, ?> result = null;
         RandomAccessFile raf = CanonicalRandomAccessFiles.acquire(file);
+        ChronicleHashResourceReleaser resourceReleaser =
+                new PersistedChronicleHashResourceReleaser(file);
         try {
-            if (raf.length() > 0)
-                return result = openWithExistingFile(file, raf, recover, overrideBuilderConfig);
+            VanillaChronicleMap<K, V, ?> result;
+            if (raf.length() > 0) {
+                result = openWithExistingFile(file, raf, resourceReleaser, recover,
+                        overrideBuilderConfig);
+            } else {
 
-            // Single-element arrays allow to modify variables within lambda
-            @SuppressWarnings("unchecked")
-            VanillaChronicleMap<K, V, ?>[] map = new VanillaChronicleMap[1];
-            ByteBuffer[] headerBuffer = new ByteBuffer[1];
-            boolean[] newFile = new boolean[1];
-            FileChannel fileChannel = raf.getChannel();
+                // Single-element arrays allow to modify variables within lambda
+                @SuppressWarnings("unchecked")
+                VanillaChronicleMap<K, V, ?>[] map = new VanillaChronicleMap[1];
+                ByteBuffer[] headerBuffer = new ByteBuffer[1];
+                boolean[] newFile = new boolean[1];
+                FileChannel fileChannel = raf.getChannel();
 
-            fileLockedIO(file, fileChannel, () -> {
-                if (raf.length() == 0) {
-                    map[0] = newMap();
-                    headerBuffer[0] = writeHeader(fileChannel, map[0]);
-                    newFile[0] = true;
+                fileLockedIO(file, fileChannel, () -> {
+                    if (raf.length() == 0) {
+                        map[0] = newMap();
+                        headerBuffer[0] = writeHeader(fileChannel, map[0]);
+                        newFile[0] = true;
+                    } else {
+                        newFile[0] = false;
+                    }
+                });
+
+                if (newFile[0]) {
+                    int headerSize = headerBuffer[0].remaining();
+                    result = createWithNewFile(map[0], file, raf, resourceReleaser, headerBuffer[0],
+                            headerSize);
                 } else {
-                    newFile[0] = false;
+                    result = openWithExistingFile(file, raf, resourceReleaser, recover,
+                            overrideBuilderConfig);
                 }
-            });
-
-            if (newFile[0]) {
-                int headerSize = headerBuffer[0].remaining();
-                return result = createWithNewFile(map[0], file, raf, headerBuffer[0], headerSize);
-            } else {
-                return result = openWithExistingFile(file, raf, recover, overrideBuilderConfig);
             }
-        } finally {
-            if (result != null) {
-                result.registerRafReleaser();
-                prepareMapPublication(result);
-            } else {
-                CanonicalRandomAccessFiles.release(file);
+            prepareMapPublication(result);
+            return result;
+        } catch (Throwable t) {
+            try {
+                resourceReleaser.releaseManually();
+            } catch (Exception e) {
+                t.addSuppressed(e);
             }
+            throw Throwables.propagateNotWrapping(t, IOException.class);
         }
     }
 
-    private static void prepareMapPublication(VanillaChronicleMap map) {
+    private void prepareMapPublication(VanillaChronicleMap map) throws IOException {
+        establishReplication(map);
+        map.registerCleaner();
         // Ensure safe publication of a ChronicleMap
         OS.memory().storeFence();
         ChronicleMapCloseOnExitHook.add(map);
@@ -1607,16 +1617,17 @@ public final class ChronicleMapBuilder<K, V> implements
 
     private VanillaChronicleMap<K, V, ?> createWithNewFile(
             VanillaChronicleMap<K, V, ?> map, File file, RandomAccessFile raf,
-            ByteBuffer headerBuffer, int headerSize) throws IOException {
+            ChronicleHashResourceReleaser resourceReleaser, ByteBuffer headerBuffer, int headerSize)
+            throws IOException {
         map.initBeforeMapping(file, raf, headerBuffer.limit(), false);
-        map.createMappedStoreAndSegments();
+        map.createMappedStoreAndSegments(resourceReleaser);
         commitChronicleMapReady(map, raf, headerBuffer, headerSize);
-        establishReplication(map);
         return map;
     }
 
     private VanillaChronicleMap<K, V, ?> openWithExistingFile(
-            File file, RandomAccessFile raf, boolean recover, boolean overrideBuilderConfig)
+            File file, RandomAccessFile raf, ChronicleHashResourceReleaser resourceReleaser,
+            boolean recover, boolean overrideBuilderConfig)
             throws IOException {
         try {
             int headerSize = waitUntilReady(raf, recover);
@@ -1645,16 +1656,15 @@ public final class ChronicleMapBuilder<K, V> implements
             }
             map.initTransientsFromBuilder(this);
             if (!recover) {
-                map.createMappedStoreAndSegments();
+                map.createMappedStoreAndSegments(resourceReleaser);
             } else {
                 if (!overrideBuilderConfig)
                     writeNotComplete(fileChannel, headerBuffer, headerSize);
                 // if overrideBuilderConfig = true, readiness bit is already set
                 // in writeHeader() call
-                map.recover();
+                map.recover(resourceReleaser);
                 commitChronicleMapReady(map, raf, headerBuffer, headerSize);
             }
-            establishReplication(map);
             return map;
         } catch (Exception e) {
             if (recover && !(e instanceof IOException) &&
@@ -1668,17 +1678,20 @@ public final class ChronicleMapBuilder<K, V> implements
         replicated = replicationIdentifier != -1;
         persisted = false;
 
+        ChronicleHashResourceReleaser resourceReleaser =
+                new InMemoryChronicleHashResourceReleaser();
         try {
             VanillaChronicleMap<K, V, ?> map = newMap();
-            BytesStore bytesStore =
-                    lazyNativeBytesStoreWithFixedCapacity(map.sizeInBytesWithoutTiers());
-            map.createMappedStoreAndSegments(bytesStore);
-            establishReplication(map);
+            map.createInMemoryStoreAndSegments(resourceReleaser);
             prepareMapPublication(map);
             return map;
-        } catch (IOException e) {
-            // file-less version should never trigger an IOException.
-            throw new AssertionError(e);
+        } catch (Throwable t) {
+            try {
+                resourceReleaser.releaseManually();
+            } catch (Exception e) {
+                t.addSuppressed(e);
+            }
+            throw Throwables.propagate(t);
         }
     }
 

src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java

@@ -150,8 +150,8 @@ public class VanillaChronicleMap<K, V, R>
         initOwnTransients();
     }
 
-    public void recover() throws IOException {
-        basicRecover();
+    public void recover(ChronicleHashResourceReleaser resourceReleaser) throws IOException {
+        basicRecover(resourceReleaser);
         try (IterationContext<K, V, ?> iterationContext = iterationContext()) {
             iterationContext.recoverSegments();
         }

src/main/java/net/openhft/chronicle/set/SetFromMap.java

@@ -116,6 +116,11 @@ class SetFromMap<E> extends AbstractSet<E> implements ChronicleSet<E> {
         return m.longSize();
     }
 
+    @Override
+    public long offHeapMemoryUsed() {
+        return m.offHeapMemoryUsed();
+    }
+
     @Override
     public Class<E> keyClass() {
         return m.keyClass();