Add direct support for sparse files under Linux (with some size tuning) #313

src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java

@@ -54,6 +54,8 @@ import java.lang.reflect.Type;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
+import java.nio.file.FileStore;
+import java.nio.file.FileSystem;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -92,6 +94,8 @@ public abstract class VanillaChronicleHash<K,
             AcquisitionStrategies.spinLoopOrFail(2, TimeUnit.SECONDS);
     private static final long GLOBAL_MUTABLE_STATE_LOCK_OFFSET = 0L;
     private static final long GLOBAL_MUTABLE_STATE_VALUE_OFFSET = 8L;
+    private final Runnable preShutdownAction;
+    private final boolean skipCloseOnExitHook;
     /////////////////////////////////////////////////
     // If the hash was created in the first place, or read from disk
     public transient boolean createdOrInMemory;
@@ -129,8 +133,6 @@ public abstract class VanillaChronicleHash<K,
     public transient CompactOffHeapLinearHashTable hashLookup;
     public transient Identity identity;
     protected int log2TiersInBulk;
-    private final Runnable preShutdownAction;
-    private final boolean skipCloseOnExitHook;
     /////////////////////////////////////////////////
     // Bytes Store (essentially, the base address) and serialization-dependent offsets
     protected transient BytesStore bs;
@@ -1053,7 +1055,7 @@ public abstract class VanillaChronicleHash<K,
             // "XFS: ... possible memory allocation deadlock size ... in kmem_alloc (mode:0x250)".
             // We can fix this by trying calling posix_fallocate to preallocate the space.
             if (OS.isLinux() && !sparseFile) {
-                PosixFallocate.fallocate(raf.getFD(), mappingOffsetInFile, minFileSize);
+                fallocate(mappingOffsetInFile, minFileSize - mappingOffsetInFile);
             }
         }
         final long address = OS.map(fileChannel, READ_WRITE, mappingOffsetInFile, mapSize);
@@ -1061,6 +1063,21 @@ public abstract class VanillaChronicleHash<K,
         return new NativeBytesStore(address, mapSize, null, false);
     }
 
+    private void fallocate(long mappingOffsetInFile, long length) throws IOException {
+        FileSystem fileSystem = file.toPath().getFileSystem();
+
+        long maxSize = 0;
+        for (FileStore fileStore : fileSystem.getFileStores()) {
+            long unallocatedSpace = fileStore.getUnallocatedSpace();
+            if (unallocatedSpace > length * 11 / 10) {
+                PosixFallocate.fallocate(raf.getFD(), mappingOffsetInFile, length);
+                return;
+            }
+            maxSize = Math.max(maxSize, unallocatedSpace);
+        }
+        throw new IOException("Not enough space to fallocate " + (length >> 20) + " MiB to " + fileSystem.getRootDirectories().iterator().next() + " unallocated was " + (maxSize >> 20) + " MiB");
+    }
+
     private long bulkOffset(final int bulkIndex) {
         return sizeInBytesWithoutTiers() + bulkIndex * tierBulkSizeInBytes;
     }

src/main/java/net/openhft/chronicle/hash/impl/util/jna/PosixFallocate.java

@@ -3,10 +3,9 @@ package net.openhft.chronicle.hash.impl.util.jna;
 import com.sun.jna.Native;
 import com.sun.jna.Platform;
 import net.openhft.chronicle.core.Jvm;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.FileDescriptor;
+import java.io.IOException;
 import java.lang.reflect.Field;
 
 public final class PosixFallocate {
@@ -18,26 +17,25 @@ public final class PosixFallocate {
     private PosixFallocate() {
     }
 
-    public static void fallocate(FileDescriptor descriptor, long offset, long length) {
+    public static void fallocate(FileDescriptor descriptor, long offset, long length) throws IOException {
         int fd = getNativeFileDescriptor(descriptor);
         if (fd != -1) {
             int ret = posix_fallocate(getNativeFileDescriptor(descriptor), offset, length);
             if (ret != 0) {
-                Jvm.warn().on(PosixFallocate.class, "posix_fallocate() returned " + ret);
+                throw new IOException("posix_fallocate() returned " + ret);
             }
         }
     }
 
     private static native int posix_fallocate(int fd, long offset, long length);
 
-    private static int getNativeFileDescriptor(FileDescriptor descriptor) {
+    private static int getNativeFileDescriptor(FileDescriptor descriptor) throws IOException {
         try {
             final Field field = descriptor.getClass().getDeclaredField("fd");
             Jvm.setAccessible(field);
             return (int) field.get(descriptor);
         } catch (final Exception e) {
-            Jvm.warn().on(PosixFallocate.class, "unsupported FileDescriptor implementation: e=" + e.getLocalizedMessage());
-            return -1;
+            throw new IOException("unsupported FileDescriptor implementation", e);
         }
     }
 }

src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java

@@ -329,11 +329,11 @@ public final class ChronicleMapBuilder<K, V> implements
         // When replicated, having 16 chunks (=> 8 bits in bitsets) per entry seems more wasteful
         // because when replicated we have bit sets per each remote node, not only allocation
         // bit set as when non-replicated
-        if (avgEntrySize >= 64 * 64)
+        if (avgEntrySize >= 2 * 64 * 64)
             return replicated ? 32 : 64;
-        if (avgEntrySize >= 32 * 32)
+        if (avgEntrySize >= 2 * 32 * 32)
             return replicated ? 16 : 32;
-        if (avgEntrySize >= 16 * 16)
+        if (avgEntrySize >= 2 * 16 * 16)
             return replicated ? 8 : 16;
         return replicated ? 4 : 8;
     }

src/test/java/net/openhft/chronicle/map/HugeSparseMapTest.java

@@ -0,0 +1,50 @@
+package net.openhft.chronicle.map;
+
+import net.openhft.chronicle.core.OS;
+import net.openhft.chronicle.core.io.IOTools;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeTrue;
+
+public class HugeSparseMapTest {
+
+    private static ChronicleMap<CharSequence, CharSequence> createMap(boolean sparseFile) throws IOException {
+        File file = IOTools.createTempFile("huge-map");
+        try {
+            file.deleteOnExit();
+            ChronicleMapBuilder<CharSequence, CharSequence> builder = ChronicleMapBuilder
+                    .of(CharSequence.class, CharSequence.class)
+                    .averageKeySize(64)
+                    // .entries(10_000_000_000L).averageValueSize(640) // 16 TB
+                    .entries(3_000_000_000L).averageValueSize(200) // 2 TB
+                    .sparseFile(sparseFile); // ~16 TB.
+            ChronicleMap<CharSequence, CharSequence> map = builder.createPersistedTo(file);
+            return map;
+        } catch (Throwable t) {
+            file.delete();
+            throw t;
+        }
+    }
+
+    @Test
+    public void hugeSparseMap() throws IOException {
+        assumeTrue(OS.isLinux());
+        try (ChronicleMap<CharSequence, CharSequence> map = createMap(true)) {
+            map.put("hi", "there");
+            assertEquals("there", map.get("hi").toString());
+        }
+    }
+
+    @Test(expected = IOException.class)
+    public void hugeAllocatedMap() throws IOException {
+        assumeTrue(OS.isLinux());
+        try (ChronicleMap<CharSequence, CharSequence> map = createMap(false)) {
+            map.put("hi", "there");
+            assertEquals("there", map.get("hi").toString());
+        }
+    }
+}

src/test/java/net/openhft/chronicle/map/LargeMapMain.java

@@ -9,6 +9,9 @@ import java.util.Comparator;
 import java.util.Random;
 
 public class LargeMapMain {
+    //    static int avgSize = 4096, maxSize = 40_000, entries= 500_000; // average ~= 4K
+    static int avgSize = 1024, maxSize = 8_000, entries = 1_000_000; // average ~= 1K
+
     public static void main(String[] args) throws IOException {
         ChronicleMap<CharSequence, CharSequence> map = createMap(false);
         ChronicleMap<CharSequence, CharSequence> map2 = createMap(true);
@@ -16,9 +19,9 @@ public class LargeMapMain {
         StringBuilder value = new StringBuilder();
         Random rand = new Random(1);
         for (int j = 0; j < 40; j++) {
-            for (int i = 0; i < 10_000; i++) {
+            for (int i = 0; i < entries / 10; i++) {
                 generate(rand, key, 1 + rand.nextInt(128));
-                generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K
+                generate(rand, value, (int) Math.pow(maxSize, rand.nextFloat()));
                 map.put(key, value);
                 map2.put(key, value);
             }
@@ -26,7 +29,7 @@ public class LargeMapMain {
             int repeat = Math.max(1, 9 - j);
             for (int r = 0; r < repeat; r++) {
                 for (CharSequence cs : map2.keySet()) {
-                    generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K
+                    generate(rand, value, (int) Math.pow(maxSize, rand.nextFloat()));
                     map.put(cs, value);
                     map2.put(cs, value);
                 }
@@ -58,10 +61,10 @@ public class LargeMapMain {
         file.delete();
         ChronicleMapBuilder<CharSequence, CharSequence> builder = ChronicleMapBuilder
                 .of(CharSequence.class, CharSequence.class)
-                .entries(100_000)
+                .entries(entries)
                 .actualSegments(64)
                 .averageKeySize(64)
-                .averageValueSize(4096) // 4 KB
+                .averageValueSize(avgSize)
                 .maxBloatFactor(4)
                 .sparseFile(sparseFile);
         ChronicleMap<CharSequence, CharSequence> map = builder.createPersistedTo(file);

src/test/java/net/openhft/chronicle/map/VeryLargeMapMain.java

@@ -0,0 +1,70 @@
+package net.openhft.chronicle.map;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.stream.IntStream;
+
+public class VeryLargeMapMain {
+    static int avgSize = 16000, maxSize = 190_000, entries = 50_000_000; // average ~= 16K
+
+    public static void main(String[] args) throws IOException {
+        ChronicleMap<CharSequence, CharSequence> map = createMap(true);
+        while (map.size() < entries) {
+            long start = System.nanoTime();
+            IntStream.range(0, 1000)
+                    .parallel()
+                    .forEach(t -> {
+                        Random rand = new Random();
+                        StringBuilder key = new StringBuilder();
+                        StringBuilder value = new StringBuilder();
+                        for (int k = 0; k < 1000; k++) {
+                            generate(rand, key, 1 + rand.nextInt(128));
+                            generate(rand, value, (int) Math.pow(maxSize, rand.nextFloat()));
+                            map.put(key, value);
+                        }
+                    });
+            long time = System.nanoTime() - start;
+            System.out.println("Entries per second: " + (long) (1e6 * 1e9 / time));
+
+            System.out.println("any key sparse.size()= " + map.size());
+            ChronicleMap.SegmentStats[] ss = map.segmentStats();
+            Arrays.sort(ss, Comparator.comparingLong(ChronicleMap.SegmentStats::usedBytes));
+            System.out.println("segments " + ss.length + ", from " + ss[0] + " to " + ss[ss.length - 1]);
+
+            ProcessBuilder pb = new ProcessBuilder("bash", "-c", "ls -l *.map ; du *.map");
+            pb.redirectErrorStream(true);
+            Process process = pb.start();
+            try (BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
+                for (String line; (line = br.readLine()) != null; ) {
+                    System.out.println(line);
+                }
+            }
+            System.out.println("...");
+        }
+    }
+
+    private static ChronicleMap<CharSequence, CharSequence> createMap(boolean sparseFile) throws IOException {
+        File file = new File("verylarge-deleteme.map");
+        file.delete();
+        ChronicleMapBuilder<CharSequence, CharSequence> builder = ChronicleMapBuilder
+                .of(CharSequence.class, CharSequence.class)
+                .entries(entries)
+                .averageKeySize(64)
+                .averageValueSize(avgSize)
+//                .maxBloatFactor(2)
+                .sparseFile(sparseFile);
+        ChronicleMap<CharSequence, CharSequence> map = builder.createPersistedTo(file);
+        return map;
+    }
+
+    private static void generate(Random rand, StringBuilder sb, int length) {
+        sb.setLength(0);
+        while (sb.length() <= length)
+            sb.append(rand.nextInt());
+    }
+}