Add direct support for sparse files under Linux #313

src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilderPrivateAPI.java

@@ -100,4 +100,9 @@ public interface ChronicleHashBuilderPrivateAPI<K, RO> {
      * Whether to skip closing a {@link ChronicleHash} using the default shutdown hook.
      */
     boolean skipCloseOnExitHook();
+
+    /**
+     * @return should the file be left as sparse if that is supported.
+     */
+    boolean sparseFile();
 }

src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java

@@ -214,10 +214,12 @@ public final class ChronicleMapBuilder<K, V> implements
     private boolean replicated;
     private boolean persisted;
     private String replicatedMapClassName = ReplicatedChronicleMap.class.getName();
+    private boolean sparseFile = Jvm.getBoolean("chronicle.map.sparseFile");
 
     ChronicleMapBuilder(@NotNull final Class<K> keyClass, @NotNull final Class<V> valueClass) {
         keyBuilder = new SerializationBuilder<>(keyClass);
         valueBuilder = new SerializationBuilder<>(valueClass);
+        System.out.println("sparseFile = " + sparseFile);
     }
 
     /**
@@ -324,10 +326,12 @@ public final class ChronicleMapBuilder<K, V> implements
         return greatestCommonDivisor(b, a % b);
     }
 
-    private static int maxDefaultChunksPerAverageEntry(final boolean replicated) {
+    private static int maxDefaultChunksPerAverageEntry(final boolean replicated, boolean sparseFile) {
         // When replicated, having 8 chunks (=> 8 bits in bitsets) per entry seems more wasteful
         // because when replicated we have bit sets per each remote node, not only allocation
         // bit set as when non-replicated
+        if (sparseFile)
+            return replicated ? 2 : 4;
         return replicated ? 4 : 8;
     }
 
@@ -785,7 +789,7 @@ public final class ChronicleMapBuilder<K, V> implements
                     worstAlignment = worstAlignmentAssumingChunkSize(
                             constantSizeBeforeAlignment, chunkSize);
                     if (size + worstAlignment + valueSize >=
-                            maxDefaultChunksPerAverageEntry(replicated) * chunkSize) {
+                            maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize) {
                         break determineAlignment;
                     }
                     chunkSize = 4;
@@ -864,7 +868,7 @@ public final class ChronicleMapBuilder<K, V> implements
             return toLong(averageEntrySize);
         final int maxChunkSize = 1 << 30;
         for (long chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) {
-            if (maxDefaultChunksPerAverageEntry(replicated) * chunkSize > averageEntrySize)
+            if (maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize > averageEntrySize)
                 return chunkSize;
         }
         return maxChunkSize;
@@ -1003,7 +1007,7 @@ public final class ChronicleMapBuilder<K, V> implements
             entriesPerSegment = this.entriesPerSegment;
         } else {
             final int actualSegments = actualSegments();
-            final double averageEntriesPerSegment = entries() * 1.0 / actualSegments;
+            final double averageEntriesPerSegment = entries() * (sparseFile ? 2.0 : 1.0) / actualSegments;
             if (actualSegments > 1) {
                 entriesPerSegment = PoissonDistribution.inverseCumulativeProbability(
                         averageEntriesPerSegment, nonTieredSegmentsPercentile);
@@ -1345,6 +1349,7 @@ public final class ChronicleMapBuilder<K, V> implements
                 ", entries=" + entries() +
                 ", putReturnsNull=" + putReturnsNull() +
                 ", removeReturnsNull=" + removeReturnsNull() +
+                ", sparseFile=" + sparseFile() +
                 ", keyBuilder=" + keyBuilder +
                 ", valueBuilder=" + valueBuilder +
                 '}';
@@ -1556,6 +1561,21 @@ public final class ChronicleMapBuilder<K, V> implements
         return this;
     }
 
+    /**
+     * Enable the use of sparse files if supported. This will double the extents however could reduce disk space used.
+     *
+     * @param sparseFile true, if allowed
+     * @return this builder
+     */
+    public ChronicleMapBuilder<K, V> sparseFile(final boolean sparseFile) {
+        this.sparseFile = sparseFile;
+        return this;
+    }
+
+    public boolean sparseFile() {
+        return sparseFile;
+    }
+
     @Override
     public ChronicleMap<K, V> createPersistedTo(@NotNull final File file) throws IOException {
         // clone() to make this builder instance thread-safe, because createWithFile() method
@@ -1964,6 +1984,8 @@ public final class ChronicleMapBuilder<K, V> implements
     private void preMapConstruction() {
         averageKeySize = preMapConstruction(keyBuilder, averageKeySize, averageKey, sampleKey, "Key");
         averageValueSize = preMapConstruction(valueBuilder, averageValueSize, averageValue, sampleValue, "Value");
+        if (sparseFile)
+            averageValueSize *= 2;
         stateChecks();
     }
 

src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderPrivateAPI.java

@@ -125,4 +125,9 @@ class ChronicleMapBuilderPrivateAPI<K, V>
     public boolean skipCloseOnExitHook() {
         return b.skipCloseOnExitHook;
     }
+
+    @Override
+    public boolean sparseFile() {
+        return b.sparseFile();
+    }
 }

src/main/java/net/openhft/chronicle/map/LargeMapMain.java

@@ -0,0 +1,49 @@
+package net.openhft.chronicle.map;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+public class LargeMapMain {
+    public static void main(String[] args) throws IOException {
+        ChronicleMap<CharSequence, CharSequence> map = createMap(false);
+        ChronicleMap<CharSequence, CharSequence> map2 = createMap(true);
+        StringBuilder key = new StringBuilder();
+        StringBuilder value = new StringBuilder();
+        Random rand = new Random();
+        for (int j = 0; j < 10; j++) {
+            for (CharSequence cs : map.keySet()) {
+                generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000
+                map.put(cs, value);
+                map2.put(cs, value);
+            }
+
+            for (int i = 0; i < 1_000_000; i++) {
+                generate(rand, key, rand.nextInt(128));
+                generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000
+                map.put(key, value);
+                map2.put(key, value);
+            }
+            System.out.println("any key map.size()= " + map.size() + ", sparse.size()= " + map2.size());
+            System.in.read();
+        }
+    }
+
+    private static ChronicleMap<CharSequence, CharSequence> createMap(boolean sparseFile) throws IOException {
+        ChronicleMapBuilder<CharSequence, CharSequence> builder = ChronicleMapBuilder
+                .of(CharSequence.class, CharSequence.class)
+                .entries(4_000_000)
+                .averageKeySize(64)
+                .averageValueSize(1000)
+                .maxBloatFactor(3)
+                .sparseFile(sparseFile);
+        ChronicleMap<CharSequence, CharSequence> map = builder.createPersistedTo(new File((sparseFile ? "sparse" : "default") + "-deleteme.map"));
+        return map;
+    }
+
+    private static void generate(Random rand, StringBuilder sb, int length) {
+        sb.setLength(0);
+        while (sb.length() <= length)
+            sb.append(rand.nextInt());
+    }
+}

src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java

@@ -147,7 +147,6 @@ public class ChronicleMapTest {
         }
 
         return map;
-
     }
 
     @Test

system.properties

@@ -2,4 +2,6 @@
 jvm.resource.tracing=false
 check.thread.safety=false
 # for profiling
-jvm.safepoint.enabled=false
+jvm.safepoint.enabled=false
+# for testing sparseFiles
+chronicle.map.sparseFile=true