diff --git a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilderPrivateAPI.java b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilderPrivateAPI.java index 0450f7eb7..efcc96a07 100644 --- a/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilderPrivateAPI.java +++ b/src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilderPrivateAPI.java @@ -100,4 +100,9 @@ public interface ChronicleHashBuilderPrivateAPI { * Whether to skip closing a {@link ChronicleHash} using the default shutdown hook. */ boolean skipCloseOnExitHook(); + + /** + * @return should the file be left as sparse if that is supported. + */ + boolean sparseFile(); } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 02cc5381e..5f6802512 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -214,10 +214,12 @@ public final class ChronicleMapBuilder implements private boolean replicated; private boolean persisted; private String replicatedMapClassName = ReplicatedChronicleMap.class.getName(); + private boolean sparseFile = Jvm.getBoolean("chronicle.map.sparseFile"); ChronicleMapBuilder(@NotNull final Class keyClass, @NotNull final Class valueClass) { keyBuilder = new SerializationBuilder<>(keyClass); valueBuilder = new SerializationBuilder<>(valueClass); + System.out.println("sparseFile = " + sparseFile); } /** @@ -324,10 +326,12 @@ public final class ChronicleMapBuilder implements return greatestCommonDivisor(b, a % b); } - private static int maxDefaultChunksPerAverageEntry(final boolean replicated) { + private static int maxDefaultChunksPerAverageEntry(final boolean replicated, boolean sparseFile) { // When replicated, having 8 chunks (=> 8 bits in bitsets) per entry seems more wasteful // because when replicated we have bit sets per each remote node, not only allocation // bit set as when non-replicated + if (sparseFile) + return replicated ? 2 : 4; return replicated ? 4 : 8; } @@ -785,7 +789,7 @@ public final class ChronicleMapBuilder implements worstAlignment = worstAlignmentAssumingChunkSize( constantSizeBeforeAlignment, chunkSize); if (size + worstAlignment + valueSize >= - maxDefaultChunksPerAverageEntry(replicated) * chunkSize) { + maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize) { break determineAlignment; } chunkSize = 4; @@ -864,7 +868,7 @@ public final class ChronicleMapBuilder implements return toLong(averageEntrySize); final int maxChunkSize = 1 << 30; for (long chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) { - if (maxDefaultChunksPerAverageEntry(replicated) * chunkSize > averageEntrySize) + if (maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize > averageEntrySize) return chunkSize; } return maxChunkSize; @@ -1003,7 +1007,7 @@ public final class ChronicleMapBuilder implements entriesPerSegment = this.entriesPerSegment; } else { final int actualSegments = actualSegments(); - final double averageEntriesPerSegment = entries() * 1.0 / actualSegments; + final double averageEntriesPerSegment = entries() * (sparseFile ? 2.0 : 1.0) / actualSegments; if (actualSegments > 1) { entriesPerSegment = PoissonDistribution.inverseCumulativeProbability( averageEntriesPerSegment, nonTieredSegmentsPercentile); @@ -1345,6 +1349,7 @@ public final class ChronicleMapBuilder implements ", entries=" + entries() + ", putReturnsNull=" + putReturnsNull() + ", removeReturnsNull=" + removeReturnsNull() + + ", sparseFile=" + sparseFile() + ", keyBuilder=" + keyBuilder + ", valueBuilder=" + valueBuilder + '}'; @@ -1556,6 +1561,21 @@ public final class ChronicleMapBuilder implements return this; } + /** + * Enable the use of sparse files if supported. This will double the extents however could reduce disk space used. + * + * @param sparseFile true, if allowed + * @return this builder + */ + public ChronicleMapBuilder sparseFile(final boolean sparseFile) { + this.sparseFile = sparseFile; + return this; + } + + public boolean sparseFile() { + return sparseFile; + } + @Override public ChronicleMap createPersistedTo(@NotNull final File file) throws IOException { // clone() to make this builder instance thread-safe, because createWithFile() method @@ -1964,6 +1984,8 @@ public final class ChronicleMapBuilder implements private void preMapConstruction() { averageKeySize = preMapConstruction(keyBuilder, averageKeySize, averageKey, sampleKey, "Key"); averageValueSize = preMapConstruction(valueBuilder, averageValueSize, averageValue, sampleValue, "Value"); + if (sparseFile) + averageValueSize *= 2; stateChecks(); } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderPrivateAPI.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderPrivateAPI.java index 62944a4f5..1fb63c217 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderPrivateAPI.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderPrivateAPI.java @@ -125,4 +125,9 @@ class ChronicleMapBuilderPrivateAPI public boolean skipCloseOnExitHook() { return b.skipCloseOnExitHook; } + + @Override + public boolean sparseFile() { + return b.sparseFile(); + } } diff --git a/src/main/java/net/openhft/chronicle/map/LargeMapMain.java b/src/main/java/net/openhft/chronicle/map/LargeMapMain.java new file mode 100644 index 000000000..9468929b0 --- /dev/null +++ b/src/main/java/net/openhft/chronicle/map/LargeMapMain.java @@ -0,0 +1,49 @@ +package net.openhft.chronicle.map; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + +public class LargeMapMain { + public static void main(String[] args) throws IOException { + ChronicleMap map = createMap(false); + ChronicleMap map2 = createMap(true); + StringBuilder key = new StringBuilder(); + StringBuilder value = new StringBuilder(); + Random rand = new Random(); + for (int j = 0; j < 10; j++) { + for (CharSequence cs : map.keySet()) { + generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000 + map.put(cs, value); + map2.put(cs, value); + } + + for (int i = 0; i < 1_000_000; i++) { + generate(rand, key, rand.nextInt(128)); + generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000 + map.put(key, value); + map2.put(key, value); + } + System.out.println("any key map.size()= " + map.size() + ", sparse.size()= " + map2.size()); + System.in.read(); + } + } + + private static ChronicleMap createMap(boolean sparseFile) throws IOException { + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(CharSequence.class, CharSequence.class) + .entries(4_000_000) + .averageKeySize(64) + .averageValueSize(1000) + .maxBloatFactor(3) + .sparseFile(sparseFile); + ChronicleMap map = builder.createPersistedTo(new File((sparseFile ? "sparse" : "default") + "-deleteme.map")); + return map; + } + + private static void generate(Random rand, StringBuilder sb, int length) { + sb.setLength(0); + while (sb.length() <= length) + sb.append(rand.nextInt()); + } +} diff --git a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java index 35fabf610..e9c6e349a 100644 --- a/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/ChronicleMapTest.java @@ -147,7 +147,6 @@ public class ChronicleMapTest { } return map; - } @Test diff --git a/system.properties b/system.properties index 9a872a39d..56ed82500 100644 --- a/system.properties +++ b/system.properties @@ -2,4 +2,6 @@ jvm.resource.tracing=false check.thread.safety=false # for profiling -jvm.safepoint.enabled=false \ No newline at end of file +jvm.safepoint.enabled=false +# for testing sparseFiles +chronicle.map.sparseFile=true \ No newline at end of file