diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java index 042220b3c..1b7d08d37 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java @@ -246,7 +246,9 @@ public interface ChronicleMap extends ConcurrentMap, Type valueType(); /** - * @return the amount of free space in the map as a percentage. When the free space gets low ( around 5% ) the map will automatically expand. The + * WARNING: This is an expensive operation which can take milli-seconds. + * + * @return the amount of free space in the map as a percentage. When the free space gets low ( around 5-25% ) the map will automatically expand. The * number of times it can automatically expand is based on the {@code net.openhft.chronicle.map.ChronicleMapBuilder#maxBloatFactor}. If the map * expands you will see an increase in the available free space. NOTE: It is not possible to expand the chronicle map manually. *

@@ -257,6 +259,7 @@ public interface ChronicleMap extends ConcurrentMap, } /** + * WARNING: This is a detailed however expensive operation which can take milliseconds * @return an array of how full each segment is */ default SegmentStats[] segmentStats() { diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index c4f431d91..085213afe 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -325,28 +325,34 @@ public final class ChronicleMapBuilder implements return greatestCommonDivisor(b, a % b); } - private static int maxDefaultChunksPerAverageEntry(final boolean replicated, boolean sparseFile) { - // When replicated, having 8 chunks (=> 8 bits in bitsets) per entry seems more wasteful + private static int maxDefaultChunksPerAverageEntry(final boolean replicated, int avgEntrySize) { + // When replicated, having 16 chunks (=> 8 bits in bitsets) per entry seems more wasteful // because when replicated we have bit sets per each remote node, not only allocation // bit set as when non-replicated - if (sparseFile) - return replicated ? 2 : 4; + if (avgEntrySize >= 64 * 64) + return replicated ? 32 : 64; + if (avgEntrySize >= 32 * 32) + return replicated ? 16 : 32; + if (avgEntrySize >= 16 * 16) + return replicated ? 8 : 16; return replicated ? 4 : 8; } private static int estimateSegmentsForEntries(final long size) { - if (size > 200 << 20) + if (size >= 256 << 20) return 256; - if (size >= 1 << 20) + if (size >= 32 << 20) return 128; - if (size >= 128 << 10) + if (size >= 4 << 20) return 64; - if (size >= 16 << 10) + if (size >= 512 << 10) return 32; - if (size >= 4 << 10) + if (size >= 64 << 10) return 16; - if (size >= 1 << 10) + if (size >= 8 << 10) return 8; + if (size >= 1 << 10) + return 4; return 1; } @@ -788,7 +794,7 @@ public final class ChronicleMapBuilder implements worstAlignment = worstAlignmentAssumingChunkSize( constantSizeBeforeAlignment, chunkSize); if (size + worstAlignment + valueSize >= - maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize) { + maxDefaultChunksPerAverageEntry(replicated, (int) size) * chunkSize) { break determineAlignment; } chunkSize = 4; @@ -866,8 +872,8 @@ public final class ChronicleMapBuilder implements if (constantlySizedEntries()) return toLong(averageEntrySize); final int maxChunkSize = 1 << 30; - for (long chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) { - if (maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize > averageEntrySize) + for (int chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) { + if ((long) maxDefaultChunksPerAverageEntry(replicated, (int) averageEntrySize) * chunkSize > averageEntrySize) return chunkSize; } return maxChunkSize; @@ -1006,7 +1012,7 @@ public final class ChronicleMapBuilder implements entriesPerSegment = this.entriesPerSegment; } else { final int actualSegments = actualSegments(); - final double averageEntriesPerSegment = entries() * (sparseFile ? 2.0 : 1.0) / actualSegments; + final double averageEntriesPerSegment = entries() * 1.0 / actualSegments; if (actualSegments > 1) { entriesPerSegment = PoissonDistribution.inverseCumulativeProbability( averageEntriesPerSegment, nonTieredSegmentsPercentile); @@ -1014,6 +1020,10 @@ public final class ChronicleMapBuilder implements // if there is only 1 segment, there is no source of variance in segments filling entriesPerSegment = roundUp(averageEntriesPerSegment); } + if (sparseFile && maxBloatFactor > 1.0) { + final double averageEntriesPerSegment2 = entries() * maxBloatFactor / actualSegments; + entriesPerSegment = Math.max(roundUp(averageEntriesPerSegment2), entriesPerSegment); + } } final boolean actualChunksDefined = actualChunksPerSegmentTier > 0; if (!actualChunksDefined) { @@ -1983,8 +1993,10 @@ public final class ChronicleMapBuilder implements private void preMapConstruction() { averageKeySize = preMapConstruction(keyBuilder, averageKeySize, averageKey, sampleKey, "Key"); averageValueSize = preMapConstruction(valueBuilder, averageValueSize, averageValue, sampleValue, "Value"); - if (sparseFile) + if (sparseFile) { + averageKeySize *= 2; averageValueSize *= 2; + } stateChecks(); } diff --git a/src/test/java/net/openhft/chronicle/map/LargeMapMain.java b/src/test/java/net/openhft/chronicle/map/LargeMapMain.java index 9468929b0..745f895ab 100644 --- a/src/test/java/net/openhft/chronicle/map/LargeMapMain.java +++ b/src/test/java/net/openhft/chronicle/map/LargeMapMain.java @@ -1,7 +1,11 @@ package net.openhft.chronicle.map; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.Comparator; import java.util.Random; public class LargeMapMain { @@ -10,34 +14,57 @@ public class LargeMapMain { ChronicleMap map2 = createMap(true); StringBuilder key = new StringBuilder(); StringBuilder value = new StringBuilder(); - Random rand = new Random(); - for (int j = 0; j < 10; j++) { - for (CharSequence cs : map.keySet()) { - generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000 - map.put(cs, value); - map2.put(cs, value); - } - - for (int i = 0; i < 1_000_000; i++) { - generate(rand, key, rand.nextInt(128)); - generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000 + Random rand = new Random(1); + for (int j = 0; j < 40; j++) { + for (int i = 0; i < 10_000; i++) { + generate(rand, key, 1 + rand.nextInt(128)); + generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K map.put(key, value); map2.put(key, value); } + + int repeat = Math.max(1, 9 - j); + for (int r = 0; r < repeat; r++) { + for (CharSequence cs : map2.keySet()) { + generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K + map.put(cs, value); + map2.put(cs, value); + } + } + System.out.println("any key map.size()= " + map.size() + ", sparse.size()= " + map2.size()); - System.in.read(); + ChronicleMap.SegmentStats[] ss = map.segmentStats(); + Arrays.sort(ss, Comparator.comparingLong(ChronicleMap.SegmentStats::usedBytes)); + System.out.println("segments " + ss.length + ", from " + ss[0] + " to " + ss[ss.length - 1]); + + ChronicleMap.SegmentStats[] ss2 = map2.segmentStats(); + Arrays.sort(ss2, Comparator.comparingLong(ChronicleMap.SegmentStats::usedBytes)); + System.out.println("segments " + ss2.length + ", from " + ss2[0] + " to " + ss2[ss2.length - 1]); + + ProcessBuilder pb = new ProcessBuilder("bash", "-c", "ls -l *.map ; du *.map"); + pb.redirectErrorStream(true); + Process process = pb.start(); + try (BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + for (String line; (line = br.readLine()) != null; ) { + System.out.println(line); + } + } + System.out.println("..."); } } private static ChronicleMap createMap(boolean sparseFile) throws IOException { + File file = new File((sparseFile ? "sparse" : "default") + "-deleteme.map"); + file.delete(); ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, CharSequence.class) - .entries(4_000_000) + .entries(100_000) + .actualSegments(64) .averageKeySize(64) - .averageValueSize(1000) - .maxBloatFactor(3) + .averageValueSize(4096) // 4 KB + .maxBloatFactor(4) .sparseFile(sparseFile); - ChronicleMap map = builder.createPersistedTo(new File((sparseFile ? "sparse" : "default") + "-deleteme.map")); + ChronicleMap map = builder.createPersistedTo(file); return map; }