Alawn/Chronicle-Map
1
0
Fork 0
mirror of https://github.com/OpenHFT/Chronicle-Map.git synced 2021-09-13 01:26:12 +00:00
Code Releases Activity

Add direct support for sparse files under Linux (with some size tuning) #313

Browse Source
This commit is contained in:
Peter Lawrey
2021-08-09 16:41:34 +01:00
parent 520464ac06
commit d1cd9b0075
3 changed files with 74 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/net/openhft/chronicle/map/ChronicleMap.java
+4 -1
View File
@@ -246,7 +246,9 @@ public interface ChronicleMap<K, V> extends ConcurrentMap<K, V>,
Type valueType();
/**
* @return the amount of free space in the map as a percentage. When the free space gets low ( around 5% ) the map will automatically expand. The
* WARNING: This is an expensive operation which can take milli-seconds.
*
* @return the amount of free space in the map as a percentage. When the free space gets low ( around 5-25% ) the map will automatically expand. The
* number of times it can automatically expand is based on the {@code net.openhft.chronicle.map.ChronicleMapBuilder#maxBloatFactor}. If the map
* expands you will see an increase in the available free space. NOTE: It is not possible to expand the chronicle map manually.
* <p>
@@ -257,6 +259,7 @@ public interface ChronicleMap<K, V> extends ConcurrentMap<K, V>,
}
/**
* WARNING: This is a detailed however expensive operation which can take milliseconds
* @return an array of how full each segment is
*/
default SegmentStats[] segmentStats() {
src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java
+27 -15
View File
@@ -325,28 +325,34 @@ public final class ChronicleMapBuilder<K, V> implements
return greatestCommonDivisor(b, a % b);
}
private static int maxDefaultChunksPerAverageEntry(final boolean replicated, boolean sparseFile) {
// When replicated, having 8 chunks (=> 8 bits in bitsets) per entry seems more wasteful
private static int maxDefaultChunksPerAverageEntry(final boolean replicated, int avgEntrySize) {
// When replicated, having 16 chunks (=> 8 bits in bitsets) per entry seems more wasteful
// because when replicated we have bit sets per each remote node, not only allocation
// bit set as when non-replicated
if (sparseFile)
return replicated ? 2 : 4;
if (avgEntrySize >= 64 * 64)
return replicated ? 32 : 64;
if (avgEntrySize >= 32 * 32)
return replicated ? 16 : 32;
if (avgEntrySize >= 16 * 16)
return replicated ? 8 : 16;
return replicated ? 4 : 8;
}
private static int estimateSegmentsForEntries(final long size) {
if (size > 200 << 20)
if (size >= 256 << 20)
return 256;
if (size >= 1 << 20)
if (size >= 32 << 20)
return 128;
if (size >= 128 << 10)
if (size >= 4 << 20)
return 64;
if (size >= 16 << 10)
if (size >= 512 << 10)
return 32;
if (size >= 4 << 10)
if (size >= 64 << 10)
return 16;
if (size >= 1 << 10)
if (size >= 8 << 10)
return 8;
if (size >= 1 << 10)
return 4;
return 1;
}
@@ -788,7 +794,7 @@ public final class ChronicleMapBuilder<K, V> implements
worstAlignment = worstAlignmentAssumingChunkSize(
constantSizeBeforeAlignment, chunkSize);
if (size + worstAlignment + valueSize >=
maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize) {
maxDefaultChunksPerAverageEntry(replicated, (int) size) * chunkSize) {
break determineAlignment;
}
chunkSize = 4;
@@ -866,8 +872,8 @@ public final class ChronicleMapBuilder<K, V> implements
if (constantlySizedEntries())
return toLong(averageEntrySize);
final int maxChunkSize = 1 << 30;
for (long chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) {
if (maxDefaultChunksPerAverageEntry(replicated, sparseFile) * chunkSize > averageEntrySize)
for (int chunkSize = 4; chunkSize <= maxChunkSize; chunkSize *= 2L) {
if ((long) maxDefaultChunksPerAverageEntry(replicated, (int) averageEntrySize) * chunkSize > averageEntrySize)
return chunkSize;
}
return maxChunkSize;
@@ -1006,7 +1012,7 @@ public final class ChronicleMapBuilder<K, V> implements
entriesPerSegment = this.entriesPerSegment;
} else {
final int actualSegments = actualSegments();
final double averageEntriesPerSegment = entries() * (sparseFile ? 2.0 : 1.0) / actualSegments;
final double averageEntriesPerSegment = entries() * 1.0 / actualSegments;
if (actualSegments > 1) {
entriesPerSegment = PoissonDistribution.inverseCumulativeProbability(
averageEntriesPerSegment, nonTieredSegmentsPercentile);
@@ -1014,6 +1020,10 @@ public final class ChronicleMapBuilder<K, V> implements
// if there is only 1 segment, there is no source of variance in segments filling
entriesPerSegment = roundUp(averageEntriesPerSegment);
}
if (sparseFile && maxBloatFactor > 1.0) {
final double averageEntriesPerSegment2 = entries() * maxBloatFactor / actualSegments;
entriesPerSegment = Math.max(roundUp(averageEntriesPerSegment2), entriesPerSegment);
}
}
final boolean actualChunksDefined = actualChunksPerSegmentTier > 0;
if (!actualChunksDefined) {
@@ -1983,8 +1993,10 @@ public final class ChronicleMapBuilder<K, V> implements
private void preMapConstruction() {
averageKeySize = preMapConstruction(keyBuilder, averageKeySize, averageKey, sampleKey, "Key");
averageValueSize = preMapConstruction(valueBuilder, averageValueSize, averageValue, sampleValue, "Value");
if (sparseFile)
if (sparseFile) {
averageKeySize *= 2;
averageValueSize *= 2;
}
stateChecks();
}
src/test/java/net/openhft/chronicle/map/LargeMapMain.java
+43 -16
View File
@@ -1,7 +1,11 @@
package net.openhft.chronicle.map;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Random;
public class LargeMapMain {
@@ -10,34 +14,57 @@ public class LargeMapMain {
ChronicleMap<CharSequence, CharSequence> map2 = createMap(true);
StringBuilder key = new StringBuilder();
StringBuilder value = new StringBuilder();
Random rand = new Random();
for (int j = 0; j < 10; j++) {
for (CharSequence cs : map.keySet()) {
generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000
map.put(cs, value);
map2.put(cs, value);
}
for (int i = 0; i < 1_000_000; i++) {
generate(rand, key, rand.nextInt(128));
generate(rand, value, (int) Math.pow(9200, rand.nextFloat())); // average ~= 1000
Random rand = new Random(1);
for (int j = 0; j < 40; j++) {
for (int i = 0; i < 10_000; i++) {
generate(rand, key, 1 + rand.nextInt(128));
generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K
map.put(key, value);
map2.put(key, value);
}
int repeat = Math.max(1, 9 - j);
for (int r = 0; r < repeat; r++) {
for (CharSequence cs : map2.keySet()) {
generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K
map.put(cs, value);
map2.put(cs, value);
}
}
System.out.println("any key map.size()= " + map.size() + ", sparse.size()= " + map2.size());
System.in.read();
ChronicleMap.SegmentStats[] ss = map.segmentStats();
Arrays.sort(ss, Comparator.comparingLong(ChronicleMap.SegmentStats::usedBytes));
System.out.println("segments " + ss.length + ", from " + ss[0] + " to " + ss[ss.length - 1]);
ChronicleMap.SegmentStats[] ss2 = map2.segmentStats();
Arrays.sort(ss2, Comparator.comparingLong(ChronicleMap.SegmentStats::usedBytes));
System.out.println("segments " + ss2.length + ", from " + ss2[0] + " to " + ss2[ss2.length - 1]);
ProcessBuilder pb = new ProcessBuilder("bash", "-c", "ls -l *.map ; du *.map");
pb.redirectErrorStream(true);
Process process = pb.start();
try (BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
for (String line; (line = br.readLine()) != null; ) {
System.out.println(line);
}
}
System.out.println("...");
}
}
private static ChronicleMap<CharSequence, CharSequence> createMap(boolean sparseFile) throws IOException {
File file = new File((sparseFile ? "sparse" : "default") + "-deleteme.map");
file.delete();
ChronicleMapBuilder<CharSequence, CharSequence> builder = ChronicleMapBuilder
.of(CharSequence.class, CharSequence.class)
.entries(4_000_000)
.entries(100_000)
.actualSegments(64)
.averageKeySize(64)
.averageValueSize(1000)
.maxBloatFactor(3)
.averageValueSize(4096) // 4 KB
.maxBloatFactor(4)
.sparseFile(sparseFile);
ChronicleMap<CharSequence, CharSequence> map = builder.createPersistedTo(new File((sparseFile ? "sparse" : "default") + "-deleteme.map"));
ChronicleMap<CharSequence, CharSequence> map = builder.createPersistedTo(file);
return map;
}