diff --git a/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java b/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java index 099a2d464..d9f291856 100644 --- a/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java +++ b/src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java @@ -54,6 +54,8 @@ import java.lang.reflect.Type; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -92,6 +94,8 @@ public abstract class VanillaChronicleHash length * 11 / 10) { + PosixFallocate.fallocate(raf.getFD(), mappingOffsetInFile, length); + return; + } + maxSize = Math.max(maxSize, unallocatedSpace); + } + throw new IOException("Not enough space to fallocate " + (length >> 20) + " MiB to " + fileSystem.getRootDirectories().iterator().next() + " unallocated was " + (maxSize >> 20) + " MiB"); + } + private long bulkOffset(final int bulkIndex) { return sizeInBytesWithoutTiers() + bulkIndex * tierBulkSizeInBytes; } diff --git a/src/main/java/net/openhft/chronicle/hash/impl/util/jna/PosixFallocate.java b/src/main/java/net/openhft/chronicle/hash/impl/util/jna/PosixFallocate.java index c477ad56e..89652f451 100644 --- a/src/main/java/net/openhft/chronicle/hash/impl/util/jna/PosixFallocate.java +++ b/src/main/java/net/openhft/chronicle/hash/impl/util/jna/PosixFallocate.java @@ -3,10 +3,9 @@ package net.openhft.chronicle.hash.impl.util.jna; import com.sun.jna.Native; import com.sun.jna.Platform; import net.openhft.chronicle.core.Jvm; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.FileDescriptor; +import java.io.IOException; import java.lang.reflect.Field; public final class PosixFallocate { @@ -18,26 +17,25 @@ public final class PosixFallocate { private PosixFallocate() { } - public static void fallocate(FileDescriptor descriptor, long offset, long length) { + public static void fallocate(FileDescriptor descriptor, long offset, long length) throws IOException { int fd = getNativeFileDescriptor(descriptor); if (fd != -1) { int ret = posix_fallocate(getNativeFileDescriptor(descriptor), offset, length); if (ret != 0) { - Jvm.warn().on(PosixFallocate.class, "posix_fallocate() returned " + ret); + throw new IOException("posix_fallocate() returned " + ret); } } } private static native int posix_fallocate(int fd, long offset, long length); - private static int getNativeFileDescriptor(FileDescriptor descriptor) { + private static int getNativeFileDescriptor(FileDescriptor descriptor) throws IOException { try { final Field field = descriptor.getClass().getDeclaredField("fd"); Jvm.setAccessible(field); return (int) field.get(descriptor); } catch (final Exception e) { - Jvm.warn().on(PosixFallocate.class, "unsupported FileDescriptor implementation: e=" + e.getLocalizedMessage()); - return -1; + throw new IOException("unsupported FileDescriptor implementation", e); } } } diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index 085213afe..e1bd0c4d1 100644 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -329,11 +329,11 @@ public final class ChronicleMapBuilder implements // When replicated, having 16 chunks (=> 8 bits in bitsets) per entry seems more wasteful // because when replicated we have bit sets per each remote node, not only allocation // bit set as when non-replicated - if (avgEntrySize >= 64 * 64) + if (avgEntrySize >= 2 * 64 * 64) return replicated ? 32 : 64; - if (avgEntrySize >= 32 * 32) + if (avgEntrySize >= 2 * 32 * 32) return replicated ? 16 : 32; - if (avgEntrySize >= 16 * 16) + if (avgEntrySize >= 2 * 16 * 16) return replicated ? 8 : 16; return replicated ? 4 : 8; } diff --git a/src/test/java/net/openhft/chronicle/map/HugeSparseMapTest.java b/src/test/java/net/openhft/chronicle/map/HugeSparseMapTest.java new file mode 100644 index 000000000..7d316eb99 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/HugeSparseMapTest.java @@ -0,0 +1,50 @@ +package net.openhft.chronicle.map; + +import net.openhft.chronicle.core.OS; +import net.openhft.chronicle.core.io.IOTools; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +public class HugeSparseMapTest { + + private static ChronicleMap createMap(boolean sparseFile) throws IOException { + File file = IOTools.createTempFile("huge-map"); + try { + file.deleteOnExit(); + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(CharSequence.class, CharSequence.class) + .averageKeySize(64) + // .entries(10_000_000_000L).averageValueSize(640) // 16 TB + .entries(3_000_000_000L).averageValueSize(200) // 2 TB + .sparseFile(sparseFile); // ~16 TB. + ChronicleMap map = builder.createPersistedTo(file); + return map; + } catch (Throwable t) { + file.delete(); + throw t; + } + } + + @Test + public void hugeSparseMap() throws IOException { + assumeTrue(OS.isLinux()); + try (ChronicleMap map = createMap(true)) { + map.put("hi", "there"); + assertEquals("there", map.get("hi").toString()); + } + } + + @Test(expected = IOException.class) + public void hugeAllocatedMap() throws IOException { + assumeTrue(OS.isLinux()); + try (ChronicleMap map = createMap(false)) { + map.put("hi", "there"); + assertEquals("there", map.get("hi").toString()); + } + } +} diff --git a/src/test/java/net/openhft/chronicle/map/LargeMapMain.java b/src/test/java/net/openhft/chronicle/map/LargeMapMain.java index 745f895ab..c8d40b897 100644 --- a/src/test/java/net/openhft/chronicle/map/LargeMapMain.java +++ b/src/test/java/net/openhft/chronicle/map/LargeMapMain.java @@ -9,6 +9,9 @@ import java.util.Comparator; import java.util.Random; public class LargeMapMain { + // static int avgSize = 4096, maxSize = 40_000, entries= 500_000; // average ~= 4K + static int avgSize = 1024, maxSize = 8_000, entries = 1_000_000; // average ~= 1K + public static void main(String[] args) throws IOException { ChronicleMap map = createMap(false); ChronicleMap map2 = createMap(true); @@ -16,9 +19,9 @@ public class LargeMapMain { StringBuilder value = new StringBuilder(); Random rand = new Random(1); for (int j = 0; j < 40; j++) { - for (int i = 0; i < 10_000; i++) { + for (int i = 0; i < entries / 10; i++) { generate(rand, key, 1 + rand.nextInt(128)); - generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K + generate(rand, value, (int) Math.pow(maxSize, rand.nextFloat())); map.put(key, value); map2.put(key, value); } @@ -26,7 +29,7 @@ public class LargeMapMain { int repeat = Math.max(1, 9 - j); for (int r = 0; r < repeat; r++) { for (CharSequence cs : map2.keySet()) { - generate(rand, value, (int) Math.pow(40_000, rand.nextFloat())); // average ~= 4K + generate(rand, value, (int) Math.pow(maxSize, rand.nextFloat())); map.put(cs, value); map2.put(cs, value); } @@ -58,10 +61,10 @@ public class LargeMapMain { file.delete(); ChronicleMapBuilder builder = ChronicleMapBuilder .of(CharSequence.class, CharSequence.class) - .entries(100_000) + .entries(entries) .actualSegments(64) .averageKeySize(64) - .averageValueSize(4096) // 4 KB + .averageValueSize(avgSize) .maxBloatFactor(4) .sparseFile(sparseFile); ChronicleMap map = builder.createPersistedTo(file); diff --git a/src/test/java/net/openhft/chronicle/map/VeryLargeMapMain.java b/src/test/java/net/openhft/chronicle/map/VeryLargeMapMain.java new file mode 100644 index 000000000..80dbdf942 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/VeryLargeMapMain.java @@ -0,0 +1,70 @@ +package net.openhft.chronicle.map; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Random; +import java.util.stream.IntStream; + +public class VeryLargeMapMain { + static int avgSize = 16000, maxSize = 190_000, entries = 50_000_000; // average ~= 16K + + public static void main(String[] args) throws IOException { + ChronicleMap map = createMap(true); + while (map.size() < entries) { + long start = System.nanoTime(); + IntStream.range(0, 1000) + .parallel() + .forEach(t -> { + Random rand = new Random(); + StringBuilder key = new StringBuilder(); + StringBuilder value = new StringBuilder(); + for (int k = 0; k < 1000; k++) { + generate(rand, key, 1 + rand.nextInt(128)); + generate(rand, value, (int) Math.pow(maxSize, rand.nextFloat())); + map.put(key, value); + } + }); + long time = System.nanoTime() - start; + System.out.println("Entries per second: " + (long) (1e6 * 1e9 / time)); + + System.out.println("any key sparse.size()= " + map.size()); + ChronicleMap.SegmentStats[] ss = map.segmentStats(); + Arrays.sort(ss, Comparator.comparingLong(ChronicleMap.SegmentStats::usedBytes)); + System.out.println("segments " + ss.length + ", from " + ss[0] + " to " + ss[ss.length - 1]); + + ProcessBuilder pb = new ProcessBuilder("bash", "-c", "ls -l *.map ; du *.map"); + pb.redirectErrorStream(true); + Process process = pb.start(); + try (BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + for (String line; (line = br.readLine()) != null; ) { + System.out.println(line); + } + } + System.out.println("..."); + } + } + + private static ChronicleMap createMap(boolean sparseFile) throws IOException { + File file = new File("verylarge-deleteme.map"); + file.delete(); + ChronicleMapBuilder builder = ChronicleMapBuilder + .of(CharSequence.class, CharSequence.class) + .entries(entries) + .averageKeySize(64) + .averageValueSize(avgSize) +// .maxBloatFactor(2) + .sparseFile(sparseFile); + ChronicleMap map = builder.createPersistedTo(file); + return map; + } + + private static void generate(Random rand, StringBuilder sb, int length) { + sb.setLength(0); + while (sb.length() <= length) + sb.append(rand.nextInt()); + } +}