Make max hashLookup load factor (for tiering) greater than max 'normal'

load factor (0.66), to avoid tiering due to natural variance in segment
load. Account non-tiered segments percentile

src/main/java/net/openhft/chronicle/hash/ChronicleHashBuilderPrivateAPI.java

@@ -30,6 +30,8 @@ public interface ChronicleHashBuilderPrivateAPI<K> {
 
     long entriesPerSegment();
 
+    long tierHashLookupCapacity();
+
     long actualChunksPerSegmentTier();
 
     int segmentHeaderSize();

src/main/java/net/openhft/chronicle/hash/impl/CompactOffHeapLinearHashTable.java

@@ -26,7 +26,14 @@ public abstract class CompactOffHeapLinearHashTable {
     // to fit 64 bits per slot.
     public static final int MAX_TIER_CHUNKS = 1 << 30;
     public static final int MAX_TIER_ENTRIES = 1 << 29;
-    public static final double MAX_LOAD_FACTOR = .2/.3;
+    /** Normally, hashLookup capacity is chosen so that the lf varies from 1/3 to 2/3, at most */
+    public static final double MAX_UPPER_BOUND_LOAD_FACTOR = .2/.3;
+    /**
+     * If average key/value sizes are configured wrong, and Chronicle Hash bloats up (size grows
+     * beyond configured .entries()), hashLookup might become too full. Stop insertion and trigger
+     * segment tiering, when hashLookup load factor > 0.8, because collision chains become too long.
+     */
+    public static final double MAX_LOAD_FACTOR = 0.8;
 
 
     public static int valueBits(long actualChunksPerSegment) {
@@ -56,12 +63,10 @@ public abstract class CompactOffHeapLinearHashTable {
 
     public static long capacityFor(long entriesPerSegment) {
         if (entriesPerSegment < 0L)
-            throw new IllegalArgumentException("entriesPerSegment should be positive");
+            throw new AssertionError("entriesPerSegment should be positive");
         long capacity = Maths.nextPower2(entriesPerSegment, 64L);
-        if (((double) entriesPerSegment) / (double) capacity > MAX_LOAD_FACTOR) {
-            // hash lookup shouldn't be too dense
-            capacity <<= 1L;
-        }
+        if (entriesPerSegment > MAX_UPPER_BOUND_LOAD_FACTOR * capacity)
+            capacity *= 2;
         return capacity;
     }
 

src/main/java/net/openhft/chronicle/hash/impl/VanillaChronicleHash.java

@@ -194,7 +194,7 @@ public abstract class VanillaChronicleHash<K,
             throw new IllegalStateException("aligned64BitMemoryOperationsAtomic() == false, " +
                     "but hash lookup slot is " + tierHashLookupEntrySize);
         }
-        tierHashLookupCapacity = CompactOffHeapLinearHashTable.capacityFor(entriesPerSegment);
+        tierHashLookupCapacity = privateAPI.tierHashLookupCapacity();
         maxEntriesPerHashLookup = (long) (tierHashLookupCapacity * MAX_LOAD_FACTOR);
         tierHashLookupInnerSize = tierHashLookupCapacity * tierHashLookupEntrySize;
         tierHashLookupOuterSize = CACHE_LINES.align(tierHashLookupInnerSize, BYTES);

src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java

@@ -24,6 +24,7 @@ import net.openhft.chronicle.core.OS;
 import net.openhft.chronicle.hash.ChronicleHashBuilder;
 import net.openhft.chronicle.hash.ChronicleHashBuilderPrivateAPI;
 import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder;
+import net.openhft.chronicle.hash.impl.CompactOffHeapLinearHashTable;
 import net.openhft.chronicle.hash.impl.stage.entry.ChecksumStrategy;
 import net.openhft.chronicle.hash.impl.util.math.PoissonDistribution;
 import net.openhft.chronicle.hash.replication.*;
@@ -971,6 +972,17 @@ public final class ChronicleMapBuilder<K, V> implements
         return segments;
     }
 
+    long tierHashLookupCapacity() {
+        long entriesPerSegment = entriesPerSegment();
+        long capacity = CompactOffHeapLinearHashTable.capacityFor(entriesPerSegment);
+        long maxEntriesPerTier = PoissonDistribution.inverseCumulativeProbability(
+                entriesPerSegment, nonTieredSegmentsPercentile);
+        while (maxEntriesPerTier > MAX_LOAD_FACTOR * capacity) {
+            capacity *= 2;
+        }
+        return capacity;
+    }
+
     int segmentHeaderSize() {
         int segments = actualSegments();
 

src/main/java/net/openhft/chronicle/map/ChronicleMapBuilderPrivateAPI.java

@@ -52,6 +52,11 @@ class ChronicleMapBuilderPrivateAPI<K> implements ChronicleHashBuilderPrivateAPI
         return b.entriesPerSegment();
     }
 
+    @Override
+    public long tierHashLookupCapacity() {
+        return b.tierHashLookupCapacity();
+    }
+
     @Override
     public long actualChunksPerSegmentTier() {
         return b.actualChunksPerSegmentTier();