Add ChronicleHash.offHeapMemoryUsed() method; Centralized management of all direct memory and file resources of ChronicleMap in one Cleaner

This commit is contained in:
leventov
2016-11-05 15:08:45 -06:00
parent 68c29c4c54
commit 9a2bb31b66
12 changed files with 478 additions and 159 deletions
@@ -1,50 +0,0 @@
/*
* Copyright (C) 2012, 2016 higherfrequencytrading.com
* Copyright (C) 2016 Roman Leventov
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map;
import org.junit.Assert;
import org.junit.Test;
import java.lang.ref.WeakReference;
public class MemoryLeakTest {
@Test
public void testChronicleMapCollected() {
WeakReference<ChronicleMap<Integer, Integer>> ref = getMapRef(false);
System.gc();
Assert.assertNull(ref.get());
}
@Test
public void testReplicatedChronicleMapCollected() {
WeakReference<ChronicleMap<Integer, Integer>> ref = getMapRef(true);
System.gc();
Assert.assertNull(ref.get());
}
private static WeakReference<ChronicleMap<Integer, Integer>> getMapRef(boolean replicated) {
ChronicleMapBuilder<Integer, Integer> builder = ChronicleMap
.of(Integer.class, Integer.class);
if (replicated)
builder.replication((byte) 1);
ChronicleMap<Integer, Integer> map = builder.entries(1).create();
map.put(1, 1);
return new WeakReference<>(map);
}
}
@@ -0,0 +1,123 @@
/*
* Copyright (C) 2012, 2016 higherfrequencytrading.com
* Copyright (C) 2016 Roman Leventov
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package net.openhft.chronicle.map;
import net.openhft.chronicle.bytes.NoBytesStore;
import net.openhft.chronicle.core.OS;
import net.openhft.chronicle.core.values.IntValue;
import net.openhft.chronicle.values.Values;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.util.Arrays;
import java.util.Collection;
import static org.junit.Assert.assertEquals;
@RunWith(Parameterized.class)
public class MemoryResourcesTest {
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{ false, false }, { false, true }, { true, false }, { true, true }
});
}
@Rule
public TemporaryFolder folder = new TemporaryFolder();
private boolean replicated;
private boolean persisted;
public MemoryResourcesTest(boolean replicated, boolean persisted) {
this.replicated = replicated;
this.persisted = persisted;
}
@Before
public void initNoBytesStore() {
Assert.assertNotEquals(0, NoBytesStore.NO_PAGE);
}
@Test(timeout = 60_000)
public void testChronicleMapCollectedAndDirectMemoryReleased() throws IOException {
long nativeMemoryUsedBeforeMap = nativeMemoryUsed();
WeakReference<ChronicleMap<IntValue, String>> ref = new WeakReference<>(getMap());
Assert.assertNotNull(ref.get());
long expectedNativeMemory = nativeMemoryUsedBeforeMap + ref.get().offHeapMemoryUsed();
assertEquals(expectedNativeMemory, nativeMemoryUsed());
// Wait until Map is collected by GC
while (ref.get() != null) {
System.gc();
Thread.yield();
}
// Wait until Cleaner is called and memory is returned to the system
while (nativeMemoryUsedBeforeMap != nativeMemoryUsed()) {
System.gc();
Thread.yield();
}
}
private long nativeMemoryUsed() {
if (persisted) {
return OS.memoryMapped();
} else {
return OS.memory().nativeMemoryUsed();
}
}
@Test
public void testExplicitChronicleMapCloseReleasesMemory() throws IOException {
long nativeMemoryUsedBeforeMap = nativeMemoryUsed();
try (ChronicleMap<IntValue, String> map = getMap()) {
long expectedNativeMemory = nativeMemoryUsedBeforeMap + map.offHeapMemoryUsed();
assertEquals(expectedNativeMemory, nativeMemoryUsed());
}
assertEquals(nativeMemoryUsedBeforeMap, nativeMemoryUsed());
}
private ChronicleMap<IntValue, String> getMap() throws IOException {
ChronicleMapBuilder<IntValue, String> builder = ChronicleMap
.of(IntValue.class, String.class);
if (replicated)
builder.replication((byte) 1);
builder.entries(1).averageValueSize(1);
VanillaChronicleMap<IntValue, String, ?> map;
if (persisted) {
map = (VanillaChronicleMap<IntValue, String, ?>)
builder.createPersistedTo(folder.newFile("map.dat"));
} else {
map = (VanillaChronicleMap<IntValue, String, ?>) builder.create();
}
IntValue key = Values.newHeapInstance(IntValue.class);
int i = 0;
while (!map.hasExtraTierBulks()) {
key.setValue(i++);
map.put(key, "string" + i);
}
return map;
}
}
@@ -214,6 +214,11 @@ public class ReplicationCheckingMap<K, V> implements ChronicleMap<K, V> {
return map1.longSize();
}
@Override
public long offHeapMemoryUsed() {
return map1.offHeapMemoryUsed();
}
@Override
public V getUsing(K key, V usingValue) {
return map1.getUsing(key, usingValue);