mirror of
https://github.com/OpenHFT/Chronicle-Map.git
synced 2021-09-13 01:26:12 +00:00
HCOLL-318 added many documentation and examples; Empowerted ReplicableEntry interface, and changed Map/SetRemoteOperations contracts to allow implementing more agile reconciliation strategies
This commit is contained in:
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue;
|
||||
public class BiMapTest {
|
||||
|
||||
enum DualLockSuccess {SUCCESS, FAIL}
|
||||
|
||||
|
||||
static class BiMapMethods<K, V> implements MapMethods<K, V, DualLockSuccess> {
|
||||
@Override
|
||||
public void remove(MapQueryContext<K, V, DualLockSuccess> q, ReturnValue<V> returnValue) {
|
||||
@@ -88,7 +88,7 @@ public class BiMapTest {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean remove(MapQueryContext<K, V, DualLockSuccess> q, net.openhft.chronicle.hash.Data<V, ?> value) {
|
||||
public boolean remove(MapQueryContext<K, V, DualLockSuccess> q, Data<V, ?> value) {
|
||||
while (true) {
|
||||
q.updateLock().lock();
|
||||
MapEntry<K, V> entry = q.entry();
|
||||
@@ -141,7 +141,7 @@ public class BiMapTest {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static class BiMapEntryOperations<K, V> implements MapEntryOperations<K, V, DualLockSuccess> {
|
||||
ChronicleMap<V, K> reverse;
|
||||
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
/*
|
||||
* Copyright 2015 Higher Frequency Trading
|
||||
*
|
||||
* http://www.higherfrequencytrading.com
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package net.openhft.chronicle.map;
|
||||
|
||||
import net.openhft.chronicle.hash.AcceptanceDecision;
|
||||
import net.openhft.chronicle.hash.Data;
|
||||
import net.openhft.chronicle.map.replication.MapRemoteOperations;
|
||||
import net.openhft.chronicle.map.replication.MapRemoteQueryContext;
|
||||
import net.openhft.chronicle.map.replication.MapReplicableEntry;
|
||||
|
||||
import static net.openhft.chronicle.hash.AcceptanceDecision.ACCEPT;
|
||||
import static net.openhft.chronicle.hash.AcceptanceDecision.DISCARD;
|
||||
|
||||
// TODO verify eventual consistency
|
||||
public class MaximizingValueRemoteOperations<K, V extends Comparable<? super V>, R>
|
||||
implements MapRemoteOperations<K, V, R> {
|
||||
|
||||
@Override
|
||||
public AcceptanceDecision put(MapRemoteQueryContext<K, V, R> q, Data<V, ?> newValue) {
|
||||
MapReplicableEntry<K, V> entry = q.entry();
|
||||
if (entry != null) {
|
||||
int compareResult = newValue.get().compareTo(entry.value().get());
|
||||
if (compareResult > 0 ||
|
||||
(compareResult == 0 && q.remoteIdentifier() <= entry.originIdentifier())) {
|
||||
// replace, if the new value is greater
|
||||
entry.doReplaceValue(newValue);
|
||||
return ACCEPT;
|
||||
} else {
|
||||
return DISCARD;
|
||||
}
|
||||
} else {
|
||||
return MapRemoteOperations.super.put(q, newValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package net.openhft.chronicle.map;
|
||||
|
||||
import net.openhft.chronicle.hash.Data;
|
||||
import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication;
|
||||
import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig;
|
||||
import net.openhft.chronicle.map.replication.MapRemoteOperations;
|
||||
import net.openhft.chronicle.map.replication.MapRemoteQueryContext;
|
||||
import net.openhft.chronicle.map.replication.MapReplicableEntry;
|
||||
import net.openhft.lang.io.ByteBufferBytes;
|
||||
import net.openhft.lang.model.Byteable;
|
||||
import net.openhft.lang.model.DataValueClasses;
|
||||
import net.openhft.lang.values.IntValue;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static net.openhft.chronicle.core.MemoryUnit.MEGABYTES;
|
||||
import static net.openhft.chronicle.map.TcpReplicationSoakCrdtTest.GrowOnlySetValuedMapEntryOperations.growOnlySetValuedMapEntryOperations;
|
||||
import static net.openhft.chronicle.map.TcpReplicationSoakCrdtTest.GrowOnlySetValuedMapRemoteOperations.growOnlySetValuedMapRemoteOperations;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class TcpReplicationSoakCrdtTest {
|
||||
|
||||
static class GrowOnlySetValuedMapEntryOperations<K, E>
|
||||
implements MapEntryOperations<K, Set<E>, Void> {
|
||||
|
||||
private static final GrowOnlySetValuedMapEntryOperations INSTANCE =
|
||||
new GrowOnlySetValuedMapEntryOperations();
|
||||
|
||||
public static <K, E>
|
||||
MapEntryOperations<K, Set<E>, Void> growOnlySetValuedMapEntryOperations() {
|
||||
return GrowOnlySetValuedMapEntryOperations.INSTANCE;
|
||||
}
|
||||
|
||||
private GrowOnlySetValuedMapEntryOperations() {}
|
||||
|
||||
@Override
|
||||
public Void remove(@NotNull MapEntry<K, Set<E>> entry) {
|
||||
throw new UnsupportedOperationException("Map with grow-only set values " +
|
||||
"doesn't support map value removals");
|
||||
}
|
||||
}
|
||||
|
||||
static class GrowOnlySetValuedMapRemoteOperations<K, E>
|
||||
implements MapRemoteOperations<K, Set<E>, Void> {
|
||||
|
||||
private static final GrowOnlySetValuedMapRemoteOperations INSTANCE =
|
||||
new GrowOnlySetValuedMapRemoteOperations();
|
||||
|
||||
public static <K, E>
|
||||
MapRemoteOperations<K, Set<E>, Void> growOnlySetValuedMapRemoteOperations() {
|
||||
return GrowOnlySetValuedMapRemoteOperations.INSTANCE;
|
||||
}
|
||||
|
||||
private GrowOnlySetValuedMapRemoteOperations() {}
|
||||
|
||||
@Override
|
||||
public void put(MapRemoteQueryContext<K, Set<E>, Void> q, Data<Set<E>, ?> newValue) {
|
||||
MapReplicableEntry<K, Set<E>> entry = q.entry();
|
||||
if (entry != null) {
|
||||
Set<E> merged = new HashSet<>(entry.value().get());
|
||||
merged.addAll(newValue.get());
|
||||
q.replaceValue(entry, q.wrapValueAsValue(merged));
|
||||
} else {
|
||||
q.insert(q.absentEntry(), newValue);
|
||||
q.entry().updateOrigin(q.remoteIdentifier(), q.remoteTimestamp());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(MapRemoteQueryContext<K, Set<E>, Void> q) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
private ChronicleMap<Integer, Set<Integer>> map1;
|
||||
private ChronicleMap<Integer, Set<Integer>> map2;
|
||||
static int s_port = 8093;
|
||||
|
||||
@Before
|
||||
public void setup() throws IOException {
|
||||
final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1);
|
||||
|
||||
ChronicleMapBuilder<Integer, Set<Integer>> builder = ChronicleMapBuilder
|
||||
.of(Integer.class, (Class<Set<Integer>>) (Class) Set.class)
|
||||
.entries(Builder.SIZE)
|
||||
.averageValueSize(3000) // more than HashSet of 10 Integers on average, serialized
|
||||
.entryOperations(growOnlySetValuedMapEntryOperations())
|
||||
.remoteOperations(growOnlySetValuedMapRemoteOperations());
|
||||
|
||||
{
|
||||
final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(s_port,
|
||||
endpoint).autoReconnectedUponDroppedConnection(true)
|
||||
.heartBeatInterval(1, TimeUnit.SECONDS)
|
||||
.tcpBufferSize((int) MEGABYTES.toBytes(64));
|
||||
|
||||
map1 = builder
|
||||
.replication(SingleChronicleHashReplication.builder()
|
||||
.tcpTransportAndNetwork(tcpConfig1)
|
||||
.name("map1")
|
||||
.createWithId((byte) 1))
|
||||
.instance()
|
||||
.name("map1")
|
||||
.create();
|
||||
}
|
||||
{
|
||||
final TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of
|
||||
(s_port + 1).autoReconnectedUponDroppedConnection(true)
|
||||
.heartBeatInterval(1, TimeUnit.SECONDS)
|
||||
.tcpBufferSize((int) MEGABYTES.toBytes(64));
|
||||
|
||||
map2 = builder
|
||||
.replication(SingleChronicleHashReplication.builder()
|
||||
.tcpTransportAndNetwork(tcpConfig2)
|
||||
.name("map2")
|
||||
.createWithId((byte) 2))
|
||||
.instance()
|
||||
.name("map2")
|
||||
.create();
|
||||
|
||||
}
|
||||
s_port += 2;
|
||||
}
|
||||
|
||||
|
||||
@After
|
||||
public void tearDown() throws InterruptedException {
|
||||
|
||||
for (final Closeable closeable : new Closeable[]{map1, map2}) {
|
||||
try {
|
||||
closeable.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
System.gc();
|
||||
}
|
||||
|
||||
Set<Thread> threads;
|
||||
|
||||
@Before
|
||||
public void sampleThreads() {
|
||||
threads = Thread.getAllStackTraces().keySet();
|
||||
}
|
||||
|
||||
@After
|
||||
public void checkThreadsShutdown() {
|
||||
StatelessClientTest.checkThreadsShutdown(threads);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testReplicatedValuesMerged() throws InterruptedException {
|
||||
HashSet<Integer> v1 = new HashSet<>();
|
||||
v1.add(1);
|
||||
map1.put(1, v1);
|
||||
HashSet<Integer> v2 = new HashSet<>();
|
||||
v2.add(2);
|
||||
map2.put(1, v2);
|
||||
waitTillEqual(5000);
|
||||
// Map1 not equal to Map2, because it use byte-to-byte comparison of serialized HashSet
|
||||
// values, that are different for some reason
|
||||
assertEquals(map1.entrySet(), map2.entrySet());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSoakTestWithRandomData() throws IOException, InterruptedException {
|
||||
try {
|
||||
System.out.print("SoakTesting ");
|
||||
for (int j = 1; j < 2 * Builder.SIZE; j++) {
|
||||
if (j % 1000 == 0)
|
||||
System.out.print(".");
|
||||
Random rnd = new Random(j);
|
||||
for (int i = 1; i < 10; i++) {
|
||||
final ChronicleMap<Integer, Set<Integer>> map = rnd.nextBoolean() ? map1 : map2;
|
||||
int key = rnd.nextInt(Builder.SIZE);
|
||||
try (ExternalMapQueryContext<Integer, Set<Integer>, ?> q =
|
||||
map.queryContext(key)) {
|
||||
q.updateLock().lock();
|
||||
MapEntry<Integer, Set<Integer>> entry = q.entry();
|
||||
if (entry != null) {
|
||||
Set<Integer> value = entry.value().get();
|
||||
value.add(rnd.nextInt(20));
|
||||
q.replaceValue(entry, q.wrapValueAsValue(value));
|
||||
} else {
|
||||
HashSet<Integer> value = new HashSet<>();
|
||||
q.insert(q.absentEntry(), q.wrapValueAsValue(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println("\nwaiting till equal");
|
||||
|
||||
waitTillEqual(15000);
|
||||
|
||||
// not map1.equals(map2), the reason is described above
|
||||
assertEquals(map1.entrySet(), map2.entrySet());
|
||||
} finally {
|
||||
map1.close();
|
||||
map2.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void waitTillEqual(final int timeOutMs) throws InterruptedException {
|
||||
|
||||
Map map1UnChanged = new HashMap();
|
||||
Map map2UnChanged = new HashMap();
|
||||
|
||||
int numberOfTimesTheSame = 0;
|
||||
long startTime = System.currentTimeMillis();
|
||||
for (int t = 0; t < timeOutMs + 100; t++) {
|
||||
// not map1.equals(map2), the reason is described above
|
||||
if (map1.entrySet().equals(map2.entrySet())) {
|
||||
if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) {
|
||||
numberOfTimesTheSame++;
|
||||
} else {
|
||||
numberOfTimesTheSame = 0;
|
||||
map1UnChanged = new HashMap(map1);
|
||||
map2UnChanged = new HashMap(map2);
|
||||
}
|
||||
Thread.sleep(1);
|
||||
if (numberOfTimesTheSame == 10) {
|
||||
System.out.println("same");
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
Thread.sleep(1);
|
||||
if (System.currentTimeMillis() - startTime > timeOutMs)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,12 +26,11 @@ public class TcpReplicationSoakTest {
|
||||
|
||||
private ChronicleMap<Integer, CharSequence> map1;
|
||||
private ChronicleMap<Integer, CharSequence> map2;
|
||||
private IntValue value;
|
||||
static int s_port = 8093;
|
||||
|
||||
@Before
|
||||
public void setup() throws IOException {
|
||||
value = DataValueClasses.newDirectReference(IntValue.class);
|
||||
IntValue value = DataValueClasses.newDirectReference(IntValue.class);
|
||||
((Byteable) value).bytes(new ByteBufferBytes(ByteBuffer.allocateDirect(4)), 0);
|
||||
|
||||
final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1);
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package net.openhft.chronicle.map.example;
|
||||
|
||||
import net.openhft.chronicle.map.*;
|
||||
|
||||
public class HashQueryContextExamples {
|
||||
|
||||
public static <K, V> void incorrectRemovePresentEntry(ChronicleMap<K, V> map, K key) {
|
||||
try (ExternalMapQueryContext<K, V, ?> q = map.queryContext(key)) {
|
||||
// q.entry(), checks if the entry is present in the map, and acquires
|
||||
// the read lock for that.
|
||||
MapEntry<K, V> entry = q.entry();
|
||||
if (entry != null) {
|
||||
// Tries to acquire the write lock to perform modification,
|
||||
// but this is an illegal upgrade: read -> write, throws IllegalMonitorStateException
|
||||
q.remove(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static <K, V> void correctRemovePresentEntry(ChronicleMap<K, V> map, K key) {
|
||||
try (ExternalMapQueryContext<K, V, ?> q = map.queryContext(key)) {
|
||||
q.updateLock().lock(); // acquire the update lock before checking the entry presence.
|
||||
MapEntry<K, V> entry = q.entry();
|
||||
if (entry != null)
|
||||
q.remove(entry);
|
||||
}
|
||||
}
|
||||
|
||||
public static <K, V, R> void possibleAcquireUsingImplementation(
|
||||
MapQueryContext<K, V, R> q, ReturnValue<V> returnValue) {
|
||||
// For acquireUsing(), it is assumed to be very probable, that the entry is already
|
||||
// present in the map, so we will perform the whole acquireUsing() without exclusive locking
|
||||
if (q.readLock().tryLock()) {
|
||||
MapEntry<K, V> entry = q.entry();
|
||||
if (entry != null) {
|
||||
// Entry is present, return
|
||||
returnValue.returnValue(entry.value());
|
||||
return;
|
||||
}
|
||||
// Key is absent
|
||||
// Need to unlock, to lock to update lock later. Direct upgrade is forbidden.
|
||||
q.readLock().unlock();
|
||||
}
|
||||
// We are here, either if we:
|
||||
// 1) Failed to acquire the read lock, this means some other thread is holding the write
|
||||
// lock now, in this case waiting for the update lock acquisition is no longer, than for
|
||||
// the read
|
||||
// 2) Seen the entry is absent under the read lock. This means we need to insert
|
||||
// the default value into the map. that requires update-level access as well
|
||||
q.updateLock().lock();
|
||||
MapEntry<K, V> entry = q.entry();
|
||||
if (entry != null) {
|
||||
// Entry is present, return
|
||||
returnValue.returnValue(entry.value());
|
||||
return;
|
||||
}
|
||||
// Key is absent
|
||||
q.insert(q.absentEntry(), q.defaultValue(q.absentEntry()));
|
||||
returnValue.returnValue(q.entry().value());
|
||||
}
|
||||
|
||||
interface Point {
|
||||
double getX();
|
||||
void setX(double x);
|
||||
double addX(double xAdd);
|
||||
|
||||
double getY();
|
||||
void setY(double y);
|
||||
double addY(double yAdd);
|
||||
}
|
||||
|
||||
<K> Point movePoint(ChronicleMap<K, Point> map, K key, double xMove, double yMove,
|
||||
Point using) {
|
||||
// Moves existing point by [xMove, yMove], if absent - assumes the default point is [0, 0].
|
||||
// Returns the resulting point
|
||||
try (ExternalMapQueryContext<K, Point, ?> q = map.queryContext(key)) {
|
||||
Point offHeapPoint;
|
||||
q.updateLock().lock();
|
||||
MapEntry<K, Point> entry = q.entry();
|
||||
if (entry != null) {
|
||||
// Key is present
|
||||
offHeapPoint = entry.value().getUsing(using);
|
||||
} else {
|
||||
// Key is absent
|
||||
q.insert(q.absentEntry(), q.defaultValue(q.absentEntry()));
|
||||
offHeapPoint = q.entry().value().getUsing(using);
|
||||
}
|
||||
offHeapPoint.addX(xMove);
|
||||
offHeapPoint.addY(yMove);
|
||||
return offHeapPoint;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package net.openhft.chronicle.map.example;
|
||||
|
||||
import net.openhft.chronicle.hash.Data;
|
||||
import net.openhft.chronicle.map.*;
|
||||
import net.openhft.lang.model.DataValueClasses;
|
||||
import net.openhft.lang.values.IntValue;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import static net.openhft.chronicle.map.example.SimpleMapOperationsListeningTest.SimpleLoggingMapEntryOperations.simpleLoggingMapEntryOperations;
|
||||
|
||||
public class SimpleMapOperationsListeningTest {
|
||||
|
||||
static class SimpleLoggingMapEntryOperations<K, V> implements MapEntryOperations<K, V, Void> {
|
||||
|
||||
private static final SimpleLoggingMapEntryOperations INSTANCE =
|
||||
new SimpleLoggingMapEntryOperations();
|
||||
|
||||
public static <K, V> MapEntryOperations<K, V, Void> simpleLoggingMapEntryOperations() {
|
||||
return SimpleLoggingMapEntryOperations.INSTANCE;
|
||||
}
|
||||
|
||||
private SimpleLoggingMapEntryOperations() {}
|
||||
|
||||
@Override
|
||||
public Void remove(@NotNull MapEntry<K, V> entry) {
|
||||
System.out.println("remove " + entry.key() + ": " + entry.value());
|
||||
entry.doRemove();
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Void replaceValue(@NotNull MapEntry<K, V> entry, Data<V, ?> newValue) {
|
||||
System.out.println("replace " + entry.key() + ": " + entry.value() + " -> " + newValue);
|
||||
entry.doReplaceValue(newValue);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Void insert(@NotNull MapAbsentEntry<K, V> absentEntry, Data<V, ?> value) {
|
||||
System.out.println("insert " + absentEntry.absentKey() + " -> " + value);
|
||||
absentEntry.doInsert(value);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Data<V, ?> defaultValue(@NotNull MapAbsentEntry<K, V> absentEntry) {
|
||||
Data<V, ?> defaultValue = absentEntry.defaultValue();
|
||||
System.out.println("default " + absentEntry.absentKey() + " -> " + defaultValue);
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Ignore("Example test, run manually to see the output")
|
||||
public void simpleLoggingTest() {
|
||||
ChronicleMap<Integer, IntValue> map = ChronicleMapBuilder
|
||||
.of(Integer.class, IntValue.class)
|
||||
.entries(100)
|
||||
.entryOperations(simpleLoggingMapEntryOperations())
|
||||
.create();
|
||||
|
||||
IntValue value = DataValueClasses.newDirectInstance(IntValue.class);
|
||||
value.setValue(2);
|
||||
map.put(1, value);
|
||||
map.remove(1);
|
||||
map.acquireUsing(3, value).addAtomicValue(1);
|
||||
IntValue value2 = DataValueClasses.newDirectInstance(IntValue.class);
|
||||
value2.setValue(5);
|
||||
map.forEachEntry(c -> c.put(value2));
|
||||
map.forEachEntry(c -> c.remove());
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user