HCOLL-318 added many documentation and examples; Empowerted ReplicableEntry interface, and changed Map/SetRemoteOperations contracts to allow implementing more agile reconciliation strategies

This commit is contained in:
leventov
2015-06-20 05:43:34 +03:00
parent 9746084998
commit 1d7f54635a
28 changed files with 4164 additions and 2862 deletions
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue;
public class BiMapTest {
enum DualLockSuccess {SUCCESS, FAIL}
static class BiMapMethods<K, V> implements MapMethods<K, V, DualLockSuccess> {
@Override
public void remove(MapQueryContext<K, V, DualLockSuccess> q, ReturnValue<V> returnValue) {
@@ -88,7 +88,7 @@ public class BiMapTest {
}
@Override
public boolean remove(MapQueryContext<K, V, DualLockSuccess> q, net.openhft.chronicle.hash.Data<V, ?> value) {
public boolean remove(MapQueryContext<K, V, DualLockSuccess> q, Data<V, ?> value) {
while (true) {
q.updateLock().lock();
MapEntry<K, V> entry = q.entry();
@@ -141,7 +141,7 @@ public class BiMapTest {
throw new UnsupportedOperationException();
}
}
static class BiMapEntryOperations<K, V> implements MapEntryOperations<K, V, DualLockSuccess> {
ChronicleMap<V, K> reverse;
@@ -1,51 +0,0 @@
/*
* Copyright 2015 Higher Frequency Trading
*
* http://www.higherfrequencytrading.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.AcceptanceDecision;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.map.replication.MapRemoteOperations;
import net.openhft.chronicle.map.replication.MapRemoteQueryContext;
import net.openhft.chronicle.map.replication.MapReplicableEntry;
import static net.openhft.chronicle.hash.AcceptanceDecision.ACCEPT;
import static net.openhft.chronicle.hash.AcceptanceDecision.DISCARD;
// TODO verify eventual consistency
public class MaximizingValueRemoteOperations<K, V extends Comparable<? super V>, R>
implements MapRemoteOperations<K, V, R> {
@Override
public AcceptanceDecision put(MapRemoteQueryContext<K, V, R> q, Data<V, ?> newValue) {
MapReplicableEntry<K, V> entry = q.entry();
if (entry != null) {
int compareResult = newValue.get().compareTo(entry.value().get());
if (compareResult > 0 ||
(compareResult == 0 && q.remoteIdentifier() <= entry.originIdentifier())) {
// replace, if the new value is greater
entry.doReplaceValue(newValue);
return ACCEPT;
} else {
return DISCARD;
}
} else {
return MapRemoteOperations.super.put(q, newValue);
}
}
}
@@ -0,0 +1,246 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.replication.SingleChronicleHashReplication;
import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig;
import net.openhft.chronicle.map.replication.MapRemoteOperations;
import net.openhft.chronicle.map.replication.MapRemoteQueryContext;
import net.openhft.chronicle.map.replication.MapReplicableEntry;
import net.openhft.lang.io.ByteBufferBytes;
import net.openhft.lang.model.Byteable;
import net.openhft.lang.model.DataValueClasses;
import net.openhft.lang.values.IntValue;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.TimeUnit;
import static net.openhft.chronicle.core.MemoryUnit.MEGABYTES;
import static net.openhft.chronicle.map.TcpReplicationSoakCrdtTest.GrowOnlySetValuedMapEntryOperations.growOnlySetValuedMapEntryOperations;
import static net.openhft.chronicle.map.TcpReplicationSoakCrdtTest.GrowOnlySetValuedMapRemoteOperations.growOnlySetValuedMapRemoteOperations;
import static org.junit.Assert.assertEquals;
public class TcpReplicationSoakCrdtTest {
static class GrowOnlySetValuedMapEntryOperations<K, E>
implements MapEntryOperations<K, Set<E>, Void> {
private static final GrowOnlySetValuedMapEntryOperations INSTANCE =
new GrowOnlySetValuedMapEntryOperations();
public static <K, E>
MapEntryOperations<K, Set<E>, Void> growOnlySetValuedMapEntryOperations() {
return GrowOnlySetValuedMapEntryOperations.INSTANCE;
}
private GrowOnlySetValuedMapEntryOperations() {}
@Override
public Void remove(@NotNull MapEntry<K, Set<E>> entry) {
throw new UnsupportedOperationException("Map with grow-only set values " +
"doesn't support map value removals");
}
}
static class GrowOnlySetValuedMapRemoteOperations<K, E>
implements MapRemoteOperations<K, Set<E>, Void> {
private static final GrowOnlySetValuedMapRemoteOperations INSTANCE =
new GrowOnlySetValuedMapRemoteOperations();
public static <K, E>
MapRemoteOperations<K, Set<E>, Void> growOnlySetValuedMapRemoteOperations() {
return GrowOnlySetValuedMapRemoteOperations.INSTANCE;
}
private GrowOnlySetValuedMapRemoteOperations() {}
@Override
public void put(MapRemoteQueryContext<K, Set<E>, Void> q, Data<Set<E>, ?> newValue) {
MapReplicableEntry<K, Set<E>> entry = q.entry();
if (entry != null) {
Set<E> merged = new HashSet<>(entry.value().get());
merged.addAll(newValue.get());
q.replaceValue(entry, q.wrapValueAsValue(merged));
} else {
q.insert(q.absentEntry(), newValue);
q.entry().updateOrigin(q.remoteIdentifier(), q.remoteTimestamp());
}
}
@Override
public void remove(MapRemoteQueryContext<K, Set<E>, Void> q) {
throw new UnsupportedOperationException();
}
}
private ChronicleMap<Integer, Set<Integer>> map1;
private ChronicleMap<Integer, Set<Integer>> map2;
static int s_port = 8093;
@Before
public void setup() throws IOException {
final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1);
ChronicleMapBuilder<Integer, Set<Integer>> builder = ChronicleMapBuilder
.of(Integer.class, (Class<Set<Integer>>) (Class) Set.class)
.entries(Builder.SIZE)
.averageValueSize(3000) // more than HashSet of 10 Integers on average, serialized
.entryOperations(growOnlySetValuedMapEntryOperations())
.remoteOperations(growOnlySetValuedMapRemoteOperations());
{
final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(s_port,
endpoint).autoReconnectedUponDroppedConnection(true)
.heartBeatInterval(1, TimeUnit.SECONDS)
.tcpBufferSize((int) MEGABYTES.toBytes(64));
map1 = builder
.replication(SingleChronicleHashReplication.builder()
.tcpTransportAndNetwork(tcpConfig1)
.name("map1")
.createWithId((byte) 1))
.instance()
.name("map1")
.create();
}
{
final TcpTransportAndNetworkConfig tcpConfig2 = TcpTransportAndNetworkConfig.of
(s_port + 1).autoReconnectedUponDroppedConnection(true)
.heartBeatInterval(1, TimeUnit.SECONDS)
.tcpBufferSize((int) MEGABYTES.toBytes(64));
map2 = builder
.replication(SingleChronicleHashReplication.builder()
.tcpTransportAndNetwork(tcpConfig2)
.name("map2")
.createWithId((byte) 2))
.instance()
.name("map2")
.create();
}
s_port += 2;
}
@After
public void tearDown() throws InterruptedException {
for (final Closeable closeable : new Closeable[]{map1, map2}) {
try {
closeable.close();
} catch (Exception e) {
e.printStackTrace();
}
}
System.gc();
}
Set<Thread> threads;
@Before
public void sampleThreads() {
threads = Thread.getAllStackTraces().keySet();
}
@After
public void checkThreadsShutdown() {
StatelessClientTest.checkThreadsShutdown(threads);
}
@Test
public void testReplicatedValuesMerged() throws InterruptedException {
HashSet<Integer> v1 = new HashSet<>();
v1.add(1);
map1.put(1, v1);
HashSet<Integer> v2 = new HashSet<>();
v2.add(2);
map2.put(1, v2);
waitTillEqual(5000);
// Map1 not equal to Map2, because it use byte-to-byte comparison of serialized HashSet
// values, that are different for some reason
assertEquals(map1.entrySet(), map2.entrySet());
}
@Test
public void testSoakTestWithRandomData() throws IOException, InterruptedException {
try {
System.out.print("SoakTesting ");
for (int j = 1; j < 2 * Builder.SIZE; j++) {
if (j % 1000 == 0)
System.out.print(".");
Random rnd = new Random(j);
for (int i = 1; i < 10; i++) {
final ChronicleMap<Integer, Set<Integer>> map = rnd.nextBoolean() ? map1 : map2;
int key = rnd.nextInt(Builder.SIZE);
try (ExternalMapQueryContext<Integer, Set<Integer>, ?> q =
map.queryContext(key)) {
q.updateLock().lock();
MapEntry<Integer, Set<Integer>> entry = q.entry();
if (entry != null) {
Set<Integer> value = entry.value().get();
value.add(rnd.nextInt(20));
q.replaceValue(entry, q.wrapValueAsValue(value));
} else {
HashSet<Integer> value = new HashSet<>();
q.insert(q.absentEntry(), q.wrapValueAsValue(value));
}
}
}
}
System.out.println("\nwaiting till equal");
waitTillEqual(15000);
// not map1.equals(map2), the reason is described above
assertEquals(map1.entrySet(), map2.entrySet());
} finally {
map1.close();
map2.close();
}
}
private void waitTillEqual(final int timeOutMs) throws InterruptedException {
Map map1UnChanged = new HashMap();
Map map2UnChanged = new HashMap();
int numberOfTimesTheSame = 0;
long startTime = System.currentTimeMillis();
for (int t = 0; t < timeOutMs + 100; t++) {
// not map1.equals(map2), the reason is described above
if (map1.entrySet().equals(map2.entrySet())) {
if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) {
numberOfTimesTheSame++;
} else {
numberOfTimesTheSame = 0;
map1UnChanged = new HashMap(map1);
map2UnChanged = new HashMap(map2);
}
Thread.sleep(1);
if (numberOfTimesTheSame == 10) {
System.out.println("same");
break;
}
}
Thread.sleep(1);
if (System.currentTimeMillis() - startTime > timeOutMs)
break;
}
}
}
@@ -26,12 +26,11 @@ public class TcpReplicationSoakTest {
private ChronicleMap<Integer, CharSequence> map1;
private ChronicleMap<Integer, CharSequence> map2;
private IntValue value;
static int s_port = 8093;
@Before
public void setup() throws IOException {
value = DataValueClasses.newDirectReference(IntValue.class);
IntValue value = DataValueClasses.newDirectReference(IntValue.class);
((Byteable) value).bytes(new ByteBufferBytes(ByteBuffer.allocateDirect(4)), 0);
final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1);
@@ -0,0 +1,93 @@
package net.openhft.chronicle.map.example;
import net.openhft.chronicle.map.*;
public class HashQueryContextExamples {
public static <K, V> void incorrectRemovePresentEntry(ChronicleMap<K, V> map, K key) {
try (ExternalMapQueryContext<K, V, ?> q = map.queryContext(key)) {
// q.entry(), checks if the entry is present in the map, and acquires
// the read lock for that.
MapEntry<K, V> entry = q.entry();
if (entry != null) {
// Tries to acquire the write lock to perform modification,
// but this is an illegal upgrade: read -> write, throws IllegalMonitorStateException
q.remove(entry);
}
}
}
public static <K, V> void correctRemovePresentEntry(ChronicleMap<K, V> map, K key) {
try (ExternalMapQueryContext<K, V, ?> q = map.queryContext(key)) {
q.updateLock().lock(); // acquire the update lock before checking the entry presence.
MapEntry<K, V> entry = q.entry();
if (entry != null)
q.remove(entry);
}
}
public static <K, V, R> void possibleAcquireUsingImplementation(
MapQueryContext<K, V, R> q, ReturnValue<V> returnValue) {
// For acquireUsing(), it is assumed to be very probable, that the entry is already
// present in the map, so we will perform the whole acquireUsing() without exclusive locking
if (q.readLock().tryLock()) {
MapEntry<K, V> entry = q.entry();
if (entry != null) {
// Entry is present, return
returnValue.returnValue(entry.value());
return;
}
// Key is absent
// Need to unlock, to lock to update lock later. Direct upgrade is forbidden.
q.readLock().unlock();
}
// We are here, either if we:
// 1) Failed to acquire the read lock, this means some other thread is holding the write
// lock now, in this case waiting for the update lock acquisition is no longer, than for
// the read
// 2) Seen the entry is absent under the read lock. This means we need to insert
// the default value into the map. that requires update-level access as well
q.updateLock().lock();
MapEntry<K, V> entry = q.entry();
if (entry != null) {
// Entry is present, return
returnValue.returnValue(entry.value());
return;
}
// Key is absent
q.insert(q.absentEntry(), q.defaultValue(q.absentEntry()));
returnValue.returnValue(q.entry().value());
}
interface Point {
double getX();
void setX(double x);
double addX(double xAdd);
double getY();
void setY(double y);
double addY(double yAdd);
}
<K> Point movePoint(ChronicleMap<K, Point> map, K key, double xMove, double yMove,
Point using) {
// Moves existing point by [xMove, yMove], if absent - assumes the default point is [0, 0].
// Returns the resulting point
try (ExternalMapQueryContext<K, Point, ?> q = map.queryContext(key)) {
Point offHeapPoint;
q.updateLock().lock();
MapEntry<K, Point> entry = q.entry();
if (entry != null) {
// Key is present
offHeapPoint = entry.value().getUsing(using);
} else {
// Key is absent
q.insert(q.absentEntry(), q.defaultValue(q.absentEntry()));
offHeapPoint = q.entry().value().getUsing(using);
}
offHeapPoint.addX(xMove);
offHeapPoint.addY(yMove);
return offHeapPoint;
}
}
}
@@ -0,0 +1,75 @@
package net.openhft.chronicle.map.example;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.map.*;
import net.openhft.lang.model.DataValueClasses;
import net.openhft.lang.values.IntValue;
import org.jetbrains.annotations.NotNull;
import org.junit.Ignore;
import org.junit.Test;
import static net.openhft.chronicle.map.example.SimpleMapOperationsListeningTest.SimpleLoggingMapEntryOperations.simpleLoggingMapEntryOperations;
public class SimpleMapOperationsListeningTest {
static class SimpleLoggingMapEntryOperations<K, V> implements MapEntryOperations<K, V, Void> {
private static final SimpleLoggingMapEntryOperations INSTANCE =
new SimpleLoggingMapEntryOperations();
public static <K, V> MapEntryOperations<K, V, Void> simpleLoggingMapEntryOperations() {
return SimpleLoggingMapEntryOperations.INSTANCE;
}
private SimpleLoggingMapEntryOperations() {}
@Override
public Void remove(@NotNull MapEntry<K, V> entry) {
System.out.println("remove " + entry.key() + ": " + entry.value());
entry.doRemove();
return null;
}
@Override
public Void replaceValue(@NotNull MapEntry<K, V> entry, Data<V, ?> newValue) {
System.out.println("replace " + entry.key() + ": " + entry.value() + " -> " + newValue);
entry.doReplaceValue(newValue);
return null;
}
@Override
public Void insert(@NotNull MapAbsentEntry<K, V> absentEntry, Data<V, ?> value) {
System.out.println("insert " + absentEntry.absentKey() + " -> " + value);
absentEntry.doInsert(value);
return null;
}
@Override
public Data<V, ?> defaultValue(@NotNull MapAbsentEntry<K, V> absentEntry) {
Data<V, ?> defaultValue = absentEntry.defaultValue();
System.out.println("default " + absentEntry.absentKey() + " -> " + defaultValue);
return defaultValue;
}
}
@Test
@Ignore("Example test, run manually to see the output")
public void simpleLoggingTest() {
ChronicleMap<Integer, IntValue> map = ChronicleMapBuilder
.of(Integer.class, IntValue.class)
.entries(100)
.entryOperations(simpleLoggingMapEntryOperations())
.create();
IntValue value = DataValueClasses.newDirectInstance(IntValue.class);
value.setValue(2);
map.put(1, value);
map.remove(1);
map.acquireUsing(3, value).addAtomicValue(1);
IntValue value2 = DataValueClasses.newDirectInstance(IntValue.class);
value2.setValue(5);
map.forEachEntry(c -> c.put(value2));
map.forEachEntry(c -> c.remove());
}
}