Fixed several replication bugs; Value -> Data; extracted ReplicableEntry as a superclass of HashReplicableEntry

This commit is contained in:
leventov
2015-06-15 21:34:45 +03:00
parent f1da352a5d
commit bf20bc98c3
55 changed files with 6708 additions and 6379 deletions
@@ -14,7 +14,6 @@ import java.util.concurrent.TimeUnit;
/**
* Created by Vanitha on 12/5/2014.
*/
@Ignore("a test written by someone on the google groups")
public class TestNesterInterfaceMarshal {
@@ -27,87 +26,62 @@ public class TestNesterInterfaceMarshal {
@Test
public void testReplicatedMap() throws Exception {
ChronicleMap chronicleMap = null;
TcpTransportAndNetworkConfig tcpTransportAndNetworkConfig =
TcpTransportAndNetworkConfig.of(8076).heartBeatInterval(1L, TimeUnit.SECONDS);
try {
ChronicleMapBuilder builder = ChronicleMapBuilder
.of(CharSequence.class, TestInstrumentVOInterface.class)
.entries(5000L)
.averageKeySize(10)
.replication((byte) 1, tcpTransportAndNetworkConfig);
chronicleMap = builder.create();
} catch (Exception e) {
System.out.println("Error(s) creating instrument cache: " + e);
}
ChronicleMap<CharSequence, TestInstrumentVOInterface> map = chronicleMap;
ChronicleMap chronicleMap1 = null;
TcpTransportAndNetworkConfig tcpTransportAndNetworkConfig1 =
TcpTransportAndNetworkConfig.of(8077, new InetSocketAddress("127.0.0.1", 8076))
.heartBeatInterval(1L, TimeUnit.SECONDS);
try {
ChronicleMapBuilder builder = ChronicleMapBuilder
.of(CharSequence.class, TestInstrumentVOInterface.class)
.putReturnsNull(true)
.removeReturnsNull(true)
.entries(5000L).averageKeySize(10)
.replication((byte) 2, tcpTransportAndNetworkConfig1);
try (ChronicleMap<CharSequence, TestInstrumentVOInterface> map =
ChronicleMapBuilder
.of(CharSequence.class, TestInstrumentVOInterface.class)
.entries(5000L)
.averageKeySize(10)
.replication((byte) 1, tcpTransportAndNetworkConfig).create();
ChronicleMap<CharSequence, TestInstrumentVOInterface> replicatedMap =
ChronicleMapBuilder
.of(CharSequence.class, TestInstrumentVOInterface.class)
.putReturnsNull(true)
.removeReturnsNull(true)
.entries(5000L).averageKeySize(10)
.replication((byte) 2, tcpTransportAndNetworkConfig1).create()) {
//Store some data into MAP1
TestInstrumentVOInterface intrumentVOInterface = map.newValueInstance();
chronicleMap1 = builder.create();
try (MapKeyContext<?, TestInstrumentVOInterface> wc = map.acquireContext
("KEY1", intrumentVOInterface)) {
intrumentVOInterface.setSymbol("Flyer");
intrumentVOInterface.setCurrencyCode("USA");
intrumentVOInterface.setSizeOfInstrumentIDArray(2);
} catch (Exception e) {
System.out.println("*********************Error(s) creating launcher " +
"instrument cache: " + e);
// TestIntrumentVOInterface.TestInstrumentIDVOInterface instrumentIDVOInterface =
intrumentVOInterface.getInstrumentIDAt(0);
// instrumentIDVOInterface.setIdSource("CUSIP");
// instrumentIDVOInterface.setSecurityId("TEST");
// intrumentVOInterface.setInstrumentIDAt(0, instrumentIDVOInterface);
// Assert.assertNotNull(map);
// Assert.assertNotNull(replicatedMap);
}
int t = 0;
for (; t < 5000; t++) {
if (map.equals(replicatedMap))
break;
Thread.sleep(1);
}
Assert.assertEquals(map, replicatedMap);
Assert.assertTrue(!replicatedMap.isEmpty());
System.out.println(replicatedMap.get("KEY1"));
}
ChronicleMap<CharSequence, TestInstrumentVOInterface> replicatedMap = chronicleMap1;
//Store some data into MAP1
TestInstrumentVOInterface intrumentVOInterface = map.newValueInstance();
try (MapKeyContext<?, TestInstrumentVOInterface> wc = map.acquireContext
("KEY1", intrumentVOInterface)) {
intrumentVOInterface.setSymbol("Flyer");
intrumentVOInterface.setCurrencyCode("USA");
intrumentVOInterface.setSizeOfInstrumentIDArray(2);
// TestIntrumentVOInterface.TestInstrumentIDVOInterface instrumentIDVOInterface =
intrumentVOInterface.getInstrumentIDAt(0);
// instrumentIDVOInterface.setIdSource("CUSIP");
// instrumentIDVOInterface.setSecurityId("TEST");
// intrumentVOInterface.setInstrumentIDAt(0, instrumentIDVOInterface);
// Assert.assertNotNull(map);
// Assert.assertNotNull(replicatedMap);
}
int t = 0;
for (; t < 5000; t++) {
if (map.equals(replicatedMap))
break;
Thread.sleep(1);
}
Assert.assertEquals(map, replicatedMap);
Assert.assertTrue(!replicatedMap.isEmpty());
System.out.println(((TestInstrumentVOInterface.TestInstrumentIDVOInterface) map.get
("KEY1")).getIdSource());
System.out.println(replicatedMap.get("KEY1"));
map.close();
replicatedMap.close();
}
protected ChronicleMap createReplicatedMap1() {
@@ -1,6 +1,6 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.Value;
import net.openhft.chronicle.hash.Data;
import org.jetbrains.annotations.NotNull;
import org.junit.Test;
@@ -11,7 +11,7 @@ import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.BiFunction;
import static net.openhft.chronicle.hash.Value.bytesEquivalent;
import static net.openhft.chronicle.hash.Data.bytesEquivalent;
import static net.openhft.chronicle.map.BiMapTest.DualLockSuccess.FAIL;
import static net.openhft.chronicle.map.BiMapTest.DualLockSuccess.SUCCESS;
import static org.junit.Assert.assertEquals;
@@ -40,7 +40,7 @@ public class BiMapTest {
}
@Override
public void put(MapQueryContext<K, V, DualLockSuccess> q, Value<V, ?> value,
public void put(MapQueryContext<K, V, DualLockSuccess> q, Data<V, ?> value,
ReturnValue<V> returnValue) {
while (true) {
q.updateLock().lock();
@@ -59,7 +59,7 @@ public class BiMapTest {
}
@Override
public void putIfAbsent(MapQueryContext<K, V, DualLockSuccess> q, Value<V, ?> value,
public void putIfAbsent(MapQueryContext<K, V, DualLockSuccess> q, Data<V, ?> value,
ReturnValue<V> returnValue) {
while (true) {
try {
@@ -88,7 +88,7 @@ public class BiMapTest {
}
@Override
public boolean remove(MapQueryContext<K, V, DualLockSuccess> q, Value<V, ?> value) {
public boolean remove(MapQueryContext<K, V, DualLockSuccess> q, net.openhft.chronicle.hash.Data<V, ?> value) {
while (true) {
q.updateLock().lock();
MapEntry<K, V> entry = q.entry();
@@ -116,14 +116,14 @@ public class BiMapTest {
}
@Override
public void replace(MapQueryContext<K, V, DualLockSuccess> q, Value<V, ?> value,
public void replace(MapQueryContext<K, V, DualLockSuccess> q, Data<V, ?> value,
ReturnValue<V> returnValue) {
throw new UnsupportedOperationException();
}
@Override
public boolean replace(MapQueryContext<K, V, DualLockSuccess> q, Value<V, ?> oldValue,
Value<V, ?> newValue) {
public boolean replace(MapQueryContext<K, V, DualLockSuccess> q, Data<V, ?> oldValue,
Data<V, ?> newValue) {
throw new UnsupportedOperationException();
}
@@ -135,7 +135,7 @@ public class BiMapTest {
}
@Override
public void merge(MapQueryContext<K, V, DualLockSuccess> q, Value<V, ?> value,
public void merge(MapQueryContext<K, V, DualLockSuccess> q, Data<V, ?> value,
BiFunction<? super V, ? super V, ? extends V> remappingFunction,
ReturnValue<V> returnValue) {
throw new UnsupportedOperationException();
@@ -171,13 +171,13 @@ public class BiMapTest {
}
@Override
public DualLockSuccess replaceValue(@NotNull MapEntry<K, V> entry, Value<V, ?> newValue) {
public DualLockSuccess replaceValue(@NotNull MapEntry<K, V> entry, Data<V, ?> newValue) {
throw new UnsupportedOperationException();
}
@Override
public DualLockSuccess insert(@NotNull MapAbsentEntry<K, V> absentEntry,
Value<V, ?> value) {
Data<V, ?> value) {
try (ExternalMapQueryContext<V, K, ?> rq = reverse.queryContext(value)) {
if (!rq.updateLock().tryLock())
return FAIL;
@@ -187,7 +187,7 @@ public class BiMapTest {
reverseAbsentEntry.doInsert(absentEntry.absentKey());
return SUCCESS;
} else {
Value<K, ?> reverseKey = rq.entry().value();
Data<K, ?> reverseKey = rq.entry().value();
if (reverseKey.equals(absentEntry.absentKey())) {
// recover
absentEntry.doInsert(value);
@@ -0,0 +1,117 @@
/*
* Copyright 2014 Higher Frequency Trading http://www.higherfrequencytrading.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.openhft.chronicle.map;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.LockSupport;
/**
* Created by Borislav Ivanov on 5/29/15.
*/
public class ChronicleMapSanityCheckTest {
enum DummyValue {
DUMMY_VALUE
}
@Test
public void testSanity1() throws IOException, InterruptedException
{
String tmp = System.getProperty("java.io.tmpdir");
String pathname = tmp + "/testSanity1-" + UUID.randomUUID().toString() + ".dat";
File file = new File(pathname);
System.out.println("Starting sanity test 1. Chronicle file :" +
file.getAbsolutePath().toString());
ScheduledExecutorService producerExecutor =
Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() - 1);
ScheduledExecutorService consumerExecutor =
Executors.newSingleThreadScheduledExecutor();
int N = 1000;
int producerPeriod = 100;
TimeUnit producerTimeUnit = TimeUnit.MILLISECONDS;
int consumerPeriod = 100;
TimeUnit consumerTimeUnit = TimeUnit.MILLISECONDS;
int totalTestTimeMS = (consumerPeriod + producerPeriod) * 20;
try (ChronicleMap<String, DummyValue> map =
ChronicleMapBuilder.of(String.class, DummyValue.class)
.entries(N)
.createPersistedTo(file)) {
map.clear();
producerExecutor.scheduleAtFixedRate(() -> {
Thread.currentThread().setName("Producer " + Thread.currentThread().getId());
Random r = new Random();
System.out.println("Before PRODUCING size is " + map.size());
for (int i = 0; i < N; i++) {
LockSupport.parkNanos(r.nextInt(5));
map.put(String.valueOf(i), DummyValue.DUMMY_VALUE);
}
System.out.println("After PRODUCING size is " + map.size());
}, 0, producerPeriod, producerTimeUnit);
consumerExecutor.scheduleAtFixedRate(() -> {
Thread.currentThread().setName("Consumer");
Set<String> keys = map.keySet();
Random r = new Random();
System.out.println("Before CONSUMING size is " + map.size());
System.out.println();
for (String key : keys) {
if (r.nextBoolean()) {
map.remove(key);
}
}
System.out.println("After CONSUMING size is " + map.size());
}, 0, consumerPeriod, consumerTimeUnit);
Thread.sleep(totalTestTimeMS);
consumerExecutor.shutdown();
producerExecutor.shutdown();
}
}
}
@@ -210,7 +210,6 @@ public class ChronicleMapTest {
}
}
@Ignore("HCOLL-265 Chronicle Maps with Identical byte[] & char[] values are not equal")
@Test
public void testEqualsCharArray() throws Exception {
@@ -233,7 +232,6 @@ public class ChronicleMapTest {
}
@Ignore("HCOLL-265 Chronicle Maps with Identical byte[] & char[] values are not equal")
@Test
public void testEqualsByteArray() throws Exception {
@@ -18,7 +18,6 @@
package net.openhft.chronicle.map;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@@ -26,7 +26,7 @@ import com.google.common.collect.testing.features.CollectionSize;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import net.openhft.chronicle.hash.Value;
import net.openhft.chronicle.hash.Data;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
@@ -141,7 +141,7 @@ public class GuavaTest extends TestCase {
@Override
public Void replaceValue(@NotNull MapEntry<CharSequence, CharSequence> entry,
Value<CharSequence, ?> newValue) {
net.openhft.chronicle.hash.Data<CharSequence, ?> newValue) {
Assert.assertEquals(m, entry.context().map());
m.put(entry.key().get().toString(), newValue.get().toString());
return MapEntryOperations.super.replaceValue(entry, newValue);
@@ -149,7 +149,7 @@ public class GuavaTest extends TestCase {
@Override
public Void insert(@NotNull MapAbsentEntry<CharSequence, CharSequence> absentEntry,
Value<CharSequence, ?> value) {
Data<CharSequence, ?> value) {
Assert.assertEquals(m, absentEntry.context().map());
m.put(absentEntry.absentKey().get().toString(), value.get().toString());
return MapEntryOperations.super.insert(absentEntry, value);
@@ -1,6 +1,6 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.Value;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.map.impl.NullReturnValue;
import org.jetbrains.annotations.NotNull;
import org.junit.Test;
@@ -23,13 +23,13 @@ public class ListenersTest {
return MapEntryOperations.super.remove(entry);
}
@Override
public Void replaceValue(@NotNull MapEntry<K, V> entry, Value<V, ?> newValue) {
public Void replaceValue(@NotNull MapEntry<K, V> entry, Data<V, ?> newValue) {
replaceValueCount.incrementAndGet();
return MapEntryOperations.super.replaceValue(entry, newValue);
}
@Override
public Void insert(@NotNull MapAbsentEntry<K, V> absentEntry, Value<V, ?> value) {
public Void insert(@NotNull MapAbsentEntry<K, V> absentEntry, Data<V, ?> value) {
insertCount.incrementAndGet();
return MapEntryOperations.super.insert(absentEntry, value);
}
@@ -152,7 +152,7 @@ public class ListenersTest {
.mapMethods(new MapMethods<Integer, Integer, Void>() {
@Override
public void put(MapQueryContext<Integer, Integer, Void> q,
Value<Integer, ?> value,
net.openhft.chronicle.hash.Data<Integer, ?> value,
ReturnValue<Integer> returnValue) {
if (q.queriedKey().get() == 2) {
MapMethods.super.put(q, value, NullReturnValue.get());
@@ -18,7 +18,7 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.Value;
import net.openhft.chronicle.hash.Data;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,7 +28,7 @@ class LoggingMapEntryOperations<K, V> implements MapEntryOperations<K, V, Void>
static final Logger LOG = LoggerFactory.getLogger(LoggingMapEntryOperations.class);
@Override
public Void replaceValue(@NotNull MapEntry<K, V> entry, Value<V, ?> newValue) {
public Void replaceValue(@NotNull MapEntry<K, V> entry, Data<V, ?> newValue) {
LOG.info("replace: old key: {}, old value: {}, new value: {}",
entry.key(), entry.value(), newValue);
entry.doReplaceValue(newValue);
@@ -19,7 +19,7 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.AcceptanceDecision;
import net.openhft.chronicle.hash.Value;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.map.replication.MapRemoteOperations;
import net.openhft.chronicle.map.replication.MapRemoteQueryContext;
import net.openhft.chronicle.map.replication.MapReplicableEntry;
@@ -32,7 +32,7 @@ public class MaximizingValueRemoteOperations<K, V extends Comparable<? super V>,
implements MapRemoteOperations<K, V, R> {
@Override
public AcceptanceDecision put(MapRemoteQueryContext<K, V, R> q, Value<V, ?> newValue) {
public AcceptanceDecision put(MapRemoteQueryContext<K, V, R> q, Data<V, ?> newValue) {
MapReplicableEntry<K, V> entry = q.entry();
if (entry != null) {
int compareResult = newValue.get().compareTo(entry.value().get());
@@ -18,7 +18,6 @@
package net.openhft.chronicle.map;
import net.openhft.chronicle.hash.Value;
import net.openhft.chronicle.hash.function.SerializableFunction;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
@@ -295,7 +294,7 @@ public class ReplicationCheckingMap<K, V> implements ChronicleMap<K, V> {
@NotNull
@Override
public ExternalMapQueryContext<K, V, ?> queryContext(Value<K, ?> key) {
public ExternalMapQueryContext<K, V, ?> queryContext(net.openhft.chronicle.hash.Data<K, ?> key) {
throw new UnsupportedOperationException();
}
@@ -137,7 +137,6 @@ public class TCPSocketReplicationTest {
waitTillEqual(5000);
assertEquals(map1, map2);
assertTrue(!map1.isEmpty());
}
@Test
@@ -185,6 +184,7 @@ public class TCPSocketReplicationTest {
Map map1UnChanged = new HashMap();
Map map2UnChanged = new HashMap();
long startTime = System.currentTimeMillis();
int numberOfTimesTheSame = 0;
for (int t = 0; t < timeOutMs + 100; t++) {
if (map1.equals(map2)) {
@@ -196,13 +196,15 @@ public class TCPSocketReplicationTest {
map2UnChanged = new HashMap(map2);
}
Thread.sleep(1);
if (numberOfTimesTheSame == 100) {
if (numberOfTimesTheSame == 10) {
System.out.println("same");
break;
}
}
Thread.sleep(1);
if (System.currentTimeMillis() - startTime > timeOutMs)
break;
}
}
@@ -141,6 +141,7 @@ public class TcpReplicationSoakTest {
Map map2UnChanged = new HashMap();
int numberOfTimesTheSame = 0;
long startTime = System.currentTimeMillis();
for (int t = 0; t < timeOutMs + 100; t++) {
if (map1.equals(map2)) {
if (map1.equals(map1UnChanged) && map2.equals(map2UnChanged)) {
@@ -151,13 +152,15 @@ public class TcpReplicationSoakTest {
map2UnChanged = new HashMap(map2);
}
Thread.sleep(1);
if (numberOfTimesTheSame == 100) {
if (numberOfTimesTheSame == 10) {
System.out.println("same");
break;
}
}
Thread.sleep(1);
if (System.currentTimeMillis() - startTime > timeOutMs)
break;
}
}
}
@@ -99,7 +99,7 @@ public class TimeBasedReplicationTest extends JSR166TestCase {
waitTillEqual(map, map2, 5000);
assertEquals(map.size(), 1);
// prove that late update to map2 ignored
assertEquals(map.get("key-1"), "value-1");
assertEquals("value-1", map.get("key-1"));
}
}
}
@@ -218,7 +218,7 @@ public class TimeBasedReplicationTest extends JSR166TestCase {
// we'll now flip the time back to the current in order to do the read the result
current(timeProvider);
waitTillEqual(map, map2, 5000);
assertEquals(map.size(), 0);
assertEquals("map: " + map + ", map2: " + map2, 0, map.size());
// prove that late put was ignored
assertEquals(null, map.get("key-1"));
}
@@ -231,6 +231,6 @@ public class TimeBasedReplicationTest extends JSR166TestCase {
private void late(TimeProvider timeProvider) {
Mockito.when(timeProvider.currentTime())
.thenReturn(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(5));
.thenReturn(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(10));
}
}
@@ -80,7 +80,7 @@ public class UDPSocketReplicationTest {
public void testBufferOverflow() throws IOException, InterruptedException {
for (int i = 0; i < 1024; i++) {
Thread.sleep(5000);
Thread.sleep(5);
map2.put(i * 2, "E");
System.out.println("" + map2);
}
@@ -18,7 +18,7 @@
package net.openhft.chronicle.map.jsr166.map;
import net.openhft.chronicle.hash.Value;
import net.openhft.chronicle.hash.Data;
import net.openhft.chronicle.hash.function.SerializableFunction;
import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig;
import net.openhft.chronicle.map.*;
@@ -123,7 +123,7 @@ public class StatelessChronicleMapTest extends JSR166TestCase {
@NotNull
@Override
public ExternalMapQueryContext<K, V, ?> queryContext(Value<K, ?> key) {
public ExternalMapQueryContext<K, V, ?> queryContext(Data<K, ?> key) {
return d.queryContext(key);
}
@@ -39,7 +39,6 @@ import static org.junit.Assert.assertTrue;
*
* @author Rob Austin.
*/
@Ignore
public class SetForTCPSocketReplication3wayTest {
private ChronicleSet<Integer> set1;
@@ -68,6 +68,7 @@ public class DoubleArrayTest {
}
@Test
@Ignore
public void addToAMap2() {
DoubleArray.HACK = false;
DoubleArray a = new DoubleArray(10);
@@ -76,7 +77,8 @@ public class DoubleArrayTest {
DoubleArray b = new DoubleArray(10);
b.setData(new double[]{5, 6, 7, 8, 9});
ChronicleMap<Integer, DoubleArray> proxyMap = ChronicleMapBuilder.of(Integer.class, DoubleArray.class)
ChronicleMap<Integer, DoubleArray> proxyMap = ChronicleMapBuilder
.of(Integer.class, DoubleArray.class)
.averageValueSize(6 * 8)
.create();
proxyMap.put(1, a);