diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/BasicCopyingMetaBytesInterop.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/BasicCopyingMetaBytesInterop.java index 250cd18f8..915038128 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/BasicCopyingMetaBytesInterop.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/BasicCopyingMetaBytesInterop.java @@ -63,6 +63,14 @@ public abstract class BasicCopyingMetaBytesInterop implements MetaBytesInt return h; } + @Override + public boolean isEqual(W write, Bytes bytes, E e, long size) { + if (buffer.buffer.remaining() != size) + return false; + + return bytes.compare(bytes.position(), buffer.buffer, buffer.buffer.position(), size); + } + @Override public void write(W writer, Bytes bytes, E e) { bytes.write(buffer.buffer, buffer.buffer.position(), buffer.buffer.remaining()); diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java index c7c740f20..a0de0ad52 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteBufferMarshaller.java @@ -31,22 +31,22 @@ public enum ByteBufferMarshaller implements BytesInterop, BytesReade @Override public boolean startsWith(Bytes bytes, ByteBuffer bb) { int inputRemaining = bb.remaining(); - if(bytes.capacity() - bytes.position() < (long) inputRemaining) + if (bytes.capacity() - bytes.position() < (long) inputRemaining) return false; long pos = bytes.position(); int inputPos = bb.position(); int i; - for(i = 0; i < inputRemaining - 7; i += 8) { + for (i = 0; i < inputRemaining - 7; i += 8) { if (bytes.readLong(pos + (long) i) != bb.getLong(inputPos + i)) return false; } - for(; i < inputRemaining - 3; i += 4) { + for (; i < inputRemaining - 3; i += 4) { if (bytes.readInt(pos + (long) i) != bb.getInt(inputPos + i)) return false; } - for(; i < inputRemaining; i++) { + for (; i < inputRemaining; i++) { if (bytes.readByte(pos + (long) i) != bb.get(inputPos + i)) return false; } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java index 2c60069c6..817bee388 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/ByteableMarshaller.java @@ -192,6 +192,13 @@ public abstract class ByteableMarshaller super(tClass); initSize(); } + + + @Override + public boolean isEqual(Object write, Bytes bytes, E e, long size) { + // todo + return false; + } } private static class WithCustomFactory extends ByteableMarshaller { @@ -201,7 +208,7 @@ public abstract class ByteableMarshaller private final ObjectFactory factory; WithCustomFactory(@NotNull Class tClass, - @NotNull ObjectFactory factory) { + @NotNull ObjectFactory factory) { super(tClass); this.factory = factory; initSize(); @@ -212,5 +219,12 @@ public abstract class ByteableMarshaller E getInstance() throws Exception { return factory.create(); } + + + @Override + public boolean isEqual(Object write, Bytes bytes, E e, long size) { + // todo + return false; + } } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java index 2aad8beb1..6e0663045 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java @@ -52,6 +52,12 @@ public final class DelegatingMetaBytesInterop> interop.write(bytes, e); } + @Override + public boolean isEqual(I interop, Bytes bytes, E e, long size) { + // todo + return false; + } + private Object readResolve() { return INSTANCE; } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DirectBytesBuffer.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DirectBytesBuffer.java index 90e70d894..649230255 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DirectBytesBuffer.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DirectBytesBuffer.java @@ -136,5 +136,10 @@ class DirectBytesBuffer private ForDataValueWriter() { super(DirectBytesBuffer.this); } + + @Override + public boolean isEqual(BytesWriter write, Bytes bytes, E e, long size) { + return false; + } } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaBytesWriter.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaBytesWriter.java index b9214320c..29575d2ed 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaBytesWriter.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/MetaBytesWriter.java @@ -24,4 +24,16 @@ public interface MetaBytesWriter extends Serializable { long size(W writer, E e); void write(W writer, Bytes bytes, E e); + + + /** + * returns true if the {@code bytes} equals {@code e} + * + * @param write + * @param bytes + * @param e + * @param size + * @return + */ + boolean isEqual(W write, Bytes bytes, E e, long size); } diff --git a/src/main/java/net/openhft/chronicle/map/MapEventListener.java b/src/main/java/net/openhft/chronicle/map/MapEventListener.java index 12be02611..ec05109db 100644 --- a/src/main/java/net/openhft/chronicle/map/MapEventListener.java +++ b/src/main/java/net/openhft/chronicle/map/MapEventListener.java @@ -17,7 +17,6 @@ package net.openhft.chronicle.map; import net.openhft.chronicle.hash.ChronicleHashBuilder; -import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,10 +63,43 @@ public abstract class MapEventListener implements Serializable { return new LoggingMapEventListener(prefix); } + /** + * This method is called whenever a new value is put for the key in the map during calls of such + * methods as {@link ChronicleMap#put put}, {@link ChronicleMap#putIfAbsent putIfAbsent}, {@link + * ChronicleMap#replace(Object, Object, Object) replace}, etc. When a new value is {@linkplain + * ChronicleMapBuilder#defaultValue(Object) default} for the map or obtained during {@link + * ChronicleMap#acquireUsing acquireUsing} call is put for the key, this method is called as + * well. + * + *

This method is called when put is already happened. + * + * @param key the key the given value is put for + * @param newValue the value which is now associated with the given key + * @param replacedValue the value which was replaced by {@code newValue}, {@code null} if + * the key was absent in the map before current {@code ChronicleMap} + * @param hasValueChanged {@code true} if the {@code newValue } equals {@code replacedValue + * } + * @param identifier the identifer used for replicaiton or zero if not definded + * @param replacedIdentifier the replaced identifer used for replicaiton or zero if not + * definded + * @param timeStamp the timestamp used for replicaiton or zero if not definded + * @param replacedTimeStamp the replaced timestamp used for replicaiton or zero if not + * definded + */ + public void onPut(K key, + V newValue, + @Nullable V replacedValue, + boolean replicationEvent, + boolean added, + boolean hasValueChanged, + byte identifier, + byte replacedIdentifier, long timeStamp, long replacedTimeStamp) { + + } + private static class LoggingMapEventListener extends MapEventListener { private static final long serialVersionUID = 0L; public final static Logger LOGGER = LoggerFactory.getLogger(LoggingMapEventListener.class); - private final String prefix; private LoggingMapEventListener(String prefix) { @@ -81,8 +113,8 @@ public abstract class MapEventListener implements Serializable { @Override public void onPut(Object key, Object value, Object replacedValue, boolean - replicationEvent, - @NotNull boolean updateResult) { + replicationEvent, boolean updateResult, boolean hasValueChanged, byte identifier, + byte replacedIdentifier, long timeStamp, long replacedTimeStamp) { LOGGER.info("{} put {} => {}", prefix, key, value); } @@ -114,18 +146,24 @@ public abstract class MapEventListener implements Serializable { * *

This method is called when put is already happened. * - * @param key the key the given value is put for - * @param newValue the value which is now associated with the given key - * @param replacedValue the value which was replaced by {@code newValue}, {@code null} if the - * key was absent in the map before current {@code ChronicleMap} + * @param key the key the given value is put for + * @param newValue the value which is now associated with the given key + * @param replacedValue the value which was replaced by {@code newValue}, {@code null} + * if the key was absent in the map before current {@code + * ChronicleMap} * @param replicationEvent + * @param lastModifiedTimeStamp + * @param lastId + * @param timestamp + * @param identifier */ public void onPut(K key, V newValue, @Nullable V replacedValue, boolean replicationEvent, - boolean added) { - // do nothing + boolean added, long lastModifiedTimeStamp, byte lastId, long timestamp, + byte identifier, boolean hasValueChanged) { + } /** diff --git a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java deleted file mode 100644 index e7205ad10..000000000 --- a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java +++ /dev/null @@ -1,1080 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.RemoteNodeValidator; -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import net.openhft.chronicle.hash.replication.UdpTransportConfig; -import net.openhft.lang.collection.ATSDirectBitSet; -import net.openhft.lang.collection.DirectBitSet; -import net.openhft.lang.io.ByteBufferBytes; -import net.openhft.lang.io.Bytes; -import net.openhft.lang.io.serialization.BytesMarshallable; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.*; -import java.nio.ByteBuffer; -import java.nio.channels.DatagramChannel; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import static java.nio.ByteBuffer.allocateDirect; -import static java.util.concurrent.TimeUnit.SECONDS; -import static net.openhft.chronicle.map.ConcurrentExpiryMap.getDefaultAddress; -import static net.openhft.chronicle.map.DiscoveryNodeBytesMarshallable.ProposedNodes; -import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.BOOTSTRAP_BYTES; -import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.LOG; - -interface NodeDiscoveryEventListener { - /** - * called when we have received a UDP message, this is called after the message has been parsed - */ - void onRemoteNodeEvent(KnownNodes remoteNode, ConcurrentExpiryMap proposedIdentifiersWithHost); -} - -/** - * @author Rob Austin. - * - * This code is in alpha, and still has to undergo refactoring - */ -class NodeDiscovery { - - public static final int DEFAULT_UDP_BROADCAST_PORT = 8129; - public static final short DEFAULT_TCP_PORT = (short) 8123; - public static final int SERIALIZED_ENTRY_SIZE = 1024; - public static final InetAddress IP_MULIT_CAST_GROUP; - - static { - InetAddress mask = null; - try { - mask = Inet4Address.getByName("224.0.8.0"); - } catch (Exception e) { - mask = null; - } - IP_MULIT_CAST_GROUP = mask; - } - - private final AddressAndPort ourAddressAndPort; - private final UdpTransportConfig udpConfig; - private final DiscoveryNodeBytesMarshallable discoveryNodeBytesMarshallable; - private final AtomicReference nodeDiscoveryEventListenerAtomicReference = - new AtomicReference(); - - private KnownNodes knownNodes; - - public NodeDiscovery() throws IOException { - this((short) DEFAULT_UDP_BROADCAST_PORT, DEFAULT_TCP_PORT, getDefaultAddress(), IP_MULIT_CAST_GROUP); - } - - public NodeDiscovery(short udpBroadcastPort, - short tcpPort, - @NotNull InetAddress tcpAddress, - @NotNull InetAddress udpMultiCastGroup) throws IOException { - this.ourAddressAndPort = new AddressAndPort(tcpAddress.getAddress(), tcpPort); - this.udpConfig = UdpTransportConfig.multiCast(udpMultiCastGroup, udpBroadcastPort, - ConcurrentExpiryMap.defaultNetworkInterface()); - knownNodes = new KnownNodes(); - - discoveryNodeBytesMarshallable = new DiscoveryNodeBytesMarshallable - (knownNodes, nodeDiscoveryEventListenerAtomicReference, ourAddressAndPort); - - final NodeDiscoveryBroadcaster nodeDiscoveryBroadcaster - = new NodeDiscoveryBroadcaster(udpConfig, SERIALIZED_ENTRY_SIZE, discoveryNodeBytesMarshallable); - - discoveryNodeBytesMarshallable.setModificationNotifier(nodeDiscoveryBroadcaster); - } - - private static Collection toInetSocketCollection(Set source) - throws UnknownHostException { - - // make a safe copy - final HashSet addressAndPorts = new HashSet(source); - - if (addressAndPorts.isEmpty()) - return Collections.emptyList(); - - ArrayList addresses = - new ArrayList(addressAndPorts.size()); - - for (final AddressAndPort addressAndPort : addressAndPorts) { - addresses.add(new InetSocketAddress(InetAddress.getByAddress(addressAndPort.address()) - .getHostAddress(), addressAndPort.port())); - } - return addresses; - } - - static byte proposeRandomUnusedIdentifier(final DirectBitSet knownIdentifiers, - boolean isFirstTime) throws UnknownHostException { - byte possible; - - // the first time, rather than choosing a random number, we will choose the last value of the IP - // address as our random number, ( or at least something that is based upon it) - if (isFirstTime) { - byte[] address = InetAddress.getLocalHost().getAddress(); - int lastAddress = address[address.length - 1]; - if (lastAddress > 127) - lastAddress = lastAddress - 127; - if (lastAddress > 127) - lastAddress = lastAddress - 127; - - possible = (byte) lastAddress; - } else - possible = (byte) (Math.random() * 128); - - int count = 0; - for (; ; ) { - if (knownIdentifiers.setIfClear(possible)) { - return possible; - } - - count++; - - if (count == 128) { - throw new IllegalStateException("The grid is full, its not possible for any more nodes to " + - "going the grid."); - } - - if (possible == Byte.MAX_VALUE) - possible = 0; - else - possible++; - } - } - - public synchronized ReplicationHubFindByName mapByName() throws - IOException, InterruptedException { - - final AtomicInteger ourProposedIdentifier = new AtomicInteger(); - final AtomicBoolean useAnotherIdentifier = new AtomicBoolean(); - - final Set knownHostPorts = new ConcurrentSkipListSet(); - final DirectBitSet knownAndProposedIdentifiers = new ATSDirectBitSet(new ByteBufferBytes(ByteBuffer.allocate - (128 / 8))); - - final AtomicReference countDownLatch - = new AtomicReference(new CountDownLatch(1)); - - final NodeDiscoveryEventListener nodeDiscoveryEventListener = new NodeDiscoveryEventListener() { - @Override - public void onRemoteNodeEvent(@NotNull final KnownNodes remoteNodes, - @NotNull final ConcurrentExpiryMap proposedIdentifiersWithHost) { - LOG.info("onRemoteNodeEvent " + remoteNodes + ", proposedIdentifiersWithHost=" + proposedIdentifiersWithHost); - - knownHostPorts.addAll(remoteNodes.addressAndPorts()); - - orBitSets(remoteNodes.identifiers(), knownAndProposedIdentifiers); - - for (ProposedNodes proposedIdentifierWithHost : - proposedIdentifiersWithHost.values()) { - if (!proposedIdentifierWithHost.addressAndPort().equals(ourAddressAndPort)) { - int proposedIdentifier = proposedIdentifierWithHost.identifier(); - if (proposedIdentifier != -1) { - knownAndProposedIdentifiers.set(proposedIdentifier, true); - } - - knownHostPorts.add(proposedIdentifierWithHost.addressAndPort()); - - if (proposedIdentifier == ourProposedIdentifier.get()) - useAnotherIdentifier.set(true); - } - } - countDownLatch.get().countDown(); - } - }; - - nodeDiscoveryEventListenerAtomicReference.set(nodeDiscoveryEventListener); - - final ProposedNodes ourHostPort = new - ProposedNodes(ourAddressAndPort, (byte) -1); - - // to start with we will send a bootstrap that just contains our hostname without and identifier - - for (int i = 0; i < 10; i++) { - discoveryNodeBytesMarshallable.sendBootStrap(ourHostPort); - - // once the count down latch is trigger we know we go something back from one of the nodes - if (countDownLatch.get().await(i * 20, TimeUnit.MILLISECONDS)) - break; - } - - // we should now get back some identifiers - // the identifiers will come back to the callback on the nio thread, the update arrives at the - // onRemoteNodeEvent - - Thread.sleep(1000); - - byte identifier; - - // now we are going to propose an identifier - boolean isFistTime = true; - - OUTER: - for (; ; ) { - useAnotherIdentifier.set(false); - identifier = proposeRandomUnusedIdentifier(knownAndProposedIdentifiers, isFistTime); - ourProposedIdentifier.set(identifier); - LOG.info("proposing to use identifier=" + identifier); - - isFistTime = false; - - final ProposedNodes proposedNodes = new - ProposedNodes(ourAddressAndPort, identifier); - - Thread.sleep(1000); - - countDownLatch.set(new CountDownLatch(1)); - - for (int i = 0; i < 20; i++) { - discoveryNodeBytesMarshallable.sendBootStrap(proposedNodes); - - // once the count down latch is trigger we know we go something back from one of the nodes - if (countDownLatch.get().await(i * 20, TimeUnit.MILLISECONDS)) { - if (useAnotherIdentifier.get()) { - // given that another node host proposed the same identifier, we will choose a different one. - LOG.info("Another node is using identifier=" + identifier + ", " + - "going to have to select another one, we will wait 500ms before continuing"); - continue OUTER; - } else { - break OUTER; - } - } else { - LOG.debug("timed-out getting a response from the server so sending another boot-strap " + - "message"); - } - } - - LOG.info("looks like we are the only node in the grid, so going to use identifier=" + identifier); - - break; - } - - // we should make a local copy as this may change - - final RemoteNodeValidator remoteNodeValidator = new RemoteNodeValidator() { - private final ConcurrentMap identifiers = new ConcurrentHashMap(); - - @Override - public boolean validate(byte remoteIdentifier, SocketAddress remoteAddress) { - final SocketAddress lastKnownAddress = identifiers.putIfAbsent(remoteIdentifier, remoteAddress); - if (remoteAddress.equals(lastKnownAddress)) - return true; - knownNodes.identifiers().set(remoteIdentifier); - return lastKnownAddress == null; - } - }; - - // remove self - final HashSet knownHostPorts0 = new HashSet<>(knownHostPorts); - knownHostPorts0.remove(ourAddressAndPort); - - // add our identifier and host:port to the list of known identifiers - knownNodes.add(ourAddressAndPort, identifier); - - /* final TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig - .of(ourAddressAndPort.getPort(), - toInetSocketCollection(knownHostPorts0)); -*/ - LOG.info("Using Remote identifier=" + identifier); - nodeDiscoveryEventListenerAtomicReference.set(null); - - TcpTransportAndNetworkConfig tcpConfig = TcpTransportAndNetworkConfig.of(8123, new - InetSocketAddress("192.168.1.253", 8123)); - - final ReplicationHub replicationHub = ReplicationHub.builder() - .tcpTransportAndNetwork(tcpConfig) - .remoteNodeValidator(remoteNodeValidator) - .createWithId(identifier); - - return new ReplicationHubFindByName(replicationHub); - } - - /** - * bitwise OR's the two bit sets or put another way, merges the source bitset into the - * destination bitset and returns the destination - * - * @param source to or - * @param destination to or - * @return result. - */ - private DirectBitSet orBitSets(@NotNull final DirectBitSet source, - @NotNull final DirectBitSet destination) { - // merges the two bit-sets together - for (int i = (int) source.nextSetBit(0); i > 0; - i = (int) source.nextSetBit(i + 1)) { - try { - destination.set(i, true); - } catch (IndexOutOfBoundsException e) { - LOG.error("", e); - } - } - - return destination; - } - -} - -/** - * Broadcast the nodes host ports and identifiers over UDP, to make it easy to join a grid of remote - * nodes just by name, this functionality requires UDP - * - * @author Rob Austin. - */ -class NodeDiscoveryBroadcaster extends UdpChannelReplicator { - - public static final Logger LOG = LoggerFactory.getLogger(NodeDiscoveryBroadcaster.class.getName()); - static final ByteBufferBytes BOOTSTRAP_BYTES; - private static final byte UNUSED = (byte) -1; - - static { - final String BOOTSTRAP = "BOOTSTRAP"; - BOOTSTRAP_BYTES = new ByteBufferBytes(ByteBuffer.allocate(2 + BOOTSTRAP.length())); - BOOTSTRAP_BYTES.write((short) BOOTSTRAP.length()); - BOOTSTRAP_BYTES.append(BOOTSTRAP); - } - - /** - * @param replicationConfig - * @param externalizable - * @throws java.io.IOException - */ - NodeDiscoveryBroadcaster( - final UdpTransportConfig replicationConfig, - final int serializedEntrySize, - final BytesMarshallable externalizable) throws IOException { - - super(replicationConfig, UNUSED); - - final UdpSocketChannelEntryWriter writer = new UdpSocketChannelEntryWriter(1024, externalizable, this); - final UdpSocketChannelEntryReader reader = new UdpSocketChannelEntryReader(1024, externalizable); - - setReader(reader); - setWriter(writer); - - start(); - } - - static String toString(DirectBitSet bitSet) { - final StringBuilder builder = new StringBuilder(); - for (int i = 0; i < bitSet.size(); i++) { - builder.append(bitSet.get(i) ? '1' : '0'); - } - return builder.toString(); - } - - static class UdpSocketChannelEntryReader implements EntryReader { - - private final ByteBuffer in; - private final ByteBufferBytes out; - private final BytesMarshallable externalizable; - - /** - * @param serializedEntrySize the maximum size of an entry include the meta data - * @param externalizable supports reading and writing serialize entries - */ - UdpSocketChannelEntryReader(final int serializedEntrySize, - @NotNull final BytesMarshallable externalizable) { - // we make the buffer twice as large just to give ourselves headroom - in = allocateDirect(serializedEntrySize * 2); - - out = new ByteBufferBytes(in); - out.limit(0); - in.clear(); - this.externalizable = externalizable; - } - - /** - * reads entries from the socket till it is empty - * - * @param socketChannel the socketChannel that we will read from - * @throws IOException - * @throws InterruptedException - */ - public void readAll(@NotNull final DatagramChannel socketChannel) throws IOException { - - out.clear(); - in.clear(); - - socketChannel.receive(in); - - final int bytesRead = in.position(); - - if (bytesRead < SIZE_OF_SIZE + SIZE_OF_SIZE) - return; - - out.limit(in.position()); - - final int invertedSize = out.readInt(); - final int size = out.readInt(); - - // check the the first 4 bytes are the inverted len followed by the len - // we do this to check that this is a valid start of entry, otherwise we throw it away - if (~size != invertedSize) - return; - - if (out.remaining() != size) - return; - - externalizable.readMarshallable(out); - } - } - - static class UdpSocketChannelEntryWriter implements EntryWriter { - - private final ByteBuffer out; - private final ByteBufferBytes in; - - @NotNull - private final BytesMarshallable externalizable; - private UdpChannelReplicator udpReplicator; - - UdpSocketChannelEntryWriter(final int serializedEntrySize, - @NotNull final BytesMarshallable externalizable, - @NotNull final UdpChannelReplicator udpReplicator) { - this.externalizable = externalizable; - this.udpReplicator = udpReplicator; - - // we make the buffer twice as large just to give ourselves headroom - out = allocateDirect(serializedEntrySize * 2); - in = new ByteBufferBytes(out); - } - - /** - * @param socketChannel the socketChannel that we will write to - * @throws InterruptedException - * @throws IOException - */ - public int writeAll(@NotNull final DatagramChannel socketChannel) - throws InterruptedException, IOException { - - out.clear(); - in.clear(); - - // skip the size inverted - in.skip(SIZE_OF_SIZE); - - // skip the size - in.skip(SIZE_OF_SIZE); - - long start = in.position(); - - // writes the contents of - externalizable.writeMarshallable(in); - - long size = in.position() - start; - - // we'll write the size inverted at the start - in.writeInt(0, ~((int) size)); - in.writeInt(SIZE_OF_SIZE, (int) size); - - out.limit((int) in.position()); - - udpReplicator.disableWrites(); - - return socketChannel.write(out); - } - } - -} - -class KnownNodes implements BytesMarshallable { - - private Bytes activeIdentifiersBitSetBytes; - private ConcurrentSkipListSet addressAndPorts = new ConcurrentSkipListSet(); - private ATSDirectBitSet atsDirectBitSet; - - KnownNodes() { - this.activeIdentifiersBitSetBytes = new ByteBufferBytes(ByteBuffer.allocate(128 / 8)); - this.addressAndPorts = new ConcurrentSkipListSet(); - this.atsDirectBitSet = new ATSDirectBitSet(this.activeIdentifiersBitSetBytes); - } - - public Set addressAndPorts() { - return addressAndPorts; - } - - public void add(AddressAndPort inetSocketAddress, byte identifier) { - identifiers().set(identifier); - addressAndPorts.add(inetSocketAddress); - } - - /** - * all the known identifiers - * - * @return - */ - public DirectBitSet identifiers() { - return atsDirectBitSet; - } - - @Override - public void readMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes in) throws IllegalStateException { - short size = in.readShort(); - - for (int i = 0; i < size; i++) { - final AddressAndPort addressAndPort = new AddressAndPort(); - addressAndPort.readMarshallable(in); - addressAndPorts.add(addressAndPort); - } - - final ByteBufferBytes activeIdentifiersBitSetBytes = new ByteBufferBytes(ByteBuffer.allocate(128 / 8)); - activeIdentifiersBitSetBytes.readMarshallable(in); - - final ATSDirectBitSet bitset = new ATSDirectBitSet(activeIdentifiersBitSetBytes); - for (long next = bitset.nextSetBit(0); next > 0; next = bitset.nextSetBit(next + 1)) { - atsDirectBitSet.set(next); - } - } - - @Override - public void writeMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes out) { - // make a safe copy - final Set addressAndPorts = new HashSet(this.addressAndPorts); - - // write the size - out.writeShort(addressAndPorts.size()); - - for (AddressAndPort bytesMarshallable : addressAndPorts) { - bytesMarshallable.writeMarshallable(out); - } - - activeIdentifiersBitSetBytes.clear(); - activeIdentifiersBitSetBytes.writeMarshallable(out); - } - - @Override - public String toString() { - return "RemoteNodes{" + - " addressAndPorts=" + addressAndPorts + - ", bitSet=" + NodeDiscoveryBroadcaster.toString(atsDirectBitSet) + - '}'; - } -} - -class AddressAndPort implements Comparable, BytesMarshallable { - final static int INADDRSZ = 16; - final static int INT16SZ = 2; - private byte[] address; - private short port; - - AddressAndPort(byte[] address, short port) { - this.address = address; - this.port = port; - } - - public AddressAndPort() { - } - - static String numericToTextFormat(byte[] src) { - if (src.length == 4) { - return (src[0] & 0xff) + "." + (src[1] & 0xff) + "." + (src[2] & 0xff) + "." + (src[3] & 0xff); - } - - StringBuilder sb = new StringBuilder(39); - for (int i = 0; i < (INADDRSZ / INT16SZ); i++) { - sb.append(Integer.toHexString(((src[i << 1] << 8) & 0xff00) - | (src[(i << 1) + 1] & 0xff))); - if (i < (INADDRSZ / INT16SZ) - 1) { - sb.append(":"); - } - } - - return sb.toString(); - } - - public byte[] getAddress() { - return address; - } - - public short getPort() { - return port; - } - - public byte[] address() { - return address; - } - - public short port() { - return port; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || AddressAndPort.class != o.getClass()) return false; - - AddressAndPort that = (AddressAndPort) o; - - if (port != that.port) return false; - return Arrays.equals(address, that.address); - - } - - @Override - public int hashCode() { - int result = Arrays.hashCode(address); - result = 31 * result + (int) port; - return result; - } - - @Override - public int compareTo(AddressAndPort o) { - int i = 0; - for (byte b : address) { - int compare = Byte.compare(b, o.address[i++]); - if (compare != 0) - return compare; - } - return Short.compare(port, o.port); - } - - @Override - public String toString() { - return "{" + - "address=" + numericToTextFormat(address) + - ", port=" + port + - '}'; - } - - @Override - public void readMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes in) throws IllegalStateException { - short len = in.readShort(); - address = new byte[len]; - - for (int i = 0; i < len; i++) { - address[i] = in.readByte(); - } - port = in.readShort(); - } - - @Override - public void writeMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes out) { - out.writeShort(getAddress().length); - for (byte address : getAddress()) { - out.write(address); - } - out.writeShort(port); - } -} - -class DiscoveryNodeBytesMarshallable implements BytesMarshallable { - - final ConcurrentExpiryMap proposedIdentifiersWithHost = new - ConcurrentExpiryMap(AddressAndPort.class, - ProposedNodes.class); - private final KnownNodes remoteNode; - - private final AtomicBoolean bootstrapRequired = new AtomicBoolean(); - private final AtomicReference nodeDiscoveryEventListener; - Replica.ModificationNotifier modificationNotifier; - private AddressAndPort sourceAddressAndPort = new AddressAndPort(); - private ProposedNodes ourProposedIdentifier; - private AddressAndPort ourAddressAndPort; - - public DiscoveryNodeBytesMarshallable(@NotNull final KnownNodes remoteNode, - @NotNull final AtomicReference - nodeDiscoveryEventListener, - @NotNull final AddressAndPort ourAddressAndPort) { - this.remoteNode = remoteNode; - this.nodeDiscoveryEventListener = nodeDiscoveryEventListener; - this.ourAddressAndPort = ourAddressAndPort; - } - - public ProposedNodes getOurProposedIdentifier() { - return ourProposedIdentifier; - } - - private void setOurProposedIdentifier(ProposedNodes ourProposedIdentifier) { - this.ourProposedIdentifier = ourProposedIdentifier; - } - - public void setModificationNotifier(Replica.ModificationNotifier modificationNotifier) { - this.modificationNotifier = modificationNotifier; - } - - public KnownNodes getRemoteNodes() { - return remoteNode; - } - - /** - * this is used to tell nodes that are connecting to us which host and ports are in our grid, - * along with all the identifiers. - */ - @Override - public void writeMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes out) { - if (bootstrapRequired.getAndSet(false)) { - writeBootstrap(out); - return; - } - - // the host and port the message came from - ourAddressAndPort.writeMarshallable(out); - - remoteNode.writeMarshallable(out); - - // we are no going to broadcast all the nodes that have been bootstaping in the last second - proposedIdentifiersWithHost.expireEntries(System.currentTimeMillis() - SECONDS.toMillis(1)); - - proposedIdentifiersWithHost.writeMarshallable(out); - } - - private boolean writeBootstrap(Bytes out) { - final ProposedNodes ourProposedIdentifier = getOurProposedIdentifier(); - if (ourProposedIdentifier == null || ourProposedIdentifier.addressAndPort == null) - return false; - - BOOTSTRAP_BYTES.clear(); - out.write(BOOTSTRAP_BYTES, BOOTSTRAP_BYTES.position(), BOOTSTRAP_BYTES.remaining()); - - ourProposedIdentifier.writeMarshallable(out); - return true; - } - - /** - * @param in - * @return returns true if the UDP message contains the text 'BOOTSTRAP' - */ - private ProposedNodes readBootstrapMessage(Bytes in) { - final long start = in.position(); - - try { - - long size = BOOTSTRAP_BYTES.limit(); - - if (size > in.remaining()) - return null; - - // reads the text bootstrap - for (int i = 0; i < size; i++) { - final byte byteRead = in.readByte(); - final byte expectedByte = BOOTSTRAP_BYTES.readByte(i); - - if (!(expectedByte == byteRead)) - return null; - } - - final ProposedNodes result = new ProposedNodes(); - result.readMarshallable(in); - - return result; - } finally { - in.position(start); - } - } - - @Override - public void readMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes in) throws IllegalStateException { - - final ProposedNodes bootstrap = readBootstrapMessage(in); - if (bootstrap != null) { - if (bootstrap.addressAndPort().equals(this.ourAddressAndPort)) - return; - - if (LOG.isDebugEnabled()) - LOG.debug("Received Bootstrap from " + bootstrap); - - proposedIdentifiersWithHost.put(bootstrap.addressAndPort, bootstrap); - - // we've received a bootstrap message so will will now rebroadcast what we know, - // after a random delay - // Thread.sleep((int) (Math.random() * 9.0)); - - // this is used to turn on the OP_WRITE, so that we can broadcast back the known host and - // ports in the grid - onChange(); - return; - } - - // the host and port the message came from - this.sourceAddressAndPort.readMarshallable(in); - if (sourceAddressAndPort.equals(ourAddressAndPort)) - return; - - LOG.debug("Received Proposal"); - - this.remoteNode.readMarshallable(in); - this.proposedIdentifiersWithHost.readMarshallable(in); - - NodeDiscoveryEventListener listener = nodeDiscoveryEventListener.get(); - if (listener != null) - listener.onRemoteNodeEvent(remoteNode, proposedIdentifiersWithHost); - } - - public void onChange() { - if (modificationNotifier != null) - modificationNotifier.onChange(); - } - - /** - * sends a bootstrap message to the other nodes in the grid, the bootstrap message contains the - * host:port and perhaps even proposed identifier of the node that sent it. - * - * @param proposedNodes - */ - public void sendBootStrap(ProposedNodes proposedNodes) { - setOurProposedIdentifier(proposedNodes); - bootstrapRequired.set(true); - onChange(); - } - - static class ProposedNodes implements BytesMarshallable { - - private byte identifier; - private long timestamp; - private AddressAndPort addressAndPort; - - public ProposedNodes() { - } - - ProposedNodes(@NotNull final AddressAndPort addressAndPort, - byte identifier) { - this.addressAndPort = addressAndPort; - this.identifier = identifier; - this.timestamp = System.currentTimeMillis(); - } - - public byte identifier() { - return identifier; - } - - AddressAndPort addressAndPort() { - return addressAndPort; - } - - @Override - public void readMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes in) throws IllegalStateException { - addressAndPort = new AddressAndPort(); - addressAndPort.readMarshallable(in); - timestamp = in.readLong(); - identifier = in.readByte(); - } - - @Override - public void writeMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes out) { - addressAndPort.writeMarshallable(out); - out.writeLong(timestamp); - out.writeByte(identifier); - } - - @Override - public String toString() { - return "{" + - "identifier=" + identifier + - ", timestamp=" + timestamp + - ", addressAndPort=" + addressAndPort + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || ProposedNodes.class != o.getClass()) return false; - - ProposedNodes that = (ProposedNodes) o; - - if (identifier != that.identifier) return false; - return addressAndPort.equals(that.addressAndPort); - - } - - @Override - public int hashCode() { - int result = (int) identifier; - result = 31 * result + addressAndPort.hashCode(); - return result; - } - } - -} - -class ConcurrentExpiryMap implements BytesMarshallable { - - final ConcurrentMap map = new - ConcurrentHashMap(); - private final Class kClass; - private final Class vClass; - // this is used for expiry - private final Queue>> queue = new ConcurrentLinkedQueue>>(); - - ConcurrentExpiryMap(Class kClass, Class vClass) { - this.kClass = kClass; - this.vClass = vClass; - } - - public static void main(String... args) { - String ip; - try { - Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); - while (interfaces.hasMoreElements()) { - NetworkInterface iface = interfaces.nextElement(); - // filters out 127.0.0.1 and inactive interfaces - if (iface.isLoopback() || !iface.isUp()) - continue; - - Enumeration addresses = iface.getInetAddresses(); - while (addresses.hasMoreElements()) { - InetAddress addr = addresses.nextElement(); - ip = addr.getHostAddress(); - System.out.println(iface.getDisplayName() + " " + ip); - } - } - } catch (SocketException e) { - throw new RuntimeException(e); - } - } - - /** - * private NetworkInterface defaultNetworkInterface() throws SocketException { NetworkInterface - * networkInterface = null - *

- * for (String suggestedName : new String[]{"en0", "eth0"}) { networkInterface = - * NetworkInterface.getByName(suggestedName); if (networkInterface != null) break; } return - * networkInterface; } - *

- * /** - * - * @return the default network interface, or - * @throws SocketException - */ - public static NetworkInterface defaultNetworkInterface() throws SocketException { - NetworkInterface networkInterface = null; - - for (String suggestedName : new String[]{"en0", "eth0"}) { - networkInterface = NetworkInterface.getByName(suggestedName); - if (networkInterface != null) - break; - } - - if (networkInterface != null) - return networkInterface; - - Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); - if (networkInterfaces == null || !networkInterfaces.hasMoreElements()) - return null; - - return networkInterfaces.nextElement(); - } - - public static InetAddress getDefaultAddress() throws SocketException { - NetworkInterface networkInterface = ConcurrentExpiryMap.defaultNetworkInterface(); - Enumeration inetAddresses = networkInterface.getInetAddresses(); - - // we will take the last one as this is likely to be an ipv address which is easier to work with - InetAddress inetAddress = null; - for (; inetAddresses.hasMoreElements(); ) { - inetAddress = inetAddresses.nextElement(); - } - - if (inetAddress == null) - throw new IllegalStateException(); - - return inetAddress; - } - - @Override - public String toString() { - return "ConcurrentExpiryMap{" + map + '}'; - } - - @Override - public void readMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes in) throws IllegalStateException { - - short size = in.readShort(); - try { - for (int i = 0; i < size; i++) { - final K k = kClass.newInstance(); - k.readMarshallable(in); - - final V v = vClass.newInstance(); - v.readMarshallable(in); - - map.put(k, v); - } - } catch (Exception e) { - LOG.error("", e); - } - } - - @Override - public void writeMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes out) { - final Map safeCopy = new HashMap(map); - out.writeShort(safeCopy.size()); - for (Map.Entry entry : safeCopy.entrySet()) { - entry.getKey().writeMarshallable(out); - entry.getValue().writeMarshallable(out); - } - } - - void put(final K k, final V v) { - map.put(k, v); - final W w = new W(v); - queue.add(new Map.Entry>() { - @Override - public K getKey() { - return k; - } - - @Override - public W getValue() { - return w; - } - - @Override - public W setValue(W value) { - throw new UnsupportedOperationException(); - } - }); - } - - java.util.Collection values() { - return map.values(); - } - - void expireEntries(long timeOlderThan) { - for (; ; ) { - final Map.Entry> e = this.queue.peek(); - - if (e == null) - break; - - if (e.getValue().timestamp < timeOlderThan) { - // only remote it if it has not changed - map.remove(e.getKey(), e.getValue().v); - } - - this.queue.poll(); - } - } - - class W { - final long timestamp; - final V v; - - W(V v) { - this.v = v; - this.timestamp = System.currentTimeMillis(); - } - } -} diff --git a/src/main/java/net/openhft/chronicle/map/PrepareValueBytesAsWriter.java b/src/main/java/net/openhft/chronicle/map/PrepareValueBytesAsWriter.java index 5b0809786..a16af5563 100644 --- a/src/main/java/net/openhft/chronicle/map/PrepareValueBytesAsWriter.java +++ b/src/main/java/net/openhft/chronicle/map/PrepareValueBytesAsWriter.java @@ -53,4 +53,10 @@ final class PrepareValueBytesAsWriter public void write(PrepareValueBytesAsWriter writer, Bytes bytes, K k) { write(bytes, k); } + + @Override + public boolean isEqual(PrepareValueBytesAsWriter write, Bytes bytes, K k, long size) { + // todo + return false; + } } diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index 49753c18e..12efa0310 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -45,7 +45,6 @@ import java.io.IOException; import java.util.Iterator; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -901,7 +900,10 @@ final class ReplicatedChronicleMap 1 ms. but making current time a param // of acquireWithoutLock would complicate code much, requiring putting // lookupUsing() back into ReplicatedChMap, and other methods - entry.writeLong(currentTime()); + final long timestamp = currentTime(); + final long replacedTimestamp = entry.readLong(entry.position()); + entry.writeLong(timestamp); + byte replacedIdentifier = entry.readByte(entry.position()); entry.writeByte(localIdentifier); // deleted flag entry.writeBoolean(false); @@ -931,7 +933,8 @@ final class ReplicatedChronicleMap toKey, GetValueInterops getValueInterops, VB value, InstanceOrBytesToInstance toValue, long hash2, - boolean replaceIfPresent, ReadValue readValue, boolean resultUnused) { + boolean replaceIfPresent, ReadValue readValue, boolean resultUnused, + final byte identifier, final byte replacedIdentifier, + final long timestamp, final long replacedTimestamp) { return putWithoutLock(copies, segmentState, metaKeyInterop, keyInterop, key, keySize, toKey, getValueInterops, value, toValue, hash2, replaceIfPresent, readValue, resultUnused, localIdentifier, currentTime(), false); @@ -1249,7 +1264,9 @@ final class ReplicatedChronicleMap currentTime() - TimeUnit.SECONDS.toMillis(1) && - timeStamp <= currentTime() : "timeStamp=" + timeStamp + ", " + - "currentTime=" + currentTime(); + // this is not true for bootstraping of replicaiton events + // assert timeStamp > currentTime() - TimeUnit.SECONDS.toMillis(1) && + // timeStamp <= currentTime() : "timeStamp=" + timeStamp + ", " + + // "currentTime=" + currentTime(); return timeStamp; } diff --git a/src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java b/src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java deleted file mode 100644 index ccabfc0c6..000000000 --- a/src/main/java/net/openhft/chronicle/map/ReplicationHubFindByName.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.ChronicleHash; -import net.openhft.chronicle.hash.ChronicleHashBuilder; -import net.openhft.chronicle.hash.ChronicleHashInstanceBuilder; -import net.openhft.chronicle.hash.replication.ReplicationChannel; -import net.openhft.chronicle.hash.replication.ReplicationHub; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author Rob Austin. - */ -class ReplicationHubFindByName implements FindByName { - public static final Logger LOG = LoggerFactory.getLogger(ReplicationHubFindByName.class.getName()); - - public static final int MAP_BY_NAME_CHANNEL = 1; - - private final AtomicInteger nextFreeChannel = new AtomicInteger(2); - private final Map map; - private final ReplicationHub replicationHub; - - /** - * @throws IOException - */ - public ReplicationHubFindByName(ReplicationHub replicationHub) throws IOException { - - LOG.info("connecting to replicationHub=" + replicationHub); - - this.replicationHub = replicationHub; - ReplicationChannel channel = replicationHub.createChannel((short) MAP_BY_NAME_CHANNEL); - - final MapEventListener listener = - new MapEventListener() { - // creates a map based on the details that are sent to the map of builders - @Override - public void onPut(CharSequence key, MapInstanceBuilder value, - MapInstanceBuilder replacedValue, boolean replicationEvent, - boolean added0) { - super.onPut(key, value, replacedValue, replicationEvent, added0); - boolean added = replacedValue == null; - if (!added || value == null) - return; - - // establish the new map based on this channel ID - LOG.info("create new map for name=" + value.name - + ",channelId=" + value.channel.channelId()); - - try { - toReplicatedViaChannel(value.mapBuilder, value.channel.channelId()).create(); - } catch (IllegalStateException e) { - // channel is already created - LOG.debug("while creating channel for name=" + value.name - + ",channelId=" + value.channel.channelId(), e); - } catch (IOException e) { - LOG.error("", e); - } - } - }; - - this.map = (Map) ChronicleMapBuilder - .of(CharSequence.class, MapInstanceBuilder.class) - .averageKeySize(10) - .averageValueSize(4000) - .entries(128) - .eventListener(listener) - .instance() - .replicatedViaChannel(channel) - .create(); - if (LOG.isDebugEnabled()) - LOG.debug("map=" + map); - } - - public T create( - MapInstanceBuilder config) - throws TimeoutException, InterruptedException { - - int withChannelId = nextFreeChannel.incrementAndGet(); - - map.put(config.name, - config.replicatedViaChannel(replicationHub.createChannel(withChannelId))); - - return (T) get(config.name).mapBuilder.create(); - } - - public T from(String name) throws IllegalArgumentException, - IOException, TimeoutException, InterruptedException { - return (T) get(name).create(); - } - - MapInstanceBuilder get(String name) throws IllegalArgumentException, - TimeoutException, - InterruptedException { - - MapInstanceBuilder config = waitTillEntryReceived(5000, name); - if (config == null) - throw new IllegalArgumentException("A map name=" + name + " can not be found."); - - return config; - } - - /** - * * waits until map1 and map2 show the same value - * - * @param timeOutMs timeout in milliseconds - * @throws InterruptedException - */ - private MapInstanceBuilder waitTillEntryReceived(final int timeOutMs, String name) - throws TimeoutException, InterruptedException { - int t = 0; - for (; t < timeOutMs; t++) { - MapInstanceBuilder config = map.get(name); - - if (config != null) - return config; - Thread.sleep(1); - } - throw new TimeoutException("timed out wait for map name=" + name); - } - - /** - * similar to {@link ChronicleMapBuilder#createPersistedTo(File file) } - * - * @param name the name of the map to be created - * @param file the file which will be used as shared off heap storage - * @param - * @return a new instance of ChronicleMap - * @throws IllegalArgumentException - * @throws IOException - * @see ChronicleMapBuilder#createPersistedTo(File file) - */ - public T createPersistedTo(String name, File file) - throws IllegalArgumentException, - IOException, TimeoutException, InterruptedException { - return (T) get(name).persistedTo(file).create(); - } - - private ChronicleHashInstanceBuilder toReplicatedViaChannel(ChronicleMapBuilder builder) { - int channelId = nextFreeChannel.incrementAndGet(); - if (channelId > replicationHub.maxNumberOfChannels()) - throw new IllegalStateException("There are no more free channels, you can increase the number " + - "of changes in the replicationHub by calling replicationHub.maxNumberOfChannels(..);"); - return builder.instance().replicatedViaChannel(replicationHub.createChannel((short) channelId)); - } - - private ChronicleHashInstanceBuilder toReplicatedViaChannel(ChronicleHashBuilder builder, int - withChannelId) { - return builder.instance().replicatedViaChannel(replicationHub.createChannel((short) withChannelId)); - } - -} diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 1748693f0..bebf31992 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -1376,6 +1376,15 @@ class VanillaChronicleMap, public void write(Void writer, Bytes bytes, Bytes value) { bytes.write(value, value.position(), value.remaining()); } + + @Override + public boolean isEqual(Void write, Bytes bytes, Bytes bytes2, long size) { + + if (size != bytes2.remaining()) + return false; + + return bytes.compare(bytes.position(), bytes2, bytes2.position(), bytes2.remaining()); + } } public static final class SegmentState implements StatefulCopyable, @@ -2091,6 +2100,12 @@ class VanillaChronicleMap, throw defaultValueOrPrepareBytesShouldBeSpecified(); } + + boolean hasValueChanged = valueSize < 0; + + if (valueSize < 0) + valueSize = valueSize * -1; + entry.positionAddr(entry.positionAddr() - valueSize); RV v = readValue.readValue(copies, entry, usingValue, valueSize); @@ -2102,7 +2117,9 @@ class VanillaChronicleMap, } if (eventListener != null) { eventListener.onPut(toKey.toInstance(copies, key, keySize), - toValue.toInstance(copies, v, valueSize), null, false, true); + toValue.toInstance(copies, v, valueSize), null, false, true, hasValueChanged, + (byte) 0, + (byte) 0, 0, 0); } return v; @@ -2171,6 +2188,8 @@ class VanillaChronicleMap, long prevValueSize = valueSizeMarshaller.readSize(entry); long sizeOfEverythingBeforeValue = entry.position(); alignment.alignPositionAddr(entry); + + boolean hasValueChanged = false; try { if (prevValueSize == valueSize && metaValueInterop.startsWith(valueInterop, entry, value)) { @@ -2193,7 +2212,8 @@ class VanillaChronicleMap, if (eventListener != null) { eventListener.onPut(toKey.toInstance(copies, key, keySize), toValue.toInstance(copies, value, valueSize), null, false, - false); + false, hasValueChanged, (byte) 0, (byte) 0, 0, + 0); } } } @@ -2202,7 +2222,8 @@ class VanillaChronicleMap, MVBI metaValueInterop = getValueInterops.getMetaValueInterop(copies, valueInterop, value); long valueSize = metaValueInterop.size(valueInterop, value); - putEntry(segmentState, metaKeyInterop, keyInterop, key, keySize, + long l = putEntry(segmentState, metaKeyInterop, keyInterop, key, + keySize, metaValueInterop, valueInterop, value, entry, false); // put callbacks @@ -2212,7 +2233,8 @@ class VanillaChronicleMap, segmentState.valueSizePos, true, false); if (eventListener != null) eventListener.onPut(toKey.toInstance(copies, key, keySize), - toValue.toInstance(copies, value, valueSize), null, false, true); + toValue.toInstance(copies, value, valueSize), null, false, true, l < + 0, (byte) 0, (byte) 0, 0, 0); return UpdateResult.INSERT; } finally { @@ -2241,7 +2263,8 @@ class VanillaChronicleMap, return putWithoutLock(copies, segmentState, metaKeyInterop, keyInterop, key, keySize, toKey, getValueInterops, value, toValue, - hash2, replaceIfPresent, readValue, resultUnused); + hash2, replaceIfPresent, readValue, resultUnused, (byte) 0, (byte) 0, 0, + 0); } finally { segmentState.close(); writeUnlock(); @@ -2257,7 +2280,8 @@ class VanillaChronicleMap, GetValueInterops getValueInterops, VB value, InstanceOrBytesToInstance toValue, long hash2, boolean replaceIfPresent, - ReadValue readValue, boolean resultUnused) { + ReadValue readValue, boolean resultUnused, final byte identifier, + final byte replacedIdentifier, final long timestamp, final long replacedTimestamp) { SearchState searchState = segmentState.searchState; hashLookup.startSearch(hash2, searchState); @@ -2274,7 +2298,7 @@ class VanillaChronicleMap, key, keySize, toKey, getValueInterops, value, toValue, entry, pos, offset, hashLookup, readValue, resultUnused, - false, false); + false, false, identifier, replacedIdentifier, timestamp, replacedTimestamp); } else { long valueSize = readValueSize(entry); return resultUnused ? null : @@ -2286,7 +2310,7 @@ class VanillaChronicleMap, MVBI metaValueInterop = getValueInterops.getMetaValueInterop(copies, valueInterop, value); long valueSize = metaValueInterop.size(valueInterop, value); - putEntry(segmentState, metaKeyInterop, keyInterop, key, keySize, + long l = putEntry(segmentState, metaKeyInterop, keyInterop, key, keySize, metaValueInterop, valueInterop, value, entry, false); // put callbacks @@ -2296,7 +2320,8 @@ class VanillaChronicleMap, segmentState.valueSizePos, true, false); if (eventListener != null) eventListener.onPut(toKey.toInstance(copies, key, keySize), - toValue.toInstance(copies, value, valueSize), null, false, true); + toValue.toInstance(copies, value, valueSize), null, false, true, l < 0, + identifier, (byte) 0, 0, 0); return resultUnused ? null : readValue.readNull(); } @@ -2309,7 +2334,7 @@ class VanillaChronicleMap, InstanceOrBytesToInstance toValue, MultiStoreBytes entry, long pos, long offset, MultiMap searchedHashLookup, ReadValue readValue, boolean resultUnused, - boolean entryIsDeleted, boolean remote) { + boolean entryIsDeleted, boolean remote, final byte identifier, byte replacedIdentifier, final long timestamp, final long replacedTimestamp) { VBI valueInterop = getValueInterops.getValueInterop(copies); MVBI metaValueInterop = getValueInterops.getMetaValueInterop( copies, valueInterop, value); @@ -2335,7 +2360,8 @@ class VanillaChronicleMap, } // putValue may relocate entry and change offset - putValue(pos, offset, entry, valueSizePos, entryEndAddr, entryIsDeleted, segmentState, + boolean hasValueChanged = putValue(pos, offset, entry, valueSizePos, entryEndAddr, + entryIsDeleted, segmentState, metaValueInterop, valueInterop, value, valueSize, searchedHashLookup, sizeOfEverythingBeforeValue); @@ -2346,7 +2372,8 @@ class VanillaChronicleMap, if (eventListener != null) { eventListener.onPut(toKey.toInstance(copies, key, keySize), toValue.toInstance(copies, value, valueSize), prevValueInstance, remote, - entryIsDeleted); + entryIsDeleted, hasValueChanged, identifier, replacedIdentifier, + timestamp, replacedTimestamp); } return resultUnused ? null : prevValue; @@ -2364,6 +2391,8 @@ class VanillaChronicleMap, * Returns value size, writes the entry (key, value, sizes) to the entry, after this method * call entry positioned after value bytes written (i. e. at the end of entry), sets entry * position (in segment) and value size position in the given segmentState + * + * @return the size or negivtive size if the value has changed. */ final , E, EW> long putEntry(SegmentState segmentState, @@ -2379,14 +2408,18 @@ class VanillaChronicleMap, clearMetaData(offset); reuse(entry, offset); + keySizeMarshaller.writeSize(entry, keySize); metaKeyInterop.write(keyInterop, entry, key); manageReplicationBytes(entry, writeDefaultInitialReplicationValues, false); segmentState.valueSizePos = entry.position(); + final long size = valueSizeMarshaller.readSize(entry); + entry.position(segmentState.valueSizePos); valueSizeMarshaller.writeSize(entry, valueSize); alignment.alignPositionAddr(entry); + boolean hasValueChanged = !metaElemWriter.isEqual(elemWriter, entry, elem, size); metaElemWriter.write(elemWriter, entry, elem); freeExtraAllocatedChunks(pos, allocatedChunks, entry); @@ -2394,7 +2427,7 @@ class VanillaChronicleMap, hashLookup.putAfterFailedSearch(segmentState.searchState, pos); incrementSize(); - return valueSize; + return hasValueChanged ? -valueSize : valueSize; } final void freeExtraAllocatedChunks(long pos, int allocatedChunks, Bytes entry) { @@ -2714,7 +2747,7 @@ class VanillaChronicleMap, copies, valueInterop, newValue); long newValueSize = metaValueInterop.size(valueInterop, newValue); boolean entryIsDeleted = false; // couldn't replace deleted entry - putValue(pos, offset, entry, valueSizePos, entryEndAddr, entryIsDeleted, segmentState, + boolean hasValueChanged = putValue(pos, offset, entry, valueSizePos, entryEndAddr, entryIsDeleted, segmentState, metaValueInterop, valueInterop, newValue, newValueSize, searchedHashLookup, sizeOfEverythingBeforeValue); @@ -2727,7 +2760,8 @@ class VanillaChronicleMap, if (eventListener != null) eventListener.onPut(toKey.toInstance(copies, key, keySize), toValue.toInstance(copies, newValue, newValueSize), - toValue.toInstance(copies, prevValue, valueSize), false, true); + toValue.toInstance(copies, prevValue, valueSize), false, true, + hasValueChanged, (byte) 0, (byte) 0, 0, 0); return expectedValue == null ? prevValue : Boolean.TRUE; } @@ -2742,16 +2776,16 @@ class VanillaChronicleMap, * @param entry relative pointer in Segment bytes * @param valueSizePos relative position of value size in entry * @param entryEndAddr absolute address of the entry end - * @return relative offset of the entry in Segment bytes after putting value (that may cause - * entry relocation) + * @return true if the value has change */ - final long putValue( + final boolean putValue( long pos, long offset, MultiStoreBytes entry, long valueSizePos, long entryEndAddr, boolean entryIsDeleted, SegmentState segmentState, MetaBytesWriter metaElemWriter, EW elemWriter, E newElem, long newElemSize, MultiMap searchedHashLookup, long sizeOfEverythingBeforeValue) { + final boolean hasValueChanged; long entryStartAddr = entry.address(); long valueSizeAddr = entryStartAddr + valueSizePos; long newValueAddr = alignment.alignAddr( @@ -2789,14 +2823,20 @@ class VanillaChronicleMap, NativeBytes.UNSAFE.copyMemory(entryStartAddr, newEntryStartAddr, valueSizeAddr - entryStartAddr); entry.position(valueSizePos); + final long originalSize = valueSizeMarshaller.readSize(entry); + entry.position(valueSizePos); valueSizeMarshaller.writeSize(entry, newElemSize); alignment.alignPositionAddr(entry); - metaElemWriter.write(elemWriter, entry, newElem); + + hasValueChanged = !metaElemWriter.isEqual(elemWriter, entry, newElem, + originalSize); + if (hasValueChanged) + metaElemWriter.write(elemWriter, entry, newElem); freeExtraAllocatedChunks(newPos, allocatedChunks, entry); segmentState.pos = newPos; - return offset; + return hasValueChanged; // END OF RELOCATION } else if (newSizeInChunks < oldSizeInChunks) { // Freeing extra chunks @@ -2809,11 +2849,15 @@ class VanillaChronicleMap, } // Common code for all cases entry.position(valueSizePos); + final long originalSize = valueSizeMarshaller.readSize(entry); + entry.position(valueSizePos); valueSizeMarshaller.writeSize(entry, newElemSize); alignment.alignPositionAddr(entry); + + hasValueChanged = !metaElemWriter.isEqual(elemWriter, entry, newElem, originalSize); metaElemWriter.write(elemWriter, entry, newElem); segmentState.pos = pos; - return offset; + return hasValueChanged; } private void clear() { diff --git a/src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java b/src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java index df8ca8941..b43d042e5 100644 --- a/src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java +++ b/src/test/java/net/openhft/chronicle/map/EventListenerWithTCPSocketReplicationTest.java @@ -55,7 +55,9 @@ public class EventListenerWithTCPSocketReplicationTest { @Override public void onPut(Integer key, CharSequence value, CharSequence replacedValue, - boolean replicationEvent, boolean added) { + boolean replicationEvent, boolean added, boolean hasValueChanged, + byte identifier, byte replacedIdentifier, long timeStamp, + long replacedTimeStamp) { putWasCalled.getAndSet(true); keyRef.set(key); valueRef.set(value); diff --git a/src/test/java/net/openhft/chronicle/map/MapByNameTest.java b/src/test/java/net/openhft/chronicle/map/MapByNameTest.java deleted file mode 100644 index 4b3d0b22a..000000000 --- a/src/test/java/net/openhft/chronicle/map/MapByNameTest.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import net.openhft.chronicle.hash.replication.ReplicationHub; -import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.concurrent.TimeoutException; - -import static junit.framework.Assert.assertEquals; - -/** - * @author Rob Austin. - */ -public class MapByNameTest { - - private ReplicationHubFindByName findMapByName; - - @Before - public void setUp() throws IOException { - final ReplicationHub replicationHub = ReplicationHub.builder() - .tcpTransportAndNetwork(TcpTransportAndNetworkConfig.of(8243)) - .createWithId((byte) 1); - - findMapByName = new ReplicationHubFindByName(replicationHub); - } - - @Test - public void testSerializingBuilder() throws TimeoutException, InterruptedException, IOException { - - final MapInstanceBuilder config = - (MapInstanceBuilder) - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .averageKeySize("hello".length()).averageValueSize("world".length()) - .entries(2) - .minSegments(2) - .replication((byte) 1, TcpTransportAndNetworkConfig.of(8244)) - .removeReturnsNull(true) - .instance().name("map1"); - - final ChronicleMap map = findMapByName.create(config); - map.put("hello", "world"); - - assertEquals(map.get("hello"), "world"); - } - - // @Test(expected = IllegalArgumentException.class) - // public void testSerializingBuilderUnknownMap() throws TimeoutException, InterruptedException { - // findMapByName.get("hello"); - // } - - @Test - @Ignore("currently work in progress") - public void testConnectByName() throws IOException, InterruptedException, TimeoutException { - - NodeDiscovery nodeDiscovery = new NodeDiscovery(); - - MapInstanceBuilder config = - (MapInstanceBuilder) - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .minSegments(2) - .removeReturnsNull(true) - .instance() - .name("myMap"); - - ReplicationHubFindByName mapByName = nodeDiscovery.mapByName(); - // mapByName.add(config); - - ChronicleMap myMap2 = mapByName.from("myMap"); - } - - @Ignore("causing issues on windows") - @Test - public void testReplicationHubSerialization() throws IOException, InterruptedException, - TimeoutException { - NodeDiscovery nodeDiscovery = new NodeDiscovery(); - - final ReplicationHubFindByName mapByName = nodeDiscovery.mapByName(); - Thread.sleep(2000); - - MapInstanceBuilder config = - (MapInstanceBuilder) - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .minSegments(2) - .entries(2) - .removeReturnsNull(true) - .instance() - .name("myMap6"); - - mapByName.create(config); - - } - - public static void main(String... args) throws IOException, InterruptedException, TimeoutException { - NodeDiscovery nodeDiscovery = new NodeDiscovery(); - - final ReplicationHubFindByName mapByName = nodeDiscovery.mapByName(); - Thread.sleep(2000); - - MapInstanceBuilder config = - (MapInstanceBuilder) - ChronicleMapBuilder.of(CharSequence.class, CharSequence.class) - .minSegments(2) - .entries(2) - .removeReturnsNull(true) - .instance() - .name("myMap6"); - - ChronicleMap map = mapByName.create(config); - map.put("hello", "world"); - - // final ChronicleMap map = mapByName.from("myMap5"); - - // allow time for replication - Thread.sleep(2000); - System.out.print(map); - - map.close(); - } - -} diff --git a/src/test/java/net/openhft/chronicle/map/MapEventListenerOnPutAddedFlagTest.java b/src/test/java/net/openhft/chronicle/map/MapEventListenerOnPutAddedFlagTest.java index 96ff62ba5..330c051a4 100644 --- a/src/test/java/net/openhft/chronicle/map/MapEventListenerOnPutAddedFlagTest.java +++ b/src/test/java/net/openhft/chronicle/map/MapEventListenerOnPutAddedFlagTest.java @@ -31,7 +31,9 @@ public class MapEventListenerOnPutAddedFlagTest { @Override public void onPut(Object key, Object newValue, @Nullable Object replacedValue, - boolean replicationEvent, boolean added) { + boolean replicationEvent, boolean added, boolean hasValueChanged, + byte identifier, byte replacedIdentifier, long timeStamp, + long replacedTimeStamp) { Assert.assertEquals(expected, added); } } diff --git a/src/test/java/net/openhft/chronicle/map/NodeDiscoveryBroadcasterTest.java b/src/test/java/net/openhft/chronicle/map/NodeDiscoveryBroadcasterTest.java deleted file mode 100644 index 53fa16ae0..000000000 --- a/src/test/java/net/openhft/chronicle/map/NodeDiscoveryBroadcasterTest.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (C) 2015 higherfrequencytrading.com - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package net.openhft.chronicle.map; - -import junit.framework.TestCase; -import net.openhft.lang.io.ByteBufferBytes; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.util.concurrent.atomic.AtomicReference; - -import static java.nio.ByteBuffer.allocate; - -public class NodeDiscoveryBroadcasterTest extends TestCase { - - public static final int SERVER2_IDENTIFER = 10; - public static final int PROPOSED_IDENTIFIER = 5; - - @Test - public void testsTheSerializationOfTheNodeDiscovery() throws IOException, InterruptedException { - - AtomicReference ref = new AtomicReference(); - final AddressAndPort ourAddressAndPort1 = new AddressAndPort(InetAddress.getLocalHost() - .getAddress(), - (short) 1024); - - final AddressAndPort ourAddressAndPort2 = new AddressAndPort(InetAddress.getLocalHost() - .getAddress(), - (short) 1025); - - // write broadcast our address on the bootstrap method - byte[] server1Address = Inet4Address.getLocalHost().getAddress(); - AddressAndPort server1AddressAndPort = new AddressAndPort(server1Address, (short) 1234); - - // write broadcast our address on the bootstrap method - byte[] server2Address = Inet4Address.getByName("129.168.0.2").getAddress(); - AddressAndPort server2AddressAndPort = new AddressAndPort(server1Address, (short) 1234); - - KnownNodes server1KnownNodes = new KnownNodes(); - DiscoveryNodeBytesMarshallable server1BytesExternalizable = new DiscoveryNodeBytesMarshallable - (server1KnownNodes, ref, ourAddressAndPort1); - - // for the unit test we are using a ByteBufferBytes, but iltimately this data would have been send - // and received via UDP - ByteBufferBytes udpData = new ByteBufferBytes(allocate(1024)); - - // SERER1 - is the new node join the cluster - // we will first send the bootstrap along with our host and port - { - final DiscoveryNodeBytesMarshallable.ProposedNodes proposedNodes = new - DiscoveryNodeBytesMarshallable.ProposedNodes(server1AddressAndPort, (byte) -1); - server1BytesExternalizable.sendBootStrap(proposedNodes); - server1BytesExternalizable.writeMarshallable(udpData); - } - - KnownNodes server2KnownNodes = new KnownNodes(); - server2KnownNodes.add(server2AddressAndPort, (byte) SERVER2_IDENTIFER); - DiscoveryNodeBytesMarshallable server2BytesExternalizable = new DiscoveryNodeBytesMarshallable - (server2KnownNodes, ref, ourAddressAndPort2); - - // add our identifier and host:port to the list of known identifiers - - // SERVER 2 - is the node already in the cluster - // read the bootstrap along with the host and port it came from - { - udpData.flip(); - server2BytesExternalizable.readMarshallable(udpData); - - Assert.assertTrue(server2BytesExternalizable.proposedIdentifiersWithHost.values().contains - (new DiscoveryNodeBytesMarshallable.ProposedNodes(server1AddressAndPort, (byte) -1))); - } - - // SERVER 2 - // write/broadcast the the response to receiving a bootstrap message - - { - //ByteBufferBytes bitSetBytes = new ByteBufferBytes(allocate(128 / 8)); - // final DirectBitSet knownIdentifiers = new ATSDirectBitSet(bitSetBytes); - - // RemoteNodes remoteNodes = new RemoteNodes(bitSetBytes); - - udpData.clear(); - - // broadcasts in response to the bootstrap all the host:ports and identifiers that it knows about - // in this case given that there is only this node in the grid and one node joining, - // it will only be the node that has the bootstrap and this node, who's data will be sent - server2BytesExternalizable.writeMarshallable(udpData); - } - - // SERER1 - // receive - - // read the bootstrap along with the host and port it came from - { - udpData.flip(); - server1BytesExternalizable.readMarshallable(udpData); - - Assert.assertTrue(server1BytesExternalizable.getRemoteNodes().addressAndPorts().contains - (server2AddressAndPort)); - - Assert.assertTrue(server1BytesExternalizable.getRemoteNodes().identifiers().get - (SERVER2_IDENTIFER)); - } - - // SERVER 1 - // send/broadcast - propose an identifier to use, by choosing one that is not used already - // for the purpose of this test we wont choose one at random ( as the code does ) but just pick - // identifier=5 - - { - udpData.clear(); - - KnownNodes knownNodes = new KnownNodes(); - - DiscoveryNodeBytesMarshallable bytesExternalizable = new DiscoveryNodeBytesMarshallable - (knownNodes, ref, ourAddressAndPort1); - - DiscoveryNodeBytesMarshallable.ProposedNodes proposedNodes = new DiscoveryNodeBytesMarshallable.ProposedNodes(server1AddressAndPort, (byte) PROPOSED_IDENTIFIER); - - // send the bootstrap along with this newly proposed identifier - bytesExternalizable.sendBootStrap(proposedNodes); - - bytesExternalizable.writeMarshallable(udpData); - } - - // SERVER 2 - is the node already in the cluster - // read the bootstrap along with the proposed identifier and host:port it came from - { - udpData.flip(); - server2BytesExternalizable.readMarshallable(udpData); - - Assert.assertTrue(server2BytesExternalizable.proposedIdentifiersWithHost.values().contains - (new DiscoveryNodeBytesMarshallable.ProposedNodes(server1AddressAndPort, (byte) PROPOSED_IDENTIFIER))); - } - - // SERVER 2 - // write/broadcast the the response to receiving a bootstrap message - { - //ByteBufferBytes bitSetBytes = new ByteBufferBytes(allocate(128 / 8)); - // final DirectBitSet knownIdentifiers = new ATSDirectBitSet(bitSetBytes); - - // RemoteNodes remoteNodes = new RemoteNodes(bitSetBytes); - - udpData.clear(); - - // broadcasts in response to the bootstrap all the host:ports and identifiers that it knows about - // in this case given that there is only this node in the grid and one node joining, - // it will only be the node that has the bootstrap and this node, who's data will be sent - server2BytesExternalizable.writeMarshallable(udpData); - } - - // SERVER 1 - // read the bootstrap along with all the proposed identifiers ( this will include our - // PROPOSED_IDENTIFIER and any that come from other server - { - udpData.flip(); - server1BytesExternalizable.readMarshallable(udpData); - - Assert.assertTrue(server1BytesExternalizable.getRemoteNodes().addressAndPorts().contains - (server2AddressAndPort)); - - Assert.assertTrue(server1BytesExternalizable.getRemoteNodes().identifiers().get - (SERVER2_IDENTIFER)); - - Assert.assertTrue(server1BytesExternalizable.proposedIdentifiersWithHost.values().contains - (new DiscoveryNodeBytesMarshallable.ProposedNodes(server1AddressAndPort, (byte) PROPOSED_IDENTIFIER))); - } - } - -} diff --git a/src/test/java/net/openhft/chronicle/map/ReplicationTest.java b/src/test/java/net/openhft/chronicle/map/ReplicationTest.java new file mode 100755 index 000000000..0739c09e4 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/map/ReplicationTest.java @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2015 higherfrequencytrading.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package net.openhft.chronicle.map; + +import net.openhft.chronicle.hash.replication.TcpTransportAndNetworkConfig; +import net.openhft.lang.io.ByteBufferBytes; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.DataValueClasses; +import net.openhft.lang.values.IntValue; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Test ReplicatedChronicleMap where the Replicated is over a TCP Socket + * + * @author Rob Austin. + */ + +public class ReplicationTest { + + + static int s_port = 8093; + Set threads; + private ChronicleMap map1; + final AtomicLong count = new AtomicLong(); + final AtomicBoolean hasValueChanged = new AtomicBoolean(); + + final AtomicInteger identifier = new AtomicInteger(); + final AtomicInteger replacedIdentifier = new AtomicInteger(); + final AtomicLong timeStamp = new AtomicLong(); + final AtomicLong replacedTimeStamp = new AtomicLong(); + + private IntValue value; + + @Before + public void setup() throws IOException { + value = DataValueClasses.newDirectReference(IntValue.class); + ((Byteable) value).bytes(new ByteBufferBytes(ByteBuffer.allocateDirect(4)), 0); + + + final InetSocketAddress endpoint = new InetSocketAddress("localhost", s_port + 1); + + { + final TcpTransportAndNetworkConfig tcpConfig1 = TcpTransportAndNetworkConfig.of(s_port, + endpoint).autoReconnectedUponDroppedConnection(true).name(" map1") + .heartBeatInterval(1, TimeUnit.SECONDS) + .tcpBufferSize(1024 * 64); + + final MapEventListener integerCharSequenceMapEventListener = + new MapEventListener() { + public void onPut(Integer key, + CharSequence newValue, + @Nullable CharSequence replacedValue, + boolean replicationEvent, + boolean added, + boolean hasValueChanged, + byte identifier, + byte replacedIdentifier, + long timeStamp, + long replacedTimeStamp) { + + System.out.println("key=" + key + ", value=" + value + ", replacedValue=" + replacedValue + ", " + + "replicationEvent=" + replicationEvent + ", added=" + + added + ", hasValueChanged=" + hasValueChanged + ", " + + "identifier=" + identifier + ", replacedIdentifier=" + + replacedIdentifier + ", timeStamp=" + timeStamp + ", " + + "replacedTimeStamp=" + replacedTimeStamp); + count.incrementAndGet(); + + ReplicationTest.this.identifier.set(identifier); + ReplicationTest.this.replacedIdentifier.set(replacedIdentifier); + ReplicationTest.this.timeStamp.set(timeStamp); + ReplicationTest.this.replacedTimeStamp.set(replacedTimeStamp); + + ReplicationTest.this.hasValueChanged.set(hasValueChanged); + + } + + }; + + + map1 = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + .entries(Builder.SIZE + Builder.SIZE) + .actualSegments(1) + .replication((byte) 1) + .eventListener(integerCharSequenceMapEventListener) + .instance() + .name("map1") + .create(); + } + + s_port += 2; + } + + @After + public void tearDown() throws InterruptedException { + System.gc(); + } + + @Before + public void sampleThreads() { + threads = Thread.getAllStackTraces().keySet(); + } + + @After + public void checkThreadsShutdown() { + StatelessClientTest.checkThreadsShutdown(threads); + } + + /** + * tests that remote puts (in otherwords, puts that provide and identifier only replicate on + * events with with same identifier as the local-identifier + * + * @throws IOException + * @throws InterruptedException + */ + @Test + public void test() throws IOException, InterruptedException { + + ReplicatedChronicleMap map = (ReplicatedChronicleMap) map1; + + final long ts = 123; + map.put(1, "test", (byte) 2, ts); + Assert.assertTrue(hasValueChanged.get()); + Assert.assertEquals((byte) 2, identifier.get()); + Assert.assertEquals((byte) 0, replacedIdentifier.get()); + Assert.assertEquals(123, timeStamp.get()); + Assert.assertEquals((byte) 0, replacedTimeStamp.get()); + + map.put(1, "test", (byte) 3, 124); + Assert.assertFalse(hasValueChanged.get()); + Assert.assertEquals((byte) 3, identifier.get()); + Assert.assertEquals((byte) 2, replacedIdentifier.get()); + Assert.assertEquals(124, timeStamp.get()); + Assert.assertEquals(123, replacedTimeStamp.get()); + + + map.put(1, "test2", (byte) 2, 125); + Assert.assertTrue(hasValueChanged.get()); + Assert.assertEquals((byte) 2, identifier.get()); + Assert.assertEquals((byte) 3, replacedIdentifier.get()); + Assert.assertEquals(125, timeStamp.get()); + Assert.assertEquals(124, replacedTimeStamp.get()); + + map.put(1, "test2", (byte) 2, 125); + Assert.assertFalse(hasValueChanged.get()); + Assert.assertEquals((byte) 2, identifier.get()); + Assert.assertEquals((byte) 2, replacedIdentifier.get()); + Assert.assertEquals(125, timeStamp.get()); + Assert.assertEquals(125, replacedTimeStamp.get()); + } + + +} + diff --git a/src/test/java/net/openhft/chronicle/map/TCPReplicationCheckUpdatesFromOtherMapsAreNotPopulated.java b/src/test/java/net/openhft/chronicle/map/TCPReplicationCheckUpdatesFromOtherMapsAreNotPopulated.java index bcaff1e16..6f57ac501 100755 --- a/src/test/java/net/openhft/chronicle/map/TCPReplicationCheckUpdatesFromOtherMapsAreNotPopulated.java +++ b/src/test/java/net/openhft/chronicle/map/TCPReplicationCheckUpdatesFromOtherMapsAreNotPopulated.java @@ -118,7 +118,7 @@ public class TCPReplicationCheckUpdatesFromOtherMapsAreNotPopulated { * @throws InterruptedException */ @Test - public void test3() throws IOException, InterruptedException { + public void test() throws IOException, InterruptedException { ReplicatedChronicleMap map = (ReplicatedChronicleMap) map1; @@ -135,7 +135,6 @@ public class TCPReplicationCheckUpdatesFromOtherMapsAreNotPopulated { Assert.assertTrue(map2.size() == 1); Assert.assertEquals("test2", map2.get(1)); - }