diff --git a/src/main/java/net/openhft/chronicle/hash/replication/TimeProvider.java b/src/main/java/net/openhft/chronicle/hash/replication/TimeProvider.java index 8c621473b..857841f06 100644 --- a/src/main/java/net/openhft/chronicle/hash/replication/TimeProvider.java +++ b/src/main/java/net/openhft/chronicle/hash/replication/TimeProvider.java @@ -20,7 +20,6 @@ import net.openhft.chronicle.hash.ChronicleHashBuilder; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.set.ChronicleSet; -import java.io.ObjectStreamException; import java.io.Serializable; /** @@ -65,7 +64,7 @@ public abstract class TimeProvider implements Serializable { return getClass().getSimpleName(); } - private Object readResolve() throws ObjectStreamException { + private Object readResolve() { return SYSTEM; } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceReader.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceReader.java index e2599f77a..301dc918d 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceReader.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceReader.java @@ -26,35 +26,26 @@ import net.openhft.lang.threadlocal.StatefulCopyable; import org.jetbrains.annotations.NotNull; import java.io.IOException; -import java.io.ObjectStreamException; import java.io.Serializable; public final class CharSequenceReader implements BytesReader, StatefulCopyable> { private static final long serialVersionUID = 0L; - - private enum NoInterningIdentity implements CharSequenceInterner { - INSTANCE; - - @Override - public CharSequence intern(CharSequence cs) { - return cs; - } - } - - private enum NoInterningToString implements CharSequenceInterner { - INSTANCE; - @Override - public String intern(CharSequence cs) { - return cs.toString(); - } - } - private static final CharSequenceReader STRING_READER = new CharSequenceReader(NoInterningToString.INSTANCE, NoInterningToString.INSTANCE); - private static final CharSequenceReader DEFAULT_READER = new CharSequenceReader(NoInterningIdentity.INSTANCE, NoInterningIdentity.INSTANCE); + private final StringBuilder sb = new StringBuilder(128); + @NotNull + private final CharSequenceInterner interner; + @NotNull + private final Serializable identity; + + private CharSequenceReader(@NotNull CharSequenceInterner interner, + @NotNull Serializable identity) { + this.interner = interner; + this.identity = identity; + } public static CharSequenceReader of() { return STRING_READER; @@ -69,18 +60,6 @@ public final class CharSequenceReader return new CharSequenceReader(interner, stateIdentity); } - private final StringBuilder sb = new StringBuilder(128); - @NotNull - private final CharSequenceInterner interner; - @NotNull - private final Serializable identity; - - private CharSequenceReader(@NotNull CharSequenceInterner interner, - @NotNull Serializable identity) { - this.interner = interner; - this.identity = identity; - } - @Override public S read(Bytes bytes, long size) { sb.setLength(0); @@ -126,11 +105,29 @@ public final class CharSequenceReader return new CharSequenceReader(interner, identity); } - private Object readResolve() throws ObjectStreamException { + private Object readResolve() { if (interner == NoInterningToString.INSTANCE) return STRING_READER; if (interner == NoInterningIdentity.INSTANCE) return DEFAULT_READER; return this; } + + private enum NoInterningIdentity implements CharSequenceInterner { + INSTANCE; + + @Override + public CharSequence intern(CharSequence cs) { + return cs; + } + } + + private enum NoInterningToString implements CharSequenceInterner { + INSTANCE; + + @Override + public String intern(CharSequence cs) { + return cs.toString(); + } + } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceWriter.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceWriter.java index 36ea3c7fb..4a8337927 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceWriter.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/CharSequenceWriter.java @@ -20,8 +20,6 @@ import net.openhft.chronicle.hash.serialization.BytesWriter; import net.openhft.lang.io.AbstractBytes; import net.openhft.lang.io.Bytes; -import java.io.ObjectStreamException; - public final class CharSequenceWriter implements BytesWriter { private static final long serialVersionUID = 0L; private static final CharSequenceWriter INSTANCE = new CharSequenceWriter(); @@ -43,7 +41,7 @@ public final class CharSequenceWriter implements BytesW AbstractBytes.writeUTF0(bytes, s, s.length()); } - private Object readResolve() throws ObjectStreamException { + private Object readResolve() { return INSTANCE; } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java index ed281b289..2aad8beb1 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInterop.java @@ -19,20 +19,19 @@ package net.openhft.chronicle.hash.serialization.internal; import net.openhft.chronicle.hash.serialization.BytesInterop; import net.openhft.lang.io.Bytes; -import java.io.ObjectStreamException; - public final class DelegatingMetaBytesInterop> implements MetaBytesInterop { private static final long serialVersionUID = 0L; private static final DelegatingMetaBytesInterop INSTANCE = new DelegatingMetaBytesInterop(); + private DelegatingMetaBytesInterop() { + } + public static > DelegatingMetaBytesInterop instance() { return INSTANCE; } - private DelegatingMetaBytesInterop() {} - @Override public long size(I interop, E e) { return interop.size(e); @@ -53,7 +52,7 @@ public final class DelegatingMetaBytesInterop> interop.write(bytes, e); } - private Object readResolve() throws ObjectStreamException { + private Object readResolve() { return INSTANCE; } } diff --git a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInteropProvider.java b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInteropProvider.java index 9ae4d8f32..22a01280f 100644 --- a/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInteropProvider.java +++ b/src/main/java/net/openhft/chronicle/hash/serialization/internal/DelegatingMetaBytesInteropProvider.java @@ -19,21 +19,20 @@ package net.openhft.chronicle.hash.serialization.internal; import net.openhft.chronicle.hash.serialization.BytesInterop; import net.openhft.lang.threadlocal.ThreadLocalCopies; -import java.io.ObjectStreamException; - public final class DelegatingMetaBytesInteropProvider> implements MetaProvider> { private static final long serialVersionUID = 0L; private static final DelegatingMetaBytesInteropProvider INSTANCE = new DelegatingMetaBytesInteropProvider(); + private DelegatingMetaBytesInteropProvider() { + } + public static > MetaProvider> instance() { return INSTANCE; } - private DelegatingMetaBytesInteropProvider() {} - @Override public MetaBytesInterop get(ThreadLocalCopies copies, MetaBytesInterop originalMetaWriter, I writer, E e) { @@ -45,7 +44,7 @@ public final class DelegatingMetaBytesInteropProvider pendingRegistrations = new ConcurrentLinkedQueue(); + @Nullable + private final Throttler throttler; + volatile boolean isClosed = false; + ThreadLocalCopies copies; + VanillaChronicleMap.SegmentState segmentState; + private volatile Thread lastThread; private final ExecutorService connectExecutorService = Executors.newCachedThreadPool( new NamedThreadFactory("TCP-connect", true) { @Override @@ -72,20 +78,10 @@ abstract class AbstractChannelReplicator implements Closeable { return lastThread = super.newThread(r); } }); - - private volatile Thread lastThread; private Throwable startedHere; private Future future; - private final Queue pendingRegistrations = new ConcurrentLinkedQueue(); - @Nullable - private final Throttler throttler; - volatile boolean isClosed = false; - ThreadLocalCopies copies; - VanillaChronicleMap.SegmentState segmentState; - - AbstractChannelReplicator(String name, ThrottlingConfig throttlingConfig) - throws IOException { + AbstractChannelReplicator(String name, ThrottlingConfig throttlingConfig) throws IOException { executorService = Executors.newSingleThreadExecutor( new NamedThreadFactory(name, true) { @Override @@ -94,6 +90,7 @@ abstract class AbstractChannelReplicator implements Closeable { } }); + selector = openSelector(closeables); throttler = throttlingConfig.throttling(DAYS) > 0 ? new Throttler(selector, throttlingConfig.bucketInterval(MILLISECONDS), @@ -102,6 +99,36 @@ abstract class AbstractChannelReplicator implements Closeable { startedHere = new Throwable("Started here"); } + static SocketChannel openSocketChannel(final CloseablesManager closeables) throws IOException { + SocketChannel result = null; + + try { + result = SocketChannel.open(); + result.socket().setTcpNoDelay(true); + } finally { + if (result != null) + try { + closeables.add(result); + } catch (IllegalStateException e) { + // already closed + } + } + return result; + } + + static ClassLoader getSystemClassLoader() { + if (System.getSecurityManager() == null) { + return ClassLoader.getSystemClassLoader(); + } else { + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public ClassLoader run() { + return ClassLoader.getSystemClassLoader(); + } + }); + } + } + Selector openSelector(final CloseablesManager closeables) throws IOException { Selector result = Selector.open(); @@ -132,23 +159,6 @@ abstract class AbstractChannelReplicator implements Closeable { return result; } - static SocketChannel openSocketChannel(final CloseablesManager closeables) throws IOException { - SocketChannel result = null; - - try { - result = SocketChannel.open(); - result.socket().setTcpNoDelay(true); - } finally { - if (result != null) - try { - closeables.add(result); - } catch (IllegalStateException e) { - // already closed - } - } - return result; - } - /** * this is similar to the code in Netty * @@ -185,24 +195,11 @@ abstract class AbstractChannelReplicator implements Closeable { return selector; } - static ClassLoader getSystemClassLoader() { - if (System.getSecurityManager() == null) { - return ClassLoader.getSystemClassLoader(); - } else { - return AccessController.doPrivileged(new PrivilegedAction() { - @Override - public ClassLoader run() { - return ClassLoader.getSystemClassLoader(); - } - }); - } - } - void addPendingRegistration(Runnable registration) { pendingRegistrations.add(registration); } - void registerPendingRegistrations() throws ClosedChannelException { + void registerPendingRegistrations() { for (Runnable runnable = pendingRegistrations.poll(); runnable != null; runnable = pendingRegistrations.poll()) { try { @@ -213,7 +210,7 @@ abstract class AbstractChannelReplicator implements Closeable { } } - abstract void processEvent() throws IOException; + abstract void processEvent(); final void start() { future = executorService.submit( @@ -315,12 +312,12 @@ abstract class AbstractChannelReplicator implements Closeable { closeables.closeQuietly(channel); } - void checkThrottleInterval() throws ClosedChannelException { + void checkThrottleInterval() { if (throttler != null) throttler.checkThrottleInterval(); } - void contemplateThrottleWrites(int bytesJustWritten) throws ClosedChannelException { + void contemplateThrottleWrites(int bytesJustWritten) { if (throttler != null) throttler.contemplateThrottleWrites(bytesJustWritten); } @@ -366,7 +363,7 @@ abstract class AbstractChannelReplicator implements Closeable { * * @throws java.nio.channels.ClosedChannelException */ - public void checkThrottleInterval() throws ClosedChannelException { + public void checkThrottleInterval() { final long time = System.currentTimeMillis(); if (lastTime + throttleInterval >= time) @@ -392,8 +389,7 @@ abstract class AbstractChannelReplicator implements Closeable { * * @throws ClosedChannelException */ - public void contemplateThrottleWrites(int bytesJustWritten) - throws ClosedChannelException { + public void contemplateThrottleWrites(int bytesJustWritten) { bytesWritten += bytesJustWritten; if (bytesWritten > maxBytesInInterval) { for (SelectableChannel channel : channels) { diff --git a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java index f0cb0a0b9..bd0f10f08 100644 --- a/src/main/java/net/openhft/chronicle/map/ChannelProvider.java +++ b/src/main/java/net/openhft/chronicle/map/ChannelProvider.java @@ -47,44 +47,17 @@ import static net.openhft.chronicle.map.Replica.ModificationIterator; * @author Rob Austin. */ final class ChannelProvider implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(ChannelProvider.class.getName()); - static final Map implMapping = new IdentityHashMap<>(); - - static synchronized ChannelProvider getProvider(ReplicationHub hub) throws IOException { - ChannelProvider channelProvider = implMapping.get(hub); - if (channelProvider != null) - return channelProvider; - channelProvider = new ChannelProvider(hub); - - - - TcpTransportAndNetworkConfig tcpConfig = hub.tcpTransportAndNetwork(); - if (tcpConfig != null) { - - final TcpReplicator tcpReplicator = new TcpReplicator( - channelProvider.asReplica, - channelProvider.asEntryExternalizable, - tcpConfig, hub.remoteNodeValidator(), null, hub.connectionListener()); - channelProvider.add(tcpReplicator); - } - - UdpTransportConfig udpConfig = hub.udpTransport(); - if (udpConfig != null) { - final UdpReplicator udpReplicator = - new UdpReplicator( - channelProvider.asReplica, - channelProvider.asEntryExternalizable, - udpConfig); - channelProvider.add(udpReplicator); - if (tcpConfig == null) - LOG.warn(Replicators.ONLY_UDP_WARN_MESSAGE); - } - implMapping.put(hub, channelProvider); - return channelProvider; - } - + private static final Logger LOG = LoggerFactory.getLogger(ChannelProvider.class.getName()); private static final byte BOOTSTRAP_MESSAGE = 'B'; + private final byte localIdentifier; + private final ReplicationHub hub; + private final ReadWriteLock channelDataLock = new ReentrantReadWriteLock(); + // start of channel data + private final Replica[] chronicleChannels; + private final List chronicleChannelList; + private final List chronicleChannelIds; + private final EntryExternalizable[] channelEntryExternalizables; final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { @Override public int sizeOfEntry(@NotNull Bytes entry, int chronicleChannel) { @@ -153,9 +126,16 @@ final class ChannelProvider implements Closeable { } } }; - - - private final byte localIdentifier; + private final AtomicReferenceArray systemModificationIterator = + new AtomicReferenceArray(128); + private final DirectBitSet systemModificationIteratorBitSet = + newBitSet(systemModificationIterator.length()); + private final AtomicReferenceArray modificationIterator = + new AtomicReferenceArray(128); + private final Set replicators = new CopyOnWriteArraySet(); + private final SystemQueue systemMessageQueue; + // end of channel data + private volatile boolean isClosed = false; final Replica asReplica = new Replica() { @Override public void put(final Bytes key, final Bytes value, final byte remoteIdentifier, final long timestamp) { @@ -236,15 +216,11 @@ final class ChannelProvider implements Closeable { channelDataLock.readLock().lock(); try { for (int i = 0, len = chronicleChannelList.size(); i < len; i++) { - try { - ModificationIterator mi = chronicleChannelList.get(i) - .acquireModificationIterator(remoteIdentifier - ); + ModificationIterator mi = chronicleChannelList.get(i) + .acquireModificationIterator(remoteIdentifier + ); - mi.dirtyEntries(fromTimeStamp); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + mi.dirtyEntries(fromTimeStamp); notifier0.onChange(); } } finally { @@ -310,30 +286,6 @@ final class ChannelProvider implements Closeable { ChannelProvider.this.close(); } }; - - private final ReplicationHub hub; - - - private final ReadWriteLock channelDataLock = new ReentrantReadWriteLock(); - - // start of channel data - private final Replica[] chronicleChannels; - private final List chronicleChannelList; - private final List chronicleChannelIds; - private final EntryExternalizable[] channelEntryExternalizables; - private final AtomicReferenceArray systemModificationIterator = - new AtomicReferenceArray(128); - private final DirectBitSet systemModificationIteratorBitSet = - newBitSet(systemModificationIterator.length()); - private final AtomicReferenceArray modificationIterator = - new AtomicReferenceArray(128); - // end of channel data - - private final Set replicators = new CopyOnWriteArraySet(); - - private volatile boolean isClosed = false; - private final SystemQueue systemMessageQueue; - private ChannelProvider(ReplicationHub hub) { localIdentifier = hub.identifier(); this.hub = hub; @@ -358,6 +310,37 @@ final class ChannelProvider implements Closeable { add((short) 0, systemMessageQueue.asReplica, systemMessageQueue.asEntryExternalizable); } + static synchronized ChannelProvider getProvider(ReplicationHub hub) throws IOException { + ChannelProvider channelProvider = implMapping.get(hub); + if (channelProvider != null) + return channelProvider; + channelProvider = new ChannelProvider(hub); + + + TcpTransportAndNetworkConfig tcpConfig = hub.tcpTransportAndNetwork(); + if (tcpConfig != null) { + + final TcpReplicator tcpReplicator = new TcpReplicator( + channelProvider.asReplica, + channelProvider.asEntryExternalizable, + tcpConfig, hub.remoteNodeValidator(), null, hub.connectionListener()); + channelProvider.add(tcpReplicator); + } + + UdpTransportConfig udpConfig = hub.udpTransport(); + if (udpConfig != null) { + final UdpReplicator udpReplicator = + new UdpReplicator( + channelProvider.asReplica, + channelProvider.asEntryExternalizable, + udpConfig); + channelProvider.add(udpReplicator); + if (tcpConfig == null) + LOG.warn(Replicators.ONLY_UDP_WARN_MESSAGE); + } + implMapping.put(hub, channelProvider); + return channelProvider; + } /** * creates a bit set based on a number of bits @@ -370,6 +353,16 @@ final class ChannelProvider implements Closeable { return new SingleThreadedDirectBitSet(bytes); } + private static ByteBufferBytes toBootstrapMessage(int chronicleChannel, final long lastModificationTime, final byte localIdentifier) { + final ByteBufferBytes writeBuffer = new ByteBufferBytes(ByteBuffer.allocate(1 + 1 + 2 + 8)); + writeBuffer.writeByte(BOOTSTRAP_MESSAGE); + writeBuffer.writeByte(localIdentifier); + writeBuffer.writeUnsignedShort(chronicleChannel); + writeBuffer.writeLong(lastModificationTime); + writeBuffer.flip(); + return writeBuffer; + } + public ChronicleChannel createChannel(int channel) { return new ChronicleChannel(channel); } @@ -389,25 +382,11 @@ final class ChannelProvider implements Closeable { // this could be null if one node has a chronicle channel before the other if (chronicleChannels[chronicleChannel] != null) { - try { - chronicleChannels[chronicleChannel].acquireModificationIterator(remoteIdentifier) - .dirtyEntries(lastModificationTime); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + chronicleChannels[chronicleChannel].acquireModificationIterator(remoteIdentifier) + .dirtyEntries(lastModificationTime); } } - private static ByteBufferBytes toBootstrapMessage(int chronicleChannel, final long lastModificationTime, final byte localIdentifier) { - final ByteBufferBytes writeBuffer = new ByteBufferBytes(ByteBuffer.allocate(1 + 1 + 2 + 8)); - writeBuffer.writeByte(BOOTSTRAP_MESSAGE); - writeBuffer.writeByte(localIdentifier); - writeBuffer.writeUnsignedShort(chronicleChannel); - writeBuffer.writeLong(lastModificationTime); - writeBuffer.flip(); - return writeBuffer; - } - private void add(int chronicleChannel, Replica replica, @NotNull EntryExternalizable entryExternalizable) { @@ -485,6 +464,8 @@ final class ChannelProvider implements Closeable { */ class SystemQueue { + private final DirectBitSet systemModificationIteratorBitSet; + private final AtomicReferenceArray systemModificationIterator; final Replica asReplica = new Replica() { @Override public void put(final Bytes key, final Bytes value, final byte remoteIdentifier, final long timestamp) { @@ -575,6 +556,7 @@ final class ChannelProvider implements Closeable { // do nothing } }; + private final MessageHandler messageHandler; final EntryExternalizable asEntryExternalizable = new EntryExternalizable() { @Override public int sizeOfEntry(@NotNull Bytes entry, int chronicleId) { @@ -599,9 +581,6 @@ final class ChannelProvider implements Closeable { messageHandler.onMessage(source); } }; - private final DirectBitSet systemModificationIteratorBitSet; - private final AtomicReferenceArray systemModificationIterator; - private final MessageHandler messageHandler; SystemQueue(DirectBitSet systemModificationIteratorBitSet, AtomicReferenceArray systemModificationIterator, diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java index a0ec91d58..0b646d49a 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMapBuilder.java @@ -68,23 +68,23 @@ import static net.openhft.lang.model.DataValueGenerator.firstPrimitiveFieldType; * ChronicleMapBuilder builder = ChronicleMapBuilder * .of(Key.class, Value.class) * .entries(100500); - * + *

* ChronicleMap map1 = builder.create(); * ChronicleMap map2 = builder.create();} * i. e. created {@code ChronicleMap} instances don't depend on the builder. - * + *

*

{@code ChronicleMapBuilder} is mutable, see a note in {@link ChronicleHashBuilder} interface * documentation. - * + *

*

Later in this documentation, "ChronicleMap" means "ChronicleMaps, created by {@code * ChronicleMapBuilder}", unless specified different, because theoretically someone might provide * {@code ChronicleMap} implementations with completely different properties. - * + *

*

{@code ChronicleMap} ("ChronicleMaps, created by {@code ChronicleMapBuilder}") currently * doesn't support resizing. That is why you must configure {@linkplain #entries(long) number * of entries} you are going to insert into the created map at most. See {@link * #entries(long)} method documentation for more information on this. - * + *

*

If you key or value type is not constantly sized and known to {@code ChronicleHashBuilder}, i. * e. it is not a boxed primitive, data value generated interface, {@link Byteable}, etc. (see the * complete list TODO insert the link to the complete list), you must provide the {@code @@ -115,19 +115,11 @@ public class ChronicleMapBuilder implements Cloneable, private static final int XML_SERIALIZATION = 1; private static final int BINARY_SERIALIZATION = 2; - - private static boolean isDefined(double config) { - return !Double.isNaN(config); - } - private static final boolean strictStateChecks = Boolean.getBoolean("chronicle.strictStateChecks"); - SerializationBuilder keyBuilder; SerializationBuilder valueBuilder; - private String name; - // used when configuring the number of segments. private int minSegments = -1; private int actualSegments = -1; @@ -148,7 +140,6 @@ public class ChronicleMapBuilder implements Cloneable, private ChronicleHashErrorListener errorListener = ChronicleHashErrorListeners.logging(); private boolean putReturnsNull = false; private boolean removeReturnsNull = false; - // replication private TimeProvider timeProvider = TimeProvider.SYSTEM; private BytesMarshallerFactory bytesMarshallerFactory; @@ -158,10 +149,8 @@ public class ChronicleMapBuilder implements Cloneable, private V defaultValue = null; private DefaultValueProvider defaultValueProvider = null; private PrepareValueBytes prepareValueBytes = null; - private SingleChronicleHashReplication singleHashReplication = null; private InetSocketAddress[] pushToAddresses; - ChronicleMapBuilder(Class keyClass, Class valueClass) { keyBuilder = new SerializationBuilder<>(keyClass, SerializationBuilder.Role.KEY); valueBuilder = new SerializationBuilder<>(valueClass, SerializationBuilder.Role.VALUE); @@ -172,6 +161,10 @@ public class ChronicleMapBuilder implements Cloneable, defaultValue = (V) EMTRY_STRING_BUILDER; } + private static boolean isDefined(double config) { + return !Double.isNaN(config); + } + /** * Returns a new {@code ChronicleMapBuilder} instance which is able to {@linkplain #create() * create} maps with the specified key and value classes. @@ -222,6 +215,95 @@ public class ChronicleMapBuilder implements Cloneable, return obj != null ? obj + "" : "not configured"; } + private static void checkAverageSize(double averageSize, String role) { + if (averageSize <= 0 || Double.isNaN(averageSize) || + Double.isInfinite(averageSize)) { + throw new IllegalArgumentException("Average " + role + " size must be a positive, " + + "finite number"); + } + } + + private static void checkSizeIsNotStaticallyKnown(SerializationBuilder builder) { + if (builder.sizeIsStaticallyKnown) + throw new IllegalStateException("Size of type " + builder.eClass + + " is statically known and shouldn't be specified manually"); + } + + private static double averageSizeEncodingSize( + SerializationBuilder builder, double averageSize) { + SizeMarshaller sizeMarshaller = builder.sizeMarshaller(); + if (averageSize == round(averageSize)) + return sizeMarshaller.sizeEncodingSize(round(averageSize)); + long lower = (long) averageSize; + long upper = lower + 1; + int lowerEncodingSize = sizeMarshaller.sizeEncodingSize(lower); + int upperEncodingSize = sizeMarshaller.sizeEncodingSize(upper); + if (lowerEncodingSize == upperEncodingSize) + return lowerEncodingSize; + return lower * (upper - averageSize) + upper * (averageSize - lower); + } + + static int greatestCommonDivisor(int a, int b) { + if (b == 0) return a; + return greatestCommonDivisor(b, a % b); + } + + private static int maxDefaultChunksPerAverageEntry(boolean replicated) { + return replicated ? 4 : 8; + } + + private static int estimateSegmentsForEntries(long size) { + if (size > 200 << 20) + return 256; + if (size >= 1 << 20) + return 128; + if (size >= 128 << 10) + return 64; + if (size >= 16 << 10) + return 32; + if (size >= 4 << 10) + return 16; + if (size >= 1 << 10) + return 8; + return 1; + } + + private static boolean trySerializeHeaderViaXStream( + VanillaChronicleMap map, ObjectOutputStream oos) throws IOException { + Class xStreamClass; + try { + xStreamClass = + Class.forName("net.openhft.xstream.MapHeaderSerializationXStream"); + } catch (ClassNotFoundException e) { + xStreamClass = null; + } + if (xStreamClass == null) { + LOG.info("xStream not found, use binary ChronicleMap header serialization"); + return false; + } + try { + oos.writeByte(XML_SERIALIZATION); + Method toXML = xStreamClass.getMethod("toXML", Object.class, OutputStream.class); + toXML.invoke(xStreamClass.newInstance(), map, oos); + return true; + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException | + InstantiationException e) { + throw new AssertionError(e); + } + } + + private static Object deserializeHeaderViaXStream(ObjectInputStream ois) { + try { + Class xStreamClass = + Class.forName("net.openhft.xstream.MapHeaderSerializationXStream"); + Method fromXML = xStreamClass.getMethod("fromXML", InputStream.class); + return fromXML.invoke(xStreamClass.newInstance(), ois); + } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException | + IllegalAccessException | InstantiationException e) { + throw new AssertionError(e); + } + } + public ChronicleMapBuilder pushTo(InetSocketAddress... addresses) { this.pushToAddresses = addresses; return this; @@ -243,7 +325,7 @@ public class ChronicleMapBuilder implements Cloneable, /** * {@inheritDoc} - * + *

*

Example: if keys in your map(s) are English words in {@link String} form, average English * word length is 5.1, configure average key size of 6:

{@code
      * ChronicleMap wordFrequencies = ChronicleMapBuilder
@@ -272,7 +354,7 @@ public class ChronicleMapBuilder implements Cloneable,
 
     /**
      * {@inheritDoc}
-     *
+     * 

*

For example, if your keys are Git commit hashes:

{@code
      * Map gitCommitMessagesByHash =
      *     ChronicleMapBuilder.of(byte[].class, String.class)
@@ -297,14 +379,14 @@ public class ChronicleMapBuilder implements Cloneable,
      * Configures the average number of bytes, taken by serialized form of values, put into maps,
      * created by this builder. If value size is always the same, call {@link
      * #constantValueSizeBySample(Object)} method instead of this one.
-     *
+     * 

*

{@code ChronicleHashBuilder} implementation heuristically chooses {@linkplain * #actualChunkSize(int) the actual chunk size} based on this configuration and the key size, * that, however, might result to quite high internal fragmentation, i. e. losses because only * integral number of chunks could be allocated for the entry. If you want to avoid this, you * should manually configure the actual chunk size in addition to this average value size * configuration, which is anyway needed. - * + *

*

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is * known statically, it is automatically accounted and shouldn't be specified by user. * @@ -324,28 +406,14 @@ public class ChronicleMapBuilder implements Cloneable, return this; } - private static void checkAverageSize(double averageSize, String role) { - if (averageSize <= 0 || Double.isNaN(averageSize) || - Double.isInfinite(averageSize)) { - throw new IllegalArgumentException("Average " + role + " size must be a positive, " + - "finite number"); - } - } - - private static void checkSizeIsNotStaticallyKnown(SerializationBuilder builder) { - if (builder.sizeIsStaticallyKnown) - throw new IllegalStateException("Size of type " + builder.eClass + - " is statically known and shouldn't be specified manually"); - } - /** * Configures the constant number of bytes, taken by serialized form of values, put into maps, * created by this builder. This is done by providing the {@code sampleValue}, all values should * take the same number of bytes in serialized form, as this sample object. - * + *

*

If values are of boxed primitive type or {@link Byteable} subclass, i. e. if value size is * known statically, it is automatically accounted and this method shouldn't be called. - * + *

*

If value size varies, method {@link #averageValueSize(double)} should be called instead of * this one. * @@ -394,16 +462,6 @@ public class ChronicleMapBuilder implements Cloneable, return this; } - static class EntrySizeInfo { - final double averageEntrySize; - final int worstAlignment; - - public EntrySizeInfo(double averageEntrySize, int worstAlignment) { - this.averageEntrySize = averageEntrySize; - this.worstAlignment = worstAlignment; - } - } - private EntrySizeInfo entrySizeInfo(boolean replicated) { double size = metaDataBytes; double keySize = averageKeySize(); @@ -467,20 +525,6 @@ public class ChronicleMapBuilder implements Cloneable, return keyBuilder.constantSizeMarshaller(); } - private static double averageSizeEncodingSize( - SerializationBuilder builder, double averageSize) { - SizeMarshaller sizeMarshaller = builder.sizeMarshaller(); - if (averageSize == round(averageSize)) - return sizeMarshaller.sizeEncodingSize(round(averageSize)); - long lower = (long) averageSize; - long upper = lower + 1; - int lowerEncodingSize = sizeMarshaller.sizeEncodingSize(lower); - int upperEncodingSize = sizeMarshaller.sizeEncodingSize(upper); - if (lowerEncodingSize == upperEncodingSize) - return lowerEncodingSize; - return lower * (upper - averageSize) + upper * (averageSize - lower); - } - private int worstAlignmentAssumingChunkSize( long constantSizeBeforeAlignment, int chunkSize) { Alignment valueAlignment = valueAlignment(); @@ -502,11 +546,6 @@ public class ChronicleMapBuilder implements Cloneable, return entrySizeInfo(replicated).worstAlignment; } - static int greatestCommonDivisor(int a, int b) { - if (b == 0) return a; - return greatestCommonDivisor(b, a % b); - } - long chunkSize(boolean replicated) { if (actualChunkSize > 0) return actualChunkSize; @@ -535,10 +574,6 @@ public class ChronicleMapBuilder implements Cloneable, return (entrySizeInfo(replicated).averageEntrySize + chunkSize - 1) / chunkSize; } - private static int maxDefaultChunksPerAverageEntry(boolean replicated) { - return replicated ? 4 : 8; - } - @Override public ChronicleMapBuilder maxChunksPerEntry(int maxChunksPerEntry) { if (maxChunksPerEntry < 1) @@ -565,14 +600,14 @@ public class ChronicleMapBuilder implements Cloneable, /** * Configures alignment strategy of address in memory of entries and independently of address in * memory of values within entries in ChronicleMaps, created by this builder. - * + *

*

Useful when values of the map are updated intensively, particularly fields with volatile * access, because it doesn't work well if the value crosses cache lines. Also, on some * (nowadays rare) architectures any misaligned memory access is more expensive than aligned. - * + *

*

If values couldn't reference off-heap memory (i. e. it is not {@link Byteable} or "data * value generated"), alignment configuration makes no sense and forbidden. - * + *

*

Default is {@link Alignment#NO_ALIGNMENT} if values couldn't reference off-heap memory, * otherwise chosen heuristically (configure explicitly for being sure and to compare * performance in your case). @@ -741,22 +776,6 @@ public class ChronicleMapBuilder implements Cloneable, : segmentsForEntries; } - private static int estimateSegmentsForEntries(long size) { - if (size > 200 << 20) - return 256; - if (size >= 1 << 20) - return 128; - if (size >= 128 << 10) - return 64; - if (size >= 16 << 10) - return 32; - if (size >= 4 << 10) - return 16; - if (size >= 1 << 10) - return 8; - return 1; - } - @Override public ChronicleMapBuilder actualSegments(int actualSegments) { checkSegments(actualSegments); @@ -1310,42 +1329,6 @@ public class ChronicleMapBuilder implements Cloneable, return establishReplication(map, singleHashReplication, channel); } - private static boolean trySerializeHeaderViaXStream( - VanillaChronicleMap map, ObjectOutputStream oos) throws IOException { - Class xStreamClass; - try { - xStreamClass = - Class.forName("net.openhft.xstream.MapHeaderSerializationXStream"); - } catch (ClassNotFoundException e) { - xStreamClass = null; - } - if (xStreamClass == null) { - LOG.info("xStream not found, use binary ChronicleMap header serialization"); - return false; - } - try { - oos.writeByte(XML_SERIALIZATION); - Method toXML = xStreamClass.getMethod("toXML", Object.class, OutputStream.class); - toXML.invoke(xStreamClass.newInstance(), map, oos); - return true; - } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException | - InstantiationException e) { - throw new AssertionError(e); - } - } - - private static Object deserializeHeaderViaXStream(ObjectInputStream ois) { - try { - Class xStreamClass = - Class.forName("net.openhft.xstream.MapHeaderSerializationXStream"); - Method fromXML = xStreamClass.getMethod("fromXML", InputStream.class); - return fromXML.invoke(xStreamClass.newInstance(), ois); - } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException | - IllegalAccessException | InstantiationException e) { - throw new AssertionError(e); - } - } - ChronicleMap createWithoutFile( SingleChronicleHashReplication singleHashReplication, ReplicationChannel channel) { try { @@ -1390,8 +1373,7 @@ public class ChronicleMapBuilder implements Cloneable, } private VanillaChronicleMap newMap( - SingleChronicleHashReplication singleHashReplication, ReplicationChannel channel) - throws IOException { + SingleChronicleHashReplication singleHashReplication, ReplicationChannel channel) { boolean replicated = singleHashReplication != null || channel != null; preMapConstruction(replicated); if (replicated) { @@ -1485,5 +1467,15 @@ public class ChronicleMapBuilder implements Cloneable, return (int) Math.min(Math.max(2L, entries() >> 10), Math.min(64, maxChunksPerEntry())); } + + static class EntrySizeInfo { + final double averageEntrySize; + final int worstAlignment; + + public EntrySizeInfo(double averageEntrySize, int worstAlignment) { + this.averageEntrySize = averageEntrySize; + this.worstAlignment = worstAlignment; + } + } } diff --git a/src/main/java/net/openhft/chronicle/map/EngineReplicationLangBytes.java b/src/main/java/net/openhft/chronicle/map/EngineReplicationLangBytes.java index d1636f9b5..65f48f25c 100644 --- a/src/main/java/net/openhft/chronicle/map/EngineReplicationLangBytes.java +++ b/src/main/java/net/openhft/chronicle/map/EngineReplicationLangBytes.java @@ -143,7 +143,7 @@ public interface EngineReplicationLangBytes extends Closeable { * * @param fromTimeStamp the timestamp from which all entries should be dirty */ - void dirtyEntries(long fromTimeStamp) throws InterruptedException; + void dirtyEntries(long fromTimeStamp); /** * the {@code modificationNotifier} is called when ever there is a change applied to the diff --git a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java index f35b2c772..68857d2e7 100644 --- a/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java +++ b/src/main/java/net/openhft/chronicle/map/NodeDiscovery.java @@ -46,6 +46,13 @@ import static net.openhft.chronicle.map.DiscoveryNodeBytesMarshallable.ProposedN import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.BOOTSTRAP_BYTES; import static net.openhft.chronicle.map.NodeDiscoveryBroadcaster.LOG; +interface NodeDiscoveryEventListener { + /** + * called when we have received a UDP message, this is called after the message has been parsed + */ + void onRemoteNodeEvent(KnownNodes remoteNode, ConcurrentExpiryMap proposedIdentifiersWithHost); +} + /** * @author Rob Austin. * @@ -56,10 +63,6 @@ class NodeDiscovery { public static final int DEFAULT_UDP_BROADCAST_PORT = 8129; public static final short DEFAULT_TCP_PORT = (short) 8123; public static final int SERIALIZED_ENTRY_SIZE = 1024; - - private final AddressAndPort ourAddressAndPort; - private final UdpTransportConfig udpConfig; - public static final InetAddress IP_MULIT_CAST_GROUP; static { @@ -72,6 +75,8 @@ class NodeDiscovery { IP_MULIT_CAST_GROUP = mask; } + private final AddressAndPort ourAddressAndPort; + private final UdpTransportConfig udpConfig; private final DiscoveryNodeBytesMarshallable discoveryNodeBytesMarshallable; private final AtomicReference nodeDiscoveryEventListenerAtomicReference = new AtomicReference(); @@ -100,6 +105,63 @@ class NodeDiscovery { discoveryNodeBytesMarshallable.setModificationNotifier(nodeDiscoveryBroadcaster); } + private static Collection toInetSocketCollection(Set source) + throws UnknownHostException { + + // make a safe copy + final HashSet addressAndPorts = new HashSet(source); + + if (addressAndPorts.isEmpty()) + return Collections.emptyList(); + + ArrayList addresses = + new ArrayList(addressAndPorts.size()); + + for (final AddressAndPort addressAndPort : addressAndPorts) { + addresses.add(new InetSocketAddress(InetAddress.getByAddress(addressAndPort.address()) + .getHostAddress(), addressAndPort.port())); + } + return addresses; + } + + static byte proposeRandomUnusedIdentifier(final DirectBitSet knownIdentifiers, + boolean isFirstTime) throws UnknownHostException { + byte possible; + + // the first time, rather than choosing a random number, we will choose the last value of the IP + // address as our random number, ( or at least something that is based upon it) + if (isFirstTime) { + byte[] address = InetAddress.getLocalHost().getAddress(); + int lastAddress = address[address.length - 1]; + if (lastAddress > 127) + lastAddress = lastAddress - 127; + if (lastAddress > 127) + lastAddress = lastAddress - 127; + + possible = (byte) lastAddress; + } else + possible = (byte) (Math.random() * 128); + + int count = 0; + for (; ; ) { + if (knownIdentifiers.setIfClear(possible)) { + return possible; + } + + count++; + + if (count == 128) { + throw new IllegalStateException("The grid is full, its not possible for any more nodes to " + + "going the grid."); + } + + if (possible == Byte.MAX_VALUE) + possible = 0; + else + possible++; + } + } + public synchronized ReplicationHubFindByName mapByName() throws IOException, InterruptedException { @@ -250,25 +312,6 @@ class NodeDiscovery { return new ReplicationHubFindByName(replicationHub); } - private static Collection toInetSocketCollection(Set source) - throws UnknownHostException { - - // make a safe copy - final HashSet addressAndPorts = new HashSet(source); - - if (addressAndPorts.isEmpty()) - return Collections.emptyList(); - - ArrayList addresses = - new ArrayList(addressAndPorts.size()); - - for (final AddressAndPort addressAndPort : addressAndPorts) { - addresses.add(new InetSocketAddress(InetAddress.getByAddress(addressAndPort.address()) - .getHostAddress(), addressAndPort.port())); - } - return addresses; - } - /** * bitwise OR's the two bit sets or put another way, merges the source bitset into the * destination bitset and returns the destination @@ -292,44 +335,6 @@ class NodeDiscovery { return destination; } - static byte proposeRandomUnusedIdentifier(final DirectBitSet knownIdentifiers, - boolean isFirstTime) throws UnknownHostException { - byte possible; - - // the first time, rather than choosing a random number, we will choose the last value of the IP - // address as our random number, ( or at least something that is based upon it) - if (isFirstTime) { - byte[] address = InetAddress.getLocalHost().getAddress(); - int lastAddress = address[address.length - 1]; - if (lastAddress > 127) - lastAddress = lastAddress - 127; - if (lastAddress > 127) - lastAddress = lastAddress - 127; - - possible = (byte) lastAddress; - } else - possible = (byte) (Math.random() * 128); - - int count = 0; - for (; ; ) { - if (knownIdentifiers.setIfClear(possible)) { - return possible; - } - - count++; - - if (count == 128) { - throw new IllegalStateException("The grid is full, its not possible for any more nodes to " + - "going the grid."); - } - - if (possible == Byte.MAX_VALUE) - possible = 0; - else - possible++; - } - } - } /** @@ -341,10 +346,8 @@ class NodeDiscovery { class NodeDiscoveryBroadcaster extends UdpChannelReplicator { public static final Logger LOG = LoggerFactory.getLogger(NodeDiscoveryBroadcaster.class.getName()); - - private static final byte UNUSED = (byte) -1; - static final ByteBufferBytes BOOTSTRAP_BYTES; + private static final byte UNUSED = (byte) -1; static { final String BOOTSTRAP = "BOOTSTRAP"; @@ -353,14 +356,6 @@ class NodeDiscoveryBroadcaster extends UdpChannelReplicator { BOOTSTRAP_BYTES.append(BOOTSTRAP); } - static String toString(DirectBitSet bitSet) { - final StringBuilder builder = new StringBuilder(); - for (int i = 0; i < bitSet.size(); i++) { - builder.append(bitSet.get(i) ? '1' : '0'); - } - return builder.toString(); - } - /** * @param replicationConfig * @param externalizable @@ -369,8 +364,7 @@ class NodeDiscoveryBroadcaster extends UdpChannelReplicator { NodeDiscoveryBroadcaster( final UdpTransportConfig replicationConfig, final int serializedEntrySize, - final BytesMarshallable externalizable) - throws IOException { + final BytesMarshallable externalizable) throws IOException { super(replicationConfig, UNUSED); @@ -383,6 +377,14 @@ class NodeDiscoveryBroadcaster extends UdpChannelReplicator { start(); } + static String toString(DirectBitSet bitSet) { + final StringBuilder builder = new StringBuilder(); + for (int i = 0; i < bitSet.size(); i++) { + builder.append(bitSet.get(i) ? '1' : '0'); + } + return builder.toString(); + } + static class UdpSocketChannelEntryReader implements EntryReader { private final ByteBuffer in; @@ -411,8 +413,7 @@ class NodeDiscoveryBroadcaster extends UdpChannelReplicator { * @throws IOException * @throws InterruptedException */ - public void readAll(@NotNull final DatagramChannel socketChannel) throws IOException, - InterruptedException { + public void readAll(@NotNull final DatagramChannel socketChannel) throws IOException { out.clear(); in.clear(); @@ -574,6 +575,8 @@ class KnownNodes implements BytesMarshallable { } class AddressAndPort implements Comparable, BytesMarshallable { + final static int INADDRSZ = 16; + final static int INT16SZ = 2; private byte[] address; private short port; @@ -585,6 +588,23 @@ class AddressAndPort implements Comparable, BytesMarshallable { public AddressAndPort() { } + static String numericToTextFormat(byte[] src) { + if (src.length == 4) { + return (src[0] & 0xff) + "." + (src[1] & 0xff) + "." + (src[2] & 0xff) + "." + (src[3] & 0xff); + } + + StringBuilder sb = new StringBuilder(39); + for (int i = 0; i < (INADDRSZ / INT16SZ); i++) { + sb.append(Integer.toHexString(((src[i << 1] << 8) & 0xff00) + | (src[(i << 1) + 1] & 0xff))); + if (i < (INADDRSZ / INT16SZ) - 1) { + sb.append(":"); + } + } + + return sb.toString(); + } + public byte[] getAddress() { return address; } @@ -601,7 +621,6 @@ class AddressAndPort implements Comparable, BytesMarshallable { return port; } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -640,26 +659,6 @@ class AddressAndPort implements Comparable, BytesMarshallable { '}'; } - final static int INADDRSZ = 16; - final static int INT16SZ = 2; - - static String numericToTextFormat(byte[] src) { - if (src.length == 4) { - return (src[0] & 0xff) + "." + (src[1] & 0xff) + "." + (src[2] & 0xff) + "." + (src[3] & 0xff); - } - - StringBuilder sb = new StringBuilder(39); - for (int i = 0; i < (INADDRSZ / INT16SZ); i++) { - sb.append(Integer.toHexString(((src[i << 1] << 8) & 0xff00) - | (src[(i << 1) + 1] & 0xff))); - if (i < (INADDRSZ / INT16SZ) - 1) { - sb.append(":"); - } - } - - return sb.toString(); - } - @Override public void readMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes in) throws IllegalStateException { short len = in.readShort(); @@ -683,20 +682,27 @@ class AddressAndPort implements Comparable, BytesMarshallable { class DiscoveryNodeBytesMarshallable implements BytesMarshallable { - private AddressAndPort sourceAddressAndPort = new AddressAndPort(); + final ConcurrentExpiryMap proposedIdentifiersWithHost = new + ConcurrentExpiryMap(AddressAndPort.class, + ProposedNodes.class); private final KnownNodes remoteNode; private final AtomicBoolean bootstrapRequired = new AtomicBoolean(); private final AtomicReference nodeDiscoveryEventListener; - + Replica.ModificationNotifier modificationNotifier; + private AddressAndPort sourceAddressAndPort = new AddressAndPort(); private ProposedNodes ourProposedIdentifier; - - final ConcurrentExpiryMap proposedIdentifiersWithHost = new - ConcurrentExpiryMap(AddressAndPort.class, - ProposedNodes.class); - private AddressAndPort ourAddressAndPort; + public DiscoveryNodeBytesMarshallable(@NotNull final KnownNodes remoteNode, + @NotNull final AtomicReference + nodeDiscoveryEventListener, + @NotNull final AddressAndPort ourAddressAndPort) { + this.remoteNode = remoteNode; + this.nodeDiscoveryEventListener = nodeDiscoveryEventListener; + this.ourAddressAndPort = ourAddressAndPort; + } + public ProposedNodes getOurProposedIdentifier() { return ourProposedIdentifier; } @@ -709,17 +715,6 @@ class DiscoveryNodeBytesMarshallable implements BytesMarshallable { this.modificationNotifier = modificationNotifier; } - Replica.ModificationNotifier modificationNotifier; - - public DiscoveryNodeBytesMarshallable(@NotNull final KnownNodes remoteNode, - @NotNull final AtomicReference - nodeDiscoveryEventListener, - @NotNull final AddressAndPort ourAddressAndPort) { - this.remoteNode = remoteNode; - this.nodeDiscoveryEventListener = nodeDiscoveryEventListener; - this.ourAddressAndPort = ourAddressAndPort; - } - public KnownNodes getRemoteNodes() { return remoteNode; } @@ -833,6 +828,18 @@ class DiscoveryNodeBytesMarshallable implements BytesMarshallable { modificationNotifier.onChange(); } + /** + * sends a bootstrap message to the other nodes in the grid, the bootstrap message contains the + * host:port and perhaps even proposed identifier of the node that sent it. + * + * @param proposedNodes + */ + public void sendBootStrap(ProposedNodes proposedNodes) { + setOurProposedIdentifier(proposedNodes); + bootstrapRequired.set(true); + onChange(); + } + static class ProposedNodes implements BytesMarshallable { private byte identifier; @@ -842,10 +849,6 @@ class DiscoveryNodeBytesMarshallable implements BytesMarshallable { public ProposedNodes() { } - public byte identifier() { - return identifier; - } - ProposedNodes(@NotNull final AddressAndPort addressAndPort, byte identifier) { this.addressAndPort = addressAndPort; @@ -853,6 +856,10 @@ class DiscoveryNodeBytesMarshallable implements BytesMarshallable { this.timestamp = System.currentTimeMillis(); } + public byte identifier() { + return identifier; + } + AddressAndPort addressAndPort() { return addressAndPort; } @@ -901,25 +908,6 @@ class DiscoveryNodeBytesMarshallable implements BytesMarshallable { } } - /** - * sends a bootstrap message to the other nodes in the grid, the bootstrap message contains the - * host:port and perhaps even proposed identifier of the node that sent it. - * - * @param proposedNodes - */ - public void sendBootStrap(ProposedNodes proposedNodes) { - setOurProposedIdentifier(proposedNodes); - bootstrapRequired.set(true); - onChange(); - } - -} - -interface NodeDiscoveryEventListener { - /** - * called when we have received a UDP message, this is called after the message has been parsed - */ - void onRemoteNodeEvent(KnownNodes remoteNode, ConcurrentExpiryMap proposedIdentifiersWithHost); } class ConcurrentExpiryMap implements BytesMarshallable { @@ -928,100 +916,14 @@ class ConcurrentExpiryMap(); private final Class kClass; private final Class vClass; + // this is used for expiry + private final Queue>> queue = new ConcurrentLinkedQueue>>(); ConcurrentExpiryMap(Class kClass, Class vClass) { this.kClass = kClass; this.vClass = vClass; } - @Override - public String toString() { - return "ConcurrentExpiryMap{" + map + '}'; - } - - @Override - public void readMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes in) throws IllegalStateException { - - short size = in.readShort(); - try { - for (int i = 0; i < size; i++) { - final K k = kClass.newInstance(); - k.readMarshallable(in); - - final V v = vClass.newInstance(); - v.readMarshallable(in); - - map.put(k, v); - } - } catch (Exception e) { - LOG.error("", e); - } - } - - @Override - public void writeMarshallable(@net.openhft.lang.model.constraints.NotNull Bytes out) { - final Map safeCopy = new HashMap(map); - out.writeShort(safeCopy.size()); - for (Map.Entry entry : safeCopy.entrySet()) { - entry.getKey().writeMarshallable(out); - entry.getValue().writeMarshallable(out); - } - } - - class W { - final long timestamp; - final V v; - - W(V v) { - this.v = v; - this.timestamp = System.currentTimeMillis(); - } - } - - void put(final K k, final V v) { - map.put(k, v); - final W w = new W(v); - queue.add(new Map.Entry>() { - @Override - public K getKey() { - return k; - } - - @Override - public W getValue() { - return w; - } - - @Override - public W setValue(W value) { - throw new UnsupportedOperationException(); - } - }); - } - - // this is used for expiry - private final Queue>> queue = new ConcurrentLinkedQueue>>(); - - java.util.Collection values() { - return map.values(); - } - - void expireEntries(long timeOlderThan) { - for (; ; ) { - final Map.Entry> e = this.queue.peek(); - - if (e == null) - break; - - if (e.getValue().timestamp < timeOlderThan) { - // only remote it if it has not changed - map.remove(e.getKey(), e.getValue().v); - } - - this.queue.poll(); - } - } - public static void main(String... args) { String ip; try { @@ -1047,11 +949,11 @@ class ConcurrentExpiryMap * for (String suggestedName : new String[]{"en0", "eth0"}) { networkInterface = * NetworkInterface.getByName(suggestedName); if (networkInterface != null) break; } return * networkInterface; } - * + *

* /** * * @return the default network interface, or @@ -1091,4 +993,89 @@ class ConcurrentExpiryMap safeCopy = new HashMap(map); + out.writeShort(safeCopy.size()); + for (Map.Entry entry : safeCopy.entrySet()) { + entry.getKey().writeMarshallable(out); + entry.getValue().writeMarshallable(out); + } + } + + void put(final K k, final V v) { + map.put(k, v); + final W w = new W(v); + queue.add(new Map.Entry>() { + @Override + public K getKey() { + return k; + } + + @Override + public W getValue() { + return w; + } + + @Override + public W setValue(W value) { + throw new UnsupportedOperationException(); + } + }); + } + + java.util.Collection values() { + return map.values(); + } + + void expireEntries(long timeOlderThan) { + for (; ; ) { + final Map.Entry> e = this.queue.peek(); + + if (e == null) + break; + + if (e.getValue().timestamp < timeOlderThan) { + // only remote it if it has not changed + map.remove(e.getKey(), e.getValue().v); + } + + this.queue.poll(); + } + } + + class W { + final long timestamp; + final V v; + + W(V v) { + this.v = v; + this.timestamp = System.currentTimeMillis(); + } + } } diff --git a/src/main/java/net/openhft/chronicle/map/Replica.java b/src/main/java/net/openhft/chronicle/map/Replica.java index 4b3ea4c62..7198e9fe2 100644 --- a/src/main/java/net/openhft/chronicle/map/Replica.java +++ b/src/main/java/net/openhft/chronicle/map/Replica.java @@ -117,7 +117,7 @@ interface Replica extends Closeable, EngineReplicationLangBytes { * * @param fromTimeStamp the timestamp from which all entries should be dirty */ - void dirtyEntries(long fromTimeStamp) throws InterruptedException; + void dirtyEntries(long fromTimeStamp); /** * the {@code modificationNotifier} is called when ever there is a change applied to the diff --git a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java index a24732d8d..cf559ea22 100644 --- a/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ReplicatedChronicleMap.java @@ -99,25 +99,22 @@ final class ReplicatedChronicleMap closeables; - private transient Bytes identifierUpdatedBytes; - - private transient ATSDirectBitSet modIterSet; private final AtomicReferenceArray modificationIterators = new AtomicReferenceArray(127 + RESERVED_MOD_ITER); - + transient Set closeables; + private transient Bytes identifierUpdatedBytes; + private transient ATSDirectBitSet modIterSet; private transient long startOfModificationIterators; private boolean bootstrapOnlyLocalEntries; public ReplicatedChronicleMap(@NotNull ChronicleMapBuilder builder, - AbstractReplication replication) - throws IOException { + AbstractReplication replication) { super(builder); this.timeProvider = builder.timeProvider(); @@ -129,6 +126,14 @@ final class ReplicatedChronicleMap 0 && source.remaining() > 0) { + destination.writeByte(source.readByte()); + } + + } + private int assignedModIterBitSetSizeInBytes() { return (int) CACHE_LINES.align(BYTES.alignAndConvert(127 + RESERVED_MOD_ITER, BITS), BYTES); } @@ -310,7 +315,6 @@ final class ReplicatedChronicleMap, RV, VB extends RV, VBI, MVBI extends MetaBytesInterop> @@ -412,14 +416,6 @@ final class ReplicatedChronicleMap 0 && source.remaining() > 0) { - destination.writeByte(source.readByte()); - } - - } - @Override public void remove(final Bytes key, final byte remoteIdentifier, @@ -516,7 +512,7 @@ final class ReplicatedChronicleMap.Segment { + final MultiStoreBytes timestampBytes = new MultiStoreBytes(); + Segment(Bytes segmentHeader, NativeBytes bytes, int index) { super(segmentHeader, bytes, index); } @@ -982,7 +986,6 @@ final class ReplicatedChronicleMap { private static final long serialVersionUID = 0L; @@ -1744,26 +1738,22 @@ final class ReplicatedChronicleMap implements FindByName { public T create( MapInstanceBuilder config) - throws IOException, TimeoutException, InterruptedException { + throws TimeoutException, InterruptedException { int withChannelId = nextFreeChannel.incrementAndGet(); diff --git a/src/main/java/net/openhft/chronicle/map/Replicators.java b/src/main/java/net/openhft/chronicle/map/Replicators.java index 7532cc8a8..04c471885 100644 --- a/src/main/java/net/openhft/chronicle/map/Replicators.java +++ b/src/main/java/net/openhft/chronicle/map/Replicators.java @@ -50,7 +50,7 @@ final class Replicators { protected Closeable applyTo(@NotNull final ChronicleMapBuilder builder, @NotNull final Replica replica, @NotNull final EntryExternalizable entryExternalizable, - final ChronicleMap chronicleMap) throws IOException { + final ChronicleMap chronicleMap) { replication.engineReplicator().set(replica); @@ -65,6 +65,47 @@ final class Replicators { }; } + static Replicator tcp(final AbstractReplication replication) { + return new Replicator() { + + + @Override + protected Closeable applyTo(@NotNull final ChronicleMapBuilder builder, + @NotNull final Replica replica, + @NotNull final EntryExternalizable entryExternalizable, + final ChronicleMap chronicleMap) throws IOException { + + TcpTransportAndNetworkConfig tcpConfig = replication.tcpTransportAndNetwork(); + + StatelessClientParameters statelessClientParameters = + new StatelessClientParameters( + (VanillaChronicleMap) chronicleMap, + builder.keyBuilder, + builder.valueBuilder); + + return new TcpReplicator(replica, entryExternalizable, + tcpConfig, + replication.remoteNodeValidator(), + statelessClientParameters, + replication.connectionListener()); + } + }; + } + + static Replicator udp( + final UdpTransportConfig replicationConfig) { + return new Replicator() { + @Override + protected Closeable applyTo(@NotNull final ChronicleMapBuilder builder, + @NotNull final Replica map, + @NotNull final EntryExternalizable entryExternalizable, + final ChronicleMap chronicleMap) throws IOException { + return new UdpReplicator(map, entryExternalizable, replicationConfig + ); + } + }; + } + static class OutBuffer implements BufferResizer { @NotNull @@ -73,6 +114,11 @@ final class Replicators { @NotNull private ByteBuffer out; + OutBuffer(final int tcpBufferSize) { + out = ByteBuffer.allocateDirect(tcpBufferSize); + in = new ByteBufferBytes(out); + } + @NotNull public ByteBufferBytes in() { return in; @@ -83,12 +129,6 @@ final class Replicators { return out; } - - OutBuffer(final int tcpBufferSize) { - out = ByteBuffer.allocateDirect(tcpBufferSize); - in = new ByteBufferBytes(out); - } - @Override public Bytes resizeBuffer(int newCapacity) { @@ -128,46 +168,4 @@ final class Replicators { } } - - static Replicator tcp(final AbstractReplication replication) { - return new Replicator() { - - - @Override - protected Closeable applyTo(@NotNull final ChronicleMapBuilder builder, - @NotNull final Replica replica, - @NotNull final EntryExternalizable entryExternalizable, - final ChronicleMap chronicleMap) throws IOException { - - TcpTransportAndNetworkConfig tcpConfig = replication.tcpTransportAndNetwork(); - - StatelessClientParameters statelessClientParameters = - new StatelessClientParameters( - (VanillaChronicleMap) chronicleMap, - builder.keyBuilder, - builder.valueBuilder); - - return new TcpReplicator(replica, entryExternalizable, - tcpConfig, - replication.remoteNodeValidator(), - statelessClientParameters, - replication.connectionListener()); - } - }; - } - - static Replicator udp( - final UdpTransportConfig replicationConfig) { - return new Replicator() { - @Override - protected Closeable applyTo(@NotNull final ChronicleMapBuilder builder, - @NotNull final Replica map, - @NotNull final EntryExternalizable entryExternalizable, - final ChronicleMap chronicleMap) - throws IOException { - return new UdpReplicator(map, entryExternalizable, replicationConfig - ); - } - }; - } } diff --git a/src/main/java/net/openhft/chronicle/map/SharedSegment.java b/src/main/java/net/openhft/chronicle/map/SharedSegment.java index 54bbc46f7..91c369e5e 100644 --- a/src/main/java/net/openhft/chronicle/map/SharedSegment.java +++ b/src/main/java/net/openhft/chronicle/map/SharedSegment.java @@ -30,7 +30,7 @@ interface SharedSegment { /** * if passed segmentState is null, null is returned instead of readLock */ - AutoCloseable readLock(@Nullable VanillaChronicleMap.SegmentState segmentState) throws InterruptedException; + AutoCloseable readLock(@Nullable VanillaChronicleMap.SegmentState segmentState); Map.Entry getEntry(@NotNull VanillaChronicleMap.SegmentState segmentState, long pos); diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index b166c97d6..ec9e5e77f 100755 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -58,13 +58,25 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon private final byte[] connectionByte = new byte[1]; private final ByteBuffer connectionOutBuffer = ByteBuffer.wrap(connectionByte); private final String name; - + private final InetSocketAddress remoteAddress; + private final long timeoutMs; + private final int tcpBufferSize; + private final ReentrantLock inBytesLock = new ReentrantLock(true); + private final ReentrantLock outBytesLock = new ReentrantLock(); + @NotNull + private final AtomicLong transactionID = new AtomicLong(0); + long largestEntrySoFar = 0; private ByteBuffer outBuffer; private ByteBufferBytes outBytes; + private final BufferResizer outBufferResizer = new BufferResizer() { + @Override + public Bytes resizeBuffer(int newCapacity) { + return resizeBufferOutBuffer(newCapacity); + } + }; private ByteBuffer inBuffer; private ByteBufferBytes inBytes; - @NotNull private ReaderWithSize keyReaderWithSize; @NotNull @@ -74,75 +86,19 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon @NotNull private WriterWithSize valueWriterWithSize; private SocketChannel clientChannel; - @Nullable private CloseablesManager closeables; - private final InetSocketAddress remoteAddress; - private final long timeoutMs; - private final int tcpBufferSize; - private Class kClass; private Class vClass; private boolean putReturnsNull; private boolean removeReturnsNull; - - - private final ReentrantLock inBytesLock = new ReentrantLock(true); - private final ReentrantLock outBytesLock = new ReentrantLock(); - - private final BufferResizer outBufferResizer = new BufferResizer() { - @Override - public Bytes resizeBuffer(int newCapacity) { - return resizeBufferOutBuffer(newCapacity); - - } - }; - - enum EventId { - HEARTBEAT, - STATEFUL_UPDATE, - LONG_SIZE, - SIZE, - IS_EMPTY, - CONTAINS_KEY, - CONTAINS_VALUE, - GET, - PUT, - PUT_WITHOUT_ACC, - REMOVE, - REMOVE_WITHOUT_ACC, - CLEAR, - KEY_SET, - VALUES, - ENTRY_SET, - REPLACE, - REPLACE_WITH_OLD_AND_NEW_VALUE, - PUT_IF_ABSENT, - REMOVE_WITH_VALUE, - TO_STRING, - APPLICATION_VERSION, - PERSISTED_DATA_VERSION, - PUT_ALL, - PUT_ALL_WITHOUT_ACC, - HASH_CODE, - MAP_FOR_KEY, - PUT_MAPPED, - KEY_BUILDER, - VALUE_BUILDER - } - - // used by the enterprise version private int identifier; - - @SuppressWarnings("UnusedDeclaration") - void identifier(int identifier) { - this.identifier = identifier; - } - - - @NotNull - private final AtomicLong transactionID = new AtomicLong(0); + // this is a transaction id and size that has been read by another thread. + private volatile long parkedTransactionId; + private volatile int parkedRemainingBytes; + private volatile long parkedTransactionTimeStamp; + private long limitOfLast = 0; StatelessChronicleMap(@NotNull final ChronicleMapStatelessClientBuilder config) { this.remoteAddress = config.remoteAddress(); @@ -164,6 +120,11 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon loadKeyValueSerializers(); } + @SuppressWarnings("UnusedDeclaration") + void identifier(int identifier) { + this.identifier = identifier; + } + private void loadKeyValueSerializers() { final SerializationBuilder keyBuilder = fetchObject(SerializationBuilder.class, KEY_BUILDER); @@ -237,7 +198,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return vClass; } - private void checkTimeout(long timeoutTime) { if (timeoutTime < System.currentTimeMillis()) throw new RemoteCallTimeoutException(); @@ -437,7 +397,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchObject(vClass, PUT_IF_ABSENT, key, value); } - @SuppressWarnings("NullableProblems") public boolean remove(Object key, Object value) { @@ -448,7 +407,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - @SuppressWarnings("NullableProblems") public boolean replace(K key, V oldValue, V newValue) { if (key == null || oldValue == null || newValue == null) @@ -525,7 +483,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchBoolean(IS_EMPTY); } - public boolean containsKey(Object key) { return fetchBooleanK(CONTAINS_KEY, (K) key); } @@ -612,7 +569,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchObject(MAP_FOR_KEY, key, function); } - @Nullable @Override public V putMapped(@Nullable K key, @NotNull UnaryOperator unaryOperator) { @@ -626,7 +582,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon throw new UnsupportedOperationException(); } - private Bytes resizeBufferOutBuffer(int newCapacity) { return resizeBufferOutBuffer(newCapacity, outBytes.position()); } @@ -673,7 +628,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return outBytes; } - private void resizeBufferInBuffer(int newCapacity, long start) { // assert !outBytesLock.isHeldByCurrentThread(); assert inBytesLock.isHeldByCurrentThread(); @@ -986,7 +940,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return result; } - private long readLong(long transactionId, long startTime) { assert !outBytesLock.isHeldByCurrentThread(); @@ -1002,7 +955,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } - private long writeEvent(@NotNull StatelessChronicleMap.EventId event) { assert outBytesLock.isHeldByCurrentThread(); assert !inBytesLock.isHeldByCurrentThread(); @@ -1038,60 +990,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return sizeLocation; } - class Entry implements Map.Entry { - - final K key; - final V value; - - /** - * Creates new entry. - */ - Entry(K k1, V v) { - value = v; - key = k1; - } - - public final K getKey() { - return key; - } - - public final V getValue() { - return value; - } - - public final V setValue(V newValue) { - V oldValue = value; - StatelessChronicleMap.this.put(getKey(), newValue); - return oldValue; - } - - public final boolean equals(Object o) { - if (!(o instanceof Map.Entry)) - return false; - final Map.Entry e = (Map.Entry) o; - final Object k1 = getKey(); - final Object k2 = e.getKey(); - if (k1 == k2 || (k1 != null && k1.equals(k2))) { - Object v1 = getValue(); - Object v2 = e.getValue(); - if (v1 == v2 || (v1 != null && v1.equals(v2))) - return true; - } - return false; - } - - public final int hashCode() { - return (key == null ? 0 : key.hashCode()) ^ - (value == null ? 0 : value.hashCode()); - } - - @NotNull - public final String toString() { - return getKey() + "=" + getValue(); - } - } - - /** * sends data to the server via TCP/IP * @@ -1165,13 +1063,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } - // this is a transaction id and size that has been read by another thread. - private volatile long parkedTransactionId; - private volatile int parkedRemainingBytes; - private volatile long parkedTransactionTimeStamp; - - private Bytes blockingFetchThrowable(long timeoutTime, long transactionId) throws IOException, - InterruptedException { + private Bytes blockingFetchThrowable(long timeoutTime, long transactionId) throws IOException { // assert !outBytesLock.isHeldByCurrentThread(); assert inBytesLock.isHeldByCurrentThread(); @@ -1357,10 +1249,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return inBytes; } - - long largestEntrySoFar = 0; - private long limitOfLast = 0; - private void writeBytesToSocket(long timeoutTime) throws IOException { assert outBytesLock.isHeldByCurrentThread(); @@ -1412,7 +1300,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private void writeSizeAndTransactionIdAt(long locationOfSize, final long transactionId) { assert outBytesLock.isHeldByCurrentThread(); @@ -1541,7 +1428,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } - private int readInt(long transactionId, long startTime) { assert !outBytesLock.isHeldByCurrentThread(); assert inBytesLock.isHeldByCurrentThread(); @@ -1652,7 +1538,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return readBoolean(transactionId, startTime); } - @SuppressWarnings("SameParameterValue") private boolean fetchBooleanK(@NotNull final EventId eventId, K key) { final long startTime = System.currentTimeMillis(); @@ -1671,7 +1556,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return readBoolean(transactionId, startTime); } - @SuppressWarnings("SameParameterValue") private long fetchLong(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); @@ -1736,7 +1620,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } - @SuppressWarnings("SameParameterValue") @Nullable private O fetchObject(final Class tClass, @NotNull final EventId eventId) { @@ -1761,7 +1644,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } - @SuppressWarnings("SameParameterValue") private int fetchInt(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); @@ -1779,7 +1661,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return readInt(transactionId, startTime); } - @Nullable private R fetchObject(Class rClass, @NotNull final EventId eventId, K key, V value) { final long startTime = System.currentTimeMillis(); @@ -1810,7 +1691,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon "supported"); } - @Nullable private R fetchObject(Class rClass, @NotNull final EventId eventId, K key) { final long startTime = System.currentTimeMillis(); @@ -1839,7 +1719,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon "supported"); } - private boolean eventReturnsNull(@NotNull EventId eventId) { switch (eventId) { @@ -1899,7 +1778,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon throw e; } - @Nullable private R fetchObject(@NotNull final EventId eventId, K key, @NotNull Object object) { final long startTime = System.currentTimeMillis(); @@ -1921,5 +1799,91 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return (R) readObject(transactionId, startTime); } + + enum EventId { + HEARTBEAT, + STATEFUL_UPDATE, + LONG_SIZE, + SIZE, + IS_EMPTY, + CONTAINS_KEY, + CONTAINS_VALUE, + GET, + PUT, + PUT_WITHOUT_ACC, + REMOVE, + REMOVE_WITHOUT_ACC, + CLEAR, + KEY_SET, + VALUES, + ENTRY_SET, + REPLACE, + REPLACE_WITH_OLD_AND_NEW_VALUE, + PUT_IF_ABSENT, + REMOVE_WITH_VALUE, + TO_STRING, + APPLICATION_VERSION, + PERSISTED_DATA_VERSION, + PUT_ALL, + PUT_ALL_WITHOUT_ACC, + HASH_CODE, + MAP_FOR_KEY, + PUT_MAPPED, + KEY_BUILDER, + VALUE_BUILDER + } + + class Entry implements Map.Entry { + + final K key; + final V value; + + /** + * Creates new entry. + */ + Entry(K k1, V v) { + value = v; + key = k1; + } + + public final K getKey() { + return key; + } + + public final V getValue() { + return value; + } + + public final V setValue(V newValue) { + V oldValue = value; + StatelessChronicleMap.this.put(getKey(), newValue); + return oldValue; + } + + public final boolean equals(Object o) { + if (!(o instanceof Map.Entry)) + return false; + final Map.Entry e = (Map.Entry) o; + final Object k1 = getKey(); + final Object k2 = e.getKey(); + if (k1 == k2 || (k1 != null && k1.equals(k2))) { + Object v1 = getValue(); + Object v2 = e.getValue(); + if (v1 == v2 || (v1 != null && v1.equals(v2))) + return true; + } + return false; + } + + public final int hashCode() { + return (key == null ? 0 : key.hashCode()) ^ + (value == null ? 0 : value.hashCode()); + } + + @NotNull + public final String toString() { + return getKey() + "=" + getValue(); + } + } } diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index aa4886684..cdcfaef38 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -42,7 +42,6 @@ import java.nio.ByteOrder; import java.nio.channels.*; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import static java.nio.channels.SelectionKey.*; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -70,12 +69,11 @@ interface Work { final class TcpReplicator extends AbstractChannelReplicator implements Closeable { public static final long TIMESTAMP_FACTOR = 10000; + public static final long SPIN_LOOP_TIME_IN_NONOSECONDS = TimeUnit.MICROSECONDS.toNanos(500); private static final int STATELESS_CLIENT = -127; private static final byte NOT_SET = (byte) HEARTBEAT.ordinal(); private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName()); private static final int BUFFER_SIZE = 0x100000; // 1MB - - public static final long SPIN_LOOP_TIME_IN_NONOSECONDS = TimeUnit.MICROSECONDS.toNanos(500); private final SelectionKey[] selectionKeysStore = new SelectionKey[Byte.MAX_VALUE + 1]; // used to instruct the selector thread to set OP_WRITE on a key correlated by the bit index // in the bitset @@ -84,40 +82,20 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo private final BitSet activeKeys = new BitSet(selectionKeysStore.length); private final long heartBeatIntervalMillis; private final ConnectionListener connectionListener; - private long largestEntrySoFar = 128; - @NotNull private final Replica replica; private final byte localIdentifier; - @NotNull private final Replica.EntryExternalizable externalizable; @NotNull private final TcpTransportAndNetworkConfig replicationConfig; - - private final @Nullable RemoteNodeValidator remoteNodeValidator; private final String name; - - private long selectorTimeout; - StatelessClientParameters statelessClientParameters; - - static class StatelessClientParameters { - public StatelessClientParameters(VanillaChronicleMap map, - SerializationBuilder keySerializationBuilder, - SerializationBuilder valueSerializationBuilder) { - this.map = map; - this.keySerializationBuilder = keySerializationBuilder; - this.valueSerializationBuilder = valueSerializationBuilder; - } - - VanillaChronicleMap map; - SerializationBuilder keySerializationBuilder; - SerializationBuilder valueSerializationBuilder; - } + private long largestEntrySoFar = 128; + private long selectorTimeout; /** * @throws IOException on an io error. @@ -127,8 +105,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo @NotNull final TcpTransportAndNetworkConfig replicationConfig, @Nullable final RemoteNodeValidator remoteNodeValidator, @Nullable final StatelessClientParameters statelessClientParameters, - @Nullable final ConnectionListener connectionListener) - throws IOException { + @Nullable final ConnectionListener connectionListener) throws IOException { super("TcpSocketReplicator-" + replica.identifier(), replicationConfig.throttlingConfig()); @@ -153,7 +130,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo } @Override - void processEvent() throws IOException { + void processEvent() { try { final InetSocketAddress serverInetSocketAddress = new InetSocketAddress(replicationConfig.serverPort()); @@ -520,7 +497,6 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo throttle(channel); } - /** * check that the version number is valid text, * @@ -586,7 +562,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo */ private void doHandShaking(@NotNull final SelectionKey key, @NotNull SocketChannel socketChannel) - throws IOException, InterruptedException { + throws IOException { final Attached attached = (Attached) key.attachment(); final TcpSocketChannelEntryWriter writer = attached.entryWriter; final TcpSocketChannelEntryReader reader = attached.entryReader; @@ -788,7 +764,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo */ private void onRead(@NotNull final SelectionKey key, - final long approxTime) throws IOException, InterruptedException { + final long approxTime) throws IOException { final SocketChannel socketChannel = (SocketChannel) key.channel(); final Attached attached = (Attached) key.attachment(); @@ -849,6 +825,20 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo return result; } + static class StatelessClientParameters { + VanillaChronicleMap map; + SerializationBuilder keySerializationBuilder; + SerializationBuilder valueSerializationBuilder; + + public StatelessClientParameters(VanillaChronicleMap map, + SerializationBuilder keySerializationBuilder, + SerializationBuilder valueSerializationBuilder) { + this.map = map; + this.keySerializationBuilder = keySerializationBuilder; + this.valueSerializationBuilder = valueSerializationBuilder; + } + } + /** * sets interestOps to "selector keys",The change to interestOps much be on the same thread as * the selector. This class, allows via {@link AbstractChannelReplicator @@ -1080,8 +1070,8 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo // is carried out @Nullable public Work uncompletedWork; - private long lastSentTime; StatelessServerConnector statelessServer; + private long lastSentTime; private TcpSocketChannelEntryWriter() { entryCallback = new EntryCallback(externalizable, replicationConfig.tcpBufferSize()); @@ -1494,14 +1484,13 @@ final class TcpReplicator extends AbstractChannelReplicator implements Clo */ class StatelessServerConnector { - private static final Logger LOG = LoggerFactory.getLogger(StatelessServerConnector.class - .getName()); public static final StatelessChronicleMap.EventId[] VALUES = StatelessChronicleMap.EventId.values(); public static final int SIZE_OF_IS_EXCEPTION = 1; public static final int HEADER_SIZE = SIZE_OF_SIZE + SIZE_OF_IS_EXCEPTION + SIZE_OF_TRANSACTION_ID; - + private static final Logger LOG = LoggerFactory.getLogger(StatelessServerConnector.class + .getName()); @NotNull private final ReaderWithSize keyReaderWithSize; diff --git a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java index 17159d452..43ae9c2d0 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpChannelReplicator.java @@ -35,6 +35,14 @@ import static java.net.StandardSocketOptions.SO_REUSEADDR; import static java.nio.channels.SelectionKey.OP_READ; import static java.nio.channels.SelectionKey.OP_WRITE; +interface EntryReader { + void readAll(@NotNull final DatagramChannel socketChannel) throws IOException; +} + +interface EntryWriter { + int writeAll(@NotNull final DatagramChannel socketChannel) throws InterruptedException, IOException; +} + /** * The UdpReplicator attempts to read the data ( but it does not enforce or grantee delivery ), * typically, you should use the UdpReplicator if you have a large number of nodes, and you wish to @@ -47,17 +55,15 @@ class UdpChannelReplicator extends AbstractChannelReplicator implements Replica. LoggerFactory.getLogger(UdpChannelReplicator.class.getName()); private final byte localIdentifier; - private EntryWriter writer; - private EntryReader reader; - private final InetAddress address; private final int port; private final NetworkInterface networkInterface; private final ServerConnector serverConnector; - + private final String name; + private EntryWriter writer; + private EntryReader reader; private SelectableChannel writeChannel; private volatile boolean shouldEnableOpWrite; - private final String name; /** * @param replicationConfig @@ -65,11 +71,9 @@ class UdpChannelReplicator extends AbstractChannelReplicator implements Replica. * @throws IOException */ UdpChannelReplicator(@NotNull final UdpTransportConfig replicationConfig, - final byte localIdentifier) - throws IOException { + final byte localIdentifier) throws IOException { - super("UdpReplicator-" + localIdentifier, replicationConfig.throttlingConfig() - ); + super("UdpReplicator-" + localIdentifier, replicationConfig.throttlingConfig()); this.localIdentifier = localIdentifier; @@ -100,7 +104,7 @@ class UdpChannelReplicator extends AbstractChannelReplicator implements Replica. @Override - void processEvent() throws IOException { + void processEvent() { try { connectClient().register(selector, OP_READ); @@ -324,11 +328,3 @@ class UdpChannelReplicator extends AbstractChannelReplicator implements Replica. } -interface EntryReader { - void readAll(@NotNull final DatagramChannel socketChannel) throws IOException, InterruptedException; -} - -interface EntryWriter { - int writeAll(@NotNull final DatagramChannel socketChannel) throws InterruptedException, IOException; -} - diff --git a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java index 8be425dc1..a38d324f0 100644 --- a/src/main/java/net/openhft/chronicle/map/UdpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/UdpReplicator.java @@ -35,14 +35,13 @@ import java.nio.channels.DatagramChannel; */ final class UdpReplicator extends UdpChannelReplicator implements Replica.ModificationNotifier, Closeable { + public static final int UPD_BUFFER_SIZE = 64 * 1204; private static final Logger LOG = LoggerFactory.getLogger(UdpReplicator.class.getName()); - public static final int UPD_BUFFER_SIZE = 64 * 1204; public UdpReplicator(@NotNull final Replica replica, @NotNull final Replica.EntryExternalizable entryExternalizable, - @NotNull final UdpTransportConfig replicationConfig) - throws IOException { + @NotNull final UdpTransportConfig replicationConfig) throws IOException { super(replicationConfig, replica.identifier()); @@ -58,63 +57,6 @@ final class UdpReplicator extends UdpChannelReplicator implements Replica.Modifi start(); } - private class UdpSocketChannelEntryReader implements EntryReader { - - private final Replica.EntryExternalizable externalizable; - private final ByteBuffer in; - private final ByteBufferBytes out; - - /** - * @param serializedEntrySize the maximum size of an entry include the meta data - * @param externalizable supports reading and writing serialize entries - */ - UdpSocketChannelEntryReader(final int serializedEntrySize, - @NotNull final Replica.EntryExternalizable externalizable) { - // we make the buffer twice as large just to give ourselves headroom - in = ByteBuffer.allocateDirect(serializedEntrySize * 2); - this.externalizable = externalizable; - out = new ByteBufferBytes(in); - out.limit(0); - in.clear(); - } - - /** - * reads entries from the socket till it is empty - * - * @param socketChannel the socketChannel that we will read from - * @throws IOException - * @throws InterruptedException - */ - public void readAll(@NotNull final DatagramChannel socketChannel) throws IOException, - InterruptedException { - - out.clear(); - in.clear(); - - socketChannel.receive(in); - - final int bytesRead = in.position(); - - if (bytesRead < SIZE_OF_SIZE + SIZE_OF_SIZE) - return; - - out.limit(in.position()); - - final int invertedSize = out.readInt(); - final int size = out.readInt(); - - // check the the first 4 bytes are the inverted len followed by the len - // we do this to check that this is a valid start of entry, otherwise we throw it away - if (~size != invertedSize) - return; - - if (out.remaining() != size) - return; - - externalizable.readExternalEntry(copies, segmentState, out); - } - } - private static class UdpSocketChannelEntryWriter implements EntryWriter { @@ -177,5 +119,61 @@ final class UdpReplicator extends UdpChannelReplicator implements Replica.Modifi } } + private class UdpSocketChannelEntryReader implements EntryReader { + + private final Replica.EntryExternalizable externalizable; + private final ByteBuffer in; + private final ByteBufferBytes out; + + /** + * @param serializedEntrySize the maximum size of an entry include the meta data + * @param externalizable supports reading and writing serialize entries + */ + UdpSocketChannelEntryReader(final int serializedEntrySize, + @NotNull final Replica.EntryExternalizable externalizable) { + // we make the buffer twice as large just to give ourselves headroom + in = ByteBuffer.allocateDirect(serializedEntrySize * 2); + this.externalizable = externalizable; + out = new ByteBufferBytes(in); + out.limit(0); + in.clear(); + } + + /** + * reads entries from the socket till it is empty + * + * @param socketChannel the socketChannel that we will read from + * @throws IOException + * @throws InterruptedException + */ + public void readAll(@NotNull final DatagramChannel socketChannel) throws IOException { + + out.clear(); + in.clear(); + + socketChannel.receive(in); + + final int bytesRead = in.position(); + + if (bytesRead < SIZE_OF_SIZE + SIZE_OF_SIZE) + return; + + out.limit(in.position()); + + final int invertedSize = out.readInt(); + final int size = out.readInt(); + + // check the the first 4 bytes are the inverted len followed by the len + // we do this to check that this is a valid start of entry, otherwise we throw it away + if (~size != invertedSize) + return; + + if (out.remaining() != size) + return; + + externalizable.readExternalEntry(copies, segmentState, out); + } + } + } diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 075199489..db352f1aa 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -137,7 +137,7 @@ class VanillaChronicleMap, } }; - public VanillaChronicleMap(ChronicleMapBuilder builder) throws IOException { + public VanillaChronicleMap(ChronicleMapBuilder builder) { SerializationBuilder keyBuilder = builder.keyBuilder; kClass = keyBuilder.eClass; @@ -233,6 +233,16 @@ class VanillaChronicleMap, } } + static void segmentStateNotNullImpliesCopiesNotNull( + @Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState) { + assert copies != null || segmentState == null; // segmentState != null -> copies != null + } + + static void expectedValueNotNullImpliesBooleanResult( + @Nullable Object expectedValue, boolean booleanResult) { + assert booleanResult || expectedValue == null; + } + @Override public K readKey(Bytes entry, long keyPos) { long initialPos = entry.position(); @@ -260,16 +270,6 @@ class VanillaChronicleMap, } } - static void segmentStateNotNullImpliesCopiesNotNull( - @Nullable ThreadLocalCopies copies, @Nullable SegmentState segmentState) { - assert copies != null || segmentState == null; // segmentState != null -> copies != null - } - - static void expectedValueNotNullImpliesBooleanResult( - @Nullable Object expectedValue, boolean booleanResult) { - assert booleanResult || expectedValue == null; - } - /** * @return the version of Chronicle Map that was used to create the current data file */ @@ -326,7 +326,7 @@ class VanillaChronicleMap, return Segment.class; } - final long createMappedStoreAndSegments(BytesStore bytesStore) throws IOException { + final long createMappedStoreAndSegments(BytesStore bytesStore) { this.ms = bytesStore; onHeaderCreated(); @@ -614,31 +614,19 @@ class VanillaChronicleMap, @SuppressWarnings("unchecked") public final V get(Object key) { checkKey(key); - try { - return lookupUsing((K) key, null, false); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + return lookupUsing((K) key, null, false); } @Override public final V getUsing(K key, V usingValue) { checkKey(key); - try { - return lookupUsing(key, usingValue, false); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + return lookupUsing(key, usingValue, false); } @Override public final V acquireUsing(@NotNull K key, V usingValue) { checkKey(key); - try { - return lookupUsing(key, usingValue, true); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + return lookupUsing(key, usingValue, true); } final void getBytes(Bytes key, TcpReplicator.TcpSocketChannelEntryWriter output) { @@ -646,18 +634,14 @@ class VanillaChronicleMap, SegmentState segmentState = SegmentState.get(copies); ReadValueToOutputBytes readValueToOutputBytes = segmentState.readValueToOutputBytes; readValueToOutputBytes.reuse(valueSizeMarshaller, output); - try { - lookupUsing(copies, segmentState, - DelegatingMetaBytesInterop.>instance(), - BytesBytesInterop.INSTANCE, key, keySizeMarshaller.readSize(key), - keyBytesToInstance, readValueToOutputBytes, null, outputValueBytesToInstance, - false, null); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + lookupUsing(copies, segmentState, + DelegatingMetaBytesInterop.>instance(), + BytesBytesInterop.INSTANCE, key, keySizeMarshaller.readSize(key), + keyBytesToInstance, readValueToOutputBytes, null, outputValueBytesToInstance, + false, null); } - private V lookupUsing(K key, V usingValue, boolean create) throws InterruptedException { + private V lookupUsing(K key, V usingValue, boolean create) { ThreadLocalCopies copies = keyInteropProvider.getCopies(null); KI keyInterop = keyInteropProvider.get(copies, originalKeyInterop); copies = metaKeyInteropProvider.getCopies(copies); @@ -673,7 +657,7 @@ class VanillaChronicleMap, MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, InstanceOrBytesToInstance toKey, ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, - boolean create, final MutableLockedEntry lock) throws InterruptedException { + boolean create, final MutableLockedEntry lock) { long hash = metaKeyInterop.hash(keyInterop, key); int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); @@ -688,8 +672,6 @@ class VanillaChronicleMap, try (ReadContext entry = lookupUsing(key, null, false, false, LockType.READ_LOCK)) { return entry.present() ? function.apply(entry.value()) : null; - } catch (InterruptedException e) { - throw new RuntimeException(e); } } @@ -717,8 +699,6 @@ class VanillaChronicleMap, usingSB.append((CharSequence) result); } return result; - } catch (InterruptedException e) { - throw new RuntimeException(e); } } @@ -733,8 +713,6 @@ class VanillaChronicleMap, V result = unaryOperator.update(entry.value()); ((MutableLockedEntry) entry).value(result); return result; - } catch (InterruptedException e) { - throw new RuntimeException(e); } } @@ -775,25 +753,17 @@ class VanillaChronicleMap, @NotNull @Override public final WriteContext acquireUsingLocked(@NotNull K key, @NotNull V usingValue) { - try { - return lookupUsing(key, usingValue, true, true, LockType.WRITE_LOCK); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + return lookupUsing(key, usingValue, true, true, LockType.WRITE_LOCK); } @NotNull @Override public final ReadContext getUsingLocked(@NotNull K key, @NotNull V usingValue) { - try { - return lookupUsing(key, usingValue, true, false, LockType.READ_LOCK); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + return lookupUsing(key, usingValue, true, false, LockType.READ_LOCK); } private T lookupUsing( - K key, V usingValue, boolean mustReuseValue, final boolean create, final LockType lockType) throws InterruptedException { + K key, V usingValue, boolean mustReuseValue, final boolean create, final LockType lockType) { checkKey(key); ThreadLocalCopies copies = SegmentState.getCopies(null); @@ -863,15 +833,11 @@ class VanillaChronicleMap, copies = metaKeyInteropProvider.getCopies(copies); MKI metaKeyInterop = metaKeyInteropProvider.get(copies, originalMetaKeyInterop, keyInterop, key); - try { - return containsKey(copies, metaKeyInterop, keyInterop, key, - metaKeyInterop.size(keyInterop, key)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + return containsKey(copies, metaKeyInterop, keyInterop, key, + metaKeyInterop.size(keyInterop, key)); } - final boolean containsBytesKey(Bytes key) throws InterruptedException { + final boolean containsBytesKey(Bytes key) { return containsKey(null, DelegatingMetaBytesInterop.>instance(), BytesBytesInterop.INSTANCE, @@ -880,7 +846,7 @@ class VanillaChronicleMap, private > boolean containsKey(ThreadLocalCopies copies, - MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize) throws InterruptedException { + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize) { long hash = metaKeyInterop.hash(keyInterop, key); int segmentNum = getSegment(hash); long segmentHash = segmentHash(hash); @@ -1167,7 +1133,7 @@ class VanillaChronicleMap, /** * For testing */ - final void checkConsistency() throws InterruptedException { + final void checkConsistency() { for (Segment segment : segments) { segment.checkConsistency(); } @@ -2037,7 +2003,7 @@ class VanillaChronicleMap, MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, InstanceOrBytesToInstance toKey, ReadValue readValue, RV usingValue, InstanceOrBytesToInstance toValue, - long hash2, boolean create, MutableLockedEntry lock) throws InterruptedException { + long hash2, boolean create, MutableLockedEntry lock) { segmentStateNotNullImpliesCopiesNotNull(copies, segmentState); if (segmentState == null) { copies = SegmentState.getCopies(copies); @@ -2634,7 +2600,7 @@ class VanillaChronicleMap, private > boolean containsKey(ThreadLocalCopies copies, - MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, long hash2) throws InterruptedException { + MKBI metaKeyInterop, KBI keyInterop, KB key, long keySize, long hash2) { readLock(null); copies = SegmentState.getCopies(copies); try (SegmentState segmentState = SegmentState.get(copies)) { @@ -2877,7 +2843,7 @@ class VanillaChronicleMap, /** * Check there is no garbage in freeList. */ - private void checkConsistency() throws InterruptedException { + private void checkConsistency() { readLock(null); try (SegmentState segmentState = SegmentState.get(SegmentState.getCopies(null))) { MultiStoreBytes entry = segmentState.tmpBytes;