mirror of
https://github.com/OpenHFT/Chronicle-Map.git
synced 2021-09-13 01:26:12 +00:00
574 lines
19 KiB
Java
574 lines
19 KiB
Java
/*
|
|
* Copyright 2014 Higher Frequency Trading
|
|
*
|
|
* http://www.higherfrequencytrading.com
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package net.openhft.chronicle.map;
|
|
|
|
import net.openhft.chronicle.hash.replication.ThrottlingConfig;
|
|
import net.openhft.lang.io.ByteBufferBytes;
|
|
import net.openhft.lang.io.Bytes;
|
|
import net.openhft.lang.thread.NamedThreadFactory;
|
|
import net.openhft.lang.threadlocal.ThreadLocalCopies;
|
|
import org.jetbrains.annotations.NotNull;
|
|
import org.jetbrains.annotations.Nullable;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import java.io.Closeable;
|
|
import java.io.IOException;
|
|
import java.lang.reflect.Field;
|
|
import java.net.InetSocketAddress;
|
|
import java.nio.channels.*;
|
|
import java.security.AccessController;
|
|
import java.security.PrivilegedAction;
|
|
import java.util.Queue;
|
|
import java.util.Set;
|
|
import java.util.concurrent.*;
|
|
|
|
import static java.lang.Math.round;
|
|
import static java.nio.channels.SelectionKey.OP_WRITE;
|
|
import static java.util.concurrent.TimeUnit.DAYS;
|
|
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
|
|
|
/**
|
|
* @author Rob Austin.
|
|
*/
|
|
abstract class AbstractChannelReplicator implements Closeable {
|
|
|
|
public static final int BITS_IN_A_BYTE = 8;
|
|
|
|
public static final int SIZE_OF_SIZE = 4;
|
|
public static final int SIZE_OF_TRANSACTION_ID = 8;
|
|
|
|
private static final Logger LOG = LoggerFactory.getLogger(AbstractChannelReplicator.class);
|
|
|
|
// currently this is not safe to use as it wont work with the stateless client
|
|
static boolean useJavaNIOSelectionKeys = Boolean.valueOf(System.getProperty("useJavaNIOSelectionKeys"));
|
|
|
|
final SelectedSelectionKeySet selectedKeys = new SelectedSelectionKeySet();
|
|
|
|
final CloseablesManager closeables = new CloseablesManager();
|
|
final Selector selector = openSelector(closeables);
|
|
private final ExecutorService executorService;
|
|
private volatile Thread lastThread;
|
|
private Throwable startedHere;
|
|
private Future<?> future;
|
|
private final Queue<Runnable> pendingRegistrations = new ConcurrentLinkedQueue<Runnable>();
|
|
@Nullable
|
|
private final Throttler throttler;
|
|
|
|
|
|
volatile boolean isClosed = false;
|
|
ThreadLocalCopies copies;
|
|
VanillaChronicleMap.SegmentState segmentState;
|
|
|
|
AbstractChannelReplicator(String name, ThrottlingConfig throttlingConfig,
|
|
int maxEntrySizeBytes)
|
|
throws IOException {
|
|
executorService = Executors.newSingleThreadExecutor(
|
|
new NamedThreadFactory(name, true) {
|
|
@Override
|
|
public Thread newThread(@net.openhft.lang.model.constraints.NotNull Runnable r) {
|
|
return lastThread = super.newThread(r);
|
|
}
|
|
});
|
|
|
|
throttler = throttlingConfig.throttling(DAYS) > 0 ?
|
|
new Throttler(selector,
|
|
throttlingConfig.bucketInterval(MILLISECONDS),
|
|
maxEntrySizeBytes, throttlingConfig.throttling(DAYS)) : null;
|
|
|
|
startedHere = new Throwable("Started here");
|
|
}
|
|
|
|
Selector openSelector(final CloseablesManager closeables) throws IOException {
|
|
|
|
Selector result = Selector.open();
|
|
closeables.add(result);
|
|
|
|
if (!useJavaNIOSelectionKeys) {
|
|
closeables.add(new Closeable() {
|
|
@Override
|
|
public void close() throws IOException {
|
|
{
|
|
SelectionKey[] keys = selectedKeys.flip();
|
|
for (int i = 0; i < keys.length && keys[i] != null; i++) {
|
|
keys[i] = null;
|
|
}
|
|
}
|
|
{
|
|
SelectionKey[] keys = selectedKeys.flip();
|
|
for (int i = 0; i < keys.length && keys[i] != null; i++) {
|
|
keys[i] = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
}
|
|
);
|
|
return openSelector(result, selectedKeys);
|
|
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
static SocketChannel openSocketChannel(final CloseablesManager closeables) throws IOException {
|
|
SocketChannel result = null;
|
|
|
|
try {
|
|
result = SocketChannel.open();
|
|
result.socket().setTcpNoDelay(true);
|
|
|
|
} finally {
|
|
if (result != null)
|
|
try {
|
|
closeables.add(result);
|
|
} catch (IllegalStateException e) {
|
|
// already closed
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
/**
|
|
* this is similar to the code in Netty
|
|
*
|
|
* @param selector
|
|
* @return
|
|
*/
|
|
private Selector openSelector(@NotNull final Selector selector,
|
|
@NotNull final SelectedSelectionKeySet selectedKeySet) {
|
|
|
|
|
|
try {
|
|
|
|
|
|
Class<?> selectorImplClass =
|
|
Class.forName("sun.nio.ch.SelectorImpl", false, getSystemClassLoader());
|
|
|
|
// Ensure the current selector implementation is what we can instrument.
|
|
if (!selectorImplClass.isAssignableFrom(selector.getClass())) {
|
|
return selector;
|
|
}
|
|
|
|
Field selectedKeysField = selectorImplClass.getDeclaredField("selectedKeys");
|
|
Field publicSelectedKeysField = selectorImplClass.getDeclaredField("publicSelectedKeys");
|
|
|
|
|
|
selectedKeysField.setAccessible(true);
|
|
publicSelectedKeysField.setAccessible(true);
|
|
|
|
selectedKeysField.set(selector, selectedKeySet);
|
|
publicSelectedKeysField.set(selector, selectedKeySet);
|
|
|
|
// logger.trace("Instrumented an optimized java.util.Set into: {}", selector);
|
|
} catch (Exception e) {
|
|
LOG.error("", e);
|
|
// logger.trace("Failed to instrument an optimized java.util.Set into: {}", selector, t);
|
|
}
|
|
|
|
return selector;
|
|
}
|
|
|
|
static ClassLoader getSystemClassLoader() {
|
|
if (System.getSecurityManager() == null) {
|
|
return ClassLoader.getSystemClassLoader();
|
|
} else {
|
|
return AccessController.doPrivileged(new PrivilegedAction<ClassLoader>() {
|
|
@Override
|
|
public ClassLoader run() {
|
|
return ClassLoader.getSystemClassLoader();
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
void addPendingRegistration(Runnable registration) {
|
|
pendingRegistrations.add(registration);
|
|
}
|
|
|
|
void registerPendingRegistrations() throws ClosedChannelException {
|
|
for (Runnable runnable = pendingRegistrations.poll(); runnable != null;
|
|
runnable = pendingRegistrations.poll()) {
|
|
try {
|
|
runnable.run();
|
|
} catch (Exception e) {
|
|
LOG.info("", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
abstract void processEvent() throws IOException;
|
|
|
|
final void start() {
|
|
future = executorService.submit(
|
|
new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
try {
|
|
copies = VanillaChronicleMap.SegmentState.getCopies(null);
|
|
segmentState = VanillaChronicleMap.SegmentState.get(copies);
|
|
processEvent();
|
|
} catch (Exception e) {
|
|
LOG.error("", e);
|
|
}
|
|
}
|
|
}
|
|
);
|
|
}
|
|
|
|
|
|
@Override
|
|
public void close() {
|
|
if (Thread.interrupted())
|
|
LOG.warn("Already interrupted");
|
|
long start = System.currentTimeMillis();
|
|
closeResources();
|
|
|
|
try {
|
|
if (future != null)
|
|
future.cancel(true);
|
|
|
|
// we HAVE to be sure we have terminated before calling close() on the ReplicatedChronicleMap
|
|
// if this thread is still running and you close() the ReplicatedChronicleMap without
|
|
// fulling terminating this, you can cause the ReplicatedChronicleMap to attempt to
|
|
// process data on a map that has been closed, which would result in a core dump.
|
|
|
|
Thread thread = lastThread;
|
|
if (thread != null && Thread.currentThread() != lastThread)
|
|
for (int i = 0; i < 10; i++) {
|
|
if (thread.isAlive()) {
|
|
thread.join(1000);
|
|
dumpThreadStackTrace(start);
|
|
}
|
|
}
|
|
|
|
} catch (InterruptedException e) {
|
|
dumpThreadStackTrace(start);
|
|
LOG.error("", e);
|
|
LOG.error("", startedHere);
|
|
}
|
|
}
|
|
|
|
public void closeResources() {
|
|
isClosed = true;
|
|
executorService.shutdown();
|
|
closeables.closeQuietly();
|
|
if (segmentState != null)
|
|
segmentState.close();
|
|
}
|
|
|
|
private void dumpThreadStackTrace(long start) {
|
|
if (lastThread != null && lastThread.isAlive()) {
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.append("Replicator thread still running after ");
|
|
sb.append((System.currentTimeMillis() - start) / 100 / 10.0);
|
|
sb.append(" secs ");
|
|
sb.append(lastThread);
|
|
sb.append(" isAlive= ");
|
|
sb.append(lastThread.isAlive());
|
|
for (StackTraceElement ste : lastThread.getStackTrace()) {
|
|
sb.append("\n\t").append(ste);
|
|
}
|
|
LOG.warn(sb.toString());
|
|
}
|
|
}
|
|
|
|
void closeEarlyAndQuietly(SelectableChannel channel) {
|
|
if (throttler != null)
|
|
throttler.remove(channel);
|
|
closeables.closeQuietly(channel);
|
|
}
|
|
|
|
|
|
void checkThrottleInterval() throws ClosedChannelException {
|
|
if (throttler != null)
|
|
throttler.checkThrottleInterval();
|
|
}
|
|
|
|
void contemplateThrottleWrites(int bytesJustWritten) throws ClosedChannelException {
|
|
if (throttler != null)
|
|
throttler.contemplateThrottleWrites(bytesJustWritten);
|
|
}
|
|
|
|
void throttle(SelectableChannel channel) {
|
|
if (throttler != null)
|
|
throttler.add(channel);
|
|
}
|
|
|
|
/**
|
|
* throttles 'writes' to ensure the network is not swamped, this is achieved by periodically
|
|
* de-registering the write selector during periods of high volume.
|
|
*/
|
|
static class Throttler {
|
|
|
|
private final Selector selector;
|
|
private final Set<SelectableChannel> channels = new CopyOnWriteArraySet<SelectableChannel>();
|
|
private final long throttleInterval;
|
|
private final long maxBytesInInterval;
|
|
|
|
private long lastTime = System.currentTimeMillis();
|
|
private long bytesWritten;
|
|
|
|
Throttler(@NotNull Selector selector,
|
|
long throttleIntervalInMillis,
|
|
long serializedEntrySize,
|
|
long bitsPerDay) {
|
|
|
|
this.selector = selector;
|
|
this.throttleInterval = throttleIntervalInMillis;
|
|
double bytesPerMs = ((double) bitsPerDay) / DAYS.toMillis(1) / BITS_IN_A_BYTE;
|
|
this.maxBytesInInterval = round(bytesPerMs * throttleInterval) - serializedEntrySize;
|
|
}
|
|
|
|
public void add(SelectableChannel selectableChannel) {
|
|
channels.add(selectableChannel);
|
|
}
|
|
|
|
public void remove(SelectableChannel socketChannel) {
|
|
channels.remove(socketChannel);
|
|
}
|
|
|
|
/**
|
|
* re register the 'write' on the selector if the throttleInterval has passed
|
|
*
|
|
* @throws java.nio.channels.ClosedChannelException
|
|
*/
|
|
public void checkThrottleInterval() throws ClosedChannelException {
|
|
final long time = System.currentTimeMillis();
|
|
|
|
if (lastTime + throttleInterval >= time)
|
|
return;
|
|
|
|
lastTime = time;
|
|
bytesWritten = 0;
|
|
|
|
if (LOG.isDebugEnabled())
|
|
LOG.debug("Restoring OP_WRITE on all channels");
|
|
|
|
for (SelectableChannel selectableChannel : channels) {
|
|
|
|
final SelectionKey selectionKey = selectableChannel.keyFor(selector);
|
|
if (selectionKey != null)
|
|
selectionKey.interestOps(selectionKey.interestOps() | OP_WRITE);
|
|
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* checks the number of bytes written in this interval, if this number of bytes exceeds a
|
|
* threshold, the selected will de-register the socket that is being written to, until the
|
|
* interval is finished.
|
|
*
|
|
* @throws ClosedChannelException
|
|
*/
|
|
public void contemplateThrottleWrites(int bytesJustWritten)
|
|
throws ClosedChannelException {
|
|
bytesWritten += bytesJustWritten;
|
|
if (bytesWritten > maxBytesInInterval) {
|
|
for (SelectableChannel channel : channels) {
|
|
final SelectionKey selectionKey = channel.keyFor(selector);
|
|
if (selectionKey != null) {
|
|
selectionKey.interestOps(selectionKey.interestOps() & ~OP_WRITE);
|
|
}
|
|
if (LOG.isDebugEnabled())
|
|
LOG.debug("Throttling UDP writes");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* details about the socket connection
|
|
*/
|
|
static class Details {
|
|
|
|
private final InetSocketAddress address;
|
|
private final byte localIdentifier;
|
|
|
|
Details(@NotNull final InetSocketAddress address, final byte localIdentifier) {
|
|
this.address = address;
|
|
this.localIdentifier = localIdentifier;
|
|
}
|
|
|
|
public InetSocketAddress address() {
|
|
return address;
|
|
}
|
|
|
|
public byte localIdentifier() {
|
|
return localIdentifier;
|
|
}
|
|
|
|
@Override
|
|
public String toString() {
|
|
return "Details{" +
|
|
"address=" + address +
|
|
", localIdentifier=" + localIdentifier +
|
|
'}';
|
|
}
|
|
}
|
|
|
|
static class EntryCallback extends Replica.EntryCallback {
|
|
|
|
private final Replica.EntryExternalizable externalizable;
|
|
private final ByteBufferBytes in;
|
|
|
|
EntryCallback(@NotNull final Replica.EntryExternalizable externalizable,
|
|
@NotNull final ByteBufferBytes in) {
|
|
this.externalizable = externalizable;
|
|
this.in = in;
|
|
}
|
|
|
|
@Override
|
|
public boolean onEntry(final Bytes entry, final int chronicleId) {
|
|
|
|
long pos0 = in.position();
|
|
|
|
// used to denote that this is not a stateless map event
|
|
in.writeByte(StatelessChronicleMap.EventId.STATEFUL_UPDATE.ordinal());
|
|
|
|
long sizeLocation = in.position();
|
|
|
|
// this is where we will store the size of the entry
|
|
in.skip(SIZE_OF_SIZE);
|
|
|
|
final long start = in.position();
|
|
externalizable.writeExternalEntry(entry, in, chronicleId);
|
|
|
|
if (in.position() == start) {
|
|
in.position(pos0);
|
|
return false;
|
|
}
|
|
|
|
// write the length of the entry, just before the start, so when we read it back
|
|
// we read the length of the entry first and hence know how many preceding writer to read
|
|
final long entrySize = (int) (in.position() - start);
|
|
if (entrySize > Integer.MAX_VALUE)
|
|
throw new IllegalStateException("entry too large, the entry size=" + entrySize + ", " +
|
|
"entries are limited to a size of " + Integer.MAX_VALUE);
|
|
|
|
|
|
int entrySize1 = (int) entrySize;
|
|
if (LOG.isDebugEnabled())
|
|
LOG.debug("sending entry of entrySize=" + entrySize1);
|
|
|
|
in.writeInt(sizeLocation, entrySize1);
|
|
|
|
return true;
|
|
}
|
|
}
|
|
|
|
abstract class AbstractConnector {
|
|
|
|
private final String name;
|
|
private int connectionAttempts = 0;
|
|
private volatile SelectableChannel socketChannel;
|
|
|
|
public AbstractConnector(String name) {
|
|
this.name = name;
|
|
}
|
|
|
|
abstract SelectableChannel doConnect() throws IOException, InterruptedException;
|
|
|
|
/**
|
|
* connects or reconnects, but first waits a period of time proportional to the {@code
|
|
* connectionAttempts}
|
|
*/
|
|
public final void connectLater() {
|
|
if (socketChannel != null) {
|
|
closeables.closeQuietly(socketChannel);
|
|
socketChannel = null;
|
|
}
|
|
|
|
final long reconnectionInterval = connectionAttempts * 100;
|
|
if (connectionAttempts < 5)
|
|
connectionAttempts++;
|
|
doConnect(reconnectionInterval);
|
|
}
|
|
|
|
/**
|
|
* connects or reconnects immediately
|
|
*/
|
|
public void connect() {
|
|
doConnect(0);
|
|
}
|
|
|
|
|
|
/**
|
|
* @param reconnectionInterval the period to wait before connecting
|
|
*/
|
|
private void doConnect(final long reconnectionInterval) {
|
|
|
|
final Thread thread = new Thread(new Runnable() {
|
|
|
|
public void run() {
|
|
SelectableChannel socketChannel = null;
|
|
try {
|
|
if (reconnectionInterval > 0)
|
|
Thread.sleep(reconnectionInterval);
|
|
|
|
socketChannel = doConnect();
|
|
|
|
try {
|
|
closeables.add(socketChannel);
|
|
} catch (IllegalStateException e) {
|
|
|
|
// close could have be called from another thread, while we were in Thread.sleep()
|
|
// which would cause a IllegalStateException
|
|
closeQuietly(socketChannel);
|
|
return;
|
|
}
|
|
|
|
AbstractConnector.this.socketChannel = socketChannel;
|
|
|
|
|
|
} catch (Exception e) {
|
|
closeQuietly(socketChannel);
|
|
LOG.debug("", e);
|
|
}
|
|
}
|
|
|
|
private void closeQuietly(SelectableChannel socketChannel) {
|
|
if (socketChannel == null)
|
|
return;
|
|
try {
|
|
socketChannel.close();
|
|
} catch (Exception e1) {
|
|
// do nothing
|
|
}
|
|
}
|
|
});
|
|
|
|
thread.setName(name);
|
|
thread.setDaemon(true);
|
|
thread.start();
|
|
|
|
|
|
}
|
|
|
|
public void setSuccessfullyConnected() {
|
|
connectionAttempts = 0;
|
|
}
|
|
}
|
|
|
|
|
|
}
|