From 87e417d38a09132d6ab95dc0f6cb09678f9c7a99 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 00:01:12 +0000 Subject: [PATCH 01/26] fixed tests --- .../net/openhft/chronicle/map/StatelessChronicleMap.java | 8 ++++---- src/test/java/eg/BGChronicleTest.java | 2 +- src/test/java/net/openhft/chronicle/map/Builder.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index 6ee6aaa18..18074c7cb 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -351,7 +351,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - public V putIfAbsent(@NotNull K key, V value) { + public V putIfAbsent(K key, V value) { if (key == null || value == null) throw new NullPointerException(); @@ -360,7 +360,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - public boolean remove(@NotNull Object key, Object value) { + public boolean remove( Object key, Object value) { if (key == null) throw new NullPointerException(); @@ -372,14 +372,14 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - public boolean replace(@NotNull K key, @NotNull V oldValue, @NotNull V newValue) { + public boolean replace(K key, V oldValue, V newValue) { if (key == null || oldValue == null || newValue == null) throw new NullPointerException(); return fetchBoolean(REPLACE_WITH_OLD_AND_NEW_VALUE, key, oldValue, newValue); } - public V replace(@NotNull K key, @NotNull V value) { + public V replace(K key, V value) { if (key == null || value == null) throw new NullPointerException(); diff --git a/src/test/java/eg/BGChronicleTest.java b/src/test/java/eg/BGChronicleTest.java index e156dce6a..f61421742 100644 --- a/src/test/java/eg/BGChronicleTest.java +++ b/src/test/java/eg/BGChronicleTest.java @@ -440,7 +440,7 @@ public class BGChronicleTest { return port; } - @Test + public void test() throws IOException, InterruptedException { main(); } diff --git a/src/test/java/net/openhft/chronicle/map/Builder.java b/src/test/java/net/openhft/chronicle/map/Builder.java index 923c0a800..79d4ffc3c 100644 --- a/src/test/java/net/openhft/chronicle/map/Builder.java +++ b/src/test/java/net/openhft/chronicle/map/Builder.java @@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit; */ public class Builder { - public static final int SIZE = 10000; + public static final int SIZE = 1000; // added to ensure uniqueness static int count; static String WIN_OS = "WINDOWS"; From 66bcd19e0158e8af730a95503ec6bab3a4332b24 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 00:09:45 +0000 Subject: [PATCH 02/26] added @NotNull ect --- .../openhft/chronicle/map/TcpReplicator.java | 127 ++++++++++++------ 1 file changed, 88 insertions(+), 39 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index be124a7a6..930d1e152 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -76,11 +76,15 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable private final BitSet activeKeys = new BitSet(selectionKeysStore.length); private final long heartBeatIntervalMillis; + @NotNull private final Replica replica; private final byte localIdentifier; private final int maxEntrySizeBytes; + @NotNull private final Replica.EntryExternalizable externalizable; + @NotNull private final TcpTransportAndNetworkConfig replicationConfig; + @Nullable private final StatelessServerConnector statelessServerConnector; private final @Nullable @@ -217,7 +221,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable } } - private void processKey(long approxTime, SelectionKey key) { + private void processKey(long approxTime, @NotNull SelectionKey key) { try { if (!key.isValid()) @@ -346,7 +350,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable } } - private void enableOpWrite(SelectionKey key) { + private void enableOpWrite(@NotNull SelectionKey key) { int ops = key.interestOps(); if ((ops & (OP_CONNECT | OP_ACCEPT)) == 0) key.interestOps(ops | OP_WRITE); @@ -504,7 +508,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * * @param key the nio SelectionKey */ - private void clearHandshaking(SelectionKey key) { + private void clearHandshaking(@NotNull SelectionKey key) { final Attached attached = (Attached) key.attachment(); activeKeys.clear(attached.remoteIdentifier); selectionKeysStore[attached.remoteIdentifier] = null; @@ -520,7 +524,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * @throws java.io.IOException * @throws InterruptedException */ - private void doHandShaking(@NotNull final SelectionKey key, SocketChannel socketChannel) + private void doHandShaking(@NotNull final SelectionKey key, @NotNull SocketChannel socketChannel) throws IOException, InterruptedException { final Attached attached = (Attached) key.attachment(); final TcpSocketChannelEntryWriter writer = attached.entryWriter; @@ -658,7 +662,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * called when the selector receives a OP_READ message */ - private void onRead(final SelectionKey key, + private void onRead(@NotNull final SelectionKey key, final long approxTime) throws IOException, InterruptedException { final SocketChannel socketChannel = (SocketChannel) key.channel(); @@ -701,6 +705,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable } } + @Nullable private ServerSocketChannel openServerSocketChannel() throws IOException { ServerSocketChannel result = null; @@ -722,11 +727,13 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable private static class KeyInterestUpdater { private final AtomicBoolean wasChanged = new AtomicBoolean(); + @NotNull private final BitSet changeOfOpWriteRequired; + @NotNull private final SelectionKey[] selectionKeys; private final int op; - KeyInterestUpdater(int op, final SelectionKey[] selectionKeys) { + KeyInterestUpdater(int op, @NotNull final SelectionKey[] selectionKeys) { this.op = op; this.selectionKeys = selectionKeys; changeOfOpWriteRequired = new BitSet(selectionKeys.length); @@ -759,6 +766,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable private class ServerConnector extends AbstractConnector { + @NotNull private final Details details; private ServerConnector(@NotNull Details details) { @@ -766,6 +774,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable this.details = details; } + @NotNull @Override public String toString() { return "ServerConnector{" + @@ -773,6 +782,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable '}'; } + @Nullable SelectableChannel doConnect() throws IOException, InterruptedException { @@ -815,6 +825,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable private class ClientConnector extends AbstractConnector { + @NotNull private final Details details; private ClientConnector(@NotNull Details details) { @@ -822,6 +833,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable this.details = details; } + @NotNull @Override public String toString() { return "ClientConnector{" + details + '}'; @@ -896,6 +908,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable public TcpSocketChannelEntryReader entryReader; public TcpSocketChannelEntryWriter entryWriter; + @Nullable public Replica.ModificationIterator remoteModificationIterator; public AbstractConnector connector; public long remoteBootstrapTimestamp = Long.MIN_VALUE; @@ -935,11 +948,15 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable */ private class TcpSocketChannelEntryWriter { + @NotNull final ByteBufferBytes in; + @NotNull private final ByteBuffer out; + @NotNull private final EntryCallback entryCallback; // if uncompletedWork is set ( not null ) , this must be completed before any further work // is carried out + @Nullable public Work uncompletedWork; private long lastSentTime; @@ -1169,7 +1186,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * @param key * @throws InterruptedException */ - void entriesFromBuffer(Attached attached, SelectionKey key) throws InterruptedException, IOException { + void entriesFromBuffer(@NotNull Attached attached, @NotNull SelectionKey key) throws InterruptedException, IOException { for (; ; ) { out.limit(in.position()); @@ -1304,10 +1321,15 @@ class StatelessServerConnector { public static final int SIZE_OF_IS_EXCEPTION = 1; public static final int HEADER_SIZE = SIZE_OF_SIZE + SIZE_OF_IS_EXCEPTION + SIZE_OF_TRANSACTION_ID; + @NotNull private final ReaderWithSize keyReaderWithSize; + @NotNull private final WriterWithSize keyWriterWithSize; + @NotNull private final ReaderWithSize valueReaderWithSize; + @NotNull private final WriterWithSize valueWriterWithSize; + @NotNull private final VanillaChronicleMap map; private double maxEntrySizeBytes; @@ -1323,6 +1345,7 @@ class StatelessServerConnector { this.maxEntrySizeBytes = maxEntrySizeBytes; } + @Nullable Work processStatelessEvent(final byte eventId, @NotNull final Bytes writer, @NotNull final ByteBufferBytes reader) { @@ -1410,7 +1433,8 @@ class StatelessServerConnector { } } - public Work mapForKey(ByteBufferBytes reader, Bytes writer, long sizeLocation) { + @Nullable + public Work mapForKey(@NotNull ByteBufferBytes reader, @NotNull Bytes writer, long sizeLocation) { final K key = keyReaderWithSize.read(reader, null); final Function function = (Function) reader.readObject(); try { @@ -1425,7 +1449,8 @@ class StatelessServerConnector { return null; } - public Work updateForKey(ByteBufferBytes reader, Bytes writer, long sizeLocation) { + @Nullable + public Work updateForKey(@NotNull ByteBufferBytes reader, @NotNull Bytes writer, long sizeLocation) { final K key = keyReaderWithSize.read(reader, null); final Mutator mutator = (Mutator) reader.readObject(); try { @@ -1440,7 +1465,8 @@ class StatelessServerConnector { return null; } - private Work removeWithValue(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work removeWithValue(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { writer.writeBoolean(map.removeWithValue(reader)); @@ -1458,7 +1484,8 @@ class StatelessServerConnector { return null; } - private Work replaceWithOldAndNew(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work replaceWithOldAndNew(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { map.replaceWithOldAndNew(reader, writer); @@ -1477,7 +1504,8 @@ class StatelessServerConnector { return null; } - private Work longSize(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work longSize(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { writer.writeLong(map.longSize()); } catch (Throwable e) { @@ -1487,7 +1515,8 @@ class StatelessServerConnector { return null; } - private Work hashCode(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work hashCode(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { writer.writeInt(map.hashCode()); } catch (Throwable e) { @@ -1498,7 +1527,8 @@ class StatelessServerConnector { return null; } - private Work toString(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work toString(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { final String str; final long remaining = writer.remaining(); @@ -1520,7 +1550,8 @@ class StatelessServerConnector { return null; } - private Work sendException(Bytes writer, long sizeLocation, Throwable e) { + @Nullable + private Work sendException(@NotNull Bytes writer, long sizeLocation, @NotNull Throwable e) { // move the position to ignore any bytes written so far writer.position(sizeLocation + HEADER_SIZE); @@ -1532,7 +1563,8 @@ class StatelessServerConnector { return null; } - private Work isEmpty(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work isEmpty(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { writer.writeBoolean(map.isEmpty()); } catch (Throwable e) { @@ -1543,7 +1575,8 @@ class StatelessServerConnector { return null; } - private Work containsKey(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work containsKey(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { writer.writeBoolean(map.containsKey(reader)); @@ -1558,7 +1591,8 @@ class StatelessServerConnector { return null; } - private Work containsValue(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work containsValue(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { // todo optimize -- eliminate final V v = valueReaderWithSize.read(reader, null); @@ -1571,7 +1605,8 @@ class StatelessServerConnector { return null; } - private Work get(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work get(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { map.get(reader, writer); @@ -1587,6 +1622,7 @@ class StatelessServerConnector { return null; } + @Nullable private Work put(Bytes reader) { if (MAP_SUPPORTS_BYTES) { map.put(reader); @@ -1601,7 +1637,8 @@ class StatelessServerConnector { return null; } - private Work put(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work put(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { map.put(reader, writer); @@ -1619,6 +1656,7 @@ class StatelessServerConnector { return null; } + @Nullable private Work remove(Bytes reader) { if (MAP_SUPPORTS_BYTES) { map.removeKeyAsBytes(reader); @@ -1630,7 +1668,8 @@ class StatelessServerConnector { return null; } - private Work remove(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work remove(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { map.remove(reader, writer); @@ -1648,7 +1687,8 @@ class StatelessServerConnector { return null; } - private Work putAll(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work putAll(@NotNull Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { map.putAll(reader); @@ -1663,7 +1703,8 @@ class StatelessServerConnector { return null; } - private Work putAll(Bytes reader) { + @Nullable + private Work putAll(@NotNull Bytes reader) { if (MAP_SUPPORTS_BYTES) { map.putAll(reader); } else { @@ -1673,7 +1714,8 @@ class StatelessServerConnector { return null; } - private Work clear(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work clear(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { map.clear(); } catch (Throwable e) { @@ -1684,7 +1726,8 @@ class StatelessServerConnector { return null; } - private Work values(Bytes reader, Bytes writer) { + @Nullable + private Work values(@NotNull Bytes reader, @NotNull Bytes writer) { final long transactionId = reader.readLong(); Collection values; @@ -1700,7 +1743,7 @@ class StatelessServerConnector { // this allows us to write more data than the buffer will allow return new Work() { @Override - public boolean doWork(Bytes out) { + public boolean doWork(@NotNull Bytes out) { final long sizeLocation = header(out, transactionId); ThreadLocalCopies copies = valueWriterWithSize.getCopies(null); @@ -1726,7 +1769,8 @@ class StatelessServerConnector { }; } - private Work keySet(Bytes reader, final Bytes writer) { + @Nullable + private Work keySet(@NotNull Bytes reader, @NotNull final Bytes writer) { final long transactionId = reader.readLong(); Set ks; @@ -1742,7 +1786,7 @@ class StatelessServerConnector { // this allows us to write more data than the buffer will allow return new Work() { @Override - public boolean doWork(Bytes out) { + public boolean doWork(@NotNull Bytes out) { final long sizeLocation = header(out, transactionId); ThreadLocalCopies copies = keyWriterWithSize.getCopies(null); @@ -1768,7 +1812,8 @@ class StatelessServerConnector { }; } - private Work entrySet(final Bytes reader, Bytes writer) { + @Nullable + private Work entrySet(@NotNull final Bytes reader, @NotNull Bytes writer) { final long transactionId = reader.readLong(); final Set> entries; @@ -1784,7 +1829,7 @@ class StatelessServerConnector { // this allows us to write more data than the buffer will allow return new Work() { @Override - public boolean doWork(Bytes out) { + public boolean doWork(@NotNull Bytes out) { if (out.remaining() <= maxEntrySizeBytes) return false; @@ -1816,7 +1861,8 @@ class StatelessServerConnector { }; } - private Work putIfAbsent(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work putIfAbsent(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { map.putIfAbsent(reader, writer); @@ -1834,7 +1880,8 @@ class StatelessServerConnector { return null; } - private Work replace(Bytes reader, Bytes writer, final long sizeLocation) { + @Nullable + private Work replace(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { try { if (MAP_SUPPORTS_BYTES) { map.replaceKV(reader, writer); @@ -1852,7 +1899,7 @@ class StatelessServerConnector { return null; } - private long reflectTransactionId(Bytes reader, Bytes writer) { + private long reflectTransactionId(@NotNull Bytes reader, @NotNull Bytes writer) { final long transactionId = reader.readLong(); final long sizeLocation = writer.position(); writer.skip(SIZE_OF_SIZE); @@ -1862,7 +1909,7 @@ class StatelessServerConnector { return sizeLocation; } - private void writeSizeAndFlags(long locationOfSize, boolean isException, Bytes out) { + private void writeSizeAndFlags(long locationOfSize, boolean isException, @NotNull Bytes out) { final long size = out.position() - locationOfSize; out.writeInt(locationOfSize, (int) size); // size @@ -1887,11 +1934,12 @@ class StatelessServerConnector { } - private void writeException(Bytes out, Throwable e) { + private void writeException(@NotNull Bytes out, Throwable e) { out.writeObject(e); } - private Map readEntries(Bytes reader) { + @NotNull + private Map readEntries(@NotNull Bytes reader) { final long numberOfEntries = reader.readStopBit(); final Map result = new HashMap(); @@ -1908,12 +1956,13 @@ class StatelessServerConnector { return result; } - private Work sendException(Bytes reader, Bytes writer, Throwable e) { + @Nullable + private Work sendException(@NotNull Bytes reader, @NotNull Bytes writer, @NotNull Throwable e) { final long sizeLocation = reflectTransactionId(reader, writer); return sendException(writer, sizeLocation, e); } - private long header(Bytes writer, final long transactionId) { + private long header(@NotNull Bytes writer, final long transactionId) { final long sizeLocation = writer.position(); writer.skip(SIZE_OF_SIZE); @@ -1930,7 +1979,7 @@ class StatelessServerConnector { return sizeLocation; } - private void writeHeader(Bytes writer, long sizeLocation, int count, + private void writeHeader(@NotNull Bytes writer, long sizeLocation, int count, final boolean hasAnotherChunk) { final long end = writer.position(); final int size = (int) (end - sizeLocation); From 81f7fdad0a916ff7909ecf0583f3e403407ecd1f Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 00:17:36 +0000 Subject: [PATCH 03/26] added @NotNull ect --- .../chronicle/map/StatelessChronicleMap.java | 79 +++++++++++++------ 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index 18074c7cb..9935f713f 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -27,6 +27,7 @@ import net.openhft.lang.io.NativeBytes; import net.openhft.lang.thread.NamedThreadFactory; import net.openhft.lang.threadlocal.ThreadLocalCopies; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +61,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon private final byte[] connectionByte = new byte[1]; private final ByteBuffer connectionOutBuffer = ByteBuffer.wrap(connectionByte); private final String name; + @NotNull private final AbstractChronicleMapBuilder chronicleMapBuilder; private volatile ByteBuffer outBuffer; @@ -68,13 +70,19 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon private volatile ByteBuffer inBuffer; private volatile ByteBufferBytes inBytes; + @NotNull private final ReaderWithSize keyReaderWithSize; + @NotNull private final WriterWithSize keyWriterWithSize; + @NotNull private final ReaderWithSize valueReaderWithSize; + @NotNull private final WriterWithSize valueWriterWithSize; private volatile SocketChannel clientChannel; + @Nullable private CloseablesManager closeables; + @NotNull private final StatelessMapConfig config; private int maxEntrySize; private final Class kClass; @@ -117,10 +125,11 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon UPDATE_FOR_KEY } + @NotNull private AtomicLong transactionID = new AtomicLong(0); - StatelessChronicleMap(final StatelessMapConfig config, - final AbstractChronicleMapBuilder chronicleMapBuilder) throws IOException { + StatelessChronicleMap(@NotNull final StatelessMapConfig config, + @NotNull final AbstractChronicleMapBuilder chronicleMapBuilder) throws IOException { this.chronicleMapBuilder = chronicleMapBuilder; this.config = config; keyReaderWithSize = new ReaderWithSize<>(chronicleMapBuilder.keyBuilder); @@ -148,9 +157,11 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @NotNull @Override public Future getLater(@NotNull final K key) { return lazyExecutorService().submit(new Callable() { + @Nullable @Override public V call() throws Exception { return StatelessChronicleMap.this.get(key); @@ -158,9 +169,11 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon }); } + @NotNull @Override public Future putLater(@NotNull final K key, @NotNull final V value) { return lazyExecutorService().submit(new Callable() { + @Nullable @Override public V call() throws Exception { return StatelessChronicleMap.this.put(key, value); @@ -312,6 +325,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon LOG.debug("Attached to a map with a remote identifier=" + remoteIdentifier); } + @NotNull public File file() { throw new UnsupportedOperationException(); } @@ -400,7 +414,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon * @return true if the contain the same data */ @Override - public boolean equals(Object object) { + public boolean equals(@Nullable Object object) { if (this == object) return true; if (object == null || object.getClass().isAssignableFrom(Map.class)) return false; @@ -421,6 +435,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchInt(HASH_CODE); } + @NotNull public String toString() { return fetchObject(String.class, TO_STRING); } @@ -434,6 +449,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchBooleanK(CONTAINS_KEY, (K) key); } + @NotNull private NullPointerException keyNotNullNPE() { return new NullPointerException("key can not be null"); } @@ -450,11 +466,13 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchObject(vClass, GET, (K) key); } + @NotNull public V getUsing(K key, V usingValue) { throw new UnsupportedOperationException("getUsing() is not supported for stateless " + "clients"); } + @NotNull public V acquireUsing(@NotNull K key, V usingValue) { throw new UnsupportedOperationException("acquireUsing() is not supported for stateless " + "clients"); @@ -485,22 +503,24 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchObject(vClass, putReturnsNull ? PUT_WITHOUT_ACC : PUT, key, value); } - public R mapForKey(K key, @NotNull Function function) { + @Nullable + public R mapForKey(@Nullable K key, @NotNull Function function) { if (key == null) throw keyNotNullNPE(); return fetchObject(MAP_FOR_KEY, key, function); } + @Nullable @Override - public R updateForKey(K key, @NotNull Mutator mutator) { + public R updateForKey(@Nullable K key, @NotNull Mutator mutator) { if (key == null) throw keyNotNullNPE(); return fetchObject(UPDATE_FOR_KEY, key, mutator); } - private synchronized void writeEntriesForPutAll(Map map) { + private synchronized void writeEntriesForPutAll(@NotNull Map map) { final int numberOfEntries = map.size(); int numberOfEntriesReadSoFar = 0; @@ -850,7 +870,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private long writeEvent(StatelessChronicleMap.EventId event) { + private long writeEvent(@NotNull StatelessChronicleMap.EventId event) { outBuffer.clear(); outBytes.clear(); @@ -862,7 +882,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon /** * skips for the transactionid */ - private long writeEventAnSkip(EventId event) { + private long writeEventAnSkip(@NotNull EventId event) { final long sizeLocation = writeEvent(event); // skips for the transaction id @@ -917,6 +937,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon (value == null ? 0 : value.hashCode()); } + @NotNull public final String toString() { return getKey() + "=" + getValue(); } @@ -952,7 +973,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon outBytes.buffer().clear(); break; - } catch (java.nio.channels.ClosedChannelException | ClosedConnectionException e) { + } catch (@NotNull java.nio.channels.ClosedChannelException | ClosedConnectionException e) { checkTimeout(timeoutTime); clientChannel = lazyConnect(config.timeoutMs(), config.remoteAddress()); } @@ -1148,7 +1169,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private void send(final Bytes out, long timeoutTime) throws IOException { + private void send(@NotNull final Bytes out, long timeoutTime) throws IOException { outBuffer.limit((int) out.position()); outBuffer.position(0); @@ -1270,7 +1291,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } - private void resizeToMessage(long start, IllegalArgumentException e) { + private void resizeToMessage(long start, @NotNull IllegalArgumentException e) { String message = e.getMessage(); if (message.startsWith(START_OF)) { String substring = message.substring("Attempt to write ".length(), message.length()); @@ -1298,6 +1319,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } + @Nullable private O readObject(long transactionId, long startTime) { long timeoutTime = startTime + this.config.timeoutMs(); @@ -1335,7 +1357,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } - private long send(final EventId eventId, final long startTime) { + private long send(@NotNull final EventId eventId, final long startTime) { // send outBytesLock.lock(); try { @@ -1350,7 +1372,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return valueReaderWithSize.readNullable(in, copies); } - private boolean fetchBoolean(final EventId eventId, K key, V value) { + private boolean fetchBoolean(@NotNull final EventId eventId, K key, V value) { final long startTime = System.currentTimeMillis(); ThreadLocalCopies copies; @@ -1373,7 +1395,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private boolean fetchBoolean(final EventId eventId, K key, V value1, V value2) { + private boolean fetchBoolean(@NotNull final EventId eventId, K key, V value1, V value2) { final long startTime = System.currentTimeMillis(); ThreadLocalCopies copies; @@ -1397,7 +1419,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private boolean fetchBooleanV(final EventId eventId, V value) { + private boolean fetchBooleanV(@NotNull final EventId eventId, V value) { final long startTime = System.currentTimeMillis(); long transactionId; @@ -1415,7 +1437,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private boolean fetchBooleanK(final EventId eventId, K key) { + private boolean fetchBooleanK(@NotNull final EventId eventId, K key) { final long startTime = System.currentTimeMillis(); long transactionId; @@ -1433,7 +1455,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private long fetchLong(final EventId eventId) { + private long fetchLong(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); long transactionId; @@ -1449,7 +1471,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return readLong(transactionId, startTime); } - private boolean fetchBoolean(final EventId eventId) { + private boolean fetchBoolean(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); long transactionId; @@ -1465,7 +1487,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return readBoolean(transactionId, startTime); } - private void fetchVoid(final EventId eventId) { + private void fetchVoid(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); @@ -1495,7 +1517,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private O fetchObject(final Class tClass, final EventId eventId) { + @Nullable + private O fetchObject(final Class tClass, @NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); long transactionId = send(eventId, startTime); @@ -1511,7 +1534,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private int fetchInt(final EventId eventId) { + private int fetchInt(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); long transactionId; @@ -1528,7 +1551,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private R fetchObject(Class rClass, final EventId eventId, K key, V value) { + @Nullable + private R fetchObject(Class rClass, @NotNull final EventId eventId, K key, V value) { final long startTime = System.currentTimeMillis(); ThreadLocalCopies copies; @@ -1559,7 +1583,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon "supported"); } - private R fetchObject(Class rClass, final EventId eventId, K key, V value1, V value2) { + @Nullable + private R fetchObject(Class rClass, @NotNull final EventId eventId, K key, V value1, V value2) { final long startTime = System.currentTimeMillis(); ThreadLocalCopies copies; @@ -1592,7 +1617,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon "supported"); } - private R fetchObject(Class rClass, final EventId eventId, K key) { + @Nullable + private R fetchObject(Class rClass, @NotNull final EventId eventId, K key) { final long startTime = System.currentTimeMillis(); ThreadLocalCopies copies; @@ -1622,7 +1648,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private boolean eventReturnsNull(EventId eventId) { + private boolean eventReturnsNull(@NotNull EventId eventId) { switch (eventId) { case PUT_ALL_WITHOUT_ACC: @@ -1654,7 +1680,8 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - private R fetchObject(final EventId eventId, K key, Object object) { + @Nullable + private R fetchObject(@NotNull final EventId eventId, K key, @NotNull Object object) { final long startTime = System.currentTimeMillis(); long transactionId; From 3ae05ec34b8a88a573f44f05fb539a565d034b51 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 00:37:09 +0000 Subject: [PATCH 04/26] minor refactors --- .../openhft/chronicle/map/TcpReplicator.java | 86 ++++++------------- 1 file changed, 27 insertions(+), 59 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 930d1e152..4bdad2b43 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -64,11 +64,11 @@ interface Work { */ final class TcpReplicator extends AbstractChannelReplicator implements Closeable { - public static final int STATELESS_CLIENT = -127; - public static final byte NOT_SET = (byte) HEARTBEAT.ordinal(); + private static final int STATELESS_CLIENT = -127; + private static final byte NOT_SET = (byte) HEARTBEAT.ordinal(); private static final Logger LOG = LoggerFactory.getLogger(TcpReplicator.class.getName()); private static final int BUFFER_SIZE = 0x100000; // 1MB - public static final int SPIN_LOOP_COUNT = 100000; + private static final int SPIN_LOOP_COUNT = 100000; private final SelectionKey[] selectionKeysStore = new SelectionKey[Byte.MAX_VALUE + 1]; // used to instruct the selector thread to set OP_WRITE on a key correlated by the bit index in the // bitset @@ -194,16 +194,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable } } } - } catch (CancelledKeyException e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } catch (ClosedSelectorException e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } catch (ClosedChannelException e) { - if (LOG.isDebugEnabled()) - LOG.debug("", e); - } catch (ConnectException e) { + } catch (CancelledKeyException | ConnectException | ClosedChannelException | ClosedSelectorException e) { if (LOG.isDebugEnabled()) LOG.debug("", e); } catch (Exception e) { @@ -249,19 +240,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable LOG.debug("onWrite - " + name); onWrite(key, approxTime); } - } catch (BufferUnderflowException e) { - if (!isClosed) - quietClose(key, e); - } catch (CancelledKeyException e) { - if (!isClosed) - quietClose(key, e); - } catch (ClosedSelectorException e) { - if (!isClosed) - quietClose(key, e); - } catch (IOException e) { - if (!isClosed) - quietClose(key, e); - } catch (InterruptedException e) { + } catch (BufferUnderflowException | InterruptedException | IOException | ClosedSelectorException | CancelledKeyException e) { if (!isClosed) quietClose(key, e); } catch (Exception e) { @@ -365,7 +344,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * @throws ConnectException */ private void heartbeatCheckHasReceived(@NotNull final SelectionKey key, - final long approxTimeOutTime) throws ConnectException { + final long approxTimeOutTime) { final Attached attached = (Attached) key.attachment(); @@ -409,7 +388,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * called when the selector receives a OP_CONNECT message */ private void onConnect(@NotNull final SelectionKey key) - throws IOException, InterruptedException { + throws IOException { SocketChannel channel = null; @@ -502,19 +481,6 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable throttle(channel); } - /** - * this can be called when a new CHM is added to a cluster, we have to rebootstrap so will clear - * all the old bootstrap information - * - * @param key the nio SelectionKey - */ - private void clearHandshaking(@NotNull SelectionKey key) { - final Attached attached = (Attached) key.attachment(); - activeKeys.clear(attached.remoteIdentifier); - selectionKeysStore[attached.remoteIdentifier] = null; - attached.clearHandShaking(); - } - /** * used to exchange identifiers and timestamps and heartbeat intervals between the server and * client @@ -525,7 +491,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * @throws InterruptedException */ private void doHandShaking(@NotNull final SelectionKey key, @NotNull SocketChannel socketChannel) - throws IOException, InterruptedException { + throws IOException { final Attached attached = (Attached) key.attachment(); final TcpSocketChannelEntryWriter writer = attached.entryWriter; final TcpSocketChannelEntryReader reader = attached.entryReader; @@ -623,7 +589,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * called when the selector receives a OP_WRITE message */ private void onWrite(@NotNull final SelectionKey key, - final long approxTime) throws InterruptedException, IOException { + final long approxTime) throws IOException { final SocketChannel socketChannel = (SocketChannel) key.channel(); final Attached attached = (Attached) key.attachment(); if (attached == null) throw new NullPointerException("No attached"); @@ -1000,8 +966,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * @param selectionKey */ void entriesToBuffer(@NotNull final Replica.ModificationIterator modificationIterator, - @NotNull final SelectionKey selectionKey) - throws InterruptedException, IOException { + @NotNull final SelectionKey selectionKey) { final SocketChannel socketChannel = (SocketChannel) selectionKey.channel(); final Attached attached = (Attached) selectionKey.attachment(); @@ -1109,7 +1074,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable } public boolean doWork() { - return uncompletedWork.doWork(in); + return uncompletedWork != null && uncompletedWork.doWork(in); } public boolean hasBytesToWrite() { @@ -1186,7 +1151,7 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * @param key * @throws InterruptedException */ - void entriesFromBuffer(@NotNull Attached attached, @NotNull SelectionKey key) throws InterruptedException, IOException { + void entriesFromBuffer(@NotNull Attached attached, @NotNull SelectionKey key) { for (; ; ) { out.limit(in.position()); @@ -1312,7 +1277,6 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable */ class StatelessServerConnector { - public static final int TRANSACTION_ID_OFFSET = SIZE_OF_SIZE + 1; public static final boolean MAP_SUPPORTS_BYTES = true; private static final Logger LOG = LoggerFactory.getLogger(StatelessServerConnector.class .getName()); @@ -1331,7 +1295,7 @@ class StatelessServerConnector { private final WriterWithSize valueWriterWithSize; @NotNull private final VanillaChronicleMap map; - private double maxEntrySizeBytes; + private final double maxEntrySizeBytes; StatelessServerConnector( SerializationBuilder keySerializationBuilder, @@ -1378,10 +1342,10 @@ class StatelessServerConnector { switch (event) { case LONG_SIZE: - return longSize(reader, writer, sizeLocation); + return longSize(writer, sizeLocation); case IS_EMPTY: - return isEmpty(reader, writer, sizeLocation); + return isEmpty(writer, sizeLocation); case CONTAINS_KEY: return containsKey(reader, writer, sizeLocation); @@ -1399,7 +1363,7 @@ class StatelessServerConnector { return remove(reader, writer, sizeLocation); case CLEAR: - return clear(reader, writer, sizeLocation); + return clear(writer, sizeLocation); case REPLACE: return replace(reader, writer, sizeLocation); @@ -1414,13 +1378,13 @@ class StatelessServerConnector { return removeWithValue(reader, writer, sizeLocation); case TO_STRING: - return toString(reader, writer, sizeLocation); + return toString(writer, sizeLocation); case PUT_ALL: return putAll(reader, writer, sizeLocation); case HASH_CODE: - return hashCode(reader, writer, sizeLocation); + return hashCode(writer, sizeLocation); case MAP_FOR_KEY: return mapForKey(reader, writer, sizeLocation); @@ -1505,7 +1469,7 @@ class StatelessServerConnector { } @Nullable - private Work longSize(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { + private Work longSize(@NotNull Bytes writer, final long sizeLocation) { try { writer.writeLong(map.longSize()); } catch (Throwable e) { @@ -1516,7 +1480,7 @@ class StatelessServerConnector { } @Nullable - private Work hashCode(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { + private Work hashCode(@NotNull Bytes writer, final long sizeLocation) { try { writer.writeInt(map.hashCode()); } catch (Throwable e) { @@ -1528,7 +1492,7 @@ class StatelessServerConnector { } @Nullable - private Work toString(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { + private Work toString(@NotNull Bytes writer, final long sizeLocation) { final String str; final long remaining = writer.remaining(); @@ -1550,6 +1514,7 @@ class StatelessServerConnector { return null; } + @SuppressWarnings("SameReturnValue") @Nullable private Work sendException(@NotNull Bytes writer, long sizeLocation, @NotNull Throwable e) { // move the position to ignore any bytes written so far @@ -1564,7 +1529,7 @@ class StatelessServerConnector { } @Nullable - private Work isEmpty(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { + private Work isEmpty(@NotNull Bytes writer, final long sizeLocation) { try { writer.writeBoolean(map.isEmpty()); } catch (Throwable e) { @@ -1622,6 +1587,7 @@ class StatelessServerConnector { return null; } + @SuppressWarnings("SameReturnValue") @Nullable private Work put(Bytes reader) { if (MAP_SUPPORTS_BYTES) { @@ -1656,6 +1622,7 @@ class StatelessServerConnector { return null; } + @SuppressWarnings("SameReturnValue") @Nullable private Work remove(Bytes reader) { if (MAP_SUPPORTS_BYTES) { @@ -1703,6 +1670,7 @@ class StatelessServerConnector { return null; } + @SuppressWarnings("SameReturnValue") @Nullable private Work putAll(@NotNull Bytes reader) { if (MAP_SUPPORTS_BYTES) { @@ -1715,7 +1683,7 @@ class StatelessServerConnector { } @Nullable - private Work clear(Bytes reader, @NotNull Bytes writer, final long sizeLocation) { + private Work clear(@NotNull Bytes writer, final long sizeLocation) { try { map.clear(); } catch (Throwable e) { From 9bd56f32c6318f0b5657e222519919ad9a31e01c Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 00:46:14 +0000 Subject: [PATCH 05/26] minor refactors --- .../chronicle/map/StatelessChronicleMap.java | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index 9935f713f..59e2dbf3a 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -54,9 +54,9 @@ import static net.openhft.chronicle.map.StatelessChronicleMap.EventId.*; */ class StatelessChronicleMap implements ChronicleMap, Closeable, Cloneable { - public static final String START_OF = "Attempt to write "; + private static final String START_OF = "Attempt to write "; private static final Logger LOG = LoggerFactory.getLogger(StatelessChronicleMap.class); - public static final byte STATELESS_CLIENT_IDENTIFIER = (byte) -127; + private static final byte STATELESS_CLIENT_IDENTIFIER = (byte) -127; private final byte[] connectionByte = new byte[1]; private final ByteBuffer connectionOutBuffer = ByteBuffer.wrap(connectionByte); @@ -126,7 +126,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } @NotNull - private AtomicLong transactionID = new AtomicLong(0); + private final AtomicLong transactionID = new AtomicLong(0); StatelessChronicleMap(@NotNull final StatelessMapConfig config, @NotNull final AbstractChronicleMapBuilder chronicleMapBuilder) throws IOException { @@ -215,7 +215,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } private synchronized SocketChannel lazyConnect(final long timeoutMs, - final InetSocketAddress remoteAddress) throws IOException { + final InetSocketAddress remoteAddress) { if (LOG.isDebugEnabled()) LOG.debug("attempting to connect to " + remoteAddress); @@ -238,9 +238,9 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon doHandShaking(result); break; } catch (IOException e) { - closeables.closeQuietly(); + if (closeables!=null) closeables.closeQuietly(); } catch (Exception e) { - closeables.closeQuietly(); + if (closeables!=null) closeables.closeQuietly(); throw e; } } @@ -256,13 +256,12 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon * the first map method call. * * @param remoteAddress the Inet Socket Address - * @return * @throws java.io.IOException * @see net.openhft.chronicle.map.StatelessChronicleMap#lazyConnect(long, * java.net.InetSocketAddress) */ - private synchronized void attemptConnect(final InetSocketAddress remoteAddress) throws IOException { + private synchronized void attemptConnect(final InetSocketAddress remoteAddress) { // ensures that the excising connection are closed closeExisting(); @@ -272,7 +271,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon clientChannel.connect(remoteAddress); doHandShaking(clientChannel); } catch (IOException e) { - closeables.closeQuietly(); + if (closeables!=null) closeables.closeQuietly(); } } @@ -365,6 +364,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @SuppressWarnings("NullableProblems") public V putIfAbsent(K key, V value) { if (key == null || value == null) @@ -374,18 +374,18 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @SuppressWarnings("NullableProblems") public boolean remove( Object key, Object value) { if (key == null) throw new NullPointerException(); - if (value == null) - return false; + return value != null && fetchBoolean(REMOVE_WITH_VALUE, (K) key, (V) value); - return fetchBoolean(REMOVE_WITH_VALUE, (K) key, (V) value); } + @SuppressWarnings("NullableProblems") public boolean replace(K key, V oldValue, V newValue) { if (key == null || oldValue == null || newValue == null) throw new NullPointerException(); @@ -393,6 +393,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return fetchBoolean(REPLACE_WITH_OLD_AND_NEW_VALUE, key, oldValue, newValue); } + @SuppressWarnings("NullableProblems") public V replace(K key, V value) { if (key == null || value == null) throw new NullPointerException(); @@ -635,7 +636,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon assert size < Integer.MAX_VALUE; - final ByteBuffer result = ByteBuffer.allocate((int) size).order(ByteOrder.nativeOrder()); + final ByteBuffer result = ByteBuffer.allocate(size).order(ByteOrder.nativeOrder()); final long bytesPosition = inBytes.position(); inBytes = new ByteBufferBytes(result.slice()); @@ -913,7 +914,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon public final V setValue(V newValue) { V oldValue = value; - StatelessChronicleMap.this.put((K) getKey(), (V) newValue); + StatelessChronicleMap.this.put(getKey(), newValue); return oldValue; } @@ -1113,6 +1114,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon InetSocketAddress address = config.remoteAddress(); stes.add(new StackTraceElement("~ remote", "tcp ~", address.getHostName(), address.getPort())); StackTraceElement[] stackTrace2 = Thread.currentThread().getStackTrace(); + //noinspection ManualArrayToCollectionCopy for (int i = 4; i < stackTrace2.length; i++) stes.add(stackTrace2[i]); stackTrace.set(throwable, stes.toArray(new StackTraceElement[stes.size()])); @@ -1131,7 +1133,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon parkedTransactionTimeStamp = 0; } - private void pause() throws InterruptedException { + private void pause() { /// don't call inBytesLock.isHeldByCurrentThread() as it not atomic inBytesLock.unlock(); @@ -1147,6 +1149,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon * @return bytes read from the TCP/IP socket * @throws IOException socket failed to read data */ + @SuppressWarnings("UnusedReturnValue") private Bytes receive(int requiredNumberOfBytes, long timeoutTime) throws IOException { inBytes.buffer().position(0); @@ -1217,6 +1220,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return writeKey(key, null); } + @SuppressWarnings("SameParameterValue") private ThreadLocalCopies writeKey(K key, ThreadLocalCopies copies) { long start = outBytes.position(); @@ -1236,6 +1240,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } + @SuppressWarnings("UnusedReturnValue") private ThreadLocalCopies writeKeyInLoop(K key, Object writer, ThreadLocalCopies copies) { long start = outBytes.position(); @@ -1273,6 +1278,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } } + @SuppressWarnings("UnusedReturnValue") private ThreadLocalCopies writeValueInLoop(V value, Object writer, ThreadLocalCopies copies) { long start = outBytes.position(); for (; ; ) { @@ -1372,6 +1378,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return valueReaderWithSize.readNullable(in, copies); } + @SuppressWarnings("SameParameterValue") private boolean fetchBoolean(@NotNull final EventId eventId, K key, V value) { final long startTime = System.currentTimeMillis(); ThreadLocalCopies copies; @@ -1395,6 +1402,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @SuppressWarnings("SameParameterValue") private boolean fetchBoolean(@NotNull final EventId eventId, K key, V value1, V value2) { final long startTime = System.currentTimeMillis(); ThreadLocalCopies copies; @@ -1419,6 +1427,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @SuppressWarnings("SameParameterValue") private boolean fetchBooleanV(@NotNull final EventId eventId, V value) { final long startTime = System.currentTimeMillis(); @@ -1437,6 +1446,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @SuppressWarnings("SameParameterValue") private boolean fetchBooleanK(@NotNull final EventId eventId, K key) { final long startTime = System.currentTimeMillis(); @@ -1455,6 +1465,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @SuppressWarnings("SameParameterValue") private long fetchLong(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); @@ -1471,6 +1482,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return readLong(transactionId, startTime); } + @SuppressWarnings("SameParameterValue") private boolean fetchBoolean(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); @@ -1487,6 +1499,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon return readBoolean(transactionId, startTime); } + @SuppressWarnings("SameParameterValue") private void fetchVoid(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); @@ -1517,6 +1530,7 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } + @SuppressWarnings("SameParameterValue") @Nullable private O fetchObject(final Class tClass, @NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); @@ -1527,13 +1541,14 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon // receive inBytesLock.lock(); try { - return (O) blockingFetchReadOnly(timeoutTime, transactionId).readObject(tClass); + return blockingFetchReadOnly(timeoutTime, transactionId).readObject(tClass); } finally { inBytesLock.unlock(); } } + @SuppressWarnings("SameParameterValue") private int fetchInt(@NotNull final EventId eventId) { final long startTime = System.currentTimeMillis(); From f79029f280599150f6f0178b8e472c4ea03f645d Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 11:04:07 +0000 Subject: [PATCH 06/26] removed getLater(0 putLater() and removeLater() because the stateless client is now mulithreaded --- .../openhft/chronicle/map/ChronicleMap.java | 62 ------------ .../chronicle/map/StatelessChronicleMap.java | 34 ------- .../chronicle/map/VanillaChronicleMap.java | 99 ------------------- .../chronicle/map/CHMUseCasesTest.java | 48 ++------- .../chronicle/map/ReplicationCheckingMap.java | 16 --- .../chronicle/map/StatelessClientTest.java | 53 ---------- .../jrs166/map/StatelessChronicleMapTest.java | 16 --- 7 files changed, 6 insertions(+), 322 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java index e0cef2c72..c7e3764fd 100755 --- a/src/main/java/net/openhft/chronicle/map/ChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/ChronicleMap.java @@ -211,68 +211,6 @@ public interface ChronicleMap extends ConcurrentMap, ChronicleHash { */ R updateForKey(K key, @NotNull Mutator mutator); - /** - * Returns a {@link java.util.concurrent.Future} containing the value to which the specified key - * is mapped, or {@code null} if this map contains no mapping for the key. This method behaves - * the same as {@link java.util.Map#get(java.lang.Object)} yet wraps result in a future a {@link - * java.util.concurrent.Future}. - * - * Calling this method may or may not block until the operating is complete depending on the - * implementation. - * - * @param key the key whose associated value is to be returned in the future - * @return a future containing the value to which the specified key is mapped after this method - * call, or {@code null} if no value is mapped - * @see java.util.Map#get(java.lang.Object) - * @see java.util.concurrent.Future - */ - Future getLater(@NotNull K key); - - /** - * Associates the specified value with the specified key in this map (optional operation). If - * the map previously contained a mapping for the key, the old value is replaced by the - * specified value. (A map m is said to contain a mapping for a key k if and - * only if {@link #containsKey(Object) m.containsKey(k)} would return true.) - * - * Calling this method may or may not block until the operating is complete depending on the - * implementation. - * - * @param key key with which the specified value is to be associated - * @param value value to be associated with the specified key * @return a future containing the - * previous value associated with key, or null if there was no - * mapping for key. (A null return can also indicate that the map - * previously associated null with key, if the implementation - * supports null values.) - * @return a future containing either, the previous value associated with key, or - * null if there was no mapping for key. - * @see java.util.Map#put(java.lang.Object, java.lang.Object) - * @see java.util.concurrent.Future - */ - Future putLater(@NotNull K key, @NotNull V value); - - /** - * Removes the mapping for a key from this map if it is present (optional operation). More - * formally, if this map contains a mapping from key k to value v such that - * (key==null ? k==null : key.equals(k)), that mapping is removed. (The map can - * contain at most one such mapping.) - * - *

Returns a future containing, either the value to which this map previously associated the - * key, or null if the map contained no mapping for the key. - * - * This method behaves the same as {@link java.util.Map#remove(java.lang.Object)} yet wraps - * result in a future a {@link java.util.concurrent.Future}. - * - * Calling this method may or may not block until the operating is complete depending on the - * implementation. - * - * @param key key whose mapping is to be removed from the map - * @return a future containing either, the previous value associated with key, or - * null if there was no mapping for key. - * @see java.util.Map#remove(java.lang.Object) - * @see java.util.concurrent.Future - */ - Future removeLater(@NotNull K key); - /** * exports all the entries to a {@link java.io.File} storing them in JSON format, an attempt is * made where possible to use standard java serialisation and keep the data human readable, data diff --git a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java index 59e2dbf3a..29fc940f8 100644 --- a/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/StatelessChronicleMap.java @@ -157,40 +157,6 @@ class StatelessChronicleMap implements ChronicleMap, Closeable, Clon } - @NotNull - @Override - public Future getLater(@NotNull final K key) { - return lazyExecutorService().submit(new Callable() { - @Nullable - @Override - public V call() throws Exception { - return StatelessChronicleMap.this.get(key); - } - }); - } - - @NotNull - @Override - public Future putLater(@NotNull final K key, @NotNull final V value) { - return lazyExecutorService().submit(new Callable() { - @Nullable - @Override - public V call() throws Exception { - return StatelessChronicleMap.this.put(key, value); - } - }); - } - - @Override - public Future removeLater(@NotNull final K key) { - return lazyExecutorService().submit(new Callable() { - @Override - public V call() throws Exception { - return StatelessChronicleMap.this.remove(key); - } - }); - } - @Override public void getAll(File toFile) { throw new UnsupportedOperationException(); diff --git a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java index 2670bec71..d83b83753 100755 --- a/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java +++ b/src/main/java/net/openhft/chronicle/map/VanillaChronicleMap.java @@ -50,10 +50,7 @@ import java.lang.reflect.Array; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import static java.lang.Class.forName; import static java.lang.Long.numberOfTrailingZeros; @@ -569,102 +566,6 @@ class VanillaChronicleMap, } } - @Override - public Future getLater(@NotNull K key) { - final V v = get(key); - return new Future() { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public V get() throws InterruptedException, ExecutionException { - return v; - } - - @Override - public V get(long timeout, @NotNull TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - return v; - } - }; - } - - @Override - public Future putLater(@NotNull K key, @NotNull V value) { - final V v = put(key, value); - return new Future() { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public V get() throws InterruptedException, ExecutionException { - return v; - } - - @Override - public V get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - return v; - } - }; - } - - @Override - public Future removeLater(@NotNull K key) { - final V v = remove(key); - return new Future() { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public V get() throws InterruptedException, ExecutionException { - return v; - } - - @Override - public V get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { - return v; - } - }; - } - @Override public synchronized void getAll(File toFile) throws IOException { diff --git a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java index 6c98eb444..98eab7796 100644 --- a/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java +++ b/src/test/java/net/openhft/chronicle/map/CHMUseCasesTest.java @@ -36,7 +36,7 @@ public class CHMUseCasesTest { */ try (ChronicleMap map = ChronicleMapBuilder .of(String.class, String.class) - .checkSerializedValues() // for testng purposes only + .checkSerializedValues() // for testing purposes only .create()) { map.put("Hello", "World"); assertEquals("World", map.get("Hello")); @@ -65,10 +65,6 @@ public class CHMUseCasesTest { // TODO replace with the specific exception. } - assertEquals(null, map.putLater("Bye", "For now").get()); - assertEquals("For now", map.getLater("Bye").get()); - assertEquals("For now", map.removeLater("Bye").get()); - assertEquals(null, map.removeLater("Bye").get()); } } @@ -81,7 +77,7 @@ public class CHMUseCasesTest { public void testCharSequenceCharSequenceMap() throws ExecutionException, InterruptedException { try (ChronicleMap map = ChronicleMapBuilder .of(CharSequence.class, CharSequence.class) - .checkSerializedValues() // for testng purposes only + .checkSerializedValues() // for testing purposes only .create()) { map.put("Hello", "World"); StringBuilder key = new StringBuilder(); @@ -129,10 +125,6 @@ public class CHMUseCasesTest { } })); - assertEquals(null, map.putLater("Bye", "For now").get()); - assertEquals("For now", map.getLater("Bye").get().toString()); - assertEquals("For now", map.removeLater("Bye").get().toString()); - assertEquals(null, map.removeLater("Bye").get()); } } @@ -144,7 +136,7 @@ public class CHMUseCasesTest { public void testStringValueStringValueMap() { try (ChronicleMap map = ChronicleMapBuilder .of(StringValue.class, StringValue.class) - .checkSerializedValues() // for testng purposes only + .checkSerializedValues() // for testing purposes only .create()) { StringValue key1 = DataValueClasses.newDirectInstance(StringValue.class); StringValue key2 = DataValueClasses.newInstance(StringValue.class); @@ -283,10 +275,7 @@ public class CHMUseCasesTest { } catch (Exception todoMoreSpecificException) { } - assertEquals(null, map.putLater(3, 4).get()); - assertEquals((Integer) 4, map.getLater(3).get()); - assertEquals((Integer) 4, map.removeLater(3).get()); - assertEquals(null, map.removeLater(3).get()); + } } @@ -331,10 +320,7 @@ public class CHMUseCasesTest { } catch (Exception todoMoreSpecificException) { } - assertEquals(null, map.putLater(3L, 4L).get()); - assertEquals((Long) 4L, map.getLater(3L).get()); - assertEquals((Long) 4L, map.removeLater(3L).get()); - assertEquals(null, map.removeLater(3L).get()); + } } @@ -379,10 +365,7 @@ public class CHMUseCasesTest { } catch (Exception todoMoreSpecificException) { } - assertEquals(null, map.putLater(3.0, 4.0).get()); - assertEquals((Double) 4.0, map.getLater(3.0).get()); - assertEquals((Double) 4.0, map.removeLater(3.0).get()); - assertEquals(null, map.removeLater(3.0).get()); + } } @@ -430,10 +413,6 @@ public class CHMUseCasesTest { byte[] key3 = {3, 3, 3, 3}; byte[] value3 = {4, 4, 4, 4}; - assertEquals(null, map.putLater(key3, value3).get()); - assertTrue(Arrays.equals(value3, map.getLater(key3).get())); - assertTrue(Arrays.equals(value3, map.removeLater(key3).get())); - assertEquals(null, map.removeLater(key3).get()); } } @@ -521,13 +500,7 @@ public class CHMUseCasesTest { assertBBEquals(bb1, valueA); } - ByteBuffer key3 = ByteBuffer.wrap(new byte[]{3, 3, 3, 3}); - ByteBuffer value3 = ByteBuffer.wrap(new byte[]{4, 4, 4, 4}); - assertEquals(null, map.putLater(key3, value3).get()); - assertBBEquals(value3, map.getLater(key3).get()); - assertBBEquals(value3, map.removeLater(key3).get()); - assertEquals(null, map.removeLater(key3).get()); } } @@ -579,15 +552,6 @@ public class CHMUseCasesTest { assertBBEquals(ByteBuffer.wrap(new byte[]{12, 10, 11, 11}), map.get(key1)); - ByteBuffer key3 = ByteBuffer.wrap(new byte[]{3, 3, 3, 3}); - ByteBuffer value3 = ByteBuffer.allocateDirect(4); - value3.put(new byte[]{4, 4, 4, 4}); - value3.flip(); - - assertEquals(null, map.putLater(key3, value3).get()); - assertBBEquals(value3, map.getLater(key3).get()); - assertBBEquals(value3, map.removeLater(key3).get()); - assertEquals(null, map.removeLater(key3).get()); } } diff --git a/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java index 10292c16f..e70843d33 100644 --- a/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java +++ b/src/test/java/net/openhft/chronicle/map/ReplicationCheckingMap.java @@ -26,7 +26,6 @@ import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; /** * @author Rob Austin. @@ -228,21 +227,6 @@ public class ReplicationCheckingMap implements ChronicleMap { return map1.updateForKey(key, mutator); } - @Override - public Future getLater(@NotNull K key) { - throw new UnsupportedOperationException(); - } - - @Override - public Future putLater(@NotNull K key, @NotNull V value) { - throw new UnsupportedOperationException(); - } - - @Override - public Future removeLater(@NotNull K key) { - throw new UnsupportedOperationException(); - } - @Override public void getAll(File toFile) { throw new UnsupportedOperationException(); diff --git a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java index 66b664e45..1d4288b61 100644 --- a/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java +++ b/src/test/java/net/openhft/chronicle/map/StatelessClientTest.java @@ -453,60 +453,7 @@ public class StatelessClientTest { } } - @Test(timeout = 10000) - public void testGetLater() throws IOException, - InterruptedException, ExecutionException { - try (ChronicleMap serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replication((byte) 2, TcpTransportAndNetworkConfig.of(8056)).create()) { - try (ChronicleMap statelessMap = ChronicleMapBuilder.of(Integer - .class, CharSequence.class) - .statelessClient(new InetSocketAddress("localhost", 8056)).create()) { - statelessMap.put(1, "some value"); - - assertEquals("some value", statelessMap.getLater(1).get()); - assertEquals(1, statelessMap.size()); - - statelessMap.remove(1); - - assertEquals(null, statelessMap.getLater(1).get()); - assertEquals(0, statelessMap.size()); - } - } - } - - @Test - public void testPutLater() throws IOException, - InterruptedException, ExecutionException { - - try (ChronicleMap serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replication((byte) 2, TcpTransportAndNetworkConfig.of(8056)).create()) { - try (ChronicleMap statelessMap = ChronicleMapBuilder.of(Integer - .class, CharSequence.class) - .statelessClient(new InetSocketAddress("localhost", 8056)).create()) { - CharSequence oldValue = statelessMap.putLater(1, "some value").get(); - assertEquals("some value", statelessMap.get(1)); - assertEquals(1, statelessMap.size()); - } - } - } - - @Test(timeout = 10000) - public void testRemoveLater() throws IOException, - InterruptedException, ExecutionException { - - try (ChronicleMap serverMap = ChronicleMapBuilder.of(Integer.class, CharSequence.class) - .replication((byte) 2, TcpTransportAndNetworkConfig.of(8056)).create()) { - try (ChronicleMap statelessMap = ChronicleMapBuilder.of(Integer - .class, CharSequence.class) - .statelessClient(new InetSocketAddress("localhost", 8056)).create()) { - statelessMap.put(1, "some value"); - statelessMap.removeLater(1).get(); - assertEquals(null, statelessMap.get(1)); - assertEquals(0, statelessMap.size()); - } - } - } @Test(timeout = 10000) public void testGetAndEntryWeDontHave() throws IOException, diff --git a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java index 3aa530b2c..1c487acf1 100755 --- a/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java +++ b/src/test/java/net/openhft/chronicle/map/jrs166/map/StatelessChronicleMapTest.java @@ -33,7 +33,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.Future; import static org.junit.Assert.*; @@ -180,21 +179,6 @@ public class StatelessChronicleMapTest extends JSR166TestCase { return d.updateForKey(key, mutator); } - @Override - public Future getLater(@NotNull K key) { - return d.getLater(key); - } - - @Override - public Future putLater(@NotNull K key, @NotNull V value) { - return d.putLater(key, value); - } - - @Override - public Future removeLater(@NotNull K key) { - return d.removeLater(key); - } - @Override public void getAll(File toFile) throws IOException { d.getAll(toFile); From 5a8079f1e363847048b571bc4db89fb8dfcc115b Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 12:03:37 +0000 Subject: [PATCH 07/26] added "There is two key differences between Windows and Linux.." --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index cdbbc3dac..9f74ae368 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,21 @@ To illustrate this with an example - On Ubuntu we can create a 100 TB chronicle `ls -l` say the process virtual size / file size is 100 TB, however the resident memory via `du` says the size is 71 MB after adding 10000 entries. You can see the size actually used with du. +There is two key differences between Windows and Linux + +- windows fails if you attempt to use more than 2^20 * 4 KB pages in a single mapping (4 GB). This doesn't fail when you map the region, rather as you use it up. We should check for this and prevent users mapping regions of more than 4 GB on some (?all?) windows. It may be this limitation doesn't apply to newer or server based versions, so this requires some testing. In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. It is possible Microsoft might fix this first. + +- windows allocates memory and disk eagerly. Linux allocates memory and disk lazily. MacOSX allocates memory lazily and disk eagerly. On windows, eager memory allocation means you can't map more than free memory, but it should reduce jitter when you use it. Chronicle Map allocates head room which is a waste on Windows (Linux's sparse allocation means the head room has little impact) We could add an option for linux to more eagerly allocate space/disk on startup. e.g. to a percentage like 50%. + +Linux systems see a performance degradation at around 200% of main memory size. + +There likely to be more subtle differences, but these are the main ones which have caused problems. + +For production systems, we recommend; +- on Windows smaller map sizes of less than 4 GB each, and less than 50% main memory in total. +- on Linux small to large maps of less than double main memory. e.g. if you have a 128 GB server, have less than 256 GB of maps on the server. +- on MacOSX, we have no specific recommendations. + ### Chronicle Map Interface The Chronicle Map interface adds a few methods above an beyond the standard ConcurrentMap, the ChronicleMapBuilder can also be used to return the ChronicleMap, see the example below : From bddefa7a70ee61a548add2f517d2ee855660c5d5 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 12:05:57 +0000 Subject: [PATCH 08/26] added heading "How Operating Systems differ " --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9f74ae368..9f89b37dd 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,8 @@ To illustrate this with an example - On Ubuntu we can create a 100 TB chronicle `ls -l` say the process virtual size / file size is 100 TB, however the resident memory via `du` says the size is 71 MB after adding 10000 entries. You can see the size actually used with du. +### How Operating Systems differ + There is two key differences between Windows and Linux - windows fails if you attempt to use more than 2^20 * 4 KB pages in a single mapping (4 GB). This doesn't fail when you map the region, rather as you use it up. We should check for this and prevent users mapping regions of more than 4 GB on some (?all?) windows. It may be this limitation doesn't apply to newer or server based versions, so this requires some testing. In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. It is possible Microsoft might fix this first. From 1a43e5cc02032e454d86bbff3ceff27a1d3cdbfa Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 12:11:19 +0000 Subject: [PATCH 09/26] improved "How Operating Systems differ" --- README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9f89b37dd..62bb65cfc 100644 --- a/README.md +++ b/README.md @@ -299,14 +299,21 @@ says the size is 71 MB after adding 10000 entries. You can see the size actually There is two key differences between Windows and Linux -- windows fails if you attempt to use more than 2^20 * 4 KB pages in a single mapping (4 GB). This doesn't fail when you map the region, rather as you use it up. We should check for this and prevent users mapping regions of more than 4 GB on some (?all?) windows. It may be this limitation doesn't apply to newer or server based versions, so this requires some testing. In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. It is possible Microsoft might fix this first. +- Windows fails if you attempt to use more than 2^20 * 4 KB pages in a single mapping (4 GB). +This doesn't fail when you map the region, rather as you use it up. +It may be this limitation doesn't apply to newer or server based versions. +In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. -- windows allocates memory and disk eagerly. Linux allocates memory and disk lazily. MacOSX allocates memory lazily and disk eagerly. On windows, eager memory allocation means you can't map more than free memory, but it should reduce jitter when you use it. Chronicle Map allocates head room which is a waste on Windows (Linux's sparse allocation means the head room has little impact) We could add an option for linux to more eagerly allocate space/disk on startup. e.g. to a percentage like 50%. +- Windows allocates memory and disk eagerly. +- Linux allocates memory and disk lazily. +- MacOSX allocates memory lazily and disk eagerly. + +On Windows, eager memory allocation means you can't map more than free memory, but it should reduce jitter when you use it. +Chronicle Map allocates head room which is a waste on Windows (Linux's sparse allocation means the head room has +little impact). Linux systems see a performance degradation at around 200% of main memory size. -There likely to be more subtle differences, but these are the main ones which have caused problems. - For production systems, we recommend; - on Windows smaller map sizes of less than 4 GB each, and less than 50% main memory in total. - on Linux small to large maps of less than double main memory. e.g. if you have a 128 GB server, have less than 256 GB of maps on the server. From b8d7e874765a37e01d42b30ea7fbd59cd5416a1e Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 12:12:42 +0000 Subject: [PATCH 10/26] improved "How Operating Systems differ" --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 62bb65cfc..6dac3fe0a 100644 --- a/README.md +++ b/README.md @@ -302,9 +302,7 @@ There is two key differences between Windows and Linux - Windows fails if you attempt to use more than 2^20 * 4 KB pages in a single mapping (4 GB). This doesn't fail when you map the region, rather as you use it up. It may be this limitation doesn't apply to newer or server based versions. -In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. - -- Windows allocates memory and disk eagerly. +In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. Windows allocates memory and disk eagerly. - Linux allocates memory and disk lazily. - MacOSX allocates memory lazily and disk eagerly. From a56f8e6aef1320547c3bbc0a99babed86f13acd8 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 12:15:32 +0000 Subject: [PATCH 11/26] improved "How Operating Systems differ" --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 6dac3fe0a..f8032281d 100644 --- a/README.md +++ b/README.md @@ -299,8 +299,7 @@ says the size is 71 MB after adding 10000 entries. You can see the size actually There is two key differences between Windows and Linux -- Windows fails if you attempt to use more than 2^20 * 4 KB pages in a single mapping (4 GB). -This doesn't fail when you map the region, rather as you use it up. +- Windows fails if you attempt to use more than 4 GB in a single mapping. This calculated as 4GB = 2^20 * 4 KB pages. This doesn't fail when you map the region, rather as you use it up. It may be this limitation doesn't apply to newer or server based versions. In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. Windows allocates memory and disk eagerly. - Linux allocates memory and disk lazily. From 35e619d0f3085bd6d02874714b7ddeea0751f280 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 12:17:57 +0000 Subject: [PATCH 12/26] improved "How Operating Systems differ" --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f8032281d..c01a5bb47 100644 --- a/README.md +++ b/README.md @@ -299,9 +299,9 @@ says the size is 71 MB after adding 10000 entries. You can see the size actually There is two key differences between Windows and Linux -- Windows fails if you attempt to use more than 4 GB in a single mapping. This calculated as 4GB = 2^20 * 4 KB pages. This doesn't fail when you map the region, rather as you use it up. -It may be this limitation doesn't apply to newer or server based versions. -In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. Windows allocates memory and disk eagerly. +- Windows allocates memory and disk eagerly - Windows fails if you attempt to use more than 4 GB in a single mapping. This calculated as 4GB = 2^20 * 4 KB pages. This doesn't fail when you map the region, rather as you use it up. +It may be this limitation doesn't apply to newer or server based versions of Windows. +In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. - Linux allocates memory and disk lazily. - MacOSX allocates memory lazily and disk eagerly. From 291857e4d2da705e7ca49f265228486bfed4d5a2 Mon Sep 17 00:00:00 2001 From: leventov Date: Sun, 7 Dec 2014 20:19:28 +0700 Subject: [PATCH 13/26] Some debug output in TcpReplicator --- .../openhft/chronicle/map/TcpReplicator.java | 162 ++++++++++-------- 1 file changed, 88 insertions(+), 74 deletions(-) diff --git a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java index 4bdad2b43..50bafefc5 100644 --- a/src/main/java/net/openhft/chronicle/map/TcpReplicator.java +++ b/src/main/java/net/openhft/chronicle/map/TcpReplicator.java @@ -974,24 +974,31 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable // this can occur when new map are added to a channel final boolean handShakingComplete = attached.isHandShakingComplete(); - for (; ; ) { - final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0); + int entriesWritten = 0; + try { + for (; ; entriesWritten++) { + final boolean wasDataRead = modificationIterator.nextEntry(entryCallback, 0); - if (!wasDataRead) { - // if we have no more data to write to the socket then we will - // un-register OP_WRITE on the selector, until more data becomes available - if (in.position() == 0 && handShakingComplete) - disableWrite(socketChannel, attached); + if (!wasDataRead) { + // if we have no more data to write to the socket then we will + // un-register OP_WRITE on the selector, until more data becomes available + if (in.position() == 0 && handShakingComplete) + disableWrite(socketChannel, attached); - return; + return; + } + + // we've filled up the buffer lets give another channel a chance to send + // some data + if (in.remaining() <= maxEntrySizeBytes) + return; + + // if we have space in the buffer to write more data and we just wrote data + // into the buffer then let try and write some more } - - // we've filled up the buffer lets give another channel a chance to send some data - if (in.remaining() <= maxEntrySizeBytes) - return; - - // if we have space in the buffer to write more data and we just wrote data into the - // buffer then let try and write some more + } finally { + if (LOG.isDebugEnabled()) + LOG.debug("Entries written: {}", entriesWritten); } } @@ -1152,82 +1159,89 @@ final class TcpReplicator extends AbstractChannelReplicator implements Closeable * @throws InterruptedException */ void entriesFromBuffer(@NotNull Attached attached, @NotNull SelectionKey key) { - for (; ; ) { - out.limit(in.position()); + int entriesRead = 0; + try { + for (; ; entriesRead++) { + out.limit(in.position()); - // its set to MIN_VALUE when it should be read again - if (state == NOT_SET) { - if (out.remaining() < SIZE_OF_SIZE + 1) - return; + // its set to MIN_VALUE when it should be read again + if (state == NOT_SET) { + if (out.remaining() < SIZE_OF_SIZE + 1) + return; - // state is used for both heartbeat and stateless - state = out.readByte(); - sizeInBytes = out.readInt(); + // state is used for both heartbeat and stateless + state = out.readByte(); + sizeInBytes = out.readInt(); - assert sizeInBytes >= 0; + assert sizeInBytes >= 0; - // if the buffer is too small to read this payload we will have to grow the - // size of the buffer - long requiredSize = sizeInBytes + SIZE_OF_SIZE + 1; - if (out.capacity() < requiredSize) { - attached.entryReader.resizeBuffer(requiredSize + HEADROOM); + // if the buffer is too small to read this payload we will have to grow the + // size of the buffer + long requiredSize = sizeInBytes + SIZE_OF_SIZE + 1; + if (out.capacity() < requiredSize) { + attached.entryReader.resizeBuffer(requiredSize + HEADROOM); + } + + // this is the : + // -- heartbeat if its 0 + // -- stateful update if its 1 + // -- the id of the stateful event + if (state == NOT_SET) + continue; } - // this is the : - // -- heartbeat if its 0 - // -- stateful update if its 1 - // -- the id of the stateful event - if (state == NOT_SET) - continue; - } + if (out.remaining() < sizeInBytes) { + return; + } - if (out.remaining() < sizeInBytes) { - return; - } + final long nextEntryPos = out.position() + sizeInBytes; + assert nextEntryPos > 0; + final long limit = out.limit(); + out.limit(nextEntryPos); - final long nextEntryPos = out.position() + sizeInBytes; - assert nextEntryPos > 0; - final long limit = out.limit(); - out.limit(nextEntryPos); + boolean isStatelessClient = (state != 1); - boolean isStatelessClient = (state != 1); + if (isStatelessClient) { + if (statelessServerConnector == null) { + LOG.error("", new IllegalArgumentException("received an event " + + "from a stateless map, stateless maps are not " + + "currently supported when using Chronicle Channels")); + } else { - if (isStatelessClient) { - if (statelessServerConnector == null) { - LOG.error("", new IllegalArgumentException("received an event " + - "from a stateless map, stateless maps are not " + - "currently supported when using Chronicle Channels")); - } else { + final Work futureWork = + statelessServerConnector.processStatelessEvent(state, + attached.entryWriter.in, attached.entryReader.out); - final Work futureWork = statelessServerConnector.processStatelessEvent(state, - attached.entryWriter.in, attached.entryReader.out); + // turn the OP_WRITE on + key.interestOps(key.interestOps() | OP_WRITE); - // turn the OP_WRITE on - key.interestOps(key.interestOps() | OP_WRITE); - - // in some cases it may not be possible to send out all the data before we - // fill out the write buffer, so this data will be send when the buffer - // is no longer full, and as such is treated as future work - if (futureWork != null) { - try { // we will complete what we can for now - boolean isComplete = futureWork.doWork(attached.entryWriter.in); - if (!isComplete) - attached.entryWriter.uncompletedWork = futureWork; - } catch (Exception e) { - LOG.error("", e); + // in some cases it may not be possible to send out all the data before + // we fill out the write buffer, so this data will be send when + // the buffer is no longer full, and as such is treated as future work + if (futureWork != null) { + try { // we will complete what we can for now + boolean isComplete = futureWork.doWork(attached.entryWriter.in); + if (!isComplete) + attached.entryWriter.uncompletedWork = futureWork; + } catch (Exception e) { + LOG.error("", e); + } } } - } - } else - externalizable.readExternalEntry(copies, segmentState, out); + } else + externalizable.readExternalEntry(copies, segmentState, out); - out.limit(limit); + out.limit(limit); - // skip onto the next entry - out.position(nextEntryPos); + // skip onto the next entry + out.position(nextEntryPos); - state = NOT_SET; - sizeInBytes = 0; + state = NOT_SET; + sizeInBytes = 0; + } + } finally { + if (LOG.isDebugEnabled()) + LOG.debug("Entries read: {}", entriesRead); } } From b179834ff16ce9b0e0127210e7414a123fc4bffa Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 19:03:08 +0000 Subject: [PATCH 14/26] added more under -"#### How many bytes is each entry" --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index c01a5bb47..df4420e4e 100644 --- a/README.md +++ b/README.md @@ -260,11 +260,13 @@ You set the maximum number of entries by the builder: ``` java ConcurrentMap map = ChronicleMapBuilder.of(Integer.class, CharSequence.class) + constantValueSizeBySample("a long sample string"), .entries(1000) // set the max number of entries here .create(); ``` In this example above we have set 1000 entries. + We have optimised chronicle, So that you can have situations where you either don't use; - all the entries you have allowed for. This works best on Unix where the disk space and memory @@ -275,6 +277,19 @@ lines (128 bytes +), only the lines you touch sit in your CPU cache and if you h (8+ Kbytes) only the pages you touch use memory or disk. The CPU cache usage matters as it can be 10000x smaller than main memory. +#### How many bytes is each entry + +For example, if you had ant entry that had both Integer keys and values, these take exactly 4 bytes themselves. +There is also multi-map overhead per entry, ~6 bytes if your segment size is < 64k, + ~9 if < 16m and ~12 if > 16m entries per segment. If you create replicated map, + there is also 10 bytes per entry for replication purposes. + +We suggest you don't configure size for constant-sized keys or values, instead you can use the +builder methods .constantKeySizeBySample(sampleKey) and +.constantValueSizeBySample(sampleValue), For common types like Integer we suggest you don't use +these methods, for example ChronicleMap knows that Integer is 4 bytes long, Long is 8, etc. + + ### Size of space reserved on disk In linux, if you looked at the size of the 'file', it will report the used entry size so if you From 7ec3cdd53366bc025113e656ffe2c2092c1d9b3b Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 20:26:52 +0000 Subject: [PATCH 15/26] updated Entry size --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index df4420e4e..aa1986f42 100644 --- a/README.md +++ b/README.md @@ -277,12 +277,15 @@ lines (128 bytes +), only the lines you touch sit in your CPU cache and if you h (8+ Kbytes) only the pages you touch use memory or disk. The CPU cache usage matters as it can be 10000x smaller than main memory. -#### How many bytes is each entry +#### Entry size -For example, if you had ant entry that had both Integer keys and values, these take exactly 4 bytes themselves. -There is also multi-map overhead per entry, ~6 bytes if your segment size is < 64k, - ~9 if < 16m and ~12 if > 16m entries per segment. If you create replicated map, - there is also 10 bytes per entry for replication purposes. +The size of each entry depends on the type of the Keys and Values, as some types are larger than others. For example, if an entry uses Integers for both the keys and values, both the key and value each take exactly 4 bytes. +There is also some other overhead which is internal to chronicle, such as its internal multi-map, which has an overhead of +* 6 bytes per entry for segment size of < 64k, +* 9 bytes per entry for segment size of < 16m but > 64k +* 12 bytes per entry for segment size of > 16m entries per segment. + +Also, if you create a replicated map, there is an additional 10 bytes per entry. We suggest you don't configure size for constant-sized keys or values, instead you can use the builder methods .constantKeySizeBySample(sampleKey) and From b659d8777b947482672b9116dad9e02b0086c514 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 21:09:33 +0000 Subject: [PATCH 16/26] added performance test results --- README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa1986f42..07efdeb28 100644 --- a/README.md +++ b/README.md @@ -1126,13 +1126,13 @@ For in memory data-structures like a HashMap this isn’t a big problem. But in So if you don’t require the old value and don’t wish to block until your `put()` has been received by the server, then you may wish to consider using the following configuration : ``` java -.putReturnsNull +.putReturnsNull(true) ``` and also for the `remove()` method ``` java -.removeReturnsNull +.removeReturnsNull(true) ``` ``` java @@ -1153,6 +1153,55 @@ ChronicleMapBuilder.of(Integer.class, CharSequence.class) .create(); ``` +##### Performance + +The throughput and latency performance for different configurations. + +Tested using a test called BGChronicleTest. + +On one machine (i7 3.5 GHz) we have two persisted replicas run as + +-Dreplicas=2 eg.BGChronicleTest server + +On another machine (Dual Xeon 8 core 2.6 GHz) connected via a pair of Solarflare SFN5121T 10 Gig-E with onload enabled. + +-Dreplicas=2 -Dclients={see below} -DmaxRate=30000 -DreadRatio=2 eg.BGChronicleTest client + +2 clients +Throughput test +messages per seconds: 58,864 + +Latency test at 30,000 msg/sec +50% / 90% / 99% // 99.9% / 99.99% / worst latency was 33 / 80 / 111 // 120 / 148 / 3,921 us + +4 clients +Throughput test +messages per seconds: 94,006 + +Latency test at 30,000 msg/sec +50% / 90% / 99% // 99.9% / 99.99% / worst latency was 32 / 94 / 106 // 131 / 177 / 2,153 us + +8 clients +Throughput test +messages per seconds: 162,961 + +Latency test at 30,000 msg/sec +50% / 90% / 99% // 99.9% / 99.99% / worst latency was 35 / 94 / 117 // 140 / 167 / 1,685 us + +16 clients +Throughput test +messages per seconds: 267,097 + +Latency test at 30,000 msg/sec +50% / 90% / 99% // 99.9% / 99.99% / worst latency was 38 / 97 / 122 // 149 / 174 / 2,771 us + +24 clients +Throughput test +messages per seconds: 253,052 + +Latency test at 30,000 msg/sec +50% / 90% / 99% // 99.9% / 99.99% / worst latency was 40 / 99 / 121 // 151 / 243 / 3,669 us + ##### Close its always important to close `ChronicleMap`'s and `ChronicleSet` 's when you have finished with them From ed2df2895ca08c0d02a7ccdda24d55246af4157c Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 21:57:54 +0000 Subject: [PATCH 17/26] updated Entry size --- README.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 07efdeb28..a7a2a24bc 100644 --- a/README.md +++ b/README.md @@ -315,24 +315,23 @@ says the size is 71 MB after adding 10000 entries. You can see the size actually ### How Operating Systems differ -There is two key differences between Windows and Linux +As a pure java library, the same chronicle map java byte code can be run on Windows, Linux and Mac OSX. +However these operating systems work with memory mapped files differently, these differences effect how +chronicle is able to map memory to a file, and hence this can impact the total number of entries that you are +able to configure. -- Windows allocates memory and disk eagerly - Windows fails if you attempt to use more than 4 GB in a single mapping. This calculated as 4GB = 2^20 * 4 KB pages. This doesn't fail when you map the region, rather as you use it up. -It may be this limitation doesn't apply to newer or server based versions of Windows. -In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. -- Linux allocates memory and disk lazily. -- MacOSX allocates memory lazily and disk eagerly. +- Windows allocates memory and disk eagerly, Windows will fail if more than 4 GB is allocated in a single memory +mapping, ( calculated as 4GB = 2^20 * 4 KB pages). Windows doesn't fail when a memory mapped region is mapped, rather it will fail when it is used up. This limitation doesn't apply to newer or server based versions of Windows. Eager memory allocation means you can't map more than free memory, but it should reduce jitter when you use it. In the future we may support multiple mappings to avoid this limitation, but there is no immediate plan to do so. +- Linux allocates memory and disk lazily. Linux systems see a performance degradation at around 200% of main memory. +- Mac OSX allocates memory lazily and disk eagerly. -On Windows, eager memory allocation means you can't map more than free memory, but it should reduce jitter when you use it. -Chronicle Map allocates head room which is a waste on Windows (Linux's sparse allocation means the head room has -little impact). +Chronicle Map allocates head room which is a waste on Windows (Linux's sparse allocation means the head room has little +impact). -Linux systems see a performance degradation at around 200% of main memory size. - -For production systems, we recommend; -- on Windows smaller map sizes of less than 4 GB each, and less than 50% main memory in total. -- on Linux small to large maps of less than double main memory. e.g. if you have a 128 GB server, have less than 256 GB of maps on the server. -- on MacOSX, we have no specific recommendations. +For production systems +- on Windows we recommend you use map sizes of less than 4 GB each, and less than 50% main memory in total. +- on Linux we recommend you use small to large maps of less than double main memory. e.g. if you have a 128 GB server, we recommend you have less than 256 GB of maps on the server. +- on Mac OSX, we have no specific recommendations. ### Chronicle Map Interface The Chronicle Map interface adds a few methods above an beyond the standard ConcurrentMap, From 60ad0c2ce9425a15b14fa671a307741b6b150459 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 21:59:12 +0000 Subject: [PATCH 18/26] updated Entry size --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a7a2a24bc..9bb5765a4 100644 --- a/README.md +++ b/README.md @@ -328,7 +328,7 @@ mapping, ( calculated as 4GB = 2^20 * 4 KB pages). Windows doesn't fail when a m Chronicle Map allocates head room which is a waste on Windows (Linux's sparse allocation means the head room has little impact). -For production systems +##### For production - on Windows we recommend you use map sizes of less than 4 GB each, and less than 50% main memory in total. - on Linux we recommend you use small to large maps of less than double main memory. e.g. if you have a 128 GB server, we recommend you have less than 256 GB of maps on the server. - on Mac OSX, we have no specific recommendations. From a2aaaa3246f260bea5ebdfff3688693fd8ff245e Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 22:05:40 +0000 Subject: [PATCH 19/26] change the example to use CharSequence --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9bb5765a4..34a51534c 100644 --- a/README.md +++ b/README.md @@ -350,18 +350,17 @@ supports the following methods : - [`ReadContext getUsingLocked(@NotNull K key, @NotNull V usingValue);`] (https://github.com/OpenHFT/Chronicle-Map#off-heap-storage-and-how-using-a-proxy-object-can-improve-performance) - [`WriteContext acquireUsingLocked(@NotNull K key, @NotNull V usingValue);`] (https://github.com/OpenHFT/Chronicle-Map#acquireusinglocked) -These methods let you provide the object which the data will be written to, even if the object is -immutable. For example +These methods let you provide the object which the data will be written to, but the value use to be mutable. For example ``` java -String myString = ""; -String myResult = map.getUsing("key", myString); +CharSequence using = new StringBuilder(); +CharSequence myResult = map.getUsing("key", using); // at this point the myString and myResult will both point to the same object ``` The `map.getUsing()` method is similar to `map.get()`, but because Chronicle Map stores its data off heap, if you were to call get("key"), a new object would be created each time, map.getUsing() works -by reusing the heap memory which was used by the original Object "myString". This technique provides +by reusing the heap memory which was used by the original Object "using". This technique provides you with better control over your object creation. Exactly like `map.getUsing()`, `map.acquireUsing()` will give you back a reference to an value From 3db6cac3a0ee9fac4d6fcf5be4a3bfbeb3816e43 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 22:21:26 +0000 Subject: [PATCH 20/26] minor corrections --- README.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 34a51534c..1cd8a83f3 100644 --- a/README.md +++ b/README.md @@ -453,7 +453,7 @@ bond.getCoupon() lets say that it is only the `coupon` field that we are interested in, then its better not to have to deserialise the whole object that implements the `BondVOInterface`. The `ChronicleMapBuilder` will look a the types of keys and values that you use, If the value type is a simple accessor/mutator interface that is exposing a non nested pojo, which uses simple types - like String and primitives with corresponding get..() and + like `CharSequence` and primitives with corresponding get..() and set..() methods, Chronicle is able to generate off heap poxies so the whole object is not desrialized each time it is accessed, The off heap poxies are able to read and write into @@ -461,7 +461,7 @@ the off heap data structures directly, this reduced serialisation can give you a Below we show you how you can work directly with the off heap entries. ``` java - ChronicleMap chm = ChronicleMapBuilder + ChronicleMap chm = ChronicleMapBuilder .of(String.class, BondVOInterface.class) .keySize(10) .create(); @@ -470,7 +470,7 @@ Below we show you how you can work directly with the off heap entries. notice that the ``` java -.of(String.class, BondVOInterface.class) +.of(CharSequence.class, BondVOInterface.class) ``` value class, in our case `BondVOInterface.class` is an `interface` rather than a `class`, now @@ -542,14 +542,13 @@ context.close() // the lock will get released when this is called ``` #### acquireUsingLocked() - -just like getUsing(), acquireUsing() will also recycle the value you pass it, the following -code is a pattern that you will often come across, acquireUsing(key,value) offers this -functionality with a single method call : +Just like getUsing(), acquireUsing() will also recycle the value you pass it, the following +code is a pattern that you will often come across, `acquireUsing(key,value)` offers this +functionality i the example, below with a single method call : ``` java V acquireUsing(key,value) { - Lock l = ...; // the segement lock of the map + Lock l = ...; // the segment lock of the map l.lock(); try { V v = map.getUsing(key,value) @@ -564,12 +563,12 @@ V acquireUsing(key,value) { } ``` -if you are only accessing ChronicleMap from a single thread and you are not doing replication -and don't care about atomic reads, then its simpler ( and faster ) to use acquireUsing() otherwise we -recommend acquireUsingLocked(key,value) +If you are only accessing ChronicleMap from a single thread. If you are not doing replication +and don't care about atomic reads. Then its simpler ( and faster ) to use acquireUsing() otherwise we +recommend you use `acquireUsingLocked(key,value)` -because acquireUsing can end up creating an entry the acquireUsingLocked(key,value) method holds -a segment write lock, this is unlike the getUsing(key,using) method that holds a segment read lock. +Since the acquireUsing() method can end up creating an entry, the acquireUsingLocked(key,value) method must hold +a segment write lock, this is unlike getUsing(key,using) which only holds a segment read lock. ``` java BondVOInterface bond = ... // create your instance @@ -584,7 +583,7 @@ try (WriteContext context = map.acquireUsingLocked("one", bo } ``` -if after you have read the 'issueDate' and 'symbol' and you wish to remove the entry based on some +If after you have read the 'issueDate' and 'symbol' and you wish to remove the entry based on some business logic, it more efficient to use the 'context' to remove the entry, as the contents is already aware when the entry is in memory. @@ -593,7 +592,7 @@ already aware when the entry is in memory. It is possible for the size of your entry to be twice as large as the maximum entry size, we refer to this type of entry as an oversized entry. Oversized entries are there to cater for the case where only a small -percentage of your entries are twise as large as the others, in this case your large entry will +percentage of your entries are twice as large as the others, in this case your large entry will span across two entries. The alternative would be to increase your maximum entry size to be similar to the size of the largest entry, but this approach is wasteful of memory, especially when most entries are no where near the max entry size. From 831539133e8adc5f727fc30d22cdee63095505ea Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 22:23:01 +0000 Subject: [PATCH 21/26] minor corrections --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1cd8a83f3..960a7357f 100644 --- a/README.md +++ b/README.md @@ -584,7 +584,7 @@ try (WriteContext context = map.acquireUsingLocked("one", bo ``` If after you have read the 'issueDate' and 'symbol' and you wish to remove the entry based on some -business logic, it more efficient to use the 'context' to remove the entry, as the contents is +business logic, its more efficient to use the 'context' to remove the entry, as the contents is already aware when the entry is in memory. ## Oversized Entries Support From 3d1cc76a055f2e50c3468fd25656a4f94f1c14d4 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 22:25:45 +0000 Subject: [PATCH 22/26] updated oversized --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 960a7357f..e2fe20761 100644 --- a/README.md +++ b/README.md @@ -589,11 +589,11 @@ already aware when the entry is in memory. ## Oversized Entries Support -It is possible for the size of your entry to be twice as large as the maximum entry size, +It is possible for the size of your entry to be 64 times as large as the maximum entry size, we refer to this type of entry as an oversized entry. Oversized entries are there to cater for the case where only a small -percentage of your entries are twice as large as the others, in this case your large entry will -span across two entries. The alternative would be to increase your maximum entry size to be similar +percentage of your entries are larger than the others, in this case your large entry will +span across a number of entries. The alternative would be to increase your maximum entry size to be similar to the size of the largest entry, but this approach is wasteful of memory, especially when most entries are no where near the max entry size. From f3f3c84c0f1332d06044b52015d29d8d97fbc260 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 22:28:38 +0000 Subject: [PATCH 23/26] updated import --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e2fe20761..cfa44ebee 100644 --- a/README.md +++ b/README.md @@ -628,10 +628,12 @@ void putAll(File fromFile) throws IOException; Its only the entries of your map that are exported, not the configuration of your map. So care must be taken to populate the data in to a map of the correct Key/Value type and with enough available entries. When importing data : + * entries that are in the map but not in the JSON file will remain untouched. * entries that are in the map and in the JSON file will be updated. -* entries that are not in the map but are in the JSON file wil added. -* In other words importing data into a Chronicle Map works like `map.putAll()`. +* entries that are not in the map but are in the JSON file will be added. + +In other words importing data into a Chronicle Map works like `map.putAll()`. When Importing data if you are also writing to the map at the same time, the last update will win. In other words a write lock is not held for the entire import process. From dba44661a03e7b70601694002856e1b9d3710b71 Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 22:32:59 +0000 Subject: [PATCH 24/26] correct comment on Close() --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index cfa44ebee..0c88baf3b 100644 --- a/README.md +++ b/README.md @@ -652,10 +652,7 @@ Its recommended that you call close() once you have finished working with a Chro map.close() ``` -You only need to close to clean up resources deterministically. If your program is exiting, -you don't need to close the collection, as Chronicle never knows when the program might crash, -so we have designed it so you don't have to close() it. - +When using replication its important to call close to clean up the NIO socket connections. Failure to call close may prevent you from restarting a replicated map on the same port. WARNING : If you call close too early before you have finished working with the map, this can cause your JVM to crash. Close MUST BE the last thing that you do with the map. From 78ca1c55fd0dcdcc499544d65ebf1f840dd6438f Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Sun, 7 Dec 2014 22:35:10 +0000 Subject: [PATCH 25/26] correct comment on Close() --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0c88baf3b..61895d93c 100644 --- a/README.md +++ b/README.md @@ -652,10 +652,13 @@ Its recommended that you call close() once you have finished working with a Chro map.close() ``` -When using replication its important to call close to clean up the NIO socket connections. Failure to call close may prevent you from restarting a replicated map on the same port. -WARNING : If you call close too early before you have finished working with the map, this can cause -your JVM to crash. Close MUST BE the last thing that you do with the map. +When using replication its important to call close to clean up the TCP/IP socket connection. +Failure to call close may prevent you from restarting a replicated map on the same port. +###### WARNING + +If you call close too early before you have finished working with the map, this can cause +your JVM to crash. Close MUST BE the last thing that you do with the map. # TCP / UDP Replication From c7bfb46b0b3d1340c629bdee1e1374c603303eac Mon Sep 17 00:00:00 2001 From: boundedbuffer Date: Mon, 8 Dec 2014 09:27:37 +0000 Subject: [PATCH 26/26] correct comment on Close() --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 61895d93c..9e26588df 100644 --- a/README.md +++ b/README.md @@ -648,12 +648,16 @@ VALUES, future versions will support a binary encoding of objects that are `net. ## Close Unlike ConcurrentHashMap, chronicle map stores its data off heap, often in a memory mapped file. Its recommended that you call close() once you have finished working with a Chronicle Map. + ``` java map.close() ``` -When using replication its important to call close to clean up the TCP/IP socket connection. -Failure to call close may prevent you from restarting a replicated map on the same port. +This is especially important when working with chronicle map replication, as failure to call close may prevent +you from restarting a replicated map on the same port. In the event that your application crashes it may not +be possible to call close(). Your operating system will usually close dangling ports automatically, +so although it is recommended that you close() when you have finished with the map, +its not something that you must do, its just something that we recommend you should do. ###### WARNING