From 08f2d5d6f112cccc997d43aeae798eea397ec61e Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Sat, 28 May 2016 10:22:47 +0200 Subject: [PATCH 01/31] Added write(ByteBuffer) to Streamable interface and a first draft for a NioNetworkHandler --- .../ch/dissem/bitmessage/entity/Addr.java | 15 +- .../bitmessage/entity/CustomMessage.java | 12 ++ .../ch/dissem/bitmessage/entity/GetData.java | 9 ++ .../java/ch/dissem/bitmessage/entity/Inv.java | 9 ++ .../bitmessage/entity/NetworkMessage.java | 39 ++++- .../bitmessage/entity/ObjectMessage.java | 11 ++ .../dissem/bitmessage/entity/Plaintext.java | 38 +++++ .../dissem/bitmessage/entity/Streamable.java | 3 + .../ch/dissem/bitmessage/entity/VerAck.java | 6 + .../ch/dissem/bitmessage/entity/Version.java | 13 ++ .../bitmessage/entity/payload/CryptoBox.java | 18 +++ .../entity/payload/GenericPayload.java | 6 + .../bitmessage/entity/payload/GetPubkey.java | 6 + .../dissem/bitmessage/entity/payload/Msg.java | 7 + .../bitmessage/entity/payload/Pubkey.java | 5 + .../bitmessage/entity/payload/V2Pubkey.java | 16 +- .../entity/payload/V4Broadcast.java | 6 + .../bitmessage/entity/payload/V4Pubkey.java | 12 ++ .../entity/valueobject/InventoryVector.java | 10 +- .../entity/valueobject/NetworkAddress.java | 28 +++- .../entity/valueobject/PrivateKey.java | 47 ++++-- .../bitmessage/ports/NetworkHandler.java | 2 + .../ch/dissem/bitmessage/utils/Encode.java | 115 +++++++++----- .../extensions/pow/ProofOfWorkRequest.java | 8 + .../networking/DefaultNetworkHandler.java | 1 - .../networking/nio/ConnectionInfo.java | 51 ++++++ .../networking/nio/NioNetworkHandler.java | 147 ++++++++++++++++++ 27 files changed, 561 insertions(+), 79 deletions(-) create mode 100644 networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java create mode 100644 networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/Addr.java b/core/src/main/java/ch/dissem/bitmessage/entity/Addr.java index 6125910..73d9995 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/Addr.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/Addr.java @@ -21,6 +21,7 @@ import ch.dissem.bitmessage.utils.Encode; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -47,10 +48,18 @@ public class Addr implements MessagePayload { } @Override - public void write(OutputStream stream) throws IOException { - Encode.varInt(addresses.size(), stream); + public void write(OutputStream out) throws IOException { + Encode.varInt(addresses.size(), out); for (NetworkAddress address : addresses) { - address.write(stream); + address.write(out); + } + } + + @Override + public void write(ByteBuffer buffer) { + Encode.varInt(addresses.size(), buffer); + for (NetworkAddress address : addresses) { + address.write(buffer); } } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/CustomMessage.java b/core/src/main/java/ch/dissem/bitmessage/entity/CustomMessage.java index e43f56d..439f003 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/CustomMessage.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/CustomMessage.java @@ -21,6 +21,7 @@ import ch.dissem.bitmessage.utils.AccessCounter; import ch.dissem.bitmessage.utils.Encode; import java.io.*; +import java.nio.ByteBuffer; import static ch.dissem.bitmessage.utils.Decode.bytes; import static ch.dissem.bitmessage.utils.Decode.varString; @@ -85,6 +86,17 @@ public class CustomMessage implements MessagePayload { } } + @Override + public void write(ByteBuffer buffer) { + if (data != null) { + Encode.varString(command, buffer); + buffer.put(data); + } else { + throw new ApplicationException("Tried to write custom message without data. " + + "Programmer: did you forget to override #write()?"); + } + } + public boolean isError() { return COMMAND_ERROR.equals(command); } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/GetData.java b/core/src/main/java/ch/dissem/bitmessage/entity/GetData.java index 44fab5b..7d14fa0 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/GetData.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/GetData.java @@ -21,6 +21,7 @@ import ch.dissem.bitmessage.utils.Encode; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; @@ -55,6 +56,14 @@ public class GetData implements MessagePayload { } } + @Override + public void write(ByteBuffer buffer) { + Encode.varInt(inventory.size(), buffer); + for (InventoryVector iv : inventory) { + iv.write(buffer); + } + } + public static final class Builder { private List inventory = new LinkedList<>(); diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/Inv.java b/core/src/main/java/ch/dissem/bitmessage/entity/Inv.java index fd2d40d..8d0f592 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/Inv.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/Inv.java @@ -21,6 +21,7 @@ import ch.dissem.bitmessage.utils.Encode; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; @@ -53,6 +54,14 @@ public class Inv implements MessagePayload { } } + @Override + public void write(ByteBuffer buffer) { + Encode.varInt(inventory.size(), buffer); + for (InventoryVector iv : inventory) { + iv.write(buffer); + } + } + public static final class Builder { private List inventory = new LinkedList<>(); diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java b/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java index 860c6ed..e549101 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java @@ -22,6 +22,7 @@ import ch.dissem.bitmessage.utils.Encode; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.NoSuchAlgorithmException; @@ -74,9 +75,7 @@ public class NetworkMessage implements Streamable { out.write('\0'); } - ByteArrayOutputStream payloadStream = new ByteArrayOutputStream(); - payload.write(payloadStream); - byte[] payloadBytes = payloadStream.toByteArray(); + byte[] payloadBytes = Encode.bytes(payload); // Length of payload in number of bytes. Because of other restrictions, there is no reason why this length would // ever be larger than 1600003 bytes. Some clients include a sanity-check to avoid processing messages which are @@ -93,4 +92,38 @@ public class NetworkMessage implements Streamable { // message payload out.write(payloadBytes); } + + @Override + public void write(ByteBuffer out) { + // magic + Encode.int32(MAGIC, out); + + // ASCII string identifying the packet content, NULL padded (non-NULL padding results in packet rejected) + String command = payload.getCommand().name().toLowerCase(); + try { + out.put(command.getBytes("ASCII")); + } catch (UnsupportedEncodingException e) { + throw new ApplicationException(e); + } + for (int i = command.length(); i < 12; i++) { + out.put((byte) 0); + } + + byte[] payloadBytes = Encode.bytes(payload); + + // Length of payload in number of bytes. Because of other restrictions, there is no reason why this length would + // ever be larger than 1600003 bytes. Some clients include a sanity-check to avoid processing messages which are + // larger than this. + Encode.int32(payloadBytes.length, out); + + // checksum + try { + out.put(getChecksum(payloadBytes)); + } catch (GeneralSecurityException e) { + throw new ApplicationException(e); + } + + // message payload + out.put(payloadBytes); + } } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/ObjectMessage.java b/core/src/main/java/ch/dissem/bitmessage/entity/ObjectMessage.java index 6f74257..fee761c 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/ObjectMessage.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/ObjectMessage.java @@ -29,6 +29,7 @@ import ch.dissem.bitmessage.utils.Encode; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Objects; @@ -168,6 +169,16 @@ public class ObjectMessage implements MessagePayload { out.write(getPayloadBytesWithoutNonce()); } + @Override + public void write(ByteBuffer buffer) { + if (nonce == null) { + buffer.put(new byte[8]); + } else { + buffer.put(nonce); + } + buffer.put(getPayloadBytesWithoutNonce()); + } + private void writeHeaderWithoutNonce(OutputStream out) throws IOException { Encode.int64(expiresTime, out); Encode.int32(objectType, out); diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java b/core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java index 4bfaba4..242eb95 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java @@ -25,6 +25,7 @@ import ch.dissem.bitmessage.factory.Factory; import ch.dissem.bitmessage.utils.*; import java.io.*; +import java.nio.ByteBuffer; import java.util.*; import java.util.Collections; @@ -197,12 +198,49 @@ public class Plaintext implements Streamable { } } } + public void write(ByteBuffer buffer, boolean includeSignature) { + Encode.varInt(from.getVersion(), buffer); + Encode.varInt(from.getStream(), buffer); + Encode.int32(from.getPubkey().getBehaviorBitfield(), buffer); + buffer.put(from.getPubkey().getSigningKey(), 1, 64); + buffer.put(from.getPubkey().getEncryptionKey(), 1, 64); + if (from.getVersion() >= 3) { + Encode.varInt(from.getPubkey().getNonceTrialsPerByte(), buffer); + Encode.varInt(from.getPubkey().getExtraBytes(), buffer); + } + if (type == Type.MSG) { + buffer.put(to.getRipe()); + } + Encode.varInt(encoding, buffer); + Encode.varInt(message.length, buffer); + buffer.put(message); + if (type == Type.MSG) { + if (to.has(Feature.DOES_ACK) && getAckMessage() != null) { + Encode.varBytes(Encode.bytes(getAckMessage()), buffer); + } else { + Encode.varInt(0, buffer); + } + } + if (includeSignature) { + if (signature == null) { + Encode.varInt(0, buffer); + } else { + Encode.varInt(signature.length, buffer); + buffer.put(signature); + } + } + } @Override public void write(OutputStream out) throws IOException { write(out, true); } + @Override + public void write(ByteBuffer buffer) { + write(buffer, true); + } + public Object getId() { return id; } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/Streamable.java b/core/src/main/java/ch/dissem/bitmessage/entity/Streamable.java index cc12050..e75a926 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/Streamable.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/Streamable.java @@ -19,10 +19,13 @@ package ch.dissem.bitmessage.entity; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; +import java.nio.ByteBuffer; /** * An object that can be written to an {@link OutputStream} */ public interface Streamable extends Serializable { void write(OutputStream stream) throws IOException; + + void write(ByteBuffer buffer); } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/VerAck.java b/core/src/main/java/ch/dissem/bitmessage/entity/VerAck.java index 815a20f..3d30f32 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/VerAck.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/VerAck.java @@ -18,6 +18,7 @@ package ch.dissem.bitmessage.entity; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; /** * The 'verack' command answers a 'version' command, accepting the other node's version. @@ -34,4 +35,9 @@ public class VerAck implements MessagePayload { public void write(OutputStream stream) throws IOException { // 'verack' doesn't have any payload, so there is nothing to write } + + @Override + public void write(ByteBuffer buffer) { + // 'verack' doesn't have any payload, so there is nothing to write + } } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/Version.java b/core/src/main/java/ch/dissem/bitmessage/entity/Version.java index 8539be1..48022c4 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/Version.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/Version.java @@ -23,6 +23,7 @@ import ch.dissem.bitmessage.utils.UnixTime; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Random; /** @@ -134,6 +135,18 @@ public class Version implements MessagePayload { Encode.varIntList(streams, stream); } + @Override + public void write(ByteBuffer buffer) { + Encode.int32(version, buffer); + Encode.int64(services, buffer); + Encode.int64(timestamp, buffer); + addrRecv.write(buffer, true); + addrFrom.write(buffer, true); + Encode.int64(nonce, buffer); + Encode.varString(userAgent, buffer); + Encode.varIntList(streams, buffer); + } + public static final class Builder { private int version; diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/CryptoBox.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/CryptoBox.java index 89d7bd7..f7f3c15 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/CryptoBox.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/CryptoBox.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.ByteBuffer; import java.util.Arrays; import static ch.dissem.bitmessage.entity.valueobject.PrivateKey.PRIVATE_KEY_SIZE; @@ -144,12 +145,29 @@ public class CryptoBox implements Streamable { out.write(x, offset, length); } + private void writeCoordinateComponent(ByteBuffer buffer, byte[] x) { + int offset = Bytes.numberOfLeadingZeros(x); + int length = x.length - offset; + Encode.int16(length, buffer); + buffer.put(x, offset, length); + } + @Override public void write(OutputStream stream) throws IOException { writeWithoutMAC(stream); stream.write(mac); } + @Override + public void write(ByteBuffer buffer) { + buffer.put(initializationVector); + Encode.int16(curveType, buffer); + writeCoordinateComponent(buffer, Points.getX(R)); + writeCoordinateComponent(buffer, Points.getY(R)); + buffer.put(encrypted); + buffer.put(mac); + } + public static final class Builder { private byte[] initializationVector; private int curveType; diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/GenericPayload.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/GenericPayload.java index 176d938..9312160 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/GenericPayload.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/GenericPayload.java @@ -21,6 +21,7 @@ import ch.dissem.bitmessage.utils.Decode; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Arrays; /** @@ -62,6 +63,11 @@ public class GenericPayload extends ObjectPayload { stream.write(data); } + @Override + public void write(ByteBuffer buffer) { + buffer.put(data); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/GetPubkey.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/GetPubkey.java index 06e623a..d889489 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/GetPubkey.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/GetPubkey.java @@ -22,6 +22,7 @@ import ch.dissem.bitmessage.utils.Decode; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; /** * Request for a public key. @@ -73,4 +74,9 @@ public class GetPubkey extends ObjectPayload { public void write(OutputStream stream) throws IOException { stream.write(ripeTag); } + + @Override + public void write(ByteBuffer buffer) { + buffer.put(ripeTag); + } } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/Msg.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/Msg.java index 64a010c..dc36bb1 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/Msg.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/Msg.java @@ -24,6 +24,7 @@ import ch.dissem.bitmessage.exception.DecryptionFailedException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Objects; import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG; @@ -111,6 +112,12 @@ public class Msg extends ObjectPayload implements Encrypted, PlaintextHolder { encrypted.write(out); } + @Override + public void write(ByteBuffer buffer) { + if (encrypted == null) throw new IllegalStateException("Msg must be signed and encrypted before writing it."); + encrypted.write(buffer); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/Pubkey.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/Pubkey.java index c476bf9..27d2da9 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/Pubkey.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/Pubkey.java @@ -18,6 +18,7 @@ package ch.dissem.bitmessage.entity.payload; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.ArrayList; import static ch.dissem.bitmessage.utils.Singleton.cryptography; @@ -60,6 +61,10 @@ public abstract class Pubkey extends ObjectPayload { write(out); } + public void writeUnencrypted(ByteBuffer buffer){ + write(buffer); + } + protected byte[] add0x04(byte[] key) { if (key.length == 65) return key; byte[] result = new byte[65]; diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/V2Pubkey.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/V2Pubkey.java index 0eceb91..d2901c1 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/V2Pubkey.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/V2Pubkey.java @@ -22,6 +22,7 @@ import ch.dissem.bitmessage.utils.Encode; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; /** * A version 2 public key. @@ -86,10 +87,17 @@ public class V2Pubkey extends Pubkey { } @Override - public void write(OutputStream os) throws IOException { - Encode.int32(behaviorBitfield, os); - os.write(publicSigningKey, 1, 64); - os.write(publicEncryptionKey, 1, 64); + public void write(OutputStream out) throws IOException { + Encode.int32(behaviorBitfield, out); + out.write(publicSigningKey, 1, 64); + out.write(publicEncryptionKey, 1, 64); + } + + @Override + public void write(ByteBuffer buffer) { + Encode.int32(behaviorBitfield, buffer); + buffer.put(publicSigningKey, 1, 64); + buffer.put(publicEncryptionKey, 1, 64); } public static class Builder { diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Broadcast.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Broadcast.java index 7781455..323da33 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Broadcast.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Broadcast.java @@ -22,6 +22,7 @@ import ch.dissem.bitmessage.entity.Plaintext; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; /** * Users who are subscribed to the sending address will see the message appear in their inbox. @@ -58,4 +59,9 @@ public class V4Broadcast extends Broadcast { public void write(OutputStream out) throws IOException { encrypted.write(out); } + + @Override + public void write(ByteBuffer buffer) { + encrypted.write(buffer); + } } diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Pubkey.java b/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Pubkey.java index 8aa0ee4..179a475 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Pubkey.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/payload/V4Pubkey.java @@ -24,6 +24,7 @@ import ch.dissem.bitmessage.utils.Decode; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Arrays; /** @@ -85,11 +86,22 @@ public class V4Pubkey extends Pubkey implements Encrypted { encrypted.write(stream); } + @Override + public void write(ByteBuffer buffer) { + buffer.put(tag); + encrypted.write(buffer); + } + @Override public void writeUnencrypted(OutputStream out) throws IOException { decrypted.write(out); } + @Override + public void writeUnencrypted(ByteBuffer buffer) { + decrypted.write(buffer); + } + @Override public void writeBytesToSign(OutputStream out) throws IOException { out.write(tag); diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/InventoryVector.java b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/InventoryVector.java index 127b90a..9a3b258 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/InventoryVector.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/InventoryVector.java @@ -22,6 +22,7 @@ import ch.dissem.bitmessage.utils.Strings; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; +import java.nio.ByteBuffer; import java.util.Arrays; public class InventoryVector implements Streamable, Serializable { @@ -56,8 +57,13 @@ public class InventoryVector implements Streamable, Serializable { } @Override - public void write(OutputStream stream) throws IOException { - stream.write(hash); + public void write(OutputStream out) throws IOException { + out.write(hash); + } + + @Override + public void write(ByteBuffer buffer) { + buffer.put(hash); } @Override diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java index d0324a4..0fef152 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.OutputStream; import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.ByteBuffer; import java.util.Arrays; /** @@ -119,14 +120,29 @@ public class NetworkAddress implements Streamable { write(stream, false); } - public void write(OutputStream stream, boolean light) throws IOException { + public void write(OutputStream out, boolean light) throws IOException { if (!light) { - Encode.int64(time, stream); - Encode.int32(this.stream, stream); + Encode.int64(time, out); + Encode.int32(stream, out); } - Encode.int64(services, stream); - stream.write(ipv6); - Encode.int16(port, stream); + Encode.int64(services, out); + out.write(ipv6); + Encode.int16(port, out); + } + + @Override + public void write(ByteBuffer buffer) { + write(buffer, false); + } + + public void write(ByteBuffer buffer, boolean light) { + if (!light) { + Encode.int64(time, buffer); + Encode.int32(stream, buffer); + } + Encode.int64(services, buffer); + buffer.put(ipv6); + Encode.int16(port, buffer); } public static final class Builder { diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/PrivateKey.java b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/PrivateKey.java index 716afc6..7621ca5 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/PrivateKey.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/PrivateKey.java @@ -27,6 +27,7 @@ import ch.dissem.bitmessage.utils.Decode; import ch.dissem.bitmessage.utils.Encode; import java.io.*; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -113,24 +114,20 @@ public class PrivateKey implements Streamable { } Builder generate() { - try { - long signingKeyNonce = nextNonce; - long encryptionKeyNonce = nextNonce + 1; - byte[] ripe; - do { - privEK = Bytes.truncate(cryptography().sha512(seed, Encode.varInt(encryptionKeyNonce)), 32); - privSK = Bytes.truncate(cryptography().sha512(seed, Encode.varInt(signingKeyNonce)), 32); - pubSK = cryptography().createPublicKey(privSK); - pubEK = cryptography().createPublicKey(privEK); - ripe = cryptography().ripemd160(cryptography().sha512(pubSK, pubEK)); + long signingKeyNonce = nextNonce; + long encryptionKeyNonce = nextNonce + 1; + byte[] ripe; + do { + privEK = Bytes.truncate(cryptography().sha512(seed, Encode.varInt(encryptionKeyNonce)), 32); + privSK = Bytes.truncate(cryptography().sha512(seed, Encode.varInt(signingKeyNonce)), 32); + pubSK = cryptography().createPublicKey(privSK); + pubEK = cryptography().createPublicKey(privEK); + ripe = cryptography().ripemd160(cryptography().sha512(pubSK, pubEK)); - signingKeyNonce += 2; - encryptionKeyNonce += 2; - } while (ripe[0] != 0 || (shorter && ripe[1] != 0)); - nextNonce = signingKeyNonce; - } catch (IOException e) { - throw new ApplicationException(e); - } + signingKeyNonce += 2; + encryptionKeyNonce += 2; + } while (ripe[0] != 0 || (shorter && ripe[1] != 0)); + nextNonce = signingKeyNonce; return this; } } @@ -182,4 +179,20 @@ public class PrivateKey implements Streamable { Encode.varInt(privateEncryptionKey.length, out); out.write(privateEncryptionKey); } + + + @Override + public void write(ByteBuffer buffer) { + Encode.varInt(pubkey.getVersion(), buffer); + Encode.varInt(pubkey.getStream(), buffer); + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + pubkey.writeUnencrypted(baos); + Encode.varBytes(baos.toByteArray(), buffer); + } catch (IOException e) { + throw new ApplicationException(e); + } + Encode.varBytes(privateSigningKey, buffer); + Encode.varBytes(privateEncryptionKey, buffer); + } } diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java index 909d3dd..bfbb48c 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java @@ -29,6 +29,8 @@ import java.util.concurrent.Future; * Handles incoming messages */ public interface NetworkHandler { + int NETWORK_MAGIC_NUMBER = 8; + /** * Connects to the trusted host, fetches and offers new messages and disconnects afterwards. *

diff --git a/core/src/main/java/ch/dissem/bitmessage/utils/Encode.java b/core/src/main/java/ch/dissem/bitmessage/utils/Encode.java index f5eac43..a60c027 100644 --- a/core/src/main/java/ch/dissem/bitmessage/utils/Encode.java +++ b/core/src/main/java/ch/dissem/bitmessage/utils/Encode.java @@ -17,10 +17,13 @@ package ch.dissem.bitmessage.utils; import ch.dissem.bitmessage.entity.Streamable; +import ch.dissem.bitmessage.exception.ApplicationException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.Buffer; import java.nio.ByteBuffer; import static ch.dissem.bitmessage.utils.AccessCounter.inc; @@ -37,62 +40,52 @@ public class Encode { } } + public static void varIntList(long[] values, ByteBuffer buffer) { + varInt(values.length, buffer); + for (long value : values) { + varInt(value, buffer); + } + } + public static void varInt(long value, OutputStream stream) throws IOException { varInt(value, stream, null); } - public static byte[] varInt(long value) throws IOException { - final byte[] result; + public static void varInt(long value, ByteBuffer buffer) { if (value < 0) { // This is due to the fact that Java doesn't really support unsigned values. // Please be aware that this might be an error due to a smaller negative value being cast to long. - // Normally, negative values shouldn't occur within the protocol, and I large enough longs - // to being recognized as negatives aren't realistic. - ByteBuffer buffer = ByteBuffer.allocate(9); + // Normally, negative values shouldn't occur within the protocol, and longs large enough for being + // recognized as negatives aren't realistic. buffer.put((byte) 0xff); - result = buffer.putLong(value).array(); + buffer.putLong(value); } else if (value < 0xfd) { - result = new byte[]{(byte) value}; + buffer.put((byte) value); } else if (value <= 0xffffL) { - ByteBuffer buffer = ByteBuffer.allocate(3); buffer.put((byte) 0xfd); - result = buffer.putShort((short) value).array(); + buffer.putShort((short) value); } else if (value <= 0xffffffffL) { - ByteBuffer buffer = ByteBuffer.allocate(5); buffer.put((byte) 0xfe); - result = buffer.putInt((int) value).array(); + buffer.putInt((int) value); } else { - ByteBuffer buffer = ByteBuffer.allocate(9); buffer.put((byte) 0xff); - result = buffer.putLong(value).array(); + buffer.putLong(value); } - return result; + } + + public static byte[] varInt(long value) { + ByteBuffer buffer = ByteBuffer.allocate(9); + varInt(value, buffer); + buffer.flip(); + return Bytes.truncate(buffer.array(), buffer.limit()); } public static void varInt(long value, OutputStream stream, AccessCounter counter) throws IOException { - if (value < 0) { - // This is due to the fact that Java doesn't really support unsigned values. - // Please be aware that this might be an error due to a smaller negative value being cast to long. - // Normally, negative values shouldn't occur within the protocol, and I large enough longs - // to being recognized as negatives aren't realistic. - stream.write(0xff); - inc(counter); - int64(value, stream, counter); - } else if (value < 0xfd) { - int8(value, stream, counter); - } else if (value <= 0xffffL) { - stream.write(0xfd); - inc(counter); - int16(value, stream, counter); - } else if (value <= 0xffffffffL) { - stream.write(0xfe); - inc(counter); - int32(value, stream, counter); - } else { - stream.write(0xff); - inc(counter); - int64(value, stream, counter); - } + ByteBuffer buffer = ByteBuffer.allocate(9); + varInt(value, buffer); + buffer.flip(); + stream.write(buffer.array(), 0, buffer.limit()); + inc(counter, buffer.limit()); } public static void int8(long value, OutputStream stream) throws IOException { @@ -113,6 +106,10 @@ public class Encode { inc(counter, 2); } + public static void int16(long value, ByteBuffer buffer) { + buffer.putShort((short) value); + } + public static void int32(long value, OutputStream stream) throws IOException { int32(value, stream, null); } @@ -122,6 +119,10 @@ public class Encode { inc(counter, 4); } + public static void int32(long value, ByteBuffer buffer) { + buffer.putInt((int) value); + } + public static void int64(long value, OutputStream stream) throws IOException { int64(value, stream, null); } @@ -131,6 +132,10 @@ public class Encode { inc(counter, 8); } + public static void int64(long value, ByteBuffer buffer) { + buffer.putLong(value); + } + public static void varString(String value, OutputStream out) throws IOException { byte[] bytes = value.getBytes("utf-8"); // Technically, it says the length in characters, but I think this one might be correct. @@ -140,23 +145,44 @@ public class Encode { out.write(bytes); } + public static void varString(String value, ByteBuffer buffer) { + try { + byte[] bytes = value.getBytes("utf-8"); + // Technically, it says the length in characters, but I think this one might be correct. + // It doesn't really matter, as only ASCII characters are being used. + // see also Decode#varString() + buffer.put(varInt(bytes.length)); + buffer.put(bytes); + } catch (UnsupportedEncodingException e) { + throw new ApplicationException(e); + } + } + public static void varBytes(byte[] data, OutputStream out) throws IOException { varInt(data.length, out); out.write(data); } + public static void varBytes(byte[] data, ByteBuffer buffer) { + varInt(data.length, buffer); + buffer.put(data); + } + /** * Serializes a {@link Streamable} object and returns the byte array. * * @param streamable the object to be serialized * @return an array of bytes representing the given streamable object. - * @throws IOException if an I/O error occurs. */ - public static byte[] bytes(Streamable streamable) throws IOException { + public static byte[] bytes(Streamable streamable) { if (streamable == null) return null; ByteArrayOutputStream stream = new ByteArrayOutputStream(); - streamable.write(stream); + try { + streamable.write(stream); + } catch (IOException e) { + throw new ApplicationException(e); + } return stream.toByteArray(); } @@ -164,11 +190,14 @@ public class Encode { * @param streamable the object to be serialized * @param padding the result will be padded such that its length is a multiple of padding * @return the bytes of the given {@link Streamable} object, 0-padded such that the final length is x*padding. - * @throws IOException if an I/O error occurs. */ - public static byte[] bytes(Streamable streamable, int padding) throws IOException { + public static byte[] bytes(Streamable streamable, int padding) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); - streamable.write(stream); + try { + streamable.write(stream); + } catch (IOException e) { + throw new ApplicationException(e); + } int offset = padding - stream.size() % padding; int length = stream.size() + offset; byte[] result = new byte[length]; diff --git a/extensions/src/main/java/ch/dissem/bitmessage/extensions/pow/ProofOfWorkRequest.java b/extensions/src/main/java/ch/dissem/bitmessage/extensions/pow/ProofOfWorkRequest.java index d661c50..e5ba4f8 100644 --- a/extensions/src/main/java/ch/dissem/bitmessage/extensions/pow/ProofOfWorkRequest.java +++ b/extensions/src/main/java/ch/dissem/bitmessage/extensions/pow/ProofOfWorkRequest.java @@ -24,6 +24,7 @@ import ch.dissem.bitmessage.utils.Encode; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Arrays; import static ch.dissem.bitmessage.utils.Decode.*; @@ -83,6 +84,13 @@ public class ProofOfWorkRequest implements Streamable { Encode.varBytes(data, out); } + @Override + public void write(ByteBuffer buffer) { + buffer.put(initialHash); + Encode.varString(request.name(), buffer); + Encode.varBytes(data, buffer); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java index 635aed2..2b06373 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java @@ -45,7 +45,6 @@ import static java.util.Collections.newSetFromMap; * Handles all the networky stuff. */ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { - public final static int NETWORK_MAGIC_NUMBER = 8; final Collection connections = new ConcurrentLinkedQueue<>(); private final ExecutorService pool = Executors.newCachedThreadPool( diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java new file mode 100644 index 0000000..3361693 --- /dev/null +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -0,0 +1,51 @@ +/* + * Copyright 2016 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.networking.nio; + +import ch.dissem.bitmessage.entity.MessagePayload; + +import java.nio.ByteBuffer; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedDeque; + +/** + * Created by chrig on 27.05.2016. + */ +public class ConnectionInfo { + private State state; + private final Queue sendingQueue = new ConcurrentLinkedDeque<>(); + private ByteBuffer in = ByteBuffer.allocate(10); + private ByteBuffer out = ByteBuffer.allocate(10); + + public State getState() { + return state; + } + + public Queue getSendingQueue() { + return sendingQueue; + } + + public ByteBuffer getInBuffer() { + return in; + } + + public ByteBuffer getOutBuffer() { + return out; + } + + public enum State {CONNECTING, ACTIVE, DISCONNECTED} +} diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java new file mode 100644 index 0000000..4f56d42 --- /dev/null +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -0,0 +1,147 @@ +/* + * Copyright 2016 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.networking.nio; + +import ch.dissem.bitmessage.InternalContext; +import ch.dissem.bitmessage.entity.CustomMessage; +import ch.dissem.bitmessage.entity.GetData; +import ch.dissem.bitmessage.entity.MessagePayload; +import ch.dissem.bitmessage.entity.NetworkMessage; +import ch.dissem.bitmessage.entity.valueobject.InventoryVector; +import ch.dissem.bitmessage.exception.ApplicationException; +import ch.dissem.bitmessage.ports.NetworkHandler; +import ch.dissem.bitmessage.utils.Property; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.Future; + +import static java.nio.channels.SelectionKey.*; + +/** + * Network handler using java.nio, resulting in less threads. + */ +public class NioNetworkHandler implements NetworkHandler, InternalContext.ContextHolder { + private static final Logger LOG = LoggerFactory.getLogger(NioNetworkHandler.class); + + private InternalContext ctx; + private Selector selector; + + @Override + public Future synchronize(InetAddress server, int port, MessageListener listener, long timeoutInSeconds) { + return null; + } + + @Override + public CustomMessage send(InetAddress server, int port, CustomMessage request) { + return null; + } + + @Override + public void start(MessageListener listener) { + if (listener == null) { + throw new IllegalStateException("Listener must be set at start"); + } + if (selector != null && selector.isOpen()) { + throw new IllegalStateException("Network already running - you need to stop first."); + } + try { + final Set requestedObjects = new HashSet<>(); + selector = Selector.open(); + { + ServerSocketChannel server = ServerSocketChannel.open(); + server.configureBlocking(false); + server.bind(new InetSocketAddress(ctx.getPort())); + server.register(selector, OP_ACCEPT); + } + while (selector.isOpen()) { + // TODO: establish outgoing connections + selector.select(); + Iterator keyIterator = selector.selectedKeys().iterator(); + + while (keyIterator.hasNext()) { + SelectionKey key = keyIterator.next(); + if (key.isAcceptable()) { + SocketChannel accepted = ((ServerSocketChannel) key.channel()).accept(); + accepted.configureBlocking(false); + accepted.register(selector, OP_READ | OP_WRITE).attach(new ConnectionInfo()); + } + if (key.attachment() instanceof ConnectionInfo) { + SocketChannel channel = (SocketChannel) key.channel(); + ConnectionInfo connection = (ConnectionInfo) key.attachment(); + + if (key.isWritable()) { + if (connection.getOutBuffer().hasRemaining()) { + channel.write(connection.getOutBuffer()); + } + while (!connection.getOutBuffer().hasRemaining() && !connection.getSendingQueue().isEmpty()) { + MessagePayload payload = connection.getSendingQueue().poll(); + if (payload instanceof GetData) { + requestedObjects.addAll(((GetData) payload).getInventory()); + } + new NetworkMessage(payload).write(connection.getOutBuffer()); + } + } + if (key.isReadable()) { + // TODO + channel.read(connection.getInBuffer()); + } + } + keyIterator.remove(); + } + } + selector.close(); + } catch (IOException e) { + throw new ApplicationException(e); + } + } + + @Override + public void stop() { + + } + + @Override + public void offer(InventoryVector iv) { + + } + + @Override + public Property getNetworkStatus() { + return null; + } + + @Override + public boolean isRunning() { + return false; + } + + @Override + public void setContext(InternalContext context) { + this.ctx = context; + } +} From cde4f7b3ce50e5649b6b60e4fdd644d8bd43a281 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Wed, 1 Jun 2016 17:38:49 +0200 Subject: [PATCH 02/31] Some refactoring to move some common code into an AbstractConnection --- .../entity/valueobject/NetworkAddress.java | 13 + .../bitmessage/factory/V3MessageFactory.java | 2 +- .../bitmessage/factory/V3MessageReader.java | 143 ++++++++ .../ports/AbstractCryptography.java | 6 + .../dissem/bitmessage/ports/Cryptography.java | 12 + .../bitmessage/ports/NetworkHandler.java | 10 + .../ch/dissem/bitmessage/utils/Decode.java | 4 + .../networking/AbstractConnection.java | 318 ++++++++++++++++++ .../bitmessage/networking/Connection.java | 281 ++-------------- .../networking/ConnectionOrganizer.java | 8 +- .../networking/DefaultNetworkHandler.java | 11 +- .../bitmessage/networking/ServerRunnable.java | 8 +- .../networking/nio/ConnectionInfo.java | 53 ++- .../networking/nio/NioNetworkHandler.java | 210 +++++++++--- .../networking/NetworkHandlerTest.java | 9 +- 15 files changed, 748 insertions(+), 340 deletions(-) create mode 100644 core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java create mode 100644 networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java index 0fef152..a496d19 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java @@ -24,6 +24,8 @@ import ch.dissem.bitmessage.utils.UnixTime; import java.io.IOException; import java.io.OutputStream; import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -215,6 +217,17 @@ public class NetworkAddress implements Streamable { return this; } + public Builder address(SocketAddress address) { + if (address instanceof InetSocketAddress) { + InetSocketAddress inetAddress = (InetSocketAddress) address; + ip(inetAddress.getAddress()); + port(inetAddress.getPort()); + } else { + throw new IllegalArgumentException("Unknown type of address: " + address.getClass()); + } + return this; + } + public NetworkAddress build() { if (time == 0) { time = UnixTime.now(); diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java index af9839f..478d77c 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java @@ -62,7 +62,7 @@ class V3MessageFactory { } } - private static MessagePayload getPayload(String command, InputStream stream, int length) throws IOException { + static MessagePayload getPayload(String command, InputStream stream, int length) throws IOException { switch (command) { case "version": return parseVersion(stream); diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java new file mode 100644 index 0000000..7006cdb --- /dev/null +++ b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java @@ -0,0 +1,143 @@ +/* + * Copyright 2016 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.factory; + +import ch.dissem.bitmessage.entity.MessagePayload; +import ch.dissem.bitmessage.entity.NetworkMessage; +import ch.dissem.bitmessage.exception.ApplicationException; +import ch.dissem.bitmessage.exception.NodeException; +import ch.dissem.bitmessage.utils.Decode; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.LinkedList; +import java.util.List; + +import static ch.dissem.bitmessage.entity.NetworkMessage.MAGIC_BYTES; +import static ch.dissem.bitmessage.ports.NetworkHandler.MAX_PAYLOAD_SIZE; +import static ch.dissem.bitmessage.utils.Singleton.cryptography; + +/** + * Similar to the {@link V3MessageFactory}, but used for NIO buffers which may or may not contain a whole message. + */ +public class V3MessageReader { + private ReaderState state = ReaderState.MAGIC; + private String command; + private int length; + private byte[] checksum; + + private List messages = new LinkedList<>(); + + public void update(ByteBuffer buffer) { + while (buffer.hasRemaining()) { + switch (state) { + case MAGIC: + if (!findMagicBytes(buffer)) return; + state = ReaderState.HEADER; + case HEADER: + if (buffer.remaining() < 20) { + buffer.compact(); + return; + } + command = getCommand(buffer); + length = (int) Decode.uint32(buffer); + if (length > MAX_PAYLOAD_SIZE) { + throw new NodeException("Payload of " + length + " bytes received, no more than 1600003 was expected."); + } + checksum = new byte[4]; + buffer.get(checksum); + state = ReaderState.DATA; + if (buffer.remaining() < length) { + // We need to compact the buffer to make sure the message fits even if it's really big. + buffer.compact(); + } + case DATA: + if (buffer.remaining() < length) return; + if (!testChecksum(buffer)) { + throw new NodeException("Checksum failed for message '" + command + "'"); + } + try { + MessagePayload payload = V3MessageFactory.getPayload( + command, + new ByteArrayInputStream(buffer.array(), buffer.arrayOffset() + buffer.position(), length), + length); + if (payload != null) { + messages.add(new NetworkMessage(payload)); + } + } catch (IOException e) { + throw new NodeException(e.getMessage()); + } + state = ReaderState.MAGIC; + } + } + } + + public List getMessages() { + return messages; + } + + private boolean findMagicBytes(ByteBuffer buffer) { + int i = 0; + while (buffer.hasRemaining()) { + if (buffer.get() == MAGIC_BYTES[i]) { + buffer.mark(); + i++; + if (i == MAGIC_BYTES.length) return true; + } else { + i = 0; + } + } + if (i > 0) { + buffer.reset(); + buffer.compact(); + } else { + buffer.clear(); + } + return false; + } + + private static String getCommand(ByteBuffer buffer) { + int start = buffer.position(); + int i = 0; + while (i < 12 && buffer.get() != 0) i++; + int end = start + i; + while (i < 12) { + if (buffer.get() != 0) throw new NodeException("'\\0' padding expected for command"); + i++; + } + try { + return new String(buffer.array(), start, end, "ASCII"); + } catch (UnsupportedEncodingException e) { + throw new ApplicationException(e); + } + } + + private boolean testChecksum(ByteBuffer buffer) { + byte[] payloadChecksum = cryptography().sha512(buffer.array(), + buffer.arrayOffset() + buffer.position(), length); + for (int i = 0; i < checksum.length; i++) { + if (checksum[i] != payloadChecksum[i]) { + return false; + } + } + return true; + } + + private enum ReaderState {MAGIC, HEADER, DATA} +} diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java b/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java index 3b08377..f67b6d4 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java @@ -61,6 +61,12 @@ public abstract class AbstractCryptography implements Cryptography, InternalCont this.context = context; } + public byte[] sha512(byte[] data, int offset, int length) { + MessageDigest mda = md("SHA-512"); + mda.update(data, offset, length); + return mda.digest(); + } + public byte[] sha512(byte[]... data) { return hash("SHA-512", data); } diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/Cryptography.java b/core/src/main/java/ch/dissem/bitmessage/ports/Cryptography.java index 48739ea..9ea6a9d 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/Cryptography.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/Cryptography.java @@ -30,6 +30,18 @@ import java.security.SecureRandom; * which should be secure enough. */ public interface Cryptography { + /** + * A helper method to calculate SHA-512 hashes. Please note that a new {@link MessageDigest} object is created at + * each call (to ensure thread safety), so you shouldn't use this if you need to do many hash calculations in + * success on the same thread. + * + * @param data to get hashed + * @param offset of the data to be hashed + * @param length of the data to be hashed + * @return SHA-512 hash of data within the given range + */ + byte[] sha512(byte[] data, int offset, int length); + /** * A helper method to calculate SHA-512 hashes. Please note that a new {@link MessageDigest} object is created at * each call (to ensure thread safety), so you shouldn't use this if you need to do many hash calculations in diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java index bfbb48c..96dec8e 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java @@ -23,6 +23,7 @@ import ch.dissem.bitmessage.utils.Property; import java.io.IOException; import java.net.InetAddress; +import java.util.Collection; import java.util.concurrent.Future; /** @@ -30,6 +31,8 @@ import java.util.concurrent.Future; */ public interface NetworkHandler { int NETWORK_MAGIC_NUMBER = 8; + int MAX_PAYLOAD_SIZE = 1600003; + int MAX_MESSAGE_SIZE = 24 + MAX_PAYLOAD_SIZE; /** * Connects to the trusted host, fetches and offers new messages and disconnects afterwards. @@ -65,6 +68,13 @@ public interface NetworkHandler { */ void offer(InventoryVector iv); + /** + * Request each of those objects from a node that knows of the requested object. + * + * @param inventoryVectors of the objects to be requested + */ + void request(Collection inventoryVectors); + Property getNetworkStatus(); boolean isRunning(); diff --git a/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java b/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java index 47b0ee3..fb2f3c8 100644 --- a/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java +++ b/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java @@ -111,6 +111,10 @@ public class Decode { return stream.read() * 16777216L + stream.read() * 65536L + stream.read() * 256L + stream.read(); } + public static long uint32(ByteBuffer buffer) { + return buffer.get() * 16777216L + buffer.get() * 65536L + buffer.get() * 256L + buffer.get(); + } + public static int int32(InputStream stream) throws IOException { return int32(stream, null); } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java new file mode 100644 index 0000000..3027f04 --- /dev/null +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -0,0 +1,318 @@ +/* + * Copyright 2016 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.networking; + +import ch.dissem.bitmessage.BitmessageContext; +import ch.dissem.bitmessage.InternalContext; +import ch.dissem.bitmessage.entity.*; +import ch.dissem.bitmessage.entity.valueobject.InventoryVector; +import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; +import ch.dissem.bitmessage.exception.InsufficientProofOfWorkException; +import ch.dissem.bitmessage.exception.NodeException; +import ch.dissem.bitmessage.ports.NetworkHandler; +import ch.dissem.bitmessage.utils.UnixTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; + +import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES; +import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; +import static ch.dissem.bitmessage.networking.AbstractConnection.State.*; +import static ch.dissem.bitmessage.utils.Singleton.cryptography; +import static ch.dissem.bitmessage.utils.UnixTime.MINUTE; + +/** + * Contains everything used by both the old streams-oriented NetworkHandler and the new NioNetworkHandler, + * respectively their connection objects. + */ +public abstract class AbstractConnection { + private static final Logger LOG = LoggerFactory.getLogger(AbstractConnection.class); + protected final InternalContext ctx; + protected final Mode mode; + protected final NetworkAddress host; + protected final NetworkAddress node; + protected final NetworkHandler.MessageListener listener; + protected final Map ivCache; + protected final Deque sendingQueue; + protected final Set commonRequestedObjects; + protected final Set requestedObjects; + + protected volatile State state; + protected long lastObjectTime; + + protected long peerNonce; + protected int version; + protected long[] streams; + + public AbstractConnection(InternalContext context, Mode mode, + NetworkAddress node, + NetworkHandler.MessageListener listener, + Set commonRequestedObjects, + boolean threadsafe) { + this.ctx = context; + this.mode = mode; + this.host = new NetworkAddress.Builder().ipv6(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).port(0).build(); + this.node = node; + this.listener = listener; + if (threadsafe) { + this.ivCache = new ConcurrentHashMap<>(); + this.sendingQueue = new ConcurrentLinkedDeque<>(); + this.requestedObjects = Collections.newSetFromMap(new ConcurrentHashMap(10_000)); + } else { + this.ivCache = new HashMap<>(); + this.sendingQueue = new LinkedList<>(); + this.requestedObjects = new HashSet<>(); + } + this.state = CONNECTING; + this.commonRequestedObjects = commonRequestedObjects; + } + + public Mode getMode() { + return mode; + } + + public NetworkAddress getNode() { + return node; + } + + public State getState() { + return state; + } + + protected void handleMessage(MessagePayload payload) { + switch (state) { + case ACTIVE: + receiveMessage(payload); + break; + + default: + handleCommand(payload); + break; + } + } + + private void receiveMessage(MessagePayload messagePayload) { + switch (messagePayload.getCommand()) { + case INV: + receiveMessage((Inv) messagePayload); + break; + case GETDATA: + receiveMessage((GetData) messagePayload); + break; + case OBJECT: + receiveMessage((ObjectMessage) messagePayload); + break; + case ADDR: + receiveMessage((Addr) messagePayload); + break; + case CUSTOM: + case VERACK: + case VERSION: + default: + throw new IllegalStateException("Unexpectedly received '" + messagePayload.getCommand() + "' command"); + } + } + + private void receiveMessage(Inv inv) { + int originalSize = inv.getInventory().size(); + updateIvCache(inv.getInventory()); + List missing = ctx.getInventory().getMissing(inv.getInventory(), streams); + missing.removeAll(commonRequestedObjects); + LOG.debug("Received inventory with " + originalSize + " elements, of which are " + + missing.size() + " missing."); + send(new GetData.Builder().inventory(missing).build()); + } + + private void receiveMessage(GetData getData) { + for (InventoryVector iv : getData.getInventory()) { + ObjectMessage om = ctx.getInventory().getObject(iv); + if (om != null) sendingQueue.offer(om); + } + } + + private void receiveMessage(ObjectMessage objectMessage) { + requestedObjects.remove(objectMessage.getInventoryVector()); + if (ctx.getInventory().contains(objectMessage)) { + LOG.trace("Received object " + objectMessage.getInventoryVector() + " - already in inventory"); + return; + } + try { + listener.receive(objectMessage); + cryptography().checkProofOfWork(objectMessage, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES); + ctx.getInventory().storeObject(objectMessage); + // offer object to some random nodes so it gets distributed throughout the network: + ctx.getNetworkHandler().offer(objectMessage.getInventoryVector()); + lastObjectTime = UnixTime.now(); + } catch (InsufficientProofOfWorkException e) { + LOG.warn(e.getMessage()); + // DebugUtils.saveToFile(objectMessage); // this line must not be committed active + } catch (IOException e) { + LOG.error("Stream " + objectMessage.getStream() + ", object type " + objectMessage.getType() + ": " + e.getMessage(), e); + } finally { + if (commonRequestedObjects.remove(objectMessage.getInventoryVector())) { + LOG.debug("Received object that wasn't requested."); + } + } + } + + private void receiveMessage(Addr addr) { + LOG.debug("Received " + addr.getAddresses().size() + " addresses."); + ctx.getNodeRegistry().offerAddresses(addr.getAddresses()); + } + + private void updateIvCache(List inventory) { + cleanupIvCache(); + Long now = UnixTime.now(); + for (InventoryVector iv : inventory) { + ivCache.put(iv, now); + } + } + + public void offer(InventoryVector iv) { + sendingQueue.offer(new Inv.Builder() + .addInventoryVector(iv) + .build()); + updateIvCache(Collections.singletonList(iv)); + } + + public boolean knowsOf(InventoryVector iv) { + return ivCache.containsKey(iv); + } + + protected void cleanupIvCache() { + Long fiveMinutesAgo = UnixTime.now(-5 * MINUTE); + for (Map.Entry entry : ivCache.entrySet()) { + if (entry.getValue() < fiveMinutesAgo) { + ivCache.remove(entry.getKey()); + } + } + } + + private void handleCommand(MessagePayload payload) { + switch (payload.getCommand()) { + case VERSION: + handleVersion((Version) payload); + break; + case VERACK: + switch (mode) { + case SERVER: + activateConnection(); + break; + case CLIENT: + case SYNC: + default: + // NO OP + break; + } + break; + case CUSTOM: + MessagePayload response = ctx.getCustomCommandHandler().handle((CustomMessage) payload); + if (response != null) { + send(response); + } + disconnect(); + break; + default: + throw new NodeException("Command 'version' or 'verack' expected, but was '" + + payload.getCommand() + "'"); + } + } + + protected void activateConnection() { + LOG.info("Successfully established connection with node " + node); + state = ACTIVE; + node.setTime(UnixTime.now()); + if (mode != SYNC) { + sendAddresses(); + ctx.getNodeRegistry().offerAddresses(Collections.singletonList(node)); + } + sendInventory(); + } + + private void sendAddresses() { + List addresses = ctx.getNodeRegistry().getKnownAddresses(1000, streams); + sendingQueue.offer(new Addr.Builder().addresses(addresses).build()); + } + + private void sendInventory() { + List inventory = ctx.getInventory().getInventory(streams); + for (int i = 0; i < inventory.size(); i += 50000) { + sendingQueue.offer(new Inv.Builder() + .inventory(inventory.subList(i, Math.min(inventory.size(), i + 50000))) + .build()); + } + } + + private void handleVersion(Version version) { + if (version.getNonce() == ctx.getClientNonce()) { + LOG.info("Tried to connect to self, disconnecting."); + disconnect(); + } else if (version.getVersion() >= BitmessageContext.CURRENT_VERSION) { + this.peerNonce = version.getNonce(); + if (peerNonce == ctx.getClientNonce()) disconnect(); + + this.version = version.getVersion(); + this.streams = version.getStreams(); + send(new VerAck()); + switch (mode) { + case SERVER: + send(new Version.Builder().defaults(ctx.getClientNonce()).addrFrom(host).addrRecv(node).build()); + break; + case CLIENT: + case SYNC: + activateConnection(); + break; + default: + // NO OP + } + } else { + LOG.info("Received unsupported version " + version.getVersion() + ", disconnecting."); + disconnect(); + } + } + + public void disconnect() { + state = DISCONNECTED; + + // Make sure objects that are still missing are requested from other nodes + ctx.getNetworkHandler().request(requestedObjects); + } + + protected abstract void send(MessagePayload payload); + + public enum Mode {SERVER, CLIENT, SYNC} + + public enum State {CONNECTING, ACTIVE, DISCONNECTED} + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AbstractConnection that = (AbstractConnection) o; + return Objects.equals(node, that.node); + } + + @Override + public int hashCode() { + return Objects.hash(node); + } +} diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java index 9ed19bc..3fe1b62 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java @@ -16,13 +16,13 @@ package ch.dissem.bitmessage.networking; -import ch.dissem.bitmessage.BitmessageContext; import ch.dissem.bitmessage.InternalContext; -import ch.dissem.bitmessage.entity.*; +import ch.dissem.bitmessage.entity.GetData; +import ch.dissem.bitmessage.entity.MessagePayload; +import ch.dissem.bitmessage.entity.NetworkMessage; +import ch.dissem.bitmessage.entity.Version; import ch.dissem.bitmessage.entity.valueobject.InventoryVector; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; -import ch.dissem.bitmessage.exception.InsufficientProofOfWorkException; -import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.factory.Factory; import ch.dissem.bitmessage.ports.NetworkHandler.MessageListener; import ch.dissem.bitmessage.utils.UnixTime; @@ -36,94 +36,62 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketTimeoutException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ConcurrentMap; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; -import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES; -import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE; -import static ch.dissem.bitmessage.networking.Connection.Mode.CLIENT; -import static ch.dissem.bitmessage.networking.Connection.Mode.SYNC; -import static ch.dissem.bitmessage.networking.Connection.State.*; -import static ch.dissem.bitmessage.utils.Singleton.cryptography; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; +import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE; +import static ch.dissem.bitmessage.networking.AbstractConnection.State.DISCONNECTED; import static ch.dissem.bitmessage.utils.UnixTime.MINUTE; /** * A connection to a specific node */ -class Connection { +class Connection extends AbstractConnection { public static final int READ_TIMEOUT = 2000; private static final Logger LOG = LoggerFactory.getLogger(Connection.class); private static final int CONNECT_TIMEOUT = 5000; private final long startTime; - private final ConcurrentMap ivCache; - private final InternalContext ctx; - private final Mode mode; private final Socket socket; - private final MessageListener listener; - private final NetworkAddress host; - private final NetworkAddress node; - private final Queue sendingQueue = new ConcurrentLinkedDeque<>(); - private final Set commonRequestedObjects; - private final Set requestedObjects; private final long syncTimeout; private final ReaderRunnable reader = new ReaderRunnable(); private final WriterRunnable writer = new WriterRunnable(); - private final DefaultNetworkHandler networkHandler; - private final long clientNonce; - private volatile State state; private InputStream in; private OutputStream out; - private int version; - private long[] streams; private int readTimeoutCounter; private boolean socketInitialized; - private long lastObjectTime; public Connection(InternalContext context, Mode mode, Socket socket, MessageListener listener, - Set requestedObjectsMap, long clientNonce) throws IOException { + Set requestedObjectsMap) throws IOException { this(context, mode, listener, socket, requestedObjectsMap, - Collections.newSetFromMap(new ConcurrentHashMap(10_000)), new NetworkAddress.Builder().ip(socket.getInetAddress()).port(socket.getPort()).stream(1).build(), - 0, clientNonce); + 0); } public Connection(InternalContext context, Mode mode, NetworkAddress node, MessageListener listener, - Set requestedObjectsMap, long clientNonce) { + Set requestedObjectsMap) { this(context, mode, listener, new Socket(), requestedObjectsMap, - Collections.newSetFromMap(new ConcurrentHashMap(10_000)), - node, 0, clientNonce); + node, 0); } private Connection(InternalContext context, Mode mode, MessageListener listener, Socket socket, - Set commonRequestedObjects, Set requestedObjects, - NetworkAddress node, long syncTimeout, long clientNonce) { + Set commonRequestedObjects, NetworkAddress node, long syncTimeout) { + super(context, mode, node, listener, commonRequestedObjects, true); this.startTime = UnixTime.now(); - this.ctx = context; - this.mode = mode; - this.state = CONNECTING; - this.listener = listener; this.socket = socket; - this.commonRequestedObjects = commonRequestedObjects; - this.requestedObjects = requestedObjects; - this.host = new NetworkAddress.Builder().ipv6(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).port(0).build(); - this.node = node; this.syncTimeout = (syncTimeout > 0 ? UnixTime.now(+syncTimeout) : 0); - this.ivCache = new ConcurrentHashMap<>(); - this.networkHandler = (DefaultNetworkHandler) ctx.getNetworkHandler(); - this.clientNonce = clientNonce; } public static Connection sync(InternalContext ctx, InetAddress address, int port, MessageListener listener, long timeoutInSeconds) throws IOException { return new Connection(ctx, SYNC, listener, new Socket(address, port), - new HashSet(), new HashSet(), new NetworkAddress.Builder().ip(address).port(port).stream(1).build(), - timeoutInSeconds, cryptography().randomNonce()); + timeoutInSeconds); } public long getStartTime() { @@ -169,133 +137,8 @@ class Connection { } } - private void activateConnection() { - LOG.info("Successfully established connection with node " + node); - state = ACTIVE; - if (mode != SYNC) { - sendAddresses(); - ctx.getNodeRegistry().offerAddresses(Collections.singletonList(node)); - } - sendInventory(); - node.setTime(UnixTime.now()); - } - - private void cleanupIvCache() { - Long fiveMinutesAgo = UnixTime.now(-5 * MINUTE); - for (Map.Entry entry : ivCache.entrySet()) { - if (entry.getValue() < fiveMinutesAgo) { - ivCache.remove(entry.getKey()); - } - } - } - - private void updateIvCache(InventoryVector... inventory) { - cleanupIvCache(); - Long now = UnixTime.now(); - for (InventoryVector iv : inventory) { - ivCache.put(iv, now); - } - } - - private void updateIvCache(List inventory) { - cleanupIvCache(); - Long now = UnixTime.now(); - for (InventoryVector iv : inventory) { - ivCache.put(iv, now); - } - } - - private void receiveMessage(MessagePayload messagePayload) { - switch (messagePayload.getCommand()) { - case INV: - receiveMessage((Inv) messagePayload); - break; - case GETDATA: - receiveMessage((GetData) messagePayload); - break; - case OBJECT: - receiveMessage((ObjectMessage) messagePayload); - break; - case ADDR: - receiveMessage((Addr) messagePayload); - break; - case CUSTOM: - case VERACK: - case VERSION: - default: - throw new IllegalStateException("Unexpectedly received '" + messagePayload.getCommand() + "' command"); - } - } - - private void receiveMessage(Inv inv) { - int originalSize = inv.getInventory().size(); - updateIvCache(inv.getInventory()); - List missing = ctx.getInventory().getMissing(inv.getInventory(), streams); - missing.removeAll(commonRequestedObjects); - LOG.debug("Received inventory with " + originalSize + " elements, of which are " - + missing.size() + " missing."); - send(new GetData.Builder().inventory(missing).build()); - } - - private void receiveMessage(GetData getData) { - for (InventoryVector iv : getData.getInventory()) { - ObjectMessage om = ctx.getInventory().getObject(iv); - if (om != null) sendingQueue.offer(om); - } - } - - private void receiveMessage(ObjectMessage objectMessage) { - requestedObjects.remove(objectMessage.getInventoryVector()); - if (ctx.getInventory().contains(objectMessage)) { - LOG.trace("Received object " + objectMessage.getInventoryVector() + " - already in inventory"); - return; - } - try { - listener.receive(objectMessage); - cryptography().checkProofOfWork(objectMessage, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES); - ctx.getInventory().storeObject(objectMessage); - // offer object to some random nodes so it gets distributed throughout the network: - networkHandler.offer(objectMessage.getInventoryVector()); - lastObjectTime = UnixTime.now(); - } catch (InsufficientProofOfWorkException e) { - LOG.warn(e.getMessage()); - // DebugUtils.saveToFile(objectMessage); // this line must not be committed active - } catch (IOException e) { - LOG.error("Stream " + objectMessage.getStream() + ", object type " + objectMessage.getType() + ": " + e.getMessage(), e); - } finally { - if (commonRequestedObjects.remove(objectMessage.getInventoryVector())) { - LOG.debug("Received object that wasn't requested."); - } - } - } - - private void receiveMessage(Addr addr) { - LOG.debug("Received " + addr.getAddresses().size() + " addresses."); - ctx.getNodeRegistry().offerAddresses(addr.getAddresses()); - } - - private void sendAddresses() { - List addresses = ctx.getNodeRegistry().getKnownAddresses(1000, streams); - sendingQueue.offer(new Addr.Builder().addresses(addresses).build()); - } - - private void sendInventory() { - List inventory = ctx.getInventory().getInventory(streams); - for (int i = 0; i < inventory.size(); i += 50000) { - sendingQueue.offer(new Inv.Builder() - .inventory(inventory.subList(i, Math.min(inventory.size(), i + 50000))) - .build()); - } - } - - public void disconnect() { - state = DISCONNECTED; - - // Make sure objects that are still missing are requested from other nodes - networkHandler.request(requestedObjects); - } - - void send(MessagePayload payload) { + @Override + protected void send(MessagePayload payload) { try { if (payload instanceof GetData) { requestedObjects.addAll(((GetData) payload).getInventory()); @@ -309,17 +152,6 @@ class Connection { } } - public void offer(InventoryVector iv) { - sendingQueue.offer(new Inv.Builder() - .addInventoryVector(iv) - .build()); - updateIvCache(iv); - } - - public boolean knowsOf(InventoryVector iv) { - return ivCache.containsKey(iv); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -354,18 +186,13 @@ class Connection { return writer; } - public enum Mode {SERVER, CLIENT, SYNC} - - public enum State {CONNECTING, ACTIVE, DISCONNECTED} - public class ReaderRunnable implements Runnable { @Override public void run() { - lastObjectTime = 0; try (Socket socket = Connection.this.socket) { initSocket(socket); if (mode == CLIENT || mode == SYNC) { - send(new Version.Builder().defaults(clientNonce).addrFrom(host).addrRecv(node).build()); + send(new Version.Builder().defaults(peerNonce).addrFrom(host).addrRecv(node).build()); } while (state != DISCONNECTED) { if (mode != SYNC) { @@ -394,75 +221,13 @@ class Connection { NetworkMessage msg = Factory.getNetworkMessage(version, in); if (msg == null) return; - switch (state) { - case ACTIVE: - receiveMessage(msg.getPayload()); - break; - - default: - handleCommand(msg.getPayload()); - break; - } + handleMessage(msg.getPayload()); if (socket.isClosed() || syncFinished(msg) || checkOpenRequests()) disconnect(); } catch (SocketTimeoutException ignore) { if (state == ACTIVE && syncFinished(null)) disconnect(); } } - private void handleCommand(MessagePayload payload) { - switch (payload.getCommand()) { - case VERSION: - handleVersion((Version) payload); - break; - case VERACK: - switch (mode) { - case SERVER: - activateConnection(); - break; - case CLIENT: - case SYNC: - default: - // NO OP - break; - } - break; - case CUSTOM: - MessagePayload response = ctx.getCustomCommandHandler().handle((CustomMessage) payload); - if (response != null) { - send(response); - } - disconnect(); - break; - default: - throw new NodeException("Command 'version' or 'verack' expected, but was '" - + payload.getCommand() + "'"); - } - } - - private void handleVersion(Version version) { - if (version.getNonce() == ctx.getClientNonce()) { - LOG.info("Tried to connect to self, disconnecting."); - disconnect(); - } else if (version.getVersion() >= BitmessageContext.CURRENT_VERSION) { - Connection.this.version = version.getVersion(); - streams = version.getStreams(); - send(new VerAck()); - switch (mode) { - case SERVER: - send(new Version.Builder().defaults(clientNonce).addrFrom(host).addrRecv(node).build()); - break; - case CLIENT: - case SYNC: - activateConnection(); - break; - default: - // NO OP - } - } else { - LOG.info("Received unsupported version " + version.getVersion() + ", disconnecting."); - disconnect(); - } - } } private boolean checkOpenRequests() { diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java b/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java index c25a31c..5c976e2 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java @@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory; import java.util.Iterator; import java.util.List; -import static ch.dissem.bitmessage.networking.Connection.Mode.CLIENT; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT; import static ch.dissem.bitmessage.networking.DefaultNetworkHandler.NETWORK_MAGIC_NUMBER; /** @@ -38,17 +38,15 @@ public class ConnectionOrganizer implements Runnable { private final InternalContext ctx; private final DefaultNetworkHandler networkHandler; private final NetworkHandler.MessageListener listener; - private final long clientNonce; private Connection initialConnection; public ConnectionOrganizer(InternalContext ctx, DefaultNetworkHandler networkHandler, - NetworkHandler.MessageListener listener, long clientNonce) { + NetworkHandler.MessageListener listener) { this.ctx = ctx; this.networkHandler = networkHandler; this.listener = listener; - this.clientNonce = clientNonce; } @Override @@ -94,7 +92,7 @@ public class ConnectionOrganizer implements Runnable { boolean first = active == 0 && initialConnection == null; for (NetworkAddress address : addresses) { Connection c = new Connection(ctx, CLIENT, address, listener, - networkHandler.requestedObjects, clientNonce); + networkHandler.requestedObjects); if (first) { initialConnection = c; first = false; diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java index 2b06373..c4a4554 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java @@ -35,8 +35,8 @@ import java.net.Socket; import java.util.*; import java.util.concurrent.*; -import static ch.dissem.bitmessage.networking.Connection.Mode.SERVER; -import static ch.dissem.bitmessage.networking.Connection.State.ACTIVE; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER; +import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE; import static ch.dissem.bitmessage.utils.DebugUtils.inc; import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; import static java.util.Collections.newSetFromMap; @@ -107,9 +107,9 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { try { running = true; connections.clear(); - server = new ServerRunnable(ctx, this, listener, ctx.getClientNonce()); + server = new ServerRunnable(ctx, this, listener); pool.execute(server); - pool.execute(new ConnectionOrganizer(ctx, this, listener, ctx.getClientNonce())); + pool.execute(new ConnectionOrganizer(ctx, this, listener)); } catch (IOException e) { throw new ApplicationException(e); } @@ -198,7 +198,8 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { ); } - void request(Set inventoryVectors) { + @Override + public void request(Collection inventoryVectors) { if (!running || inventoryVectors.isEmpty()) return; Map> distribution = new HashMap<>(); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java b/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java index 5a866b9..99b6a88 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java @@ -26,7 +26,7 @@ import java.io.IOException; import java.net.ServerSocket; import java.net.Socket; -import static ch.dissem.bitmessage.networking.Connection.Mode.SERVER; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER; /** * @author Christian Basler @@ -37,15 +37,13 @@ public class ServerRunnable implements Runnable, Closeable { private final ServerSocket serverSocket; private final DefaultNetworkHandler networkHandler; private final NetworkHandler.MessageListener listener; - private final long clientNonce; public ServerRunnable(InternalContext ctx, DefaultNetworkHandler networkHandler, - NetworkHandler.MessageListener listener, long clientNonce) throws IOException { + NetworkHandler.MessageListener listener) throws IOException { this.ctx = ctx; this.networkHandler = networkHandler; this.listener = listener; this.serverSocket = new ServerSocket(ctx.getPort()); - this.clientNonce = clientNonce; } @Override @@ -55,7 +53,7 @@ public class ServerRunnable implements Runnable, Closeable { Socket socket = serverSocket.accept(); socket.setSoTimeout(Connection.READ_TIMEOUT); networkHandler.startConnection(new Connection(ctx, SERVER, socket, listener, - networkHandler.requestedObjects, clientNonce)); + networkHandler.requestedObjects)); } catch (IOException e) { LOG.debug(e.getMessage(), e); } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 3361693..223722a 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -16,25 +16,43 @@ package ch.dissem.bitmessage.networking.nio; +import ch.dissem.bitmessage.InternalContext; import ch.dissem.bitmessage.entity.MessagePayload; +import ch.dissem.bitmessage.entity.NetworkMessage; +import ch.dissem.bitmessage.entity.valueobject.InventoryVector; +import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; +import ch.dissem.bitmessage.factory.V3MessageReader; +import ch.dissem.bitmessage.networking.AbstractConnection; +import ch.dissem.bitmessage.ports.NetworkHandler; import java.nio.ByteBuffer; -import java.util.Queue; +import java.util.*; import java.util.concurrent.ConcurrentLinkedDeque; +import static ch.dissem.bitmessage.ports.NetworkHandler.MAX_MESSAGE_SIZE; + /** - * Created by chrig on 27.05.2016. + * Represents the current state of a connection. */ -public class ConnectionInfo { - private State state; - private final Queue sendingQueue = new ConcurrentLinkedDeque<>(); - private ByteBuffer in = ByteBuffer.allocate(10); - private ByteBuffer out = ByteBuffer.allocate(10); +public class ConnectionInfo extends AbstractConnection { + private ByteBuffer in = ByteBuffer.allocate(MAX_MESSAGE_SIZE); + private ByteBuffer out = ByteBuffer.allocate(MAX_MESSAGE_SIZE); + private V3MessageReader reader = new V3MessageReader(); + + public ConnectionInfo(InternalContext context, Mode mode, + NetworkAddress node, NetworkHandler.MessageListener listener, + Set commonRequestedObjects) { + super(context, mode, node, listener, commonRequestedObjects, false); + } public State getState() { return state; } + public boolean knowsOf(InventoryVector iv) { + return ivCache.containsKey(iv); + } + public Queue getSendingQueue() { return sendingQueue; } @@ -47,5 +65,24 @@ public class ConnectionInfo { return out; } - public enum State {CONNECTING, ACTIVE, DISCONNECTED} + public void updateReader() { + reader.update(in); + if (!reader.getMessages().isEmpty()) { + Iterator iterator = reader.getMessages().iterator(); + while (iterator.hasNext()) { + NetworkMessage msg = iterator.next(); + handleMessage(msg.getPayload()); + iterator.remove(); + } + } + } + + public List getMessages() { + return reader.getMessages(); + } + + @Override + protected void send(MessagePayload payload) { + sendingQueue.addFirst(payload); + } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 4f56d42..c3689af 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -22,7 +22,10 @@ import ch.dissem.bitmessage.entity.GetData; import ch.dissem.bitmessage.entity.MessagePayload; import ch.dissem.bitmessage.entity.NetworkMessage; import ch.dissem.bitmessage.entity.valueobject.InventoryVector; +import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.exception.ApplicationException; +import ch.dissem.bitmessage.exception.NodeException; +import ch.dissem.bitmessage.factory.V3MessageReader; import ch.dissem.bitmessage.ports.NetworkHandler; import ch.dissem.bitmessage.utils.Property; import org.slf4j.Logger; @@ -31,16 +34,16 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.nio.channels.ServerSocketChannel; -import java.nio.channels.SocketChannel; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; +import java.nio.ByteBuffer; +import java.nio.channels.*; +import java.util.*; import java.util.concurrent.Future; -import static java.nio.channels.SelectionKey.*; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER; +import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE; +import static ch.dissem.bitmessage.utils.DebugUtils.inc; +import static java.nio.channels.SelectionKey.OP_READ; +import static java.nio.channels.SelectionKey.OP_WRITE; /** * Network handler using java.nio, resulting in less threads. @@ -50,6 +53,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private InternalContext ctx; private Selector selector; + private ServerSocketChannel serverChannel; @Override public Future synchronize(InetAddress server, int port, MessageListener listener, long timeoutInSeconds) { @@ -58,11 +62,38 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex @Override public CustomMessage send(InetAddress server, int port, CustomMessage request) { - return null; + try (SocketChannel channel = SocketChannel.open(new InetSocketAddress(server, port))) { + channel.configureBlocking(true); + ByteBuffer buffer = ByteBuffer.allocate(MAX_MESSAGE_SIZE); + new NetworkMessage(request).write(buffer); + channel.write(buffer); + buffer.clear(); + + V3MessageReader reader = new V3MessageReader(); + while (reader.getMessages().isEmpty()) { + channel.read(buffer); + buffer.flip(); + reader.update(buffer); + } + NetworkMessage networkMessage = reader.getMessages().get(0); + + if (networkMessage != null && networkMessage.getPayload() instanceof CustomMessage) { + return (CustomMessage) networkMessage.getPayload(); + } else { + if (networkMessage == null) { + throw new NodeException("No response from node " + server); + } else { + throw new NodeException("Unexpected response from node " + + server + ": " + networkMessage.getPayload().getCommand()); + } + } + } catch (IOException e) { + throw new ApplicationException(e); + } } @Override - public void start(MessageListener listener) { + public void start(final MessageListener listener) { if (listener == null) { throw new IllegalStateException("Listener must be set at start"); } @@ -70,50 +101,83 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex throw new IllegalStateException("Network already running - you need to stop first."); } try { - final Set requestedObjects = new HashSet<>(); selector = Selector.open(); - { - ServerSocketChannel server = ServerSocketChannel.open(); - server.configureBlocking(false); - server.bind(new InetSocketAddress(ctx.getPort())); - server.register(selector, OP_ACCEPT); - } - while (selector.isOpen()) { - // TODO: establish outgoing connections - selector.select(); - Iterator keyIterator = selector.selectedKeys().iterator(); + } catch (IOException e) { + throw new ApplicationException(e); + } + final Set requestedObjects = new HashSet<>(); + new Thread(new Runnable() { + @Override + public void run() { + try { + serverChannel = ServerSocketChannel.open(); + serverChannel.bind(new InetSocketAddress(ctx.getPort())); - while (keyIterator.hasNext()) { - SelectionKey key = keyIterator.next(); - if (key.isAcceptable()) { - SocketChannel accepted = ((ServerSocketChannel) key.channel()).accept(); - accepted.configureBlocking(false); - accepted.register(selector, OP_READ | OP_WRITE).attach(new ConnectionInfo()); - } - if (key.attachment() instanceof ConnectionInfo) { - SocketChannel channel = (SocketChannel) key.channel(); - ConnectionInfo connection = (ConnectionInfo) key.attachment(); - - if (key.isWritable()) { - if (connection.getOutBuffer().hasRemaining()) { - channel.write(connection.getOutBuffer()); - } - while (!connection.getOutBuffer().hasRemaining() && !connection.getSendingQueue().isEmpty()) { - MessagePayload payload = connection.getSendingQueue().poll(); - if (payload instanceof GetData) { - requestedObjects.addAll(((GetData) payload).getInventory()); - } - new NetworkMessage(payload).write(connection.getOutBuffer()); - } - } - if (key.isReadable()) { - // TODO - channel.read(connection.getInBuffer()); - } - } - keyIterator.remove(); + SocketChannel accepted = serverChannel.accept(); + accepted.configureBlocking(false); + // FIXME: apparently it isn't good practice to generally listen for OP_WRITE + accepted.register(selector, OP_READ | OP_WRITE).attach( + new ConnectionInfo(ctx, SERVER, + new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), + listener, + requestedObjects + )); + } catch (ClosedSelectorException | AsynchronousCloseException ignore) { + } catch (IOException e) { + throw new ApplicationException(e); } } + }, "Server").start(); + new Thread(new Runnable() { + @Override + public void run() { + try { + while (selector.isOpen()) { + // TODO: establish outgoing connections + Iterator keyIterator = selector.selectedKeys().iterator(); + + while (keyIterator.hasNext()) { + SelectionKey key = keyIterator.next(); + if (key.attachment() instanceof ConnectionInfo) { + SocketChannel channel = (SocketChannel) key.channel(); + ConnectionInfo connection = (ConnectionInfo) key.attachment(); + + if (key.isWritable()) { + if (connection.getOutBuffer().hasRemaining()) { + channel.write(connection.getOutBuffer()); + } + while (!connection.getOutBuffer().hasRemaining() && !connection.getSendingQueue().isEmpty()) { + MessagePayload payload = connection.getSendingQueue().poll(); + if (payload instanceof GetData) { + requestedObjects.addAll(((GetData) payload).getInventory()); + } + new NetworkMessage(payload).write(connection.getOutBuffer()); + } + } + if (key.isReadable()) { + channel.read(connection.getInBuffer()); + connection.updateReader(); + } + } + keyIterator.remove(); + } + } + selector.close(); + } catch (ClosedSelectorException ignore) { + } catch (IOException e) { + throw new ApplicationException(e); + } + } + }, "Connections").start(); + } + + @Override + public void stop() { + try { + serverChannel.close(); + for (SelectionKey key : selector.keys()) { + key.channel().close(); + } selector.close(); } catch (IOException e) { throw new ApplicationException(e); @@ -121,23 +185,57 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } @Override - public void stop() { - + public void offer(InventoryVector iv) { + // TODO } @Override - public void offer(InventoryVector iv) { - + public void request(Collection inventoryVectors) { + // TODO } @Override public Property getNetworkStatus() { - return null; + TreeSet streams = new TreeSet<>(); + TreeMap incomingConnections = new TreeMap<>(); + TreeMap outgoingConnections = new TreeMap<>(); + + for (SelectionKey key : selector.keys()) { + if (key.attachment() instanceof ConnectionInfo) { + ConnectionInfo connection = (ConnectionInfo) key.attachment(); + if (connection.getState() == ACTIVE) { + long stream = connection.getNode().getStream(); + streams.add(stream); + if (connection.getMode() == SERVER) { + inc(incomingConnections, stream); + } else { + inc(outgoingConnections, stream); + } + } + } + } + Property[] streamProperties = new Property[streams.size()]; + int i = 0; + for (Long stream : streams) { + int incoming = incomingConnections.containsKey(stream) ? incomingConnections.get(stream) : 0; + int outgoing = outgoingConnections.containsKey(stream) ? outgoingConnections.get(stream) : 0; + streamProperties[i] = new Property("stream " + stream, + null, new Property("nodes", incoming + outgoing), + new Property("incoming", incoming), + new Property("outgoing", outgoing) + ); + i++; + } + return new Property("network", null, + new Property("connectionManager", isRunning() ? "running" : "stopped"), + new Property("connections", null, streamProperties), + new Property("requestedObjects", "requestedObjects.size()") // TODO + ); } @Override public boolean isRunning() { - return false; + return selector != null && selector.isOpen(); } @Override diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 3841be3..5dbf077 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -24,7 +24,9 @@ import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.ports.*; import ch.dissem.bitmessage.utils.Property; -import org.junit.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.net.InetAddress; import java.util.concurrent.Future; @@ -89,6 +91,9 @@ public class NetworkHandlerTest { break; case 3: data[0] = 0; + break; + default: + break; } } return new CustomMessage("test response", request.getData()); @@ -115,7 +120,7 @@ public class NetworkHandlerTest { } while (ctx.isRunning()); } - @Test(timeout = 5_000) + @Test//(timeout = 5_000) public void ensureNodesAreConnecting() { node.startup(); Property status; From 12fb794203ef99ed09bf3f155ef1b147b86b7424 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Thu, 9 Jun 2016 17:40:26 +0200 Subject: [PATCH 03/31] Minor test improvements --- .../networking/NetworkHandlerTest.java | 74 ++++++++++++------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 5dbf077..2cadfb0 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -27,8 +27,9 @@ import ch.dissem.bitmessage.utils.Property; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.net.InetAddress; import java.util.concurrent.Future; import static ch.dissem.bitmessage.utils.Singleton.cryptography; @@ -42,7 +43,8 @@ import static org.mockito.Mockito.mock; * FIXME: there really should be sensible tests for the network handler */ public class NetworkHandlerTest { - private static NetworkAddress localhost = new NetworkAddress.Builder().ipv4(127, 0, 0, 1).port(6001).build(); + private static final Logger LOG = LoggerFactory.getLogger(NetworkHandlerTest.class); + private static NetworkAddress peerAddress = new NetworkAddress.Builder().ipv4(127, 0, 0, 1).port(6001).build(); private TestInventory peerInventory; private TestInventory nodeInventory; @@ -59,26 +61,11 @@ public class NetworkHandlerTest { .inventory(peerInventory) .messageRepo(mock(MessageRepository.class)) .powRepo(mock(ProofOfWorkRepository.class)) - .port(6001) + .port(peerAddress.getPort()) .nodeRegistry(new TestNodeRegistry()) .networkHandler(new DefaultNetworkHandler()) .cryptography(new BouncyCryptography()) .listener(mock(BitmessageContext.Listener.class)) - .build(); - peer.startup(); - - nodeInventory = new TestInventory(); - networkHandler = new DefaultNetworkHandler(); - node = new BitmessageContext.Builder() - .addressRepo(mock(AddressRepository.class)) - .inventory(nodeInventory) - .messageRepo(mock(MessageRepository.class)) - .powRepo(mock(ProofOfWorkRepository.class)) - .port(6002) - .nodeRegistry(new TestNodeRegistry(localhost)) - .networkHandler(networkHandler) - .cryptography(new BouncyCryptography()) - .listener(mock(BitmessageContext.Listener.class)) .customCommandHandler(new CustomCommandHandler() { @Override public MessagePayload handle(CustomMessage request) { @@ -100,12 +87,28 @@ public class NetworkHandlerTest { } }) .build(); + peer.startup(); + + nodeInventory = new TestInventory(); + networkHandler = new DefaultNetworkHandler(); + node = new BitmessageContext.Builder() + .addressRepo(mock(AddressRepository.class)) + .inventory(nodeInventory) + .messageRepo(mock(MessageRepository.class)) + .powRepo(mock(ProofOfWorkRepository.class)) + .port(6002) + .nodeRegistry(new TestNodeRegistry(peerAddress)) + .networkHandler(networkHandler) + .cryptography(new BouncyCryptography()) + .listener(mock(BitmessageContext.Listener.class)) + .build(); } @After public void cleanUp() { shutdown(peer); shutdown(node); + shutdown(networkHandler); } private static void shutdown(BitmessageContext ctx) { @@ -120,12 +123,29 @@ public class NetworkHandlerTest { } while (ctx.isRunning()); } - @Test//(timeout = 5_000) - public void ensureNodesAreConnecting() { + private static void shutdown(NetworkHandler networkHandler) { + if (!networkHandler.isRunning()) return; + + networkHandler.stop(); + do { + try { + Thread.sleep(100); + } catch (InterruptedException ignore) { + if (networkHandler.isRunning()) { + LOG.warn("Thread interrupted while waiting for network shutdown - " + + "this could cause problems in subsequent tests."); + } + return; + } + } while (networkHandler.isRunning()); + } + + @Test(timeout = 5_000) + public void ensureNodesAreConnecting() throws Exception { node.startup(); Property status; do { - Thread.yield(); + Thread.sleep(100); status = node.status().getProperty("network", "connections", "stream 0"); } while (status == null); assertEquals(1, status.getProperty("outgoing").getValue()); @@ -138,7 +158,7 @@ public class NetworkHandlerTest { CustomMessage request = new CustomMessage("test request", data); node.startup(); - CustomMessage response = networkHandler.send(InetAddress.getLocalHost(), 6002, request); + CustomMessage response = networkHandler.send(peerAddress.toInetAddress(), peerAddress.getPort(), request); assertThat(response, notNullValue()); assertThat(response.getCustomCommand(), is("test response")); @@ -146,13 +166,13 @@ public class NetworkHandlerTest { } @Test(timeout = 5_000, expected = NodeException.class) - public void ensureCustomMessageWithoutResponsYieldsException() throws Exception { + public void ensureCustomMessageWithoutResponseYieldsException() throws Exception { byte[] data = cryptography().randomBytes(8); data[0] = (byte) 0; CustomMessage request = new CustomMessage("test request", data); node.startup(); - CustomMessage response = networkHandler.send(InetAddress.getLocalHost(), 6002, request); + CustomMessage response = networkHandler.send(peerAddress.toInetAddress(), peerAddress.getPort(), request); assertThat(response, notNullValue()); assertThat(response.getCustomCommand(), is("test response")); @@ -171,7 +191,7 @@ public class NetworkHandlerTest { "V4Pubkey.payload" ); - Future future = networkHandler.synchronize(InetAddress.getLocalHost(), 6001, + Future future = networkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), 10); future.get(); @@ -188,7 +208,7 @@ public class NetworkHandlerTest { nodeInventory.init(); - Future future = networkHandler.synchronize(InetAddress.getLocalHost(), 6001, + Future future = networkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), 10); future.get(); @@ -204,7 +224,7 @@ public class NetworkHandlerTest { "V1Msg.payload" ); - Future future = networkHandler.synchronize(InetAddress.getLocalHost(), 6001, + Future future = networkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), 10); future.get(); From 62d40fb2c3cf65138e73fab56887bc7ca30dc58d Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Sun, 12 Jun 2016 20:43:23 +0200 Subject: [PATCH 04/31] Improved unsigned byte comparison --- core/src/main/java/ch/dissem/bitmessage/utils/Bytes.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/ch/dissem/bitmessage/utils/Bytes.java b/core/src/main/java/ch/dissem/bitmessage/utils/Bytes.java index 8107eb0..986c288 100644 --- a/core/src/main/java/ch/dissem/bitmessage/utils/Bytes.java +++ b/core/src/main/java/ch/dissem/bitmessage/utils/Bytes.java @@ -23,6 +23,8 @@ package ch.dissem.bitmessage.utils; * situations. */ public class Bytes { + public static final byte BYTE_0x80 = (byte) 0x80; + public static void inc(byte[] nonce) { for (int i = nonce.length - 1; i >= 0; i--) { nonce[i]++; @@ -82,11 +84,7 @@ public class Bytes { } private static boolean lt(byte a, byte b) { - if (a < 0) return b < 0 && a < b; - if (b < 0) return a >= 0 || a < b; - return a < b; - // This would be easier to understand, but is (slightly) slower: - // return (a & 0xff) < (b & 0xff); + return (a ^ BYTE_0x80) < (b ^ BYTE_0x80); } /** From ed4fd1002b17042dce1fb944e5e53cbbd55bbd0a Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Sun, 12 Jun 2016 20:53:05 +0200 Subject: [PATCH 05/31] Improved uint reading --- .../ch/dissem/bitmessage/utils/Decode.java | 102 +++++++++--------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java b/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java index fb2f3c8..c15f397 100644 --- a/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java +++ b/core/src/main/java/ch/dissem/bitmessage/utils/Decode.java @@ -27,30 +27,29 @@ import static ch.dissem.bitmessage.utils.AccessCounter.inc; * https://bitmessage.org/wiki/Protocol_specification#Common_structures */ public class Decode { - public static byte[] shortVarBytes(InputStream stream, AccessCounter counter) throws IOException { - int length = uint16(stream, counter); - return bytes(stream, length, counter); + public static byte[] shortVarBytes(InputStream in, AccessCounter counter) throws IOException { + int length = uint16(in, counter); + return bytes(in, length, counter); } - public static byte[] varBytes(InputStream stream) throws IOException { - int length = (int) varInt(stream, null); - return bytes(stream, length, null); + public static byte[] varBytes(InputStream in) throws IOException { + return varBytes(in, null); } - public static byte[] varBytes(InputStream stream, AccessCounter counter) throws IOException { - int length = (int) varInt(stream, counter); - return bytes(stream, length, counter); + public static byte[] varBytes(InputStream in, AccessCounter counter) throws IOException { + int length = (int) varInt(in, counter); + return bytes(in, length, counter); } - public static byte[] bytes(InputStream stream, int count) throws IOException { - return bytes(stream, count, null); + public static byte[] bytes(InputStream in, int count) throws IOException { + return bytes(in, count, null); } - public static byte[] bytes(InputStream stream, int count, AccessCounter counter) throws IOException { + public static byte[] bytes(InputStream in, int count, AccessCounter counter) throws IOException { byte[] result = new byte[count]; int off = 0; while (off < count) { - int read = stream.read(result, off, count - off); + int read = in.read(result, off, count - off); if (read < 0) { throw new IOException("Unexpected end of stream, wanted to read " + count + " bytes but only got " + off); } @@ -60,87 +59,94 @@ public class Decode { return result; } - public static long[] varIntList(InputStream stream) throws IOException { - int length = (int) varInt(stream); + public static long[] varIntList(InputStream in) throws IOException { + int length = (int) varInt(in); long[] result = new long[length]; for (int i = 0; i < length; i++) { - result[i] = varInt(stream); + result[i] = varInt(in); } return result; } - public static long varInt(InputStream stream) throws IOException { - return varInt(stream, null); + public static long varInt(InputStream in) throws IOException { + return varInt(in, null); } - public static long varInt(InputStream stream, AccessCounter counter) throws IOException { - int first = stream.read(); + public static long varInt(InputStream in, AccessCounter counter) throws IOException { + int first = in.read(); inc(counter); switch (first) { case 0xfd: - return uint16(stream, counter); + return uint16(in, counter); case 0xfe: - return uint32(stream, counter); + return uint32(in, counter); case 0xff: - return int64(stream, counter); + return int64(in, counter); default: return first; } } - public static int uint8(InputStream stream) throws IOException { - return stream.read(); + public static int uint8(InputStream in) throws IOException { + return in.read(); } - public static int uint16(InputStream stream) throws IOException { - return uint16(stream, null); + public static int uint16(InputStream in) throws IOException { + return uint16(in, null); } - public static int uint16(InputStream stream, AccessCounter counter) throws IOException { + public static int uint16(InputStream in, AccessCounter counter) throws IOException { inc(counter, 2); - return stream.read() * 256 + stream.read(); + return in.read() << 8 | in.read(); } - public static long uint32(InputStream stream) throws IOException { - return uint32(stream, null); + public static long uint32(InputStream in) throws IOException { + return uint32(in, null); } - public static long uint32(InputStream stream, AccessCounter counter) throws IOException { + public static long uint32(InputStream in, AccessCounter counter) throws IOException { inc(counter, 4); - return stream.read() * 16777216L + stream.read() * 65536L + stream.read() * 256L + stream.read(); + return in.read() << 24 | in.read() << 16 | in.read() << 8 | in.read(); } - public static long uint32(ByteBuffer buffer) { - return buffer.get() * 16777216L + buffer.get() * 65536L + buffer.get() * 256L + buffer.get(); + public static long uint32(ByteBuffer in) { + return u(in.get()) << 24 | u(in.get()) << 16 | u(in.get()) << 8 | u(in.get()); } - public static int int32(InputStream stream) throws IOException { - return int32(stream, null); + public static int int32(InputStream in) throws IOException { + return int32(in, null); } - public static int int32(InputStream stream, AccessCounter counter) throws IOException { + public static int int32(InputStream in, AccessCounter counter) throws IOException { inc(counter, 4); - return ByteBuffer.wrap(bytes(stream, 4)).getInt(); + return ByteBuffer.wrap(bytes(in, 4)).getInt(); } - public static long int64(InputStream stream) throws IOException { - return int64(stream, null); + public static long int64(InputStream in) throws IOException { + return int64(in, null); } - public static long int64(InputStream stream, AccessCounter counter) throws IOException { + public static long int64(InputStream in, AccessCounter counter) throws IOException { inc(counter, 8); - return ByteBuffer.wrap(bytes(stream, 8)).getLong(); + return ByteBuffer.wrap(bytes(in, 8)).getLong(); } - public static String varString(InputStream stream) throws IOException { - return varString(stream, null); + public static String varString(InputStream in) throws IOException { + return varString(in, null); } - public static String varString(InputStream stream, AccessCounter counter) throws IOException { - int length = (int) varInt(stream, counter); + public static String varString(InputStream in, AccessCounter counter) throws IOException { + int length = (int) varInt(in, counter); // FIXME: technically, it says the length in characters, but I think this one might be correct // otherwise it will get complicated, as we'll need to read UTF-8 char by char... - return new String(bytes(stream, length, counter), "utf-8"); + return new String(bytes(in, length, counter), "utf-8"); + } + + /** + * Returns the given byte as if it were unsigned. + */ + private static int u(byte b) { + return b & 0xFF; } } From 0fadb40c6cca6d97c0f8aa02c8ba7695fea93663 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Thu, 16 Jun 2016 19:47:59 +0200 Subject: [PATCH 06/31] Improved tests and fixed some --- core/build.gradle | 2 +- .../ch/dissem/bitmessage/entity/Version.java | 1 - .../ch/dissem/bitmessage/factory/Factory.java | 2 +- .../bitmessage/factory/V3MessageReader.java | 31 ++-- .../ports/AbstractCryptography.java | 2 +- cryptography-bc/build.gradle | 2 +- cryptography-sc/build.gradle | 2 +- demo/build.gradle | 2 +- .../java/ch/dissem/bitmessage/SystemTest.java | 4 +- extensions/build.gradle | 2 +- networking/build.gradle | 2 +- .../networking/AbstractConnection.java | 35 ++-- .../networking/nio/ConnectionInfo.java | 18 +- .../networking/nio/NioNetworkHandler.java | 173 ++++++++++++++---- .../networking/NetworkHandlerTest.java | 59 ++++-- wif/build.gradle | 2 +- 16 files changed, 234 insertions(+), 105 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index dd100e6..73f8fdf 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -25,7 +25,7 @@ artifacts { dependencies { compile 'org.slf4j:slf4j-api:1.7.12' - testCompile 'junit:junit:4.11' + testCompile 'junit:junit:4.12' testCompile 'org.hamcrest:hamcrest-library:1.3' testCompile 'org.mockito:mockito-core:1.10.19' testCompile project(':cryptography-bc') diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/Version.java b/core/src/main/java/ch/dissem/bitmessage/entity/Version.java index 48022c4..4d0fd05 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/Version.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/Version.java @@ -24,7 +24,6 @@ import ch.dissem.bitmessage.utils.UnixTime; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.Random; /** * The 'version' command advertises this node's latest supported protocol version upon initiation. diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/Factory.java b/core/src/main/java/ch/dissem/bitmessage/factory/Factory.java index 0597f6a..07b3161 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/Factory.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/Factory.java @@ -40,7 +40,7 @@ import static ch.dissem.bitmessage.utils.Singleton.cryptography; * Creates {@link NetworkMessage} objects from {@link InputStream InputStreams} */ public class Factory { - public static final Logger LOG = LoggerFactory.getLogger(Factory.class); + private static final Logger LOG = LoggerFactory.getLogger(Factory.class); public static NetworkMessage getNetworkMessage(int version, InputStream stream) throws SocketTimeoutException { try { diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java index 7006cdb..80220f1 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java @@ -52,23 +52,21 @@ public class V3MessageReader { state = ReaderState.HEADER; case HEADER: if (buffer.remaining() < 20) { - buffer.compact(); return; } command = getCommand(buffer); length = (int) Decode.uint32(buffer); if (length > MAX_PAYLOAD_SIZE) { - throw new NodeException("Payload of " + length + " bytes received, no more than 1600003 was expected."); + throw new NodeException("Payload of " + length + " bytes received, no more than " + + MAX_PAYLOAD_SIZE + " was expected."); } checksum = new byte[4]; buffer.get(checksum); state = ReaderState.DATA; - if (buffer.remaining() < length) { - // We need to compact the buffer to make sure the message fits even if it's really big. - buffer.compact(); - } case DATA: - if (buffer.remaining() < length) return; + if (buffer.remaining() < length) { + return; + } if (!testChecksum(buffer)) { throw new NodeException("Checksum failed for message '" + command + "'"); } @@ -95,34 +93,35 @@ public class V3MessageReader { private boolean findMagicBytes(ByteBuffer buffer) { int i = 0; while (buffer.hasRemaining()) { - if (buffer.get() == MAGIC_BYTES[i]) { + if (i == 0) { buffer.mark(); + } + if (buffer.get() == MAGIC_BYTES[i]) { i++; - if (i == MAGIC_BYTES.length) return true; + if (i == MAGIC_BYTES.length) { + return true; + } } else { i = 0; } } if (i > 0) { buffer.reset(); - buffer.compact(); - } else { - buffer.clear(); } return false; } private static String getCommand(ByteBuffer buffer) { int start = buffer.position(); - int i = 0; - while (i < 12 && buffer.get() != 0) i++; - int end = start + i; + int l = 0; + while (l < 12 && buffer.get() != 0) l++; + int i = l + 1; while (i < 12) { if (buffer.get() != 0) throw new NodeException("'\\0' padding expected for command"); i++; } try { - return new String(buffer.array(), start, end, "ASCII"); + return new String(buffer.array(), start, l, "ASCII"); } catch (UnsupportedEncodingException e) { throw new ApplicationException(e); } diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java b/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java index f67b6d4..b02b12d 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java @@ -43,7 +43,7 @@ import static ch.dissem.bitmessage.utils.Numbers.max; * Implements everything that isn't directly dependent on either Spongy- or Bouncycastle. */ public abstract class AbstractCryptography implements Cryptography, InternalContext.ContextHolder { - public static final Logger LOG = LoggerFactory.getLogger(Cryptography.class); + protected static final Logger LOG = LoggerFactory.getLogger(Cryptography.class); private static final SecureRandom RANDOM = new SecureRandom(); private static final BigInteger TWO = BigInteger.valueOf(2); private static final BigInteger TWO_POW_64 = TWO.pow(64); diff --git a/cryptography-bc/build.gradle b/cryptography-bc/build.gradle index c09b0db..0b87c17 100644 --- a/cryptography-bc/build.gradle +++ b/cryptography-bc/build.gradle @@ -13,6 +13,6 @@ uploadArchives { dependencies { compile project(':core') compile 'org.bouncycastle:bcprov-jdk15on:1.52' - testCompile 'junit:junit:4.11' + testCompile 'junit:junit:4.12' testCompile 'org.mockito:mockito-core:1.10.19' } diff --git a/cryptography-sc/build.gradle b/cryptography-sc/build.gradle index 16771fc..a052c2e 100644 --- a/cryptography-sc/build.gradle +++ b/cryptography-sc/build.gradle @@ -13,5 +13,5 @@ uploadArchives { dependencies { compile project(':core') compile 'com.madgag.spongycastle:prov:1.52.0.0' - testCompile 'junit:junit:4.11' + testCompile 'junit:junit:4.12' } diff --git a/demo/build.gradle b/demo/build.gradle index c1571ed..0cf1781 100644 --- a/demo/build.gradle +++ b/demo/build.gradle @@ -32,6 +32,6 @@ dependencies { compile 'args4j:args4j:2.32' compile 'com.h2database:h2:1.4.190' compile 'org.apache.commons:commons-lang3:3.4' - testCompile 'junit:junit:4.11' + testCompile 'junit:junit:4.12' testCompile 'org.mockito:mockito-core:1.10.19' } diff --git a/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java b/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java index 03cbf5e..f2f02f2 100644 --- a/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java +++ b/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java @@ -88,7 +88,7 @@ public class SystemTest { bob.shutdown(); } - @Test + @Test(timeout = 60_000) public void ensureAliceCanSendMessageToBob() throws Exception { String originalMessage = UUID.randomUUID().toString(); alice.send(aliceIdentity, new BitmessageAddress(bobIdentity.getAddress()), "Subject", originalMessage); @@ -102,7 +102,7 @@ public class SystemTest { .markAsAcknowledged(any()); } - @Test + @Test(timeout = 30_000) public void ensureBobCanReceiveBroadcastFromAlice() throws Exception { String originalMessage = UUID.randomUUID().toString(); bob.addSubscribtion(new BitmessageAddress(aliceIdentity.getAddress())); diff --git a/extensions/build.gradle b/extensions/build.gradle index d44f900..42b175b 100644 --- a/extensions/build.gradle +++ b/extensions/build.gradle @@ -28,7 +28,7 @@ uploadArchives { dependencies { compile project(':core') - testCompile 'junit:junit:4.11' + testCompile 'junit:junit:4.12' testCompile 'org.slf4j:slf4j-simple:1.7.12' testCompile 'org.mockito:mockito-core:1.10.19' testCompile project(path: ':core', configuration: 'testArtifacts') diff --git a/networking/build.gradle b/networking/build.gradle index 984f585..49cafa5 100644 --- a/networking/build.gradle +++ b/networking/build.gradle @@ -12,7 +12,7 @@ uploadArchives { dependencies { compile project(':core') - testCompile 'junit:junit:4.11' + testCompile 'junit:junit:4.12' testCompile 'org.slf4j:slf4j-simple:1.7.12' testCompile 'org.mockito:mockito-core:1.10.19' testCompile project(path: ':core', configuration: 'testArtifacts') diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 3027f04..54897cb 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentLinkedDeque; import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES; import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; import static ch.dissem.bitmessage.networking.AbstractConnection.State.*; import static ch.dissem.bitmessage.utils.Singleton.cryptography; @@ -62,6 +63,8 @@ public abstract class AbstractConnection { protected long peerNonce; protected int version; protected long[] streams; + private boolean verackSent; + private boolean verackReceived; public AbstractConnection(InternalContext context, Mode mode, NetworkAddress node, @@ -198,7 +201,7 @@ public abstract class AbstractConnection { return ivCache.containsKey(iv); } - protected void cleanupIvCache() { + private void cleanupIvCache() { Long fiveMinutesAgo = UnixTime.now(-5 * MINUTE); for (Map.Entry entry : ivCache.entrySet()) { if (entry.getValue() < fiveMinutesAgo) { @@ -213,16 +216,10 @@ public abstract class AbstractConnection { handleVersion((Version) payload); break; case VERACK: - switch (mode) { - case SERVER: - activateConnection(); - break; - case CLIENT: - case SYNC: - default: - // NO OP - break; + if (verackSent) { + activateConnection(); } + verackReceived = true; break; case CUSTOM: MessagePayload response = ctx.getCustomCommandHandler().handle((CustomMessage) payload); @@ -237,7 +234,7 @@ public abstract class AbstractConnection { } } - protected void activateConnection() { + private void activateConnection() { LOG.info("Successfully established connection with node " + node); state = ACTIVE; node.setTime(UnixTime.now()); @@ -272,17 +269,13 @@ public abstract class AbstractConnection { this.version = version.getVersion(); this.streams = version.getStreams(); + verackSent = true; send(new VerAck()); - switch (mode) { - case SERVER: - send(new Version.Builder().defaults(ctx.getClientNonce()).addrFrom(host).addrRecv(node).build()); - break; - case CLIENT: - case SYNC: - activateConnection(); - break; - default: - // NO OP + if (mode == SERVER) { + send(new Version.Builder().defaults(ctx.getClientNonce()).addrFrom(host).addrRecv(node).build()); + } + if (verackReceived) { + activateConnection(); } } else { LOG.info("Received unsupported version " + version.getVersion() + ", disconnecting."); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 223722a..36bb463 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -19,6 +19,7 @@ package ch.dissem.bitmessage.networking.nio; import ch.dissem.bitmessage.InternalContext; import ch.dissem.bitmessage.entity.MessagePayload; import ch.dissem.bitmessage.entity.NetworkMessage; +import ch.dissem.bitmessage.entity.Version; import ch.dissem.bitmessage.entity.valueobject.InventoryVector; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.factory.V3MessageReader; @@ -26,9 +27,12 @@ import ch.dissem.bitmessage.networking.AbstractConnection; import ch.dissem.bitmessage.ports.NetworkHandler; import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.Iterator; +import java.util.Queue; +import java.util.Set; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; import static ch.dissem.bitmessage.ports.NetworkHandler.MAX_MESSAGE_SIZE; /** @@ -43,6 +47,10 @@ public class ConnectionInfo extends AbstractConnection { NetworkAddress node, NetworkHandler.MessageListener listener, Set commonRequestedObjects) { super(context, mode, node, listener, commonRequestedObjects, false); + out.flip(); + if (mode == CLIENT || mode == SYNC) { + send(new Version.Builder().defaults(peerNonce).addrFrom(host).addrRecv(node).build()); + } } public State getState() { @@ -77,12 +85,8 @@ public class ConnectionInfo extends AbstractConnection { } } - public List getMessages() { - return reader.getMessages(); - } - @Override protected void send(MessagePayload payload) { - sendingQueue.addFirst(payload); + sendingQueue.add(payload); } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index c3689af..cfb6c2e 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -37,11 +37,14 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.*; import java.util.*; -import java.util.concurrent.Future; +import java.util.concurrent.*; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE; import static ch.dissem.bitmessage.utils.DebugUtils.inc; +import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; import static java.nio.channels.SelectionKey.OP_READ; import static java.nio.channels.SelectionKey.OP_WRITE; @@ -51,13 +54,40 @@ import static java.nio.channels.SelectionKey.OP_WRITE; public class NioNetworkHandler implements NetworkHandler, InternalContext.ContextHolder { private static final Logger LOG = LoggerFactory.getLogger(NioNetworkHandler.class); + private final ExecutorService pool = Executors.newCachedThreadPool( + pool("network") + .lowPrio() + .daemon() + .build()); + private InternalContext ctx; private Selector selector; private ServerSocketChannel serverChannel; @Override - public Future synchronize(InetAddress server, int port, MessageListener listener, long timeoutInSeconds) { - return null; + public Future synchronize(final InetAddress server, final int port, final MessageListener listener, long timeoutInSeconds) { + return pool.submit(new Callable() { + @Override + public Void call() throws Exception { + Set requestedObjects = new HashSet<>(); + try (SocketChannel channel = SocketChannel.open(new InetSocketAddress(server, port))) { + channel.finishConnect(); + channel.configureBlocking(false); + ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, + new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), + listener, new HashSet()); + while (channel.isConnected() && + (connection.getState() != ACTIVE + || connection.getSendingQueue().isEmpty() + || requestedObjects.isEmpty())) { + write(requestedObjects, channel, connection); + read(channel, connection); + Thread.sleep(10); + } + } + return null; + } + }); } @Override @@ -66,7 +96,9 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex channel.configureBlocking(true); ByteBuffer buffer = ByteBuffer.allocate(MAX_MESSAGE_SIZE); new NetworkMessage(request).write(buffer); - channel.write(buffer); + while (buffer.hasRemaining()) { + channel.write(buffer); + } buffer.clear(); V3MessageReader reader = new V3MessageReader(); @@ -106,34 +138,74 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex throw new ApplicationException(e); } final Set requestedObjects = new HashSet<>(); - new Thread(new Runnable() { + start("connection listener", new Runnable() { @Override public void run() { try { serverChannel = ServerSocketChannel.open(); - serverChannel.bind(new InetSocketAddress(ctx.getPort())); - - SocketChannel accepted = serverChannel.accept(); - accepted.configureBlocking(false); - // FIXME: apparently it isn't good practice to generally listen for OP_WRITE - accepted.register(selector, OP_READ | OP_WRITE).attach( - new ConnectionInfo(ctx, SERVER, - new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), - listener, - requestedObjects - )); + serverChannel.socket().bind(new InetSocketAddress(ctx.getPort())); + while (selector.isOpen() && serverChannel.isOpen()) { + try { + SocketChannel accepted = serverChannel.accept(); + accepted.configureBlocking(false); + accepted.register(selector, OP_READ | OP_WRITE, + new ConnectionInfo(ctx, SERVER, + new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), + listener, + requestedObjects + )); + } catch (AsynchronousCloseException ignore) { + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } } catch (ClosedSelectorException | AsynchronousCloseException ignore) { } catch (IOException e) { throw new ApplicationException(e); + } catch (RuntimeException e) { + e.printStackTrace(); + throw e; } } - }, "Server").start(); - new Thread(new Runnable() { + }); + + start("connection starter", new Runnable() { + @Override + public void run() { + while (selector.isOpen()) { + List addresses = ctx.getNodeRegistry().getKnownAddresses( + 2, ctx.getStreams()); + for (NetworkAddress address : addresses) { + try { + SocketChannel channel = SocketChannel.open( + new InetSocketAddress(address.toInetAddress(), address.getPort())); + channel.configureBlocking(false); + channel.register(selector, OP_READ | OP_WRITE, + new ConnectionInfo(ctx, CLIENT, + address, + listener, + requestedObjects + )); + } catch (AsynchronousCloseException ignore) { + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + try { + Thread.sleep(30_000); + } catch (InterruptedException e) { + return; + } + } + } + }); + + start("processor", new Runnable() { @Override public void run() { try { while (selector.isOpen()) { - // TODO: establish outgoing connections + selector.select(1000); Iterator keyIterator = selector.selectedKeys().iterator(); while (keyIterator.hasNext()) { @@ -141,22 +213,16 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex if (key.attachment() instanceof ConnectionInfo) { SocketChannel channel = (SocketChannel) key.channel(); ConnectionInfo connection = (ConnectionInfo) key.attachment(); - if (key.isWritable()) { - if (connection.getOutBuffer().hasRemaining()) { - channel.write(connection.getOutBuffer()); - } - while (!connection.getOutBuffer().hasRemaining() && !connection.getSendingQueue().isEmpty()) { - MessagePayload payload = connection.getSendingQueue().poll(); - if (payload instanceof GetData) { - requestedObjects.addAll(((GetData) payload).getInventory()); - } - new NetworkMessage(payload).write(connection.getOutBuffer()); - } + write(requestedObjects, channel, connection); } if (key.isReadable()) { - channel.read(connection.getInBuffer()); - connection.updateReader(); + read(channel, connection); + } + if (connection.getSendingQueue().isEmpty()) { + key.interestOps(OP_READ); + } else { + key.interestOps(OP_READ | OP_WRITE); } } keyIterator.remove(); @@ -168,13 +234,52 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex throw new ApplicationException(e); } } - }, "Connections").start(); + }); + } + + private static void write(Set requestedObjects, SocketChannel channel, ConnectionInfo connection) + throws IOException { + if (!connection.getSendingQueue().isEmpty()) { + ByteBuffer buffer = connection.getOutBuffer(); + if (buffer.hasRemaining()) { + channel.write(buffer); + } + while (!buffer.hasRemaining() + && !connection.getSendingQueue().isEmpty()) { + buffer.clear(); + MessagePayload payload = connection.getSendingQueue().poll(); + if (payload instanceof GetData) { + requestedObjects.addAll(((GetData) payload).getInventory()); + } + new NetworkMessage(payload).write(buffer); + buffer.flip(); + if (buffer.hasRemaining()) { + channel.write(buffer); + } + } + } + } + + private static void read(SocketChannel channel, ConnectionInfo connection) throws IOException { + ByteBuffer buffer = connection.getInBuffer(); + while (channel.read(buffer) > 0) { + buffer.flip(); + connection.updateReader(); + buffer.compact(); + } + } + + private void start(String threadName, Runnable runnable) { + Thread thread = new Thread(runnable, threadName); + thread.setDaemon(true); + thread.setPriority(Thread.MIN_PRIORITY); + thread.start(); } @Override public void stop() { try { - serverChannel.close(); + serverChannel.socket().close(); for (SelectionKey key : selector.keys()) { key.channel().close(); } diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 2cadfb0..1b7994a 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -22,14 +22,23 @@ import ch.dissem.bitmessage.entity.CustomMessage; import ch.dissem.bitmessage.entity.MessagePayload; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.exception.NodeException; +import ch.dissem.bitmessage.networking.nio.NioNetworkHandler; import ch.dissem.bitmessage.ports.*; import ch.dissem.bitmessage.utils.Property; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.Future; import static ch.dissem.bitmessage.utils.Singleton.cryptography; @@ -42,6 +51,7 @@ import static org.mockito.Mockito.mock; /** * FIXME: there really should be sensible tests for the network handler */ +@RunWith(Parameterized.class) public class NetworkHandlerTest { private static final Logger LOG = LoggerFactory.getLogger(NetworkHandlerTest.class); private static NetworkAddress peerAddress = new NetworkAddress.Builder().ipv4(127, 0, 0, 1).port(6001).build(); @@ -51,7 +61,27 @@ public class NetworkHandlerTest { private BitmessageContext peer; private BitmessageContext node; - private NetworkHandler networkHandler; + + private final NetworkHandler peerNetworkHandler; + private final NetworkHandler nodeNetworkHandler; + + @Rule + public final TestRule timeout = new DisableOnDebug(Timeout.seconds(5)); + + public NetworkHandlerTest(NetworkHandler peer, NetworkHandler node) { + this.peerNetworkHandler = peer; + this.nodeNetworkHandler = node; + } + + @Parameterized.Parameters + public static List parameters() { + return Arrays.asList(new Object[][]{ + {new DefaultNetworkHandler(), new DefaultNetworkHandler()}, + {new DefaultNetworkHandler(), new NioNetworkHandler()}, + {new NioNetworkHandler(), new DefaultNetworkHandler()}, + {new NioNetworkHandler(), new NioNetworkHandler()} + }); + } @Before public void setUp() { @@ -63,7 +93,7 @@ public class NetworkHandlerTest { .powRepo(mock(ProofOfWorkRepository.class)) .port(peerAddress.getPort()) .nodeRegistry(new TestNodeRegistry()) - .networkHandler(new DefaultNetworkHandler()) + .networkHandler(peerNetworkHandler) .cryptography(new BouncyCryptography()) .listener(mock(BitmessageContext.Listener.class)) .customCommandHandler(new CustomCommandHandler() { @@ -90,7 +120,6 @@ public class NetworkHandlerTest { peer.startup(); nodeInventory = new TestInventory(); - networkHandler = new DefaultNetworkHandler(); node = new BitmessageContext.Builder() .addressRepo(mock(AddressRepository.class)) .inventory(nodeInventory) @@ -98,7 +127,7 @@ public class NetworkHandlerTest { .powRepo(mock(ProofOfWorkRepository.class)) .port(6002) .nodeRegistry(new TestNodeRegistry(peerAddress)) - .networkHandler(networkHandler) + .networkHandler(nodeNetworkHandler) .cryptography(new BouncyCryptography()) .listener(mock(BitmessageContext.Listener.class)) .build(); @@ -108,7 +137,7 @@ public class NetworkHandlerTest { public void cleanUp() { shutdown(peer); shutdown(node); - shutdown(networkHandler); + shutdown(nodeNetworkHandler); } private static void shutdown(BitmessageContext ctx) { @@ -140,7 +169,7 @@ public class NetworkHandlerTest { } while (networkHandler.isRunning()); } - @Test(timeout = 5_000) + @Test public void ensureNodesAreConnecting() throws Exception { node.startup(); Property status; @@ -151,14 +180,14 @@ public class NetworkHandlerTest { assertEquals(1, status.getProperty("outgoing").getValue()); } - @Test(timeout = 5_000) + @Test public void ensureCustomMessageIsSentAndResponseRetrieved() throws Exception { byte[] data = cryptography().randomBytes(8); data[0] = (byte) 1; CustomMessage request = new CustomMessage("test request", data); node.startup(); - CustomMessage response = networkHandler.send(peerAddress.toInetAddress(), peerAddress.getPort(), request); + CustomMessage response = nodeNetworkHandler.send(peerAddress.toInetAddress(), peerAddress.getPort(), request); assertThat(response, notNullValue()); assertThat(response.getCustomCommand(), is("test response")); @@ -172,14 +201,14 @@ public class NetworkHandlerTest { CustomMessage request = new CustomMessage("test request", data); node.startup(); - CustomMessage response = networkHandler.send(peerAddress.toInetAddress(), peerAddress.getPort(), request); + CustomMessage response = nodeNetworkHandler.send(peerAddress.toInetAddress(), peerAddress.getPort(), request); assertThat(response, notNullValue()); assertThat(response.getCustomCommand(), is("test response")); assertThat(response.getData(), is(request.getData())); } - @Test(timeout = 5_000) + @Test public void ensureObjectsAreSynchronizedIfBothHaveObjects() throws Exception { peerInventory.init( "V4Pubkey.payload", @@ -191,7 +220,7 @@ public class NetworkHandlerTest { "V4Pubkey.payload" ); - Future future = networkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), + Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), 10); future.get(); @@ -199,7 +228,7 @@ public class NetworkHandlerTest { assertInventorySize(3, peerInventory); } - @Test(timeout = 5_000) + @Test public void ensureObjectsAreSynchronizedIfOnlyPeerHasObjects() throws Exception { peerInventory.init( "V4Pubkey.payload", @@ -208,7 +237,7 @@ public class NetworkHandlerTest { nodeInventory.init(); - Future future = networkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), + Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), 10); future.get(); @@ -216,7 +245,7 @@ public class NetworkHandlerTest { assertInventorySize(2, peerInventory); } - @Test(timeout = 5_000) + @Test public void ensureObjectsAreSynchronizedIfOnlyNodeHasObjects() throws Exception { peerInventory.init(); @@ -224,7 +253,7 @@ public class NetworkHandlerTest { "V1Msg.payload" ); - Future future = networkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), + Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), 10); future.get(); diff --git a/wif/build.gradle b/wif/build.gradle index 93a0248..0c1ae14 100644 --- a/wif/build.gradle +++ b/wif/build.gradle @@ -13,7 +13,7 @@ uploadArchives { dependencies { compile project(':core') compile 'org.ini4j:ini4j:0.5.4' - testCompile 'junit:junit:4.11' + testCompile 'junit:junit:4.12' testCompile 'org.mockito:mockito-core:1.10.19' testCompile project(':cryptography-bc') } From ae2120675f86148f186a9a35a5499f0cd4823227 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Sat, 18 Jun 2016 23:09:23 +0200 Subject: [PATCH 07/31] Tests with NioNetworkHandler as peer work now --- .../networking/AbstractConnection.java | 36 ++++++++++++++++++- .../bitmessage/networking/Connection.java | 32 +---------------- .../networking/nio/ConnectionInfo.java | 13 +++++-- .../networking/nio/NioNetworkHandler.java | 29 ++++++++------- .../networking/NetworkHandlerTest.java | 3 +- 5 files changed, 64 insertions(+), 49 deletions(-) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 54897cb..7f7b6ec 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -60,6 +60,9 @@ public abstract class AbstractConnection { protected volatile State state; protected long lastObjectTime; + private final long syncTimeout; + private int readTimeoutCounter; + protected long peerNonce; protected int version; protected long[] streams; @@ -70,12 +73,13 @@ public abstract class AbstractConnection { NetworkAddress node, NetworkHandler.MessageListener listener, Set commonRequestedObjects, - boolean threadsafe) { + long syncTimeout, boolean threadsafe) { this.ctx = context; this.mode = mode; this.host = new NetworkAddress.Builder().ipv6(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).port(0).build(); this.node = node; this.listener = listener; + this.syncTimeout = (syncTimeout > 0 ? UnixTime.now(+syncTimeout) : 0); if (threadsafe) { this.ivCache = new ConcurrentHashMap<>(); this.sendingQueue = new ConcurrentLinkedDeque<>(); @@ -107,6 +111,9 @@ public abstract class AbstractConnection { receiveMessage(payload); break; + case DISCONNECTED: + break; + default: handleCommand(payload); break; @@ -283,6 +290,33 @@ public abstract class AbstractConnection { } } + @SuppressWarnings("RedundantIfStatement") + protected boolean syncFinished(NetworkMessage msg) { + if (mode != SYNC) { + return false; + } + if (Thread.interrupted()) { + return true; + } + if (state != ACTIVE) { + return false; + } + if (syncTimeout < UnixTime.now()) { + LOG.info("Synchronization timed out"); + return true; + } + if (msg == null) { + if (requestedObjects.isEmpty() && sendingQueue.isEmpty()) + return true; + + readTimeoutCounter++; + return readTimeoutCounter > 1; + } else { + readTimeoutCounter = 0; + return false; + } + } + public void disconnect() { state = DISCONNECTED; diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java index 3fe1b62..d9cb2cc 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java @@ -56,13 +56,11 @@ class Connection extends AbstractConnection { private final long startTime; private final Socket socket; - private final long syncTimeout; private final ReaderRunnable reader = new ReaderRunnable(); private final WriterRunnable writer = new WriterRunnable(); private InputStream in; private OutputStream out; - private int readTimeoutCounter; private boolean socketInitialized; public Connection(InternalContext context, Mode mode, Socket socket, MessageListener listener, @@ -80,10 +78,9 @@ class Connection extends AbstractConnection { private Connection(InternalContext context, Mode mode, MessageListener listener, Socket socket, Set commonRequestedObjects, NetworkAddress node, long syncTimeout) { - super(context, mode, node, listener, commonRequestedObjects, true); + super(context, mode, node, listener, commonRequestedObjects, syncTimeout, true); this.startTime = UnixTime.now(); this.socket = socket; - this.syncTimeout = (syncTimeout > 0 ? UnixTime.now(+syncTimeout) : 0); } public static Connection sync(InternalContext ctx, InetAddress address, int port, MessageListener listener, @@ -110,33 +107,6 @@ class Connection extends AbstractConnection { return node; } - @SuppressWarnings("RedundantIfStatement") - private boolean syncFinished(NetworkMessage msg) { - if (mode != SYNC) { - return false; - } - if (Thread.interrupted()) { - return true; - } - if (state != ACTIVE) { - return false; - } - if (syncTimeout < UnixTime.now()) { - LOG.info("Synchronization timed out"); - return true; - } - if (msg == null) { - if (requestedObjects.isEmpty() && sendingQueue.isEmpty()) - return true; - - readTimeoutCounter++; - return readTimeoutCounter > 1; - } else { - readTimeoutCounter = 0; - return false; - } - } - @Override protected void send(MessagePayload payload) { try { diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 36bb463..1626c87 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -42,11 +42,12 @@ public class ConnectionInfo extends AbstractConnection { private ByteBuffer in = ByteBuffer.allocate(MAX_MESSAGE_SIZE); private ByteBuffer out = ByteBuffer.allocate(MAX_MESSAGE_SIZE); private V3MessageReader reader = new V3MessageReader(); + private boolean syncFinished; public ConnectionInfo(InternalContext context, Mode mode, NetworkAddress node, NetworkHandler.MessageListener listener, - Set commonRequestedObjects) { - super(context, mode, node, listener, commonRequestedObjects, false); + Set commonRequestedObjects, long syncTimeout) { + super(context, mode, node, listener, commonRequestedObjects, syncTimeout, false); out.flip(); if (mode == CLIENT || mode == SYNC) { send(new Version.Builder().defaults(peerNonce).addrFrom(host).addrRecv(node).build()); @@ -77,14 +78,20 @@ public class ConnectionInfo extends AbstractConnection { reader.update(in); if (!reader.getMessages().isEmpty()) { Iterator iterator = reader.getMessages().iterator(); + NetworkMessage msg = null; while (iterator.hasNext()) { - NetworkMessage msg = iterator.next(); + msg = iterator.next(); handleMessage(msg.getPayload()); iterator.remove(); } + syncFinished = syncFinished(msg); } } + public boolean isSyncFinished() { + return syncFinished; + } + @Override protected void send(MessagePayload payload) { sendingQueue.add(payload); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index cfb6c2e..c7b4412 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -43,6 +43,7 @@ import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE; +import static ch.dissem.bitmessage.networking.AbstractConnection.State.DISCONNECTED; import static ch.dissem.bitmessage.utils.DebugUtils.inc; import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; import static java.nio.channels.SelectionKey.OP_READ; @@ -65,7 +66,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private ServerSocketChannel serverChannel; @Override - public Future synchronize(final InetAddress server, final int port, final MessageListener listener, long timeoutInSeconds) { + public Future synchronize(final InetAddress server, final int port, final MessageListener listener, final long timeoutInSeconds) { return pool.submit(new Callable() { @Override public Void call() throws Exception { @@ -75,11 +76,9 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex channel.configureBlocking(false); ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), - listener, new HashSet()); + listener, new HashSet(), timeoutInSeconds); while (channel.isConnected() && - (connection.getState() != ACTIVE - || connection.getSendingQueue().isEmpty() - || requestedObjects.isEmpty())) { + (connection.getState() != ACTIVE || connection.isSyncFinished())) { write(requestedObjects, channel, connection); read(channel, connection); Thread.sleep(10); @@ -138,7 +137,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex throw new ApplicationException(e); } final Set requestedObjects = new HashSet<>(); - start("connection listener", new Runnable() { + thread("connection listener", new Runnable() { @Override public void run() { try { @@ -152,9 +151,10 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex new ConnectionInfo(ctx, SERVER, new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), listener, - requestedObjects + requestedObjects, 0 )); } catch (AsynchronousCloseException ignore) { + LOG.trace(ignore.getMessage()); } catch (IOException e) { LOG.error(e.getMessage(), e); } @@ -169,7 +169,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } }); - start("connection starter", new Runnable() { + thread("connection starter", new Runnable() { @Override public void run() { while (selector.isOpen()) { @@ -184,7 +184,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex new ConnectionInfo(ctx, CLIENT, address, listener, - requestedObjects + requestedObjects, 0 )); } catch (AsynchronousCloseException ignore) { } catch (IOException e) { @@ -200,7 +200,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } }); - start("processor", new Runnable() { + thread("processor", new Runnable() { @Override public void run() { try { @@ -220,7 +220,12 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex read(channel, connection); } if (connection.getSendingQueue().isEmpty()) { - key.interestOps(OP_READ); + if (connection.getState() == DISCONNECTED) { + key.interestOps(0); + key.channel().close(); + } else { + key.interestOps(OP_READ); + } } else { key.interestOps(OP_READ | OP_WRITE); } @@ -269,7 +274,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } } - private void start(String threadName, Runnable runnable) { + private void thread(String threadName, Runnable runnable) { Thread thread = new Thread(runnable, threadName); thread.setDaemon(true); thread.setPriority(Thread.MIN_PRIORITY); diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 1b7994a..cc0fc04 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -194,12 +194,11 @@ public class NetworkHandlerTest { assertThat(response.getData(), is(data)); } - @Test(timeout = 5_000, expected = NodeException.class) + @Test(expected = NodeException.class) public void ensureCustomMessageWithoutResponseYieldsException() throws Exception { byte[] data = cryptography().randomBytes(8); data[0] = (byte) 0; CustomMessage request = new CustomMessage("test request", data); - node.startup(); CustomMessage response = nodeNetworkHandler.send(peerAddress.toInetAddress(), peerAddress.getPort(), request); From abc2f63aa6abcae08c3b371f8e62f1afe03884dd Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 20 Jun 2016 16:33:47 +0200 Subject: [PATCH 08/31] Some further fixes and improvements, not all tests working yet --- .../java/ch/dissem/bitmessage/SystemTest.java | 26 ++++++++++++- .../networking/AbstractConnection.java | 5 ++- .../networking/nio/ConnectionInfo.java | 6 +++ .../networking/nio/NioNetworkHandler.java | 38 +++++++++++-------- .../networking/NetworkHandlerTest.java | 2 +- 5 files changed, 58 insertions(+), 19 deletions(-) diff --git a/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java b/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java index f2f02f2..dac6f74 100644 --- a/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java +++ b/demo/src/test/java/ch/dissem/bitmessage/SystemTest.java @@ -4,17 +4,23 @@ import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography; import ch.dissem.bitmessage.entity.BitmessageAddress; import ch.dissem.bitmessage.entity.Plaintext; import ch.dissem.bitmessage.networking.DefaultNetworkHandler; +import ch.dissem.bitmessage.networking.nio.NioNetworkHandler; import ch.dissem.bitmessage.ports.DefaultLabeler; import ch.dissem.bitmessage.ports.Labeler; +import ch.dissem.bitmessage.ports.NetworkHandler; import ch.dissem.bitmessage.repository.*; import ch.dissem.bitmessage.utils.TTL; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -27,8 +33,11 @@ import static org.mockito.Matchers.any; /** * @author Christian Basler */ +@RunWith(Parameterized.class) public class SystemTest { private static int port = 6000; + private final NetworkHandler aliceNetworkHandler; + private final NetworkHandler bobNetworkHandler; private BitmessageContext alice; private TestListener aliceListener = new TestListener(); @@ -39,6 +48,19 @@ public class SystemTest { private TestListener bobListener = new TestListener(); private BitmessageAddress bobIdentity; + public SystemTest(NetworkHandler peer, NetworkHandler node) { + this.aliceNetworkHandler = peer; + this.bobNetworkHandler = node; + } + + @Parameterized.Parameters + public static List parameters() { + return Arrays.asList(new Object[][]{ + {new NioNetworkHandler(), new DefaultNetworkHandler()}, + {new NioNetworkHandler(), new NioNetworkHandler()} + }); + } + @Before public void setUp() { int alicePort = port++; @@ -54,7 +76,7 @@ public class SystemTest { .powRepo(new JdbcProofOfWorkRepository(aliceDB)) .port(alicePort) .nodeRegistry(new TestNodeRegistry(bobPort)) - .networkHandler(new DefaultNetworkHandler()) + .networkHandler(aliceNetworkHandler) .cryptography(new BouncyCryptography()) .listener(aliceListener) .labeler(aliceLabeler) @@ -70,7 +92,7 @@ public class SystemTest { .powRepo(new JdbcProofOfWorkRepository(bobDB)) .port(bobPort) .nodeRegistry(new TestNodeRegistry(alicePort)) - .networkHandler(new DefaultNetworkHandler()) + .networkHandler(bobNetworkHandler) .cryptography(new BouncyCryptography()) .listener(bobListener) .labeler(new DebugLabeler("Bob")) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 7f7b6ec..ac80aff 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -305,8 +305,11 @@ public abstract class AbstractConnection { LOG.info("Synchronization timed out"); return true; } + if (!sendingQueue.isEmpty()) { + return false; + } if (msg == null) { - if (requestedObjects.isEmpty() && sendingQueue.isEmpty()) + if (requestedObjects.isEmpty()) return true; readTimeoutCounter++; diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 1626c87..afe3d14 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -88,6 +88,12 @@ public class ConnectionInfo extends AbstractConnection { } } + public void updateSyncStatus() { + if (!syncFinished) { + syncFinished = reader.getMessages().isEmpty() && syncFinished(null); + } + } + public boolean isSyncFinished() { return syncFinished; } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index c7b4412..5e6f056 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -77,12 +77,12 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), listener, new HashSet(), timeoutInSeconds); - while (channel.isConnected() && - (connection.getState() != ACTIVE || connection.isSyncFinished())) { + while (channel.isConnected() && !connection.isSyncFinished()) { write(requestedObjects, channel, connection); read(channel, connection); Thread.sleep(10); } + LOG.info("Synchronization finished"); } return null; } @@ -95,28 +95,34 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex channel.configureBlocking(true); ByteBuffer buffer = ByteBuffer.allocate(MAX_MESSAGE_SIZE); new NetworkMessage(request).write(buffer); + buffer.flip(); while (buffer.hasRemaining()) { channel.write(buffer); } buffer.clear(); V3MessageReader reader = new V3MessageReader(); - while (reader.getMessages().isEmpty()) { - channel.read(buffer); - buffer.flip(); - reader.update(buffer); + while (channel.isConnected() && reader.getMessages().isEmpty()) { + if (channel.read(buffer) > 0) { + buffer.flip(); + reader.update(buffer); + buffer.compact(); + } else { + throw new NodeException("No response from node " + server); + } + } + NetworkMessage networkMessage; + if (reader.getMessages().isEmpty()) { + throw new NodeException("No response from node " + server); + } else { + networkMessage = reader.getMessages().get(0); } - NetworkMessage networkMessage = reader.getMessages().get(0); if (networkMessage != null && networkMessage.getPayload() instanceof CustomMessage) { return (CustomMessage) networkMessage.getPayload(); } else { - if (networkMessage == null) { - throw new NodeException("No response from node " + server); - } else { - throw new NodeException("Unexpected response from node " + - server + ": " + networkMessage.getPayload().getCommand()); - } + throw new NodeException("Unexpected response from node " + + server + ": " + networkMessage.getPayload().getCommand()); } } catch (IOException e) { throw new ApplicationException(e); @@ -272,6 +278,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex connection.updateReader(); buffer.compact(); } + connection.updateSyncStatus(); } private void thread(String threadName, Runnable runnable) { @@ -285,8 +292,9 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex public void stop() { try { serverChannel.socket().close(); - for (SelectionKey key : selector.keys()) { - key.channel().close(); + Iterator iterator = selector.keys().iterator(); + while (iterator.hasNext()) { + iterator.next().channel().close(); } selector.close(); } catch (IOException e) { diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index cc0fc04..24cedf7 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -254,7 +254,7 @@ public class NetworkHandlerTest { Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), - 10); + 100); future.get(); assertInventorySize(1, nodeInventory); assertInventorySize(1, peerInventory); From d130080df223a201aca925230335c6c62ce32f2c Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Fri, 8 Jul 2016 18:14:41 +0200 Subject: [PATCH 09/31] Implemented methods offer and request, system test works now but synchronization is still broken. --- .../networking/AbstractConnection.java | 3 +- .../networking/nio/NioNetworkHandler.java | 131 +++++++++++++----- .../networking/NetworkHandlerTest.java | 2 +- .../repository/JdbcMessageRepository.java | 2 +- 4 files changed, 100 insertions(+), 38 deletions(-) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index ac80aff..f7d6fc3 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -82,13 +82,12 @@ public abstract class AbstractConnection { this.syncTimeout = (syncTimeout > 0 ? UnixTime.now(+syncTimeout) : 0); if (threadsafe) { this.ivCache = new ConcurrentHashMap<>(); - this.sendingQueue = new ConcurrentLinkedDeque<>(); this.requestedObjects = Collections.newSetFromMap(new ConcurrentHashMap(10_000)); } else { this.ivCache = new HashMap<>(); - this.sendingQueue = new LinkedList<>(); this.requestedObjects = new HashSet<>(); } + this.sendingQueue = new ConcurrentLinkedDeque<>(); this.state = CONNECTING; this.commonRequestedObjects = commonRequestedObjects; } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 5e6f056..096c229 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -37,17 +37,21 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.*; import java.util.*; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; -import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT; -import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER; -import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; +import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.*; import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE; import static ch.dissem.bitmessage.networking.AbstractConnection.State.DISCONNECTED; +import static ch.dissem.bitmessage.utils.Collections.selectRandom; import static ch.dissem.bitmessage.utils.DebugUtils.inc; import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; import static java.nio.channels.SelectionKey.OP_READ; import static java.nio.channels.SelectionKey.OP_WRITE; +import static java.util.Collections.newSetFromMap; +import static java.util.Collections.synchronizedSet; /** * Network handler using java.nio, resulting in less threads. @@ -64,6 +68,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private InternalContext ctx; private Selector selector; private ServerSocketChannel serverChannel; + private Set connections = synchronizedSet(newSetFromMap(new WeakHashMap())); @Override public Future synchronize(final InetAddress server, final int port, final MessageListener listener, final long timeoutInSeconds) { @@ -77,6 +82,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), listener, new HashSet(), timeoutInSeconds); + connections.add(connection); while (channel.isConnected() && !connection.isSyncFinished()) { write(requestedObjects, channel, connection); read(channel, connection); @@ -121,8 +127,12 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex if (networkMessage != null && networkMessage.getPayload() instanceof CustomMessage) { return (CustomMessage) networkMessage.getPayload(); } else { - throw new NodeException("Unexpected response from node " + - server + ": " + networkMessage.getPayload().getCommand()); + if (networkMessage == null || networkMessage.getPayload() == null) { + throw new NodeException("Empty response from node " + server); + } else { + throw new NodeException("Unexpected response from node " + server + ": " + + networkMessage.getPayload().getClass()); + } } } catch (IOException e) { throw new ApplicationException(e); @@ -153,12 +163,13 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex try { SocketChannel accepted = serverChannel.accept(); accepted.configureBlocking(false); - accepted.register(selector, OP_READ | OP_WRITE, - new ConnectionInfo(ctx, SERVER, - new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), - listener, - requestedObjects, 0 - )); + ConnectionInfo connection = new ConnectionInfo(ctx, SERVER, + new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), + listener, + requestedObjects, 0 + ); + accepted.register(selector, OP_READ | OP_WRITE, connection); + connections.add(connection); } catch (AsynchronousCloseException ignore) { LOG.trace(ignore.getMessage()); } catch (IOException e) { @@ -186,12 +197,13 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex SocketChannel channel = SocketChannel.open( new InetSocketAddress(address.toInetAddress(), address.getPort())); channel.configureBlocking(false); - channel.register(selector, OP_READ | OP_WRITE, - new ConnectionInfo(ctx, CLIENT, - address, - listener, - requestedObjects, 0 - )); + ConnectionInfo connection = new ConnectionInfo(ctx, CLIENT, + address, + listener, + requestedObjects, 0 + ); + channel.register(selector, OP_READ | OP_WRITE, connection); + connections.add(connection); } catch (AsynchronousCloseException ignore) { } catch (IOException e) { LOG.error(e.getMessage(), e); @@ -235,6 +247,9 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } else { key.interestOps(OP_READ | OP_WRITE); } + if (connection.getState() == DISCONNECTED) { + connections.remove(connection); + } } keyIterator.remove(); } @@ -292,9 +307,8 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex public void stop() { try { serverChannel.socket().close(); - Iterator iterator = selector.keys().iterator(); - while (iterator.hasNext()) { - iterator.next().channel().close(); + for (SelectionKey selectionKey : selector.keys()) { + selectionKey.channel().close(); } selector.close(); } catch (IOException e) { @@ -304,12 +318,64 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex @Override public void offer(InventoryVector iv) { - // TODO + List target = new LinkedList<>(); + for (ConnectionInfo connection : connections) { + if (connection.getState() == ACTIVE && !connection.knowsOf(iv)) { + target.add(connection); + } + } + List randomSubset = selectRandom(NETWORK_MAGIC_NUMBER, target); + for (ConnectionInfo connection : randomSubset) { + connection.offer(iv); + } } @Override public void request(Collection inventoryVectors) { - // TODO + if (!isRunning()) return; + Iterator iterator = inventoryVectors.iterator(); + if (!iterator.hasNext()) { + return; + } + + Map> distribution = new HashMap<>(); + for (ConnectionInfo connection : connections) { + if (connection.getState() == ACTIVE) { + distribution.put(connection, new LinkedList()); + } + } + InventoryVector next = iterator.next(); + ConnectionInfo previous = null; + do { + for (ConnectionInfo connection : distribution.keySet()) { + if (connection == previous) { + next = iterator.next(); + } + if (connection.knowsOf(next)) { + List ivs = distribution.get(connection); + if (ivs.size() == GetData.MAX_INVENTORY_SIZE) { + connection.send(new GetData.Builder().inventory(ivs).build()); + ivs.clear(); + } + ivs.add(next); + iterator.remove(); + + if (iterator.hasNext()) { + next = iterator.next(); + previous = connection; + } else { + break; + } + } + } + } while (iterator.hasNext()); + + for (ConnectionInfo connection : distribution.keySet()) { + List ivs = distribution.get(connection); + if (!ivs.isEmpty()) { + connection.send(new GetData.Builder().inventory(ivs).build()); + } + } } @Override @@ -318,17 +384,14 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex TreeMap incomingConnections = new TreeMap<>(); TreeMap outgoingConnections = new TreeMap<>(); - for (SelectionKey key : selector.keys()) { - if (key.attachment() instanceof ConnectionInfo) { - ConnectionInfo connection = (ConnectionInfo) key.attachment(); - if (connection.getState() == ACTIVE) { - long stream = connection.getNode().getStream(); - streams.add(stream); - if (connection.getMode() == SERVER) { - inc(incomingConnections, stream); - } else { - inc(outgoingConnections, stream); - } + for (ConnectionInfo connection : connections) { + if (connection.getState() == ACTIVE) { + long stream = connection.getNode().getStream(); + streams.add(stream); + if (connection.getMode() == SERVER) { + inc(incomingConnections, stream); + } else { + inc(outgoingConnections, stream); } } } diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 24cedf7..85dec9e 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -66,7 +66,7 @@ public class NetworkHandlerTest { private final NetworkHandler nodeNetworkHandler; @Rule - public final TestRule timeout = new DisableOnDebug(Timeout.seconds(5)); + public final TestRule timeout = new DisableOnDebug(Timeout.seconds(10)); public NetworkHandlerTest(NetworkHandler peer, NetworkHandler node) { this.peerNetworkHandler = peer; diff --git a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java index 403754a..e2e247a 100644 --- a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java +++ b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java @@ -121,7 +121,7 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements builder.retries(rs.getInt("retries")); builder.nextTry(rs.getLong("next_try")); builder.labels(findLabels(connection, - "WHERE id IN (SELECT label_id FROM Message_Label WHERE message_id=" + id + ") ORDER BY ord")); + "id IN (SELECT label_id FROM Message_Label WHERE message_id=" + id + ") ORDER BY ord")); Plaintext message = builder.build(); message.setInitialHash(rs.getBytes("initial_hash")); result.add(message); From 50f2c7e0809251344f3d78a9969a6675cb80bd50 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Sat, 9 Jul 2016 16:37:12 +0200 Subject: [PATCH 10/31] Fixed synchronisation --- .../bitmessage/networking/AbstractConnection.java | 11 ++++------- .../bitmessage/networking/nio/NioNetworkHandler.java | 11 +++++++++-- .../bitmessage/networking/NetworkHandlerTest.java | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index f7d6fc3..edb26a2 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -61,7 +61,7 @@ public abstract class AbstractConnection { protected long lastObjectTime; private final long syncTimeout; - private int readTimeoutCounter; + private long syncReadTimeout = Long.MAX_VALUE; protected long peerNonce; protected int version; @@ -305,16 +305,13 @@ public abstract class AbstractConnection { return true; } if (!sendingQueue.isEmpty()) { + syncReadTimeout = System.currentTimeMillis() + 1000; return false; } if (msg == null) { - if (requestedObjects.isEmpty()) - return true; - - readTimeoutCounter++; - return readTimeoutCounter > 1; + return syncReadTimeout < System.currentTimeMillis(); } else { - readTimeoutCounter = 0; + syncReadTimeout = System.currentTimeMillis() + 1000; return false; } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 096c229..c49d324 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -77,7 +77,6 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex public Void call() throws Exception { Set requestedObjects = new HashSet<>(); try (SocketChannel channel = SocketChannel.open(new InetSocketAddress(server, port))) { - channel.finishConnect(); channel.configureBlocking(false); ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), @@ -88,6 +87,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex read(channel, connection); Thread.sleep(10); } + connections.remove(connection); LOG.info("Synchronization finished"); } return null; @@ -225,7 +225,6 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex while (selector.isOpen()) { selector.select(1000); Iterator keyIterator = selector.selectedKeys().iterator(); - while (keyIterator.hasNext()) { SelectionKey key = keyIterator.next(); if (key.attachment() instanceof ConnectionInfo) { @@ -253,6 +252,14 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } keyIterator.remove(); } + for (SelectionKey key : selector.keys()) { + if ((key.interestOps() & OP_WRITE) == 0) { + if (key.attachment() instanceof ConnectionInfo && + !((ConnectionInfo) key.attachment()).getSendingQueue().isEmpty()) { + key.interestOps(OP_READ | OP_WRITE); + } + } + } } selector.close(); } catch (ClosedSelectorException ignore) { diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 85dec9e..444f836 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -254,7 +254,7 @@ public class NetworkHandlerTest { Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), mock(NetworkHandler.MessageListener.class), - 100); + 10); future.get(); assertInventorySize(1, nodeInventory); assertInventorySize(1, peerInventory); From 82ee4d05bb62e00b5c48324d32d9b7c352ff12e2 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Sun, 10 Jul 2016 06:55:24 +0200 Subject: [PATCH 11/31] Raised test timeout, as the Jacoco run seems to be considerably slower. --- .../ch/dissem/bitmessage/networking/NetworkHandlerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 444f836..475cbb1 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -66,7 +66,7 @@ public class NetworkHandlerTest { private final NetworkHandler nodeNetworkHandler; @Rule - public final TestRule timeout = new DisableOnDebug(Timeout.seconds(10)); + public final TestRule timeout = new DisableOnDebug(Timeout.seconds(20)); public NetworkHandlerTest(NetworkHandler peer, NetworkHandler node) { this.peerNetworkHandler = peer; From 48ff975ffd91e38a0d9909e45072d8562b26a5bb Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 25 Jul 2016 07:52:27 +0200 Subject: [PATCH 12/31] Better memory management for the in buffer (the same TODO for the out buffer. --- .editorconfig | 7 + .../dissem/bitmessage/factory/BufferPool.java | 107 +++++++++ .../bitmessage/factory/V3MessageReader.java | 166 ++++++++++---- .../bitmessage/ports/MemoryNodeRegistry.java | 6 +- .../dissem/bitmessage/demo/Application.java | 4 +- .../java/ch/dissem/bitmessage/demo/Main.java | 4 +- .../networking/AbstractConnection.java | 13 +- .../bitmessage/networking/Connection.java | 2 +- .../networking/DefaultNetworkHandler.java | 24 +- .../networking/nio/ConnectionInfo.java | 50 ++++- .../networking/nio/NioNetworkHandler.java | 212 ++++++++++-------- 11 files changed, 427 insertions(+), 168 deletions(-) create mode 100644 .editorconfig create mode 100644 core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..656384c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_size = 4 diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java b/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java new file mode 100644 index 0000000..15e5856 --- /dev/null +++ b/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java @@ -0,0 +1,107 @@ +/* + * Copyright 2016 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.factory; + +import ch.dissem.bitmessage.ports.NetworkHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.*; + +/** + * A pool for {@link ByteBuffer}s. As they may use up a lot of memory, + * they should be reused as efficiently as possible. + */ +class BufferPool { + private static final Logger LOG = LoggerFactory.getLogger(BufferPool.class); + + public static final BufferPool bufferPool = new BufferPool(256, 2048); + + private final Map capacities = new EnumMap<>(Size.class); + private final Map> pools = new EnumMap<>(Size.class); + + private BufferPool(int small, int medium) { + capacities.put(Size.HEADER, 24); + capacities.put(Size.SMALL, small); + capacities.put(Size.MEDIUM, medium); + capacities.put(Size.LARGE, NetworkHandler.MAX_PAYLOAD_SIZE); + pools.put(Size.HEADER, new Stack()); + pools.put(Size.SMALL, new Stack()); + pools.put(Size.MEDIUM, new Stack()); + pools.put(Size.LARGE, new Stack()); + } + + public synchronized ByteBuffer allocate(int capacity) { + Size targetSize = getTargetSize(capacity); + Size s = targetSize; + do { + Stack pool = pools.get(s); + if (!pool.isEmpty()) { + return pool.pop(); + } + s = s.next(); + } while (s != null); + LOG.debug("Creating new buffer of size " + targetSize); + return ByteBuffer.allocate(capacities.get(targetSize)); + } + + public synchronized ByteBuffer allocate() { + Stack pool = pools.get(Size.HEADER); + if (!pool.isEmpty()) { + return pool.pop(); + } else { + return ByteBuffer.allocate(capacities.get(Size.HEADER)); + } + } + + public synchronized void deallocate(ByteBuffer buffer) { + buffer.clear(); + Size size = getTargetSize(buffer.capacity()); + if (buffer.capacity() != capacities.get(size)) { + throw new IllegalArgumentException("Illegal buffer capacity " + buffer.capacity() + + " one of " + capacities.values() + " expected."); + } + pools.get(size).push(buffer); + } + + private Size getTargetSize(int capacity) { + for (Size s : Size.values()) { + if (capacity <= capacities.get(s)) { + return s; + } + } + throw new IllegalArgumentException("Requested capacity too large: " + + "requested=" + capacity + "; max=" + capacities.get(Size.LARGE)); + } + + + private enum Size { + HEADER, SMALL, MEDIUM, LARGE; + + public Size next() { + switch (this) { + case SMALL: + return MEDIUM; + case MEDIUM: + return LARGE; + default: + return null; + } + } + } +} diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java index 80220f1..bb4460f 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java @@ -21,15 +21,20 @@ import ch.dissem.bitmessage.entity.NetworkMessage; import ch.dissem.bitmessage.exception.ApplicationException; import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.utils.Decode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; +import java.util.UUID; import static ch.dissem.bitmessage.entity.NetworkMessage.MAGIC_BYTES; +import static ch.dissem.bitmessage.factory.BufferPool.bufferPool; import static ch.dissem.bitmessage.ports.NetworkHandler.MAX_PAYLOAD_SIZE; import static ch.dissem.bitmessage.utils.Singleton.cryptography; @@ -37,53 +42,89 @@ import static ch.dissem.bitmessage.utils.Singleton.cryptography; * Similar to the {@link V3MessageFactory}, but used for NIO buffers which may or may not contain a whole message. */ public class V3MessageReader { + private static final Logger LOG = LoggerFactory.getLogger(V3MessageReader.class); + + private ByteBuffer headerBuffer; + private ByteBuffer dataBuffer; + private ReaderState state = ReaderState.MAGIC; private String command; private int length; private byte[] checksum; private List messages = new LinkedList<>(); + private SizeInfo sizeInfo = new SizeInfo(); - public void update(ByteBuffer buffer) { - while (buffer.hasRemaining()) { - switch (state) { - case MAGIC: - if (!findMagicBytes(buffer)) return; - state = ReaderState.HEADER; - case HEADER: - if (buffer.remaining() < 20) { - return; - } - command = getCommand(buffer); - length = (int) Decode.uint32(buffer); - if (length > MAX_PAYLOAD_SIZE) { - throw new NodeException("Payload of " + length + " bytes received, no more than " + - MAX_PAYLOAD_SIZE + " was expected."); - } - checksum = new byte[4]; - buffer.get(checksum); - state = ReaderState.DATA; - case DATA: - if (buffer.remaining() < length) { - return; - } - if (!testChecksum(buffer)) { - throw new NodeException("Checksum failed for message '" + command + "'"); - } - try { - MessagePayload payload = V3MessageFactory.getPayload( - command, - new ByteArrayInputStream(buffer.array(), buffer.arrayOffset() + buffer.position(), length), - length); - if (payload != null) { - messages.add(new NetworkMessage(payload)); - } - } catch (IOException e) { - throw new NodeException(e.getMessage()); - } - state = ReaderState.MAGIC; + public ByteBuffer getActiveBuffer() { + if (state != null && state != ReaderState.DATA) { + if (headerBuffer == null) { + headerBuffer = bufferPool.allocate(); } } + return state == ReaderState.DATA ? dataBuffer : headerBuffer; + } + + public void update() { + if (state != ReaderState.DATA) { + getActiveBuffer(); + headerBuffer.flip(); + } + switch (state) { + case MAGIC: + if (!findMagicBytes(headerBuffer)) { + headerBuffer.compact(); + return; + } + state = ReaderState.HEADER; + case HEADER: + if (headerBuffer.remaining() < 20) { + headerBuffer.compact(); + headerBuffer.limit(20); + return; + } + command = getCommand(headerBuffer); + length = (int) Decode.uint32(headerBuffer); + if (length > MAX_PAYLOAD_SIZE) { + throw new NodeException("Payload of " + length + " bytes received, no more than " + + MAX_PAYLOAD_SIZE + " was expected."); + } + sizeInfo.add(length); // FIXME: remove this once we have some values to work with + checksum = new byte[4]; + headerBuffer.get(checksum); + state = ReaderState.DATA; + bufferPool.deallocate(headerBuffer); + headerBuffer = null; + dataBuffer = bufferPool.allocate(length); + dataBuffer.clear(); + dataBuffer.limit(length); + case DATA: + if (dataBuffer.position() < length) { + return; + } else { + dataBuffer.flip(); + } + if (!testChecksum(dataBuffer)) { + state = ReaderState.MAGIC; + throw new NodeException("Checksum failed for message '" + command + "'"); + } + try { + MessagePayload payload = V3MessageFactory.getPayload( + command, + new ByteArrayInputStream(dataBuffer.array(), + dataBuffer.arrayOffset() + dataBuffer.position(), length), + length); + if (payload != null) { + messages.add(new NetworkMessage(payload)); + } + } catch (IOException e) { + throw new NodeException(e.getMessage()); + } finally { + state = ReaderState.MAGIC; + bufferPool.deallocate(dataBuffer); + dataBuffer = null; + dataBuffer = null; + } + } } public List getMessages() { @@ -129,7 +170,7 @@ public class V3MessageReader { private boolean testChecksum(ByteBuffer buffer) { byte[] payloadChecksum = cryptography().sha512(buffer.array(), - buffer.arrayOffset() + buffer.position(), length); + buffer.arrayOffset() + buffer.position(), length); for (int i = 0; i < checksum.length; i++) { if (checksum[i] != payloadChecksum[i]) { return false; @@ -138,5 +179,52 @@ public class V3MessageReader { return true; } + /** + * De-allocates all buffers. This method should be called iff the reader isn't used anymore, i.e. when its + * connection is severed. + */ + public void cleanup() { + state = null; + if (headerBuffer != null) { + bufferPool.deallocate(headerBuffer); + } + if (dataBuffer != null) { + bufferPool.deallocate(dataBuffer); + } + } + private enum ReaderState {MAGIC, HEADER, DATA} + + private class SizeInfo { + private FileWriter file; + private long min = Long.MAX_VALUE; + private long avg = 0; + private long max = Long.MIN_VALUE; + private long count = 0; + + private SizeInfo() { + try { + file = new FileWriter("D:/message_size_info-" + UUID.randomUUID() + ".csv"); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + + private void add(long length) { + avg = (count * avg + length) / (count + 1); + if (length < min) { + min = length; + } + if (length > max) { + max = length; + } + count++; + LOG.info("Received message with data size " + length + "; Min: " + min + "; Max: " + max + "; Avg: " + avg); + try { + file.write(length + "\n"); + } catch (IOException e) { + e.printStackTrace(); + } + } + } } diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java b/core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java index ff2ad95..a56b4ff 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java @@ -87,7 +87,7 @@ public class MemoryNodeRegistry implements NodeRegistry { } } if (result.isEmpty()) { - if (stableNodes.isEmpty()) { + if (stableNodes.isEmpty() || stableNodes.get(stream).isEmpty()) { loadStableNodes(); } Set nodes = stableNodes.get(stream); @@ -108,8 +108,8 @@ public class MemoryNodeRegistry implements NodeRegistry { synchronized (knownNodes) { if (!knownNodes.containsKey(node.getStream())) { knownNodes.put( - node.getStream(), - newSetFromMap(new ConcurrentHashMap()) + node.getStream(), + newSetFromMap(new ConcurrentHashMap()) ); } } diff --git a/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java b/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java index 583669b..4f155fa 100644 --- a/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java +++ b/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java @@ -22,7 +22,7 @@ import ch.dissem.bitmessage.entity.BitmessageAddress; import ch.dissem.bitmessage.entity.Plaintext; import ch.dissem.bitmessage.entity.payload.Pubkey; import ch.dissem.bitmessage.entity.valueobject.Label; -import ch.dissem.bitmessage.networking.DefaultNetworkHandler; +import ch.dissem.bitmessage.networking.nio.NioNetworkHandler; import ch.dissem.bitmessage.ports.MemoryNodeRegistry; import ch.dissem.bitmessage.repository.*; import org.apache.commons.lang3.text.WordUtils; @@ -53,7 +53,7 @@ public class Application { .nodeRegistry(new MemoryNodeRegistry()) .messageRepo(new JdbcMessageRepository(jdbcConfig)) .powRepo(new JdbcProofOfWorkRepository(jdbcConfig)) - .networkHandler(new DefaultNetworkHandler()) + .networkHandler(new NioNetworkHandler()) .cryptography(new BouncyCryptography()) .port(48444) .listener(plaintext -> System.out.println("New Message from " + plaintext.getFrom() + ": " + plaintext.getSubject())) diff --git a/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java b/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java index 2532796..402fa91 100644 --- a/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java +++ b/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java @@ -18,7 +18,7 @@ package ch.dissem.bitmessage.demo; import ch.dissem.bitmessage.BitmessageContext; import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography; -import ch.dissem.bitmessage.networking.DefaultNetworkHandler; +import ch.dissem.bitmessage.networking.nio.NioNetworkHandler; import ch.dissem.bitmessage.ports.MemoryNodeRegistry; import ch.dissem.bitmessage.repository.*; import ch.dissem.bitmessage.wif.WifExporter; @@ -53,7 +53,7 @@ public class Main { .nodeRegistry(new MemoryNodeRegistry()) .messageRepo(new JdbcMessageRepository(jdbcConfig)) .powRepo(new JdbcProofOfWorkRepository(jdbcConfig)) - .networkHandler(new DefaultNetworkHandler()) + .networkHandler(new NioNetworkHandler()) .cryptography(new BouncyCryptography()) .port(48444) .build(); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index edb26a2..73f26f5 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -73,20 +73,15 @@ public abstract class AbstractConnection { NetworkAddress node, NetworkHandler.MessageListener listener, Set commonRequestedObjects, - long syncTimeout, boolean threadsafe) { + long syncTimeout) { this.ctx = context; this.mode = mode; this.host = new NetworkAddress.Builder().ipv6(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).port(0).build(); this.node = node; this.listener = listener; this.syncTimeout = (syncTimeout > 0 ? UnixTime.now(+syncTimeout) : 0); - if (threadsafe) { - this.ivCache = new ConcurrentHashMap<>(); - this.requestedObjects = Collections.newSetFromMap(new ConcurrentHashMap(10_000)); - } else { - this.ivCache = new HashMap<>(); - this.requestedObjects = new HashSet<>(); - } + this.requestedObjects = Collections.newSetFromMap(new ConcurrentHashMap(10_000)); + this.ivCache = new ConcurrentHashMap<>(); this.sendingQueue = new ConcurrentLinkedDeque<>(); this.state = CONNECTING; this.commonRequestedObjects = commonRequestedObjects; @@ -177,7 +172,7 @@ public abstract class AbstractConnection { } catch (IOException e) { LOG.error("Stream " + objectMessage.getStream() + ", object type " + objectMessage.getType() + ": " + e.getMessage(), e); } finally { - if (commonRequestedObjects.remove(objectMessage.getInventoryVector())) { + if (!commonRequestedObjects.remove(objectMessage.getInventoryVector())) { LOG.debug("Received object that wasn't requested."); } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java index d9cb2cc..bb5f370 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java @@ -78,7 +78,7 @@ class Connection extends AbstractConnection { private Connection(InternalContext context, Mode mode, MessageListener listener, Socket socket, Set commonRequestedObjects, NetworkAddress node, long syncTimeout) { - super(context, mode, node, listener, commonRequestedObjects, syncTimeout, true); + super(context, mode, node, listener, commonRequestedObjects, syncTimeout); this.startTime = UnixTime.now(); this.socket = socket; } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java index c4a4554..b5f93b1 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java @@ -48,10 +48,10 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { final Collection connections = new ConcurrentLinkedQueue<>(); private final ExecutorService pool = Executors.newCachedThreadPool( - pool("network") - .lowPrio() - .daemon() - .build()); + pool("network") + .lowPrio() + .daemon() + .build()); private InternalContext ctx; private ServerRunnable server; private volatile boolean running; @@ -88,11 +88,11 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { throw new NodeException("No response from node " + server); } else { throw new NodeException("Unexpected response from node " + - server + ": " + networkMessage.getPayload().getCommand()); + server + ": " + networkMessage.getPayload().getCommand()); } } } catch (IOException e) { - throw new ApplicationException(e); + throw new NodeException(e.getMessage(), e); } } @@ -185,16 +185,16 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { int incoming = incomingConnections.containsKey(stream) ? incomingConnections.get(stream) : 0; int outgoing = outgoingConnections.containsKey(stream) ? outgoingConnections.get(stream) : 0; streamProperties[i] = new Property("stream " + stream, - null, new Property("nodes", incoming + outgoing), - new Property("incoming", incoming), - new Property("outgoing", outgoing) + null, new Property("nodes", incoming + outgoing), + new Property("incoming", incoming), + new Property("outgoing", outgoing) ); i++; } return new Property("network", null, - new Property("connectionManager", running ? "running" : "stopped"), - new Property("connections", null, streamProperties), - new Property("requestedObjects", requestedObjects.size()) + new Property("connectionManager", running ? "running" : "stopped"), + new Property("connections", null, streamProperties), + new Property("requestedObjects", requestedObjects.size()) ); } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index afe3d14..ed0462d 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -17,11 +17,13 @@ package ch.dissem.bitmessage.networking.nio; import ch.dissem.bitmessage.InternalContext; +import ch.dissem.bitmessage.entity.GetData; import ch.dissem.bitmessage.entity.MessagePayload; import ch.dissem.bitmessage.entity.NetworkMessage; import ch.dissem.bitmessage.entity.Version; import ch.dissem.bitmessage.entity.valueobject.InventoryVector; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; +import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.factory.V3MessageReader; import ch.dissem.bitmessage.networking.AbstractConnection; import ch.dissem.bitmessage.ports.NetworkHandler; @@ -39,15 +41,15 @@ import static ch.dissem.bitmessage.ports.NetworkHandler.MAX_MESSAGE_SIZE; * Represents the current state of a connection. */ public class ConnectionInfo extends AbstractConnection { - private ByteBuffer in = ByteBuffer.allocate(MAX_MESSAGE_SIZE); private ByteBuffer out = ByteBuffer.allocate(MAX_MESSAGE_SIZE); private V3MessageReader reader = new V3MessageReader(); private boolean syncFinished; + private long lastUpdate = Long.MAX_VALUE; public ConnectionInfo(InternalContext context, Mode mode, NetworkAddress node, NetworkHandler.MessageListener listener, Set commonRequestedObjects, long syncTimeout) { - super(context, mode, node, listener, commonRequestedObjects, syncTimeout, false); + super(context, mode, node, listener, commonRequestedObjects, syncTimeout); out.flip(); if (mode == CLIENT || mode == SYNC) { send(new Version.Builder().defaults(peerNonce).addrFrom(host).addrRecv(node).build()); @@ -67,7 +69,20 @@ public class ConnectionInfo extends AbstractConnection { } public ByteBuffer getInBuffer() { - return in; + if (reader == null) { + throw new NodeException("Node is disconnected"); + } + return reader.getActiveBuffer(); + } + + public void updateWriter() { + if ((out == null || !out.hasRemaining()) && !sendingQueue.isEmpty()) { + out.clear(); + MessagePayload payload = sendingQueue.poll(); + new NetworkMessage(payload).write(out); + out.flip(); + lastUpdate = System.currentTimeMillis(); + } } public ByteBuffer getOutBuffer() { @@ -75,7 +90,7 @@ public class ConnectionInfo extends AbstractConnection { } public void updateReader() { - reader.update(in); + reader.update(); if (!reader.getMessages().isEmpty()) { Iterator iterator = reader.getMessages().iterator(); NetworkMessage msg = null; @@ -86,6 +101,7 @@ public class ConnectionInfo extends AbstractConnection { } syncFinished = syncFinished(msg); } + lastUpdate = System.currentTimeMillis(); } public void updateSyncStatus() { @@ -94,6 +110,28 @@ public class ConnectionInfo extends AbstractConnection { } } + public boolean isExpired() { + switch (state) { + case CONNECTING: + return lastUpdate < System.currentTimeMillis() - 30000; + case ACTIVE: + return lastUpdate < System.currentTimeMillis() - 30000; + case DISCONNECTED: + return true; + default: + throw new IllegalStateException("Unknown state: " + state); + } + } + + @Override + public synchronized void disconnect() { + super.disconnect(); + if (reader != null) { + reader.cleanup(); + reader = null; + } + } + public boolean isSyncFinished() { return syncFinished; } @@ -101,5 +139,9 @@ public class ConnectionInfo extends AbstractConnection { @Override protected void send(MessagePayload payload) { sendingQueue.add(payload); + if (payload instanceof GetData) { + requestedObjects.addAll(((GetData) payload).getInventory()); + commonRequestedObjects.addAll(((GetData) payload).getInventory()); + } } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index c49d324..7ccea71 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -19,7 +19,6 @@ package ch.dissem.bitmessage.networking.nio; import ch.dissem.bitmessage.InternalContext; import ch.dissem.bitmessage.entity.CustomMessage; import ch.dissem.bitmessage.entity.GetData; -import ch.dissem.bitmessage.entity.MessagePayload; import ch.dissem.bitmessage.entity.NetworkMessage; import ch.dissem.bitmessage.entity.valueobject.InventoryVector; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; @@ -50,8 +49,7 @@ import static ch.dissem.bitmessage.utils.DebugUtils.inc; import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; import static java.nio.channels.SelectionKey.OP_READ; import static java.nio.channels.SelectionKey.OP_WRITE; -import static java.util.Collections.newSetFromMap; -import static java.util.Collections.synchronizedSet; +import static java.util.Collections.synchronizedMap; /** * Network handler using java.nio, resulting in less threads. @@ -59,31 +57,33 @@ import static java.util.Collections.synchronizedSet; public class NioNetworkHandler implements NetworkHandler, InternalContext.ContextHolder { private static final Logger LOG = LoggerFactory.getLogger(NioNetworkHandler.class); - private final ExecutorService pool = Executors.newCachedThreadPool( - pool("network") - .lowPrio() - .daemon() - .build()); + private final ExecutorService threadPool = Executors.newCachedThreadPool( + pool("network") + .lowPrio() + .daemon() + .build()); private InternalContext ctx; private Selector selector; private ServerSocketChannel serverChannel; - private Set connections = synchronizedSet(newSetFromMap(new WeakHashMap())); + private Map connections = synchronizedMap(new WeakHashMap()); + private int requestedObjectsCount; + + private Thread starter; @Override public Future synchronize(final InetAddress server, final int port, final MessageListener listener, final long timeoutInSeconds) { - return pool.submit(new Callable() { + return threadPool.submit(new Callable() { @Override public Void call() throws Exception { - Set requestedObjects = new HashSet<>(); try (SocketChannel channel = SocketChannel.open(new InetSocketAddress(server, port))) { channel.configureBlocking(false); ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, - new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), - listener, new HashSet(), timeoutInSeconds); - connections.add(connection); + new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), + listener, new HashSet(), timeoutInSeconds); + connections.put(connection, null); while (channel.isConnected() && !connection.isSyncFinished()) { - write(requestedObjects, channel, connection); + write(channel, connection); read(channel, connection); Thread.sleep(10); } @@ -109,10 +109,8 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex V3MessageReader reader = new V3MessageReader(); while (channel.isConnected() && reader.getMessages().isEmpty()) { - if (channel.read(buffer) > 0) { - buffer.flip(); - reader.update(buffer); - buffer.compact(); + if (channel.read(reader.getActiveBuffer()) > 0) { + reader.update(); } else { throw new NodeException("No response from node " + server); } @@ -131,7 +129,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex throw new NodeException("Empty response from node " + server); } else { throw new NodeException("Unexpected response from node " + server + ": " - + networkMessage.getPayload().getClass()); + + networkMessage.getPayload().getClass()); } } } catch (IOException e) { @@ -164,12 +162,14 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex SocketChannel accepted = serverChannel.accept(); accepted.configureBlocking(false); ConnectionInfo connection = new ConnectionInfo(ctx, SERVER, - new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), - listener, - requestedObjects, 0 + new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), + listener, + requestedObjects, 0 + ); + connections.put( + connection, + accepted.register(selector, OP_READ | OP_WRITE, connection) ); - accepted.register(selector, OP_READ | OP_WRITE, connection); - connections.add(connection); } catch (AsynchronousCloseException ignore) { LOG.trace(ignore.getMessage()); } catch (IOException e) { @@ -186,27 +186,53 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } }); - thread("connection starter", new Runnable() { + starter = thread("connection starter", new Runnable() { @Override public void run() { while (selector.isOpen()) { - List addresses = ctx.getNodeRegistry().getKnownAddresses( - 2, ctx.getStreams()); - for (NetworkAddress address : addresses) { - try { - SocketChannel channel = SocketChannel.open( - new InetSocketAddress(address.toInetAddress(), address.getPort())); - channel.configureBlocking(false); - ConnectionInfo connection = new ConnectionInfo(ctx, CLIENT, + int missing = NETWORK_MAGIC_NUMBER; + for (ConnectionInfo connectionInfo : connections.keySet()) { + if (connectionInfo.getState() == ACTIVE) { + missing--; + if (missing == 0) break; + } + } + if (missing > 0) { + List addresses = ctx.getNodeRegistry().getKnownAddresses(missing, ctx.getStreams()); + for (NetworkAddress address : addresses) { + try { + SocketChannel channel = SocketChannel.open(); + channel.configureBlocking(false); + channel.connect(new InetSocketAddress(address.toInetAddress(), address.getPort())); + channel.finishConnect(); + ConnectionInfo connection = new ConnectionInfo(ctx, CLIENT, address, listener, requestedObjects, 0 - ); - channel.register(selector, OP_READ | OP_WRITE, connection); - connections.add(connection); - } catch (AsynchronousCloseException ignore) { - } catch (IOException e) { - LOG.error(e.getMessage(), e); + ); + connections.put( + connection, + channel.register(selector, OP_READ | OP_WRITE, connection) + ); + } catch (AsynchronousCloseException ignore) { + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + } + + Iterator> it = connections.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry e = it.next(); + if (!e.getValue().isValid() || e.getKey().isExpired()) { + try { + e.getValue().channel().close(); + } catch (Exception ignore) { + } + e.getValue().cancel(); + e.getValue().attach(null); + it.remove(); + e.getKey().disconnect(); } } try { @@ -230,33 +256,37 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex if (key.attachment() instanceof ConnectionInfo) { SocketChannel channel = (SocketChannel) key.channel(); ConnectionInfo connection = (ConnectionInfo) key.attachment(); - if (key.isWritable()) { - write(requestedObjects, channel, connection); - } - if (key.isReadable()) { - read(channel, connection); - } - if (connection.getSendingQueue().isEmpty()) { - if (connection.getState() == DISCONNECTED) { - key.interestOps(0); - key.channel().close(); - } else { - key.interestOps(OP_READ); + try { + if (key.isWritable()) { + write(channel, connection); } - } else { - key.interestOps(OP_READ | OP_WRITE); + if (key.isReadable()) { + read(channel, connection); + } + if (connection.getSendingQueue().isEmpty()) { + if (connection.getState() == DISCONNECTED) { + key.interestOps(0); + key.channel().close(); + } else { + key.interestOps(OP_READ); + } + } else { + key.interestOps(OP_READ | OP_WRITE); + } + } catch (NodeException | IOException e) { + connection.disconnect(); } if (connection.getState() == DISCONNECTED) { connections.remove(connection); } } keyIterator.remove(); + requestedObjectsCount = requestedObjects.size(); } - for (SelectionKey key : selector.keys()) { - if ((key.interestOps() & OP_WRITE) == 0) { - if (key.attachment() instanceof ConnectionInfo && - !((ConnectionInfo) key.attachment()).getSendingQueue().isEmpty()) { - key.interestOps(OP_READ | OP_WRITE); + for (Map.Entry e : connections.entrySet()) { + if (e.getValue().isValid() && (e.getValue().interestOps() & OP_WRITE) == 0) { + if (!e.getKey().getSendingQueue().isEmpty()) { + e.getValue().interestOps(OP_READ | OP_WRITE); } } } @@ -270,54 +300,44 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex }); } - private static void write(Set requestedObjects, SocketChannel channel, ConnectionInfo connection) - throws IOException { - if (!connection.getSendingQueue().isEmpty()) { - ByteBuffer buffer = connection.getOutBuffer(); - if (buffer.hasRemaining()) { - channel.write(buffer); - } - while (!buffer.hasRemaining() - && !connection.getSendingQueue().isEmpty()) { - buffer.clear(); - MessagePayload payload = connection.getSendingQueue().poll(); - if (payload instanceof GetData) { - requestedObjects.addAll(((GetData) payload).getInventory()); - } - new NetworkMessage(payload).write(buffer); - buffer.flip(); - if (buffer.hasRemaining()) { - channel.write(buffer); - } - } + private static void write(SocketChannel channel, ConnectionInfo connection) + throws IOException { + writeBuffer(connection.getOutBuffer(), channel); + + connection.updateWriter(); + + writeBuffer(connection.getOutBuffer(), channel); + } + + private static void writeBuffer(ByteBuffer buffer, SocketChannel channel) throws IOException { + if (buffer != null && buffer.hasRemaining()) { + channel.write(buffer); } } private static void read(SocketChannel channel, ConnectionInfo connection) throws IOException { - ByteBuffer buffer = connection.getInBuffer(); - while (channel.read(buffer) > 0) { - buffer.flip(); + while (channel.read(connection.getInBuffer()) > 0) { connection.updateReader(); - buffer.compact(); } connection.updateSyncStatus(); } - private void thread(String threadName, Runnable runnable) { + private Thread thread(String threadName, Runnable runnable) { Thread thread = new Thread(runnable, threadName); thread.setDaemon(true); thread.setPriority(Thread.MIN_PRIORITY); thread.start(); + return thread; } @Override public void stop() { try { serverChannel.socket().close(); - for (SelectionKey selectionKey : selector.keys()) { + selector.close(); + for (SelectionKey selectionKey : connections.values()) { selectionKey.channel().close(); } - selector.close(); } catch (IOException e) { throw new ApplicationException(e); } @@ -326,7 +346,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex @Override public void offer(InventoryVector iv) { List target = new LinkedList<>(); - for (ConnectionInfo connection : connections) { + for (ConnectionInfo connection : connections.keySet()) { if (connection.getState() == ACTIVE && !connection.knowsOf(iv)) { target.add(connection); } @@ -346,7 +366,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } Map> distribution = new HashMap<>(); - for (ConnectionInfo connection : connections) { + for (ConnectionInfo connection : connections.keySet()) { if (connection.getState() == ACTIVE) { distribution.put(connection, new LinkedList()); } @@ -391,7 +411,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex TreeMap incomingConnections = new TreeMap<>(); TreeMap outgoingConnections = new TreeMap<>(); - for (ConnectionInfo connection : connections) { + for (ConnectionInfo connection : connections.keySet()) { if (connection.getState() == ACTIVE) { long stream = connection.getNode().getStream(); streams.add(stream); @@ -408,22 +428,22 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex int incoming = incomingConnections.containsKey(stream) ? incomingConnections.get(stream) : 0; int outgoing = outgoingConnections.containsKey(stream) ? outgoingConnections.get(stream) : 0; streamProperties[i] = new Property("stream " + stream, - null, new Property("nodes", incoming + outgoing), - new Property("incoming", incoming), - new Property("outgoing", outgoing) + null, new Property("nodes", incoming + outgoing), + new Property("incoming", incoming), + new Property("outgoing", outgoing) ); i++; } return new Property("network", null, - new Property("connectionManager", isRunning() ? "running" : "stopped"), - new Property("connections", null, streamProperties), - new Property("requestedObjects", "requestedObjects.size()") // TODO + new Property("connectionManager", isRunning() ? "running" : "stopped"), + new Property("connections", null, streamProperties), + new Property("requestedObjects", requestedObjectsCount) ); } @Override public boolean isRunning() { - return selector != null && selector.isOpen(); + return selector != null && selector.isOpen() && starter.isAlive(); } @Override From 56ebb7b8fad9810363abb7b52a759cd9181bb505 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Wed, 27 Jul 2016 07:38:39 +0200 Subject: [PATCH 13/31] Better memory management for the out buffer --- .../bitmessage/entity/NetworkMessage.java | 28 +++++++++++++++++-- .../networking/nio/ConnectionInfo.java | 24 ++++++++++------ .../networking/nio/NioNetworkHandler.java | 24 ++++++++-------- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java b/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java index e549101..f27384e 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java @@ -19,7 +19,6 @@ package ch.dissem.bitmessage.entity; import ch.dissem.bitmessage.exception.ApplicationException; import ch.dissem.bitmessage.utils.Encode; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; @@ -93,8 +92,31 @@ public class NetworkMessage implements Streamable { out.write(payloadBytes); } + /** + * A more efficient implementation of the write method, writing header data to the provided buffer and returning + * a new buffer containing the payload. + * + * @param headerBuffer where the header data is written to (24 bytes) + * @return a buffer containing the payload, ready to be read. + */ + public ByteBuffer writeHeaderAndGetPayloadBuffer(ByteBuffer headerBuffer) { + return ByteBuffer.wrap(writeHeader(headerBuffer)); + } + + /** + * For improved memory efficiency, you should use {@link #writeHeaderAndGetPayloadBuffer(ByteBuffer)} + * and write the header buffer as well as the returned payload buffer into the channel. + * + * @param buffer where everything gets written to. Needs to be large enough for the whole message + * to be written. + */ @Override - public void write(ByteBuffer out) { + public void write(ByteBuffer buffer) { + byte[] payloadBytes = writeHeader(buffer); + buffer.put(payloadBytes); + } + + private byte[] writeHeader(ByteBuffer out) { // magic Encode.int32(MAGIC, out); @@ -124,6 +146,6 @@ public class NetworkMessage implements Streamable { } // message payload - out.put(payloadBytes); + return payloadBytes; } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index ed0462d..647bc2c 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -35,13 +35,13 @@ import java.util.Set; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC; -import static ch.dissem.bitmessage.ports.NetworkHandler.MAX_MESSAGE_SIZE; /** * Represents the current state of a connection. */ public class ConnectionInfo extends AbstractConnection { - private ByteBuffer out = ByteBuffer.allocate(MAX_MESSAGE_SIZE); + private final ByteBuffer headerOut = ByteBuffer.allocate(24); + private ByteBuffer payloadOut; private V3MessageReader reader = new V3MessageReader(); private boolean syncFinished; private long lastUpdate = Long.MAX_VALUE; @@ -50,7 +50,7 @@ public class ConnectionInfo extends AbstractConnection { NetworkAddress node, NetworkHandler.MessageListener listener, Set commonRequestedObjects, long syncTimeout) { super(context, mode, node, listener, commonRequestedObjects, syncTimeout); - out.flip(); + headerOut.flip(); if (mode == CLIENT || mode == SYNC) { send(new Version.Builder().defaults(peerNonce).addrFrom(host).addrRecv(node).build()); } @@ -76,17 +76,23 @@ public class ConnectionInfo extends AbstractConnection { } public void updateWriter() { - if ((out == null || !out.hasRemaining()) && !sendingQueue.isEmpty()) { - out.clear(); + if (!headerOut.hasRemaining() && !sendingQueue.isEmpty()) { + headerOut.clear(); MessagePayload payload = sendingQueue.poll(); - new NetworkMessage(payload).write(out); - out.flip(); + payloadOut = new NetworkMessage(payload).writeHeaderAndGetPayloadBuffer(headerOut); + headerOut.flip(); lastUpdate = System.currentTimeMillis(); } } - public ByteBuffer getOutBuffer() { - return out; + public ByteBuffer[] getOutBuffers() { + return new ByteBuffer[]{headerOut, payloadOut}; + } + + public void cleanupBuffers() { + if (payloadOut != null && !payloadOut.hasRemaining()) { + payloadOut = null; + } } public void updateReader() { diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 7ccea71..069662b 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -36,10 +36,7 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.*; import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.*; import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE; @@ -66,7 +63,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private InternalContext ctx; private Selector selector; private ServerSocketChannel serverChannel; - private Map connections = synchronizedMap(new WeakHashMap()); + private Map connections = new ConcurrentHashMap<>(); private int requestedObjectsCount; private Thread starter; @@ -81,13 +78,11 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), listener, new HashSet(), timeoutInSeconds); - connections.put(connection, null); while (channel.isConnected() && !connection.isSyncFinished()) { write(channel, connection); read(channel, connection); Thread.sleep(10); } - connections.remove(connection); LOG.info("Synchronization finished"); } return null; @@ -302,16 +297,21 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private static void write(SocketChannel channel, ConnectionInfo connection) throws IOException { - writeBuffer(connection.getOutBuffer(), channel); + writeBuffer(connection.getOutBuffers(), channel); connection.updateWriter(); - writeBuffer(connection.getOutBuffer(), channel); + writeBuffer(connection.getOutBuffers(), channel); + connection.cleanupBuffers(); } - private static void writeBuffer(ByteBuffer buffer, SocketChannel channel) throws IOException { - if (buffer != null && buffer.hasRemaining()) { - channel.write(buffer); + private static void writeBuffer(ByteBuffer[] buffers, SocketChannel channel) throws IOException { + if (buffers[1] == null) { + if (buffers[0].hasRemaining()) { + channel.write(buffers[0]); + } + } else if (buffers[1].hasRemaining() || buffers[0].hasRemaining()) { + channel.write(buffers); } } From 334a510743ceba8fd52e84c304a9810534b2dbaa Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Fri, 29 Jul 2016 07:49:53 +0200 Subject: [PATCH 14/31] Fixes, improved tests and other improvements --- .../entity/valueobject/NetworkAddress.java | 8 +- .../dissem/bitmessage/factory/BufferPool.java | 88 ++++++++----------- .../bitmessage/factory/V3MessageFactory.java | 34 ++++--- .../bitmessage/factory/V3MessageReader.java | 2 +- .../bitmessage/ports/NetworkHandler.java | 3 +- .../networking/AbstractConnection.java | 4 + .../networking/DefaultNetworkHandler.java | 13 +-- .../networking/nio/ConnectionInfo.java | 2 +- .../networking/nio/NioNetworkHandler.java | 19 ++-- .../networking/NetworkHandlerTest.java | 19 ++-- 10 files changed, 99 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java index a496d19..94bc7c0 100644 --- a/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java +++ b/core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java @@ -41,19 +41,19 @@ public class NetworkAddress implements Streamable { /** * Stream number for this node */ - private long stream; + private final long stream; /** * same service(s) listed in version */ - private long services; + private final long services; /** * IPv6 address. IPv4 addresses are written into the message as a 16 byte IPv4-mapped IPv6 address * (12 bytes 00 00 00 00 00 00 00 00 00 00 FF FF, followed by the 4 bytes of the IPv4 address). */ - private byte[] ipv6; - private int port; + private final byte[] ipv6; + private final int port; private NetworkAddress(Builder builder) { time = builder.time; diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java b/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java index 15e5856..1b977d7 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java @@ -16,12 +16,16 @@ package ch.dissem.bitmessage.factory; -import ch.dissem.bitmessage.ports.NetworkHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Map; +import java.util.Stack; +import java.util.TreeMap; + +import static ch.dissem.bitmessage.ports.NetworkHandler.HEADER_SIZE; +import static ch.dissem.bitmessage.ports.NetworkHandler.MAX_PAYLOAD_SIZE; /** * A pool for {@link ByteBuffer}s. As they may use up a lot of memory, @@ -30,78 +34,58 @@ import java.util.*; class BufferPool { private static final Logger LOG = LoggerFactory.getLogger(BufferPool.class); - public static final BufferPool bufferPool = new BufferPool(256, 2048); + public static final BufferPool bufferPool = new BufferPool(); - private final Map capacities = new EnumMap<>(Size.class); - private final Map> pools = new EnumMap<>(Size.class); + private final Map> pools = new TreeMap<>(); - private BufferPool(int small, int medium) { - capacities.put(Size.HEADER, 24); - capacities.put(Size.SMALL, small); - capacities.put(Size.MEDIUM, medium); - capacities.put(Size.LARGE, NetworkHandler.MAX_PAYLOAD_SIZE); - pools.put(Size.HEADER, new Stack()); - pools.put(Size.SMALL, new Stack()); - pools.put(Size.MEDIUM, new Stack()); - pools.put(Size.LARGE, new Stack()); + private BufferPool() { + pools.put(HEADER_SIZE, new Stack()); + pools.put(54, new Stack()); + pools.put(1000, new Stack()); + pools.put(60000, new Stack()); + pools.put(MAX_PAYLOAD_SIZE, new Stack()); } public synchronized ByteBuffer allocate(int capacity) { - Size targetSize = getTargetSize(capacity); - Size s = targetSize; - do { - Stack pool = pools.get(s); - if (!pool.isEmpty()) { - return pool.pop(); + for (Map.Entry> e : pools.entrySet()) { + if (e.getKey() >= capacity && !e.getValue().isEmpty()) { + return e.getValue().pop(); } - s = s.next(); - } while (s != null); + } + Integer targetSize = getTargetSize(capacity); LOG.debug("Creating new buffer of size " + targetSize); - return ByteBuffer.allocate(capacities.get(targetSize)); + return ByteBuffer.allocate(targetSize); } - public synchronized ByteBuffer allocate() { - Stack pool = pools.get(Size.HEADER); + /** + * Returns a buffer that has the size of the Bitmessage network message header, 24 bytes. + * + * @return a buffer of size 24 + */ + public synchronized ByteBuffer allocateHeaderBuffer() { + Stack pool = pools.get(HEADER_SIZE); if (!pool.isEmpty()) { return pool.pop(); } else { - return ByteBuffer.allocate(capacities.get(Size.HEADER)); + return ByteBuffer.allocate(HEADER_SIZE); } } public synchronized void deallocate(ByteBuffer buffer) { buffer.clear(); - Size size = getTargetSize(buffer.capacity()); - if (buffer.capacity() != capacities.get(size)) { + + if (!pools.keySet().contains(buffer.capacity())) { throw new IllegalArgumentException("Illegal buffer capacity " + buffer.capacity() + - " one of " + capacities.values() + " expected."); + " one of " + pools.keySet() + " expected."); } - pools.get(size).push(buffer); + pools.get(buffer.capacity()).push(buffer); } - private Size getTargetSize(int capacity) { - for (Size s : Size.values()) { - if (capacity <= capacities.get(s)) { - return s; - } + private Integer getTargetSize(int capacity) { + for (Integer size : pools.keySet()) { + if (size >= capacity) return size; } throw new IllegalArgumentException("Requested capacity too large: " + - "requested=" + capacity + "; max=" + capacities.get(Size.LARGE)); - } - - - private enum Size { - HEADER, SMALL, MEDIUM, LARGE; - - public Size next() { - switch (this) { - case SMALL: - return MEDIUM; - case MEDIUM: - return LARGE; - default: - return null; - } - } + "requested=" + capacity + "; max=" + MAX_PAYLOAD_SIZE); } } diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java index 478d77c..7b27d13 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java @@ -107,12 +107,12 @@ class V3MessageFactory { } return new ObjectMessage.Builder() - .nonce(nonce) - .expiresTime(expiresTime) - .objectType(objectType) - .stream(stream) - .payload(payload) - .build(); + .nonce(nonce) + .expiresTime(expiresTime) + .objectType(objectType) + .stream(stream) + .payload(payload) + .build(); } private static GetData parseGetData(InputStream stream) throws IOException { @@ -153,13 +153,13 @@ class V3MessageFactory { long[] streamNumbers = Decode.varIntList(stream); return new Version.Builder() - .version(version) - .services(services) - .timestamp(timestamp) - .addrRecv(addrRecv).addrFrom(addrFrom) - .nonce(nonce) - .userAgent(userAgent) - .streams(streamNumbers).build(); + .version(version) + .services(services) + .timestamp(timestamp) + .addrRecv(addrRecv).addrFrom(addrFrom) + .nonce(nonce) + .userAgent(userAgent) + .streams(streamNumbers).build(); } private static InventoryVector parseInventoryVector(InputStream stream) throws IOException { @@ -179,7 +179,13 @@ class V3MessageFactory { long services = Decode.int64(stream); byte[] ipv6 = Decode.bytes(stream, 16); int port = Decode.uint16(stream); - return new NetworkAddress.Builder().time(time).stream(streamNumber).services(services).ipv6(ipv6).port(port).build(); + return new NetworkAddress.Builder() + .time(time) + .stream(streamNumber) + .services(services) + .ipv6(ipv6) + .port(port) + .build(); } private static boolean testChecksum(byte[] checksum, byte[] payload) { diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java index bb4460f..d26cabe 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java @@ -58,7 +58,7 @@ public class V3MessageReader { public ByteBuffer getActiveBuffer() { if (state != null && state != ReaderState.DATA) { if (headerBuffer == null) { - headerBuffer = bufferPool.allocate(); + headerBuffer = bufferPool.allocateHeaderBuffer(); } } return state == ReaderState.DATA ? dataBuffer : headerBuffer; diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java index 96dec8e..9597625 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java @@ -31,8 +31,9 @@ import java.util.concurrent.Future; */ public interface NetworkHandler { int NETWORK_MAGIC_NUMBER = 8; + int HEADER_SIZE = 24; int MAX_PAYLOAD_SIZE = 1600003; - int MAX_MESSAGE_SIZE = 24 + MAX_PAYLOAD_SIZE; + int MAX_MESSAGE_SIZE = HEADER_SIZE + MAX_PAYLOAD_SIZE; /** * Connects to the trusted host, fetches and offers new messages and disconnects afterwards. diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 73f26f5..411a57a 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -99,6 +99,10 @@ public abstract class AbstractConnection { return state; } + public long[] getStreams() { + return streams; + } + protected void handleMessage(MessagePayload payload) { switch (state) { case ACTIVE: diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java index b5f93b1..f5bfd47 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java @@ -170,12 +170,13 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { for (Connection connection : connections) { if (connection.getState() == ACTIVE) { - long stream = connection.getNode().getStream(); - streams.add(stream); - if (connection.getMode() == SERVER) { - inc(incomingConnections, stream); - } else { - inc(outgoingConnections, stream); + for (long stream : connection.getStreams()) { + streams.add(stream); + if (connection.getMode() == SERVER) { + inc(incomingConnections, stream); + } else { + inc(outgoingConnections, stream); + } } } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 647bc2c..217743e 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -44,7 +44,7 @@ public class ConnectionInfo extends AbstractConnection { private ByteBuffer payloadOut; private V3MessageReader reader = new V3MessageReader(); private boolean syncFinished; - private long lastUpdate = Long.MAX_VALUE; + private long lastUpdate = System.currentTimeMillis(); public ConnectionInfo(InternalContext context, Mode mode, NetworkAddress node, NetworkHandler.MessageListener listener, diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 069662b..6c4ae10 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.NoRouteToHostException; import java.nio.ByteBuffer; import java.nio.channels.*; import java.util.*; @@ -46,7 +47,6 @@ import static ch.dissem.bitmessage.utils.DebugUtils.inc; import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; import static java.nio.channels.SelectionKey.OP_READ; import static java.nio.channels.SelectionKey.OP_WRITE; -import static java.util.Collections.synchronizedMap; /** * Network handler using java.nio, resulting in less threads. @@ -209,7 +209,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex connection, channel.register(selector, OP_READ | OP_WRITE, connection) ); - } catch (AsynchronousCloseException ignore) { + } catch (NoRouteToHostException | AsynchronousCloseException ignore) { } catch (IOException e) { LOG.error(e.getMessage(), e); } @@ -268,7 +268,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } else { key.interestOps(OP_READ | OP_WRITE); } - } catch (NodeException | IOException e) { + } catch (CancelledKeyException | NodeException | IOException e) { connection.disconnect(); } if (connection.getState() == DISCONNECTED) { @@ -413,12 +413,13 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex for (ConnectionInfo connection : connections.keySet()) { if (connection.getState() == ACTIVE) { - long stream = connection.getNode().getStream(); - streams.add(stream); - if (connection.getMode() == SERVER) { - inc(incomingConnections, stream); - } else { - inc(outgoingConnections, stream); + for (long stream : connection.getStreams()) { + streams.add(stream); + if (connection.getMode() == SERVER) { + inc(incomingConnections, stream); + } else { + inc(outgoingConnections, stream); + } } } } diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 475cbb1..e6aeb51 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -169,15 +169,24 @@ public class NetworkHandlerTest { } while (networkHandler.isRunning()); } - @Test - public void ensureNodesAreConnecting() throws Exception { - node.startup(); + private Property waitForNetworkStatus(BitmessageContext ctx) throws InterruptedException { Property status; do { Thread.sleep(100); - status = node.status().getProperty("network", "connections", "stream 0"); + status = ctx.status().getProperty("network", "connections", "stream 1"); } while (status == null); - assertEquals(1, status.getProperty("outgoing").getValue()); + return status; + } + + @Test + public void ensureNodesAreConnecting() throws Exception { + node.startup(); + + Property nodeStatus = waitForNetworkStatus(node); + Property peerStatus = waitForNetworkStatus(peer); + + assertEquals(1, nodeStatus.getProperty("outgoing").getValue()); + assertEquals(1, peerStatus.getProperty("incoming").getValue()); } @Test From 92229151a534938a7db347bc03775599098a9bce Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Fri, 29 Jul 2016 11:47:05 +0200 Subject: [PATCH 15/31] It seems travis is a bit slow sometimes, so I raised the timeout for the NetworkHandlerTest --- .../ch/dissem/bitmessage/networking/NetworkHandlerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index e6aeb51..29fe020 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -66,7 +66,7 @@ public class NetworkHandlerTest { private final NetworkHandler nodeNetworkHandler; @Rule - public final TestRule timeout = new DisableOnDebug(Timeout.seconds(20)); + public final TestRule timeout = new DisableOnDebug(Timeout.seconds(60)); public NetworkHandlerTest(NetworkHandler peer, NetworkHandler node) { this.peerNetworkHandler = peer; From 505818a712f52defbfc2ac7e1829ac0444720b91 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 8 Aug 2016 18:00:50 +0200 Subject: [PATCH 16/31] Added option to connect to local Bitmessage client (This makes it easier to debug some problem or make some tests) --- .../dissem/bitmessage/demo/Application.java | 23 +--- .../java/ch/dissem/bitmessage/demo/Main.java | 57 +++++++-- .../networking/NetworkHandlerTest.java | 116 +++++++++--------- 3 files changed, 109 insertions(+), 87 deletions(-) diff --git a/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java b/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java index 4f155fa..3861831 100644 --- a/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java +++ b/demo/src/main/java/ch/dissem/bitmessage/demo/Application.java @@ -17,14 +17,10 @@ package ch.dissem.bitmessage.demo; import ch.dissem.bitmessage.BitmessageContext; -import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography; import ch.dissem.bitmessage.entity.BitmessageAddress; import ch.dissem.bitmessage.entity.Plaintext; import ch.dissem.bitmessage.entity.payload.Pubkey; import ch.dissem.bitmessage.entity.valueobject.Label; -import ch.dissem.bitmessage.networking.nio.NioNetworkHandler; -import ch.dissem.bitmessage.ports.MemoryNodeRegistry; -import ch.dissem.bitmessage.repository.*; import org.apache.commons.lang3.text.WordUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,19 +41,10 @@ public class Application { private BitmessageContext ctx; - public Application(InetAddress syncServer, int syncPort) { - JdbcConfig jdbcConfig = new JdbcConfig(); - ctx = new BitmessageContext.Builder() - .addressRepo(new JdbcAddressRepository(jdbcConfig)) - .inventory(new JdbcInventory(jdbcConfig)) - .nodeRegistry(new MemoryNodeRegistry()) - .messageRepo(new JdbcMessageRepository(jdbcConfig)) - .powRepo(new JdbcProofOfWorkRepository(jdbcConfig)) - .networkHandler(new NioNetworkHandler()) - .cryptography(new BouncyCryptography()) - .port(48444) - .listener(plaintext -> System.out.println("New Message from " + plaintext.getFrom() + ": " + plaintext.getSubject())) - .build(); + public Application(BitmessageContext.Builder ctxBuilder, InetAddress syncServer, int syncPort) { + ctx = ctxBuilder + .listener(plaintext -> System.out.println("New Message from " + plaintext.getFrom() + ": " + plaintext.getSubject())) + .build(); if (syncServer == null) { ctx.startup(); @@ -392,7 +379,7 @@ public class Application { System.out.println(WordUtils.wrap(message.getText(), 120)); System.out.println(); System.out.println(message.getLabels().stream().map(Label::toString).collect( - Collectors.joining(", ", "Labels: ", ""))); + Collectors.joining(", ", "Labels: ", ""))); System.out.println(); ctx.labeler().markAsRead(message); ctx.messages().save(message); diff --git a/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java b/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java index 402fa91..84e8ce0 100644 --- a/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java +++ b/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java @@ -18,20 +18,29 @@ package ch.dissem.bitmessage.demo; import ch.dissem.bitmessage.BitmessageContext; import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography; +import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.networking.nio.NioNetworkHandler; import ch.dissem.bitmessage.ports.MemoryNodeRegistry; +import ch.dissem.bitmessage.ports.NodeRegistry; import ch.dissem.bitmessage.repository.*; import ch.dissem.bitmessage.wif.WifExporter; import ch.dissem.bitmessage.wif.WifImporter; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; import org.kohsuke.args4j.Option; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.net.InetAddress; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; public class Main { + private static final Logger LOG = LoggerFactory.getLogger(Main.class); + public static void main(String[] args) throws IOException { if (System.getProperty("org.slf4j.simpleLogger.defaultLogLevel") == null) System.setProperty("org.slf4j.simpleLogger.defaultLogLevel", "ERROR"); @@ -45,18 +54,39 @@ public class Main { } catch (CmdLineException e) { parser.printUsage(System.err); } + + JdbcConfig jdbcConfig = new JdbcConfig(); + BitmessageContext.Builder ctxBuilder = new BitmessageContext.Builder() + .addressRepo(new JdbcAddressRepository(jdbcConfig)) + .inventory(new JdbcInventory(jdbcConfig)) + .messageRepo(new JdbcMessageRepository(jdbcConfig)) + .powRepo(new JdbcProofOfWorkRepository(jdbcConfig)) + .networkHandler(new NioNetworkHandler()) + .cryptography(new BouncyCryptography()) + .port(48444); + if (options.localPort != null) { + ctxBuilder.nodeRegistry(new NodeRegistry() { + @Override + public List getKnownAddresses(int limit, long... streams) { + return Arrays.stream(streams) + .mapToObj(s -> new NetworkAddress.Builder() + .ipv4(127, 0, 0, 1) + .port(options.localPort) + .stream(s).build()) + .collect(Collectors.toList()); + } + + @Override + public void offerAddresses(List addresses) { + LOG.info("Local node registry ignored offered addresses: " + addresses); + } + }); + } else { + ctxBuilder.nodeRegistry(new MemoryNodeRegistry()); + } + if (options.exportWIF != null || options.importWIF != null) { - JdbcConfig jdbcConfig = new JdbcConfig(); - BitmessageContext ctx = new BitmessageContext.Builder() - .addressRepo(new JdbcAddressRepository(jdbcConfig)) - .inventory(new JdbcInventory(jdbcConfig)) - .nodeRegistry(new MemoryNodeRegistry()) - .messageRepo(new JdbcMessageRepository(jdbcConfig)) - .powRepo(new JdbcProofOfWorkRepository(jdbcConfig)) - .networkHandler(new NioNetworkHandler()) - .cryptography(new BouncyCryptography()) - .port(48444) - .build(); + BitmessageContext ctx = ctxBuilder.build(); if (options.exportWIF != null) { new WifExporter(ctx).addAll().write(options.exportWIF); @@ -66,11 +96,14 @@ public class Main { } } else { InetAddress syncServer = options.syncServer == null ? null : InetAddress.getByName(options.syncServer); - new Application(syncServer, options.syncPort); + new Application(ctxBuilder, syncServer, options.syncPort); } } private static class CmdLineOptions { + @Option(name = "-local", usage = "Connect to local Bitmessage client on given port, instead of the usual connections from node.txt") + private Integer localPort; + @Option(name = "-import", usage = "Import from keys.dat or other WIF file.") private File importWIF; diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 29fe020..ba24bbd 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -49,7 +49,9 @@ import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; /** - * FIXME: there really should be sensible tests for the network handler + * Tests network handlers. This test is parametrized, so it can test both the nio and classic implementation + * as well as their combinations. It might be slightly over the top and will most probably be cleaned up once + * the nio implementation is deemed stable. */ @RunWith(Parameterized.class) public class NetworkHandlerTest { @@ -76,10 +78,10 @@ public class NetworkHandlerTest { @Parameterized.Parameters public static List parameters() { return Arrays.asList(new Object[][]{ - {new DefaultNetworkHandler(), new DefaultNetworkHandler()}, - {new DefaultNetworkHandler(), new NioNetworkHandler()}, - {new NioNetworkHandler(), new DefaultNetworkHandler()}, - {new NioNetworkHandler(), new NioNetworkHandler()} + {new DefaultNetworkHandler(), new DefaultNetworkHandler()}, + {new DefaultNetworkHandler(), new NioNetworkHandler()}, + {new NioNetworkHandler(), new DefaultNetworkHandler()}, + {new NioNetworkHandler(), new NioNetworkHandler()} }); } @@ -87,50 +89,50 @@ public class NetworkHandlerTest { public void setUp() { peerInventory = new TestInventory(); peer = new BitmessageContext.Builder() - .addressRepo(mock(AddressRepository.class)) - .inventory(peerInventory) - .messageRepo(mock(MessageRepository.class)) - .powRepo(mock(ProofOfWorkRepository.class)) - .port(peerAddress.getPort()) - .nodeRegistry(new TestNodeRegistry()) - .networkHandler(peerNetworkHandler) - .cryptography(new BouncyCryptography()) - .listener(mock(BitmessageContext.Listener.class)) - .customCommandHandler(new CustomCommandHandler() { - @Override - public MessagePayload handle(CustomMessage request) { - byte[] data = request.getData(); - if (data.length > 0) { - switch (data[0]) { - case 0: - return null; - case 1: - break; - case 3: - data[0] = 0; - break; - default: - break; - } + .addressRepo(mock(AddressRepository.class)) + .inventory(peerInventory) + .messageRepo(mock(MessageRepository.class)) + .powRepo(mock(ProofOfWorkRepository.class)) + .port(peerAddress.getPort()) + .nodeRegistry(new TestNodeRegistry()) + .networkHandler(peerNetworkHandler) + .cryptography(new BouncyCryptography()) + .listener(mock(BitmessageContext.Listener.class)) + .customCommandHandler(new CustomCommandHandler() { + @Override + public MessagePayload handle(CustomMessage request) { + byte[] data = request.getData(); + if (data.length > 0) { + switch (data[0]) { + case 0: + return null; + case 1: + break; + case 3: + data[0] = 0; + break; + default: + break; } - return new CustomMessage("test response", request.getData()); } - }) - .build(); + return new CustomMessage("test response", request.getData()); + } + }) + .build(); peer.startup(); nodeInventory = new TestInventory(); node = new BitmessageContext.Builder() - .addressRepo(mock(AddressRepository.class)) - .inventory(nodeInventory) - .messageRepo(mock(MessageRepository.class)) - .powRepo(mock(ProofOfWorkRepository.class)) - .port(6002) - .nodeRegistry(new TestNodeRegistry(peerAddress)) - .networkHandler(nodeNetworkHandler) - .cryptography(new BouncyCryptography()) - .listener(mock(BitmessageContext.Listener.class)) - .build(); + .addressRepo(mock(AddressRepository.class)) + .inventory(nodeInventory) + .messageRepo(mock(MessageRepository.class)) + .powRepo(mock(ProofOfWorkRepository.class)) + .port(6002) + .nodeRegistry(new TestNodeRegistry(peerAddress)) + .networkHandler(nodeNetworkHandler) + .cryptography(new BouncyCryptography()) + .listener(mock(BitmessageContext.Listener.class)) + .build(); } @After @@ -162,7 +164,7 @@ public class NetworkHandlerTest { } catch (InterruptedException ignore) { if (networkHandler.isRunning()) { LOG.warn("Thread interrupted while waiting for network shutdown - " + - "this could cause problems in subsequent tests."); + "this could cause problems in subsequent tests."); } return; } @@ -219,18 +221,18 @@ public class NetworkHandlerTest { @Test public void ensureObjectsAreSynchronizedIfBothHaveObjects() throws Exception { peerInventory.init( - "V4Pubkey.payload", - "V5Broadcast.payload" + "V4Pubkey.payload", + "V5Broadcast.payload" ); nodeInventory.init( - "V1Msg.payload", - "V4Pubkey.payload" + "V1Msg.payload", + "V4Pubkey.payload" ); Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), - mock(NetworkHandler.MessageListener.class), - 10); + mock(NetworkHandler.MessageListener.class), + 10); future.get(); assertInventorySize(3, nodeInventory); assertInventorySize(3, peerInventory); @@ -239,15 +241,15 @@ public class NetworkHandlerTest { @Test public void ensureObjectsAreSynchronizedIfOnlyPeerHasObjects() throws Exception { peerInventory.init( - "V4Pubkey.payload", - "V5Broadcast.payload" + "V4Pubkey.payload", + "V5Broadcast.payload" ); nodeInventory.init(); Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), - mock(NetworkHandler.MessageListener.class), - 10); + mock(NetworkHandler.MessageListener.class), + 10); future.get(); assertInventorySize(2, nodeInventory); assertInventorySize(2, peerInventory); @@ -258,12 +260,12 @@ public class NetworkHandlerTest { peerInventory.init(); nodeInventory.init( - "V1Msg.payload" + "V1Msg.payload" ); Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), - mock(NetworkHandler.MessageListener.class), - 10); + mock(NetworkHandler.MessageListener.class), + 10); future.get(); assertInventorySize(1, nodeInventory); assertInventorySize(1, peerInventory); From cd3a801704c405eee6b6f4cc97dbc48ae14660b6 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Tue, 9 Aug 2016 19:50:11 +0200 Subject: [PATCH 17/31] Used wrong nonce for version message --- .../bitmessage/networking/AbstractConnection.java | 12 ++++++------ .../ch/dissem/bitmessage/networking/Connection.java | 2 +- .../bitmessage/networking/nio/ConnectionInfo.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 411a57a..f3900b1 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -146,7 +146,7 @@ public abstract class AbstractConnection { List missing = ctx.getInventory().getMissing(inv.getInventory(), streams); missing.removeAll(commonRequestedObjects); LOG.debug("Received inventory with " + originalSize + " elements, of which are " - + missing.size() + " missing."); + + missing.size() + " missing."); send(new GetData.Builder().inventory(missing).build()); } @@ -197,8 +197,8 @@ public abstract class AbstractConnection { public void offer(InventoryVector iv) { sendingQueue.offer(new Inv.Builder() - .addInventoryVector(iv) - .build()); + .addInventoryVector(iv) + .build()); updateIvCache(Collections.singletonList(iv)); } @@ -235,7 +235,7 @@ public abstract class AbstractConnection { break; default: throw new NodeException("Command 'version' or 'verack' expected, but was '" - + payload.getCommand() + "'"); + + payload.getCommand() + "'"); } } @@ -259,8 +259,8 @@ public abstract class AbstractConnection { List inventory = ctx.getInventory().getInventory(streams); for (int i = 0; i < inventory.size(); i += 50000) { sendingQueue.offer(new Inv.Builder() - .inventory(inventory.subList(i, Math.min(inventory.size(), i + 50000))) - .build()); + .inventory(inventory.subList(i, Math.min(inventory.size(), i + 50000))) + .build()); } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java index bb5f370..e2c1856 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java @@ -162,7 +162,7 @@ class Connection extends AbstractConnection { try (Socket socket = Connection.this.socket) { initSocket(socket); if (mode == CLIENT || mode == SYNC) { - send(new Version.Builder().defaults(peerNonce).addrFrom(host).addrRecv(node).build()); + send(new Version.Builder().defaults(ctx.getClientNonce()).addrFrom(host).addrRecv(node).build()); } while (state != DISCONNECTED) { if (mode != SYNC) { diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 217743e..e416da5 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -52,7 +52,7 @@ public class ConnectionInfo extends AbstractConnection { super(context, mode, node, listener, commonRequestedObjects, syncTimeout); headerOut.flip(); if (mode == CLIENT || mode == SYNC) { - send(new Version.Builder().defaults(peerNonce).addrFrom(host).addrRecv(node).build()); + send(new Version.Builder().defaults(ctx.getClientNonce()).addrFrom(host).addrRecv(node).build()); } } From 52422d3398e7797252a425faf8f95f0fb7d2b6cc Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Fri, 12 Aug 2016 11:04:04 +0200 Subject: [PATCH 18/31] Updated gradle --- gradle/wrapper/gradle-wrapper.jar | Bin 51017 -> 53637 bytes gradle/wrapper/gradle-wrapper.properties | 4 ++-- gradlew | 10 +++------- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 3d0dee6e8edfecc92e04653ec780de06f7b34f8b..05ef575b0cd0173fc735f2857ce4bd594ce4f6bd 100644 GIT binary patch delta 27739 zcmZ6SQ;;UWvaQ>;ZQI7QZQHipf7`Zg+nBa(YudJZ?(BUYZk+R!8Bz84Mb*k&nGKE~6A=iM*QVT6fO(&cHzb|I!!zze^QFVE>KN zl>ZZ78UGuTX@K`p{&O(08@h@H0tDm;1_UGulswQ*kj&IUm23=x4~UI~W|pgxlNGcx z0=1~~Fm*_LsDFyKW4niZD~@UYl}ZGRdOI|G#rHVP$J_mSbh>s6H0fzg5*q{!3l3^S zz0J`W7mLQBqPcp5kE+vQLxM2&yDiWMZUR$(zfv!(Lxs;&a(S@KP_xP>&~u5O8{>gq z@P*oVZ3yBOvGeK4AAlxX(T2%fL-<5WF8^Y-j;!oOLUP+;a9z~~m9n-^lLEJTNrcpN z@<@d;?4^ZhHf1NZ`xLsB+#{mu!FUDkKc1ca2&;~jrXzf<(y33CZIXcr0f#H!_!9{D zV^5UA%U@}{F90gp$EqUJSpBO7wJ|8dw}?-G>p9r)2^caP|5oP)u9?jN5^?|a zR8Ia!`Ai`weFwGs#bhi)&@H)cPk3 znv{N1XJ7`{&Mg4|t1+?pr&MVeJ)=V&V`G`|zYIbDm$d&dk{kwh@?UD29l?u1{wKqP zLV-Z|&*3z@zIft4fZidJ3+b?v6%lX&$#x2;LP%qOw@z1s1%d*Kib$@Ba|-rZp%K-X z)!@)2eMRaV(@2lww)NQWRjf#u??VZHBbw#(>J-#$4Rf+Kvp(j3U+aE;&dnJBo!n7{ z%VbO#azx|Bz}+|rj?M6KzIJ;3o)N~0w&F9Lf3Xw1yHe+D&UlM$;`D|YJS%uLmddbW?jaqTjSWw>)dOM znalI?we=d6|D&OITD)`PO<_R{*!JCmS!L+yQLLSCnEP~AY4mOkZ%!*!T3DhfKVqn> zysFL$uloB^B`(pOWFfkNT-k1MK#y3n zul0iij}Et>Z(z?*&wkOZGxKqBQyt6c1|K}m3YX(g{6l-sj+U^dcvvk3<`JyDnseGC=Di^W}a5HO$GsV#;-=nj4S56#V_9 zdo{sg$i@p*d zadNrQkb;yGoosRFq>Fk8Ais#axA*%1%XCT6&B!I1nQnw(t zm?VmqdMrVdR0@!#7E)#zcOjQlgHK8lN|S0{ge)QZWi6f10;f!>5p-t#FKLp`TJ!rg z31Z5#y!{RPzu(2?btFb`c!Nh{*PZEa3CPUBsmO#tb>JxnKOflt&xjMX!57S zq9WQ@fBG0rJ@creYzAsBjF?8412aXKDKs)6v^m9RZAx~Xj8!38?B6IrD9*?*;TPyv zam;BOJc?#>LSCNBEqDGS4dCbJ1AZS#Az5fSl3fFicb)BJa=kpu8NpI+|zI;f)W4F>TS)QZ2tt@M-$^?p+ zE2gu2G^B936<==w-_6kZxVSh@d|Rojt3p8xh*!NS_&D)pnIBis?H8dK4nb%GwPqs>75@xK4C&HStaUP<4 zgsR~-{KS(oKx(jw_AcTXc zBv~9GA)Tx-P1b@$FSWPFJ7=)<;?iydQN)eRmN? zR~rnV5xy+>lVh^EE2&vFQlQ(CkA8SCvSbiRJuA`c8Vck04Lx6QzJYdB0uax6f?Jg` zpWqP=@&Bd#15J_cPxO#l1H~a(M=JfmV!Tcvjm$mw&=Z*k$tS01mAfuXY}f~vq4my7 zlArh$X6YBU&e8z}ZMIm?5YX)6m?q!G?}!ZG1%6Wlntk6-L5%B9@24xI4e667{)9c- z#>*W-m%b^U0@f~OFnOB&Jd632?qQ*+6RH8NPb#?+JmtTRmxs77yj8kOlGi4oy)5HN z-;=%ww`k21y2wsdg1smc%7fm!eRufs7x;hUIO}{Fg$@Y>WRCy@0z{D{M+*2K{!-Q7 zO>i}ke^yt~y^^`2T-IBIjVWX4HZY`AsH9NuRzz#-KWOAPJ8W!79cXU5x~qr7oKdM> z{lGv{J|Qr*sfSyD{r|*+^lj+rw9zcRK=yLpTJ27AvhHQK|NWW!<^;|e?LXv-8YIXW zQO1L?AExdl5>W~}rjiA0w3#-dj6^#PD<$CTkWoTtoG9ZdZKqu8u+)e*VvGs-lEuYO z0xa&8;RvG-kDPo#@?#EEoP2Te6Gtq>y6{I%O@Dmx#_pQn3?dGrm3*ng7{?Yu>xb{u zlzy_{3^23zkj}H2ePvD@#_n>l`f$fy@B9XwD8mSc??kx0D5C(~FbFZ4(b`Vqj=Ec# zRGndFG>R@|m0Q7LGfu7&1Bo)p-y)<#kE|*~$&cw{65*%iWt1)^>1br)_LiKjwo0t9 z8`L;Vu&3x(_${(yE3zKyv}vy3pA-~h`FI35yF9J+hBog-0!L>rW;R-EzN}%XeuGpO zY{*{QOz^KoL8AcG*P5XDg4FWOALOCsG3hDTF8*42G`D-keGB%j1h)ok=XAKD8@$FV z;Z={+UNak7+OFK(`R!7F@m5*6>^OuH_bv>FPuQ+*R>H?h@3y>q3vr(vsuWu}*Ex^= z+BH-Msg6IkPTlC>ZZpen6+~?X+#eYfkuk?stC-ogQ^o+Sw*II|LS67hbQ7@w&x_g> zA55p!O5YY!Oh%ho6!Me#bb`90(wl?csjeeSaKCY?v$*PlAr0rJktDkgn~jBQfQn#G z!wC#8ZXE)ZBGyr%oqD(i=5s2MAo7^Yz~JUblbZ^mzMI?9{URPq?-kW_Sdz(X(Bvog zC0s&wS2iFwbNU*ksl;WJ!9LMbC3{pu(;eIi2j4KX=gQ+IFP7$0L|Ks_r%C#yh#}kb z0|DWA0va9DQE9}Bk6cudcuJ(5Gw>RsIhR#qf<4<5IRe+i z9ySF9yJZv<)OPV|vFMp-?Y&rh7O2sjB$pjiq)Tp>7}ce0mnh^4H0l|NQ*96Jscw&j zMFIL{#BZXzPc|Vc58->HoWXMAF`I|Y*U~w!s0b?F!qLt^ex&SSSZB$ldP@$-WKLkq z+@cMW9MX-!%Xbgt$h%ZM=-ClhTRMQ_9qB8((CXFl?kw?x_Xf%}gi7jJINT9CYtU1QYrwl4=mmAZcY4*+Yh zt3?AkE3xv?UGt#&tD2>(ql+nj6iVA@G{-e}V3z2kuU(!=hi>tf!*5Tx8X67waYsM< zt*&%IclAl;o5ls_FCvY2M|H07C3LEK?<&GWtW!8wqT$$XP!6Qr@G<6GYa=JZ)JsN zMN?47)5Y3@RY`FAOPzxj;z8yS#jZv|D?Fn#$c11va}D9{tKeOjYYqmdc!2JJ%y=UP zJf9n?g0l4=tz?Bw6WJM}XNim;rt7N_s!}NS_igZrjsbvhPJC%(Lt~5xIv#-x9&~?T zu}f@bWvAlpGJi|mTFMQZjQgla>WhLyA<6wPs-gGVl0^&iyp5=Gg`B7b-Jb zGEXj@JLVMq+bl~T)FyKzb)KcMc^R_^onDH?eGI1D9>#}&@4CnykpVdHqRlc!QL8Bm zV!G3X@TXn~>t*KX3ybnf4_S|8o^h z0xP`0OIOLuZ@6DIbdM|WZ@QN}3rmU=lF%If@M3+T5r6X8v-*titLPrTDXr&iR|@&Q z?;+cHE^=M|#-DdmL{aMm0huxZiTX9oV6PmaxQkHyO<|r8Uq#>@UI6~>qvlw6YO62W z)zhim))&Tnpc3iJ7=~j(jjC2xA}=r>KrTVDhH@6RQPNK0Y>Qh~s1`Y2^A2_V)W zD+q)Q*lKj59H)SVp#BaF@jiPFoa#5KnQanVGN|sjN=YWpK1m)%0dI$9);SC$C|jz< z_*W*(+{5D)(43Ln!4jw|&K&$5HlKpGbP2c0!8>r)&=IKaq#1D4k@I^^LYZ<_m6Pra zI}GF9qi`tZM9c_q)|^ZUpfW#w0^rga;gc4Vi(g<|_xuRsRY$k);;9xecl-$AS*_yf z7LXQ8K4mm1 zk~-9bdKiM|)R*UZ7L^z$POIKGx=fgVUj_rLfX%X|!x5 zo0GlPq0?1lr>s>rbSC|>8}O7BL2y}P)rw+X;{v|)_TfoqbGaaFX34=ZHhljsxt7)Y}$&&8?1d#Ap%Vtv68g-tu z7{73N#LnHhZl;sb^L*bhz}C1CgH2y)*`2svVxq{0gm|7v?>(4u>sr2c!ICV?s>dDe zK!Pku2t^LJ!-6wiGUD1+4WW}uiMMIwUWU?THflmUiC3O|kJA|0L6X8dtLi>kQlNx& zg=+32-N#8uB8!Um0AL=f+URM7R*7;hZ*f15PGhi<6U(HK5Pc5I5elsr@pv4!r)Z_d z0KJVsC>~^zk+**GDvtPtBnD>1gsO!dE?cnKWXsxpj9Uh78)L7PruxQuhXNr|300Fz z`geQrKy`9EXky9iiKQ{p02Ob`D0wcLMA)2rDOsEG!XAW4r^xM%q+PF%^gwgNBClET!DkLW#Q4ss#utZvAsykxuSEssk8 z?6;FiH7`D`2jI@xqS5M}O613ndDauRv{3eKZgFjCplx4L-5r}?7IKXY7!E{JUD`fKsIJmbAUh#8Z;Nlq<}@B=O0X!Rwq)lu=}MKx zD%N94DvOpXqlB++#4KnUj>3%*U6!Q)ACIU>TyBG!0+?cuo_RDFg>d3{uS+FL(c&p+ zX%TJKtPP_o?KE9M??Nf#oVm7>@}FY4iI6Z%xVufpaHTX9E~G4=nhs-yJw9*j>m-5) z_7+h?q(!Q7#^m#o?~?_!-cy~$cnDk^EmRPQq*Q7mE7VMRugKc7o2-`=w3fHn;-bY+ z>bPoY0gyOI!b+AxqDz^Sq{^Kns(UlhJ>jE(YqHz4qbFjsMnC+ z#iym8%>0bDuw`r#l5xyLlhE)IXMj<-ut_dmK0;e(7I8UDO%y}wsI{88!kta%(Ak@$ zbURs+-=ZX36hvH&sdjhLlY=v=Wb|B;yHycZ0fzMPu@nlTbeQvY(X%=YhQKP#mk4Pa zH={XG58jxP&5BuKA~hH=w68NPLz8ZwQ>$Fq!qFk~lZ~j8wSg=x82LG?QZ6!Kt0;wP z>=@cxVHk|5Bba%Ehm9vIJ-Z_!x@pkq=qBzS_ARE8^n}fAO&u%prwE5DPd(O|Ws8qb zfFSAqLJu__*>9Tf9RI>aBL3nxW&cv*1CG>M!=6uF2Os|J2)K z2T7Ow$=7HU#ELJXZ#dk6`NP=C3^Vaq+d=cLD_b2};oCY&dHu#c%vKFHKcoxg)r)Mv##WR_Q0e+vyXX@|S0N-D5 zFA0--hu^Zi@>HT(9w>B&+E2++1rI}CcBwbS8D~TQiDL;v^qVGHdREhNJ^J0k&WZWL zA)$jM8DsKBj^E2}@~XE>T~B&ptM*vTGmFJLO%^u%Wh-iWD9tiYqN>eZuCTF%GamH6 zPi=N0xiGZ-aOid@d+-|6N(a<)0R-WjS5Hagli&L#} zqI^~7S*eobBl`02YV=W7og-yoO;zf&Qfb31nM%n~ieWi7H6tSD4|S{VXycY{?y(%! zzm6rqW8*7*BM3F#pi}eF#t2XRuU17en;8=4Kgs;6_c*CVEc|_nZ97j{s1hMamZm=?3&7QHsQd!WplBCF+pOc-SP_ZIL5;6_4p0$ zxGANwR8H<82N0zJMuq1zi6;+Gw%Qoy4LT>KIL2!(PwOJ6X-&2obv@U-5fxa76+VD! zydDx!?=3WD3-H>Q0)+15cRXO*x2mYm_L4oGpC8HQUTzGFbG!^H!e9%UuPlpjMa~p{#;EAFi_Vg&ub>dl^*QFg**9wzlA#N9!`Qnc4I*aV4J-Za=ty$2&e zmi0MmgE?4b>L$q&|rHD4Z?HNu3e_7F194P$JFvkI4Ay;u~_Ltqk8W* zDUFVIZCU6BqUGQTkq^nsugn^4M*eiYnDQkx!9|uIWSbJwfW9pR)04cy7=o5y^UTUS zoriIE9n+F%0A$4Lt$fz+x%#e_HHb7=K`cK{ifxy3r?^-*5~$aSf(XJ`=eXWeF*Rj_ z&YFPxzbPh|=OjC5Gm5=1-*lO^0HM@zCX+Vnq7U8L_Owj03|Je^ZX zJ#-Dt!j0wO&`(XP4>V$ycIIA-R#FNJpX3ssKFIO10oRuWsV2fF6*A-#=^MPb69l-6 zr8G;K<)%sK*g4$5WR*G$l^pHne@_!x8wO{AXYL8e%%8N+=Mr(vIn2>B7CO%# z#5{P~0gVI1oN-+>l@qN44bc^-2WC~c^pydmPR!Q+h%DVg=h9tK8Fix? z1b$gC$DXmT!rV4n3OPyFlyTFZ@&ntr=M*hIcW(Gpp7%K#0>8kIgV8K$@i;T}35cGV zVj1hXWTm~1jUG}p@XMgXZ4vTCd~n|9XGf%C0cDHW2}T4nWhgzlQk*6w+l5l?^R=Qc z+2g}6*^;Ri{gJu^PyTs*0R)LBP~egSE^1$};cCY`~1kHoZa)g6OL!Qzp{2 zh~G!SeL`8A>I6^VkF>N^5ET{qW&9E&@v#Zba6DA zBna(=4S+~F(c}e%2i$^PwRJ{-W-RaN9_8~y=iY1{qkFtQt<2S!`t!{h9x=Rbxqs3p z4g1QWq-S{?zk(hyGm-d2$TtD85=jBPn%{7G+mhHk<(Uv5wz&Km`gV$`~3}x>v zpkZAhvn2Q@Pz WwEC7X3IhpZj0!apYtB{Vz#L6zXqlnKJ>9F0~iWQpWwUV0bfO~ zt2LbS-%KbZ5Y+?)ki;XRn__z%#YI-?L4J@(6V zK?Q~5%YYx@#KElTj~i_KWmZ&NDTX+gAg%m$j4EHXXlx!;+X#pYx(~6$X%nSi%EPP0 z5q$EBF8HKD$CE(|Xy+-2Tu=(+0Kfv>6?xt+0lP^nEh7*RTMqF9Bwm`GX$T8P1+QQe zW~k*pd7=t+^5K<&wM)$n0giSx-ia4ew2&2Lt2*Qr29B8Y5JVXE^tvVMMH($_UJH#K zt@Qm>tuvTUQ8caZ@k=NJOUk#%+8k)jE_UU*@L_B7TwKO9B+L<-e6U6N|J*)Xpa+LG zli%S2sSrf|0_J(K6+M&c&CJ#2r2J;5cwVSqryC9mTTV&U%S3p8sGd>z$X72QthgI| z)4*OgJ}sFETItm!(!Kbn9-@)pUlY~UQu-@aF`u57qWntKA^a2;CiynIcnKx_+nvWb zo9%+yg=)7BvUw{~`hpsM05o8I!Ly;7nLv!-o44)3t~6@DyCfR@c^v(x#dqM|p16CC zqaC%tFBkovg0nms887+EPoOO-b18|uotPX0*g3}*7E+cfv7CCOIU56_<{#@Jpi&xkzI%a&$dQq-aBd2{u`}|rwY@~rHJ^U z^J#Ud{s&zO3bPfjzzSi@&J+K%M}He+J@uGnk2$Fm4nsdEU1Yu&>i!GKIf5_$7lPAo zcWCtD9mLB$*yx>bIl!sEAeAN`>}`)Uj6pbygN8gZY4knF9*Fkb|fC)A5SRtv$< zCu+l!(HA^O+!)ZX2T;%6k&N#ayyFWIPQL=<;KiEJr&pvt#27zYv_3G}_kLP|#yPuZ zXPWyW;#v4zlWFSOxx^W-ZU6Ur*h{$98~~~!>a=_w_elkJ6CifxH5BHar)yryIcxWZ zvYxN*d5+w5<+S)&+1pvfw5)K@14*^#Gw;HL>mAAwU;KNHISZmi4$@jQ*a{((%^{Uy zUdLVYcp}s*Yc4oJxuLUXFfQDhFUBoTt(m3~I!E7rsM>jzWZx#Brzy}-=K*i~I#F|A zIaEZL>;5kZ9N-B>vp2@j6P&^ao%)KM>?_?0c^R!REkF<2yjQM2WE~ZWAZeao(}m(2 z1_3EC-D0LS$KNiSR>A&rse4y!pG>I1BZ0@X|I#&Kh5URY<;nK&*g1^YoyRf$s@Qm; zL|LF{zBJ$X*Ew!58su>IR#+_jJUVE~FJPo6X$_V$TmX^Gejd-SBmDjHpvgwiF$c`D z?vf>ke(>)x1fgPlS8hY0H?0@e09{YiJk9}W&e(O5y}`3BHfV3i!gsziO=$YL!q)`m zJ;>6%;8j8@`M*HYfM53Cq?UdmJePz*=cKAgfLV${uXXFa;=RE(-iDbUglh7U+nG%# zM!$qzF2GAr)o**0i(*8f6;SM#fFVL+ABy%iqc4U1{XeTZ^OQ)`nO9-_1(#gUa!nKV z97`egNo~~BbmVr6Gh$OaX{1srIO8j}i&;gh6E|#H>sKKg8w5}=R8%xYcqC||Vzuo; z=MWJAMHnNvx1R#{gP;y~uQRe7&0BWxfUE4+-q+sSH@`8T&p$4Rf-r`?C=W;U{@cjE zb%Ln|a{af_MX?be0EPSZTAgF@?ck2}gbs*@_?{L>=pBp^(s)ltdP1s4hTzXJ%IdKY6x$#q1DlDI_K>5goac}n_fIGaz+)vXNEkOay;SDaV z-^mZ(zCZ?0`_aI@)B%i;cd0;qG*b%i_pL--A=j4=-(heHYKPfxX_D`4gnQ6Mn0x5t zd$5O}Z-E&vLqz@|gpl{BUu*&S)D?S+xcJyN*Esm+_;~C5JlpL)mY$9k7M`ZELKU|x zcJ-@?%gU(u0HrC}5(*A#%|*MC#k900cgu;GIzzL(fVx@BcnRU36UEbUj=7Vlo|&_@ z*&WxCf6|*dkYbHT+8<}ienAfj?k(qaOGz~94e*VvA z&9K3W!8N<#b4@n`l2Sy|vQak|(Ks2t*o?I8!c6GG9kIZf)-d8ix4PejAUTtql*-VT zzTl_mfYw%|a}&=7kw9T1x4i4TZ)U$uu`vrd3asAI5$-PA&E;}-mAxcMR^BmI64PZ_ zDy~AW++w0q9sJ^tCRX!+8AW$zI$ zJ49VjfSgrYX*PGcEm_+vql%E2uaKtN;{2fqAWzgz8!anHhbYk)`C?O59(C@iETee# zfkM!KphDp+4HGeFAhm~1b$;o6hCci?F>WpE=$}Ua0 z@!fh;$%;32{~;bdM^CQgCKMI#x(4(XKo>_SW=Qz!e1_vvma7VveS$2uy5`>8qLsxA z-8c?j`pwyxgBoA#UgY3jvo)p`KaKm58O>;e%t)zcV^sQnDGkEDG&(gF%};=!vevdr zy`HwV#yurp@u6FX)JVRV9>G|Gxm?v-wBwf=$7N6Uf>dm}B^zt$p+j6uwi+KT!1f|F z@|>!;kQe8Nq5JRnmLu3-6^5dwnv?|-jrZ{eZ2cR>9j<}G%zqupVrBeV5cv>SWN$6qSaDED-Frq zQ3A?P36#d=e_*(spJ};XV7ayS04STFUr9w68t@h>r7H8WB8vMvXxzOyEk)geM#qxl z$DK07Hslr2IJ0+(bGC+M!^G0-y3pIf*i?G{%m%f}T^}S$@z;jGk|#~vbr5hoQ=2fN zm%)>+5D-XT!1-_`yUQipPukS_!K*x3jht#sg7rG1x5NBQ^*Xe_?i0P1OgO9$)0 z)Jm0M?$DXOqN-?QQfT^>YiTH{WYAp^u#CDp=kaV}9y(htoSgde9ofSr5q9R8r+$xXAoe3Wo(wlOo$X-`< zJO|c}*Nf%*Mmz(I=$hdUCp)ZPqE&^qitJt57skJ>upc`c-@!|V0ZI;oo5ph{;N@PW z?h4lCDVS|}QRiOf$KSA8+6fOYGSWP(BCEPjxt6DDiSQt3GB_x0sbl^|%rVR?Ixodr zbul9XWbrU6jWR9k(J}6&^t=JFVte`EH%DkcpS55}qWoW z($*A14NXsOff8?YQS#C#goHj^3Dt@>W)lO)jndx6$`8pc5we3n+DwbCy}S_pVWP+% zgSMG#LCeN*Rei&TlG}!o*T%@$T^gf0cSI$3L_FImwz+Lp%cf~93$f6scrJ|e@b?0V zSdhTA?M{&Iy2zm+RkNf4FtfUYhl7;2Wh6}fj^l#63% z*sBBhR7IS!R5!rKiMZ*k4VW)@emnZO11?>KKu8!xg(Ibzqaunw8IGpW0>pv()EC=1 zrDD`e{sjkG(}qyV)(7O-{e&+di*B=XaR6@euq87M5Na4^Yo}BBO;XX~%Fb{P*0RuA z6!Xp>OKTY_H96*x^{FoZ>h1%Nv@TNktYym;RO-T&tJUp~VKIF;jjipbZiE1@bnISs z7E!D0CfY&M8^yvcHpjAY{0Clo4{z2SBo`GrR|a0o7S8Sv3EIxR^3nFWk zBNuG}Acnm2U0X_bz{3lo!1(n(o!Fi zX6{Vgm=+3w;SUYEGKBQp+;K$bkZclm+?%eiE9Wq>2lahvQFJvT@NjG9CV2<`f~yNB zeP?~uo;?nNBlIq@Uvw{ROnjij9=^*A;_Czez+U%;^!JHCA&j@jdg%xkS1C%2@{G1J zV1}fEsdNc}uc#{-MQo^!Ie;EufDUmiS38UKPR9-nLB9OKCp4hjh0Fhmq~|i<=KSf_}J72 zTx7lkzhav1u@66Jr`@@Af}}WSFm@y9i#?(Hxbu@L8_($nuRYi%nD0p{`f>x2{d{pA z@-%GFkNheTGrHv}oe#g{C&w7Kg4>68uMu06t1GEZQN56~rl&X;S15=}BWvb#RyUsN z*+}(*tz_v|YS7$3=dgWR`HU;*+2vdUc!QbMvR2=SiiN!Q+KFq;w`Vo|>JCcLl+z;a z$t2hwjK1ffn7-cb9||@xnN1tbp^vSjXYoN*fQ_T2Hd+_IT*dPixp`hTT(VZ<1KlY?=7cdovDt#p`XT)rN;B-xnE z#V>z|mcvE45hZtpMk(*%eLHjB%lmHO;l{H=FKp0HjNR^4AGvgFCwzUo_Xc-RRHA5{yc75G ztdHV6n1R_XDp~~YR~td-k$dp%*4-se_o(VXyj8xl2fw{P@`v`Wk2?0u4|=WE3`MNa z3~PoVP*fuA+zv;ic}pNftg09ZS1VyA?x#3@NYEvCF`0~yuedux?gRi-2e{D*M@(o4 z#>P~A@$wb#rT7Z>RbT{ajal>=3u7Anl*jr>&{-Fhoe_at` zKT^e}S3zrU^^qgMVGIyB)a~8Hi|~4Nd3m)#i*&|QmOnql%rAg)(~7e2HjWXP#KD@^ z^3=gzFs2;#iz>dSx46Ewxw@EtsC-atV{dO`Aw7^G8+?Y8W*a9~qSg)s_VNM}%-8OO zsWjD1>}Yg%c^#Q_p`0)hTUm;=Xbj??2XdoVFozRM0lw;<83{s6AcjBV#D^LI_W?+`{EPbbPj7ii_p)THj=^QekrZ7q~ilkcjxK(sJvou ziE+mmolZoqyb{w_I%`6d(l`<3k;hI(SNIo8y&ux$m+c3 zN?~KFbuFT^8j>Y(h#$AaVe&&1GnygShL0&pvSQv zmBWcUpa7^B!|IX2_YW_*s@3sgv;U;>mdH^XbHuVxvwgwztKZ?m_mA90xabz_d(v3X zTj2hUr>!`6eG3i~JXC=BsX1VLqfNFx5x!nbF5da*2XzZQkLKYgA|#*GZN# zi9u&4zWOZi};%`at(sOF)~R!QApSix`*uPZ1!btQt&BawMF(bg{a`rbxVIVOU#NkCSAfSjmJ7EjB+K&`ff{rW9|ZmB&8`OS)!WCds)avLxVi%r0Ysi?&b^ z=~fiErj%f(iJHF8Ay$+AdW6Aq&@&>c?2vFxjKQk#GUgERi1Q@N8JpGh@y@m@yaZTz zb)LI7M8M!*a&wJqT0UZ7Bv?tVjZ9q5KyGI!(GIK6fsPZGP=?m8VfQlQ+{4;09-T1J^QjUofG?VXnEV)8yZ2e1HTW$u$A1%aSM_KOCIwC>5|^46jI8p{dp{V8Bct zt?Rp^5;F6`gmmHUR;hLOkw{eUuge~@wD|oXlVkKqtaA}g4;8PXI+^hV$J^6clkPXt#@tR)zN+Rt zMqwo6^)f3tSvh&>+ij+vO7?HGD(I4Hxe!(4$5%FHFL32DD06+dn6jnHQQLOMD~$Rj zV~(i##R2rQ6dpL2X&=F&h5%x$CG0U;z96UZ;8EIyXnxCUUQ@~h_KT3qU%M6hw_+Z7 z_dBOC&MR(tr^?#*vO)qg;dk#|bLxGA!Gxs9k(o_qY}2{L&7g@}j}O+o!9#hRJAe6h zMl)=mRlkLMSn-{&znI>MiifuUF5cldddzz7xM!ZgS2~Rr-WYxZsRN8a+ko$gBmPYp ziuPr7gHs*)Q`yuz1DI?K2EPky{HUozd{-TN;}V$ubikvr$^POqEXF(0wglDK_pZwd z2Evzk(+PnJixe|<3OkH~_VGo5?ZTmKY{-swxJ82rqflHwg#B1lgb3~nlGrEI#-h5w ze*j}YYWTIt0P}&f2-SinUsNOpO%sA4UzkJ{y`-5W zu9iT#LjP0HZ-pY%skN!x!l20(8Lhosu*no^CNfY&O?Qf;$sC!A3MFT>qBt)YxPsc@ z3B&%Ul_Iv4HSJbB;1Y+&o8GRrbm&4U6oO8IlwpMDN|^O!$qt}l!r$AB3{hC#0NrlP zYA*q+A5_Z~OnZ-|ZFd{XOF*)##OoXXz|Ackk8waO`c5wLKtz~UtSQO;dIT{X(MmYR z9LCk_&);jGBty)tD`N=)W7g_Kg&&%~$Z@}rM$*o@d(rIlmO>BOTuqclEG8&NR-J>>!v;BMrOOid+TVCKF=F@jq16|t&zD3t?Ra$rY66qdt==c#1f!C2my61LjIA-6D(U)?FB$I@6nh@ z*oit#=*y9M)a8qJ?DwVb?h=?m!npGY=z`c>wA@>|1CzG(y!uo}HGX_dypr_4*rIFw&ENw*$? zdlDbeLi-^nYa%sg5O_A?7n9vMUhi6zGkmWjQRzvW?Cn&zCZ%S7Tj3=Li;+ zp&||~mHnFur)gw>JFzpE%kVnG@VZF-lxPJ_E?XeT8E-#*sC?m!++&S$S^9t5z8ka1 zo&^vlet`hio_u9G@|0@Z9a6o|>VXWy? z(;>yPw!rHy>uFx|E%&Fu)7)QQXM*42Uw+tKPMX6WqP`l#ucCYaqQQtlNE2jJlL!*b zKx`2>6BbWJ?{})v(pFJc;=>3ionceBV#gv!GsT2~*W$b3F0A1AB+If*N3^lw{SG)n z>mkIuA&xI7L5zDxwD^dR1uXxdF=J2N!hnCL`<7_{g?P)H_?<2oC8?)= zq2FW!uN|k&da*j-Kz@n?UEP!-hMLr_(+c~Qx+PL>mgWOAt=v(|?vIuU`JQtwXdXZ3 z1oQV(pkHuKi}Z9;-D5#2qgU8EwX2yfWB#m<_$Uh&Kl!)%`VBg_sEy~8op_=OJ>Pnpi5n-OAO zys6FTtrV@~fv-CMv4hv#$6~^LqG;tj{q%xsq8#oiKIuGvM~AJEnpL9=4D(TN#1v$Q z7EQsvM%S1z8>`zcZb7H&(^TEq#|&s<()d=CkJ>hNqtPY6V)=CUp6Bd@Swnr48satf z4t#eWV{-wF0KZn4bc@Ri#iC^1Vz8W~SS2%xuIiss3RMQ@ImPWr0uFFb82mwYAkr``_YbGc}`FZ8W5M!1e!GxSSn#S$*Bo(d*A*i&E@=2s;WL`F52nV3Y^XI|<(!|taqBPK!}d&iKX*vD%1(74)}MRg zJDho{4C@s6;NDV&_`rmnUcQx9074B|wQG%CyX5)4DS|xQdcswBcE=!*ztqQW?|kqH z4-o-8KM_mFz9mNrZ!Tbjml?={W>=?1Oop3fjPjDW6wA+XOFP)p;*RQLlw6nt2CiS| zrYRXzK|6}J#Hp_j=s^0lK!iU%T$-36M6pO}8MthCdOpCnsGt$?zHrK`}6OR8kT}*B)M^oOePr zdcl5Ef5Uc|yzUG8j0?jWYcEO%k_9$azE~m|SDk%ga7TzAvGgqEODhL@)C}DXU}Wp= z%c8e?fOjA-^6=c+kH3b*>j~#Xw#WeUH~mnS&(ob5@Y=8qS|Cr~dk$&d8tr<{ILC0X zal^!q6%`F}h!X??B8!%G4(Z+ShFdyzUPEfy`p&6oueznjV&kW-5r6&GM>n=MwxTga z*LO3t1wm80yddZ+Z?Hp@q3s~FX^``TxKb^1g={Y1NC+32zLvnqFb?=++`;HCdxs z&t}8rjae|{llwzv%^hE^OsqdRB%9Qhh1d8?8JdfK(OrJRyQVbxNdzpM%vPBE(FIjX zQxgl+ur(m}`iZ-4n$MsLc^v_5<|Sd~7;A9%OWL8e+QPiB`c+Ho481n!0wS(99mJz< zF^Nzvz`!Zi38J1DYw?k<2yp zemZ{RX7tx;lech0qq+3>lNILHH3(PnM6IOsN+yZuj2FRmx@7c1j^_YSO=qMYPDIpS z_-g+K=Du;dAJ9Hkb3v#=?jhPZrFY>6TZ6n-9F12buNUhNBzWiYp_W}8)fb>%?MRh{ zak=o{7&FE&PGoj7L&>l|~TWN3OCT8Ukfdd=+J)aDMX7 zQISmy3Wi<%>`SXPf=g;?$Uc!=X&re`sh6ooRqG@@ioec!;`?9n?mT6;$CH~VzA+tk zY)XUXLi>qJvK*uhjt(Gz7#|dpA^&eY^MAhQbEA-_2O$1xR9i}PGg6SVyD1eFMy+}B zfsLc)G5_FRQ012;pX0yDuwRv?bejkcD2VSlr&qjq9lnebI;4tExbjclA;M4QV6QQk z2pDwXnM%{kX3L0l#6pp@X67bz81KQcG5@cuuYii9S=wG)f&_=f-9m5)8l1352(AHw zyE|;~;Ie_>5Zv9}J-7t-;P&DY_%|f?zF+P?d**CU%~RDq)jf0i^wV7(|A9Korkc#a zxESwD-b-p!Vw)l5f%?iyNxO5^LtjqD&5Qnf3yN?9+`(#ykop|U^7ON&f$AY(TvH++ld2|80kl{6*cuGcx#(6$RH(7lQ#8>o z3tSy9P;}qgR|?F%SQ@r6H9=9aZYjL5sUcI%*2B||im$a|7+Y6W5!5PC)4nITCz#q! zUOK>18)m(Uyt-T7@J!$K+#ZSpr-KpNUG9~QKLnP;y&7L+1%kDWHyG?xH|^zmGf@%x zR8!jFn1ll`!+ua3j`}l^bxIq1$@TWBa&6!+C7ry%!`x6dyqQELQ-ThMsd1$%pSpXA z_7I%Xjtc@a5BqSvG<(!f>ch*Ju9HH)U3&nQMswutSJ~U1S+c!jufM2G z5Cjj32?v17+0rU`xW7bM3u_}_|i z@(8cc-wc6MhO*CUD>yoqQGl79Rk2KrGT*UT5KVzc?dK}RxE8%Y!XYDcRyj?@9ILLc z$G!6<5|qY#HoY(AY>KDjGL8h&xEu_}V@`2Uz75%Y$hG{Qn`SKNF3{9@!pBW~7HNQ) z7lFMqd~fArqwsNGK(I4<_`dTqWvA`noSK#duf-NPQ0d)|ms&Odlm^r~2twR>zqlrP zh`-Np7IEx$o1Jx|9g?GF^xMLCAGBX${4+tbgp7`Fz)Q_FVt0#ywm8CNIx9x@xYz(RM#aPu) z{ledyUDNvHtj)Tq_8OzNb=Vr;D|F%A>vfS!fZ52~$rJCwS3^@r8>D1=)47_1W@(EV zyM#39N(1^G@aGAf^IBvp-b;vc7rgIH=e|dPYm}f-3hWdeK`uhf9Xt7aJ_-wi`@% z;4(NigMunx(F@x~GJ#swypc2QxW+=WP&7mSi6HIuaTFBoEN7+UYBFkJ^Ay`DDUZ$Xm6a#3^D6iL zbjSY(_2t6e>bpncnSOXxRcVy{LJk#ep4lm^D*OdKQcMZVD5o9^21otY`fC2fIt19E zvw2zV&lC51;W$M)BYS%->EkmfS+a7Euh_HHgrrdQAGWvC(4H$pUfLF@k)_Ta)`Mk= z6S*v8gbohz$(^ODYRMMSOvgv+HsQXK5MqIu1TnsQpa)Zqf>XT@uOWfD-15MkB_l&Z!c zt%oo#NeX;gdwE>axO~RyPL1U|N*qE6VPyna$r;V&01rl&wd?S!A(NaIE{jGNs*K2wUyHB-8@ZXqnK#{06>oSlmvu zWG(!RITrBd;7W&UttBDTB;|t$EfQavo(tX2Fh<>PaYYjWT!cG}8TM>$F!CYK%oN5f zhp;HHHIh znri(tF$QGHfG$Z%yHu;gFK5B7N2A4NpqO_hTJlD4O6`^R*ZS$KSu9-*GIg5*DEt!$ zc+fq+HKZb^Anpds33G6MbTz_aY9+V{^c5ruy|xN2B{kgxlyrC z+M)a~)j(?!yw~byjTbWDjPlt;nMb$!WR+I>#V!}B#UYn5@*^Bwibd8&eYe+64%hJ} zTgFz3TyKjg(@w6saeQSc(vYS6P6J3{Hc)bS)66g~D=4kkw@)Is2Mdl*NJhp^TjWGx zSp=6o$ef>HHpKNEY*0#s=7##lVUVouktla43OP?M_gERg{%8pkV=T1|q82F$UuAfN zt>%`o~ZT?Of0?U{DE@9Pn_rOmpbI;4b76(B^Jt@6l}=IpD?ph%@?; zJO5YkrDJGl5sM%GOry}qB0vbHIysm-s>!B{4NcPZK^Du`<2ptZR0}H^kFWboh)~*> zmMGTh{A`~KrOvZ|2sF<<{Lm}~lFUm%J|Oi9l&MLqH}h@YV0fOhXZ#rC={v+8ba_;> zlt>;)%^#@8n0IS)Vk3c@O2{w3L>)A=u{{9;*DPg(Tuqjgl*8tlPoaWzp>Lo#j&JPj5hr|-XEeZ z3;H@1n3IE$6z|O$=9|rhLMIZ>dwn`WY}H9n!e^V*mp3yfS+*ID3Mb)K{(X&)QR}nDuMyI3F4cbZ5BuWQm z;7P$Qoa$hw_Zbt7(x1Y%KJN?TXiZqHV5J?4{9@$w%GZX_-b=FiedGq-sgJ5zGo6AD zcSzYi@F6|FbzBQdqnegSr11t@nxj zbfH-wile;}H>z^u*HniWk>OacRBNk4O}-zg<&0k!n<#H}^riy*F`{!=l0aG=NV4YY zRv^;4&}qqa$E+F*Q`s`)ofhI+Fi%Hh#9ok1)SI2*8OD@yZjjLX38~u=nNM;F2=Z>D zCnkkq+uZvor}APRgDL7DQxy<_rEOKuRtjB!7(^b0(z`6+lK+_GL#_E~@;jGnC%Cxi zC8nkDuJe(DIX5~P0YlcEIKUE}*WRE>#N{PL5RDVpM7d6>l0DyR@cGdZIHWop-uMH^ zRI+!yR+IVq((3(N?9U+8$(i1W&g_C`nf?%RCl0mKXk+z!VkpRndE+hV zn<||V1PtP3=bO9g_*zK)KfEC?d)n6ySmKX^H7xz%u?Bc$=CI6U1_ zN5=Om9iu0XC`c{#d?~vHo175@F-@~>+9557?aJG%mC<;^tDpVw?)}t9>6HClC55wA zzg*)5OO^RC$}uMZz&Ag+)r{|@I_TN+R_m@wrQMgp_49D2-+e2}Qvbmn6`WeCn7EuK zYun3qC@0#~zVqx(viD_wJ#1cdXOcZ~ez!S>Sxt)hwirHYE z;$I0$nj2woIOSm1N3M@-Mn}!DO$dyx#9te z-eqf~P3@@JHFYF_y>lCe!OMknlk~o1MYI4S^39b4yy?jva>o{?TV6rTy6)2)fTlu<>@|+tXBV>0zSJ^GQJBbe!d00lEXhuk zLDZF_k!;2FH?k=tUa?|cerf><;7008dZ&><$8b<7Qw@6|d7v^$&sGX_OBz^pgaIHAj!&iiBG%0t0?`~`dGzN;x z2$ydJx3>1=TM^*A;TM!=DVJ(6%+m)dILVqr&5WF(_0U!1gUToW2f#RQO%;~dqd2@8Wrd)h=+lVKSerf&_jaJV^?;bBRATZKQoWF@zVp}Wk z4BR!cu#*Tn4&$3%WJSHi_jrtbUp8NjW8f>FtJM0Mo`8g&^QC^O_)8JS+G4Yo=UZn9 zR|!C;1h~x1H$1fx1*No^WEsSgzKRi2OJD}Fi^_cD=Tb4o7dk7^?Hr)Hce=26E20RJ zeWGlDC2_81_#>NDW2w)&%ZX;*T@@j*oC?%Z(c09N3zzn zpQYTfULPHK9TJbsq7tV?ay$k9JIM3YmpJ=4_NSB3=T``e{!12^ zQ%zus4FL2)d;(d)&)ig|YxJ1U_#hBCnizn;AQk4ohgg8v#}f_-mCx{O$U(twG77Q2 z19Az80HyrQDW(ghqD*~n^@0*L$v2;ric5|St&6;ugsqMbFP60zJZ(J}JS??$+b$Cm zhvhKnE;lv~ggnbyX6E-#k850)@2?whB)Gy9e@tN9f^Y7@lEUNz+hxF?EMFcJ$oDtD zD)S%ePZWe`Zjhupnd|Zs9n8#i=_Sr^U7DvP_wjG zeQ zCl%kZruWQAYw(O)FT*}UVaV;PLQS*GCw$9hiM+EORFHGyTNC`V2H@NXB~=`i}lct=di%{KQ`#*W)V}h+_TPkZrrLgwjEOIPVo4Xo2UD}Wun}VXIAx; z2-`pgTq6ACx{ZpFgGe%H=IX}#V z+6|-jf&ETb6_}GhF|O#Pw%Q0x)`Pkhx?*$xt^PFhtjT@)Gy;nd}HL zIPs>DKCpkPHMPvLwm|hPu#V7H^5n|34=}F=z7ipf=pk;Z6)VHwT!cj&DxQ&<8U)VP za~gk~*EN|Mj1Cj*A0lk`iEy%HPpz=Hn8zyuf@`^^W>{A$8(j|Px4v+(-Kg7ex+baW zp6cf_S=4AY6{#iLo8js8b1@k*Hx&yEd{6;8x6Gj=E%xebe-C$+tmxWGtO`3wr{r8z zGv?H&=fJiuvIt9Jp)4wVgm3+~rx1k*ZWQr_%ZnmjRP_zS2L2iMv`D*v+Pq&Mq&;SQwv15T2+lj{o9F7Z(j_1C*dc zo3o#==p2I&;(1mBY~-)i8m|#UR$`|9W@45~gg$uOoguJH764QhHn6QVmGF8aTNWTo z#VX<%Jut}#O860HMPk)6VWkub1|_|=1NU2!U88hhSOHRpTzRzzK6aD+_>!$u@{z8x zYb}KHb0-Rz{TtB)a1zlNv)--5?imN-k)H2ZD9wiYi{p&ej{zn)-#X`I#A=6dFALDs^hoTQ-6lzuJ`JEhLcx_jQi&-2W!t-`^wF~!4P`D8?K zz9UoJ6G{i^C&Q}&*p>|94aD4Y)u@3HKV%kIHcg{{|85@CNHO&xa6{>)Wzi;#BgQ=m z7_S;rS`VL*71aMaMVxexuiwbt$AOTGn)7pzL)!CV-}t7hpX${Z$vH-^V1mL1jXr=ofeW@CBi8muOF* zE!NJVn3!$2lVq||7fNIlQFrD$Fd?B6!IJfuXyarC>zFm-lBg?3gyKJX(|E@Rp@YlYHE&*6wP~Hm2;ODSUH=;C zFM1Hu3w&>ULbl9|CPJNFlvo%ncuBIarRg2fdxzEK5NEqN$wTxM#uEA#w& zhbA#vjSO^(jWZ^AVAT`aq4Pf`5EpEBCA_B)F*@n;>R)A@6hai~jI!|RUM)kevqHic zQtx+2C^G`HEulh(_uMpzgMwi})?&H^%wrC1F)wNI(uVObs zOm}twO6gGiN0~^wAVTV!%_|NrAD}N51JBNvb7udjP?+>JQ*_~8MX>qehcU&5+0_lr zA?3VJ9p2dXN4pifYvJux?DMPJ*NFaiH}1%ZrOoHCu#9RtFP@lD=N_X0G5(Z_a9-0s?gdiq|B`>U1m;`8`oibLcR2fTc(IXZg zdPThFy+&6u5*x_P@#JyVQzeF7X(3i_31vwhl12A{U^OcTg@b8zFh)pY&HA_f5yUI` zmO!0$>q|GOzUG&CZPMsSp^9fx?5=Kh{5u1N^*1R$?K*I;xxVDAt$P2+lD^s9C4I;G zvw1hNrzBDo6Xnvr*)F4als{ipQ^aG81?=DW!}KfupimhR&Ot1y*~V&PxszR@2o+po z`)j=WRj$(Pb0=_4)+1M7(GJ(lO>b1x`%qRaFyPKjSN z@I@zdDKyyoEgqZYxl+cz9j-4fYRt}PP-TY6s`16IQ`3NVCTq((44yp>A1<7xniz>C z5Nfu}4wLjczMiQH#z(8uXxh?kk61w$pCb4P-=%CP3kp=zN3p zgL!b1Cq4}9Q_52?JmL~6l-I5R_e^!7!7hzDVLmgabmT%bMmqrm{hv)rd)^N_!-^7S z_?0xHvfSH&<@0ZMa8{9aGe3wI1pE7ilhl#xOPNq*SdX0*YdCTxcpdJ4rVqct;33M5 zYhEa^F;0ttgIwfl^ijj0v|UeGhGw3M-t*r@^n{gyNrf+&2il*BdLX00?C-KN;-qkR zQhXOI`C3X?4LmfGC&}&#U+TVV_fyQVG1WN zl&M#0wFu9qmcz=3wXBYS#c5S1eUq0dBKiIsa>9gRkDpQzturPL{7z@UOt_tD-I=0{ zM{rvxcucd}Kj~9HXd}l>idEX?xcgjkx{dVFK1Thge<90fPhq78z6R#?D=4KnA9?)c z#FD*Xyvs~e#62l8w(d){-akSmgo|Nik@04X5!sHr!slqoDMxuopKyziaEf^#(&Gv* z_H%T~CzCxGj|m&bxRVB#TDaB;PXQZZGIUx&;A7bGqAV1rG;FqBli24JEloR=pT0|D zPJUrGwM@;*cNiLsn(QY7UZ{kYsPEP>MX1_@s=gEwi!yvbzO6hgTYc<0Cu|5P*rzxj zg(G$q+2XxW7FC|B+T6<$%hlu0`XV)ZrL3f^`O(u{PTZY5qG!7A(DiDZFIU{_orh1Z zEqLr2s2I31DQ-%(PJf@)i6X>x`h4qZGMi45zhMn|C2Vb-jDP=m1a#>(+&ZHWS55E! z)#Vu1^8=U;C&{eIB!&ssw&Z10=Es{*mbsHM zr?5ZYG2IrRhMVAz?0uaR6Ko0|iE*)r7(L$q`YJeo>=J~ zIICd3kM;hRKhCMShZ64*y0sYev%za~YDn6Q(P`j~IxjnPW;15am*=~K#%I7tFF|ls zX7CRrnlv4zw8(dWl?bj#o^h3KHx^w5f^mE;ka~0oc)^_6S(#H;p2Z1AQ>H>qLX%hs zba;-9u-jA};e%Ls#UMfc4kLs5%d=>kEeJ3PS zZe!G?zH52C{06dX?*ZE)9c{&tE6UlG7Wk4U713OIXaAgLmqdI-BoMD*%%UjiLe$14 zQ(d-fax!Bw!@p%FIQ1vYj0bj`<;^P(L?O($GlBAM_BR(Kt{j--p76J8Gwt9uNgGn> zc!jO^O5{RF$yfC3Qc6PUF(o|REgv6HJlK8t9oeaf%HkR*bFBc#R;!J~vLv!_FIfGb7Y+!=0( zdL7>lUzc7F=kU=~IO4PvmQM!vgVC2q-?$Hwxr-B>%HP&nW6d(1z$N_y*8W*tUC{hx z1i@Z%; zGM~cR7zQ>C*)&6jXRKZ}JS@V0ah?FZu6{}J?$GCB?zOi9;iUA1zDocdrko<7p!c3q zTUq%UUx^=Xe!157R$2GZE3m-?>a2eI;poM_w^Imc$CQdA zWT>E@;&h+6bq&%QCFmt-b2ev-dF zrZ~0l;Kvqp+!|UK6%pMb0v+ReZhLeAYw>x?ly0=Jpx1N+CgPs_*pO^ZyJ`^aEipV`LigG3ras0upGA94YVEJ)Sjl%e1AoirZ2P zuH=~mn&{c@1OhIvrof@oV;2maOCW9|@y|$==mmP+%!Ya5|^LuIX zA?Ciz&@dgyH#$CvJ`y5Q*w;6jwiT- za-$>5T?CBW|G0p8b)c2K4O@ASoK}O_p7U-(?cA4+%8@1;f2<6K>=7fwo&5D#Is?Mv zv$zFYvsd03%@y$T@NaE?GXEsL{2?s@bIZpBj&RQQIPR*2B^cTfX{Sb73>;0}%_!Re z`_Ba3Vgf3qck93?(bmDnMzi#T`*8H}^@U54Mv?iooA46ud_F< zvO`svn&R}1USq>7KSLv7o#ujF5g9kT<)rv2o}>u&wH5|v4(-@7P(3JkvbnN^J3QkQ z8sTIAE;+i9|3>W2>@%VCG<&i?+Hq-YEM_yysiH6l*0eZBu>gL5PP zp)+B6fQ!idXl4u9nWULW@Fik3Z=>T5V1Cs9fv%dD(BFo1?3~p0NaTO$uqyO~lRu0V zO!CT~)Mjc8{BM3LH4_%@iSnv#FLz9UQeK5n%1h|Kl-Ei(e$`1%<{!2Ae4RgoV3!52 z{}|ve9sMIiFIWD9GpdY*L-{pC)+@reDFu0~5f=OZxlA6F|DVhJo56&E#fAHKml?>% zTF(D&GXrU=<^R2X%uVgf-wEbA?%xTcI-%c*oH}`w|L!aUxvMj$_;s~?py5~uxdN&hVd5@d`W17g?o+Y$e+SA{ND{YxQH z{7K=RAOWQR5BT3X|NmQQ5QQgjvlJ$Tu=%&;r#Y1{0Du~FAuQitmK{~0V19_tAK>3p z_x?pz`+BSYn#c#~YyRz=Cr%!EUlDpz|BHh*h3WT@n`U;3Uyc30Bs(-~!oNT!`ViR` zx+l;tIkZhZ9ncAbe~#yXpQ4rqkmwfDr?(PP(E@}`GJt$*5&3=nycGybYXqstp@t~7 z;yf98Cx$|r54Cg!eP0FtV+ZkJfCvL>>&KD%-jKlJ^TojF+%J*NS~DdrOf|VdCw10(!uov3KQkF@`8H& z8MLzb|0-H%CwWwc!rmqN1d<*%If#XVQlQ^~zd;+>PY>19Mfe1Cqgz+ehkj9j&j0{{ zzslR22d!`-2wFGW?=rvc2Eu}ip$|$0@#+5Kr6i~HyPU zMf}U4TP1|DhxfNJogN_U_o^ocB=^uiDMA1OiN8E|4^@2imm*0$M5a_A_CJk>r56Z$ z-0-9;T#ofCB&JEB%(X04U0 zLW)2qVnGoUWkA7TfPkQ&fF#YR#1at5q5da=6thz&@MxlAKRQ1+0|Whki^_?>{A{eAjG6um1j1C8;*g zBAUbEa=P@8`a~rIN(^j^A_V#g&9cEO!K9cT_^+pDS>~pjntTNO{r@85M~_h~Ql-}` z{;E$MltaB*mo#DtF}vAiZV1_;ICO-w#cdyo_dNyA-82j5837FyrMHC+E$PGzU+XCn zA>O+e$s=*+vvvOYEu(M;7#j$_^h6%|{3cR9v{VSyBTk_td5ncz$-N90^pdL;o-piz z0U@^Gk0TMvK6_DY-Dy3hH)!{p=n{1^2>#oT=w;*LKL9i2tW(ZMvxaD91M)QZ+h~Y- zRp35u8$Q_Cf>I4`+YD&JV|GQk_IzMC)&mR11qFNhZ#`w!VM*B-Ac-qTT01fX4x?fe zPYW&zzofIImj7`}QWEryE_r_u{|k7w9cP0{vDJ|f(KRd#f`IAzd(RQuxlc_Vxav&) zH!%;{C$D8!5kH)jpF)v6XefOde?nm~W;5+bp9q|SIoUTU8xKAyGEKExd$sh0l;|Ke zyEWVgHa!lT-wy2=01wsXIZbQc0FTM9>Nz~s@+`3gGbH2&b(lX1RU~!w_}6;Bn0Ad4 zh^;PhmLS3hQ-`!t%t;^d_pE%j@qqm1D7CVI;{n)dS%vc*<>ZOP3DordX8X2EV?zT5 zyN--~@~_px9&}y8MfI=`4#$jYIei>4-N1+{(*(J{Apd{_`~Sc#DH!bZKhSBi2QLEo z-zRh^6o~(z$fPcy)9@dPkRX77=>LNvc?4WQ;+FlAAX2DB5~n;q$7}&T{h44vtj;Aw zJCKrEGCN+eppvjoMAoEK^cBr|!p-ktl>pJ;VQ|EL5n}T-`7Aoz2-vyl*;#=nuj@U4 z|2N2QZeB&f0BwY5uJoW6JX=LTT&A40taYnt`IKYc?@gB>Do387@fSXjkkgEp31&V3 z4>O`pT`m%p{H{QgaF!rf?S)RRN$bO`%X?>=x~sI{_aTBYc+8=$`_{|0WtgCL9X|5- z^#@<~T4t?9AGZJX`T$SWk_F3G?TSDnNLp@^DFwHi2uMpQA!jM7W>2r3KSK3DKf?6U z`rXGaRZ}VTH`Ifvc*C^UTH9KVn^hw15_#VrfTMk^xup9BK zsgzX;DG+G_T7JG*IZupHA#+%|aD~u+WWav_hS18* z*y7aO>0a~?Bg1LwbD#y|mYDah(O!YynGc}hO7by8cF8QH@tWCY4PP-6mT zq~HujS7y=b_rQ3wI7aT+e7A56lp?av&+^Q@BDYswO>{E)IY=^RhqUQ&BRrZpg)5_< zP|lcCFjKptaw%?_kyVKZtdmWETCjv#Siwc!@k*p)h*+fG#a>AGY%SPC&q4D8YQ*8g zLAFpr_7Os41g5ZTFW5xdAkI2ai6=q?Fhc4hL|O#L$kqgNK-3gO9>GM+h>i4M9kdV$ zUh!q+8#%zr(t@k$;yHe0|^A=ga8CY z@gI2JW1$27m~5$|eMYLQked)6cq(y_6u+r^p&DuCwyQitD;`46 z%aNJjuS;8XO!AmsYV%4Cd=oJM4qOneMAiiOo0M5A&y`}flC6?DMKfk`4;xwWu(=1f zQH4`*o5wn>ya}7qYer*ZQ<;&gbiKkfSt*=K0nQz7i7>P9nx@*75mzD_Usu*mY38Fx z%T+qqiV3;LbQ#fxJeyr}{Y*kMFJ~wSykIt>T!u!$tWDOo+dRYyCq4A6PVyr}3vdUh z>7f3~@$`Aw70Cro!NhO!6>CrjmmD_g^_UWnU}6KgFSzWrgIx% zV)G%*+56zw2utIz#jxCJaRLS@@v;nCyZ(&qJXJw!*hW&-QK`Y=Sh?=3xlEmMmGpo{ z2}dH9X#N;^v%GpsxT?&_&6~MNWMctvJEUH5@3vB$*{FKO#W>L<>iquR*|608jkK1G zfB`qAtK^k5yh~Pu%uj5BmS(CuY(nBsMjAyp_0E}_+7M1@<_o6cn&#o1`heO^V}ecV zTBV~D&7<8>p2_S#$u2N77m?5$ZE$am)|dQ1oBJ214u4LLN}-?hfcd2+nC4R~e6V<=Oi`!Y1zD2Pk$)oZe^RZc9b}+838+kPwW&LtqrbM`L;dIXePcwzkw2{WRrXX9x5$cN&=?d?v=q>uvG7e(uG$f9Kb zMuaf+wrf-y8t^+T_*t6n9!+xD%c4y_y4bu0%>4Mq9;$UB~GCrrYQAbH*>3>vDHJQp$!@ zOHsFzT=*P}efjmbylIhF+Gp9)>I=UU_p_wUjA{Tiy zp@5Pn{T`oJKGl}CA;|O+zL~B2np9ITyBlwe;JGUo!DqFXGM5QJ2a!N-kURIIOqP(_ zmQ+ErX#;hVB*GS4mN-;_LxL%+gM|4}X9@4qN<_apFGSlB+0)?1>KbNQZVZ6W!5rLb zh0L@yk!!uREq9`73<-*wxOMa>4y~Q>=H?5ZPtR9{XdslrBTi+(8su)6s?d87TvO?a znQE5M9(Wp8qTd9ZFU#=?HMw(FEz#caDkTyBkt!sT^?xN%XoAvOyqcvvg7}MCc-BBh zY+HAz2p?8vEHUVCcy8=dS&xOour2atE^{HTGBMj;p16C#U1`DqGLXa6%ib7mv?;bS zddDJgtH(V?DPhd_cCVN#d6k2MQ(Nc&JBA?=eB#G8%yR@#!`qv7FfwFv543iD47mE> zMDOa0XpykujYCj2p}c6!i{ux~VD4>7HUR5M7YmSgMPhMDr=$aAHRxBdXD4-gCVelD z>W%4x@hw(JVuP)rMfYisR4MB5K$f>F-SnxFWc7^b*oan&8J%zmZJ+IYINIIt4rb7G zBMmoof+qyHoXF)7`_m;=Zzv!>Fr^b3pJaC~O8|$lhkXjqsV3Np^2%le&=69Let?!H zUE&pWvR5vu#BBzV^NKJHa)OPlM9Qv(9l5BT&Zbuyy7l%!OA zrp@m>t2+X%y*wY0ZsCPI0Q`4_ecw5Xxc^s0bBL0D5m5p(p?%TL+yDwb)7ETIuEg}G znkLY$GTWMlwt86PozY011HUCU6R^6wNatn)MXx!X935>btt`Fwo8>OKOYCAxO` z#4Z$yKS}|tdx@tCG7DcB(ciq1ZqKE{d#^}D_}#)ghzp-*4I%IN zZ|%%JrE`ZMRUrVS&X0b4 z+r!OBPp_?9SVnh`xvGUWmnGTrfvT@?8-kT4+MkRh?dy7T?8ltk=gqQcvZc+!>=8g? z1C}V9ymqJgm7n;cd1p&`y2LiYZ8~$QN~$@oxv@gh74y(F(#Xhe(#Mso++8{vYh+-L znQZZ~&vqnJO4l)J=MIyc!8|~MiByP#MP=sdjju7J9BS*0Ege}C-s6<4c9ugb zKOdJBHcM^fZn=?Ai`p%8^%yrq1Ez&Krm&Gc1!&lPVvi4O$IZ}&;ItZc@{mlio%ZYm z&xz&T3SAWe18=LUw}Y&LVE|;hG#s4tSoB4(c^REw2UBVyBZ;n_a@!w^mkB^J{b%&0 z`nDv+$SKf(vUOMpImTt<6JJFI_0D1uq#ONZsQF@SCiQy>Mcj-3Ql!nlAG{w$GRtRS z322T#t5%vyCc2ZtW=cbu>pz*|7>~>)M?{7YZ{*xaatm`YCIU`Xw;o+B`p^kvFh@*x zn%6q;f6jbpPZj)DaT;t6?FT@0K#Z~_N<@P#k~Y7xvg3yYnmcPIC6y@gmC6L`h|z__ zNHd=l(~n0nvi}a6P*I#+1a7W08BuPcq#5?D80G?L4 zQHOG&>o;^!WZJ5dNxx`2)38ZW{n|@kRoKDP*KTfU*TY|Pb(#QIV<%j7BW{U5gTKoV z_O@OB_faU5xaBDHBpxtiVm_ICu^qDRVk)wVw ziz8J$8!1R!{RE^+3 z3Hgg51@RGU>A?mj;WP&fmIV*sfl}@pK9&IO(tp!RXD2^j^=RZRenRToHi?Xle_}%O zH&BzF(}-*_6ceCyC-m5*4o|JP0B>GAJ)z=+>b2WsGlq~7_XJ(wtbF35rQ#`kB1NFW zS-F^2Jig4pmsf&Euw-YMLY;Mxp}Pa8kfE-jc|x75{(*)?^pdCe-aPsCpf}z>B16gW zDCV5qRq7mlL}DR~)6{Riu~=ECpDPIAAjwWPob+bViVjGXOrhUs{l2LpQ*o)wh!n%m zC$B4_xosZ@j)CSkpBn?&V~3C z#>{S@P5@BjXT0?irK;7tOic*v)Jh{9`j9zHK#U5Cch)ajL-$ubiMz*#5h#5;`U;*T zJc%P-t;=C>1fQ6e%q}>!?|JZX8(oVPdFl?QjnYNMfrKdD`$8~4pxTzH-GnL`EiF+9 zyMH-0<@O>)%=383F4rZgaKWE~$)ep9x}NH#g8=Nh3DGtOV6D`ZIC$-d)!87N$1_l> zU@-W*OE7XuVwusn>wkwNpMZ{Jrt2&gc<5>!I|`&kXUr0lg4UOFpNV70+hGA;RPmdF=wy>O25jV=QwNny%~ z?Ya2BMTBqeraMFphN;RefCC*Qr{Xcw_Xw1ZC2#=^HwN?DlnJB>3*hF#Twm}j{slbz z+0J7T<^nMI7`Pqd7VY17((kKc7Lj9CrBkog&^W_{qTEZ8bFY%RKjpN*HADqERLHrq zh*iXfccYQyl!ja37?8V0(wxZ0IhE*6-4v%OqZnV7wJBdou*j1Upq+EiEa4j8FGAp!xC0Gw&Ib7Y<_GGi zBK`|`VRYUC0U#V4mpH>8V&)6X@x(2WdlgguCBwSC-6*(w(3(~rdV#V+zj!HWR`Z5l z8Re-xre8b{QsV_*#W4_-C(l@?*Lm#~v6`Z&O&V9UfFp5<++5_is;8=8vMe7~o`%4q z&DOQuqz%vG+{EIIhD0?m0Sb6V_J(^Vp!8C9@;-&0P9FDRyb0KosLQ|8FMde?-~0sz z8O*rA?#hAK*3-FkwQ*7Pu;Q6l=%^FpV3Q_roL3f;CTR!%VNj?eeLn6*Xwdt4vzmQ> zg&!z!uxWzyO~84=>9z>niGH+YHK?=?^vbFX?uTdm$LJm76ZsQtb`7v#-YF>n^I{*k zm^Un`yOpEX%u{0tv@F__SnYi~jhgz%jx`a=^--7>xO7hy27xg4>z@K^y9dN+sqaB_FdB5pzcNaNvqf$ znp--xeDpl0WWX&iuSma1o$ilw%3KbUpcRz99f~#|GCUttoDUe#wW$9?oc_gx_16Na zN?|D|pIu^($e2Iu!6iKF44H*->Cc7Q^%q9r8INNWr+et6V=w-|J(=t`65D*)d0GDh zNKH{5#2rxk4q|0qXjE7K56g3t;{(h3AGE)Cr@0>D3Ow)rv{$cf=S{I&Rp&O(PgoJ5 z1|hO42>;p;xiP?rl5dLSmnwm00z_`n+w_tTqz(<_nN|6*)uUJ53vbp8P6|StgrxtI zCj7@GP+oD893G6-F3sZ*BeztGs6UO=(+6EKn&GXrQ(yu9&V9VaUgxStUSU=izHLkG z7K|f(yhT~t=pV8V-nra2$$Y~rN%`=YUr@w&=wQW>mgN9(JXRuE+m**=hlDP-I{5nE43b5y_FT27L?$xIte-RFAJM@&OS%J~m_$3&?1cF7`S)0!?_ zUHs%@Hj0U<{(U4;#F#Ek0pAE~u_W~mP7uA*3wy)ot0*+!{y1@%RgCj>ke#Iy2Lvy( zOV9B>K}rDFH6`N`4CX7P$fQkLoxgJWE^QL91sY7ZJ2(zLB-$_mhHE@O)VZ!809d{w zzGcXh@b?T&KN;Bg@(^|n(1_jdA$-RbC(aAjYSTesj890zyfDMKr}`1_=e*xy z%-H3*x22et1g(6rkIjON+*WVKtFf3DC9c{%Q1t-VYxM#>(a$%*P>Bol{=F=EWIoL? z7WN!X!m(_@;(|dL130+0{7kEaoLvPaFPp3~h)MW=o|H4ydn2YJ6=kFCWmMLw= z@hJmpQcf|F)d02mpZI|`k1(CWeJSSU^BRlnO6P4Q%fFBB=I=_S-1F1esJD6#!PGp0 zWRSx8r2EqNw^L&E`&87T^4yD-^3ccbB0;$gMCfUSDH#v&o;rE%Op-#h`n7 z{J>Z~Mt>mw8Zi-faNgKJMQjK-zy2`A(?{S|gGqh(NRD0)Lw^4i`0tZ)6MzNwf7#~& zF``M4e~fes1`rU{{}o3dlX6E804bUthPW%(KRKiljm}QtwiXA!p@4ZB%~)h4g)MnO zO7%76mBU&uv^v)@O>;PNn9n73EH&xvq%h~f@h|KxBjDQwWwz!Qd)^p!eCFjo{5&i! zigg$7!}hw~b~ylho(lfg6OaTTjfb(36C?(-D06_llSZgy)vO)A;&k!-6E$Rle1mN zSBhYFsULjI-5h~$6JP$={73_kX5aB}VMFbdfrwGFaZqtcc?iIgc+3zX+la0ls+A`@|W9vHK>-zPy1FB=4eNdC~hSjee}r5=HA&rg^C! zEX;U%zsK&i5*|B#@e_(M_7X|=VvVHTn=#$cD*2)tL>*qei-YZ{)_0dNvT-l2wU{+K zSUQfSCVb-$k%|oUkQSN182{~R-qE&DpOSMewVfK5tJbKm-t!kT@I%YAy}{o}#vsHztLLdU!aLyf@TS*Xl4Z6{ciV4->MPa{x71nUZp32OirJK)56~zSmwD9X zGALuI1$WNC;p;3r96t{nF1Opqz;uDl$l7ewWSWg{ZlSI7+Mv-Dx#jjB&}5%F{BjQM zQ5ql*XjLcm!$WMPQKYJ#@YwF+s5#?v6@D!qIIrR7l1MwI2k_NtdB-Uk!o*^VjbltH zI!a&RXb8!-@w7!hOVz`eFG!~y>G4OT!chf3(2wpdwnURIZX26r#G+pjyP! z!Rg`yXD!<5v`NczN!sRal?HaoYH9kdE;A4HirR5evx2vUCzzC6Y^qA3dVM-iD|z_% zVeOeOQ@U=&0U!tV<;5^)EN%R_sc~2F$0K&sYBcb5-h>iAixEwm0xa}Ot(N%d1@a># zGRdhTf{14_iAdmqxh;baB30*oNkU+7#z2n=%fvzDjKteb)3Q{ay?_s%NCrIfcD$>d zYyz*RtBU6J%JXW&ZC)L6-ye7^WEH~}FX&<`-qKs2GIBBUEqll_sNCGG!0_GJtrju3c zu~n+k0OP30n)TAd4J;L-qVmCh26TY|g$lYO+D3+&{Hk|s0*wdf?efF3!rDM%%_d5< zFJUTw>afceaelFMx3m`x=liQj(6l2C=X)J4OQ`eo;PNdxa~1@+MXf_05TvGQYRCqJ zfdda3S*OD{l6S=SqDX;>wY-L?9#WMB$)>{@KtPA+Jx*z4cRWLl{Bmt!!i;1=NR+yJ zuv^I?<2!Aap9MOKJs*{n_wn9tr4IO55&8GdZT0B$u2uNHB4mNm0|zfZq_|m`+Wk26 zQ%GCO3^U^}O9=+Snv1GfCC1!&>Z_+rOT&s!ZV9~;eOd)Cwa(TeGGWd#thaAm!p!Ix z;HAApA)~9J-Y~Nx_T5q4V*O|&&?57hU7s1>;)Ki~59;VHa92Qeo8+DG1+%d3K!)?D z;?UsT1M5fW!JwkrZykL!iRODukSelrnd%A67Kimnel4Ht)%45SFR!XKer?IvJ`@7X zd=}50u2$u*RQ#p{)!XM{eSs?>{D=D-fcLw~y!N2sX3i$XsaItL+@hnWFjY@kxm}|s zFbdSW;%C+|N}xT19sI?i(m=JmmmRZBogXc@vf$D=<AL27DS#XE9ENtTAMLPm@s*(WxxBKzj`&mwLm;EUOw zCZTM5S}G|3xS~alWEje7Rz^YSvDjX!&P>s9nEkA<`@jnPExk?MWWGsdfXlyPCn+;! zv5xw(qgmfbedNmDN$ruF5$&SEm%I(4w=WwGnv+QIqMF)B$LT8#b|{^q;fB{d2Tx!F za+Apjrt>UNpS!urkOx>7|Gl;f;7n+7Uv>oY-CsN83}+Z$MO9ra$_U())6u z8~46Cqg&>Cr4~M6Hg%W2-8MT-yO!aF>Y^{1GdJKe@xWUP!IY=lAE`xU(o~KuGkiKj z#2U5PfIaPyCG=iT>H#_SaSW_|?;9(l%U~Ic@PR^K4HBJZiVMY~Ix)-&An`EVpt~G3 zUYHNk{Kd|g(1~~nhrv(hjT&%y$rNXT8`8WqgOImbe3t3kVNEnDw;>eOJJSIO$f|1@|9JIcJ;2!xz9-(~X3my6NKEr`60j(bj@O5s@Y;S%=t!9fe z?nokfD>TA^zWt?JVAV`$Pdi3;YI#-gs$bq7eX)KPwPigQsa^W8GQJo|T`QdJ!pkAk zS?Br+B*cLsBVTkxg)vu{G?9@#z{SG(_N#`wj#~|)!Z|#H^RQf14>?!5uh3U&0d?By zfH(hn?$Gyi0z6_J5VHI$#$uS^Fh0EIMIDo;$_>5hW!D-s%t}`n8JGHSG2nHb#b$?$ zQnjK%kuD>1wA4@R;HzEaYDcfJBHN*v1B@t%P$JBuoM%2Hqhc3 zFt&H`um%GFusyZ#pKhW&!A4X@Nu`R*utc|a5X%38^qCyoaop{{-yY{VqG&DB zng6ZvtGF%oF{V%&M&Si*a!u|lku{F$rczq@d80w43FK&_E4H17Kt=sNelT-|~QjH>q} z=qJt{cmPKAdlW|f%aI!AS7vm^FQc#C>^mj{|s^TlIT!#@mD|2jvDF2X)7~NQ&(;WAxp^{TiB8>EtfQzQ8uG2Y2+V z2L$xH;jZI4kEsJbXU{s!vTjeo5KP2Lu7KULL%`qn-;r?UYSRZnGz5d)fr!+enjrrvzEQBwICNTUA0Dob7ei{zKB*8fE2Px=hYDNN$d#j2mdgu3OOpb{=i=OxF z$bSPFFums^X~zeQRqrspG+}iW@7%owhgt8bf`4xlZg&;;4uXpONWX zmH@?m%uacMQb-eTV;W9t@}h_ehJZ@~mW0OVV7Ah+#ZbZgV^4>@fZK0H0|$bCnkOrA zi#tqmOLHu=r}s1zA_J@#>Zl<%zZDtcc@mSzBg>Gtt*nCh32)bVLunCp06Z}Z7kakz zWkNbKZ|<^%d9U!K8Z&hZX~;*&@}{BCQG4sA8*1&S*)Hlp~zq}5-=JF zd!?)Zc|{bmV=`J&)o6afV-&oUDpirSv(40-F1^`wC%Qg9p;aY8GBOFx+CsB83jEALx6Wd$d^(7jOrC2bri!yaB}BMXDU?^ z(`Y*yZ8}+6KJ`E)oo0d%fO?Kb4zMboc4fA8Et51kcM3rF+W16ev6|94RkCBoj}a4( zu1C&kcr<1AEl9>}$=>=QzpN^1MUuy=qnju~kg7F;JHm0rTZ#`YAqI&==G5@Dd1vnx z5sQiu%6c-0adCCYOwKC1P=?BkySnl~F6p3wvf0by;78J)?_J)miP2IjB44(UfW; zN|aUSkgdxj%nHQ~E#h(|)76QL=TV$S1&Q#MR4TA>*c^vtAsP=M0GfTE5{c%#Naj@; z%PQiAN1w~1v0_SGtX?_Dt*i~Y?_^`za?4Wg-}yQ2SxNG{=$R7LNVFF#LM(uP?(x z%RpfMWZL~1I;XRR9h&KC&a{r#{U$-U6YEW_$Z184Zfos-N@v-k6RA;-Ofzq-ZMQ_B z$_1H(2-|TdK*EYcy)g|uK<`ZH0#;G; zsZ%@Z^szLtwbP96sReCGd4rk74&A-j$hu=bVKukwLIs`T(9;Rgy>j`tOl^k(s=z*u|>pw;b3y`g0j}f z{$=;H>2^94Ks(~? zk>L5@b*hyB!A#C)xJIcvkoBon5QRCpi~GPWUqTcC=yf3E<}^)fQv5N6F6p*mN7KIR zcU|PYJ)T%Mj`ttKE;_Ih>C7K&=dT&FtV0hO%c)&k@rH*^FL%n>-MBr>5n-^{-fkK^ zsm{T6*VJ5!?Gl$}=&7+kn{`Bf<xR>>0~pZuspeYo$xqt$|apns*d%?R@vCdoQh*JI*3!Ar~# z_Y><*IgsZ@mL)`ni3JaV6{F-pz*a+D_1$PT=eXnD|f&#eW)77^|Y3(6-*mI^a0=owsE~WV-O2 zT*r)fa|{q~4{U4`TPufipq|!HNcKjTlBh_(YkqkSiab+)XO!}ZG#jCP zI%v;H>z3MlfoLWMPO@ZS@9fuqJZho>T($o~PQC9+c+DL9j9}jj)#9=$sA!bA0hfWj zWGvzAPbnW!TmNmZlahjfrfdrTM`KuBEM;zzZpdy!vZ63cJEyNK^01_hNWaq7<~Py> zuc$|A?Jfpq5c{{yo86<`Y`zJ`fN?t?xZ{f#C%;#dqsNk=@@;^&?S!EX1g4oIzzHJl zx`%M3z)P2Vfj??WK56Wv_XgW_HvppJ{OXZ68XwinuE?d`7=uT_BBa9^LnEirYh|8& zt1*!^5%Sk~%U@l5Vf;{RDpI%^wZsA=H9Nesjp0*;dU_$ydRttqz#OTksfv>e>N`DRNF1_E<^ zD~oCuRxv2tTo0ULg$XWKv&vFdd0REjjwglvsW+-d-pG8)ftDC7@761_Y~Hr#eB&6F;mSH>*hR6koMRpkyDuC1ksUeXth4q4MOC97&6|O{Ah(f$bDs3d zUlZq76<%M(ms#_~ORJh*2?Vx`u$1gxRY0DUNG4-psM-}eP744NpG||jf-%8xgg`+> zei`O;B=@fA2U&{;u?XhDv$BabYx?+YPMFqe!@rfxoNC(?Dm%1j404FXAbi&C3$B=8hE*)e$+EVUB zPHK(RxUY;{yXQ8m7~rlwgqvGCfJb*Ytd4AJ_sTN6+Xv_YJBSE3;iKOjR3qM1y#o<0 zhIXhT+rtONygSN|NW5!~P`)pR&8hmvGOGGU)T#Q$IQYJ+k9hf>4S}QYJs}7X2~$?d z;Vc>H zGX`8@qXNXcDbi%o!ut<(?nHcxT9@%Q#N8c~WUnGzsY3XO4l9TbTJa zQm52I8Sqz?`7R!_Wq5Ty6H^nMj|pZj%K9xA$^o*ABkh^BqvLi4USiwn(P~!FR}!Zu z?LM3r{5B~+*>s4}le!6JgnxR!L`%CA^+XeA$FhE%lBx!9_>DIdPtUgUphs*}PFi8aLr>7K3#7>&$P|`Kx-DWLfUpIeB_3Ax8+4v|8 zgaJNR8y3HBZw>Vx&I*h>lLvR|$azkr0u`ra(~3G`tycFCW~ljV&qiW?{?MOVR;m>7 zMRXsuT0Km7Feb`#4U;U-pUH$1T&5^aPJO&WmOG5UKL4A`^~N~|oe}xDs?V+Fm@kJX zzgO<#Y<7ktX|{$VFZg8g6jqzWN}fmeWCSF(g1sr}NZSaWzo}H2k48NJ;SA(KxsD~G z6Q^F2bt|L58%SIX%$p?_A9Nx74A7>Z%z{`YtCcSoGyQ?_BTiUO8r?=#t&=VC1%s6$ z(mba*jVUr-*3!^Z<>V&){(}CR{ba>>))zKgEjK2v7+01uLDnjEf&|=g z#L}d$7>pPrUh3uZ__)Z9HWVkRu7G4E)11(x!bBUA#-&1k#V*S2QopkW8=R0IGK5@1 zsO>s!ulRiD;i}Nxxo0G|x1vw&RFpYVXSLH=mj#Eu?}-#_6r0YcIBm-u=q%`zM4Km0 zLthDzx4=UwL)#XUgex8*swBq7zXDiG9^gvuK&)zwQ0)~#915`kMp00PRhqvs8lbjB zd9+9Po1^?~F)8as;0j4qfb^b${WBByWW`Pk)M)EN#Pz=-YMyNZ2Fc^;SF33Gr0L&dA%GXsA9$5r(In2N4T)j67gfq>LOfq*#w zuQ3BYsc;l4X+jMiu+rXRc||Yf7rr&6evze&RH6t)L&g7Y$dd&(G2V$wk@f1(GOu_Z zB#x-E5yRgQk@wNPuq6f{`EF+FT0=!gE+L&CWgs-AQbd z;-q-o+y_YpJP--~*8Ytx3wh6UQzXCs4T!rQ)s_-ljfhK>*`R?c3Dq)lL&3HFvhe(x zOGd&$#0wckQmnv=Y8}cNrpt55*6R=#A`0ua!}@G)%h$-BuQC2pPEJgKVMiw_%P*>x9?N7V51hR#Yp zVR3rPk7U{@!Wggq(K$GQKO8%_1y^-e@13z}5AgVP`&l^R$A>!3dsWJhm7xwco^nGT ztiBpUZ>;~$`d2jm`A1=%A|qKe{>6u5F!}wGTB(i{Fr`;88bSu2fXb_g!)TMV;G>g} zsg0B>%E{fv`flIVXIlv&SM%3tmZqeO=Z^@Bvr+|GMP(Rw+4a<`j8qN>jLAZ=(#m`{ z-`nMiiH)a=w2s$Zd!R@J;E&yH+VdPOl>9o^sAblqj&Y~1F48RQsqjFYwZ+U}_r>u^Ck>qU zlviQB$E16zcp4sy22xQPekn=zLei^Qm0=baMH{7!PSi(9BWdfasLy=A!%22jrPq*3&0Ds52N-(cG5uLokRqabu z=#Tk&@(J`9GwgKPHCiiIZlPK8HO$#?V!5)oC+1{SR3I4SYzzx|m2NgV^WWA)6UK>a zit9#K>A}1rjjn6z^V$=a((9B-f&xtQgx#Rw@Q>R~loU4Oh-%qX9uTifkJW1x^&+q6 zW78?{B~+IzfELZU^fN~ldJfo%wP+gdX-G;^%4TOnc_sH{rMJ|^tvI^4ZJUzk-~_{x z)+Pgw7IA@jh=Y26__GO7gcGQQ<8nW$GbeB$hTd!UqNJ31$h6Lg0{j=8fQI82FnY_+ z4=ZSU+-(8Un`pw?2#|j4@C8Aqu;=By5GP_WJC`1?!W;;zn@-xI6QI-f<{x# zX{>+GX{{ee9llyd)0aS;K~0OPvWjNY4tL5}uW<7={G_SSrB%%Kv-=s(Q3A2OCj#`A z>8Gy`U^GdpSeL*8i>c{Av&(T0%wK!}YDwfN*Id!*eHX-ur!ubUG9M^G^8Sa71_4>F=(G`;t&g7&Vj^fE^JKWYh!gMt18E`XKsLs;p3BJ@+Jz zRO79oRPUn`PLE&K)e*WAx;`D5HhJCbreElL6IxolUl97XyII4X6}|z2-{%*KayI%$ zC8owcG?<;q9%-qPOrzOa9UrW)3R{%5>-w0_xtQ6K<^-VaD*@hXMx|4)*U&=Z7k2Cb zSa6S|7~)$X+wQ2ifvnW>J4D{q{q!oq5=QFs;_{Au8tQHWyH~0eSgrTRW7^>4iksM- zun6xYi*@o%q)qbi)Dt}Gr%4VN*n)zC+YyJ=7|7}kD&uq)*^PMpwXQFE|BF%a&v`B8 z81>E0ud$`~c7WaG^APp7ECk&Bn7XJSWCg~3Gzi_5QUpJ2bFHNqgR5{%#Hl7Qj5$sM#NnvH-p}IL>?wS=#=}U1 z>2Xx&+!wLSgb)4tLF_P`A8$?DU7Tj!=WpLz@$CW3>xjr({)hb&XEnIwdqRRkbVF zvj_>mu=OirbxXxPH-FY*bTq%bqDaxRoJrVs!{d-D*rTgbh|sV+G4k_S6ddtfxJKvo zdrAi6aGEa{pqUDk@S#a#9tZPXdzLxjApXSaIom;#YAJN>y9o6m&6$}O^it|xk#lJA z94JTVIgKgMu>GI5t^%y8rd!jYfPm5sf;7@0h;)O}jdX)Zh=3fr`_Nqy(s1bR<{TQN zyBj40yvKw8_g(&bd7gQgJ-oBtS!>Up+0U%C7waa-)1)nLDs{oN$2!t;7w~uI5pr1h z`Wa@jsy@cn{Td&K7g04GB70}l3vUj20|V>PYnEQ=m&e52PzWQSo+18gyFY3IKN~V3 z{i#$H7zQHX{Hbvmm`3J7HQ$C7dSFG^bD)Vy?LCIV^bX;^MrE=6w}xpQ0}tybOknZ5 z7y7~z_!JIK2-Xj8fVAq^W)$_lOScm_;(cxFlfd5^t@{}AecLauD|oCw^fX)nZyBLt z>ac-q?s&lBVjKXgv5ldlW1Pm{cc-}^Lk!8DN(rN4zW092IeZG*s5sit&(ztMpXc_> z*p9)6cfZY|%E4i`1eVGqk$E3v1-K@L2fA_k95T5=ZalcKl7y( zbl^oMhj8PuS#0-I2C(YxbP15+vAL`oU%HaA6}oEW%)a2-Z-1fYAuIN1(7{8yI3swi z%IJ%PMH3^l@N_V^6K9SM4b?#PM4PnCny@WF;KbBJV%tMaaPjR)o{ydTWc9x6kFM+D zJAoGY2eGHLz4YG=kwLOiE34=Zc>v|%T;-Daldqw5D;XgHu0v} z=~9(#{SUV-J^^QA)$r0$YDjI@7 z6_zk3j&QD~`E0q$ zU_Si*^B&!#d+&K@3JBK9mN5oB=OXCN)oVR2eb$JMA z>qV;@Z4&Z^xU1&ty87RnBG0xF%w-A^14uaux4K!1C$jc$8qQ2<3Um${V`lpYWo)q~L`kAULqYj5KXFz*N z1nLPnp27HuvxitWud~Nv4dAcW>uY(@My4rv(-@mSI@0`mT5SfB8Fr)1D<%NS7eU+) zuh?$wU!)j3Hg=vYe#X0{v)A?_dE&Y=x2iC+o;sGsFLGoH>;4pZ5$!#e0=abipyS@0=cW*kwT`MO-T zom?Mox4TasPTK69t|LuP7@n8iqOUEj>;Sf4{^c&NT%o#-@79?(S9b;uf7R-)LCrt1 zg(h$%Pvz5mqm?dB@zXF0e6dC@zpAgv=S2XCByhlHqY9TQ5RyT{2ed{4vb_k+*!|cA z2LNz%!5*pcICpri6)y;B0`u4BqT0fq@}W!bu%}W>cksWKqH7T1@`Wj@e%$2W&QP)EH)$^~=^E>1w1hwF z-%p7MbN7psMAC~H2=rSAs1OMq^MG_$(@E++u_o|QCa8ie3CNe~&MxLL&s(efmoR?Q z5C>dp(9$%%=*1$U0(|~>4mV0p(}T)BtP*{~S2eJVeSut=ysYw)5GRR;(1VyWrk#%k zydpzAy1m7a9+;wur(K1qo{`mszE|ZI@FJU0XqJZe!w1E;vgbS404cLHNd-p10jC~Q zBN=dJwD^-y-ON|Wj&;kxYkZC9-Ir+#(-Vzfek?9<0!_i>4Ivim-+R&k{$<>CvbZJN zrLQ+#g6dE-3#lq8Ma_3Wk?6r*t0-Sd^tk7x%?Z!yX2#T3*Od`}2hyK7uC0}g$!Tr} zJ?=0$dct0RxHG?04*1Q3viZ~(XVzLu+j?NtUgiMr!Hh!Ub@_1&cV=!zxGBuT(D1fga$zDDINpH2``Ibbp_BkYI zoXy;ynwFhFbYhW(0jDOx@}S#uUTU1B@tq_I792?QO!jH!p=74rj{b4ObKBzoebxlq z=OG~w`1>t4H|!e%Y|Eub*Hcr)g@Y?(gl@T%Kx`1o-z_&y3*AF?vi^6=&_{DSR%~i` z6fNpd6Iw4^N}GV7kQg|WcC?^`bvmNNA-^PaY=1`C677UJ7L$hLS?PZEM!Juro~0dm?EUuGS|5(gc%>-rcsWdY z0Lc8E0M6JrBe=Rz%V|a4IXbUUd@YbQjzyQ2oX9;gE|V~o1N0X8S*etFOKeSyddlC| zpqRsNkdMH_MV_UC6FIc?PB#jiQ~&mLzJ;Jn*|-T7va8R{Yqi%; z{f%N2HBQ#yl%k`Ju2Sh}R1gI*l~2#%zyQdlQyOEU=~fbP!nSHR>9Vrr;JOMc6XUuH z1BMy;664hnKL2C`|CL|@I7-R8$9xy$w`HkYGR~1+c7hO2unS7HcKWsn>{2*!+4v5Jp$86uw0(13wyxU-u|3*xceVylvBV?zoCn1f zsA*m`l5mQj#%9dfrXzuSnWMFZUxP5u2(%@x<-`G2a1d9oI^%Fgm*dJCYkZ>XtgtZV zAdje->=qChu1gB*=EmYwDCB%P1sJgC2%+ofKUQQ)2r6vTPph{h8NUq8p3AAwd*}W! zoJdLYq$f8C#I+bf#>UsL>t_iX{D7+R?h9XZ+tg1R+a?7Ck%aG4xp=iwPl=_HdJ3W) zJ5IzaE3BE;cVFhPyf2@(3hyp~E>Omnh!ck#YwA-iY4TTEh|(A|4&Of|)YdbM`l{+RMvYI8$rW@;K7q=g|mwo`}SxXo~tB?#EVLuNZzj}%V`<3#lA zU3}TPZMo+-Skz z8o@&oJyUvpY**CHPVm$K0TVFc8sGX-WyZ+4r^HO<3Ze0h>kMS^6={*O7|E(3hVKuC zaoX5btDV^uzj|(?_SL)Y6$`%l*nSn|hV*_N0_zHKt)NNBWW{7i*;7%QYQ`xW$*Gi+ z@*5YF=Kh~AyArR_8)nm^59G!^@>BPFBG>!Mw`CvM{X9NNHG&OiQH7L=|ko*K~kETfWAY#~BFfITx8lWc6hF?i1$Ss6h=f zf;!0u8-v&^%VxzL%Xv2U57_<1^j!rwm}#ivUiV!~?5x;u<9tf152>%4RQRmN;&kdU z1^H&qB*gRTTf-}r=#5PPW!;lqcEFyqc{Pg&4bEQjq!cXxKOHLJYb z%J!Q@c@e-Nq%t=fj8NM65CCldC{m_MaCpDMANHQeIsN7AT1K0S=50ZJyEr= zi(<3SMWZ(}VJ-o5xN7dO3$2X)+&~Iszais~LQa9SkcQcL8(Q4)t>N@xt{f8!Y`3u!_0N2W#BcWg5r7CotWVbl%>9D zB9Es_HpeVC$iDU)b6%#jgScO@x?{9VQI^MkcS9u!OV$A}*J z?N7!QP=vb-rY;SADRAuSu}EVrP$)Lba*e1+8fcwb;x?9@9R_`TlQuD`TuAC0cb+at_{yj?6FBwBHuaR%Zmh-#3 zl+JK`GXN_uxICvaGmhW+x_^w%e}e7nPP^K~iOZR-WXB0JqH<8Z*>y9lYiRXD_DmS% zl}K&c!VS1Dxp7R4`twZPd7Sm%k-#rClZ-bHu%ryiVtS@6 zdA$P!hSAPH9wCmg=u7bR7lWokpy~Lqx^NrpZF!jADvF63qI4?AZtwa*V7uC=?ke?{ zS-a|q<)KVx`>*|rh*RKrW3K+@=kYXYZ8G0Q?0ul?=o^U)N69&jD^gccnJwO@2+GPE zHl(`0;;>jZq)ZH-5f)^Vr}iYorWXrhq~-Df@REA!mv-5!z7TegK&HF1jv5^&?EHPd zz8QYzQo7LwVgA8Uy65jJ1>bgHmfik6O*N9I0JHhs4`X?mLLF~Ch@p#p9EqD?QUg1F zTwQEwLLW(21S@AuAC+|dV^m%)Xl$8qW*ks1v}c}&d8TK%B;VeSl< zBaf{QvbvPK#;t{{HC{Lxi!T@ka|@Ju#xBlqHAbasv}9s=5m)zIET2^%)!g7PeUG=% zFnol#-kXhCVKLy19K!Ln=TS6xoQ`&82VhsRg+D{|KFf;t36>p_ijLsaWIYx0FMKmm zUqNf>`C(Flvoy3}k@$(Mg3X(mVjkQpq)U;8?YH!|d@lu!SlFZlJOqv;{PGsJ6u)+T@%Cr#NKTQ+kF{eyq};Og2%vjh3+YElhqUEAHKf z`RxH^xV3|`7{zLLNvn_`|%Fj6od| ze8bBuZW`T5%$uh%iyLedPQ(Y9bku;;aiqeS(Lo@I0Q*ATb?urk>)toOn`-}$2`?B4 z*%Cx=$dR>mhfi*CJz1#O+z46rUKotyoLa=>QiGI*kssG3)ZE-^GN~+(tbX0XwjwS;4$uSEO_vl9t0v{c+G+A>1)19#` zITm1NLFS23;FojkFGP+tkk28En(Cfml0P8@`C~S>vQ3jj#XgqmB5sM~?Kg(!5_@IK zzs1h!Kl>GK!Clc)*fia$LMD`QQU=m#l_>ByevC0RbLF*Ugd`6z)+O}t#9oZ@={ht~ z*;9m@3-sB`uyvg~C%;`Q!nksSW`**-aqZLKs|chc%k}Y0o_{Wv&qccF{!({dET1eF zJrm>VjbL?P-sGj&9eFaAj}T>I7MhvA2eO>o_PJH7f)-Lpl4y}X2;1{Bn~Jqu`L-@I zqgtq<3P?Xg>`@I>hOh$f23j0hHuc*C-)roR3l(HiwJYj)uYw*ntH7Ht04}B{@SN(d zS7~H%PUq>Jjgl&{$zt_ol^NgKEL>Jn$&;D79No7uEb}LkM7_xc&M5H$5Ez`P9xYII zo)LRYwr6K17GH~K!<8J%ohNJMm{(1Z#k(bMGzHv>y!Ke#=mcR@uA7GJ4N>F~iS^Tp zH+-DE3FxE%m+ogFmP0OT@%p{D=QA`(qRhPJ`D=U+(q&F}9 z>NCy3s_n#ZniK!gSEfsp@12nleNT+m$g$VDJ7p^xmcp%K(h!K7ZO@2w5wO(84J)&Qs)1X@r#I^B_$44=LJHizdCL&=P8Do!S`_|)EfbKc@ASr;T=Mx^H3wS_0|^`g zN$sHzHY(=_Ik~kd_)qwseKqG1@h=+6X97(C{bud@(xvD~kL=FJI<8Lx2G%C^m8#47 zG}gahGia@+=&Y4VZ>TGfPw2%;k^KCH-6~wRozpFk8OM_dsgnhu%EhB~sGdW{I$yk| zCkg5=Ea!pmM<^dQnK9-0Zslsoa#IfP*@EQRr~&^G`^-N=!%6&TvmydI^lZ%)svB573Wcg90tv08vDOaXAZYwXt2%*K8bYtw+ z;Y~rqDqNdqxYTQ4CH8tad3CBP?i9I~&z$Ot6Vee!8@!~eFFccpAJrMkQ%w5K8YXkR z6mmE|50(=3?g%gj5jVc^yWHlWe;BwD==A93;R%iN1mfIoRwMz~j$Z}vMU{GA(I z4B3YMF~{8T5Dy=v$zBV+>L)8ya;(Hnxtz@=&6-)z&1b{vOoizY+s~Fa3JQ-h-|h1J zghpAi>;Jxo&hhf;Z{y{{2&I{TomIv#p82cUG|Rf&7J0B zpDvlNS*J%Y>cST{^v=nbM1_|%wxw=sDV|zSR z{_Eh})(vSI$qMksi8i5k>eq+aY@EQNjO`HhMs8{jcB_jWn|#3T-13O?M{>(|k! zJE6|w2rGFXfjIo_FBzU*QBO_>&&95BVko=7W06$eft#lbQ9spBMXLz5UoZ|ow;U0- z9=sX{r3xb!e0RSkHRg%r9TQxX(%wDw*`E+@ee;-q`^K{YV}xt3S#HpP2C(v7&GZY1 zyXs{Yks`oQc`d~$$On(_EQMFK&v2Sr=tjAxQJR_($@X)`pI!0 z;-?iyDY|W@YlEEXtUEEUeVprX#L9#6FpDouO)aZ{OJ$|%+P+siBHz{Ct*7a%sr2nS zT1CGVuI9n3;YVM#7L+__ekSQpBh$jd-756y4o#wqgG}g|&^n~+qvcO`_?hX8@fbP= zA@{-e)*~ ze4)ff0I_6!QWCjwD&c|mFO-mwT}snQsr)rpR5QHV&2uU*!yvJe>co=RO&YkM#Uvy=M!tF zP`yOMuhpf-!&Kg;wBM0;%aw_N6`cD&de6TW0JMj(75MyJ&fk)19tz3pgE>{z6Zs_j z&FN1;->6VR{H)gdJkIZOU9QozSP1z;8b0VfZ66V9B+;m`1103>Mf`jOa)%ruM}jk( zm9HF756*U-xW8M!d`)!V3CfUgXW0yecw!wmTa;aFj$&E}Ke;9uT|{d5L7MyNwjBWX z7W1F@DA<|RZPxK$Z_`Qc(%;%del^nlIaYWv0wT-wSf|o%_C`(k^sgiEURv z|0hE^aJSu(^3T=QzNX_hsDEw_TBxDrKR+OES0o^O2Ocn}g9HXdDs)@Wg_`6YG#@a_ ze^AihAa5gN7;vV84hAG$)tl=^hJ(9?4wmx)5FHH$F z&-A#P7$KM$)+X3}uh)aL6AwBkvcrIiSbwrm5W|M`??-}gaEuSYRDv+@K{pKy_;2v{ zf7?G)^6!vPq2b`qVBiPgeGjB=$iu+Lgny#_p2NTo!p0te4;7(cG9Y3vE({3sdP4z9 z&_eS7Xr>N4F`xx%_hL}~nZW;evY`=%gb#q1>cD_rI@m)5Ebk>oxYPg+^^)CRA3(2P zXad2VxV9j#(ledA2C9uHc-Tbk@>CMzmxk9k+LW{SpjseT|gf{!3RX?I?xU@ zz_+F(z?=UF!ssVP$kYdBl`#M{E%1Q{L-(WIrolXwf{s)UDv14okT6525K>@SKMhQ+ z2kzzvKx$)Pg^e(*g>7m!Vd(P*};ZO=t}-a z$h{HW`n6-Ak4F+Tnvd=QIM)F-EG2ggU_%cE#UDeY+USDwp*rOKiRXL(b_2E#JO}<+ zm5K++V1oa3cmKEGPoIG)#niyZL%8>rry3$g*zf`_h6w(-8?eqCHX1duP!*7%Ba?hU zRPO^*p>K#C2K@Jg|G!zx{h>f=;H%-kEboSi5q>AYI!6tY!#Ycin(fCxSyG`a50{3s zWSCXyzheBK$>2)2s`?(f=ny}KgM0O0Xj5s>StWC?lg9`#LS!~nPu4%igRwnW#2<8w z`wC_I-}eek z{12FzOM&DNDcHSa!L#r|g#sF(x^O%I{RW&- +cd "`dirname \"$PRG\"`/" >/dev/null APP_HOME="`pwd -P`" -cd "$SAVED" >&- +cd "$SAVED" >/dev/null CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -114,6 +109,7 @@ fi if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` From 86cfc66a40083091ebbd0b3a721bc8ab92db1acd Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Fri, 12 Aug 2016 16:27:57 +0200 Subject: [PATCH 19/31] Automatically set version from git - Uses last tag on master branch (this is set on 'git flow release finish') - branch name snapshot otherwise, e.g. 'feature-nio-SNAPSHOT', 'develop-SNAPSHOT' --- build.gradle | 6 +-- .../ch/dissem/gradle/GitFlowVersion.groovy | 53 +++++++++++++++++++ .../gradle-plugins/gitflow-version.properties | 1 + gradle/wrapper/gradle-wrapper.properties | 4 +- 4 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/gitflow-version.properties diff --git a/build.gradle b/build.gradle index 0e96284..311c3f9 100644 --- a/build.gradle +++ b/build.gradle @@ -3,12 +3,10 @@ subprojects { apply plugin: 'maven' apply plugin: 'signing' apply plugin: 'jacoco' + apply plugin: 'gitflow-version' sourceCompatibility = 1.7 group = 'ch.dissem.jabit' - version = '1.1.0-SNAPSHOT' - - ext.isReleaseVersion = !version.endsWith("SNAPSHOT") repositories { mavenCentral() @@ -35,7 +33,7 @@ subprojects { } signing { - required { isReleaseVersion && project.getProperties().get("signing.keyId")?.length() > 0 } + required { isRelease && project.getProperties().get("signing.keyId")?.length() > 0 } sign configurations.archives } diff --git a/buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy b/buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy new file mode 100644 index 0000000..81af76b --- /dev/null +++ b/buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy @@ -0,0 +1,53 @@ +package ch.dissem.gradle + +import org.gradle.api.Plugin +import org.gradle.api.Project + +/** + * Sets the version as follows: + *

    + *
  • If the branch is 'master', the version is set to the latest tag (which is expected to be set by Git flow)
    • + *
  • Otherwise, the version is set to the branch name, with '-SNAPSHOT' appended
    • + *
+ */ +class GitFlowVersion implements Plugin { + def getBranch(Project project) { + def stdout = new ByteArrayOutputStream() + project.exec { + commandLine 'git', 'rev-parse', '--abbrev-ref', 'HEAD' + standardOutput = stdout + } + return stdout.toString().trim() + } + + def getTag(Project project) { + def stdout = new ByteArrayOutputStream() + project.exec { + commandLine 'git', 'describe', '--abbrev=0' + standardOutput = stdout + } + return stdout.toString().trim() + } + + def isRelease(Project project) { + return "master" == getBranch(project); + } + + def getVersion(Project project) { + if (project.ext.isRelease) { + return getTag(project) + } else { + return getBranch(project).replaceAll("/", "-") + "-SNAPSHOT" + } + } + + @Override + void apply(Project project) { + project.ext.isRelease = isRelease(project) + project.version = getVersion(project) + + project.task('version') << { + println "Version deduced from git: '${project.version}'" + } + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/gitflow-version.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/gitflow-version.properties new file mode 100644 index 0000000..1fb4f78 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/gitflow-version.properties @@ -0,0 +1 @@ +implementation-class=ch.dissem.gradle.GitFlowVersion diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a657ea5..ebcdff0 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Thu Aug 11 17:36:39 CEST 2016 +#Fri Aug 12 11:52:04 CEST 2016 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-all.zip From caa2219a6301056600f9c89ae82d0572760d5c5e Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Wed, 24 Aug 2016 22:17:02 +0200 Subject: [PATCH 20/31] It looks like the NIO network handler works now - some testing needed to see how reliably --- .../bitmessage/factory/V3MessageReader.java | 41 ------------ core/src/main/resources/nodes.txt | 2 +- .../networking/AbstractConnection.java | 5 +- .../networking/nio/ConnectionInfo.java | 19 +++--- .../networking/nio/NioNetworkHandler.java | 62 ++++++++++++------- repositories/build.gradle | 4 +- 6 files changed, 59 insertions(+), 74 deletions(-) diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java index d26cabe..89677cd 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/V3MessageReader.java @@ -21,17 +21,13 @@ import ch.dissem.bitmessage.entity.NetworkMessage; import ch.dissem.bitmessage.exception.ApplicationException; import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.utils.Decode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; -import java.util.UUID; import static ch.dissem.bitmessage.entity.NetworkMessage.MAGIC_BYTES; import static ch.dissem.bitmessage.factory.BufferPool.bufferPool; @@ -42,8 +38,6 @@ import static ch.dissem.bitmessage.utils.Singleton.cryptography; * Similar to the {@link V3MessageFactory}, but used for NIO buffers which may or may not contain a whole message. */ public class V3MessageReader { - private static final Logger LOG = LoggerFactory.getLogger(V3MessageReader.class); - private ByteBuffer headerBuffer; private ByteBuffer dataBuffer; @@ -53,7 +47,6 @@ public class V3MessageReader { private byte[] checksum; private List messages = new LinkedList<>(); - private SizeInfo sizeInfo = new SizeInfo(); public ByteBuffer getActiveBuffer() { if (state != null && state != ReaderState.DATA) { @@ -88,7 +81,6 @@ public class V3MessageReader { throw new NodeException("Payload of " + length + " bytes received, no more than " + MAX_PAYLOAD_SIZE + " was expected."); } - sizeInfo.add(length); // FIXME: remove this once we have some values to work with checksum = new byte[4]; headerBuffer.get(checksum); state = ReaderState.DATA; @@ -194,37 +186,4 @@ public class V3MessageReader { } private enum ReaderState {MAGIC, HEADER, DATA} - - private class SizeInfo { - private FileWriter file; - private long min = Long.MAX_VALUE; - private long avg = 0; - private long max = Long.MIN_VALUE; - private long count = 0; - - private SizeInfo() { - try { - file = new FileWriter("D:/message_size_info-" + UUID.randomUUID() + ".csv"); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - } - - private void add(long length) { - avg = (count * avg + length) / (count + 1); - if (length < min) { - min = length; - } - if (length > max) { - max = length; - } - count++; - LOG.info("Received message with data size " + length + "; Min: " + min + "; Max: " + max + "; Avg: " + avg); - try { - file.write(length + "\n"); - } catch (IOException e) { - e.printStackTrace(); - } - } - } } diff --git a/core/src/main/resources/nodes.txt b/core/src/main/resources/nodes.txt index 9466a85..bce982e 100644 --- a/core/src/main/resources/nodes.txt +++ b/core/src/main/resources/nodes.txt @@ -5,4 +5,4 @@ bootstrap8080.bitmessage.org:8080 bootstrap8444.bitmessage.org:8444 [stream 2] -# none yet \ No newline at end of file +# none yet diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index f3900b1..66f7dca 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -228,10 +228,11 @@ public abstract class AbstractConnection { break; case CUSTOM: MessagePayload response = ctx.getCustomCommandHandler().handle((CustomMessage) payload); - if (response != null) { + if (response == null) { + disconnect(); + } else { send(response); } - disconnect(); break; default: throw new NodeException("Command 'version' or 'verack' expected, but was '" diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index e416da5..fb4cf58 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -17,10 +17,7 @@ package ch.dissem.bitmessage.networking.nio; import ch.dissem.bitmessage.InternalContext; -import ch.dissem.bitmessage.entity.GetData; -import ch.dissem.bitmessage.entity.MessagePayload; -import ch.dissem.bitmessage.entity.NetworkMessage; -import ch.dissem.bitmessage.entity.Version; +import ch.dissem.bitmessage.entity.*; import ch.dissem.bitmessage.entity.valueobject.InventoryVector; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.exception.NodeException; @@ -112,16 +109,18 @@ public class ConnectionInfo extends AbstractConnection { public void updateSyncStatus() { if (!syncFinished) { - syncFinished = reader.getMessages().isEmpty() && syncFinished(null); + syncFinished = (reader == null || reader.getMessages().isEmpty()) && syncFinished(null); } } public boolean isExpired() { switch (state) { case CONNECTING: - return lastUpdate < System.currentTimeMillis() - 30000; + // the TCP timeout starts out at 20 seconds + return lastUpdate < System.currentTimeMillis() - 20_000; case ACTIVE: - return lastUpdate < System.currentTimeMillis() - 30000; + // after verack messages are exchanged, the timeout is raised to 10 minutes + return lastUpdate < System.currentTimeMillis() - 600_000; case DISCONNECTED: return true; default: @@ -150,4 +149,10 @@ public class ConnectionInfo extends AbstractConnection { commonRequestedObjects.addAll(((GetData) payload).getInventory()); } } + + public boolean isWritePending() { + return !sendingQueue.isEmpty() + || headerOut != null && headerOut.hasRemaining() + || payloadOut != null && payloadOut.hasRemaining(); + } } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 6c4ae10..b1d1cb6 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -64,7 +64,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private Selector selector; private ServerSocketChannel serverChannel; private Map connections = new ConcurrentHashMap<>(); - private int requestedObjectsCount; + private volatile int requestedObjectsCount; private Thread starter; @@ -94,13 +94,15 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex public CustomMessage send(InetAddress server, int port, CustomMessage request) { try (SocketChannel channel = SocketChannel.open(new InetSocketAddress(server, port))) { channel.configureBlocking(true); - ByteBuffer buffer = ByteBuffer.allocate(MAX_MESSAGE_SIZE); - new NetworkMessage(request).write(buffer); - buffer.flip(); - while (buffer.hasRemaining()) { - channel.write(buffer); + ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_SIZE); + ByteBuffer payloadBuffer = new NetworkMessage(request).writeHeaderAndGetPayloadBuffer(headerBuffer); + headerBuffer.flip(); + while (headerBuffer.hasRemaining()) { + channel.write(headerBuffer); + } + while (payloadBuffer.hasRemaining()) { + channel.write(payloadBuffer); } - buffer.clear(); V3MessageReader reader = new V3MessageReader(); while (channel.isConnected() && reader.getMessages().isEmpty()) { @@ -195,11 +197,25 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex if (missing > 0) { List addresses = ctx.getNodeRegistry().getKnownAddresses(missing, ctx.getStreams()); for (NetworkAddress address : addresses) { + if (isConnectedTo(address)) { + continue; + } try { SocketChannel channel = SocketChannel.open(); channel.configureBlocking(false); channel.connect(new InetSocketAddress(address.toInetAddress(), address.getPort())); - channel.finishConnect(); + long timeout = System.currentTimeMillis() + 20_000; + while (!channel.finishConnect() && System.currentTimeMillis() < timeout) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + break; + } + } + if (!channel.finishConnect()) { + channel.close(); + continue; + } ConnectionInfo connection = new ConnectionInfo(ctx, CLIENT, address, listener, @@ -248,6 +264,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex Iterator keyIterator = selector.selectedKeys().iterator(); while (keyIterator.hasNext()) { SelectionKey key = keyIterator.next(); + keyIterator.remove(); if (key.attachment() instanceof ConnectionInfo) { SocketChannel channel = (SocketChannel) key.channel(); ConnectionInfo connection = (ConnectionInfo) key.attachment(); @@ -258,24 +275,18 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex if (key.isReadable()) { read(channel, connection); } - if (connection.getSendingQueue().isEmpty()) { - if (connection.getState() == DISCONNECTED) { - key.interestOps(0); - key.channel().close(); - } else { - key.interestOps(OP_READ); - } - } else { + if (connection.getState() == DISCONNECTED) { + key.interestOps(0); + channel.close(); + } else if (connection.isWritePending()) { key.interestOps(OP_READ | OP_WRITE); + } else { + key.interestOps(OP_READ); } } catch (CancelledKeyException | NodeException | IOException e) { connection.disconnect(); } - if (connection.getState() == DISCONNECTED) { - connections.remove(connection); - } } - keyIterator.remove(); requestedObjectsCount = requestedObjects.size(); } for (Map.Entry e : connections.entrySet()) { @@ -316,7 +327,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } private static void read(SocketChannel channel, ConnectionInfo connection) throws IOException { - while (channel.read(connection.getInBuffer()) > 0) { + if (channel.read(connection.getInBuffer()) > 0) { connection.updateReader(); } connection.updateSyncStatus(); @@ -442,6 +453,15 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex ); } + private boolean isConnectedTo(NetworkAddress address) { + for (ConnectionInfo c : connections.keySet()) { + if (c.getNode().equals(address)) { + return true; + } + } + return false; + } + @Override public boolean isRunning() { return selector != null && selector.isOpen() && starter.isAlive(); diff --git a/repositories/build.gradle b/repositories/build.gradle index 2ab092c..032ffbc 100644 --- a/repositories/build.gradle +++ b/repositories/build.gradle @@ -14,10 +14,10 @@ sourceCompatibility = 1.8 dependencies { compile project(':core') - compile 'org.flywaydb:flyway-core:3.2.1' + compile 'org.flywaydb:flyway-core:4.0.3' testCompile 'junit:junit:4.12' testCompile 'com.h2database:h2:1.4.190' testCompile 'org.mockito:mockito-core:1.10.19' testCompile project(path: ':core', configuration: 'testArtifacts') testCompile project(':cryptography-bc') -} \ No newline at end of file +} From 102d63e2c64d0902bb3895efe822d477be4417bd Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Thu, 25 Aug 2016 08:50:06 +0200 Subject: [PATCH 21/31] Fixed message count per label --- demo/build.gradle | 2 +- repositories/build.gradle | 2 +- .../ch/dissem/bitmessage/repository/JdbcMessageRepository.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/demo/build.gradle b/demo/build.gradle index 0cf1781..79f5834 100644 --- a/demo/build.gradle +++ b/demo/build.gradle @@ -30,7 +30,7 @@ dependencies { compile project(':wif') compile 'org.slf4j:slf4j-simple:1.7.12' compile 'args4j:args4j:2.32' - compile 'com.h2database:h2:1.4.190' + compile 'com.h2database:h2:1.4.192' compile 'org.apache.commons:commons-lang3:3.4' testCompile 'junit:junit:4.12' testCompile 'org.mockito:mockito-core:1.10.19' diff --git a/repositories/build.gradle b/repositories/build.gradle index 032ffbc..3f5874b 100644 --- a/repositories/build.gradle +++ b/repositories/build.gradle @@ -16,7 +16,7 @@ dependencies { compile project(':core') compile 'org.flywaydb:flyway-core:4.0.3' testCompile 'junit:junit:4.12' - testCompile 'com.h2database:h2:1.4.190' + testCompile 'com.h2database:h2:1.4.192' testCompile 'org.mockito:mockito-core:1.10.19' testCompile project(path: ':core', configuration: 'testArtifacts') testCompile project(':cryptography-bc') diff --git a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java index e2e247a..3788d96 100644 --- a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java +++ b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java @@ -81,8 +81,7 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements try ( Connection connection = config.getConnection(); Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery("SELECT count(*) FROM Message WHERE " + where - + " ORDER BY received DESC") + ResultSet rs = stmt.executeQuery("SELECT count(*) FROM Message WHERE " + where) ) { if (rs.next()) { return rs.getInt(1); From 53aa2c6804977bac68da8d4e2bbd3d241a488d2c Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Thu, 25 Aug 2016 12:44:06 +0200 Subject: [PATCH 22/31] Slightly improved the problem with stale objects in the requested objects list. But I don't know how to properly solve it, there may always be some left. --- .../bitmessage/networking/AbstractConnection.java | 2 +- .../networking/nio/NioNetworkHandler.java | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 66f7dca..4a209aa 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -183,7 +183,7 @@ public abstract class AbstractConnection { } private void receiveMessage(Addr addr) { - LOG.debug("Received " + addr.getAddresses().size() + " addresses."); + LOG.trace("Received " + addr.getAddresses().size() + " addresses."); ctx.getNodeRegistry().offerAddresses(addr.getAddresses()); } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index b1d1cb6..7c6a5b6 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -64,7 +64,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private Selector selector; private ServerSocketChannel serverChannel; private Map connections = new ConcurrentHashMap<>(); - private volatile int requestedObjectsCount; + private final Set requestedObjects = Collections.newSetFromMap(new ConcurrentHashMap(10_000)); private Thread starter; @@ -147,7 +147,6 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } catch (IOException e) { throw new ApplicationException(e); } - final Set requestedObjects = new HashSet<>(); thread("connection listener", new Runnable() { @Override public void run() { @@ -287,7 +286,6 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex connection.disconnect(); } } - requestedObjectsCount = requestedObjects.size(); } for (Map.Entry e : connections.entrySet()) { if (e.getValue().isValid() && (e.getValue().interestOps() & OP_WRITE) == 0) { @@ -370,7 +368,10 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex @Override public void request(Collection inventoryVectors) { - if (!isRunning()) return; + if (!isRunning()) { + requestedObjects.clear(); + return; + } Iterator iterator = inventoryVectors.iterator(); if (!iterator.hasNext()) { return; @@ -408,6 +409,9 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } } while (iterator.hasNext()); + // remove objects nobody knows of + requestedObjects.removeAll(inventoryVectors); + for (ConnectionInfo connection : distribution.keySet()) { List ivs = distribution.get(connection); if (!ivs.isEmpty()) { @@ -449,7 +453,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex return new Property("network", null, new Property("connectionManager", isRunning() ? "running" : "stopped"), new Property("connections", null, streamProperties), - new Property("requestedObjects", requestedObjectsCount) + new Property("requestedObjects", requestedObjects.size()) ); } From 827973f64278231e5c03b64d74cb779c5e6dfdb8 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 29 Aug 2016 12:30:26 +0200 Subject: [PATCH 23/31] Improved connecting to the network --- .../networking/nio/NioNetworkHandler.java | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 7c6a5b6..7f04aa1 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -45,8 +45,7 @@ import static ch.dissem.bitmessage.networking.AbstractConnection.State.DISCONNEC import static ch.dissem.bitmessage.utils.Collections.selectRandom; import static ch.dissem.bitmessage.utils.DebugUtils.inc; import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; -import static java.nio.channels.SelectionKey.OP_READ; -import static java.nio.channels.SelectionKey.OP_WRITE; +import static java.nio.channels.SelectionKey.*; /** * Network handler using java.nio, resulting in less threads. @@ -203,18 +202,6 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex SocketChannel channel = SocketChannel.open(); channel.configureBlocking(false); channel.connect(new InetSocketAddress(address.toInetAddress(), address.getPort())); - long timeout = System.currentTimeMillis() + 20_000; - while (!channel.finishConnect() && System.currentTimeMillis() < timeout) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - } - if (!channel.finishConnect()) { - channel.close(); - continue; - } ConnectionInfo connection = new ConnectionInfo(ctx, CLIENT, address, listener, @@ -222,9 +209,18 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex ); connections.put( connection, - channel.register(selector, OP_READ | OP_WRITE, connection) + channel.register(selector, OP_CONNECT, connection) ); - } catch (NoRouteToHostException | AsynchronousCloseException ignore) { + } catch (NoRouteToHostException ignore) { + // We'll try to connect to many offline nodes, so + // this is expected to happen quite a lot. + } catch (AsynchronousCloseException e) { + // The exception is expected if the network is being + // shut down, as we actually do asynchronously close + // the connections. + if (isRunning()) { + LOG.error(e.getMessage(), e); + } } catch (IOException e) { LOG.error(e.getMessage(), e); } @@ -268,6 +264,11 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex SocketChannel channel = (SocketChannel) key.channel(); ConnectionInfo connection = (ConnectionInfo) key.attachment(); try { + if (key.isConnectable()) { + if (!channel.finishConnect()) { + continue; + } + } if (key.isWritable()) { write(channel, connection); } @@ -288,10 +289,11 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } } for (Map.Entry e : connections.entrySet()) { - if (e.getValue().isValid() && (e.getValue().interestOps() & OP_WRITE) == 0) { - if (!e.getKey().getSendingQueue().isEmpty()) { - e.getValue().interestOps(OP_READ | OP_WRITE); - } + if (e.getValue().isValid() + && (e.getValue().interestOps() & OP_WRITE) == 0 + && (e.getValue().interestOps() & OP_CONNECT) == 0 + && !e.getKey().getSendingQueue().isEmpty()) { + e.getValue().interestOps(OP_READ | OP_WRITE); } } } From dad05d835b8cab9fdbebb60c58c546df809f7d8b Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Thu, 1 Sep 2016 07:35:46 +0200 Subject: [PATCH 24/31] Created an improved JdbcNodeRegistry and removed MemoryNodeRegistry, as it doesn't properly work with the way nodes are handled and disseminated in the new PyBitmessage client. The new one should work a lot more stable. --- .../bitmessage/ports/MemoryNodeRegistry.java | 125 ------------- .../bitmessage/ports/NodeRegistryHelper.java | 54 ++++++ .../bitmessage/ports/NodeRegistryTest.java | 99 ----------- .../java/ch/dissem/bitmessage/demo/Main.java | 3 +- .../networking/nio/ConnectionInfo.java | 2 +- .../networking/nio/NioNetworkHandler.java | 8 +- .../repository/JdbcNodeRegistry.java | 167 ++++++++++++++++++ .../db/migration/V3.3__Create_table_node.sql | 9 + .../repository/JdbcNodeRegistryTest.java | 48 +++-- 9 files changed, 267 insertions(+), 248 deletions(-) delete mode 100644 core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java create mode 100644 core/src/main/java/ch/dissem/bitmessage/ports/NodeRegistryHelper.java delete mode 100644 core/src/test/java/ch/dissem/bitmessage/ports/NodeRegistryTest.java create mode 100644 repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java create mode 100644 repositories/src/main/resources/db/migration/V3.3__Create_table_node.sql diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java b/core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java deleted file mode 100644 index a56b4ff..0000000 --- a/core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2015 Christian Basler - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ch.dissem.bitmessage.ports; - -import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; -import ch.dissem.bitmessage.exception.ApplicationException; -import ch.dissem.bitmessage.utils.UnixTime; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -import static ch.dissem.bitmessage.utils.Collections.selectRandom; -import static ch.dissem.bitmessage.utils.UnixTime.HOUR; -import static java.util.Collections.newSetFromMap; - -public class MemoryNodeRegistry implements NodeRegistry { - private static final Logger LOG = LoggerFactory.getLogger(MemoryNodeRegistry.class); - - private final Map> stableNodes = new ConcurrentHashMap<>(); - private final Map> knownNodes = new ConcurrentHashMap<>(); - - private void loadStableNodes() { - try (InputStream in = getClass().getClassLoader().getResourceAsStream("nodes.txt")) { - Scanner scanner = new Scanner(in); - long stream = 0; - Set streamSet = null; - while (scanner.hasNext()) { - try { - String line = scanner.nextLine().trim(); - if (line.startsWith("[stream")) { - stream = Long.parseLong(line.substring(8, line.lastIndexOf(']'))); - streamSet = new HashSet<>(); - stableNodes.put(stream, streamSet); - } else if (streamSet != null && !line.isEmpty() && !line.startsWith("#")) { - int portIndex = line.lastIndexOf(':'); - InetAddress[] inetAddresses = InetAddress.getAllByName(line.substring(0, portIndex)); - int port = Integer.valueOf(line.substring(portIndex + 1)); - for (InetAddress inetAddress : inetAddresses) { - streamSet.add(new NetworkAddress.Builder().ip(inetAddress).port(port).stream(stream).build()); - } - } - } catch (IOException e) { - LOG.warn(e.getMessage(), e); - } - } - if (LOG.isDebugEnabled()) { - for (Map.Entry> e : stableNodes.entrySet()) { - LOG.debug("Stream " + e.getKey() + ": loaded " + e.getValue().size() + " bootstrap nodes."); - } - } - } catch (IOException e) { - throw new ApplicationException(e); - } - } - - @Override - public List getKnownAddresses(int limit, long... streams) { - List result = new LinkedList<>(); - for (long stream : streams) { - Set known = knownNodes.get(stream); - if (known != null && !known.isEmpty()) { - for (NetworkAddress node : known) { - if (node.getTime() > UnixTime.now(-3 * HOUR)) { - result.add(node); - } else { - known.remove(node); - } - } - } - if (result.isEmpty()) { - if (stableNodes.isEmpty() || stableNodes.get(stream).isEmpty()) { - loadStableNodes(); - } - Set nodes = stableNodes.get(stream); - if (nodes != null && !nodes.isEmpty()) { - // To reduce load on stable nodes, only return one - result.add(selectRandom(nodes)); - } - } - } - return selectRandom(limit, result); - } - - @Override - public void offerAddresses(List addresses) { - for (NetworkAddress node : addresses) { - if (node.getTime() <= UnixTime.now()) { - if (!knownNodes.containsKey(node.getStream())) { - synchronized (knownNodes) { - if (!knownNodes.containsKey(node.getStream())) { - knownNodes.put( - node.getStream(), - newSetFromMap(new ConcurrentHashMap()) - ); - } - } - } - if (node.getTime() <= UnixTime.now()) { - // TODO: This isn't quite correct - // If the node is already known, the one with the more recent time should be used - knownNodes.get(node.getStream()).add(node); - } - } - } - } -} diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/NodeRegistryHelper.java b/core/src/main/java/ch/dissem/bitmessage/ports/NodeRegistryHelper.java new file mode 100644 index 0000000..63d70eb --- /dev/null +++ b/core/src/main/java/ch/dissem/bitmessage/ports/NodeRegistryHelper.java @@ -0,0 +1,54 @@ +package ch.dissem.bitmessage.ports; + +import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; +import ch.dissem.bitmessage.exception.ApplicationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.util.*; + +/** + * Helper class to kick start node registries. + */ +public class NodeRegistryHelper { + private static final Logger LOG = LoggerFactory.getLogger(NodeRegistryHelper.class); + + public static Map> loadStableNodes() { + try (InputStream in = NodeRegistryHelper.class.getClassLoader().getResourceAsStream("nodes.txt")) { + Scanner scanner = new Scanner(in); + long stream = 0; + Map> result = new HashMap<>(); + Set streamSet = null; + while (scanner.hasNext()) { + try { + String line = scanner.nextLine().trim(); + if (line.startsWith("[stream")) { + stream = Long.parseLong(line.substring(8, line.lastIndexOf(']'))); + streamSet = new HashSet<>(); + result.put(stream, streamSet); + } else if (streamSet != null && !line.isEmpty() && !line.startsWith("#")) { + int portIndex = line.lastIndexOf(':'); + InetAddress[] inetAddresses = InetAddress.getAllByName(line.substring(0, portIndex)); + int port = Integer.valueOf(line.substring(portIndex + 1)); + for (InetAddress inetAddress : inetAddresses) { + streamSet.add(new NetworkAddress.Builder().ip(inetAddress).port(port).stream(stream).build()); + } + } + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + } + } + if (LOG.isDebugEnabled()) { + for (Map.Entry> e : result.entrySet()) { + LOG.debug("Stream " + e.getKey() + ": loaded " + e.getValue().size() + " bootstrap nodes."); + } + } + return result; + } catch (IOException e) { + throw new ApplicationException(e); + } + } +} diff --git a/core/src/test/java/ch/dissem/bitmessage/ports/NodeRegistryTest.java b/core/src/test/java/ch/dissem/bitmessage/ports/NodeRegistryTest.java deleted file mode 100644 index 993e3a9..0000000 --- a/core/src/test/java/ch/dissem/bitmessage/ports/NodeRegistryTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2016 Christian Basler - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ch.dissem.bitmessage.ports; - -import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; -import ch.dissem.bitmessage.utils.UnixTime; -import org.junit.Test; - -import java.util.Arrays; - -import static ch.dissem.bitmessage.utils.UnixTime.HOUR; -import static org.hamcrest.Matchers.*; -import static org.junit.Assert.assertThat; - -public class NodeRegistryTest { - private NodeRegistry registry = new MemoryNodeRegistry(); - - @Test - public void ensureGetKnownNodesWithoutStreamsYieldsEmpty() { - assertThat(registry.getKnownAddresses(10), empty()); - } - - /** - * Please note that this test fails if there is no internet connection, - * as the initial nodes' IP addresses are determined by DNS lookup. - */ - @Test - public void ensureGetKnownNodesForStream1YieldsResult() { - assertThat(registry.getKnownAddresses(10, 1), hasSize(1)); - } - - @Test - public void ensureNodeIsStored() { - registry.offerAddresses(Arrays.asList( - new NetworkAddress.Builder() - .ipv4(127, 0, 0, 1) - .port(42) - .stream(1) - .time(UnixTime.now()) - .build(), - new NetworkAddress.Builder() - .ipv4(127, 0, 0, 2) - .port(42) - .stream(1) - .time(UnixTime.now()) - .build(), - new NetworkAddress.Builder() - .ipv4(127, 0, 0, 2) - .port(42) - .stream(2) - .time(UnixTime.now()) - .build() - )); - assertThat(registry.getKnownAddresses(10, 1).size(), is(2)); - assertThat(registry.getKnownAddresses(10, 2).size(), is(1)); - assertThat(registry.getKnownAddresses(10, 1, 2).size(), is(3)); - } - - @Test - public void ensureOldNodesAreRemoved() { - registry.offerAddresses(Arrays.asList( - new NetworkAddress.Builder() - .ipv4(127, 0, 0, 1) - .port(42) - .stream(1) - .time(UnixTime.now()) - .build(), - new NetworkAddress.Builder() - .ipv4(127, 0, 0, 2) - .port(42) - .stream(1) - .time(UnixTime.now(-4 * HOUR)) - .build(), - new NetworkAddress.Builder() - .ipv4(127, 0, 0, 2) - .port(42) - .stream(2) - .time(UnixTime.now()) - .build() - )); - assertThat(registry.getKnownAddresses(10, 1).size(), is(1)); - assertThat(registry.getKnownAddresses(10, 2).size(), is(1)); - assertThat(registry.getKnownAddresses(10, 1, 2).size(), is(2)); - } -} diff --git a/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java b/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java index 84e8ce0..c2cc7a5 100644 --- a/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java +++ b/demo/src/main/java/ch/dissem/bitmessage/demo/Main.java @@ -20,7 +20,6 @@ import ch.dissem.bitmessage.BitmessageContext; import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.networking.nio.NioNetworkHandler; -import ch.dissem.bitmessage.ports.MemoryNodeRegistry; import ch.dissem.bitmessage.ports.NodeRegistry; import ch.dissem.bitmessage.repository.*; import ch.dissem.bitmessage.wif.WifExporter; @@ -82,7 +81,7 @@ public class Main { } }); } else { - ctxBuilder.nodeRegistry(new MemoryNodeRegistry()); + ctxBuilder.nodeRegistry(new JdbcNodeRegistry(jdbcConfig)); } if (options.exportWIF != null || options.importWIF != null) { diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index fb4cf58..8ba5b66 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -129,7 +129,7 @@ public class ConnectionInfo extends AbstractConnection { } @Override - public synchronized void disconnect() { + public void disconnect() { super.disconnect(); if (reader != null) { reader.cleanup(); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 7f04aa1..0cb0af9 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -26,7 +26,7 @@ import ch.dissem.bitmessage.exception.ApplicationException; import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.factory.V3MessageReader; import ch.dissem.bitmessage.ports.NetworkHandler; -import ch.dissem.bitmessage.utils.Property; +import ch.dissem.bitmessage.utils.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +37,7 @@ import java.net.NoRouteToHostException; import java.nio.ByteBuffer; import java.nio.channels.*; import java.util.*; +import java.util.Collections; import java.util.concurrent.*; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.*; @@ -193,7 +194,8 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } } if (missing > 0) { - List addresses = ctx.getNodeRegistry().getKnownAddresses(missing, ctx.getStreams()); + List addresses = ctx.getNodeRegistry().getKnownAddresses(100, ctx.getStreams()); + addresses = selectRandom(missing, addresses); for (NetworkAddress address : addresses) { if (isConnectedTo(address)) { continue; @@ -389,7 +391,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex ConnectionInfo previous = null; do { for (ConnectionInfo connection : distribution.keySet()) { - if (connection == previous) { + if (connection == previous || previous == null) { next = iterator.next(); } if (connection.knowsOf(next)) { diff --git a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java new file mode 100644 index 0000000..7374447 --- /dev/null +++ b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java @@ -0,0 +1,167 @@ +package ch.dissem.bitmessage.repository; + +import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; +import ch.dissem.bitmessage.exception.ApplicationException; +import ch.dissem.bitmessage.ports.NodeRegistry; +import ch.dissem.bitmessage.utils.Collections; +import ch.dissem.bitmessage.utils.SqlStrings; +import ch.dissem.bitmessage.utils.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.*; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static ch.dissem.bitmessage.ports.NodeRegistryHelper.loadStableNodes; +import static ch.dissem.bitmessage.utils.UnixTime.*; + +public class JdbcNodeRegistry extends JdbcHelper implements NodeRegistry { + private static final Logger LOG = LoggerFactory.getLogger(JdbcNodeRegistry.class); + private Map> stableNodes; + + public JdbcNodeRegistry(JdbcConfig config) { + super(config); + cleanUp(); + } + + private void cleanUp() { + try ( + Connection connection = config.getConnection(); + PreparedStatement ps = connection.prepareStatement( + "DELETE FROM Node WHERE time getKnownAddresses(int limit, long... streams) { + List result = new LinkedList<>(); + String query = + "SELECT stream, address, port, services, time" + + " FROM Node WHERE stream IN (" + SqlStrings.join(streams) + ")" + + " ORDER BY TIME DESC" + + " LIMIT " + limit; + try ( + Connection connection = config.getConnection(); + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(query) + ) { + while (rs.next()) { + result.add( + new NetworkAddress.Builder() + .stream(rs.getLong("stream")) + .ipv6(rs.getBytes("address")) + .port(rs.getInt("port")) + .services(rs.getLong("services")) + .time(rs.getLong("time")) + .build() + ); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new ApplicationException(e); + } + if (result.isEmpty()) { + synchronized (this) { + if (stableNodes == null) { + stableNodes = loadStableNodes(); + } + } + for (long stream : streams) { + Set nodes = stableNodes.get(stream); + if (nodes != null) { + result.add(Collections.selectRandom(nodes)); + } + } + } + return result; + } + + @Override + public void offerAddresses(List nodes) { + cleanUp(); + nodes.stream() + .filter(node -> node.getTime() < now(+24 * HOUR) && node.getTime() > now(-28 * DAY)) + .forEach(node -> { + synchronized (this) { + NetworkAddress existing = loadExisting(node); + if (existing == null) { + insert(node); + } else if (node.getTime() > existing.getTime()) { + update(node); + } + } + }); + } + + private void insert(NetworkAddress node) { + try ( + Connection connection = config.getConnection(); + PreparedStatement ps = connection.prepareStatement( + "INSERT INTO Node (stream, address, port, services, time) " + + "VALUES (?, ?, ?, ?, ?)") + ) { + ps.setLong(1, node.getStream()); + ps.setBytes(2, node.getIPv6()); + ps.setInt(3, node.getPort()); + ps.setLong(4, node.getServices()); + ps.setLong(5, node.getTime()); + ps.executeUpdate(); + } catch (SQLException e) { + LOG.error(e.getMessage(), e); + } + } + + private void update(NetworkAddress node) { + try ( + Connection connection = config.getConnection(); + PreparedStatement ps = connection.prepareStatement( + "UPDATE Node SET services=?, time=? WHERE stream=? AND address=? AND port=?") + ) { + ps.setLong(1, node.getServices()); + ps.setLong(2, node.getTime()); + ps.setLong(3, node.getStream()); + ps.setBytes(4, node.getIPv6()); + ps.setInt(5, node.getPort()); + ps.executeUpdate(); + } catch (SQLException e) { + LOG.error(e.getMessage(), e); + } + } +} diff --git a/repositories/src/main/resources/db/migration/V3.3__Create_table_node.sql b/repositories/src/main/resources/db/migration/V3.3__Create_table_node.sql new file mode 100644 index 0000000..5d03bb5 --- /dev/null +++ b/repositories/src/main/resources/db/migration/V3.3__Create_table_node.sql @@ -0,0 +1,9 @@ +CREATE TABLE Node ( + stream BIGINT NOT NULL, + address BINARY(32) NOT NULL, + port INT NOT NULL, + services BIGINT NOT NULL, + time BIGINT NOT NULL, + PRIMARY KEY (stream, address, port) +); +CREATE INDEX idx_time on Node(time); diff --git a/repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcNodeRegistryTest.java b/repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcNodeRegistryTest.java index f56d973..48ae664 100644 --- a/repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcNodeRegistryTest.java +++ b/repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcNodeRegistryTest.java @@ -17,8 +17,8 @@ package ch.dissem.bitmessage.repository; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; -import ch.dissem.bitmessage.ports.MemoryNodeRegistry; import ch.dissem.bitmessage.ports.NodeRegistry; +import ch.dissem.bitmessage.utils.UnixTime; import org.junit.Before; import org.junit.Test; @@ -27,8 +27,15 @@ import java.util.Collections; import java.util.List; import static ch.dissem.bitmessage.utils.UnixTime.now; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +/** + * Please note that some tests fail if there is no internet connection, + * as the initial nodes' IP addresses are determined by DNS lookup. + */ public class JdbcNodeRegistryTest extends TestBase { private TestJdbcConfig config; private NodeRegistry registry; @@ -37,21 +44,26 @@ public class JdbcNodeRegistryTest extends TestBase { public void setUp() throws Exception { config = new TestJdbcConfig(); config.reset(); - registry = new MemoryNodeRegistry(); + registry = new JdbcNodeRegistry(config); registry.offerAddresses(Arrays.asList( - createAddress(1, 8444, 1, now()), - createAddress(2, 8444, 1, now()), - createAddress(3, 8444, 1, now()), - createAddress(4, 8444, 2, now()) + createAddress(1, 8444, 1, now()), + createAddress(2, 8444, 1, now()), + createAddress(3, 8444, 1, now()), + createAddress(4, 8444, 2, now()) )); } @Test - public void testInitNodes() throws Exception { + public void ensureGetKnownNodesWithoutStreamsYieldsEmpty() { + assertThat(registry.getKnownAddresses(10), empty()); + } + + @Test + public void ensurePredefinedNodeIsReturnedWhenDatabaseIsEmpty() throws Exception { config.reset(); List knownAddresses = registry.getKnownAddresses(2, 1); - assertEquals(2, knownAddresses.size()); + assertEquals(1, knownAddresses.size()); } @Test @@ -66,16 +78,16 @@ public class JdbcNodeRegistryTest extends TestBase { @Test public void testOfferAddresses() throws Exception { registry.offerAddresses(Arrays.asList( - createAddress(1, 8444, 1, now()), - createAddress(10, 8444, 1, now()), - createAddress(11, 8444, 1, now()) + createAddress(1, 8444, 1, now()), + createAddress(10, 8444, 1, now()), + createAddress(11, 8444, 1, now()) )); List knownAddresses = registry.getKnownAddresses(1000, 1); assertEquals(5, knownAddresses.size()); registry.offerAddresses(Collections.singletonList( - createAddress(1, 8445, 1, now()) + createAddress(1, 8445, 1, now()) )); knownAddresses = registry.getKnownAddresses(1000, 1); @@ -84,10 +96,10 @@ public class JdbcNodeRegistryTest extends TestBase { private NetworkAddress createAddress(int lastByte, int port, long stream, long time) { return new NetworkAddress.Builder() - .ipv6(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, lastByte) - .port(port) - .stream(stream) - .time(time) - .build(); + .ipv6(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, lastByte) + .port(port) + .stream(stream) + .time(time) + .build(); } -} \ No newline at end of file +} From a240606909f9aa8343f472216d17f6e38784da13 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 5 Sep 2016 19:35:36 +0200 Subject: [PATCH 25/31] Minor improvements and fixes --- .../dissem/bitmessage/factory/BufferPool.java | 27 ++++++++++--------- .../networking/AbstractConnection.java | 2 +- .../networking/nio/ConnectionInfo.java | 1 + .../networking/nio/NioNetworkHandler.java | 12 +++++++-- .../repository/JdbcNodeRegistry.java | 2 +- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java b/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java index 1b977d7..65c1d34 100644 --- a/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java +++ b/core/src/main/java/ch/dissem/bitmessage/factory/BufferPool.java @@ -47,14 +47,14 @@ class BufferPool { } public synchronized ByteBuffer allocate(int capacity) { - for (Map.Entry> e : pools.entrySet()) { - if (e.getKey() >= capacity && !e.getValue().isEmpty()) { - return e.getValue().pop(); - } - } Integer targetSize = getTargetSize(capacity); - LOG.debug("Creating new buffer of size " + targetSize); - return ByteBuffer.allocate(targetSize); + Stack pool = pools.get(targetSize); + if (pool.isEmpty()) { + LOG.trace("Creating new buffer of size " + targetSize); + return ByteBuffer.allocate(targetSize); + } else { + return pool.pop(); + } } /** @@ -64,21 +64,22 @@ class BufferPool { */ public synchronized ByteBuffer allocateHeaderBuffer() { Stack pool = pools.get(HEADER_SIZE); - if (!pool.isEmpty()) { - return pool.pop(); - } else { + if (pool.isEmpty()) { return ByteBuffer.allocate(HEADER_SIZE); + } else { + return pool.pop(); } } public synchronized void deallocate(ByteBuffer buffer) { buffer.clear(); - - if (!pools.keySet().contains(buffer.capacity())) { + Stack pool = pools.get(buffer.capacity()); + if (pool == null) { throw new IllegalArgumentException("Illegal buffer capacity " + buffer.capacity() + " one of " + pools.keySet() + " expected."); + } else { + pool.push(buffer); } - pools.get(buffer.capacity()).push(buffer); } private Integer getTargetSize(int capacity) { diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 4a209aa..5d0f1c5 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -145,7 +145,7 @@ public abstract class AbstractConnection { updateIvCache(inv.getInventory()); List missing = ctx.getInventory().getMissing(inv.getInventory(), streams); missing.removeAll(commonRequestedObjects); - LOG.debug("Received inventory with " + originalSize + " elements, of which are " + LOG.trace("Received inventory with " + originalSize + " elements, of which are " + missing.size() + " missing."); send(new GetData.Builder().inventory(missing).build()); } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 8ba5b66..3e53eb4 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -135,6 +135,7 @@ public class ConnectionInfo extends AbstractConnection { reader.cleanup(); reader = null; } + payloadOut = null; } public boolean isSyncFinished() { diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 0cb0af9..a17f378 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -239,8 +239,8 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } e.getValue().cancel(); e.getValue().attach(null); - it.remove(); e.getKey().disconnect(); + it.remove(); } } try { @@ -387,12 +387,20 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex distribution.put(connection, new LinkedList()); } } + if (distribution.isEmpty()){ + return; + } InventoryVector next = iterator.next(); ConnectionInfo previous = null; do { for (ConnectionInfo connection : distribution.keySet()) { if (connection == previous || previous == null) { - next = iterator.next(); + if (iterator.hasNext()) { + previous = connection; + next = iterator.next(); + } else { + break; + } } if (connection.knowsOf(next)) { List ivs = distribution.get(connection); diff --git a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java index 7374447..ae5c432 100644 --- a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java +++ b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java @@ -117,7 +117,7 @@ public class JdbcNodeRegistry extends JdbcHelper implements NodeRegistry { public void offerAddresses(List nodes) { cleanUp(); nodes.stream() - .filter(node -> node.getTime() < now(+24 * HOUR) && node.getTime() > now(-28 * DAY)) + .filter(node -> node.getTime() < now(+2 * MINUTE) && node.getTime() > now(-28 * DAY)) .forEach(node -> { synchronized (this) { NetworkAddress existing = loadExisting(node); From 489b8968e00a3779e27b88135440fc8360efe7da Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 12 Sep 2016 08:18:30 +0200 Subject: [PATCH 26/31] Refactored use of the DefaultMessageListener so it's retrieved from the InternalContext --- .../dissem/bitmessage/BitmessageContext.java | 61 ++++---- .../bitmessage/DefaultMessageListener.java | 16 +- .../ch/dissem/bitmessage/InternalContext.java | 33 ++-- .../dissem/bitmessage/ProofOfWorkService.java | 18 ++- .../bitmessage/ports/NetworkHandler.java | 4 +- .../DefaultMessageListenerTest.java | 2 +- .../networking/AbstractConnection.java | 3 +- .../bitmessage/networking/Connection.java | 14 +- .../networking/ConnectionOrganizer.java | 8 +- .../networking/DefaultNetworkHandler.java | 13 +- .../bitmessage/networking/ServerRunnable.java | 8 +- .../networking/nio/ConnectionInfo.java | 11 +- .../networking/nio/NioNetworkHandler.java | 148 +++++++++--------- .../networking/NetworkHandlerTest.java | 12 +- .../repository/JdbcNodeRegistry.java | 2 +- 15 files changed, 180 insertions(+), 173 deletions(-) diff --git a/core/src/main/java/ch/dissem/bitmessage/BitmessageContext.java b/core/src/main/java/ch/dissem/bitmessage/BitmessageContext.java index b06cf9a..9e155ad 100644 --- a/core/src/main/java/ch/dissem/bitmessage/BitmessageContext.java +++ b/core/src/main/java/ch/dissem/bitmessage/BitmessageContext.java @@ -62,16 +62,16 @@ public class BitmessageContext { private final InternalContext ctx; private final Labeler labeler; - private final NetworkHandler.MessageListener networkListener; private final boolean sendPubkeyOnIdentityCreation; private BitmessageContext(Builder builder) { + if (builder.listener instanceof Listener.WithContext) { + ((Listener.WithContext) builder.listener).setContext(this); + } ctx = new InternalContext(builder); labeler = builder.labeler; ctx.getProofOfWorkService().doMissingProofOfWork(30_000); // TODO: this should be configurable - - networkListener = new DefaultMessageListener(ctx, labeler, builder.listener); sendPubkeyOnIdentityCreation = builder.sendPubkeyOnIdentityCreation; } @@ -89,11 +89,11 @@ public class BitmessageContext { public BitmessageAddress createIdentity(boolean shorter, Feature... features) { final BitmessageAddress identity = new BitmessageAddress(new PrivateKey( - shorter, - ctx.getStreams()[0], - NETWORK_NONCE_TRIALS_PER_BYTE, - NETWORK_EXTRA_BYTES, - features + shorter, + ctx.getStreams()[0], + NETWORK_NONCE_TRIALS_PER_BYTE, + NETWORK_EXTRA_BYTES, + features )); ctx.getAddressRepository().save(identity); if (sendPubkeyOnIdentityCreation) { @@ -117,9 +117,9 @@ public class BitmessageContext { } public List createDeterministicAddresses( - String passphrase, int numberOfAddresses, long version, long stream, boolean shorter) { + String passphrase, int numberOfAddresses, long version, long stream, boolean shorter) { List result = BitmessageAddress.deterministic( - passphrase, numberOfAddresses, version, stream, shorter); + passphrase, numberOfAddresses, version, stream, shorter); for (int i = 0; i < result.size(); i++) { BitmessageAddress address = result.get(i); address.setAlias("deterministic (" + (i + 1) + ")"); @@ -130,9 +130,9 @@ public class BitmessageContext { public void broadcast(final BitmessageAddress from, final String subject, final String message) { Plaintext msg = new Plaintext.Builder(BROADCAST) - .from(from) - .message(subject, message) - .build(); + .from(from) + .message(subject, message) + .build(); send(msg); } @@ -141,10 +141,10 @@ public class BitmessageContext { throw new IllegalArgumentException("'From' must be an identity, i.e. have a private key."); } Plaintext msg = new Plaintext.Builder(MSG) - .from(from) - .to(to) - .message(subject, message) - .build(); + .from(from) + .to(to) + .message(subject, message) + .build(); send(msg); } @@ -170,17 +170,17 @@ public class BitmessageContext { ctx.send(msg); } else { ctx.send( - msg.getFrom(), - to, - Factory.getBroadcast(msg), - msg.getTTL() + msg.getFrom(), + to, + Factory.getBroadcast(msg), + msg.getTTL() ); } } } public void startup() { - ctx.getNetworkHandler().start(networkListener); + ctx.getNetworkHandler().start(); } public void shutdown() { @@ -195,7 +195,7 @@ public class BitmessageContext { * @param wait waits for the synchronization thread to finish */ public void synchronize(InetAddress host, int port, long timeoutInSeconds, boolean wait) { - Future future = ctx.getNetworkHandler().synchronize(host, port, networkListener, timeoutInSeconds); + Future future = ctx.getNetworkHandler().synchronize(host, port, timeoutInSeconds); if (wait) { try { future.get(); @@ -271,7 +271,7 @@ public class BitmessageContext { broadcast.decrypt(address); // This decrypts it twice, but on the other hand it doesn't try to decrypt the objects with // other subscriptions and the interface stays as simple as possible. - networkListener.receive(object); + ctx.getNetworkListener().receive(object); } catch (DecryptionFailedException ignore) { } catch (Exception e) { LOG.debug(e.getMessage(), e); @@ -281,8 +281,8 @@ public class BitmessageContext { public Property status() { return new Property("status", null, - ctx.getNetworkHandler().getNetworkStatus(), - new Property("unacknowledged", ctx.getMessageRepository().findMessagesToResend().size()) + ctx.getNetworkHandler().getNetworkStatus(), + new Property("unacknowledged", ctx.getMessageRepository().findMessagesToResend().size()) ); } @@ -296,6 +296,13 @@ public class BitmessageContext { public interface Listener { void receive(Plaintext plaintext); + + /** + * A message listener that needs a {@link BitmessageContext}, i.e. for implementing some sort of chat bot. + */ + interface WithContext extends Listener { + void setContext(BitmessageContext ctx); + } } public static final class Builder { @@ -429,7 +436,7 @@ public class BitmessageContext { @Override public MessagePayload handle(CustomMessage request) { throw new IllegalStateException( - "Received custom request, but no custom command handler configured."); + "Received custom request, but no custom command handler configured."); } }; } diff --git a/core/src/main/java/ch/dissem/bitmessage/DefaultMessageListener.java b/core/src/main/java/ch/dissem/bitmessage/DefaultMessageListener.java index b5d70d1..b61f747 100644 --- a/core/src/main/java/ch/dissem/bitmessage/DefaultMessageListener.java +++ b/core/src/main/java/ch/dissem/bitmessage/DefaultMessageListener.java @@ -24,7 +24,6 @@ import ch.dissem.bitmessage.entity.valueobject.InventoryVector; import ch.dissem.bitmessage.exception.DecryptionFailedException; import ch.dissem.bitmessage.ports.Labeler; import ch.dissem.bitmessage.ports.NetworkHandler; -import ch.dissem.bitmessage.utils.TTL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,21 +31,24 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import static ch.dissem.bitmessage.entity.Plaintext.Status.*; -import static ch.dissem.bitmessage.utils.UnixTime.DAY; +import static ch.dissem.bitmessage.entity.Plaintext.Status.PUBKEY_REQUESTED; -class DefaultMessageListener implements NetworkHandler.MessageListener { +class DefaultMessageListener implements NetworkHandler.MessageListener, InternalContext.ContextHolder { private final static Logger LOG = LoggerFactory.getLogger(DefaultMessageListener.class); - private final InternalContext ctx; private final Labeler labeler; private final BitmessageContext.Listener listener; + private InternalContext ctx; - public DefaultMessageListener(InternalContext context, Labeler labeler, BitmessageContext.Listener listener) { - this.ctx = context; + public DefaultMessageListener(Labeler labeler, BitmessageContext.Listener listener) { this.labeler = labeler; this.listener = listener; } + @Override + public void setContext(InternalContext context) { + this.ctx = context; + } + @Override @SuppressWarnings("ConstantConditions") public void receive(ObjectMessage object) throws IOException { diff --git a/core/src/main/java/ch/dissem/bitmessage/InternalContext.java b/core/src/main/java/ch/dissem/bitmessage/InternalContext.java index 057bfef..e01a90a 100644 --- a/core/src/main/java/ch/dissem/bitmessage/InternalContext.java +++ b/core/src/main/java/ch/dissem/bitmessage/InternalContext.java @@ -56,6 +56,7 @@ public class InternalContext { private final CustomCommandHandler customCommandHandler; private final ProofOfWorkService proofOfWorkService; private final Labeler labeler; + private final NetworkHandler.MessageListener networkListener; private final TreeSet streams = new TreeSet<>(); private final int port; @@ -79,6 +80,7 @@ public class InternalContext { this.connectionLimit = builder.connectionLimit; this.connectionTTL = builder.connectionTTL; this.labeler = builder.labeler; + this.networkListener = new DefaultMessageListener(labeler, builder.listener); Singleton.initialize(cryptography); @@ -94,7 +96,8 @@ public class InternalContext { } init(cryptography, inventory, nodeRegistry, networkHandler, addressRepository, messageRepository, - proofOfWorkRepository, proofOfWorkService, proofOfWorkEngine, customCommandHandler, builder.labeler); + proofOfWorkRepository, proofOfWorkService, proofOfWorkEngine, customCommandHandler, builder.labeler, + networkListener); for (BitmessageAddress identity : addressRepository.getIdentities()) { streams.add(identity.getStream()); } @@ -148,6 +151,10 @@ public class InternalContext { return labeler; } + public NetworkHandler.MessageListener getNetworkListener() { + return networkListener; + } + public long[] getStreams() { long[] result = new long[streams.size()]; int i = 0; @@ -178,10 +185,10 @@ public class InternalContext { long expires = UnixTime.now(+timeToLive); LOG.info("Expires at " + expires); final ObjectMessage object = new ObjectMessage.Builder() - .stream(recipient.getStream()) - .expiresTime(expires) - .payload(payload) - .build(); + .stream(recipient.getStream()) + .expiresTime(expires) + .payload(payload) + .build(); if (object.isSigned()) { object.sign(from.getPrivateKey()); } @@ -201,10 +208,10 @@ public class InternalContext { long expires = UnixTime.now(TTL.pubkey()); LOG.info("Expires at " + expires); final ObjectMessage response = new ObjectMessage.Builder() - .stream(targetStream) - .expiresTime(expires) - .payload(identity.getPubkey()) - .build(); + .stream(targetStream) + .expiresTime(expires) + .payload(identity.getPubkey()) + .build(); response.sign(identity.getPrivateKey()); response.encrypt(cryptography.createPublicKey(identity.getPublicDecryptionKey())); // TODO: remember that the pubkey is just about to be sent, and on which stream! @@ -239,10 +246,10 @@ public class InternalContext { long expires = UnixTime.now(TTL.getpubkey()); LOG.info("Expires at " + expires); final ObjectMessage request = new ObjectMessage.Builder() - .stream(contact.getStream()) - .expiresTime(expires) - .payload(new GetPubkey(contact)) - .build(); + .stream(contact.getStream()) + .expiresTime(expires) + .payload(new GetPubkey(contact)) + .build(); proofOfWorkService.doProofOfWork(request); } diff --git a/core/src/main/java/ch/dissem/bitmessage/ProofOfWorkService.java b/core/src/main/java/ch/dissem/bitmessage/ProofOfWorkService.java index e9a27db..7c72bea 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ProofOfWorkService.java +++ b/core/src/main/java/ch/dissem/bitmessage/ProofOfWorkService.java @@ -11,6 +11,7 @@ import ch.dissem.bitmessage.ports.ProofOfWorkRepository.Item; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.List; import java.util.Timer; import java.util.TimerTask; @@ -42,7 +43,7 @@ public class ProofOfWorkService implements ProofOfWorkEngine.Callback, InternalC for (byte[] initialHash : items) { Item item = powRepo.getItem(initialHash); cryptography.doProofOfWork(item.object, item.nonceTrialsPerByte, item.extraBytes, - ProofOfWorkService.this); + ProofOfWorkService.this); } } }, delayInMilliseconds); @@ -71,7 +72,7 @@ public class ProofOfWorkService implements ProofOfWorkEngine.Callback, InternalC final ObjectMessage ack = plaintext.getAckMessage(); messageRepo.save(plaintext); Item item = new Item(ack, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES, - expirationTime, plaintext); + expirationTime, plaintext); powRepo.putObject(item); cryptography.doProofOfWork(ack, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES, this); } @@ -89,15 +90,20 @@ public class ProofOfWorkService implements ProofOfWorkEngine.Callback, InternalC ctx.getLabeler().markAsSent(plaintext); messageRepo.save(plaintext); } + try { + ctx.getNetworkListener().receive(object); + } catch (IOException e) { + LOG.debug(e.getMessage(), e); + } ctx.getInventory().storeObject(object); ctx.getNetworkHandler().offer(object.getInventoryVector()); } else { item.message.getAckMessage().setNonce(nonce); final ObjectMessage object = new ObjectMessage.Builder() - .stream(item.message.getStream()) - .expiresTime(item.expirationTime) - .payload(new Msg(item.message)) - .build(); + .stream(item.message.getStream()) + .expiresTime(item.expirationTime) + .payload(new Msg(item.message)) + .build(); if (object.isSigned()) { object.sign(item.message.getFrom().getPrivateKey()); } diff --git a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java index 9597625..e9c164e 100644 --- a/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java +++ b/core/src/main/java/ch/dissem/bitmessage/ports/NetworkHandler.java @@ -41,7 +41,7 @@ public interface NetworkHandler { * An implementation should disconnect if either the timeout is reached or the returned thread is interrupted. *

*/ - Future synchronize(InetAddress server, int port, MessageListener listener, long timeoutInSeconds); + Future synchronize(InetAddress server, int port, long timeoutInSeconds); /** * Send a custom message to a specific node (that should implement handling for this message type) and returns @@ -57,7 +57,7 @@ public interface NetworkHandler { /** * Start a full network node, accepting incoming connections and relaying objects. */ - void start(MessageListener listener); + void start(); /** * Stop the full network node. diff --git a/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java b/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java index 170cb93..a9bbd8c 100644 --- a/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java +++ b/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java @@ -68,7 +68,7 @@ public class DefaultMessageListenerTest extends TestBase { when(ctx.getNetworkHandler()).thenReturn(networkHandler); when(ctx.getLabeler()).thenReturn(mock(Labeler.class)); - listener = new DefaultMessageListener(ctx, mock(Labeler.class), mock(BitmessageContext.Listener.class)); + listener = new DefaultMessageListener(mock(Labeler.class), mock(BitmessageContext.Listener.class)); } @Test diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java index 5d0f1c5..73dce63 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java @@ -71,14 +71,13 @@ public abstract class AbstractConnection { public AbstractConnection(InternalContext context, Mode mode, NetworkAddress node, - NetworkHandler.MessageListener listener, Set commonRequestedObjects, long syncTimeout) { this.ctx = context; this.mode = mode; this.host = new NetworkAddress.Builder().ipv6(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).port(0).build(); this.node = node; - this.listener = listener; + this.listener = context.getNetworkListener(); this.syncTimeout = (syncTimeout > 0 ? UnixTime.now(+syncTimeout) : 0); this.requestedObjects = Collections.newSetFromMap(new ConcurrentHashMap(10_000)); this.ivCache = new ConcurrentHashMap<>(); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java index e2c1856..64772e5 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java @@ -63,29 +63,29 @@ class Connection extends AbstractConnection { private OutputStream out; private boolean socketInitialized; - public Connection(InternalContext context, Mode mode, Socket socket, MessageListener listener, + public Connection(InternalContext context, Mode mode, Socket socket, Set requestedObjectsMap) throws IOException { - this(context, mode, listener, socket, requestedObjectsMap, + this(context, mode, socket, requestedObjectsMap, new NetworkAddress.Builder().ip(socket.getInetAddress()).port(socket.getPort()).stream(1).build(), 0); } - public Connection(InternalContext context, Mode mode, NetworkAddress node, MessageListener listener, + public Connection(InternalContext context, Mode mode, NetworkAddress node, Set requestedObjectsMap) { - this(context, mode, listener, new Socket(), requestedObjectsMap, + this(context, mode, new Socket(), requestedObjectsMap, node, 0); } - private Connection(InternalContext context, Mode mode, MessageListener listener, Socket socket, + private Connection(InternalContext context, Mode mode, Socket socket, Set commonRequestedObjects, NetworkAddress node, long syncTimeout) { - super(context, mode, node, listener, commonRequestedObjects, syncTimeout); + super(context, mode, node, commonRequestedObjects, syncTimeout); this.startTime = UnixTime.now(); this.socket = socket; } public static Connection sync(InternalContext ctx, InetAddress address, int port, MessageListener listener, long timeoutInSeconds) throws IOException { - return new Connection(ctx, SYNC, listener, new Socket(address, port), + return new Connection(ctx, SYNC, new Socket(address, port), new HashSet(), new NetworkAddress.Builder().ip(address).port(port).stream(1).build(), timeoutInSeconds); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java b/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java index 5c976e2..42021f6 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java @@ -42,11 +42,10 @@ public class ConnectionOrganizer implements Runnable { private Connection initialConnection; public ConnectionOrganizer(InternalContext ctx, - DefaultNetworkHandler networkHandler, - NetworkHandler.MessageListener listener) { + DefaultNetworkHandler networkHandler) { this.ctx = ctx; this.networkHandler = networkHandler; - this.listener = listener; + this.listener = ctx.getNetworkListener(); } @Override @@ -91,8 +90,7 @@ public class ConnectionOrganizer implements Runnable { NETWORK_MAGIC_NUMBER - active, ctx.getStreams()); boolean first = active == 0 && initialConnection == null; for (NetworkAddress address : addresses) { - Connection c = new Connection(ctx, CLIENT, address, listener, - networkHandler.requestedObjects); + Connection c = new Connection(ctx, CLIENT, address, networkHandler.requestedObjects); if (first) { initialConnection = c; first = false; diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java index f5bfd47..762d1be 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java @@ -64,9 +64,9 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { } @Override - public Future synchronize(InetAddress server, int port, MessageListener listener, long timeoutInSeconds) { + public Future synchronize(InetAddress server, int port, long timeoutInSeconds) { try { - Connection connection = Connection.sync(ctx, server, port, listener, timeoutInSeconds); + Connection connection = Connection.sync(ctx, server, port, ctx.getNetworkListener(), timeoutInSeconds); Future reader = pool.submit(connection.getReader()); pool.execute(connection.getWriter()); return reader; @@ -97,19 +97,16 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { } @Override - public void start(final MessageListener listener) { - if (listener == null) { - throw new IllegalStateException("Listener must be set at start"); - } + public void start() { if (running) { throw new IllegalStateException("Network already running - you need to stop first."); } try { running = true; connections.clear(); - server = new ServerRunnable(ctx, this, listener); + server = new ServerRunnable(ctx, this); pool.execute(server); - pool.execute(new ConnectionOrganizer(ctx, this, listener)); + pool.execute(new ConnectionOrganizer(ctx, this)); } catch (IOException e) { throw new ApplicationException(e); } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java b/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java index 99b6a88..3c67b95 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java @@ -38,11 +38,10 @@ public class ServerRunnable implements Runnable, Closeable { private final DefaultNetworkHandler networkHandler; private final NetworkHandler.MessageListener listener; - public ServerRunnable(InternalContext ctx, DefaultNetworkHandler networkHandler, - NetworkHandler.MessageListener listener) throws IOException { + public ServerRunnable(InternalContext ctx, DefaultNetworkHandler networkHandler) throws IOException { this.ctx = ctx; this.networkHandler = networkHandler; - this.listener = listener; + this.listener = ctx.getNetworkListener(); this.serverSocket = new ServerSocket(ctx.getPort()); } @@ -52,8 +51,7 @@ public class ServerRunnable implements Runnable, Closeable { try { Socket socket = serverSocket.accept(); socket.setSoTimeout(Connection.READ_TIMEOUT); - networkHandler.startConnection(new Connection(ctx, SERVER, socket, listener, - networkHandler.requestedObjects)); + networkHandler.startConnection(new Connection(ctx, SERVER, socket, networkHandler.requestedObjects)); } catch (IOException e) { LOG.debug(e.getMessage(), e); } diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java index 3e53eb4..2e24883 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java @@ -17,13 +17,15 @@ package ch.dissem.bitmessage.networking.nio; import ch.dissem.bitmessage.InternalContext; -import ch.dissem.bitmessage.entity.*; +import ch.dissem.bitmessage.entity.GetData; +import ch.dissem.bitmessage.entity.MessagePayload; +import ch.dissem.bitmessage.entity.NetworkMessage; +import ch.dissem.bitmessage.entity.Version; import ch.dissem.bitmessage.entity.valueobject.InventoryVector; import ch.dissem.bitmessage.entity.valueobject.NetworkAddress; import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.factory.V3MessageReader; import ch.dissem.bitmessage.networking.AbstractConnection; -import ch.dissem.bitmessage.ports.NetworkHandler; import java.nio.ByteBuffer; import java.util.Iterator; @@ -43,10 +45,9 @@ public class ConnectionInfo extends AbstractConnection { private boolean syncFinished; private long lastUpdate = System.currentTimeMillis(); - public ConnectionInfo(InternalContext context, Mode mode, - NetworkAddress node, NetworkHandler.MessageListener listener, + public ConnectionInfo(InternalContext context, Mode mode, NetworkAddress node, Set commonRequestedObjects, long syncTimeout) { - super(context, mode, node, listener, commonRequestedObjects, syncTimeout); + super(context, mode, node, commonRequestedObjects, syncTimeout); headerOut.flip(); if (mode == CLIENT || mode == SYNC) { send(new Version.Builder().defaults(ctx.getClientNonce()).addrFrom(host).addrRecv(node).build()); diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index a17f378..6baa68b 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -26,7 +26,7 @@ import ch.dissem.bitmessage.exception.ApplicationException; import ch.dissem.bitmessage.exception.NodeException; import ch.dissem.bitmessage.factory.V3MessageReader; import ch.dissem.bitmessage.ports.NetworkHandler; -import ch.dissem.bitmessage.utils.*; +import ch.dissem.bitmessage.utils.Property; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,6 @@ import java.net.NoRouteToHostException; import java.nio.ByteBuffer; import java.nio.channels.*; import java.util.*; -import java.util.Collections; import java.util.concurrent.*; import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.*; @@ -47,6 +46,7 @@ import static ch.dissem.bitmessage.utils.Collections.selectRandom; import static ch.dissem.bitmessage.utils.DebugUtils.inc; import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool; import static java.nio.channels.SelectionKey.*; +import static java.util.Collections.newSetFromMap; /** * Network handler using java.nio, resulting in less threads. @@ -63,13 +63,14 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private InternalContext ctx; private Selector selector; private ServerSocketChannel serverChannel; + private Queue connectionQueue = new ConcurrentLinkedQueue<>(); private Map connections = new ConcurrentHashMap<>(); - private final Set requestedObjects = Collections.newSetFromMap(new ConcurrentHashMap(10_000)); + private final Set requestedObjects = newSetFromMap(new ConcurrentHashMap(10_000)); private Thread starter; @Override - public Future synchronize(final InetAddress server, final int port, final MessageListener listener, final long timeoutInSeconds) { + public Future synchronize(final InetAddress server, final int port, final long timeoutInSeconds) { return threadPool.submit(new Callable() { @Override public Void call() throws Exception { @@ -77,7 +78,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex channel.configureBlocking(false); ConnectionInfo connection = new ConnectionInfo(ctx, SYNC, new NetworkAddress.Builder().ip(server).port(port).stream(1).build(), - listener, new HashSet(), timeoutInSeconds); + new HashSet(), timeoutInSeconds); while (channel.isConnected() && !connection.isSyncFinished()) { write(channel, connection); read(channel, connection); @@ -135,10 +136,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } @Override - public void start(final MessageListener listener) { - if (listener == null) { - throw new IllegalStateException("Listener must be set at start"); - } + public void start() { if (selector != null && selector.isOpen()) { throw new IllegalStateException("Network already running - you need to stop first."); } @@ -147,42 +145,8 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } catch (IOException e) { throw new ApplicationException(e); } - thread("connection listener", new Runnable() { - @Override - public void run() { - try { - serverChannel = ServerSocketChannel.open(); - serverChannel.socket().bind(new InetSocketAddress(ctx.getPort())); - while (selector.isOpen() && serverChannel.isOpen()) { - try { - SocketChannel accepted = serverChannel.accept(); - accepted.configureBlocking(false); - ConnectionInfo connection = new ConnectionInfo(ctx, SERVER, - new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), - listener, - requestedObjects, 0 - ); - connections.put( - connection, - accepted.register(selector, OP_READ | OP_WRITE, connection) - ); - } catch (AsynchronousCloseException ignore) { - LOG.trace(ignore.getMessage()); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - } - } catch (ClosedSelectorException | AsynchronousCloseException ignore) { - } catch (IOException e) { - throw new ApplicationException(e); - } catch (RuntimeException e) { - e.printStackTrace(); - throw e; - } - } - }); - starter = thread("connection starter", new Runnable() { + starter = thread("connection manager", new Runnable() { @Override public void run() { while (selector.isOpen()) { @@ -197,34 +161,8 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex List addresses = ctx.getNodeRegistry().getKnownAddresses(100, ctx.getStreams()); addresses = selectRandom(missing, addresses); for (NetworkAddress address : addresses) { - if (isConnectedTo(address)) { - continue; - } - try { - SocketChannel channel = SocketChannel.open(); - channel.configureBlocking(false); - channel.connect(new InetSocketAddress(address.toInetAddress(), address.getPort())); - ConnectionInfo connection = new ConnectionInfo(ctx, CLIENT, - address, - listener, - requestedObjects, 0 - ); - connections.put( - connection, - channel.register(selector, OP_CONNECT, connection) - ); - } catch (NoRouteToHostException ignore) { - // We'll try to connect to many offline nodes, so - // this is expected to happen quite a lot. - } catch (AsynchronousCloseException e) { - // The exception is expected if the network is being - // shut down, as we actually do asynchronously close - // the connections. - if (isRunning()) { - LOG.error(e.getMessage(), e); - } - } catch (IOException e) { - LOG.error(e.getMessage(), e); + if (!isConnectedTo(address)) { + connectionQueue.offer(address); } } } @@ -252,17 +190,47 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } }); - thread("processor", new Runnable() { + thread("selector worker", new Runnable() { @Override public void run() { try { + serverChannel = ServerSocketChannel.open(); + serverChannel.configureBlocking(false); + serverChannel.socket().bind(new InetSocketAddress(ctx.getPort())); + serverChannel.register(selector, OP_ACCEPT, null); + while (selector.isOpen()) { selector.select(1000); Iterator keyIterator = selector.selectedKeys().iterator(); while (keyIterator.hasNext()) { SelectionKey key = keyIterator.next(); keyIterator.remove(); - if (key.attachment() instanceof ConnectionInfo) { + if (key.attachment() == null) { + try { + if (key.isAcceptable()) { + // handle accept + try { + SocketChannel accepted = ((ServerSocketChannel) key.channel()).accept(); + accepted.configureBlocking(false); + ConnectionInfo connection = new ConnectionInfo(ctx, SERVER, + new NetworkAddress.Builder().address(accepted.getRemoteAddress()).stream(1).build(), + requestedObjects, 0 + ); + connections.put( + connection, + accepted.register(selector, OP_READ | OP_WRITE, connection) + ); + } catch (AsynchronousCloseException e) { + LOG.trace(e.getMessage()); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + } catch (CancelledKeyException e) { + LOG.error(e.getMessage(), e); + } + } else { + // handle read/write SocketChannel channel = (SocketChannel) key.channel(); ConnectionInfo connection = (ConnectionInfo) key.attachment(); try { @@ -290,6 +258,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } } } + // set interest ops for (Map.Entry e : connections.entrySet()) { if (e.getValue().isValid() && (e.getValue().interestOps() & OP_WRITE) == 0 @@ -298,6 +267,35 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex e.getValue().interestOps(OP_READ | OP_WRITE); } } + // start new connections + if (!connectionQueue.isEmpty()) { + NetworkAddress address = connectionQueue.poll(); + try { + SocketChannel channel = SocketChannel.open(); + channel.configureBlocking(false); + channel.connect(new InetSocketAddress(address.toInetAddress(), address.getPort())); + ConnectionInfo connection = new ConnectionInfo(ctx, CLIENT, + address, + requestedObjects, 0 + ); + connections.put( + connection, + channel.register(selector, OP_CONNECT, connection) + ); + } catch (NoRouteToHostException ignore) { + // We'll try to connect to many offline nodes, so + // this is expected to happen quite a lot. + } catch (AsynchronousCloseException e) { + // The exception is expected if the network is being + // shut down, as we actually do asynchronously close + // the connections. + if (isRunning()) { + LOG.error(e.getMessage(), e); + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } } selector.close(); } catch (ClosedSelectorException ignore) { @@ -387,7 +385,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex distribution.put(connection, new LinkedList()); } } - if (distribution.isEmpty()){ + if (distribution.isEmpty()) { return; } InventoryVector next = iterator.next(); diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index ba24bbd..48c8cd1 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -230,9 +230,7 @@ public class NetworkHandlerTest { "V4Pubkey.payload" ); - Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), - mock(NetworkHandler.MessageListener.class), - 10); + Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), 10); future.get(); assertInventorySize(3, nodeInventory); assertInventorySize(3, peerInventory); @@ -247,9 +245,7 @@ public class NetworkHandlerTest { nodeInventory.init(); - Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), - mock(NetworkHandler.MessageListener.class), - 10); + Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), 10); future.get(); assertInventorySize(2, nodeInventory); assertInventorySize(2, peerInventory); @@ -263,9 +259,7 @@ public class NetworkHandlerTest { "V1Msg.payload" ); - Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), - mock(NetworkHandler.MessageListener.class), - 10); + Future future = nodeNetworkHandler.synchronize(peerAddress.toInetAddress(), peerAddress.getPort(), 10); future.get(); assertInventorySize(1, nodeInventory); assertInventorySize(1, peerInventory); diff --git a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java index ae5c432..07d343a 100644 --- a/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java +++ b/repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java @@ -105,7 +105,7 @@ public class JdbcNodeRegistry extends JdbcHelper implements NodeRegistry { } for (long stream : streams) { Set nodes = stableNodes.get(stream); - if (nodes != null) { + if (nodes != null && !nodes.isEmpty()) { result.add(Collections.selectRandom(nodes)); } } From 71124d7b01166d78bae1121bbf09f5e495af6890 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 12 Sep 2016 10:02:52 +0200 Subject: [PATCH 27/31] Fixed tests --- .../ch/dissem/bitmessage/DefaultMessageListenerTest.java | 2 ++ .../java/ch/dissem/bitmessage/ProofOfWorkServiceTest.java | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java b/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java index a9bbd8c..32a20e4 100644 --- a/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java +++ b/core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java @@ -69,6 +69,8 @@ public class DefaultMessageListenerTest extends TestBase { when(ctx.getLabeler()).thenReturn(mock(Labeler.class)); listener = new DefaultMessageListener(mock(Labeler.class), mock(BitmessageContext.Listener.class)); + when(ctx.getNetworkListener()).thenReturn(listener); + listener.setContext(ctx); } @Test diff --git a/core/src/test/java/ch/dissem/bitmessage/ProofOfWorkServiceTest.java b/core/src/test/java/ch/dissem/bitmessage/ProofOfWorkServiceTest.java index c4529e3..039e7ad 100644 --- a/core/src/test/java/ch/dissem/bitmessage/ProofOfWorkServiceTest.java +++ b/core/src/test/java/ch/dissem/bitmessage/ProofOfWorkServiceTest.java @@ -66,6 +66,7 @@ public class ProofOfWorkServiceTest { when(ctx.getNetworkHandler()).thenReturn(networkHandler); when(ctx.getMessageRepository()).thenReturn(messageRepo); when(ctx.getLabeler()).thenReturn(mock(Labeler.class)); + when(ctx.getNetworkListener()).thenReturn(mock(NetworkHandler.MessageListener.class)); proofOfWorkService = new ProofOfWorkService(); proofOfWorkService.setContext(ctx); @@ -80,7 +81,7 @@ public class ProofOfWorkServiceTest { proofOfWorkService.doMissingProofOfWork(10); verify(cryptography, timeout(1000)).doProofOfWork((ObjectMessage) isNull(), eq(1001L), eq(1002L), - any(ProofOfWorkEngine.Callback.class)); + any(ProofOfWorkEngine.Callback.class)); } @Test @@ -89,8 +90,8 @@ public class ProofOfWorkServiceTest { BitmessageAddress address = TestUtils.loadContact(); Plaintext plaintext = new Plaintext.Builder(MSG).from(identity).to(address).message("", "").build(); ObjectMessage object = new ObjectMessage.Builder() - .payload(new Msg(plaintext)) - .build(); + .payload(new Msg(plaintext)) + .build(); object.sign(identity.getPrivateKey()); object.encrypt(address.getPubkey()); byte[] initialHash = new byte[64]; From e6d40cde769b354ebfc3ed3301d4e14ba2c49a31 Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Mon, 12 Sep 2016 11:15:07 +0200 Subject: [PATCH 28/31] Try to fix tests on travis. Unfortunately they work fine locally, so there may be several of those attempts. --- .../ch/dissem/bitmessage/networking/NetworkHandlerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java index 48c8cd1..d64a855 100644 --- a/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java +++ b/networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java @@ -86,7 +86,7 @@ public class NetworkHandlerTest { } @Before - public void setUp() { + public void setUp() throws InterruptedException { peerInventory = new TestInventory(); peer = new BitmessageContext.Builder() .addressRepo(mock(AddressRepository.class)) @@ -120,6 +120,7 @@ public class NetworkHandlerTest { }) .build(); peer.startup(); + Thread.sleep(100); nodeInventory = new TestInventory(); node = new BitmessageContext.Builder() From 83abce0f520935c039c1fd438b1ecf884ee503aa Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Tue, 13 Sep 2016 07:25:23 +0200 Subject: [PATCH 29/31] Deprecate DefaultNetworkHandler in favor of NioNetworkHandler --- .../ch/dissem/bitmessage/networking/DefaultNetworkHandler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java index 762d1be..1af62b5 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java @@ -43,7 +43,10 @@ import static java.util.Collections.newSetFromMap; /** * Handles all the networky stuff. + * + * @deprecated use {@link ch.dissem.bitmessage.networking.nio.NioNetworkHandler NioNetworkHandler} instead. */ +@Deprecated public class DefaultNetworkHandler implements NetworkHandler, ContextHolder { final Collection connections = new ConcurrentLinkedQueue<>(); From 7e201dd2cfedd3e5b7956795be5babfe482a5a9f Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Tue, 13 Sep 2016 07:36:28 +0200 Subject: [PATCH 30/31] Change version of 'develop' branch to 'development-SNAPSHOT' --- .../src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy b/buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy index 81af76b..869d57e 100644 --- a/buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy +++ b/buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy @@ -37,7 +37,11 @@ class GitFlowVersion implements Plugin { if (project.ext.isRelease) { return getTag(project) } else { - return getBranch(project).replaceAll("/", "-") + "-SNAPSHOT" + def branch = getBranch(project) + if ("develop" == branch) { + return "development-SNAPSHOT" + } + return branch.replaceAll("/", "-") + "-SNAPSHOT" } } From a18f76f8649a900428058c03fceed7793bb509ad Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Wed, 21 Sep 2016 19:37:17 +0200 Subject: [PATCH 31/31] Cleaning up requested objects from time to time, to work around a leak that sometimes happens. --- .../networking/nio/NioNetworkHandler.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java index 6baa68b..5800b38 100644 --- a/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java +++ b/networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java @@ -53,6 +53,7 @@ import static java.util.Collections.newSetFromMap; */ public class NioNetworkHandler implements NetworkHandler, InternalContext.ContextHolder { private static final Logger LOG = LoggerFactory.getLogger(NioNetworkHandler.class); + private static final long REQUESTED_OBJECTS_MAX_TIME = 30 * 60_000; // 30 minutes private final ExecutorService threadPool = Executors.newCachedThreadPool( pool("network") @@ -66,6 +67,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex private Queue connectionQueue = new ConcurrentLinkedQueue<>(); private Map connections = new ConcurrentHashMap<>(); private final Set requestedObjects = newSetFromMap(new ConcurrentHashMap(10_000)); + private long requestedObjectsTimeout = 0; private Thread starter; @@ -145,6 +147,8 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex } catch (IOException e) { throw new ApplicationException(e); } + requestedObjectsTimeout = System.currentTimeMillis() + REQUESTED_OBJECTS_MAX_TIME; + requestedObjects.clear(); starter = thread("connection manager", new Runnable() { @Override @@ -181,6 +185,20 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex it.remove(); } } + + // The list 'requested objects' helps to prevent downloading an object + // twice. From time to time there is an error though, and an object is + // never downloaded. To prevent a large list of failed objects and give + // them a chance to get downloaded again, let's clear the list from time + // to time. The timeout should be such that most of the initial object + // sync should be done by then, but small enough to prevent objects with + // a normal time out from not being downloaded at all. + long now = System.currentTimeMillis(); + if (now > requestedObjectsTimeout) { + requestedObjectsTimeout = now + REQUESTED_OBJECTS_MAX_TIME; + requestedObjects.clear(); + } + try { Thread.sleep(30_000); } catch (InterruptedException e) {