Merge branch 'release/2.0.4'

Version 2.0.3

core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java

@@ -198,6 +198,7 @@ public class Plaintext implements Streamable {
             }
         }
     }
+
     public void write(ByteBuffer buffer, boolean includeSignature) {
         Encode.varInt(from.getVersion(), buffer);
         Encode.varInt(from.getStream(), buffer);
@@ -263,6 +264,9 @@ public class Plaintext implements Streamable {
     }
 
     public void setStatus(Status status) {
+        if (status != Status.RECEIVED && sent == null && status != Status.DRAFT) {
+            sent = UnixTime.now();
+        }
         this.status = status;
     }
 
@@ -279,6 +283,7 @@ public class Plaintext implements Streamable {
     }
 
     public void updateNextTry() {
+        if (to != null) {
             if (nextTry == null) {
                 if (sent != null && to.has(Feature.DOES_ACK)) {
                     nextTry = UnixTime.now(+ttl);
@@ -289,6 +294,7 @@ public class Plaintext implements Streamable {
                 retries++;
             }
         }
+    }
 
     public String getSubject() {
         Scanner s = new Scanner(new ByteArrayInputStream(message), "UTF-8");
@@ -323,7 +329,7 @@ public class Plaintext implements Streamable {
             Objects.equals(from, plaintext.from) &&
             Arrays.equals(message, plaintext.message) &&
             Objects.equals(getAckMessage(), plaintext.getAckMessage()) &&
-                Arrays.equals(to.getRipe(), plaintext.to.getRipe()) &&
+            Arrays.equals(to == null ? null : to.getRipe(), plaintext.to == null ? null : plaintext.to.getRipe()) &&
             Arrays.equals(signature, plaintext.signature) &&
             Objects.equals(status, plaintext.status) &&
             Objects.equals(sent, plaintext.sent) &&

core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java

@@ -33,6 +33,7 @@ import java.io.IOException;
 import java.math.BigInteger;
 import java.security.GeneralSecurityException;
 import java.security.MessageDigest;
+import java.security.Provider;
 import java.security.SecureRandom;
 
 import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES;
@@ -49,10 +50,10 @@ public abstract class AbstractCryptography implements Cryptography, InternalCont
     private static final BigInteger TWO_POW_64 = TWO.pow(64);
     private static final BigInteger TWO_POW_16 = TWO.pow(16);
 
-    private final String provider;
+    protected final Provider provider;
     private InternalContext context;
 
-    protected AbstractCryptography(String provider) {
+    protected AbstractCryptography(Provider provider) {
         this.provider = provider;
     }
 
@@ -137,7 +138,6 @@ public abstract class AbstractCryptography implements Cryptography, InternalCont
         if (extraBytes == 0) extraBytes = NETWORK_EXTRA_BYTES;
 
         BigInteger TTL = BigInteger.valueOf(object.getExpiresTime() - UnixTime.now());
-        BigInteger numerator = TWO_POW_64;
         BigInteger powLength = BigInteger.valueOf(object.getPayloadBytesWithoutNonce().length + extraBytes);
         BigInteger denominator = BigInteger.valueOf(nonceTrialsPerByte)
                 .multiply(
@@ -145,7 +145,7 @@ public abstract class AbstractCryptography implements Cryptography, InternalCont
                                 powLength.multiply(TTL).divide(TWO_POW_16)
                         )
                 );
-        return Bytes.expand(numerator.divide(denominator).toByteArray(), 8);
+        return Bytes.expand(TWO_POW_64.divide(denominator).toByteArray(), 8);
     }
 
     private byte[] hash(String algorithm, byte[]... data) {

cryptography-bc/src/main/java/ch/dissem/bitmessage/cryptography/bc/BouncyCryptography.java

@@ -52,14 +52,9 @@ import java.util.Arrays;
 public class BouncyCryptography extends AbstractCryptography {
     private static final X9ECParameters EC_CURVE_PARAMETERS = CustomNamedCurves.getByName("secp256k1");
     private static final String ALGORITHM_ECDSA = "ECDSA";
-    private static final String PROVIDER = "BC";
-
-    static {
-        java.security.Security.addProvider(new BouncyCastleProvider());
-    }
 
     public BouncyCryptography() {
-        super(PROVIDER);
+        super(new BouncyCastleProvider());
     }
 
     @Override
@@ -109,9 +104,9 @@ public class BouncyCryptography extends AbstractCryptography {
 
             ECPoint Q = keyToPoint(pubkey.getSigningKey());
             KeySpec keySpec = new ECPublicKeySpec(Q, spec);
-            PublicKey publicKey = KeyFactory.getInstance(ALGORITHM_ECDSA, PROVIDER).generatePublic(keySpec);
+            PublicKey publicKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider).generatePublic(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, PROVIDER);
+            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
             sig.initVerify(publicKey);
             sig.update(data);
             return sig.verify(signature);
@@ -133,10 +128,10 @@ public class BouncyCryptography extends AbstractCryptography {
 
             BigInteger d = keyToBigInt(privateKey.getPrivateSigningKey());
             KeySpec keySpec = new ECPrivateKeySpec(d, spec);
-            java.security.PrivateKey privKey = KeyFactory.getInstance(ALGORITHM_ECDSA, PROVIDER)
+            java.security.PrivateKey privKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider)
                 .generatePrivate(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, PROVIDER);
+            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
             sig.initSign(privKey);
             sig.update(data);
             return sig.sign();

cryptography-sc/build.gradle

@@ -14,4 +14,5 @@ dependencies {
     compile project(':core')
     compile 'com.madgag.spongycastle:prov:1.52.0.0'
     testCompile 'junit:junit:4.12'
+    testCompile 'org.mockito:mockito-core:1.10.19'
 }

cryptography-sc/src/main/java/ch/dissem/bitmessage/cryptography/sc/SpongyCryptography.java

@@ -52,14 +52,9 @@ import java.util.Arrays;
 public class SpongyCryptography extends AbstractCryptography {
     private static final X9ECParameters EC_CURVE_PARAMETERS = CustomNamedCurves.getByName("secp256k1");
     private static final String ALGORITHM_ECDSA = "ECDSA";
-    private static final String PROVIDER = "SC";
-
-    static {
-        java.security.Security.addProvider(new BouncyCastleProvider());
-    }
 
     public SpongyCryptography() {
-        super(PROVIDER);
+        super(new BouncyCastleProvider());
     }
 
     @Override
@@ -109,9 +104,9 @@ public class SpongyCryptography extends AbstractCryptography {
 
             ECPoint Q = keyToPoint(pubkey.getSigningKey());
             KeySpec keySpec = new ECPublicKeySpec(Q, spec);
-            PublicKey publicKey = KeyFactory.getInstance(ALGORITHM_ECDSA, PROVIDER).generatePublic(keySpec);
+            PublicKey publicKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider).generatePublic(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, PROVIDER);
+            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
             sig.initVerify(publicKey);
             sig.update(data);
             return sig.verify(signature);
@@ -133,10 +128,10 @@ public class SpongyCryptography extends AbstractCryptography {
 
             BigInteger d = keyToBigInt(privateKey.getPrivateSigningKey());
             KeySpec keySpec = new ECPrivateKeySpec(d, spec);
-            java.security.PrivateKey privKey = KeyFactory.getInstance(ALGORITHM_ECDSA, PROVIDER)
+            java.security.PrivateKey privKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider)
                 .generatePrivate(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, PROVIDER);
+            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
             sig.initSign(privKey);
             sig.update(data);
             return sig.sign();

cryptography-sc/src/test/java/ch/dissem/bitmessage/security/CryptographyTest.java

@@ -0,0 +1,157 @@
+package ch.dissem.bitmessage.security;
+
+import ch.dissem.bitmessage.InternalContext;
+import ch.dissem.bitmessage.cryptography.sc.SpongyCryptography;
+import ch.dissem.bitmessage.entity.ObjectMessage;
+import ch.dissem.bitmessage.entity.payload.GenericPayload;
+import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
+import ch.dissem.bitmessage.exception.InsufficientProofOfWorkException;
+import ch.dissem.bitmessage.ports.MultiThreadedPOWEngine;
+import ch.dissem.bitmessage.ports.ProofOfWorkEngine;
+import ch.dissem.bitmessage.utils.CallbackWaiter;
+import ch.dissem.bitmessage.utils.Singleton;
+import ch.dissem.bitmessage.utils.UnixTime;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import javax.xml.bind.DatatypeConverter;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import static ch.dissem.bitmessage.utils.UnixTime.DAY;
+import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author Christian Basler
+ */
+public class CryptographyTest {
+    public static final byte[] TEST_VALUE = "teststring".getBytes();
+    public static final byte[] TEST_SHA1 = DatatypeConverter.parseHexBinary(""
+            + "b8473b86d4c2072ca9b08bd28e373e8253e865c4");
+    public static final byte[] TEST_SHA512 = DatatypeConverter.parseHexBinary(""
+            + "6253b39071e5df8b5098f59202d414c37a17d6a38a875ef5f8c7d89b0212b028"
+            + "692d3d2090ce03ae1de66c862fa8a561e57ed9eb7935ce627344f742c0931d72");
+    public static final byte[] TEST_RIPEMD160 = DatatypeConverter.parseHexBinary(""
+            + "cd566972b5e50104011a92b59fa8e0b1234851ae");
+
+    private static SpongyCryptography crypto;
+
+    @BeforeClass
+    public static void setUp() {
+        crypto = new SpongyCryptography();
+        Singleton.initialize(crypto);
+        InternalContext ctx = mock(InternalContext.class);
+        when(ctx.getProofOfWorkEngine()).thenReturn(new MultiThreadedPOWEngine());
+        crypto.setContext(ctx);
+    }
+
+    @Test
+    public void testRipemd160() {
+        assertArrayEquals(TEST_RIPEMD160, crypto.ripemd160(TEST_VALUE));
+    }
+
+    @Test
+    public void testSha1() {
+        assertArrayEquals(TEST_SHA1, crypto.sha1(TEST_VALUE));
+    }
+
+    @Test
+    public void testSha512() {
+        assertArrayEquals(TEST_SHA512, crypto.sha512(TEST_VALUE));
+    }
+
+    @Test
+    public void testChaining() {
+        assertArrayEquals(TEST_SHA512, crypto.sha512("test".getBytes(), "string".getBytes()));
+    }
+
+    @Test
+    public void ensureDoubleHashYieldsSameResultAsHashOfHash() {
+        assertArrayEquals(crypto.sha512(TEST_SHA512), crypto.doubleSha512(TEST_VALUE));
+    }
+
+    @Test(expected = IOException.class)
+    public void ensureExceptionForInsufficientProofOfWork() throws IOException {
+        ObjectMessage objectMessage = new ObjectMessage.Builder()
+                .nonce(new byte[8])
+                .expiresTime(UnixTime.now(+28 * DAY))
+                .objectType(0)
+                .payload(GenericPayload.read(0, 1, new ByteArrayInputStream(new byte[0]), 0))
+                .build();
+        crypto.checkProofOfWork(objectMessage, 1000, 1000);
+    }
+
+    @Test
+    public void testDoProofOfWork() throws Exception {
+        ObjectMessage objectMessage = new ObjectMessage.Builder()
+                .nonce(new byte[8])
+                .expiresTime(UnixTime.now(+2 * MINUTE))
+                .objectType(0)
+                .payload(GenericPayload.read(0, 1, new ByteArrayInputStream(new byte[0]), 0))
+                .build();
+        final CallbackWaiter<byte[]> waiter = new CallbackWaiter<>();
+        crypto.doProofOfWork(objectMessage, 1000, 1000,
+                new ProofOfWorkEngine.Callback() {
+                    @Override
+                    public void onNonceCalculated(byte[] initialHash, byte[] nonce) {
+                        waiter.setValue(nonce);
+                    }
+                });
+        objectMessage.setNonce(waiter.waitForValue());
+        try {
+            crypto.checkProofOfWork(objectMessage, 1000, 1000);
+        } catch (InsufficientProofOfWorkException e) {
+            fail(e.getMessage());
+        }
+    }
+
+    @Test
+    public void ensureEncryptionAndDecryptionWorks() {
+        byte[] data = crypto.randomBytes(100);
+        byte[] key_e = crypto.randomBytes(32);
+        byte[] iv = crypto.randomBytes(16);
+        byte[] encrypted = crypto.crypt(true, data, key_e, iv);
+        byte[] decrypted = crypto.crypt(false, encrypted, key_e, iv);
+        assertArrayEquals(data, decrypted);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void ensureDecryptionFailsWithInvalidCypherText() {
+        byte[] data = crypto.randomBytes(128);
+        byte[] key_e = crypto.randomBytes(32);
+        byte[] iv = crypto.randomBytes(16);
+        crypto.crypt(false, data, key_e, iv);
+    }
+
+    @Test
+    public void testMultiplication() {
+        byte[] a = crypto.randomBytes(PrivateKey.PRIVATE_KEY_SIZE);
+        byte[] A = crypto.createPublicKey(a);
+
+        byte[] b = crypto.randomBytes(PrivateKey.PRIVATE_KEY_SIZE);
+        byte[] B = crypto.createPublicKey(b);
+
+        assertArrayEquals(crypto.multiply(A, b), crypto.multiply(B, a));
+    }
+
+    @Test
+    public void ensureSignatureIsValid() {
+        byte[] data = crypto.randomBytes(100);
+        PrivateKey privateKey = new PrivateKey(false, 1, 1000, 1000);
+        byte[] signature = crypto.getSignature(data, privateKey);
+        assertThat(crypto.isSignatureValid(data, signature, privateKey.getPubkey()), is(true));
+    }
+
+    @Test
+    public void ensureSignatureIsInvalidForTemperedData() {
+        byte[] data = crypto.randomBytes(100);
+        PrivateKey privateKey = new PrivateKey(false, 1, 1000, 1000);
+        byte[] signature = crypto.getSignature(data, privateKey);
+        data[0]++;
+        assertThat(crypto.isSignatureValid(data, signature, privateKey.getPubkey()), is(false));
+    }
+}

networking/src/main/java/ch/dissem/bitmessage/networking/AbstractConnection.java

@@ -54,7 +54,7 @@ public abstract class AbstractConnection {
     protected final NetworkHandler.MessageListener listener;
     protected final Map<InventoryVector, Long> ivCache;
     protected final Deque<MessagePayload> sendingQueue;
-    protected final Set<InventoryVector> commonRequestedObjects;
+    protected final Map<InventoryVector, Long> commonRequestedObjects;
     protected final Set<InventoryVector> requestedObjects;
 
     protected volatile State state;
@@ -71,7 +71,7 @@ public abstract class AbstractConnection {
 
     public AbstractConnection(InternalContext context, Mode mode,
                               NetworkAddress node,
-                              Set<InventoryVector> commonRequestedObjects,
+                              Map<InventoryVector, Long> commonRequestedObjects,
                               long syncTimeout) {
         this.ctx = context;
         this.mode = mode;
@@ -143,7 +143,7 @@ public abstract class AbstractConnection {
         int originalSize = inv.getInventory().size();
         updateIvCache(inv.getInventory());
         List<InventoryVector> missing = ctx.getInventory().getMissing(inv.getInventory(), streams);
-        missing.removeAll(commonRequestedObjects);
+        missing.removeAll(commonRequestedObjects.keySet());
         LOG.trace("Received inventory with " + originalSize + " elements, of which are "
             + missing.size() + " missing.");
         send(new GetData.Builder().inventory(missing).build());
@@ -175,7 +175,7 @@ public abstract class AbstractConnection {
         } catch (IOException e) {
             LOG.error("Stream " + objectMessage.getStream() + ", object type " + objectMessage.getType() + ": " + e.getMessage(), e);
         } finally {
-            if (!commonRequestedObjects.remove(objectMessage.getInventoryVector())) {
+            if (commonRequestedObjects.remove(objectMessage.getInventoryVector()) == null) {
                 LOG.debug("Received object that wasn't requested.");
             }
         }
@@ -205,6 +205,10 @@ public abstract class AbstractConnection {
         return ivCache.containsKey(iv);
     }
 
+    public boolean requested(InventoryVector iv) {
+        return requestedObjects.contains(iv);
+    }
+
     private void cleanupIvCache() {
         Long fiveMinutesAgo = UnixTime.now(-5 * MINUTE);
         for (Map.Entry<InventoryVector, Long> entry : ivCache.entrySet()) {

networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java

@@ -36,9 +36,9 @@ import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
-import java.util.HashSet;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
 import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT;
 import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC;
@@ -64,20 +64,20 @@ class Connection extends AbstractConnection {
     private boolean socketInitialized;
 
     public Connection(InternalContext context, Mode mode, Socket socket,
-                      Set<InventoryVector> requestedObjectsMap) throws IOException {
+                      Map<InventoryVector, Long> requestedObjectsMap) throws IOException {
         this(context, mode, socket, requestedObjectsMap,
             new NetworkAddress.Builder().ip(socket.getInetAddress()).port(socket.getPort()).stream(1).build(),
             0);
     }
 
     public Connection(InternalContext context, Mode mode, NetworkAddress node,
-                      Set<InventoryVector> requestedObjectsMap) {
+                      Map<InventoryVector, Long> requestedObjectsMap) {
         this(context, mode, new Socket(), requestedObjectsMap,
             node, 0);
     }
 
     private Connection(InternalContext context, Mode mode, Socket socket,
-                       Set<InventoryVector> commonRequestedObjects, NetworkAddress node, long syncTimeout) {
+                       Map<InventoryVector, Long> commonRequestedObjects, NetworkAddress node, long syncTimeout) {
         super(context, mode, node, commonRequestedObjects, syncTimeout);
         this.startTime = UnixTime.now();
         this.socket = socket;
@@ -86,7 +86,7 @@ class Connection extends AbstractConnection {
     public static Connection sync(InternalContext ctx, InetAddress address, int port, MessageListener listener,
                                   long timeoutInSeconds) throws IOException {
         return new Connection(ctx, SYNC, new Socket(address, port),
-                new HashSet<InventoryVector>(),
+            new HashMap<InventoryVector, Long>(),
             new NetworkAddress.Builder().ip(address).port(port).stream(1).build(),
             timeoutInSeconds);
     }

networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java

@@ -18,7 +18,6 @@ package ch.dissem.bitmessage.networking;
 
 import ch.dissem.bitmessage.InternalContext;
 import ch.dissem.bitmessage.entity.valueobject.NetworkAddress;
-import ch.dissem.bitmessage.ports.NetworkHandler;
 import ch.dissem.bitmessage.utils.UnixTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,12 +31,13 @@ import static ch.dissem.bitmessage.networking.DefaultNetworkHandler.NETWORK_MAGI
 /**
  * @author Christian Basler
  */
+@Deprecated
+@SuppressWarnings("deprecation")
 public class ConnectionOrganizer implements Runnable {
     private static final Logger LOG = LoggerFactory.getLogger(ConnectionOrganizer.class);
 
     private final InternalContext ctx;
     private final DefaultNetworkHandler networkHandler;
-    private final NetworkHandler.MessageListener listener;
 
     private Connection initialConnection;
 
@@ -45,7 +45,6 @@ public class ConnectionOrganizer implements Runnable {
                                DefaultNetworkHandler networkHandler) {
         this.ctx = ctx;
         this.networkHandler = networkHandler;
-        this.listener = ctx.getNetworkListener();
     }
 
     @Override

networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java

@@ -39,7 +39,6 @@ import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER;
 import static ch.dissem.bitmessage.networking.AbstractConnection.State.ACTIVE;
 import static ch.dissem.bitmessage.utils.DebugUtils.inc;
 import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool;
-import static java.util.Collections.newSetFromMap;
 
 /**
  * Handles all the networky stuff.
@@ -59,7 +58,7 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder {
     private ServerRunnable server;
     private volatile boolean running;
 
-    final Set<InventoryVector> requestedObjects = newSetFromMap(new ConcurrentHashMap<InventoryVector, Boolean>(50_000));
+    final Map<InventoryVector, Long> requestedObjects = new ConcurrentHashMap<>(50_000);
 
     @Override
     public void setContext(InternalContext context) {

networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java

@@ -31,6 +31,7 @@ import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SERVER;
 /**
  * @author Christian Basler
  */
+@Deprecated
 public class ServerRunnable implements Runnable, Closeable {
     private static final Logger LOG = LoggerFactory.getLogger(ServerRunnable.class);
     private final InternalContext ctx;

networking/src/main/java/ch/dissem/bitmessage/networking/nio/ConnectionInfo.java

@@ -26,11 +26,10 @@ import ch.dissem.bitmessage.entity.valueobject.NetworkAddress;
 import ch.dissem.bitmessage.exception.NodeException;
 import ch.dissem.bitmessage.factory.V3MessageReader;
 import ch.dissem.bitmessage.networking.AbstractConnection;
+import ch.dissem.bitmessage.utils.UnixTime;
 
 import java.nio.ByteBuffer;
-import java.util.Iterator;
+import java.util.*;
-import java.util.Queue;
-import java.util.Set;
 
 import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.CLIENT;
 import static ch.dissem.bitmessage.networking.AbstractConnection.Mode.SYNC;
@@ -46,7 +45,7 @@ public class ConnectionInfo extends AbstractConnection {
     private long lastUpdate = System.currentTimeMillis();
 
     public ConnectionInfo(InternalContext context, Mode mode, NetworkAddress node,
-                          Set<InventoryVector> commonRequestedObjects, long syncTimeout) {
+                          Map<InventoryVector, Long> commonRequestedObjects, long syncTimeout) {
         super(context, mode, node, commonRequestedObjects, syncTimeout);
         headerOut.flip();
         if (mode == CLIENT || mode == SYNC) {
@@ -147,8 +146,12 @@ public class ConnectionInfo extends AbstractConnection {
     protected void send(MessagePayload payload) {
         sendingQueue.add(payload);
         if (payload instanceof GetData) {
-            requestedObjects.addAll(((GetData) payload).getInventory());
+            Long now = UnixTime.now();
-            commonRequestedObjects.addAll(((GetData) payload).getInventory());
+            List<InventoryVector> inventory = ((GetData) payload).getInventory();
+            requestedObjects.addAll(inventory);
+            for (InventoryVector iv : inventory) {
+                commonRequestedObjects.put(iv, now);
+            }
         }
     }
 

networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java

@@ -46,14 +46,14 @@ import static ch.dissem.bitmessage.utils.Collections.selectRandom;
 import static ch.dissem.bitmessage.utils.DebugUtils.inc;
 import static ch.dissem.bitmessage.utils.ThreadFactoryBuilder.pool;
 import static java.nio.channels.SelectionKey.*;
-import static java.util.Collections.newSetFromMap;
 
 /**
  * Network handler using java.nio, resulting in less threads.
  */
 public class NioNetworkHandler implements NetworkHandler, InternalContext.ContextHolder {
     private static final Logger LOG = LoggerFactory.getLogger(NioNetworkHandler.class);
-    private static final long REQUESTED_OBJECTS_MAX_TIME = 30 * 60_000; // 30 minutes
+    private static final long REQUESTED_OBJECTS_MAX_TIME = 2 * 60_000; // 2 minutes
+    private static final Long DELAYED = Long.MIN_VALUE;
 
     private final ExecutorService threadPool = Executors.newCachedThreadPool(
         pool("network")
@@ -66,8 +66,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
     private ServerSocketChannel serverChannel;
     private Queue<NetworkAddress> connectionQueue = new ConcurrentLinkedQueue<>();
     private Map<ConnectionInfo, SelectionKey> connections = new ConcurrentHashMap<>();
-    private final Set<InventoryVector> requestedObjects = newSetFromMap(new ConcurrentHashMap<InventoryVector, Boolean>(10_000));
+    private final Map<InventoryVector, Long> requestedObjects = new ConcurrentHashMap<>(10_000);
-    private long requestedObjectsTimeout = 0;
 
     private Thread starter;
 
@@ -80,7 +79,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
                     channel.configureBlocking(false);
                     ConnectionInfo connection = new ConnectionInfo(ctx, SYNC,
                         new NetworkAddress.Builder().ip(server).port(port).stream(1).build(),
-                        new HashSet<InventoryVector>(), timeoutInSeconds);
+                        new HashMap<InventoryVector, Long>(), timeoutInSeconds);
                     while (channel.isConnected() && !connection.isSyncFinished()) {
                         write(channel, connection);
                         read(channel, connection);
@@ -147,7 +146,6 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
         } catch (IOException e) {
             throw new ApplicationException(e);
         }
-        requestedObjectsTimeout = System.currentTimeMillis() + REQUESTED_OBJECTS_MAX_TIME;
         requestedObjects.clear();
 
         starter = thread("connection manager", new Runnable() {
@@ -189,15 +187,22 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
                     // The list 'requested objects' helps to prevent downloading an object
                     // twice. From time to time there is an error though, and an object is
                     // never downloaded. To prevent a large list of failed objects and give
-                    // them a chance to get downloaded again, let's clear the list from time
+                    // them a chance to get downloaded again, we will attempt to download an
-                    // to time. The timeout should be such that most of the initial object
+                    // object from another node after some time out.
-                    // sync should be done by then, but small enough to prevent objects with
+                    long timedOut = System.currentTimeMillis() - REQUESTED_OBJECTS_MAX_TIME;
-                    // a normal time out from not being downloaded at all.
+                    List<InventoryVector> delayed = new LinkedList<>();
-                    long now = System.currentTimeMillis();
+                    Iterator<Map.Entry<InventoryVector, Long>> iterator = requestedObjects.entrySet().iterator();
-                    if (now > requestedObjectsTimeout) {
+                    while (iterator.hasNext()) {
-                        requestedObjectsTimeout = now + REQUESTED_OBJECTS_MAX_TIME;
+                        Map.Entry<InventoryVector, Long> e = iterator.next();
-                        requestedObjects.clear();
+                        //noinspection NumberEquality
+                        if (e.getValue() == DELAYED) {
+                            iterator.remove();
+                        } else if (e.getValue() < timedOut) {
+                            delayed.add(e.getKey());
+                            e.setValue(DELAYED);
                         }
+                    }
+                    request(delayed);
 
                     try {
                         Thread.sleep(30_000);
@@ -422,7 +427,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
                         break;
                     }
                 }
-                if (connection.knowsOf(next)) {
+                if (connection.knowsOf(next) && !connection.requested(next)) {
                     List<InventoryVector> ivs = distribution.get(connection);
                     if (ivs.size() == GetData.MAX_INVENTORY_SIZE) {
                         connection.send(new GetData.Builder().inventory(ivs).build());
@@ -442,7 +447,9 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
         } while (iterator.hasNext());
 
         // remove objects nobody knows of
-        requestedObjects.removeAll(inventoryVectors);
+        for (InventoryVector iv : inventoryVectors) {
+            requestedObjects.remove(iv);
+        }
 
         for (ConnectionInfo connection : distribution.keySet()) {
             List<InventoryVector> ivs = distribution.get(connection);

networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java

@@ -76,6 +76,7 @@ public class NetworkHandlerTest {
     }
 
     @Parameterized.Parameters
+    @SuppressWarnings("deprecation")
     public static List<Object[]> parameters() {
         return Arrays.asList(new Object[][]{
             {new DefaultNetworkHandler(), new DefaultNetworkHandler()},

wif/src/main/java/ch/dissem/bitmessage/wif/WifImporter.java

@@ -66,12 +66,12 @@ public class WifImporter {
                 entry.getKey(),
                 getSecret(section.get("privsigningkey")),
                 getSecret(section.get("privencryptionkey")),
-                    section.get("noncetrialsperbyte", long.class),
+                Long.parseLong(section.get("noncetrialsperbyte")),
-                    section.get("payloadlengthextrabytes", long.class),
+                Long.parseLong(section.get("payloadlengthextrabytes")),
                 Pubkey.Feature.bitfield(features)
             );
             if (section.containsKey("chan")) {
-                address.setChan(section.get("chan", boolean.class));
+                address.setChan(Boolean.parseBoolean(section.get("chan")));
             }
             address.setAlias(section.get("label"));
             identities.add(address);