Merge branch 'feature/ACK' into develop

.gitignore

@@ -5,6 +5,7 @@
 ### Gradle ###
 .gradle
 build/
+classes/
 
 # Ignore Gradle GUI config
 gradle-app.setting

README.md

@@ -5,10 +5,11 @@ Jabit
 [![Apache 2](https://img.shields.io/badge/license-Apache_2.0-blue.svg)](https://raw.githubusercontent.com/Dissem/Jabit/master/LICENSE)
 [![Visit our IRC channel](https://img.shields.io/badge/irc-%23jabit-blue.svg)](https://kiwiirc.com/client/irc.freenode.net/#jabit)
 
-A Java implementation for the Bitmessage protocol. To build, use command `gradle build` or `./gradlew build`.
+A Java implementation for the Bitmessage protocol. To build, use command `./gradlew build`.
 
-Please note that it still has its limitations, but the API should now be stable. Jabit uses Semantic Versioning, meaning
+Please note that it still has its limitations, but the API should now be stable. Jabit uses Semantic Versioning, meaning as long as the major version doesn't change, nothing should break if you update.
-as long as the major version doesn't change, nothing should break if you update.
+
+Be aware though that this doesn't necessarily applies for SNAPSHOT builds and the development branch, notably when it comes to database updates. _In other words, they may break your installation!_ 
 
 #### Master
 [![Build Status](https://travis-ci.org/Dissem/Jabit.svg?branch=master)](https://travis-ci.org/Dissem/Jabit) 
@@ -23,9 +24,7 @@ as long as the major version doesn't change, nothing should break if you update.
 Security
 --------
 
-There are most probably some security issues, me programming this thing all by myself. Jabit doesn't do anything against
+There are most probably some security issues, me programming this thing all by myself. Jabit doesn't do anything against timing attacks yet, for example. Please feel free to use the library, report bugs and maybe even help out. I hope the code is easy to understand and work with.
-timing attacks yet, for example. Please feel free to use the library, report bugs and maybe even help out. I hope the
-code is easy to understand and work with.
 
 Project Status
 --------------
@@ -74,17 +73,22 @@ BitmessageContext ctx = new BitmessageContext.Builder()
         .nodeRegistry(new MemoryNodeRegistry())
         .networkHandler(new NetworkNode())
         .cryptography(new BouncyCryptography())
+        .listener(System.out::println)
         .build();
 ```
-This creates a simple context using a H2 database that will be created in the user's home directory. Next you'll need to
+This creates a simple context using a H2 database that will be created in the user's home directory. In the listener you decide what happens when a message arrives. If you can't use lambdas, you may instead write
-start the context and decide what happens if a message arrives:
 ```Java
-ctx.startup(new BitmessageContext.Listener() {
+        .listener(new BitmessageContext.Listener() {
             @Override
             public void receive(Plaintext plaintext) {
                 // TODO: Notify the user
             }
-});
+        })
+```
+
+Next you'll need to start the context:
+```Java
+ctx.startup()
 ```
 Then you might want to create an identity
 ```Java
@@ -100,3 +104,22 @@ to which you can send some messages
 ```Java
 ctx.send(identity, contact, "Test", "Hello Chris, this is a message.");
 ```
+
+### Housekeeping
+
+As Bitmessage stores all currently valid messages, we'll need to delete expired objects from time to time:
+```Java
+ctx.cleanup();
+```
+If the client runs all the time, it might be a good idea to do this daily or at least weekly. Otherwise, you might just want to clean up on shutdown.
+
+Also, if some messages weren't acknowledged when it expired, they can be resent:
+```Java
+ctx.resendUnacknowledgedMessages();
+```
+This could be triggered periodically, or manually by the user. Please be aware that _if_ there is a message to resend, proof of work needs to be calculated, so to not annoy your users you might not want to trigger it on shutdown. As the client might have been offline for some time, it might as well be wise to wait until it caught up downloading new messages before resending those messages, after all they might be acknowledged by now.
+
+There probably won't happen extremely bad things if you don't - at least not more than otherwise - but you can properly shutdown the network connection by calling
+```Java
+ctx.shutdown();
+```

core/build.gradle

@@ -26,6 +26,7 @@ artifacts {
 dependencies {
     compile 'org.slf4j:slf4j-api:1.7.12'
     testCompile 'junit:junit:4.11'
+    testCompile 'org.hamcrest:hamcrest-library:1.3'
     testCompile 'org.mockito:mockito-core:1.10.19'
     testCompile project(':cryptography-bc')
 }

core/src/main/java/ch/dissem/bitmessage/BaseMessageCallback.java

@@ -1,47 +0,0 @@
-/*
- * Copyright 2016 Christian Basler
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.dissem.bitmessage;
-
-import ch.dissem.bitmessage.entity.payload.ObjectPayload;
-import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
-
-/**
- * Default implementation that doesn't do anything.
- *
- * @author Christian Basler
- */
-public class BaseMessageCallback implements MessageCallback {
-    @Override
-    public void proofOfWorkStarted(ObjectPayload message) {
-        // No op
-    }
-
-    @Override
-    public void proofOfWorkCompleted(ObjectPayload message) {
-        // No op
-    }
-
-    @Override
-    public void messageOffered(ObjectPayload message, InventoryVector iv) {
-        // No op
-    }
-
-    @Override
-    public void messageAcknowledged(InventoryVector iv) {
-        // No op
-    }
-}

core/src/main/java/ch/dissem/bitmessage/BitmessageContext.java

@@ -18,13 +18,9 @@ package ch.dissem.bitmessage;
 
 import ch.dissem.bitmessage.entity.*;
 import ch.dissem.bitmessage.entity.payload.Broadcast;
-import ch.dissem.bitmessage.entity.payload.Msg;
-import ch.dissem.bitmessage.entity.payload.ObjectPayload;
 import ch.dissem.bitmessage.entity.payload.ObjectType;
 import ch.dissem.bitmessage.entity.payload.Pubkey.Feature;
-import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
-import ch.dissem.bitmessage.exception.ApplicationException;
 import ch.dissem.bitmessage.exception.DecryptionFailedException;
 import ch.dissem.bitmessage.factory.Factory;
 import ch.dissem.bitmessage.ports.*;
@@ -35,15 +31,12 @@ import org.slf4j.LoggerFactory;
 
 import java.net.InetAddress;
 import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 
 import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES;
 import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE;
-import static ch.dissem.bitmessage.entity.Plaintext.Status.*;
 import static ch.dissem.bitmessage.entity.Plaintext.Type.BROADCAST;
 import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
 import static ch.dissem.bitmessage.utils.UnixTime.*;
@@ -76,16 +69,10 @@ public class BitmessageContext {
     private BitmessageContext(Builder builder) {
         ctx = new InternalContext(builder);
         labeler = builder.labeler;
+        ctx.getProofOfWorkService().doMissingProofOfWork(30_000); // TODO: this should be configurable
+
         networkListener = new DefaultMessageListener(ctx, labeler, builder.listener);
-
         sendPubkeyOnIdentityCreation = builder.sendPubkeyOnIdentityCreation;
-
-        new Timer().schedule(new TimerTask() {
-            @Override
-            public void run() {
-                ctx.getProofOfWorkService().doMissingProofOfWork();
-            }
-        }, 30_000); // After 30 seconds
     }
 
     public AddressRepository addresses() {
@@ -157,7 +144,6 @@ public class BitmessageContext {
                 .from(from)
                 .to(to)
                 .message(subject, message)
-                .labels(messages().getLabels(Label.Type.SENT))
                 .build();
         send(msg);
     }
@@ -166,6 +152,7 @@ public class BitmessageContext {
         if (msg.getFrom() == null || msg.getFrom().getPrivateKey() == null) {
             throw new IllegalArgumentException("'From' must be an identity, i.e. have a private key.");
         }
+        labeler().markAsSending(msg);
         BitmessageAddress to = msg.getTo();
         if (to != null) {
             if (to.getPubkey() == null) {
@@ -173,36 +160,23 @@ public class BitmessageContext {
                 ctx.requestPubkey(to);
             }
             if (to.getPubkey() == null) {
-                msg.setStatus(PUBKEY_REQUESTED);
-                msg.addLabels(ctx.getMessageRepository().getLabels(Label.Type.OUTBOX));
                 ctx.getMessageRepository().save(msg);
             }
         }
         if (to == null || to.getPubkey() != null) {
             LOG.info("Sending message.");
-            msg.setStatus(DOING_PROOF_OF_WORK);
             ctx.getMessageRepository().save(msg);
+            if (msg.getType() == MSG) {
+                ctx.send(msg);
+            } else {
                 ctx.send(
                         msg.getFrom(),
                         to,
-                    wrapInObjectPayload(msg),
+                        Factory.getBroadcast(msg),
-                    TTL.msg()
+                        msg.getTTL()
                 );
-            msg.setStatus(SENT);
-            msg.addLabels(ctx.getMessageRepository().getLabels(Label.Type.SENT));
-            ctx.getMessageRepository().save(msg);
             }
         }
-
-    private ObjectPayload wrapInObjectPayload(Plaintext msg) {
-        switch (msg.getType()) {
-            case MSG:
-                return new Msg(msg);
-            case BROADCAST:
-                return Factory.getBroadcast(msg);
-            default:
-                throw new ApplicationException("Unknown message type " + msg.getType());
-        }
     }
 
     public void startup() {
@@ -247,10 +221,32 @@ public class BitmessageContext {
         return ctx.getNetworkHandler().send(server, port, request);
     }
 
+    /**
+     * Removes expired objects from the inventory. You should call this method regularly,
+     * e.g. daily and on each shutdown.
+     */
     public void cleanup() {
         ctx.getInventory().cleanup();
     }
 
+    /**
+     * Sends messages again whose time to live expired without being acknowledged. (And whose
+     * recipient is expected to send acknowledgements.
+     * <p>
+     * You should call this method regularly, but be aware of the following:
+     * <ul>
+     * <li>As messages might be sent, POW will be done. It is therefore not advised to
+     * call it on shutdown.</li>
+     * <li>It shouldn't be called right after startup, as it's possible the missing
+     * acknowledgement was sent while the client was offline.</li>
+     * <li>Other than that, the call isn't expensive as long as there is no message
+     * to send, so it might be a good idea to just call it every few minutes.</li>
+     * </ul>
+     */
+    public void resendUnacknowledgedMessages() {
+        ctx.resendUnacknowledged();
+    }
+
     public boolean isRunning() {
         return ctx.getNetworkHandler().isRunning();
     }
@@ -285,7 +281,8 @@ public class BitmessageContext {
 
     public Property status() {
         return new Property("status", null,
-                ctx.getNetworkHandler().getNetworkStatus()
+                ctx.getNetworkHandler().getNetworkStatus(),
+                new Property("unacknowledged", ctx.getMessageRepository().findMessagesToResend().size())
         );
     }
 
@@ -311,7 +308,6 @@ public class BitmessageContext {
         ProofOfWorkRepository proofOfWorkRepository;
         ProofOfWorkEngine proofOfWorkEngine;
         Cryptography cryptography;
-        MessageCallback messageCallback;
         CustomCommandHandler customCommandHandler;
         Labeler labeler;
         Listener listener;
@@ -359,11 +355,6 @@ public class BitmessageContext {
             return this;
         }
 
-        public Builder messageCallback(MessageCallback callback) {
-            this.messageCallback = callback;
-            return this;
-        }
-
         public Builder customCommandHandler(CustomCommandHandler handler) {
             this.customCommandHandler = handler;
             return this;
@@ -430,9 +421,6 @@ public class BitmessageContext {
             if (proofOfWorkEngine == null) {
                 proofOfWorkEngine = new MultiThreadedPOWEngine();
             }
-            if (messageCallback == null) {
-                messageCallback = new BaseMessageCallback();
-            }
             if (labeler == null) {
                 labeler = new DefaultLabeler();
             }

core/src/main/java/ch/dissem/bitmessage/DefaultMessageListener.java

@@ -24,6 +24,7 @@ import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.exception.DecryptionFailedException;
 import ch.dissem.bitmessage.ports.Labeler;
 import ch.dissem.bitmessage.ports.NetworkHandler;
+import ch.dissem.bitmessage.utils.TTL;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -47,9 +48,15 @@ class DefaultMessageListener implements NetworkHandler.MessageListener {
     }
 
     @Override
+    @SuppressWarnings("ConstantConditions")
     public void receive(ObjectMessage object) throws IOException {
         ObjectPayload payload = object.getPayload();
-        if (payload.getType() == null) return;
+        if (payload.getType() == null) {
+            if (payload instanceof GenericPayload) {
+                receive((GenericPayload) payload);
+            }
+            return;
+        }
 
         switch (payload.getType()) {
             case GET_PUBKEY: {
@@ -109,16 +116,9 @@ class DefaultMessageListener implements NetworkHandler.MessageListener {
         List<Plaintext> messages = ctx.getMessageRepository().findMessages(PUBKEY_REQUESTED, address);
         LOG.info("Sending " + messages.size() + " messages for contact " + address);
         for (Plaintext msg : messages) {
-            msg.setStatus(DOING_PROOF_OF_WORK);
+            ctx.getLabeler().markAsSending(msg);
-            ctx.getMessageRepository().save(msg);
-            ctx.send(
-                    msg.getFrom(),
-                    msg.getTo(),
-                    new Msg(msg),
-                    +2 * DAY
-            );
-            msg.setStatus(SENT);
             ctx.getMessageRepository().save(msg);
+            ctx.send(msg);
         }
     }
 
@@ -126,11 +126,12 @@ class DefaultMessageListener implements NetworkHandler.MessageListener {
         for (BitmessageAddress identity : ctx.getAddressRepository().getIdentities()) {
             try {
                 msg.decrypt(identity.getPrivateKey().getPrivateEncryptionKey());
-                msg.getPlaintext().setTo(identity);
+                Plaintext plaintext = msg.getPlaintext();
-                if (!object.isSignatureValid(msg.getPlaintext().getFrom().getPubkey())) {
+                plaintext.setTo(identity);
+                if (!object.isSignatureValid(plaintext.getFrom().getPubkey())) {
                     LOG.warn("Msg with IV " + object.getInventoryVector() + " was successfully decrypted, but signature check failed. Ignoring.");
                 } else {
-                    receive(object.getInventoryVector(), msg.getPlaintext());
+                    receive(object.getInventoryVector(), plaintext);
                 }
                 break;
             } catch (DecryptionFailedException ignore) {
@@ -138,6 +139,16 @@ class DefaultMessageListener implements NetworkHandler.MessageListener {
         }
     }
 
+    protected void receive(GenericPayload ack) {
+        if (ack.getData().length == Msg.ACK_LENGTH) {
+            Plaintext msg = ctx.getMessageRepository().getMessageForAck(ack.getData());
+            if (msg != null) {
+                ctx.getLabeler().markAsAcknowledged(msg);
+                ctx.getMessageRepository().save(msg);
+            }
+        }
+    }
+
     protected void receive(ObjectMessage object, Broadcast broadcast) throws IOException {
         byte[] tag = broadcast instanceof V5Broadcast ? ((V5Broadcast) broadcast).getTag() : null;
         for (BitmessageAddress subscription : ctx.getAddressRepository().getSubscriptions(broadcast.getVersion())) {
@@ -157,11 +168,18 @@ class DefaultMessageListener implements NetworkHandler.MessageListener {
     }
 
     protected void receive(InventoryVector iv, Plaintext msg) {
-        msg.setStatus(RECEIVED);
         msg.setInventoryVector(iv);
         labeler.setLabels(msg);
         ctx.getMessageRepository().save(msg);
         listener.receive(msg);
         updatePubkey(msg.getFrom(), msg.getFrom().getPubkey());
+
+        if (msg.getType() == Plaintext.Type.MSG && msg.getTo().has(Pubkey.Feature.DOES_ACK)) {
+            ObjectMessage ack = msg.getAckMessage();
+            if (ack != null) {
+                ctx.getInventory().storeObject(ack);
+                ctx.getNetworkHandler().offer(ack.getInventoryVector());
+            }
+        }
     }
 }

core/src/main/java/ch/dissem/bitmessage/InternalContext.java

@@ -16,9 +16,7 @@
 
 package ch.dissem.bitmessage;
 
-import ch.dissem.bitmessage.entity.BitmessageAddress;
+import ch.dissem.bitmessage.entity.*;
-import ch.dissem.bitmessage.entity.Encrypted;
-import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.payload.*;
 import ch.dissem.bitmessage.exception.ApplicationException;
 import ch.dissem.bitmessage.ports.*;
@@ -30,6 +28,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
 import java.util.TreeSet;
 
 /**
@@ -54,9 +53,9 @@ public class InternalContext {
     private final MessageRepository messageRepository;
     private final ProofOfWorkRepository proofOfWorkRepository;
     private final ProofOfWorkEngine proofOfWorkEngine;
-    private final MessageCallback messageCallback;
     private final CustomCommandHandler customCommandHandler;
     private final ProofOfWorkService proofOfWorkService;
+    private final Labeler labeler;
 
     private final TreeSet<Long> streams = new TreeSet<>();
     private final int port;
@@ -75,11 +74,11 @@ public class InternalContext {
         this.proofOfWorkService = new ProofOfWorkService();
         this.proofOfWorkEngine = builder.proofOfWorkEngine;
         this.clientNonce = cryptography.randomNonce();
-        this.messageCallback = builder.messageCallback;
         this.customCommandHandler = builder.customCommandHandler;
         this.port = builder.port;
         this.connectionLimit = builder.connectionLimit;
         this.connectionTTL = builder.connectionTTL;
+        this.labeler = builder.labeler;
 
         Singleton.initialize(cryptography);
 
@@ -95,8 +94,7 @@ public class InternalContext {
         }
 
         init(cryptography, inventory, nodeRegistry, networkHandler, addressRepository, messageRepository,
-                proofOfWorkRepository, proofOfWorkService, proofOfWorkEngine,
+                proofOfWorkRepository, proofOfWorkService, proofOfWorkEngine, customCommandHandler, builder.labeler);
-                messageCallback, customCommandHandler, builder.labeler);
         for (BitmessageAddress identity : addressRepository.getIdentities()) {
             streams.add(identity.getStream());
         }
@@ -146,6 +144,10 @@ public class InternalContext {
         return proofOfWorkService;
     }
 
+    public Labeler getLabeler() {
+        return labeler;
+    }
+
     public long[] getStreams() {
         long[] result = new long[streams.size()];
         int i = 0;
@@ -159,14 +161,24 @@ public class InternalContext {
         return port;
     }
 
+    public void send(final Plaintext plaintext) {
+        if (plaintext.getAckMessage() != null) {
+            long expires = UnixTime.now(+plaintext.getTTL());
+            LOG.info("Expires at " + expires);
+            proofOfWorkService.doProofOfWorkWithAck(plaintext, expires);
+        } else {
+            send(plaintext.getFrom(), plaintext.getTo(), new Msg(plaintext), plaintext.getTTL());
+        }
+    }
+
     public void send(final BitmessageAddress from, BitmessageAddress to, final ObjectPayload payload,
                      final long timeToLive) {
         try {
-            if (to == null) to = from;
+            final BitmessageAddress recipient = (to != null ? to : from);
             long expires = UnixTime.now(+timeToLive);
             LOG.info("Expires at " + expires);
             final ObjectMessage object = new ObjectMessage.Builder()
-                    .stream(to.getStream())
+                    .stream(recipient.getStream())
                     .expiresTime(expires)
                     .payload(payload)
                     .build();
@@ -176,9 +188,8 @@ public class InternalContext {
             if (payload instanceof Broadcast) {
                 ((Broadcast) payload).encrypt();
             } else if (payload instanceof Encrypted) {
-                object.encrypt(to.getPubkey());
+                object.encrypt(recipient.getPubkey());
             }
-            messageCallback.proofOfWorkStarted(payload);
             proofOfWorkService.doProofOfWork(to, object);
         } catch (IOException e) {
             throw new ApplicationException(e);
@@ -196,7 +207,6 @@ public class InternalContext {
                     .build();
             response.sign(identity.getPrivateKey());
             response.encrypt(cryptography.createPublicKey(identity.getPublicDecryptionKey()));
-            messageCallback.proofOfWorkStarted(identity.getPubkey());
             // TODO: remember that the pubkey is just about to be sent, and on which stream!
             proofOfWorkService.doProofOfWork(response);
         } catch (IOException e) {
@@ -233,7 +243,6 @@ public class InternalContext {
                 .expiresTime(expires)
                 .payload(new GetPubkey(contact))
                 .build();
-        messageCallback.proofOfWorkStarted(request.getPayload());
         proofOfWorkService.doProofOfWork(request);
     }
 
@@ -271,6 +280,14 @@ public class InternalContext {
         }
     }
 
+    public void resendUnacknowledged() {
+        List<Plaintext> messages = messageRepository.findMessagesToResend();
+        for (Plaintext message : messages) {
+            send(message);
+            messageRepository.save(message);
+        }
+    }
+
     public long getClientNonce() {
         return clientNonce;
     }

core/src/main/java/ch/dissem/bitmessage/MessageCallback.java

@@ -1,52 +0,0 @@
-/*
- * Copyright 2015 Christian Basler
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.dissem.bitmessage;
-
-import ch.dissem.bitmessage.entity.payload.ObjectPayload;
-import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
-
-/**
- * Callback for message sending events, mostly so the user can be notified when POW is done.
- */
-public interface MessageCallback {
-    /**
-     * Called before calculation of proof of work begins.
-     */
-    void proofOfWorkStarted(ObjectPayload message);
-
-    /**
-     * Called after calculation of proof of work finished.
-     */
-    void proofOfWorkCompleted(ObjectPayload message);
-
-    /**
-     * Called once the message is offered to the network. Please note that this doesn't mean the message was sent,
-     * if the client is not connected to the network it's just stored in the inventory.
-     * <p>
-     * Also, please note that this is where the original payload as well as the {@link InventoryVector} of the sent
-     * message is available. If the callback needs the IV for some reason, it should be retrieved here. (Plaintext
-     * and Broadcast messages will have their IV property set automatically though.)
-     * </p>
-     */
-    void messageOffered(ObjectPayload message, InventoryVector iv);
-
-    /**
-     * This isn't called yet, as ACK messages aren't being processed yet. Also, this is only relevant for Plaintext
-     * messages.
-     */
-    void messageAcknowledged(InventoryVector iv);
-}

core/src/main/java/ch/dissem/bitmessage/ProofOfWorkService.java

@@ -1,22 +1,23 @@
 package ch.dissem.bitmessage;
 
-import ch.dissem.bitmessage.entity.BitmessageAddress;
+import ch.dissem.bitmessage.entity.*;
-import ch.dissem.bitmessage.entity.ObjectMessage;
+import ch.dissem.bitmessage.entity.payload.Msg;
-import ch.dissem.bitmessage.entity.Plaintext;
-import ch.dissem.bitmessage.entity.PlaintextHolder;
 import ch.dissem.bitmessage.entity.payload.Pubkey;
 import ch.dissem.bitmessage.ports.Cryptography;
 import ch.dissem.bitmessage.ports.MessageRepository;
 import ch.dissem.bitmessage.ports.ProofOfWorkEngine;
 import ch.dissem.bitmessage.ports.ProofOfWorkRepository;
+import ch.dissem.bitmessage.ports.ProofOfWorkRepository.Item;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
 
 import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES;
 import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * @author Christian Basler
@@ -29,16 +30,23 @@ public class ProofOfWorkService implements ProofOfWorkEngine.Callback, InternalC
     private ProofOfWorkRepository powRepo;
     private MessageRepository messageRepo;
 
-    public void doMissingProofOfWork() {
+    public void doMissingProofOfWork(long delayInMilliseconds) {
-        List<byte[]> items = powRepo.getItems();
+        final List<byte[]> items = powRepo.getItems();
         if (items.isEmpty()) return;
 
+        // Wait for 30 seconds, to let the application start up before putting heavy load on the CPU
+        new Timer().schedule(new TimerTask() {
+            @Override
+            public void run() {
                 LOG.info("Doing POW for " + items.size() + " tasks.");
                 for (byte[] initialHash : items) {
-            ProofOfWorkRepository.Item item = powRepo.getItem(initialHash);
+                    Item item = powRepo.getItem(initialHash);
-            cryptography.doProofOfWork(item.object, item.nonceTrialsPerByte, item.extraBytes, this);
+                    cryptography.doProofOfWork(item.object, item.nonceTrialsPerByte, item.extraBytes,
+                            ProofOfWorkService.this);
                 }
             }
+        }, delayInMilliseconds);
+    }
 
     public void doProofOfWork(ObjectMessage object) {
         doProofOfWork(null, object);
@@ -59,24 +67,52 @@ public class ProofOfWorkService implements ProofOfWorkEngine.Callback, InternalC
         cryptography.doProofOfWork(object, nonceTrialsPerByte, extraBytes, this);
     }
 
+    public void doProofOfWorkWithAck(Plaintext plaintext, long expirationTime) {
+        final ObjectMessage ack = plaintext.getAckMessage();
+        messageRepo.save(plaintext);
+        Item item = new Item(ack, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES,
+                expirationTime, plaintext);
+        powRepo.putObject(item);
+        cryptography.doProofOfWork(ack, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES, this);
+    }
+
     @Override
     public void onNonceCalculated(byte[] initialHash, byte[] nonce) {
+        Item item = powRepo.getItem(initialHash);
+        if (item.message == null) {
             ObjectMessage object = powRepo.getItem(initialHash).object;
             object.setNonce(nonce);
             Plaintext plaintext = messageRepo.getMessage(initialHash);
             if (plaintext != null) {
                 plaintext.setInventoryVector(object.getInventoryVector());
+                plaintext.updateNextTry();
+                ctx.getLabeler().markAsSent(plaintext);
                 messageRepo.save(plaintext);
             }
             ctx.getInventory().storeObject(object);
-        powRepo.removeObject(initialHash);
             ctx.getNetworkHandler().offer(object.getInventoryVector());
+        } else {
+            item.message.getAckMessage().setNonce(nonce);
+            final ObjectMessage object = new ObjectMessage.Builder()
+                    .stream(item.message.getStream())
+                    .expiresTime(item.expirationTime)
+                    .payload(new Msg(item.message))
+                    .build();
+            if (object.isSigned()) {
+                object.sign(item.message.getFrom().getPrivateKey());
+            }
+            if (object.getPayload() instanceof Encrypted) {
+                object.encrypt(item.message.getTo().getPubkey());
+            }
+            doProofOfWork(item.message.getTo(), object);
+        }
+        powRepo.removeObject(initialHash);
     }
 
     @Override
     public void setContext(InternalContext ctx) {
         this.ctx = ctx;
-        this.cryptography = security();
+        this.cryptography = cryptography();
         this.powRepo = ctx.getProofOfWorkRepository();
         this.messageRepo = ctx.getMessageRepository();
     }

core/src/main/java/ch/dissem/bitmessage/entity/BitmessageAddress.java

@@ -17,6 +17,7 @@
 package ch.dissem.bitmessage.entity;
 
 import ch.dissem.bitmessage.entity.payload.Pubkey;
+import ch.dissem.bitmessage.entity.payload.Pubkey.Feature;
 import ch.dissem.bitmessage.entity.payload.V4Pubkey;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
 import ch.dissem.bitmessage.exception.ApplicationException;
@@ -36,7 +37,7 @@ import java.util.Objects;
 
 import static ch.dissem.bitmessage.utils.Decode.bytes;
 import static ch.dissem.bitmessage.utils.Decode.varInt;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * A Bitmessage address. Can be a user's private address, an address string without public keys or a recipient's address
@@ -73,19 +74,19 @@ public class BitmessageAddress implements Serializable {
             Encode.varInt(version, os);
             Encode.varInt(stream, os);
             if (version < 4) {
-                byte[] checksum = security().sha512(os.toByteArray(), ripe);
+                byte[] checksum = cryptography().sha512(os.toByteArray(), ripe);
                 this.tag = null;
                 this.publicDecryptionKey = Arrays.copyOfRange(checksum, 0, 32);
             } else {
                 // for tag and decryption key, the checksum has to be created with 0x00 padding
-                byte[] checksum = security().doubleSha512(os.toByteArray(), ripe);
+                byte[] checksum = cryptography().doubleSha512(os.toByteArray(), ripe);
                 this.tag = Arrays.copyOfRange(checksum, 32, 64);
                 this.publicDecryptionKey = Arrays.copyOfRange(checksum, 0, 32);
             }
             // but for the address and its checksum they need to be stripped
             int offset = Bytes.numberOfLeadingZeros(ripe);
             os.write(ripe, offset, ripe.length - offset);
-            byte[] checksum = security().doubleSha512(os.toByteArray());
+            byte[] checksum = cryptography().doubleSha512(os.toByteArray());
             os.write(checksum, 0, 4);
             this.address = "BM-" + Base58.encode(os.toByteArray());
         } catch (IOException e) {
@@ -146,18 +147,18 @@ public class BitmessageAddress implements Serializable {
             this.ripe = Bytes.expand(bytes(in, bytes.length - counter.length() - 4), 20);
 
             // test checksum
-            byte[] checksum = security().doubleSha512(bytes, bytes.length - 4);
+            byte[] checksum = cryptography().doubleSha512(bytes, bytes.length - 4);
             byte[] expectedChecksum = bytes(in, 4);
             for (int i = 0; i < 4; i++) {
                 if (expectedChecksum[i] != checksum[i])
                     throw new IllegalArgumentException("Checksum of address failed");
             }
             if (version < 4) {
-                checksum = security().sha512(Arrays.copyOfRange(bytes, 0, counter.length()), ripe);
+                checksum = cryptography().sha512(Arrays.copyOfRange(bytes, 0, counter.length()), ripe);
                 this.tag = null;
                 this.publicDecryptionKey = Arrays.copyOfRange(checksum, 0, 32);
             } else {
-                checksum = security().doubleSha512(Arrays.copyOfRange(bytes, 0, counter.length()), ripe);
+                checksum = cryptography().doubleSha512(Arrays.copyOfRange(bytes, 0, counter.length()), ripe);
                 this.tag = Arrays.copyOfRange(checksum, 32, 64);
                 this.publicDecryptionKey = Arrays.copyOfRange(checksum, 0, 32);
             }
@@ -172,7 +173,7 @@ public class BitmessageAddress implements Serializable {
             Encode.varInt(version, out);
             Encode.varInt(stream, out);
             out.write(ripe);
-            return Arrays.copyOfRange(security().doubleSha512(out.toByteArray()), 32, 64);
+            return Arrays.copyOfRange(cryptography().doubleSha512(out.toByteArray()), 32, 64);
         } catch (IOException e) {
             throw new ApplicationException(e);
         }
@@ -266,4 +267,11 @@ public class BitmessageAddress implements Serializable {
     public void setChan(boolean chan) {
         this.chan = chan;
     }
+
+    public boolean has(Feature feature) {
+        if (pubkey == null || feature == null) {
+            return false;
+        }
+        return feature.isActive(pubkey.getBehaviorBitfield());
+    }
 }

core/src/main/java/ch/dissem/bitmessage/entity/NetworkMessage.java

@@ -27,7 +27,7 @@ import java.security.GeneralSecurityException;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * A network message is exchanged between two nodes.
@@ -51,7 +51,7 @@ public class NetworkMessage implements Streamable {
      * First 4 bytes of sha512(payload)
      */
     private byte[] getChecksum(byte[] bytes) throws NoSuchProviderException, NoSuchAlgorithmException {
-        byte[] d = security().sha512(bytes);
+        byte[] d = cryptography().sha512(bytes);
         return new byte[]{d[0], d[1], d[2], d[3]};
     }
 

core/src/main/java/ch/dissem/bitmessage/entity/ObjectMessage.java

@@ -29,8 +29,10 @@ import ch.dissem.bitmessage.utils.Encode;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Objects;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * The 'object' command sends an object that is shared throughout the network.
@@ -55,7 +57,7 @@ public class ObjectMessage implements MessagePayload {
         expiresTime = builder.expiresTime;
         objectType = builder.objectType;
         version = builder.payload.getVersion();
-        stream = builder.streamNumber;
+        stream = builder.streamNumber > 0 ? builder.streamNumber : builder.payload.getStream();
         payload = builder.payload;
     }
 
@@ -94,7 +96,7 @@ public class ObjectMessage implements MessagePayload {
 
     public InventoryVector getInventoryVector() {
         return new InventoryVector(
-                Bytes.truncate(security().doubleSha512(nonce, getPayloadBytesWithoutNonce()), 32)
+                Bytes.truncate(cryptography().doubleSha512(nonce, getPayloadBytesWithoutNonce()), 32)
         );
     }
 
@@ -119,7 +121,7 @@ public class ObjectMessage implements MessagePayload {
 
     public void sign(PrivateKey key) {
         if (payload.isSigned()) {
-            payload.setSignature(security().getSignature(getBytesToSign(), key));
+            payload.setSignature(cryptography().getSignature(getBytesToSign(), key));
         }
     }
 
@@ -153,7 +155,7 @@ public class ObjectMessage implements MessagePayload {
 
     public boolean isSignatureValid(Pubkey pubkey) throws IOException {
         if (isEncrypted()) throw new IllegalStateException("Payload must be decrypted first");
-        return security().isSignatureValid(getBytesToSign(), payload.getSignature(), pubkey);
+        return cryptography().isSignatureValid(getBytesToSign(), payload.getSignature(), pubkey);
     }
 
     @Override
@@ -230,4 +232,29 @@ public class ObjectMessage implements MessagePayload {
             return new ObjectMessage(this);
         }
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ObjectMessage that = (ObjectMessage) o;
+
+        return expiresTime == that.expiresTime &&
+                objectType == that.objectType &&
+                version == that.version &&
+                stream == that.stream &&
+                Objects.equals(payload, that.payload);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Arrays.hashCode(nonce);
+        result = 31 * result + (int) (expiresTime ^ (expiresTime >>> 32));
+        result = 31 * result + (int) (objectType ^ (objectType >>> 32));
+        result = 31 * result + (int) (version ^ (version >>> 32));
+        result = 31 * result + (int) (stream ^ (stream >>> 32));
+        result = 31 * result + (payload != null ? payload.hashCode() : 0);
+        return result;
+    }
 }

core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java

@@ -16,16 +16,19 @@
 
 package ch.dissem.bitmessage.entity;
 
+import ch.dissem.bitmessage.entity.payload.Msg;
+import ch.dissem.bitmessage.entity.payload.Pubkey.Feature;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.exception.ApplicationException;
 import ch.dissem.bitmessage.factory.Factory;
-import ch.dissem.bitmessage.utils.Decode;
+import ch.dissem.bitmessage.utils.*;
-import ch.dissem.bitmessage.utils.Encode;
-import ch.dissem.bitmessage.utils.UnixTime;
 
 import java.io.*;
 import java.util.*;
+import java.util.Collections;
+
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * The unencrypted message to be sent by 'msg' or 'broadcast'.
@@ -37,7 +40,8 @@ public class Plaintext implements Streamable {
     private final BitmessageAddress from;
     private final long encoding;
     private final byte[] message;
-    private final byte[] ack;
+    private final byte[] ackData;
+    private ObjectMessage ackMessage;
     private Object id;
     private InventoryVector inventoryVector;
     private BitmessageAddress to;
@@ -49,6 +53,10 @@ public class Plaintext implements Streamable {
     private Set<Label> labels;
     private byte[] initialHash;
 
+    private long ttl;
+    private int retries;
+    private Long nextTry;
+
     private Plaintext(Builder builder) {
         id = builder.id;
         inventoryVector = builder.inventoryVector;
@@ -57,12 +65,21 @@ public class Plaintext implements Streamable {
         to = builder.to;
         encoding = builder.encoding;
         message = builder.message;
-        ack = builder.ack;
+        ackData = builder.ackData;
+        if (builder.ackMessage != null && builder.ackMessage.length > 0) {
+            ackMessage = Factory.getObjectMessage(
+                    3,
+                    new ByteArrayInputStream(builder.ackMessage),
+                    builder.ackMessage.length);
+        }
         signature = builder.signature;
         status = builder.status;
         sent = builder.sent;
         received = builder.received;
         labels = builder.labels;
+        ttl = builder.ttl;
+        retries = builder.retries;
+        nextTry = builder.nextTry;
     }
 
     public static Plaintext read(Type type, InputStream in) throws IOException {
@@ -85,7 +102,7 @@ public class Plaintext implements Streamable {
                 .destinationRipe(type == Type.MSG ? Decode.bytes(in, 20) : null)
                 .encoding(Decode.varInt(in))
                 .message(Decode.varBytes(in))
-                .ack(type == Type.MSG ? Decode.varBytes(in) : null);
+                .ackMessage(type == Type.MSG ? Decode.varBytes(in) : null);
     }
 
     public InventoryVector getInventoryVector() {
@@ -163,8 +180,13 @@ public class Plaintext implements Streamable {
         Encode.varInt(message.length, out);
         out.write(message);
         if (type == Type.MSG) {
-            Encode.varInt(ack.length, out);
+            if (to.has(Feature.DOES_ACK) && getAckMessage() != null) {
-            out.write(ack);
+                ByteArrayOutputStream ack = new ByteArrayOutputStream();
+                getAckMessage().write(ack);
+                Encode.varBytes(ack.toByteArray(), out);
+            } else {
+                Encode.varInt(0, out);
+            }
         }
         if (includeSignature) {
             if (signature == null) {
@@ -206,6 +228,30 @@ public class Plaintext implements Streamable {
         this.status = status;
     }
 
+    public long getTTL() {
+        return ttl;
+    }
+
+    public int getRetries() {
+        return retries;
+    }
+
+    public Long getNextTry() {
+        return nextTry;
+    }
+
+    public void updateNextTry() {
+        if (nextTry == null) {
+            if (sent != null && to.has(Feature.DOES_ACK)) {
+                nextTry = UnixTime.now(+ttl);
+                retries++;
+            }
+        } else {
+            nextTry = nextTry + (1 << retries) * ttl;
+            retries++;
+        }
+    }
+
     public String getSubject() {
         Scanner s = new Scanner(new ByteArrayInputStream(message), "UTF-8");
         String firstLine = s.nextLine();
@@ -238,7 +284,7 @@ public class Plaintext implements Streamable {
         return Objects.equals(encoding, plaintext.encoding) &&
                 Objects.equals(from, plaintext.from) &&
                 Arrays.equals(message, plaintext.message) &&
-                Arrays.equals(ack, plaintext.ack) &&
+                Objects.equals(getAckMessage(), plaintext.getAckMessage()) &&
                 Arrays.equals(to.getRipe(), plaintext.to.getRipe()) &&
                 Arrays.equals(signature, plaintext.signature) &&
                 Objects.equals(status, plaintext.status) &&
@@ -249,7 +295,7 @@ public class Plaintext implements Streamable {
 
     @Override
     public int hashCode() {
-        return Objects.hash(from, encoding, message, ack, to, signature, status, sent, received, labels);
+        return Objects.hash(from, encoding, message, ackData, to, signature, status, sent, received, labels);
     }
 
     public void addLabels(Label... labels) {
@@ -260,9 +306,32 @@ public class Plaintext implements Streamable {
 
     public void addLabels(Collection<Label> labels) {
         if (labels != null) {
-            this.labels.addAll(labels);
+            for (Label label : labels) {
+                this.labels.add(label);
             }
         }
+    }
+
+    public void removeLabel(Label.Type type) {
+        Iterator<Label> iterator = labels.iterator();
+        while (iterator.hasNext()) {
+            Label label = iterator.next();
+            if (label.getType() == type) {
+                iterator.remove();
+            }
+        }
+    }
+
+    public byte[] getAckData() {
+        return ackData;
+    }
+
+    public ObjectMessage getAckMessage() {
+        if (ackMessage == null) {
+            ackMessage = Factory.createAck(this);
+        }
+        return ackMessage;
+    }
 
     public void setInitialHash(byte[] initialHash) {
         this.initialHash = initialHash;
@@ -316,12 +385,16 @@ public class Plaintext implements Streamable {
         private byte[] destinationRipe;
         private long encoding;
         private byte[] message = new byte[0];
-        private byte[] ack = new byte[0];
+        private byte[] ackData;
+        private byte[] ackMessage;
         private byte[] signature;
         private long sent;
         private long received;
         private Status status;
         private Set<Label> labels = new HashSet<>();
+        private long ttl;
+        private int retries;
+        private Long nextTry;
 
         public Builder(Type type) {
             this.type = type;
@@ -415,9 +488,16 @@ public class Plaintext implements Streamable {
             return this;
         }
 
-        public Builder ack(byte[] ack) {
+        public Builder ackMessage(byte[] ack) {
-            if (type != Type.MSG && ack != null) throw new IllegalArgumentException("ack only allowed for msg");
+            if (type != Type.MSG && ack != null) throw new IllegalArgumentException("ackMessage only allowed for msg");
-            this.ack = ack;
+            this.ackMessage = ack;
+            return this;
+        }
+
+        public Builder ackData(byte[] ackData) {
+            if (type != Type.MSG && ackData != null)
+                throw new IllegalArgumentException("ackMessage only allowed for msg");
+            this.ackData = ackData;
             return this;
         }
 
@@ -446,6 +526,21 @@ public class Plaintext implements Streamable {
             return this;
         }
 
+        public Builder ttl(long ttl) {
+            this.ttl = ttl;
+            return this;
+        }
+
+        public Builder retries(int retries) {
+            this.retries = retries;
+            return this;
+        }
+
+        public Builder nextTry(Long nextTry) {
+            this.nextTry = nextTry;
+            return this;
+        }
+
         public Plaintext build() {
             if (from == null) {
                 from = new BitmessageAddress(Factory.createPubkey(
@@ -461,6 +556,12 @@ public class Plaintext implements Streamable {
             if (to == null && type != Type.BROADCAST && destinationRipe != null) {
                 to = new BitmessageAddress(0, 0, destinationRipe);
             }
+            if (type == Type.MSG && ackMessage == null && ackData == null) {
+                ackData = cryptography().randomBytes(Msg.ACK_LENGTH);
+            }
+            if (ttl <= 0) {
+                ttl = TTL.msg();
+            }
             return new Plaintext(this);
         }
     }

core/src/main/java/ch/dissem/bitmessage/entity/Version.java

@@ -145,13 +145,13 @@ public class Version implements MessagePayload {
         private String userAgent;
         private long[] streamNumbers;
 
-        public Builder defaults() {
+        public Builder defaults(long clientNonce) {
             version = BitmessageContext.CURRENT_VERSION;
             services = 1;
             timestamp = UnixTime.now();
-            nonce = new Random().nextInt();
             userAgent = "/Jabit:0.0.1/";
             streamNumbers = new long[]{1};
+            nonce = clientNonce;
             return this;
         }
 

core/src/main/java/ch/dissem/bitmessage/entity/payload/Broadcast.java

@@ -23,9 +23,10 @@ import ch.dissem.bitmessage.entity.PlaintextHolder;
 import ch.dissem.bitmessage.exception.DecryptionFailedException;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import static ch.dissem.bitmessage.entity.Plaintext.Type.BROADCAST;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * Users who are subscribed to the sending address will see the message appear in their inbox.
@@ -80,7 +81,7 @@ public abstract class Broadcast extends ObjectPayload implements Encrypted, Plai
     }
 
     public void encrypt() throws IOException {
-        encrypt(security().createPublicKey(plaintext.getFrom().getPublicDecryptionKey()));
+        encrypt(cryptography().createPublicKey(plaintext.getFrom().getPublicDecryptionKey()));
     }
 
     @Override
@@ -96,4 +97,18 @@ public abstract class Broadcast extends ObjectPayload implements Encrypted, Plai
     public boolean isDecrypted() {
         return plaintext != null;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Broadcast broadcast = (Broadcast) o;
+        return stream == broadcast.stream &&
+                (Objects.equals(encrypted, broadcast.encrypted) || Objects.equals(plaintext, broadcast.plaintext));
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(stream);
+    }
 }

core/src/main/java/ch/dissem/bitmessage/entity/payload/CryptoBox.java

@@ -27,7 +27,7 @@ import java.io.*;
 import java.util.Arrays;
 
 import static ch.dissem.bitmessage.entity.valueobject.PrivateKey.PRIVATE_KEY_SIZE;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 
 public class CryptoBox implements Streamable {
@@ -50,22 +50,22 @@ public class CryptoBox implements Streamable {
 
         // 1. The destination public key is called K.
         // 2. Generate 16 random bytes using a secure random number generator. Call them IV.
-        initializationVector = security().randomBytes(16);
+        initializationVector = cryptography().randomBytes(16);
 
         // 3. Generate a new random EC key pair with private key called r and public key called R.
-        byte[] r = security().randomBytes(PRIVATE_KEY_SIZE);
+        byte[] r = cryptography().randomBytes(PRIVATE_KEY_SIZE);
-        R = security().createPublicKey(r);
+        R = cryptography().createPublicKey(r);
         // 4. Do an EC point multiply with public key K and private key r. This gives you public key P.
-        byte[] P = security().multiply(K, r);
+        byte[] P = cryptography().multiply(K, r);
         byte[] X = Points.getX(P);
         // 5. Use the X component of public key P and calculate the SHA512 hash H.
-        byte[] H = security().sha512(X);
+        byte[] H = cryptography().sha512(X);
         // 6. The first 32 bytes of H are called key_e and the last 32 bytes are called key_m.
         byte[] key_e = Arrays.copyOfRange(H, 0, 32);
         byte[] key_m = Arrays.copyOfRange(H, 32, 64);
         // 7. Pad the input text to a multiple of 16 bytes, in accordance to PKCS7.
         // 8. Encrypt the data with AES-256-CBC, using IV as initialization vector, key_e as encryption key and the padded input text as payload. Call the output cipher text.
-        encrypted = security().crypt(true, data, key_e, initializationVector);
+        encrypted = cryptography().crypt(true, data, key_e, initializationVector);
         // 9. Calculate a 32 byte MAC with HMACSHA256, using key_m as salt and IV + R + cipher text as data. Call the output MAC.
         mac = calculateMac(key_m);
 
@@ -75,7 +75,7 @@ public class CryptoBox implements Streamable {
     private CryptoBox(Builder builder) {
         initializationVector = builder.initializationVector;
         curveType = builder.curveType;
-        R = security().createPoint(builder.xComponent, builder.yComponent);
+        R = cryptography().createPoint(builder.xComponent, builder.yComponent);
         encrypted = builder.encrypted;
         mac = builder.mac;
     }
@@ -101,9 +101,9 @@ public class CryptoBox implements Streamable {
     public InputStream decrypt(byte[] k) throws DecryptionFailedException {
         // 1. The private key used to decrypt is called k.
         // 2. Do an EC point multiply with private key k and public key R. This gives you public key P.
-        byte[] P = security().multiply(R, k);
+        byte[] P = cryptography().multiply(R, k);
         // 3. Use the X component of public key P and calculate the SHA512 hash H.
-        byte[] H = security().sha512(Arrays.copyOfRange(P, 1, 33));
+        byte[] H = cryptography().sha512(Arrays.copyOfRange(P, 1, 33));
         // 4. The first 32 bytes of H are called key_e and the last 32 bytes are called key_m.
         byte[] key_e = Arrays.copyOfRange(H, 0, 32);
         byte[] key_m = Arrays.copyOfRange(H, 32, 64);
@@ -116,14 +116,14 @@ public class CryptoBox implements Streamable {
 
         // 7. Decrypt the cipher text with AES-256-CBC, using IV as initialization vector, key_e as decryption key
         //    and the cipher text as payload. The output is the padded input text.
-        return new ByteArrayInputStream(security().crypt(false, encrypted, key_e, initializationVector));
+        return new ByteArrayInputStream(cryptography().crypt(false, encrypted, key_e, initializationVector));
     }
 
     private byte[] calculateMac(byte[] key_m) {
         try {
             ByteArrayOutputStream macData = new ByteArrayOutputStream();
             writeWithoutMAC(macData);
-            return security().mac(key_m, macData.toByteArray());
+            return cryptography().mac(key_m, macData.toByteArray());
         } catch (IOException e) {
             throw new ApplicationException(e);
         }

core/src/main/java/ch/dissem/bitmessage/entity/payload/GenericPayload.java

@@ -39,7 +39,7 @@ public class GenericPayload extends ObjectPayload {
         this.data = data;
     }
 
-    public static GenericPayload read(long version, InputStream is, long stream, int length) throws IOException {
+    public static GenericPayload read(long version, long stream, InputStream is, int length) throws IOException {
         return new GenericPayload(version, stream, Decode.bytes(is, length));
     }
 
@@ -53,6 +53,10 @@ public class GenericPayload extends ObjectPayload {
         return stream;
     }
 
+    public byte[] getData() {
+        return data;
+    }
+
     @Override
     public void write(OutputStream stream) throws IOException {
         stream.write(data);

core/src/main/java/ch/dissem/bitmessage/entity/payload/Msg.java

@@ -24,6 +24,7 @@ import ch.dissem.bitmessage.exception.DecryptionFailedException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Objects;
 
 import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
 
@@ -32,6 +33,7 @@ import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
  */
 public class Msg extends ObjectPayload implements Encrypted, PlaintextHolder {
     private static final long serialVersionUID = 4327495048296365733L;
+    public static final int ACK_LENGTH = 32;
 
     private long stream;
     private CryptoBox encrypted;
@@ -108,4 +110,19 @@ public class Msg extends ObjectPayload implements Encrypted, PlaintextHolder {
         if (encrypted == null) throw new IllegalStateException("Msg must be signed and encrypted before writing it.");
         encrypted.write(out);
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        Msg msg = (Msg) o;
+        return stream == msg.stream &&
+                (Objects.equals(encrypted, msg.encrypted) || Objects.equals(plaintext, msg.plaintext));
+    }
+
+    @Override
+    public int hashCode() {
+        return (int) stream;
+    }
 }

core/src/main/java/ch/dissem/bitmessage/entity/payload/Pubkey.java

@@ -20,7 +20,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * Public keys for signing and encryption, the answer to a 'getpubkey' request.
@@ -35,7 +35,7 @@ public abstract class Pubkey extends ObjectPayload {
     }
 
     public static byte[] getRipe(byte[] publicSigningKey, byte[] publicEncryptionKey) {
-        return security().ripemd160(security().sha512(publicSigningKey, publicEncryptionKey));
+        return cryptography().ripemd160(cryptography().sha512(publicSigningKey, publicEncryptionKey));
     }
 
     public abstract byte[] getSigningKey();
@@ -45,7 +45,7 @@ public abstract class Pubkey extends ObjectPayload {
     public abstract int getBehaviorBitfield();
 
     public byte[] getRipe() {
-        return security().ripemd160(security().sha512(getSigningKey(), getEncryptionKey()));
+        return cryptography().ripemd160(cryptography().sha512(getSigningKey(), getEncryptionKey()));
     }
 
     public long getNonceTrialsPerByte() {
@@ -76,16 +76,19 @@ public abstract class Pubkey extends ObjectPayload {
          * Receiving node expects that the RIPE hash encoded in their address preceedes the encrypted message data of msg
          * messages bound for them.
          */
-        INCLUDE_DESTINATION(1 << 30),
+        INCLUDE_DESTINATION(30),
         /**
          * If true, the receiving node does send acknowledgements (rather than dropping them).
          */
-        DOES_ACK(1 << 31);
+        DOES_ACK(31);
 
         private int bit;
 
-        Feature(int bit) {
+        Feature(int bitNumber) {
-            this.bit = bit;
+            // The Bitmessage Protocol Specification starts counting at the most significant bit,
+            // thus the slightly awkward calculation.
+            // https://bitmessage.org/wiki/Protocol_specification#Pubkey_bitfield_features
+            this.bit = 1 << (31 - bitNumber);
         }
 
         public static int bitfield(Feature... features) {
@@ -105,5 +108,9 @@ public abstract class Pubkey extends ObjectPayload {
             }
             return features.toArray(new Feature[features.size()]);
         }
+
+        public boolean isActive(int bitfield) {
+            return (bitfield & bit) != 0;
+        }
     }
 }

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/PrivateKey.java

@@ -30,7 +30,7 @@ import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * Represents a private key. Additional information (stream, version, features, ...) is stored in the accompanying
@@ -53,15 +53,15 @@ public class PrivateKey implements Streamable {
         byte[] pubEK;
         byte[] ripe;
         do {
-            privSK = security().randomBytes(PRIVATE_KEY_SIZE);
+            privSK = cryptography().randomBytes(PRIVATE_KEY_SIZE);
-            privEK = security().randomBytes(PRIVATE_KEY_SIZE);
+            privEK = cryptography().randomBytes(PRIVATE_KEY_SIZE);
-            pubSK = security().createPublicKey(privSK);
+            pubSK = cryptography().createPublicKey(privSK);
-            pubEK = security().createPublicKey(privEK);
+            pubEK = cryptography().createPublicKey(privEK);
             ripe = Pubkey.getRipe(pubSK, pubEK);
         } while (ripe[0] != 0 || (shorter && ripe[1] != 0));
         this.privateSigningKey = privSK;
         this.privateEncryptionKey = privEK;
-        this.pubkey = security().createPubkey(Pubkey.LATEST_VERSION, stream, privateSigningKey, privateEncryptionKey,
+        this.pubkey = cryptography().createPubkey(Pubkey.LATEST_VERSION, stream, privateSigningKey, privateEncryptionKey,
                 nonceTrialsPerByte, extraBytes, features);
     }
 
@@ -118,11 +118,11 @@ public class PrivateKey implements Streamable {
                 long encryptionKeyNonce = nextNonce + 1;
                 byte[] ripe;
                 do {
-                    privEK = Bytes.truncate(security().sha512(seed, Encode.varInt(encryptionKeyNonce)), 32);
+                    privEK = Bytes.truncate(cryptography().sha512(seed, Encode.varInt(encryptionKeyNonce)), 32);
-                    privSK = Bytes.truncate(security().sha512(seed, Encode.varInt(signingKeyNonce)), 32);
+                    privSK = Bytes.truncate(cryptography().sha512(seed, Encode.varInt(signingKeyNonce)), 32);
-                    pubSK = security().createPublicKey(privSK);
+                    pubSK = cryptography().createPublicKey(privSK);
-                    pubEK = security().createPublicKey(privEK);
+                    pubEK = cryptography().createPublicKey(privEK);
-                    ripe = security().ripemd160(security().sha512(pubSK, pubEK));
+                    ripe = cryptography().ripemd160(cryptography().sha512(pubSK, pubEK));
 
                     signingKeyNonce += 2;
                     encryptionKeyNonce += 2;

core/src/main/java/ch/dissem/bitmessage/factory/Factory.java

@@ -23,6 +23,8 @@ import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.entity.payload.*;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
 import ch.dissem.bitmessage.exception.NodeException;
+import ch.dissem.bitmessage.utils.TTL;
+import ch.dissem.bitmessage.utils.UnixTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,7 +33,8 @@ import java.io.InputStream;
 import java.net.SocketException;
 import java.net.SocketTimeoutException;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.entity.payload.ObjectType.MSG;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * Creates {@link NetworkMessage} objects from {@link InputStream InputStreams}
@@ -116,8 +119,8 @@ public class Factory {
         BitmessageAddress temp = new BitmessageAddress(address);
         PrivateKey privateKey = new PrivateKey(privateSigningKey, privateEncryptionKey,
                 createPubkey(temp.getVersion(), temp.getStream(),
-                        security().createPublicKey(privateSigningKey),
+                        cryptography().createPublicKey(privateSigningKey),
-                        security().createPublicKey(privateEncryptionKey),
+                        cryptography().createPublicKey(privateEncryptionKey),
                         nonceTrialsPerByte, extraBytes, behaviourBitfield));
         BitmessageAddress result = new BitmessageAddress(privateKey);
         if (!result.getAddress().equals(address)) {
@@ -155,7 +158,7 @@ public class Factory {
         }
         // fallback: just store the message - we don't really care what it is
         LOG.trace("Unexpected object type: " + objectType);
-        return GenericPayload.read(version, stream, streamNumber, length);
+        return GenericPayload.read(version, streamNumber, stream, length);
     }
 
     private static ObjectPayload parseGetPubkey(long version, long streamNumber, InputStream stream, int length) throws IOException {
@@ -177,7 +180,7 @@ public class Factory {
 
     private static ObjectPayload parsePubkey(long version, long streamNumber, InputStream stream, int length) throws IOException {
         Pubkey pubkey = readPubkey(version, streamNumber, stream, length, true);
-        return pubkey != null ? pubkey : GenericPayload.read(version, stream, streamNumber, length);
+        return pubkey != null ? pubkey : GenericPayload.read(version, streamNumber, stream, length);
     }
 
     private static ObjectPayload parseMsg(long version, long streamNumber, InputStream stream, int length) throws IOException {
@@ -192,7 +195,7 @@ public class Factory {
                 return V5Broadcast.read(stream, streamNumber, length);
             default:
                 LOG.debug("Encountered unknown broadcast version " + version);
-                return GenericPayload.read(version, stream, streamNumber, length);
+                return GenericPayload.read(version, streamNumber, stream, length);
         }
     }
 
@@ -204,4 +207,11 @@ public class Factory {
             return new V5Broadcast(sendingAddress, plaintext);
         }
     }
+
+    public static ObjectMessage createAck(Plaintext plaintext) {
+        if (plaintext == null || plaintext.getAckData() == null)
+            return null;
+        GenericPayload ack = new GenericPayload(3, plaintext.getFrom().getStream(), plaintext.getAckData());
+        return new ObjectMessage.Builder().objectType(MSG).payload(ack).expiresTime(UnixTime.now(plaintext.getTTL())).build();
+    }
 }

core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java

@@ -32,7 +32,7 @@ import java.io.IOException;
 import java.io.InputStream;
 
 import static ch.dissem.bitmessage.entity.NetworkMessage.MAGIC_BYTES;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * Creates protocol v3 network messages from {@link InputStream InputStreams}
@@ -183,7 +183,7 @@ class V3MessageFactory {
     }
 
     private static boolean testChecksum(byte[] checksum, byte[] payload) {
-        byte[] payloadChecksum = security().sha512(payload);
+        byte[] payloadChecksum = cryptography().sha512(payload);
         for (int i = 0; i < checksum.length; i++) {
             if (checksum[i] != payloadChecksum[i]) {
                 return false;

core/src/main/java/ch/dissem/bitmessage/ports/AbstractMessageRepository.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2016 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.ports;
+
+import ch.dissem.bitmessage.InternalContext;
+import ch.dissem.bitmessage.entity.BitmessageAddress;
+import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.valueobject.Label;
+import ch.dissem.bitmessage.exception.ApplicationException;
+import ch.dissem.bitmessage.utils.Strings;
+import ch.dissem.bitmessage.utils.UnixTime;
+
+import java.util.Collection;
+import java.util.List;
+
+import static ch.dissem.bitmessage.utils.SqlStrings.join;
+
+public abstract class AbstractMessageRepository implements MessageRepository, InternalContext.ContextHolder {
+    protected InternalContext ctx;
+
+    @Override
+    public void setContext(InternalContext context) {
+        this.ctx = context;
+    }
+
+    protected void safeSenderIfNecessary(Plaintext message) {
+        if (message.getId() == null) {
+            BitmessageAddress savedAddress = ctx.getAddressRepository().getAddress(message.getFrom().getAddress());
+            if (savedAddress == null) {
+                ctx.getAddressRepository().save(message.getFrom());
+            } else if (savedAddress.getPubkey() == null && message.getFrom().getPubkey() != null) {
+                savedAddress.setPubkey(message.getFrom().getPubkey());
+                ctx.getAddressRepository().save(savedAddress);
+            }
+        }
+    }
+
+    @Override
+    public Plaintext getMessage(Object id) {
+        if (id instanceof Long) {
+            return single(find("id=" + id));
+        } else {
+            throw new IllegalArgumentException("Long expected for ID");
+        }
+    }
+
+    @Override
+    public Plaintext getMessage(byte[] initialHash) {
+        return single(find("initial_hash=X'" + Strings.hex(initialHash) + "'"));
+    }
+
+    @Override
+    public Plaintext getMessageForAck(byte[] ackData) {
+        return single(find("ack_data=X'" + Strings.hex(ackData) + "' AND status='" + Plaintext.Status.SENT + "'"));
+    }
+
+    @Override
+    public List<Plaintext> findMessages(Label label) {
+        return find("id IN (SELECT message_id FROM Message_Label WHERE label_id=" + label.getId() + ")");
+    }
+
+    @Override
+    public List<Plaintext> findMessages(Plaintext.Status status, BitmessageAddress recipient) {
+        return find("status='" + status.name() + "' AND recipient='" + recipient.getAddress() + "'");
+    }
+
+    @Override
+    public List<Plaintext> findMessages(Plaintext.Status status) {
+        return find("status='" + status.name() + "'");
+    }
+
+    @Override
+    public List<Plaintext> findMessages(BitmessageAddress sender) {
+        return find("sender='" + sender.getAddress() + "'");
+    }
+
+    @Override
+    public List<Plaintext> findMessagesToResend() {
+        return find("status='" + Plaintext.Status.SENT.name() + "'" +
+                " AND next_try < " + UnixTime.now());
+    }
+
+    @Override
+    public List<Label> getLabels() {
+        return findLabels("1=1");
+    }
+
+    @Override
+    public List<Label> getLabels(Label.Type... types) {
+        return findLabels("type IN (" + join(types) + ")");
+    }
+
+    protected abstract List<Label> findLabels(String where);
+
+
+    protected <T> T single(Collection<T> collection) {
+        switch (collection.size()) {
+            case 0:
+                return null;
+            case 1:
+                return collection.iterator().next();
+            default:
+                throw new ApplicationException("This shouldn't happen, found " + collection.size() +
+                        " items, one or none was expected");
+        }
+    }
+
+    protected abstract List<Plaintext> find(String where);
+}

core/src/main/java/ch/dissem/bitmessage/ports/DefaultLabeler.java

@@ -20,13 +20,14 @@ import ch.dissem.bitmessage.InternalContext;
 import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 
-import java.util.Iterator;
+import static ch.dissem.bitmessage.entity.Plaintext.Status.*;
 
 public class DefaultLabeler implements Labeler, InternalContext.ContextHolder {
     private InternalContext ctx;
 
     @Override
     public void setLabels(Plaintext msg) {
+        msg.setStatus(RECEIVED);
         if (msg.getType() == Plaintext.Type.BROADCAST) {
             msg.addLabels(ctx.getMessageRepository().getLabels(Label.Type.INBOX, Label.Type.BROADCAST, Label.Type.UNREAD));
         } else {
@@ -34,15 +35,38 @@ public class DefaultLabeler implements Labeler, InternalContext.ContextHolder {
         }
     }
 
+    @Override
+    public void markAsDraft(Plaintext msg) {
+        msg.setStatus(DRAFT);
+        msg.addLabels(ctx.getMessageRepository().getLabels(Label.Type.DRAFT));
+    }
+
+    @Override
+    public void markAsSending(Plaintext msg) {
+        if (msg.getTo() != null && msg.getTo().getPubkey() == null) {
+            msg.setStatus(PUBKEY_REQUESTED);
+        } else {
+            msg.setStatus(DOING_PROOF_OF_WORK);
+        }
+        msg.removeLabel(Label.Type.DRAFT);
+        msg.addLabels(ctx.getMessageRepository().getLabels(Label.Type.OUTBOX));
+    }
+
+    @Override
+    public void markAsSent(Plaintext msg) {
+        msg.setStatus(SENT);
+        msg.removeLabel(Label.Type.OUTBOX);
+        msg.addLabels(ctx.getMessageRepository().getLabels(Label.Type.SENT));
+    }
+
+    @Override
+    public void markAsAcknowledged(Plaintext msg) {
+        msg.setStatus(SENT_ACKNOWLEDGED);
+    }
+
     @Override
     public void markAsRead(Plaintext msg) {
-        Iterator<Label> iterator = msg.getLabels().iterator();
+        msg.removeLabel(Label.Type.UNREAD);
-        while (iterator.hasNext()) {
-            Label label = iterator.next();
-            if (label.getType() == Label.Type.UNREAD) {
-                iterator.remove();
-            }
-        }
     }
 
     @Override

core/src/main/java/ch/dissem/bitmessage/ports/Labeler.java

@@ -19,7 +19,13 @@ package ch.dissem.bitmessage.ports;
 import ch.dissem.bitmessage.entity.Plaintext;
 
 /**
- * Defines and sets labels
+ * Defines and sets labels. Note that it should also update the status field of a message.
+ * Generally it's highly advised to override the {@link DefaultLabeler} whenever possible,
+ * instead of directly implementing the interface.
+ * <p>
+ * As the labeler gets called whenever the state of a message changes, it can also be used
+ * as a listener.
+ * </p>
  */
 public interface Labeler {
     /**
@@ -29,6 +35,18 @@ public interface Labeler {
      */
     void setLabels(Plaintext msg);
 
+    void markAsDraft(Plaintext msg);
+
+    /**
+     * It is paramount that this methods marks the {@link Plaintext} object with status
+     * {@link Plaintext.Status#PUBKEY_REQUESTED} (see {@link DefaultLabeler})
+     */
+    void markAsSending(Plaintext msg);
+
+    void markAsSent(Plaintext msg);
+
+    void markAsAcknowledged(Plaintext msg);
+
     void markAsRead(Plaintext msg);
 
     void markAsUnread(Plaintext msg);

core/src/main/java/ch/dissem/bitmessage/ports/MemoryNodeRegistry.java

@@ -46,15 +46,11 @@ public class MemoryNodeRegistry implements NodeRegistry {
             while (scanner.hasNext()) {
                 try {
                     String line = scanner.nextLine().trim();
-                    if (line.startsWith("#") || line.isEmpty()) {
-                        // Ignore
-                        continue;
-                    }
                     if (line.startsWith("[stream")) {
                         stream = Long.parseLong(line.substring(8, line.lastIndexOf(']')));
                         streamSet = new HashSet<>();
                         stableNodes.put(stream, streamSet);
-                    } else if (streamSet != null) {
+                    } else if (streamSet != null && !line.isEmpty() && !line.startsWith("#")) {
                         int portIndex = line.lastIndexOf(':');
                         InetAddress[] inetAddresses = InetAddress.getAllByName(line.substring(0, portIndex));
                         int port = Integer.valueOf(line.substring(portIndex + 1));
@@ -89,12 +85,12 @@ public class MemoryNodeRegistry implements NodeRegistry {
                         known.remove(node);
                     }
                 }
-            } else {
-                Set<NetworkAddress> nodes = stableNodes.get(stream);
-                if (nodes == null || nodes.isEmpty()) {
-                    loadStableNodes();
-                    nodes = stableNodes.get(stream);
             }
+            if (result.isEmpty()) {
+                if (stableNodes.isEmpty()) {
+                    loadStableNodes();
+                }
+                Set<NetworkAddress> nodes = stableNodes.get(stream);
                 if (nodes != null && !nodes.isEmpty()) {
                     // To reduce load on stable nodes, only return one
                     result.add(selectRandom(nodes));

core/src/main/java/ch/dissem/bitmessage/ports/MessageRepository.java

@@ -30,8 +30,12 @@ public interface MessageRepository {
 
     int countUnread(Label label);
 
+    Plaintext getMessage(Object id);
+
     Plaintext getMessage(byte[] initialHash);
 
+    Plaintext getMessageForAck(byte[] ackData);
+
     List<Plaintext> findMessages(Label label);
 
     List<Plaintext> findMessages(Status status);
@@ -40,6 +44,8 @@ public interface MessageRepository {
 
     List<Plaintext> findMessages(BitmessageAddress sender);
 
+    List<Plaintext> findMessagesToResend();
+
     void save(Plaintext message);
 
     void remove(Plaintext message);

core/src/main/java/ch/dissem/bitmessage/ports/ProofOfWorkRepository.java

@@ -1,6 +1,7 @@
 package ch.dissem.bitmessage.ports;
 
 import ch.dissem.bitmessage.entity.ObjectMessage;
+import ch.dissem.bitmessage.entity.Plaintext;
 
 import java.util.List;
 
@@ -16,6 +17,8 @@ public interface ProofOfWorkRepository {
 
     void putObject(ObjectMessage object, long nonceTrialsPerByte, long extraBytes);
 
+    void putObject(Item item);
+
     void removeObject(byte[] initialHash);
 
     class Item {
@@ -23,10 +26,20 @@ public interface ProofOfWorkRepository {
         public final long nonceTrialsPerByte;
         public final long extraBytes;
 
+        // Needed for ACK POW calculation
+        public final Long expirationTime;
+        public final Plaintext message;
+
         public Item(ObjectMessage object, long nonceTrialsPerByte, long extraBytes) {
+            this(object, nonceTrialsPerByte, extraBytes, 0, null);
+        }
+
+        public Item(ObjectMessage object, long nonceTrialsPerByte, long extraBytes, long expirationTime, Plaintext message) {
             this.object = object;
             this.nonceTrialsPerByte = nonceTrialsPerByte;
             this.extraBytes = extraBytes;
+            this.expirationTime = expirationTime;
+            this.message = message;
         }
     }
 }

core/src/main/java/ch/dissem/bitmessage/utils/Singleton.java

@@ -30,7 +30,7 @@ public class Singleton {
         }
     }
 
-    public static Cryptography security() {
+    public static Cryptography cryptography() {
         return cryptography;
     }
 }

core/src/main/java/ch/dissem/bitmessage/utils/SqlStrings.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2016 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.utils;
+
+import ch.dissem.bitmessage.entity.payload.ObjectType;
+
+import static ch.dissem.bitmessage.utils.Strings.hex;
+
+public class SqlStrings {
+    public static StringBuilder join(long... objects) {
+        StringBuilder streamList = new StringBuilder();
+        for (int i = 0; i < objects.length; i++) {
+            if (i > 0) streamList.append(", ");
+            streamList.append(objects[i]);
+        }
+        return streamList;
+    }
+
+    public static StringBuilder join(byte[]... objects) {
+        StringBuilder streamList = new StringBuilder();
+        for (int i = 0; i < objects.length; i++) {
+            if (i > 0) streamList.append(", ");
+            streamList.append(hex(objects[i]));
+        }
+        return streamList;
+    }
+
+    public static StringBuilder join(ObjectType... types) {
+        StringBuilder streamList = new StringBuilder();
+        for (int i = 0; i < types.length; i++) {
+            if (i > 0) streamList.append(", ");
+            streamList.append(types[i].getNumber());
+        }
+        return streamList;
+    }
+
+    public static StringBuilder join(Enum... types) {
+        StringBuilder streamList = new StringBuilder();
+        for (int i = 0; i < types.length; i++) {
+            if (i > 0) streamList.append(", ");
+            streamList.append('\'').append(types[i].name()).append('\'');
+        }
+        return streamList;
+    }
+}

core/src/test/java/ch/dissem/bitmessage/BitmessageContextTest.java

@@ -20,11 +20,14 @@ import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography;
 import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.Plaintext.Type;
 import ch.dissem.bitmessage.entity.payload.ObjectType;
 import ch.dissem.bitmessage.entity.payload.Pubkey;
+import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.ports.*;
 import ch.dissem.bitmessage.utils.MessageMatchers;
 import ch.dissem.bitmessage.utils.Singleton;
+import ch.dissem.bitmessage.utils.TTL;
 import ch.dissem.bitmessage.utils.TestUtils;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
@@ -35,6 +38,8 @@ import java.util.*;
 
 import static ch.dissem.bitmessage.entity.payload.ObjectType.*;
 import static ch.dissem.bitmessage.utils.MessageMatchers.object;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
+import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.junit.Assert.*;
@@ -56,13 +61,50 @@ public class BitmessageContextTest {
                 .cryptography(new BouncyCryptography())
                 .inventory(mock(Inventory.class))
                 .listener(listener)
-                .messageCallback(mock(MessageCallback.class))
                 .messageRepo(mock(MessageRepository.class))
                 .networkHandler(mock(NetworkHandler.class))
                 .nodeRegistry(mock(NodeRegistry.class))
-                .powRepo(mock(ProofOfWorkRepository.class))
+                .labeler(spy(new DefaultLabeler()))
-                .proofOfWorkEngine(mock(ProofOfWorkEngine.class))
+                .powRepo(spy(new ProofOfWorkRepository() {
+                    Map<InventoryVector, Item> items = new HashMap<>();
+
+                    @Override
+                    public Item getItem(byte[] initialHash) {
+                        return items.get(new InventoryVector(initialHash));
+                    }
+
+                    @Override
+                    public List<byte[]> getItems() {
+                        List<byte[]> result = new LinkedList<>();
+                        for (InventoryVector iv : items.keySet()) {
+                            result.add(iv.getHash());
+                        }
+                        return result;
+                    }
+
+                    @Override
+                    public void putObject(Item item) {
+                        items.put(new InventoryVector(cryptography().getInitialHash(item.object)), item);
+                    }
+
+                    @Override
+                    public void putObject(ObjectMessage object, long nonceTrialsPerByte, long extraBytes) {
+                        items.put(new InventoryVector(cryptography().getInitialHash(object)), new Item(object, nonceTrialsPerByte, extraBytes));
+                    }
+
+                    @Override
+                    public void removeObject(byte[] initialHash) {
+                        items.remove(initialHash);
+                    }
+                }))
+                .proofOfWorkEngine(spy(new ProofOfWorkEngine() {
+                    @Override
+                    public void calculateNonce(byte[] initialHash, byte[] target, Callback callback) {
+                        callback.onNonceCalculated(initialHash, new byte[8]);
+                    }
+                }))
                 .build();
+        TTL.msg(2 * MINUTE);
     }
 
     @Test
@@ -162,9 +204,10 @@ public class BitmessageContextTest {
     public void ensureMessageIsSent() throws Exception {
         ctx.send(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"), TestUtils.loadContact(),
                 "Subject", "Message");
+        assertEquals(2, ctx.internals().getProofOfWorkRepository().getItems().size());
         verify(ctx.internals().getProofOfWorkRepository(), timeout(10000).atLeastOnce())
                 .putObject(object(MSG), eq(1000L), eq(1000L));
-        verify(ctx.messages(), timeout(10000).atLeastOnce()).save(MessageMatchers.plaintext(Plaintext.Type.MSG));
+        verify(ctx.messages(), timeout(10000).atLeastOnce()).save(MessageMatchers.plaintext(Type.MSG));
     }
 
     @Test
@@ -174,7 +217,7 @@ public class BitmessageContextTest {
                 "Subject", "Message");
         verify(ctx.internals().getProofOfWorkRepository(), timeout(10000).atLeastOnce())
                 .putObject(object(GET_PUBKEY), eq(1000L), eq(1000L));
-        verify(ctx.messages(), timeout(10000).atLeastOnce()).save(MessageMatchers.plaintext(Plaintext.Type.MSG));
+        verify(ctx.messages(), timeout(10000).atLeastOnce()).save(MessageMatchers.plaintext(Type.MSG));
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -193,12 +236,12 @@ public class BitmessageContextTest {
         verify(ctx.internals().getProofOfWorkEngine())
                 .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
         verify(ctx.messages(), timeout(10000).atLeastOnce())
-                .save(MessageMatchers.plaintext(Plaintext.Type.BROADCAST));
+                .save(MessageMatchers.plaintext(Type.BROADCAST));
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void ensureSenderWithoutPrivateKeyThrowsException() {
-        Plaintext msg = new Plaintext.Builder(Plaintext.Type.BROADCAST)
+        Plaintext msg = new Plaintext.Builder(Type.BROADCAST)
                 .from(new BitmessageAddress("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
                 .message("Subject", "Message")
                 .build();
@@ -254,4 +297,19 @@ public class BitmessageContextTest {
         assertEquals(chan.getVersion(), Pubkey.LATEST_VERSION);
         assertTrue(chan.isChan());
     }
+
+    @Test
+    public void ensureUnacknowledgedMessageIsResent() throws Exception {
+        Plaintext plaintext = new Plaintext.Builder(Type.MSG)
+                .ttl(1)
+                .message("subject", "message")
+                .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+                .to(TestUtils.loadContact())
+                .build();
+        assertTrue(plaintext.getTo().has(Pubkey.Feature.DOES_ACK));
+        when(ctx.messages().findMessagesToResend()).thenReturn(Collections.singletonList(plaintext));
+        when(ctx.messages().getMessage(any(byte[].class))).thenReturn(plaintext);
+        ctx.resendUnacknowledgedMessages();
+        verify(ctx.labeler(), timeout(1000).times(1)).markAsSent(eq(plaintext));
+    }
 }

core/src/test/java/ch/dissem/bitmessage/DefaultMessageListenerTest.java

@@ -24,9 +24,7 @@ import ch.dissem.bitmessage.entity.payload.Broadcast;
 import ch.dissem.bitmessage.entity.payload.GetPubkey;
 import ch.dissem.bitmessage.entity.payload.Msg;
 import ch.dissem.bitmessage.factory.Factory;
-import ch.dissem.bitmessage.ports.AddressRepository;
+import ch.dissem.bitmessage.ports.*;
-import ch.dissem.bitmessage.ports.Labeler;
-import ch.dissem.bitmessage.ports.MessageRepository;
 import ch.dissem.bitmessage.utils.Singleton;
 import ch.dissem.bitmessage.utils.TestBase;
 import ch.dissem.bitmessage.utils.TestUtils;
@@ -51,6 +49,10 @@ public class DefaultMessageListenerTest extends TestBase {
     private AddressRepository addressRepo;
     @Mock
     private MessageRepository messageRepo;
+    @Mock
+    private Inventory inventory;
+    @Mock
+    private NetworkHandler networkHandler;
 
     private InternalContext ctx;
     private DefaultMessageListener listener;
@@ -62,6 +64,9 @@ public class DefaultMessageListenerTest extends TestBase {
         Singleton.initialize(new BouncyCryptography());
         when(ctx.getAddressRepository()).thenReturn(addressRepo);
         when(ctx.getMessageRepository()).thenReturn(messageRepo);
+        when(ctx.getInventory()).thenReturn(inventory);
+        when(ctx.getNetworkHandler()).thenReturn(networkHandler);
+        when(ctx.getLabeler()).thenReturn(mock(Labeler.class));
 
         listener = new DefaultMessageListener(ctx, mock(Labeler.class), mock(BitmessageContext.Listener.class));
     }
@@ -94,7 +99,7 @@ public class DefaultMessageListenerTest extends TestBase {
                 .payload(identity.getPubkey())
                 .build();
         objectMessage.sign(identity.getPrivateKey());
-        objectMessage.encrypt(Singleton.security().createPublicKey(identity.getPublicDecryptionKey()));
+        objectMessage.encrypt(Singleton.cryptography().createPublicKey(identity.getPublicDecryptionKey()));
         listener.receive(objectMessage);
 
         verify(addressRepo).save(any(BitmessageAddress.class));
@@ -104,6 +109,7 @@ public class DefaultMessageListenerTest extends TestBase {
     public void ensureIncomingMessageIsSaved() throws Exception {
         BitmessageAddress identity = TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8");
         BitmessageAddress contact = new BitmessageAddress(identity.getAddress());
+        contact.setPubkey(identity.getPubkey());
 
         when(addressRepo.getIdentities()).thenReturn(Collections.singletonList(identity));
 

core/src/test/java/ch/dissem/bitmessage/EncryptionTest.java

@@ -30,19 +30,19 @@ import org.junit.Test;
 
 import java.io.IOException;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
 public class EncryptionTest extends TestBase {
     @Test
     public void ensureDecryptedDataIsSameAsBeforeEncryption() throws IOException, DecryptionFailedException {
-        GenericPayload before = new GenericPayload(0, 1, security().randomBytes(100));
+        GenericPayload before = new GenericPayload(0, 1, cryptography().randomBytes(100));
 
         PrivateKey privateKey = new PrivateKey(false, 1, 1000, 1000);
         CryptoBox cryptoBox = new CryptoBox(before, privateKey.getPubkey().getEncryptionKey());
 
-        GenericPayload after = GenericPayload.read(0, cryptoBox.decrypt(privateKey.getPrivateEncryptionKey()), 1, 100);
+        GenericPayload after = GenericPayload.read(0, 1, cryptoBox.decrypt(privateKey.getPrivateEncryptionKey()), 100);
 
         assertEquals(before, after);
     }

core/src/test/java/ch/dissem/bitmessage/ProofOfWorkServiceTest.java

@@ -65,6 +65,7 @@ public class ProofOfWorkServiceTest {
         when(ctx.getInventory()).thenReturn(inventory);
         when(ctx.getNetworkHandler()).thenReturn(networkHandler);
         when(ctx.getMessageRepository()).thenReturn(messageRepo);
+        when(ctx.getLabeler()).thenReturn(mock(Labeler.class));
 
         proofOfWorkService = new ProofOfWorkService();
         proofOfWorkService.setContext(ctx);
@@ -76,9 +77,9 @@ public class ProofOfWorkServiceTest {
         when(proofOfWorkRepo.getItem(any(byte[].class))).thenReturn(new ProofOfWorkRepository.Item(null, 1001, 1002));
         doNothing().when(cryptography).doProofOfWork(any(ObjectMessage.class), anyLong(), anyLong(), any(ProofOfWorkEngine.Callback.class));
 
-        proofOfWorkService.doMissingProofOfWork();
+        proofOfWorkService.doMissingProofOfWork(10);
 
-        verify(cryptography).doProofOfWork((ObjectMessage) isNull(), eq(1001L), eq(1002L),
+        verify(cryptography, timeout(1000)).doProofOfWork((ObjectMessage) isNull(), eq(1001L), eq(1002L),
                 any(ProofOfWorkEngine.Callback.class));
     }
 

core/src/test/java/ch/dissem/bitmessage/entity/BitmessageAddressTest.java

@@ -20,20 +20,25 @@ import ch.dissem.bitmessage.entity.payload.Pubkey;
 import ch.dissem.bitmessage.entity.payload.V4Pubkey;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
 import ch.dissem.bitmessage.exception.DecryptionFailedException;
-import ch.dissem.bitmessage.utils.Base58;
+import ch.dissem.bitmessage.utils.*;
-import ch.dissem.bitmessage.utils.Bytes;
-import ch.dissem.bitmessage.utils.Strings;
-import ch.dissem.bitmessage.utils.TestUtils;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.Arrays;
 
 import static ch.dissem.bitmessage.entity.payload.Pubkey.Feature.DOES_ACK;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.entity.payload.Pubkey.Feature.INCLUDE_DESTINATION;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 import static org.junit.Assert.*;
 
-public class BitmessageAddressTest {
+public class BitmessageAddressTest extends TestBase {
+    @Test
+    public void ensureFeatureFlagIsCalculatedCorrectly() {
+        assertEquals(1, Pubkey.Feature.bitfield(DOES_ACK));
+        assertEquals(2, Pubkey.Feature.bitfield(INCLUDE_DESTINATION));
+        assertEquals(3, Pubkey.Feature.bitfield(DOES_ACK, INCLUDE_DESTINATION));
+    }
+
     @Test
     public void ensureBase58DecodesCorrectly() {
         assertHexEquals("800C28FCA386C7A227600B2FE50B7CAE11EC86D3BF1FBE471BE89827E19D72AA1D507A5B8D",
@@ -61,6 +66,7 @@ public class BitmessageAddressTest {
     public void ensureIdentityCanBeCreated() {
         BitmessageAddress address = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000, DOES_ACK));
         assertNotNull(address.getPubkey());
+        assertTrue(address.has(DOES_ACK));
     }
 
     @Test
@@ -90,6 +96,7 @@ public class BitmessageAddressTest {
         }
 
         assertArrayEquals(Bytes.fromHex("007402be6e76c3cb87caa946d0c003a3d4d8e1d5"), pubkey.getRipe());
+        assertTrue(address.has(DOES_ACK));
     }
 
     @Test
@@ -104,6 +111,7 @@ public class BitmessageAddressTest {
         } catch (Exception e) {
             fail(e.getMessage());
         }
+        assertTrue(address.has(DOES_ACK));
     }
 
     @Test
@@ -118,7 +126,7 @@ public class BitmessageAddressTest {
         System.out.println("\n\n" + Strings.hex(privsigningkey) + "\n\n");
 
         BitmessageAddress address = new BitmessageAddress(new PrivateKey(privsigningkey, privencryptionkey,
-                security().createPubkey(3, 1, privsigningkey, privencryptionkey, 320, 14000)));
+                cryptography().createPubkey(3, 1, privsigningkey, privencryptionkey, 320, 14000)));
         assertEquals(address_string, address.getAddress());
     }
 
@@ -128,7 +136,7 @@ public class BitmessageAddressTest {
         byte[] privsigningkey = getSecret("5KMWqfCyJZGFgW6QrnPJ6L9Gatz25B51y7ErgqNr1nXUVbtZbdU");
         byte[] privencryptionkey = getSecret("5JXXWEuhHQEPk414SzEZk1PHDRi8kCuZd895J7EnKeQSahJPxGz");
         BitmessageAddress address = new BitmessageAddress(new PrivateKey(privsigningkey, privencryptionkey,
-                security().createPubkey(4, 1, privsigningkey, privencryptionkey, 320, 14000)));
+                cryptography().createPubkey(4, 1, privsigningkey, privencryptionkey, 320, 14000)));
         assertEquals("BM-2cV5f9EpzaYARxtoruSpa6pDoucSf9ZNke", address.getAddress());
     }
 
@@ -143,7 +151,7 @@ public class BitmessageAddressTest {
         if (bytes.length != 37)
             throw new IOException("Unknown format: 37 bytes expected, but secret " + walletImportFormat + " was " + bytes.length + " long");
 
-        byte[] hash = security().doubleSha256(bytes, 33);
+        byte[] hash = cryptography().doubleSha256(bytes, 33);
         for (int i = 0; i < 4; i++) {
             if (hash[i] != bytes[33 + i]) throw new IOException("Hash check failed for secret " + walletImportFormat);
         }

core/src/test/java/ch/dissem/bitmessage/entity/SerializationTest.java

@@ -30,7 +30,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 
 import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 import static org.junit.Assert.*;
 
 public class SerializationTest extends TestBase {
@@ -82,7 +82,7 @@ public class SerializationTest extends TestBase {
                 .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
                 .to(TestUtils.loadContact())
                 .message("Subject", "Message")
-                .ack("ack".getBytes())
+                .ackData("ackMessage".getBytes())
                 .signature(new byte[0])
                 .build();
         ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -98,11 +98,37 @@ public class SerializationTest extends TestBase {
         assertEquals(p1, p2);
     }
 
+    @Test
+    public void ensurePlaintextWithAckMessageIsSerializedAndDeserializedCorrectly() throws Exception {
+        Plaintext p1 = new Plaintext.Builder(MSG)
+                .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+                .to(TestUtils.loadContact())
+                .message("Subject", "Message")
+                .ackData("ackMessage".getBytes())
+                .signature(new byte[0])
+                .build();
+        ObjectMessage ackMessage1 = p1.getAckMessage();
+        assertNotNull(ackMessage1);
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        p1.write(out);
+        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+        Plaintext p2 = Plaintext.read(MSG, in);
+
+        // Received is automatically set on deserialization, so we'll need to set it to 0
+        Field received = Plaintext.class.getDeclaredField("received");
+        received.setAccessible(true);
+        received.set(p2, 0L);
+
+        assertEquals(p1, p2);
+        assertEquals(ackMessage1, p2.getAckMessage());
+    }
+
     @Test
     public void ensureNetworkMessageIsSerializedAndDeserializedCorrectly() throws Exception {
         ArrayList<InventoryVector> ivs = new ArrayList<>(50000);
         for (int i = 0; i < 50000; i++) {
-            ivs.add(new InventoryVector(security().randomBytes(32)));
+            ivs.add(new InventoryVector(cryptography().randomBytes(32)));
         }
 
         Inv inv = new Inv.Builder().inventory(ivs).build();

core/src/test/java/ch/dissem/bitmessage/ports/NodeRegistryTest.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2016 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.ports;
+
+import ch.dissem.bitmessage.entity.valueobject.NetworkAddress;
+import ch.dissem.bitmessage.utils.UnixTime;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static ch.dissem.bitmessage.utils.UnixTime.HOUR;
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.assertThat;
+
+public class NodeRegistryTest {
+    private NodeRegistry registry = new MemoryNodeRegistry();
+
+    @Test
+    public void ensureGetKnownNodesWithoutStreamsYieldsEmpty() {
+        assertThat(registry.getKnownAddresses(10), empty());
+    }
+
+    /**
+     * Please note that this test fails if there is no internet connection,
+     * as the initial nodes' IP addresses are determined by DNS lookup.
+     */
+    @Test
+    public void ensureGetKnownNodesForStream1YieldsResult() {
+        assertThat(registry.getKnownAddresses(10, 1), hasSize(1));
+    }
+
+    @Test
+    public void ensureNodeIsStored() {
+        registry.offerAddresses(Arrays.asList(
+                new NetworkAddress.Builder()
+                        .ipv4(127, 0, 0, 1)
+                        .port(42)
+                        .stream(1)
+                        .time(UnixTime.now())
+                        .build(),
+                new NetworkAddress.Builder()
+                        .ipv4(127, 0, 0, 2)
+                        .port(42)
+                        .stream(1)
+                        .time(UnixTime.now())
+                        .build(),
+                new NetworkAddress.Builder()
+                        .ipv4(127, 0, 0, 2)
+                        .port(42)
+                        .stream(2)
+                        .time(UnixTime.now())
+                        .build()
+        ));
+        assertThat(registry.getKnownAddresses(10, 1).size(), is(2));
+        assertThat(registry.getKnownAddresses(10, 2).size(), is(1));
+        assertThat(registry.getKnownAddresses(10, 1, 2).size(), is(3));
+    }
+
+    @Test
+    public void ensureOldNodesAreRemoved() {
+        registry.offerAddresses(Arrays.asList(
+                new NetworkAddress.Builder()
+                        .ipv4(127, 0, 0, 1)
+                        .port(42)
+                        .stream(1)
+                        .time(UnixTime.now())
+                        .build(),
+                new NetworkAddress.Builder()
+                        .ipv4(127, 0, 0, 2)
+                        .port(42)
+                        .stream(1)
+                        .time(UnixTime.now(-4 * HOUR))
+                        .build(),
+                new NetworkAddress.Builder()
+                        .ipv4(127, 0, 0, 2)
+                        .port(42)
+                        .stream(2)
+                        .time(UnixTime.now())
+                        .build()
+        ));
+        assertThat(registry.getKnownAddresses(10, 1).size(), is(1));
+        assertThat(registry.getKnownAddresses(10, 2).size(), is(1));
+        assertThat(registry.getKnownAddresses(10, 1, 2).size(), is(2));
+    }
+}

core/src/test/java/ch/dissem/bitmessage/ports/ProofOfWorkEngineTest.java

@@ -21,7 +21,7 @@ import ch.dissem.bitmessage.utils.CallbackWaiter;
 import ch.dissem.bitmessage.utils.TestBase;
 import org.junit.Test;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 import static org.junit.Assert.assertTrue;
 
 public class ProofOfWorkEngineTest extends TestBase {
@@ -36,7 +36,7 @@ public class ProofOfWorkEngineTest extends TestBase {
     }
 
     private void testPOW(ProofOfWorkEngine engine) throws InterruptedException {
-        byte[] initialHash = security().sha512(new byte[]{1, 3, 6, 4});
+        byte[] initialHash = cryptography().sha512(new byte[]{1, 3, 6, 4});
         byte[] target = {0, 0, 0, -1, -1, -1, -1, -1};
 
         final CallbackWaiter<byte[]> waiter1 = new CallbackWaiter<>();
@@ -49,10 +49,10 @@ public class ProofOfWorkEngineTest extends TestBase {
                 });
         byte[] nonce = waiter1.waitForValue();
         System.out.println("Calculating nonce took " + waiter1.getTime() + "ms");
-        assertTrue(Bytes.lt(security().doubleSha512(nonce, initialHash), target, 8));
+        assertTrue(Bytes.lt(cryptography().doubleSha512(nonce, initialHash), target, 8));
 
         // Let's add a second (shorter) run to find possible multi threading issues
-        byte[] initialHash2 = security().sha512(new byte[]{1, 3, 6, 5});
+        byte[] initialHash2 = cryptography().sha512(new byte[]{1, 3, 6, 5});
         byte[] target2 = {0, 0, -1, -1, -1, -1, -1, -1};
 
         final CallbackWaiter<byte[]> waiter2 = new CallbackWaiter<>();
@@ -65,7 +65,7 @@ public class ProofOfWorkEngineTest extends TestBase {
                 });
         byte[] nonce2 = waiter2.waitForValue();
         System.out.println("Calculating nonce took " + waiter2.getTime() + "ms");
-        assertTrue(Bytes.lt(security().doubleSha512(nonce2, initialHash2), target2, 8));
+        assertTrue(Bytes.lt(cryptography().doubleSha512(nonce2, initialHash2), target2, 8));
         assertTrue("Second nonce must be quicker to find", waiter1.getTime() > waiter2.getTime());
     }
 

core/src/test/java/ch/dissem/bitmessage/utils/SqlStringsTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Christian Basler
+ * Copyright 2016 Christian Basler
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 
-package ch.dissem.bitmessage.repository;
+package ch.dissem.bitmessage.utils;
 
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 
-public class JdbcHelperTest {
+public class SqlStringsTest {
     @Test
     public void ensureJoinWorksWithLongArray() {
         long[] test = {1L, 2L};
-        assertEquals("1, 2", JdbcHelper.join(test).toString());
+        assertEquals("1, 2", SqlStrings.join(test).toString());
     }
 }

core/src/test/java/ch/dissem/bitmessage/utils/TestUtils.java

@@ -72,7 +72,7 @@ public class TestUtils {
 
     public static BitmessageAddress loadContact() throws IOException, DecryptionFailedException {
         BitmessageAddress address = new BitmessageAddress("BM-2cXxfcSetKnbHJX2Y85rSkaVpsdNUZ5q9h");
-        ObjectMessage object = TestUtils.loadObjectMessage(4, "V4Pubkey.payload");
+        ObjectMessage object = TestUtils.loadObjectMessage(3, "V4Pubkey.payload");
         object.decrypt(address.getPublicDecryptionKey());
         address.setPubkey((V4Pubkey) object.getPayload());
         return address;

cryptography-bc/src/test/java/ch/dissem/bitmessage/security/CryptographyTest.java

@@ -21,9 +21,7 @@ import java.io.IOException;
 import static ch.dissem.bitmessage.utils.UnixTime.DAY;
 import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.*;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -82,7 +80,7 @@ public class CryptographyTest {
                 .nonce(new byte[8])
                 .expiresTime(UnixTime.now(+28 * DAY))
                 .objectType(0)
-                .payload(GenericPayload.read(0, new ByteArrayInputStream(new byte[0]), 1, 0))
+                .payload(GenericPayload.read(0, 1, new ByteArrayInputStream(new byte[0]), 0))
                 .build();
         crypto.checkProofOfWork(objectMessage, 1000, 1000);
     }
@@ -93,7 +91,7 @@ public class CryptographyTest {
                 .nonce(new byte[8])
                 .expiresTime(UnixTime.now(+2 * MINUTE))
                 .objectType(0)
-                .payload(GenericPayload.read(0, new ByteArrayInputStream(new byte[0]), 1, 0))
+                .payload(GenericPayload.read(0, 1, new ByteArrayInputStream(new byte[0]), 0))
                 .build();
         final CallbackWaiter<byte[]> waiter = new CallbackWaiter<>();
         crypto.doProofOfWork(objectMessage, 1000, 1000,

demo/build.gradle

@@ -31,6 +31,7 @@ dependencies {
     compile 'org.slf4j:slf4j-simple:1.7.12'
     compile 'args4j:args4j:2.32'
     compile 'com.h2database:h2:1.4.190'
+    compile 'org.apache.commons:commons-lang3:3.4'
     testCompile 'junit:junit:4.11'
     testCompile 'org.mockito:mockito-core:1.10.19'
 }

demo/src/main/java/ch/dissem/bitmessage/demo/Application.java

@@ -25,10 +25,10 @@ import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.networking.DefaultNetworkHandler;
 import ch.dissem.bitmessage.ports.MemoryNodeRegistry;
 import ch.dissem.bitmessage.repository.*;
+import org.apache.commons.lang3.text.WordUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -56,13 +56,7 @@ public class Application {
                 .networkHandler(new DefaultNetworkHandler())
                 .cryptography(new BouncyCryptography())
                 .port(48444)
-                .listener(plaintext -> {
+                .listener(plaintext -> System.out.println("New Message from " + plaintext.getFrom() + ": " + plaintext.getSubject()))
-                    try {
-                        System.out.println(new String(plaintext.getMessage(), "UTF-8"));
-                    } catch (UnsupportedEncodingException e) {
-                        LOG.error(e.getMessage(), e);
-                    }
-                })
                 .build();
 
         if (syncServer == null) {
@@ -99,7 +93,7 @@ public class Application {
                         subscriptions();
                         break;
                     case "m":
-                        messages();
+                        labels();
                         break;
                     case "?":
                         info();
@@ -124,8 +118,27 @@ public class Application {
     }
 
     private void info() {
+        String command;
+        do {
             System.out.println();
             System.out.println(ctx.status());
+            System.out.println();
+            System.out.println("c) cleanup inventory");
+            System.out.println("r) resend unacknowledged messages");
+            System.out.println(COMMAND_BACK);
+
+            command = commandLine.nextCommand();
+            switch (command) {
+                case "c":
+                    ctx.cleanup();
+                    break;
+                case "r":
+                    ctx.resendUnacknowledgedMessages();
+                    break;
+                case "b":
+                    return;
+            }
+        } while (!"b".equals(command));
     }
 
     private void identities() {
@@ -280,22 +293,71 @@ public class Application {
         }
     }
 
-    private void messages() {
+    private void labels() {
+        List<Label> labels = ctx.messages().getLabels();
         String command;
         do {
-            List<Plaintext> messages = ctx.messages().findMessages(Plaintext.Status.RECEIVED);
+            System.out.println();
+            int i = 0;
+            for (Label label : labels) {
+                i++;
+                System.out.print(i + ") " + label);
+                int unread = ctx.messages().countUnread(label);
+                if (unread > 0) {
+                    System.out.println(" [" + unread + "]");
+                } else {
+                    System.out.println();
+                }
+            }
+            System.out.println("a) Archive");
+            System.out.println();
+            System.out.println("c) compose message");
+            System.out.println("s) compose broadcast");
+            System.out.println(COMMAND_BACK);
+
+            command = commandLine.nextCommand();
+            switch (command) {
+                case "a":
+                    messages(null);
+                    break;
+                case "c":
+                    compose(false);
+                    break;
+                case "s":
+                    compose(true);
+                    break;
+                case "b":
+                    return;
+                default:
+                    try {
+                        int index = Integer.parseInt(command) - 1;
+                        messages(labels.get(index));
+                    } catch (NumberFormatException | IndexOutOfBoundsException e) {
+                        System.out.println(ERROR_UNKNOWN_COMMAND);
+                    }
+            }
+        } while (!"b".equalsIgnoreCase(command));
+    }
+
+    private void messages(Label label) {
+        String command;
+        do {
+            List<Plaintext> messages = ctx.messages().findMessages(label);
             System.out.println();
             int i = 0;
             for (Plaintext message : messages) {
                 i++;
-                System.out.println(i + ") From: " + message.getFrom() + "; Subject: " + message.getSubject());
+                System.out.println(i + (message.isUnread() ? ">" : ")") + " From: " + message.getFrom() + "; Subject: " + message.getSubject());
             }
             if (i == 0) {
-                System.out.println("You have no messages.");
+                System.out.println("There are no messages.");
             }
             System.out.println();
             System.out.println("c) compose message");
             System.out.println("s) compose broadcast");
+            if (label.getType() == Label.Type.TRASH) {
+                System.out.println("e) empty trash");
+            }
             System.out.println(COMMAND_BACK);
 
             command = commandLine.nextCommand();
@@ -306,6 +368,8 @@ public class Application {
                 case "s":
                     compose(true);
                     break;
+                case "e":
+                    messages.forEach(ctx.messages()::remove);
                 case "b":
                     return;
                 default:
@@ -325,16 +389,18 @@ public class Application {
         System.out.println("To:      " + message.getTo());
         System.out.println("Subject: " + message.getSubject());
         System.out.println();
-        System.out.println(message.getText());
+        System.out.println(WordUtils.wrap(message.getText(), 120));
         System.out.println();
         System.out.println(message.getLabels().stream().map(Label::toString).collect(
-                Collectors.joining("Labels: ", ", ", "")));
+                Collectors.joining(", ", "Labels: ", "")));
         System.out.println();
         ctx.labeler().markAsRead(message);
+        ctx.messages().save(message);
         String command;
         do {
             System.out.println("r) reply");
             System.out.println("d) delete");
+            System.out.println("a) archive");
             System.out.println(COMMAND_BACK);
             command = commandLine.nextCommand();
             switch (command) {
@@ -342,7 +408,13 @@ public class Application {
                     compose(message.getTo(), message.getFrom(), "RE: " + message.getSubject());
                     break;
                 case "d":
-                    ctx.messages().remove(message);
+                    ctx.labeler().delete(message);
+                    ctx.messages().save(message);
+                    return;
+                case "a":
+                    ctx.labeler().archive(message);
+                    ctx.messages().save(message);
+                    return;
                 case "b":
                     return;
                 default:

demo/src/test/java/ch/dissem/bitmessage/SystemTest.java

@@ -4,16 +4,25 @@ import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography;
 import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.networking.DefaultNetworkHandler;
+import ch.dissem.bitmessage.ports.DefaultLabeler;
+import ch.dissem.bitmessage.ports.Labeler;
 import ch.dissem.bitmessage.repository.*;
 import ch.dissem.bitmessage.utils.TTL;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
+import static ch.dissem.bitmessage.entity.payload.Pubkey.Feature.DOES_ACK;
 import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
 
 /**
  * @author Christian Basler
@@ -23,6 +32,7 @@ public class SystemTest {
 
     private BitmessageContext alice;
     private TestListener aliceListener = new TestListener();
+    private Labeler aliceLabeler = Mockito.spy(new DebugLabeler("Alice"));
     private BitmessageAddress aliceIdentity;
 
     private BitmessageContext bob;
@@ -47,9 +57,10 @@ public class SystemTest {
                 .networkHandler(new DefaultNetworkHandler())
                 .cryptography(new BouncyCryptography())
                 .listener(aliceListener)
+                .labeler(aliceLabeler)
                 .build();
         alice.startup();
-        aliceIdentity = alice.createIdentity(false);
+        aliceIdentity = alice.createIdentity(false, DOES_ACK);
 
         JdbcConfig bobDB = new JdbcConfig("jdbc:h2:mem:bob;DB_CLOSE_DELAY=-1", "sa", "");
         bob = new BitmessageContext.Builder()
@@ -62,9 +73,13 @@ public class SystemTest {
                 .networkHandler(new DefaultNetworkHandler())
                 .cryptography(new BouncyCryptography())
                 .listener(bobListener)
+                .labeler(new DebugLabeler("Bob"))
                 .build();
         bob.startup();
-        bobIdentity = bob.createIdentity(false);
+        bobIdentity = bob.createIdentity(false, DOES_ACK);
+
+        ((DebugLabeler) alice.labeler()).init(aliceIdentity, bobIdentity);
+        ((DebugLabeler) bob.labeler()).init(aliceIdentity, bobIdentity);
     }
 
     @After
@@ -82,6 +97,9 @@ public class SystemTest {
 
         assertThat(plaintext.getType(), equalTo(Plaintext.Type.MSG));
         assertThat(plaintext.getText(), equalTo(originalMessage));
+
+        Mockito.verify(aliceLabeler, Mockito.timeout(TimeUnit.MINUTES.toMillis(15)).atLeastOnce())
+                .markAsAcknowledged(any());
     }
 
     @Test
@@ -95,4 +113,83 @@ public class SystemTest {
         assertThat(plaintext.getType(), equalTo(Plaintext.Type.BROADCAST));
         assertThat(plaintext.getText(), equalTo(originalMessage));
     }
+
+    private static class DebugLabeler extends DefaultLabeler {
+        private final Logger LOG = LoggerFactory.getLogger("Labeler");
+        final String name;
+        String alice;
+        String bob;
+
+        private DebugLabeler(String name) {
+            this.name = name;
+        }
+
+        private void init(BitmessageAddress alice, BitmessageAddress bob) {
+            this.alice = alice.getAddress();
+            this.bob = bob.getAddress();
+        }
+
+        @Override
+        public void setLabels(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Received");
+            super.setLabels(msg);
+        }
+
+        @Override
+        public void markAsDraft(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Draft");
+            super.markAsDraft(msg);
+        }
+
+        @Override
+        public void markAsSending(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Sending");
+            super.markAsSending(msg);
+        }
+
+        @Override
+        public void markAsSent(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Sent");
+            super.markAsSent(msg);
+        }
+
+        @Override
+        public void markAsAcknowledged(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Acknowledged");
+            super.markAsAcknowledged(msg);
+        }
+
+        @Override
+        public void markAsRead(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Read");
+            super.markAsRead(msg);
+        }
+
+        @Override
+        public void markAsUnread(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Unread");
+            super.markAsUnread(msg);
+        }
+
+        @Override
+        public void delete(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Cleared");
+            super.delete(msg);
+        }
+
+        @Override
+        public void archive(Plaintext msg) {
+            LOG.info(name + ": From " + name(msg.getFrom()) + ": Archived");
+            super.archive(msg);
+        }
+
+        private String name(BitmessageAddress address) {
+            if (alice.equals(address.getAddress()))
+                return "Alice";
+            else if (bob.equals(address.getAddress()))
+                return "Bob";
+            else
+                return "Unknown (" + address.getAddress() + ")";
+        }
+    }
 }

extensions/src/main/java/ch/dissem/bitmessage/extensions/CryptoCustomMessage.java

@@ -28,7 +28,7 @@ import ch.dissem.bitmessage.utils.Encode;
 import java.io.*;
 
 import static ch.dissem.bitmessage.utils.Decode.*;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * A {@link CustomMessage} implementation that contains signed and encrypted data.
@@ -80,7 +80,7 @@ public class CryptoCustomMessage<T extends Streamable> extends CustomMessage {
         }
 
         data.write(out);
-        Encode.varBytes(security().getSignature(out.toByteArray(), identity.getPrivateKey()), out);
+        Encode.varBytes(cryptography().getSignature(out.toByteArray(), identity.getPrivateKey()), out);
         container = new CryptoBox(out.toByteArray(), publicKey);
     }
 
@@ -138,7 +138,7 @@ public class CryptoCustomMessage<T extends Streamable> extends CustomMessage {
         }
 
         public void checkSignature(Pubkey pubkey) throws IOException, IllegalStateException {
-            if (!security().isSignatureValid(out.toByteArray(), varBytes(wrapped), pubkey)) {
+            if (!cryptography().isSignatureValid(out.toByteArray(), varBytes(wrapped), pubkey)) {
                 throw new IllegalStateException("Signature check failed");
             }
         }

extensions/src/test/java/ch/dissem/bitmessage/extensions/CryptoCustomMessageTest.java

@@ -30,7 +30,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 import static org.junit.Assert.assertEquals;
 
 public class CryptoCustomMessageTest extends TestBase {
@@ -39,9 +39,9 @@ public class CryptoCustomMessageTest extends TestBase {
         PrivateKey privateKey = PrivateKey.read(TestUtils.getResource("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8.privkey"));
         BitmessageAddress sendingIdentity = new BitmessageAddress(privateKey);
 
-        GenericPayload payloadBefore = new GenericPayload(0, 1, security().randomBytes(100));
+        GenericPayload payloadBefore = new GenericPayload(0, 1, cryptography().randomBytes(100));
         CryptoCustomMessage<GenericPayload> messageBefore = new CryptoCustomMessage<>(payloadBefore);
-        messageBefore.signAndEncrypt(sendingIdentity, security().createPublicKey(sendingIdentity.getPublicDecryptionKey()));
+        messageBefore.signAndEncrypt(sendingIdentity, cryptography().createPublicKey(sendingIdentity.getPublicDecryptionKey()));
 
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         messageBefore.write(out);
@@ -52,7 +52,7 @@ public class CryptoCustomMessageTest extends TestBase {
                 new CryptoCustomMessage.Reader<GenericPayload>() {
                     @Override
                     public GenericPayload read(BitmessageAddress ignore, InputStream in) throws IOException {
-                        return GenericPayload.read(0, in, 1, 100);
+                        return GenericPayload.read(0, 1, in, 100);
                     }
                 });
         GenericPayload payloadAfter = messageAfter.decrypt(sendingIdentity.getPublicDecryptionKey());
@@ -65,11 +65,11 @@ public class CryptoCustomMessageTest extends TestBase {
         PrivateKey privateKey = PrivateKey.read(TestUtils.getResource("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8.privkey"));
         final BitmessageAddress sendingIdentity = new BitmessageAddress(privateKey);
 
-        ProofOfWorkRequest requestBefore = new ProofOfWorkRequest(sendingIdentity, security().randomBytes(64),
+        ProofOfWorkRequest requestBefore = new ProofOfWorkRequest(sendingIdentity, cryptography().randomBytes(64),
                 ProofOfWorkRequest.Request.CALCULATE);
 
         CryptoCustomMessage<ProofOfWorkRequest> messageBefore = new CryptoCustomMessage<>(requestBefore);
-        messageBefore.signAndEncrypt(sendingIdentity, security().createPublicKey(sendingIdentity.getPublicDecryptionKey()));
+        messageBefore.signAndEncrypt(sendingIdentity, cryptography().createPublicKey(sendingIdentity.getPublicDecryptionKey()));
 
 
         ByteArrayOutputStream out = new ByteArrayOutputStream();

networking/src/main/java/ch/dissem/bitmessage/networking/Connection.java

@@ -46,7 +46,7 @@ import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE
 import static ch.dissem.bitmessage.networking.Connection.Mode.CLIENT;
 import static ch.dissem.bitmessage.networking.Connection.Mode.SYNC;
 import static ch.dissem.bitmessage.networking.Connection.State.*;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 
 /**
@@ -72,6 +72,7 @@ class Connection {
     private final ReaderRunnable reader = new ReaderRunnable();
     private final WriterRunnable writer = new WriterRunnable();
     private final DefaultNetworkHandler networkHandler;
+    private final long clientNonce;
 
     private volatile State state;
     private InputStream in;
@@ -83,22 +84,23 @@ class Connection {
     private long lastObjectTime;
 
     public Connection(InternalContext context, Mode mode, Socket socket, MessageListener listener,
-                      Set<InventoryVector> requestedObjectsMap) throws IOException {
+                      Set<InventoryVector> requestedObjectsMap, long clientNonce) throws IOException {
         this(context, mode, listener, socket, requestedObjectsMap,
                 Collections.newSetFromMap(new ConcurrentHashMap<InventoryVector, Boolean>(10_000)),
                 new NetworkAddress.Builder().ip(socket.getInetAddress()).port(socket.getPort()).stream(1).build(),
-                0);
+                0, clientNonce);
     }
 
     public Connection(InternalContext context, Mode mode, NetworkAddress node, MessageListener listener,
-                      Set<InventoryVector> requestedObjectsMap) {
+                      Set<InventoryVector> requestedObjectsMap, long clientNonce) {
         this(context, mode, listener, new Socket(), requestedObjectsMap,
                 Collections.newSetFromMap(new ConcurrentHashMap<InventoryVector, Boolean>(10_000)),
-                node, 0);
+                node, 0, clientNonce);
     }
 
     private Connection(InternalContext context, Mode mode, MessageListener listener, Socket socket,
-                       Set<InventoryVector> commonRequestedObjects, Set<InventoryVector> requestedObjects, NetworkAddress node, long syncTimeout) {
+                       Set<InventoryVector> commonRequestedObjects, Set<InventoryVector> requestedObjects,
+                       NetworkAddress node, long syncTimeout, long clientNonce) {
         this.startTime = UnixTime.now();
         this.ctx = context;
         this.mode = mode;
@@ -112,6 +114,7 @@ class Connection {
         this.syncTimeout = (syncTimeout > 0 ? UnixTime.now(+syncTimeout) : 0);
         this.ivCache = new ConcurrentHashMap<>();
         this.networkHandler = (DefaultNetworkHandler) ctx.getNetworkHandler();
+        this.clientNonce = clientNonce;
     }
 
     public static Connection sync(InternalContext ctx, InetAddress address, int port, MessageListener listener,
@@ -120,7 +123,7 @@ class Connection {
                 new HashSet<InventoryVector>(),
                 new HashSet<InventoryVector>(),
                 new NetworkAddress.Builder().ip(address).port(port).stream(1).build(),
-                timeoutInSeconds);
+                timeoutInSeconds, cryptography().randomNonce());
     }
 
     public long getStartTime() {
@@ -249,7 +252,7 @@ class Connection {
         }
         try {
             listener.receive(objectMessage);
-            security().checkProofOfWork(objectMessage, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES);
+            cryptography().checkProofOfWork(objectMessage, NETWORK_NONCE_TRIALS_PER_BYTE, NETWORK_EXTRA_BYTES);
             ctx.getInventory().storeObject(objectMessage);
             // offer object to some random nodes so it gets distributed throughout the network:
             networkHandler.offer(objectMessage.getInventoryVector());
@@ -362,7 +365,7 @@ class Connection {
             try (Socket socket = Connection.this.socket) {
                 initSocket(socket);
                 if (mode == CLIENT || mode == SYNC) {
-                    send(new Version.Builder().defaults().addrFrom(host).addrRecv(node).build());
+                    send(new Version.Builder().defaults(clientNonce).addrFrom(host).addrRecv(node).build());
                 }
                 while (state != DISCONNECTED) {
                     if (mode != SYNC) {
@@ -446,7 +449,7 @@ class Connection {
                 send(new VerAck());
                 switch (mode) {
                     case SERVER:
-                        send(new Version.Builder().defaults().addrFrom(host).addrRecv(node).build());
+                        send(new Version.Builder().defaults(clientNonce).addrFrom(host).addrRecv(node).build());
                         break;
                     case CLIENT:
                     case SYNC:

networking/src/main/java/ch/dissem/bitmessage/networking/ConnectionOrganizer.java

@@ -38,15 +38,17 @@ public class ConnectionOrganizer implements Runnable {
     private final InternalContext ctx;
     private final DefaultNetworkHandler networkHandler;
     private final NetworkHandler.MessageListener listener;
+    private final long clientNonce;
 
     private Connection initialConnection;
 
     public ConnectionOrganizer(InternalContext ctx,
                                DefaultNetworkHandler networkHandler,
-                               NetworkHandler.MessageListener listener) {
+                               NetworkHandler.MessageListener listener, long clientNonce) {
         this.ctx = ctx;
         this.networkHandler = networkHandler;
         this.listener = listener;
+        this.clientNonce = clientNonce;
     }
 
     @Override
@@ -91,7 +93,8 @@ public class ConnectionOrganizer implements Runnable {
                                 NETWORK_MAGIC_NUMBER - active, ctx.getStreams());
                         boolean first = active == 0 && initialConnection == null;
                         for (NetworkAddress address : addresses) {
-                            Connection c = new Connection(ctx, CLIENT, address, listener, networkHandler.requestedObjects);
+                            Connection c = new Connection(ctx, CLIENT, address, listener,
+                                    networkHandler.requestedObjects, clientNonce);
                             if (first) {
                                 initialConnection = c;
                                 first = false;

networking/src/main/java/ch/dissem/bitmessage/networking/DefaultNetworkHandler.java

@@ -28,7 +28,6 @@ import ch.dissem.bitmessage.factory.Factory;
 import ch.dissem.bitmessage.ports.NetworkHandler;
 import ch.dissem.bitmessage.utils.Collections;
 import ch.dissem.bitmessage.utils.Property;
-import ch.dissem.bitmessage.utils.ThreadFactoryBuilder;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -109,9 +108,9 @@ public class DefaultNetworkHandler implements NetworkHandler, ContextHolder {
         try {
             running = true;
             connections.clear();
-            server = new ServerRunnable(ctx, this, listener);
+            server = new ServerRunnable(ctx, this, listener, ctx.getClientNonce());
             pool.execute(server);
-            pool.execute(new ConnectionOrganizer(ctx, this, listener));
+            pool.execute(new ConnectionOrganizer(ctx, this, listener, ctx.getClientNonce()));
         } catch (IOException e) {
             throw new ApplicationException(e);
         }

networking/src/main/java/ch/dissem/bitmessage/networking/ServerRunnable.java

@@ -37,12 +37,15 @@ public class ServerRunnable implements Runnable, Closeable {
     private final ServerSocket serverSocket;
     private final DefaultNetworkHandler networkHandler;
     private final NetworkHandler.MessageListener listener;
+    private final long clientNonce;
 
-    public ServerRunnable(InternalContext ctx, DefaultNetworkHandler networkHandler, NetworkHandler.MessageListener listener) throws IOException {
+    public ServerRunnable(InternalContext ctx, DefaultNetworkHandler networkHandler,
+                          NetworkHandler.MessageListener listener, long clientNonce) throws IOException {
         this.ctx = ctx;
         this.networkHandler = networkHandler;
         this.listener = listener;
         this.serverSocket = new ServerSocket(ctx.getPort());
+        this.clientNonce = clientNonce;
     }
 
     @Override
@@ -52,7 +55,7 @@ public class ServerRunnable implements Runnable, Closeable {
                 Socket socket = serverSocket.accept();
                 socket.setSoTimeout(Connection.READ_TIMEOUT);
                 networkHandler.startConnection(new Connection(ctx, SERVER, socket, listener,
-                        networkHandler.requestedObjects));
+                        networkHandler.requestedObjects, clientNonce));
             } catch (IOException e) {
                 LOG.debug(e.getMessage(), e);
             }

repositories/build.gradle

@@ -18,5 +18,6 @@ dependencies {
     testCompile 'junit:junit:4.12'
     testCompile 'com.h2database:h2:1.4.190'
     testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile project(path: ':core', configuration: 'testArtifacts')
     testCompile project(':cryptography-bc')
 }

repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcHelper.java

@@ -18,6 +18,7 @@ package ch.dissem.bitmessage.repository;
 
 import ch.dissem.bitmessage.entity.Streamable;
 import ch.dissem.bitmessage.entity.payload.ObjectType;
+import ch.dissem.bitmessage.exception.ApplicationException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -25,6 +26,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.Collection;
 
 import static ch.dissem.bitmessage.utils.Strings.hex;
 
@@ -40,43 +42,7 @@ public abstract class JdbcHelper {
         this.config = config;
     }
 
-    public static StringBuilder join(long... objects) {
+    public static void writeBlob(PreparedStatement ps, int parameterIndex, Streamable data) throws SQLException, IOException {
-        StringBuilder streamList = new StringBuilder();
-        for (int i = 0; i < objects.length; i++) {
-            if (i > 0) streamList.append(", ");
-            streamList.append(objects[i]);
-        }
-        return streamList;
-    }
-
-    public static StringBuilder join(byte[]... objects) {
-        StringBuilder streamList = new StringBuilder();
-        for (int i = 0; i < objects.length; i++) {
-            if (i > 0) streamList.append(", ");
-            streamList.append(hex(objects[i]));
-        }
-        return streamList;
-    }
-
-    public static StringBuilder join(ObjectType... types) {
-        StringBuilder streamList = new StringBuilder();
-        for (int i = 0; i < types.length; i++) {
-            if (i > 0) streamList.append(", ");
-            streamList.append(types[i].getNumber());
-        }
-        return streamList;
-    }
-
-    public static StringBuilder join(Enum... types) {
-        StringBuilder streamList = new StringBuilder();
-        for (int i = 0; i < types.length; i++) {
-            if (i > 0) streamList.append(", ");
-            streamList.append('\'').append(types[i].name()).append('\'');
-        }
-        return streamList;
-    }
-
-    protected void writeBlob(PreparedStatement ps, int parameterIndex, Streamable data) throws SQLException, IOException {
         if (data == null) {
             ps.setBytes(parameterIndex, null);
         } else {

repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcInventory.java

@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import static ch.dissem.bitmessage.utils.SqlStrings.join;
 import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 import static ch.dissem.bitmessage.utils.UnixTime.now;
 

repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java

@@ -16,14 +16,12 @@
 
 package ch.dissem.bitmessage.repository;
 
-import ch.dissem.bitmessage.InternalContext;
-import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.exception.ApplicationException;
+import ch.dissem.bitmessage.ports.AbstractMessageRepository;
 import ch.dissem.bitmessage.ports.MessageRepository;
-import ch.dissem.bitmessage.utils.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,34 +29,30 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.sql.*;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 
-public class JdbcMessageRepository extends JdbcHelper implements MessageRepository, InternalContext.ContextHolder {
+import static ch.dissem.bitmessage.repository.JdbcHelper.writeBlob;
+
+public class JdbcMessageRepository extends AbstractMessageRepository implements MessageRepository {
     private static final Logger LOG = LoggerFactory.getLogger(JdbcMessageRepository.class);
 
-    private InternalContext ctx;
+    private final JdbcConfig config;
 
     public JdbcMessageRepository(JdbcConfig config) {
-        super(config);
+        this.config = config;
     }
 
     @Override
-    public List<Label> getLabels() {
+    protected List<Label> findLabels(String where) {
-        List<Label> result = new LinkedList<>();
         try (
-                Connection connection = config.getConnection();
+                Connection connection = config.getConnection()
-                Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery("SELECT id, label, type, color FROM Label ORDER BY ord")
         ) {
-            while (rs.next()) {
+            return findLabels(connection, where);
-                result.add(getLabel(rs));
-            }
         } catch (SQLException e) {
-            throw new ApplicationException(e);
+            LOG.error(e.getMessage(), e);
         }
-        return result;
+        return new ArrayList<>();
     }
 
     private Label getLabel(ResultSet rs) throws SQLException {
@@ -73,24 +67,6 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
         return label;
     }
 
-    @Override
-    public List<Label> getLabels(Label.Type... types) {
-        List<Label> result = new LinkedList<>();
-        try (
-                Connection connection = config.getConnection();
-                Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery("SELECT id, label, type, color FROM Label WHERE type IN (" + join(types) +
-                        ") ORDER BY ord")
-        ) {
-            while (rs.next()) {
-                result.add(getLabel(rs));
-            }
-        } catch (SQLException e) {
-            LOG.error(e.getMessage(), e);
-        }
-        return result;
-    }
-
     @Override
     public int countUnread(Label label) {
         String where;
@@ -105,7 +81,8 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
         try (
                 Connection connection = config.getConnection();
                 Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery("SELECT count(*) FROM Message WHERE " + where)
+                ResultSet rs = stmt.executeQuery("SELECT count(*) FROM Message WHERE " + where
+                        + " ORDER BY received DESC")
         ) {
             if (rs.next()) {
                 return rs.getInt(1);
@@ -117,45 +94,13 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
     }
 
     @Override
-    public Plaintext getMessage(byte[] initialHash) {
+    protected List<Plaintext> find(String where) {
-        List<Plaintext> plaintexts = find("initial_hash=X'" + Strings.hex(initialHash) + "'");
-        switch (plaintexts.size()) {
-            case 0:
-                return null;
-            case 1:
-                return plaintexts.get(0);
-            default:
-                throw new ApplicationException("This shouldn't happen, found " + plaintexts.size() +
-                        " messages, one or none was expected");
-        }
-    }
-
-    @Override
-    public List<Plaintext> findMessages(Label label) {
-        return find("id IN (SELECT message_id FROM Message_Label WHERE label_id=" + label.getId() + ")");
-    }
-
-    @Override
-    public List<Plaintext> findMessages(Plaintext.Status status, BitmessageAddress recipient) {
-        return find("status='" + status.name() + "' AND recipient='" + recipient.getAddress() + "'");
-    }
-
-    @Override
-    public List<Plaintext> findMessages(Plaintext.Status status) {
-        return find("status='" + status.name() + "'");
-    }
-
-    @Override
-    public List<Plaintext> findMessages(BitmessageAddress sender) {
-        return find("sender='" + sender.getAddress() + "'");
-    }
-
-    private List<Plaintext> find(String where) {
         List<Plaintext> result = new LinkedList<>();
         try (
                 Connection connection = config.getConnection();
                 Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery("SELECT id, iv, type, sender, recipient, data, sent, received, status " +
+                ResultSet rs = stmt.executeQuery(
+                        "SELECT id, iv, type, sender, recipient, data, ack_data, sent, received, initial_hash, status, ttl, retries, next_try " +
                                 "FROM Message WHERE " + where)
         ) {
             while (rs.next()) {
@@ -168,11 +113,18 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
                 builder.IV(new InventoryVector(iv));
                 builder.from(ctx.getAddressRepository().getAddress(rs.getString("sender")));
                 builder.to(ctx.getAddressRepository().getAddress(rs.getString("recipient")));
+                builder.ackData(rs.getBytes("ack_data"));
                 builder.sent(rs.getLong("sent"));
                 builder.received(rs.getLong("received"));
                 builder.status(Plaintext.Status.valueOf(rs.getString("status")));
-                builder.labels(findLabels(connection, id));
+                builder.ttl(rs.getLong("ttl"));
-                result.add(builder.build());
+                builder.retries(rs.getInt("retries"));
+                builder.nextTry(rs.getLong("next_try"));
+                builder.labels(findLabels(connection,
+                        "WHERE id IN (SELECT label_id FROM Message_Label WHERE message_id=" + id + ") ORDER BY ord"));
+                Plaintext message = builder.build();
+                message.setInitialHash(rs.getBytes("initial_hash"));
+                result.add(message);
             }
         } catch (IOException | SQLException e) {
             LOG.error(e.getMessage(), e);
@@ -180,12 +132,11 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
         return result;
     }
 
-    private Collection<Label> findLabels(Connection connection, long messageId) {
+    private List<Label> findLabels(Connection connection, String where) {
         List<Label> result = new ArrayList<>();
         try (
                 Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery("SELECT id, label, type, color FROM Label " +
+                ResultSet rs = stmt.executeQuery("SELECT id, label, type, color FROM Label WHERE " + where)
-                        "WHERE id IN (SELECT label_id FROM Message_Label WHERE message_id=" + messageId + ")")
         ) {
             while (rs.next()) {
                 result.add(getLabel(rs));
@@ -198,16 +149,7 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
 
     @Override
     public void save(Plaintext message) {
-        // save from address if necessary
+        safeSenderIfNecessary(message);
-        if (message.getId() == null) {
-            BitmessageAddress savedAddress = ctx.getAddressRepository().getAddress(message.getFrom().getAddress());
-            if (savedAddress == null) {
-                ctx.getAddressRepository().save(message.getFrom());
-            } else if (savedAddress.getPubkey() == null && message.getFrom().getPubkey() != null) {
-                savedAddress.setPubkey(message.getFrom().getPubkey());
-                ctx.getAddressRepository().save(savedAddress);
-            }
-        }
 
         try (Connection connection = config.getConnection()) {
             try {
@@ -249,8 +191,9 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
 
     private void insert(Connection connection, Plaintext message) throws SQLException, IOException {
         try (PreparedStatement ps = connection.prepareStatement(
-                "INSERT INTO Message (iv, type, sender, recipient, data, sent, received, status, initial_hash) " +
+                "INSERT INTO Message (iv, type, sender, recipient, data, ack_data, sent, received, " +
-                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                        "status, initial_hash, ttl, retries, next_try) " +
+                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                 Statement.RETURN_GENERATED_KEYS)
         ) {
             ps.setBytes(1, message.getInventoryVector() == null ? null : message.getInventoryVector().getHash());
@@ -258,10 +201,14 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
             ps.setString(3, message.getFrom().getAddress());
             ps.setString(4, message.getTo() == null ? null : message.getTo().getAddress());
             writeBlob(ps, 5, message);
-            ps.setLong(6, message.getSent());
+            ps.setBytes(6, message.getAckData());
-            ps.setLong(7, message.getReceived());
+            ps.setLong(7, message.getSent());
-            ps.setString(8, message.getStatus() == null ? null : message.getStatus().name());
+            ps.setLong(8, message.getReceived());
-            ps.setBytes(9, message.getInitialHash());
+            ps.setString(9, message.getStatus() == null ? null : message.getStatus().name());
+            ps.setBytes(10, message.getInitialHash());
+            ps.setLong(11, message.getTTL());
+            ps.setInt(12, message.getRetries());
+            ps.setObject(13, message.getNextTry());
 
             ps.executeUpdate();
             // get generated id
@@ -274,13 +221,23 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
 
     private void update(Connection connection, Plaintext message) throws SQLException, IOException {
         try (PreparedStatement ps = connection.prepareStatement(
-                "UPDATE Message SET iv=?, sent=?, received=?, status=?, initial_hash=? WHERE id=?")) {
+                "UPDATE Message SET iv=?, type=?, sender=?, recipient=?, data=?, ack_data=?, sent=?, received=?, " +
+                        "status=?, initial_hash=?, ttl=?, retries=?, next_try=? " +
+                        "WHERE id=?")) {
             ps.setBytes(1, message.getInventoryVector() == null ? null : message.getInventoryVector().getHash());
-            ps.setLong(2, message.getSent());
+            ps.setString(2, message.getType().name());
-            ps.setLong(3, message.getReceived());
+            ps.setString(3, message.getFrom().getAddress());
-            ps.setString(4, message.getStatus() == null ? null : message.getStatus().name());
+            ps.setString(4, message.getTo() == null ? null : message.getTo().getAddress());
-            ps.setBytes(5, message.getInitialHash());
+            writeBlob(ps, 5, message);
-            ps.setLong(6, (Long) message.getId());
+            ps.setBytes(6, message.getAckData());
+            ps.setLong(7, message.getSent());
+            ps.setLong(8, message.getReceived());
+            ps.setString(9, message.getStatus() == null ? null : message.getStatus().name());
+            ps.setBytes(10, message.getInitialHash());
+            ps.setLong(11, message.getTTL());
+            ps.setInt(12, message.getRetries());
+            ps.setObject(13, message.getNextTry());
+            ps.setLong(14, (Long) message.getId());
             ps.executeUpdate();
         }
     }
@@ -305,9 +262,4 @@ public class JdbcMessageRepository extends JdbcHelper implements MessageReposito
             LOG.error(e.getMessage(), e);
         }
     }
-
-    @Override
-    public void setContext(InternalContext context) {
-        this.ctx = context;
-    }
 }

repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcProofOfWorkRepository.java

@@ -1,6 +1,8 @@
 package ch.dissem.bitmessage.repository;
 
+import ch.dissem.bitmessage.InternalContext;
 import ch.dissem.bitmessage.entity.ObjectMessage;
+import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.exception.ApplicationException;
 import ch.dissem.bitmessage.factory.Factory;
 import ch.dissem.bitmessage.ports.ProofOfWorkRepository;
@@ -8,18 +10,21 @@ import ch.dissem.bitmessage.utils.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.sql.*;
 import java.util.LinkedList;
 import java.util.List;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * @author Christian Basler
  */
-public class JdbcProofOfWorkRepository extends JdbcHelper implements ProofOfWorkRepository {
+public class JdbcProofOfWorkRepository extends JdbcHelper implements ProofOfWorkRepository, InternalContext.ContextHolder {
     private static final Logger LOG = LoggerFactory.getLogger(JdbcProofOfWorkRepository.class);
+    private InternalContext ctx;
 
     public JdbcProofOfWorkRepository(JdbcConfig config) {
         super(config);
@@ -30,17 +35,27 @@ public class JdbcProofOfWorkRepository extends JdbcHelper implements ProofOfWork
         try (
                 Connection connection = config.getConnection();
                 PreparedStatement ps = connection.prepareStatement("SELECT data, version, nonce_trials_per_byte, " +
-                        "extra_bytes FROM POW WHERE initial_hash=?")
+                        "extra_bytes, expiration_time, message_id FROM POW WHERE initial_hash=?")
         ) {
             ps.setBytes(1, initialHash);
             try (ResultSet rs = ps.executeQuery()) {
                 if (rs.next()) {
                     Blob data = rs.getBlob("data");
+                    if (rs.getObject("message_id") == null) {
                         return new Item(
                                 Factory.getObjectMessage(rs.getInt("version"), data.getBinaryStream(), (int) data.length()),
                                 rs.getLong("nonce_trials_per_byte"),
                                 rs.getLong("extra_bytes")
                         );
+                    } else {
+                        return new Item(
+                                Factory.getObjectMessage(rs.getInt("version"), data.getBinaryStream(), (int) data.length()),
+                                rs.getLong("nonce_trials_per_byte"),
+                                rs.getLong("extra_bytes"),
+                                rs.getLong("expiration_time"),
+                                ctx.getMessageRepository().getMessage(rs.getLong("message_id"))
+                        );
+                    }
                 } else {
                     throw new IllegalArgumentException("Object requested that we don't have. Initial hash: " + Strings.hex(initialHash));
                 }
@@ -70,24 +85,38 @@ public class JdbcProofOfWorkRepository extends JdbcHelper implements ProofOfWork
     }
 
     @Override
-    public void putObject(ObjectMessage object, long nonceTrialsPerByte, long extraBytes) {
+    public void putObject(Item item) {
         try (
                 Connection connection = config.getConnection();
                 PreparedStatement ps = connection.prepareStatement("INSERT INTO POW (initial_hash, data, version, " +
-                        "nonce_trials_per_byte, extra_bytes) VALUES (?, ?, ?, ?, ?)")
+                        "nonce_trials_per_byte, extra_bytes, expiration_time, message_id) " +
+                        "VALUES (?, ?, ?, ?, ?, ?, ?)")
         ) {
-            ps.setBytes(1, security().getInitialHash(object));
+            ps.setBytes(1, cryptography().getInitialHash(item.object));
-            writeBlob(ps, 2, object);
+            writeBlob(ps, 2, item.object);
-            ps.setLong(3, object.getVersion());
+            ps.setLong(3, item.object.getVersion());
-            ps.setLong(4, nonceTrialsPerByte);
+            ps.setLong(4, item.nonceTrialsPerByte);
-            ps.setLong(5, extraBytes);
+            ps.setLong(5, item.extraBytes);
+
+            if (item.message == null) {
+                ps.setObject(6, null);
+                ps.setObject(7, null);
+            } else {
+                ps.setLong(6, item.expirationTime);
+                ps.setLong(7, (Long) item.message.getId());
+            }
             ps.executeUpdate();
         } catch (IOException | SQLException e) {
-            LOG.debug("Error storing object of type " + object.getPayload().getClass().getSimpleName(), e);
+            LOG.debug("Error storing object of type " + item.object.getPayload().getClass().getSimpleName(), e);
             throw new ApplicationException(e);
         }
     }
 
+    @Override
+    public void putObject(ObjectMessage object, long nonceTrialsPerByte, long extraBytes) {
+        putObject(new Item(object, nonceTrialsPerByte, extraBytes));
+    }
+
     @Override
     public void removeObject(byte[] initialHash) {
         try (
@@ -100,4 +129,9 @@ public class JdbcProofOfWorkRepository extends JdbcHelper implements ProofOfWork
             LOG.debug(e.getMessage(), e);
         }
     }
+
+    @Override
+    public void setContext(InternalContext context) {
+        this.ctx = context;
+    }
 }

repositories/src/main/resources/db/migration/V3.1__Update_table_POW.sql

@@ -0,0 +1,2 @@
+ALTER TABLE POW ADD COLUMN expiration_time BIGINT;
+ALTER TABLE POW ADD COLUMN message_id BIGINT;

repositories/src/main/resources/db/migration/V3.2__Update_table_message.sql

@@ -0,0 +1,4 @@
+ALTER TABLE Message ADD COLUMN ack_data BINARY(32);
+ALTER TABLE Message ADD COLUMN ttl      BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE Message ADD COLUMN retries  INT NOT NULL DEFAULT 0;
+ALTER TABLE Message ADD COLUMN next_try BIGINT;

repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcAddressRepositoryTest.java

@@ -17,12 +17,14 @@
 package ch.dissem.bitmessage.repository;
 
 import ch.dissem.bitmessage.entity.BitmessageAddress;
+import ch.dissem.bitmessage.entity.payload.Pubkey;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.util.List;
 
+import static ch.dissem.bitmessage.entity.payload.Pubkey.Feature.DOES_ACK;
 import static org.junit.Assert.*;
 
 public class JdbcAddressRepositoryTest extends TestBase {
@@ -46,7 +48,7 @@ public class JdbcAddressRepositoryTest extends TestBase {
         repo.save(new BitmessageAddress(CONTACT_B));
         repo.save(new BitmessageAddress(CONTACT_C));
 
-        BitmessageAddress identityA = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000));
+        BitmessageAddress identityA = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000, DOES_ACK));
         repo.save(identityA);
         IDENTITY_A = identityA.getAddress();
         BitmessageAddress identityB = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000));
@@ -66,8 +68,11 @@ public class JdbcAddressRepositoryTest extends TestBase {
     public void testFindIdentity() throws Exception {
         BitmessageAddress identity = new BitmessageAddress(IDENTITY_A);
         assertEquals(4, identity.getVersion());
-        assertEquals(identity, repo.findIdentity(identity.getTag()));
         assertNull(repo.findContact(identity.getTag()));
+
+        BitmessageAddress storedIdentity = repo.findIdentity(identity.getTag());
+        assertEquals(identity, storedIdentity);
+        assertTrue(storedIdentity.has(Pubkey.Feature.DOES_ACK));
     }
 
     @Test

repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcMessageRepositoryTest.java

@@ -19,12 +19,15 @@ package ch.dissem.bitmessage.repository;
 import ch.dissem.bitmessage.BitmessageContext;
 import ch.dissem.bitmessage.InternalContext;
 import ch.dissem.bitmessage.entity.BitmessageAddress;
+import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
 import ch.dissem.bitmessage.ports.AddressRepository;
 import ch.dissem.bitmessage.ports.MessageRepository;
+import ch.dissem.bitmessage.utils.UnixTime;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -32,8 +35,10 @@ import java.util.Arrays;
 import java.util.List;
 
 import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.entity.payload.Pubkey.Feature.DOES_ACK;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.*;
 
 public class JdbcMessageRepositoryTest extends TestBase {
@@ -54,19 +59,19 @@ public class JdbcMessageRepositoryTest extends TestBase {
         AddressRepository addressRepo = new JdbcAddressRepository(config);
         repo = new JdbcMessageRepository(config);
         new InternalContext(new BitmessageContext.Builder()
-                .cryptography(security())
+                .cryptography(cryptography())
                 .addressRepo(addressRepo)
                 .messageRepo(repo)
         );
 
-        BitmessageAddress tmp = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000));
+        BitmessageAddress tmp = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000, DOES_ACK));
         contactA = new BitmessageAddress(tmp.getAddress());
         contactA.setPubkey(tmp.getPubkey());
         addressRepo.save(contactA);
         contactB = new BitmessageAddress("BM-2cTtkBnb4BUYDndTKun6D9PjtueP2h1bQj");
         addressRepo.save(contactB);
 
-        identity = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000));
+        identity = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000, DOES_ACK));
         addressRepo.save(identity);
 
         inbox = repo.getLabels(Label.Type.INBOX).get(0);
@@ -123,6 +128,18 @@ public class JdbcMessageRepositoryTest extends TestBase {
         assertThat(other, is(message));
     }
 
+    @Test
+    public void ensureAckMessageCanBeUpdatedAndRetrieved() {
+        byte[] initialHash = new byte[64];
+        Plaintext message = repo.findMessages(contactA).get(0);
+        message.setInitialHash(initialHash);
+        ObjectMessage ackMessage = message.getAckMessage();
+        repo.save(message);
+        Plaintext other = repo.getMessage(initialHash);
+        assertThat(other, is(message));
+        assertThat(other.getAckMessage(), is(ackMessage));
+    }
+
     @Test
     public void testFindMessagesByStatus() throws Exception {
         List<Plaintext> messages = repo.findMessages(Plaintext.Status.RECEIVED);
@@ -146,7 +163,7 @@ public class JdbcMessageRepositoryTest extends TestBase {
     @Test
     public void testSave() throws Exception {
         Plaintext message = new Plaintext.Builder(MSG)
-                .IV(new InventoryVector(security().randomBytes(32)))
+                .IV(new InventoryVector(cryptography().randomBytes(32)))
                 .from(identity)
                 .to(contactA)
                 .message("Subject", "Message")
@@ -169,7 +186,7 @@ public class JdbcMessageRepositoryTest extends TestBase {
     public void testUpdate() throws Exception {
         List<Plaintext> messages = repo.findMessages(Plaintext.Status.DRAFT, contactA);
         Plaintext message = messages.get(0);
-        message.setInventoryVector(new InventoryVector(security().randomBytes(32)));
+        message.setInventoryVector(new InventoryVector(cryptography().randomBytes(32)));
         repo.save(message);
 
         messages = repo.findMessages(Plaintext.Status.DRAFT, contactA);
@@ -178,11 +195,40 @@ public class JdbcMessageRepositoryTest extends TestBase {
     }
 
     @Test
-    public void testRemove() throws Exception {
+    public void ensureMessageIsRemoved() throws Exception {
         Plaintext toRemove = repo.findMessages(Plaintext.Status.DRAFT, contactB).get(0);
-        repo.remove(toRemove);
         List<Plaintext> messages = repo.findMessages(Plaintext.Status.DRAFT);
-        assertEquals(1, messages.size());
+        assertEquals(2, messages.size());
+        repo.remove(toRemove);
+        messages = repo.findMessages(Plaintext.Status.DRAFT);
+        assertThat(messages, hasSize(1));
+    }
+
+    @Test
+    public void ensureUnacknowledgedMessagesAreFoundForResend() throws Exception {
+        Plaintext message = new Plaintext.Builder(MSG)
+                .IV(new InventoryVector(cryptography().randomBytes(32)))
+                .from(identity)
+                .to(contactA)
+                .message("Subject", "Message")
+                .status(Plaintext.Status.SENT)
+                .ttl(1)
+                .build();
+        message.updateNextTry();
+        assertThat(message.getRetries(), is(1));
+        assertThat(message.getNextTry(), greaterThan(UnixTime.now()));
+        assertThat(message.getNextTry(), lessThanOrEqualTo(UnixTime.now(+1)));
+        repo.save(message);
+        Thread.sleep(2100);
+        List<Plaintext> messagesToResend = repo.findMessagesToResend();
+        assertThat(messagesToResend, hasSize(1));
+
+        message.updateNextTry();
+        assertThat(message.getRetries(), is(2));
+        assertThat(message.getNextTry(), greaterThan(UnixTime.now()));
+        repo.save(message);
+        messagesToResend = repo.findMessagesToResend();
+        assertThat(messagesToResend, empty());
     }
 
     private void addMessage(BitmessageAddress from, BitmessageAddress to, Plaintext.Status status, Label... labels) {

repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcProofOfWorkRepositoryTest.java

@@ -16,13 +16,24 @@
 
 package ch.dissem.bitmessage.repository;
 
+import ch.dissem.bitmessage.BitmessageContext;
+import ch.dissem.bitmessage.InternalContext;
 import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.ObjectMessage;
+import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.payload.GenericPayload;
 import ch.dissem.bitmessage.entity.payload.GetPubkey;
-import ch.dissem.bitmessage.ports.ProofOfWorkRepository;
+import ch.dissem.bitmessage.ports.AddressRepository;
+import ch.dissem.bitmessage.ports.MessageRepository;
+import ch.dissem.bitmessage.ports.ProofOfWorkRepository.Item;
+import ch.dissem.bitmessage.utils.TestUtils;
+import ch.dissem.bitmessage.utils.UnixTime;
 import org.junit.Before;
 import org.junit.Test;
 
+import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
+import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 import static org.hamcrest.CoreMatchers.*;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -33,17 +44,51 @@ import static org.junit.Assert.assertTrue;
 public class JdbcProofOfWorkRepositoryTest extends TestBase {
     private TestJdbcConfig config;
     private JdbcProofOfWorkRepository repo;
+    private AddressRepository addressRepo;
+    private MessageRepository messageRepo;
+
+    private byte[] initialHash1;
+    private byte[] initialHash2;
 
     @Before
     public void setUp() throws Exception {
         config = new TestJdbcConfig();
         config.reset();
 
+        addressRepo = new JdbcAddressRepository(config);
+        messageRepo = new JdbcMessageRepository(config);
         repo = new JdbcProofOfWorkRepository(config);
+        InternalContext ctx = new InternalContext(new BitmessageContext.Builder()
+                .addressRepo(addressRepo)
+                .messageRepo(messageRepo)
+                .powRepo(repo)
+                .cryptography(cryptography())
+        );
 
         repo.putObject(new ObjectMessage.Builder()
                         .payload(new GetPubkey(new BitmessageAddress("BM-2DAjcCFrqFrp88FUxExhJ9kPqHdunQmiyn"))).build(),
                 1000, 1000);
+        initialHash1 = repo.getItems().get(0);
+
+        BitmessageAddress sender = TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8");
+        BitmessageAddress recipient = TestUtils.loadContact();
+        addressRepo.save(sender);
+        addressRepo.save(recipient);
+        Plaintext plaintext = new Plaintext.Builder(MSG)
+                .ackData(cryptography().randomBytes(32))
+                .from(sender)
+                .to(recipient)
+                .message("Subject", "Message")
+                .status(Plaintext.Status.DOING_PROOF_OF_WORK)
+                .build();
+        messageRepo.save(plaintext);
+        initialHash2 = cryptography().getInitialHash(plaintext.getAckMessage());
+        repo.putObject(new Item(
+                plaintext.getAckMessage(),
+                1000, 1000,
+                UnixTime.now(+10 * MINUTE),
+                plaintext
+        ));
     }
 
     @Test
@@ -55,16 +100,52 @@ public class JdbcProofOfWorkRepositoryTest extends TestBase {
         assertThat(repo.getItems().size(), is(sizeBefore + 1));
     }
 
+    @Test
+    public void ensureAckObjectsAreStored() throws Exception {
+        int sizeBefore = repo.getItems().size();
+        BitmessageAddress sender = TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8");
+        BitmessageAddress recipient = TestUtils.loadContact();
+        addressRepo.save(sender);
+        addressRepo.save(recipient);
+        Plaintext plaintext = new Plaintext.Builder(MSG)
+                .ackData(cryptography().randomBytes(32))
+                .from(sender)
+                .to(recipient)
+                .message("Subject", "Message")
+                .status(Plaintext.Status.DOING_PROOF_OF_WORK)
+                .build();
+        messageRepo.save(plaintext);
+        repo.putObject(new Item(
+                plaintext.getAckMessage(),
+                1000, 1000,
+                UnixTime.now(+10 * MINUTE),
+                plaintext
+        ));
+        assertThat(repo.getItems().size(), is(sizeBefore + 1));
+    }
+
     @Test
     public void ensureItemCanBeRetrieved() {
-        byte[] initialHash = repo.getItems().get(0);
+        Item item = repo.getItem(initialHash1);
-        ProofOfWorkRepository.Item item = repo.getItem(initialHash);
         assertThat(item, notNullValue());
         assertThat(item.object.getPayload(), instanceOf(GetPubkey.class));
         assertThat(item.nonceTrialsPerByte, is(1000L));
         assertThat(item.extraBytes, is(1000L));
     }
 
+    @Test
+    public void ensureAckItemCanBeRetrieved() {
+        Item item = repo.getItem(initialHash2);
+        assertThat(item, notNullValue());
+        assertThat(item.object.getPayload(), instanceOf(GenericPayload.class));
+        assertThat(item.nonceTrialsPerByte, is(1000L));
+        assertThat(item.extraBytes, is(1000L));
+        assertThat(item.expirationTime, not(0));
+        assertThat(item.message, notNullValue());
+        assertThat(item.message.getFrom().getPrivateKey(), notNullValue());
+        assertThat(item.message.getTo().getPubkey(), notNullValue());
+    }
+
     @Test(expected = RuntimeException.class)
     public void ensureRetrievingNonexistingItemThrowsException() {
         repo.getItem(new byte[0]);
@@ -72,8 +153,8 @@ public class JdbcProofOfWorkRepositoryTest extends TestBase {
 
     @Test
     public void ensureItemCanBeDeleted() {
-        byte[] initialHash = repo.getItems().get(0);
+        repo.removeObject(initialHash1);
-        repo.removeObject(initialHash);
+        repo.removeObject(initialHash2);
         assertTrue(repo.getItems().isEmpty());
     }
 

wif/src/main/java/ch/dissem/bitmessage/wif/WifExporter.java

@@ -27,7 +27,7 @@ import java.io.*;
 import java.util.Collection;
 
 import static ch.dissem.bitmessage.entity.valueobject.PrivateKey.PRIVATE_KEY_SIZE;
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * @author Christian Basler
@@ -77,7 +77,7 @@ public class WifExporter {
         byte[] result = new byte[37];
         result[0] = (byte) 0x80;
         System.arraycopy(privateKey, 0, result, 1, PRIVATE_KEY_SIZE);
-        byte[] hash = security().doubleSha256(result, PRIVATE_KEY_SIZE + 1);
+        byte[] hash = cryptography().doubleSha256(result, PRIVATE_KEY_SIZE + 1);
         System.arraycopy(hash, 0, result, PRIVATE_KEY_SIZE + 1, 4);
         return Base58.encode(result);
     }

wif/src/main/java/ch/dissem/bitmessage/wif/WifImporter.java

@@ -31,7 +31,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map.Entry;
 
-import static ch.dissem.bitmessage.utils.Singleton.security;
+import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
  * @author Christian Basler
@@ -87,7 +87,7 @@ public class WifImporter {
             throw new IOException("Unknown format: " + WIF_SECRET_LENGTH +
                     " bytes expected, but secret " + walletImportFormat + " was " + bytes.length + " long");
 
-        byte[] hash = security().doubleSha256(bytes, 33);
+        byte[] hash = cryptography().doubleSha256(bytes, 33);
         for (int i = 0; i < 4; i++) {
             if (hash[i] != bytes[33 + i]) throw new IOException("Hash check failed for secret " + walletImportFormat);
         }