Merge branch 'feature/extended-encoding' into develop

README.md

@@ -9,7 +9,7 @@ A Java implementation for the Bitmessage protocol. To build, use command `./grad
 
 Please note that it still has its limitations, but the API should now be stable. Jabit uses Semantic Versioning, meaning as long as the major version doesn't change, nothing should break if you update.
 
-Be aware though that this doesn't necessarily applies for SNAPSHOT builds and the development branch, notably when it comes to database updates. _In other words, they may break your installation!_ 
+Be aware though that this doesn't necessarily applies for SNAPSHOT builds and the development branch, notably when it comes to database updates. In other words, they may break your installation!_ 
 
 #### Master
 [![Build Status](https://travis-ci.org/Dissem/Jabit.svg?branch=master)](https://travis-ci.org/Dissem/Jabit) 

build.gradle

@@ -1,15 +1,21 @@
+plugins {
+    id 'com.github.ben-manes.versions' version '0.14.0'
+}
+
 subprojects {
     apply plugin: 'java'
     apply plugin: 'maven'
     apply plugin: 'signing'
     apply plugin: 'jacoco'
     apply plugin: 'gitflow-version'
+    apply plugin: 'com.github.ben-manes.versions'
 
     sourceCompatibility = 1.7
     group = 'ch.dissem.jabit'
 
     repositories {
         mavenCentral()
+        maven { url 'https://oss.sonatype.org/content/repositories/snapshots/' }
     }
 
     test {

buildSrc/src/main/groovy/ch/dissem/gradle/GitFlowVersion.groovy

@@ -50,8 +50,8 @@ class GitFlowVersion implements Plugin<Project> {
         project.ext.isRelease = isRelease(project)
         project.version = getVersion(project)
 
-        project.task('version') << {
+        project.task('version') {
-            println "Version deduced from git: '${project.version}'"
+            doLast { println "Version deduced from git: '${project.version}'" }
         }
     }
 }

core/build.gradle

@@ -24,9 +24,10 @@ artifacts {
 }
 
 dependencies {
-    compile 'org.slf4j:slf4j-api:1.7.12'
+    compile 'org.slf4j:slf4j-api:1.7.25'
+    compile 'ch.dissem.msgpack:msgpack:1.0.0'
     testCompile 'junit:junit:4.12'
     testCompile 'org.hamcrest:hamcrest-library:1.3'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
     testCompile project(':cryptography-bc')
 }

core/src/main/java/ch/dissem/bitmessage/BitmessageContext.java

@@ -25,7 +25,6 @@ import ch.dissem.bitmessage.exception.DecryptionFailedException;
 import ch.dissem.bitmessage.factory.Factory;
 import ch.dissem.bitmessage.ports.*;
 import ch.dissem.bitmessage.utils.Property;
-import ch.dissem.bitmessage.utils.TTL;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,7 +38,8 @@ import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES;
 import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE;
 import static ch.dissem.bitmessage.entity.Plaintext.Type.BROADCAST;
 import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
-import static ch.dissem.bitmessage.utils.UnixTime.*;
+import static ch.dissem.bitmessage.utils.UnixTime.HOUR;
+import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 
 /**
  * <p>Use this class if you want to create a Bitmessage client.</p>
@@ -66,13 +66,13 @@ public class BitmessageContext {
     private final boolean sendPubkeyOnIdentityCreation;
 
     private BitmessageContext(Builder builder) {
-        if (builder.listener instanceof Listener.WithContext) {
-            ((Listener.WithContext) builder.listener).setContext(this);
-        }
         ctx = new InternalContext(builder);
         labeler = builder.labeler;
         ctx.getProofOfWorkService().doMissingProofOfWork(30_000); // TODO: this should be configurable
         sendPubkeyOnIdentityCreation = builder.sendPubkeyOnIdentityCreation;
+        if (builder.listener instanceof Listener.WithContext) {
+            ((Listener.WithContext) builder.listener).setContext(this);
+        }
     }
 
     public AddressRepository addresses() {
@@ -254,7 +254,10 @@ public class BitmessageContext {
     public void addContact(BitmessageAddress contact) {
         ctx.getAddressRepository().save(contact);
         if (contact.getPubkey() == null) {
-            ctx.requestPubkey(contact);
+            BitmessageAddress stored = ctx.getAddressRepository().getAddress(contact.getAddress());
+            if (stored.getPubkey() == null) {
+                ctx.requestPubkey(contact);
+            }
         }
     }
 
@@ -401,23 +404,6 @@ public class BitmessageContext {
             return this;
         }
 
-        /**
-         * Time to live in seconds for public keys the client sends. Defaults to the maximum of 28 days,
-         * but on weak devices smaller values might be desirable.
-         * <p>
-         * Please be aware that this might cause some problems where you can't receive a message (the
-         * sender can't receive your public key) in some special situations. Also note that it's probably
-         * not a good idea to set it too low.
-         * </p>
-         *
-         * @deprecated use {@link TTL#pubkey(long)} instead.
-         */
-        public Builder pubkeyTTL(long days) {
-            if (days < 0 || days > 28 * DAY) throw new IllegalArgumentException("TTL must be between 1 and 28 days");
-            TTL.pubkey(days);
-            return this;
-        }
-
         public BitmessageContext build() {
             nonNull("inventory", inventory);
             nonNull("nodeRegistry", nodeRegistry);
@@ -435,8 +421,8 @@ public class BitmessageContext {
                 customCommandHandler = new CustomCommandHandler() {
                     @Override
                     public MessagePayload handle(CustomMessage request) {
-                        throw new IllegalStateException(
+                        LOG.debug("Received custom request, but no custom command handler configured.");
-                            "Received custom request, but no custom command handler configured.");
+                        return null;
                     }
                 };
             }

core/src/main/java/ch/dissem/bitmessage/DefaultMessageListener.java

@@ -87,7 +87,7 @@ class DefaultMessageListener implements NetworkHandler.MessageListener, Internal
         BitmessageAddress identity = ctx.getAddressRepository().findIdentity(getPubkey.getRipeTag());
         if (identity != null && identity.getPrivateKey() != null && !identity.isChan()) {
             LOG.info("Got pubkey request for identity " + identity);
-            // FIXME: only send pubkey if it wasn't sent in the last 28 days
+            // FIXME: only send pubkey if it wasn't sent in the last TTL.pubkey() days
             ctx.sendPubkey(identity, object.getStream());
         }
     }

core/src/main/java/ch/dissem/bitmessage/InternalContext.java

@@ -30,6 +30,8 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.TreeSet;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
 
 /**
  * The internal context should normally only be used for port implementations. If you need it in your client
@@ -45,6 +47,8 @@ public class InternalContext {
     public final static long NETWORK_NONCE_TRIALS_PER_BYTE = 1000;
     public final static long NETWORK_EXTRA_BYTES = 1000;
 
+    private final Executor threadPool = Executors.newCachedThreadPool();
+
     private final Cryptography cryptography;
     private final Inventory inventory;
     private final NodeRegistry nodeRegistry;
@@ -226,31 +230,36 @@ public class InternalContext {
      * for freshly received pubkeys will not be called. Instead the pubkey is added to the contact and stored on DB.
      */
     public void requestPubkey(final BitmessageAddress contact) {
-        BitmessageAddress stored = addressRepository.getAddress(contact.getAddress());
+        threadPool.execute(new Runnable() {
+            @Override
+            public void run() {
+                BitmessageAddress stored = addressRepository.getAddress(contact.getAddress());
 
-        tryToFindMatchingPubkey(contact);
+                tryToFindMatchingPubkey(contact);
-        if (contact.getPubkey() != null) {
+                if (contact.getPubkey() != null) {
-            if (stored != null) {
+                    if (stored != null) {
-                stored.setPubkey(contact.getPubkey());
+                        stored.setPubkey(contact.getPubkey());
-                addressRepository.save(stored);
+                        addressRepository.save(stored);
-            } else {
+                    } else {
-                addressRepository.save(contact);
+                        addressRepository.save(contact);
+                    }
+                    return;
+                }
+
+                if (stored == null) {
+                    addressRepository.save(contact);
+                }
+
+                long expires = UnixTime.now(TTL.getpubkey());
+                LOG.info("Expires at " + expires);
+                final ObjectMessage request = new ObjectMessage.Builder()
+                    .stream(contact.getStream())
+                    .expiresTime(expires)
+                    .payload(new GetPubkey(contact))
+                    .build();
+                proofOfWorkService.doProofOfWork(request);
             }
-            return;
+        });
-        }
-
-        if (stored == null) {
-            addressRepository.save(contact);
-        }
-
-        long expires = UnixTime.now(TTL.getpubkey());
-        LOG.info("Expires at " + expires);
-        final ObjectMessage request = new ObjectMessage.Builder()
-            .stream(contact.getStream())
-            .expiresTime(expires)
-            .payload(new GetPubkey(contact))
-            .build();
-        proofOfWorkService.doProofOfWork(request);
     }
 
     private void tryToFindMatchingPubkey(BitmessageAddress address) {

core/src/main/java/ch/dissem/bitmessage/ProofOfWorkService.java

@@ -81,7 +81,7 @@ public class ProofOfWorkService implements ProofOfWorkEngine.Callback, InternalC
     public void onNonceCalculated(byte[] initialHash, byte[] nonce) {
         Item item = powRepo.getItem(initialHash);
         if (item.message == null) {
-            ObjectMessage object = powRepo.getItem(initialHash).object;
+            ObjectMessage object = item.object;
             object.setNonce(nonce);
             Plaintext plaintext = messageRepo.getMessage(initialHash);
             if (plaintext != null) {

core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java

@@ -18,9 +18,13 @@ package ch.dissem.bitmessage.entity;
 
 import ch.dissem.bitmessage.entity.payload.Msg;
 import ch.dissem.bitmessage.entity.payload.Pubkey.Feature;
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
+import ch.dissem.bitmessage.entity.valueobject.extended.Attachment;
+import ch.dissem.bitmessage.entity.valueobject.extended.Message;
 import ch.dissem.bitmessage.exception.ApplicationException;
+import ch.dissem.bitmessage.factory.ExtendedEncodingFactory;
 import ch.dissem.bitmessage.factory.Factory;
 import ch.dissem.bitmessage.utils.*;
 
@@ -29,6 +33,8 @@ import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.Collections;
 
+import static ch.dissem.bitmessage.entity.Plaintext.Encoding.EXTENDED;
+import static ch.dissem.bitmessage.entity.Plaintext.Encoding.SIMPLE;
 import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
@@ -42,6 +48,8 @@ public class Plaintext implements Streamable {
     private final long encoding;
     private final byte[] message;
     private final byte[] ackData;
+    private final UUID conversationId;
+    private ExtendedEncoding extendedData;
     private ObjectMessage ackMessage;
     private Object id;
     private InventoryVector inventoryVector;
@@ -81,6 +89,7 @@ public class Plaintext implements Streamable {
         ttl = builder.ttl;
         retries = builder.retries;
         nextTry = builder.nextTry;
+        conversationId = builder.conversation;
     }
 
     public static Plaintext read(Type type, InputStream in) throws IOException {
@@ -143,6 +152,10 @@ public class Plaintext implements Streamable {
         return labels;
     }
 
+    public Encoding getEncoding() {
+        return Encoding.fromCode(encoding);
+    }
+
     public long getStream() {
         return from.getStream();
     }
@@ -167,12 +180,23 @@ public class Plaintext implements Streamable {
     public void write(OutputStream out, boolean includeSignature) throws IOException {
         Encode.varInt(from.getVersion(), out);
         Encode.varInt(from.getStream(), out);
-        Encode.int32(from.getPubkey().getBehaviorBitfield(), out);
+        if (from.getPubkey() == null) {
-        out.write(from.getPubkey().getSigningKey(), 1, 64);
+            Encode.int32(0, out);
-        out.write(from.getPubkey().getEncryptionKey(), 1, 64);
+            byte[] empty = new byte[64];
-        if (from.getVersion() >= 3) {
+            out.write(empty);
-            Encode.varInt(from.getPubkey().getNonceTrialsPerByte(), out);
+            out.write(empty);
-            Encode.varInt(from.getPubkey().getExtraBytes(), out);
+            if (from.getVersion() >= 3) {
+                Encode.varInt(0, out);
+                Encode.varInt(0, out);
+            }
+        } else {
+            Encode.int32(from.getPubkey().getBehaviorBitfield(), out);
+            out.write(from.getPubkey().getSigningKey(), 1, 64);
+            out.write(from.getPubkey().getEncryptionKey(), 1, 64);
+            if (from.getVersion() >= 3) {
+                Encode.varInt(from.getPubkey().getNonceTrialsPerByte(), out);
+                Encode.varInt(from.getPubkey().getExtraBytes(), out);
+            }
         }
         if (type == Type.MSG) {
             out.write(to.getRipe());
@@ -202,12 +226,23 @@ public class Plaintext implements Streamable {
     public void write(ByteBuffer buffer, boolean includeSignature) {
         Encode.varInt(from.getVersion(), buffer);
         Encode.varInt(from.getStream(), buffer);
-        Encode.int32(from.getPubkey().getBehaviorBitfield(), buffer);
+        if (from.getPubkey() == null) {
-        buffer.put(from.getPubkey().getSigningKey(), 1, 64);
+            Encode.int32(0, buffer);
-        buffer.put(from.getPubkey().getEncryptionKey(), 1, 64);
+            byte[] empty = new byte[64];
-        if (from.getVersion() >= 3) {
+            buffer.put(empty);
-            Encode.varInt(from.getPubkey().getNonceTrialsPerByte(), buffer);
+            buffer.put(empty);
-            Encode.varInt(from.getPubkey().getExtraBytes(), buffer);
+            if (from.getVersion() >= 3) {
+                Encode.varInt(0, buffer);
+                Encode.varInt(0, buffer);
+            }
+        } else {
+            Encode.int32(from.getPubkey().getBehaviorBitfield(), buffer);
+            buffer.put(from.getPubkey().getSigningKey(), 1, 64);
+            buffer.put(from.getPubkey().getEncryptionKey(), 1, 64);
+            if (from.getVersion() >= 3) {
+                Encode.varInt(from.getPubkey().getNonceTrialsPerByte(), buffer);
+                Encode.varInt(from.getPubkey().getExtraBytes(), buffer);
+            }
         }
         if (type == Type.MSG) {
             buffer.put(to.getRipe());
@@ -299,7 +334,13 @@ public class Plaintext implements Streamable {
     public String getSubject() {
         Scanner s = new Scanner(new ByteArrayInputStream(message), "UTF-8");
         String firstLine = s.nextLine();
-        if (encoding == 2) {
+        if (encoding == EXTENDED.code) {
+            if (Message.TYPE.equals(getExtendedData().getType())) {
+                return ((Message) extendedData.getContent()).getSubject();
+            } else {
+                return null;
+            }
+        } else if (encoding == SIMPLE.code) {
             return firstLine.substring("Subject:".length()).trim();
         } else if (firstLine.length() > 50) {
             return firstLine.substring(0, 50).trim() + "...";
@@ -309,17 +350,65 @@ public class Plaintext implements Streamable {
     }
 
     public String getText() {
-        try {
+        if (encoding == EXTENDED.code) {
-            String text = new String(message, "UTF-8");
+            if (Message.TYPE.equals(getExtendedData().getType())) {
-            if (encoding == 2) {
+                return ((Message) extendedData.getContent()).getBody();
-                return text.substring(text.indexOf("\nBody:") + 6);
+            } else {
+                return null;
+            }
+        } else {
+            try {
+                String text = new String(message, "UTF-8");
+                if (encoding == SIMPLE.code) {
+                    return text.substring(text.indexOf("\nBody:") + 6);
+                }
+                return text;
+            } catch (UnsupportedEncodingException e) {
+                throw new ApplicationException(e);
             }
-            return text;
-        } catch (UnsupportedEncodingException e) {
-            throw new ApplicationException(e);
         }
     }
 
+    protected ExtendedEncoding getExtendedData() {
+        if (extendedData == null && encoding == EXTENDED.code) {
+            // TODO: make sure errors are properly handled
+            extendedData = ExtendedEncodingFactory.getInstance().unzip(message);
+        }
+        return extendedData;
+    }
+
+    @SuppressWarnings("unchecked")
+    public <T extends ExtendedEncoding.ExtendedType> T getExtendedData(Class<T> type) {
+        ExtendedEncoding extendedData = getExtendedData();
+        if (extendedData == null) {
+            return null;
+        }
+        if (type == null || type.isInstance(extendedData.getContent())) {
+            return (T) extendedData.getContent();
+        }
+        return null;
+    }
+
+    public List<InventoryVector> getParents() {
+        if (getExtendedData() != null && Message.TYPE.equals(getExtendedData().getType())) {
+            return ((Message) extendedData.getContent()).getParents();
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    public List<Attachment> getFiles() {
+        if (Message.TYPE.equals(getExtendedData().getType())) {
+            return ((Message) extendedData.getContent()).getFiles();
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    public UUID getConversationId() {
+        return conversationId;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -350,9 +439,7 @@ public class Plaintext implements Streamable {
 
     public void addLabels(Collection<Label> labels) {
         if (labels != null) {
-            for (Label label : labels) {
+            this.labels.addAll(labels);
-                this.labels.add(label);
-            }
         }
     }
 
@@ -385,8 +472,18 @@ public class Plaintext implements Streamable {
         return initialHash;
     }
 
+    @Override
+    public String toString() {
+        String subject = getSubject();
+        if (subject == null || subject.length() == 0) {
+            return Strings.hex(initialHash).toString();
+        } else {
+            return subject;
+        }
+    }
+
     public enum Encoding {
-        IGNORE(0), TRIVIAL(1), SIMPLE(2);
+        IGNORE(0), TRIVIAL(1), SIMPLE(2), EXTENDED(3);
 
         long code;
 
@@ -397,6 +494,15 @@ public class Plaintext implements Streamable {
         public long getCode() {
             return code;
         }
+
+        public static Encoding fromCode(long code) {
+            for (Encoding e : values()) {
+                if (e.getCode() == code) {
+                    return e;
+                }
+            }
+            return null;
+        }
     }
 
     public enum Status {
@@ -432,13 +538,14 @@ public class Plaintext implements Streamable {
         private byte[] ackData;
         private byte[] ackMessage;
         private byte[] signature;
-        private long sent;
+        private Long sent;
-        private long received;
+        private Long received;
         private Status status;
-        private Set<Label> labels = new HashSet<>();
+        private Set<Label> labels = new LinkedHashSet<>();
         private long ttl;
         private int retries;
         private Long nextTry;
+        private UUID conversation;
 
         public Builder(Type type) {
             this.type = type;
@@ -517,9 +624,15 @@ public class Plaintext implements Streamable {
             return this;
         }
 
+        public Builder message(ExtendedEncoding message) {
+            this.encoding = EXTENDED.getCode();
+            this.message = message.zip();
+            return this;
+        }
+
         public Builder message(String subject, String message) {
             try {
-                this.encoding = Encoding.SIMPLE.getCode();
+                this.encoding = SIMPLE.getCode();
                 this.message = ("Subject:" + subject + '\n' + "Body:" + message).getBytes("UTF-8");
             } catch (UnsupportedEncodingException e) {
                 throw new ApplicationException(e);
@@ -550,12 +663,12 @@ public class Plaintext implements Streamable {
             return this;
         }
 
-        public Builder sent(long sent) {
+        public Builder sent(Long sent) {
             this.sent = sent;
             return this;
         }
 
-        public Builder received(long received) {
+        public Builder received(Long received) {
             this.received = received;
             return this;
         }
@@ -585,6 +698,11 @@ public class Plaintext implements Streamable {
             return this;
         }
 
+        public Builder conversation(UUID id) {
+            this.conversation = id;
+            return this;
+        }
+
         public Plaintext build() {
             if (from == null) {
                 from = new BitmessageAddress(Factory.createPubkey(
@@ -606,6 +724,9 @@ public class Plaintext implements Streamable {
             if (ttl <= 0) {
                 ttl = TTL.msg();
             }
+            if (conversation == null) {
+                conversation = UUID.randomUUID();
+            }
             return new Plaintext(this);
         }
     }

core/src/main/java/ch/dissem/bitmessage/entity/Version.java

@@ -20,6 +20,8 @@ import ch.dissem.bitmessage.BitmessageContext;
 import ch.dissem.bitmessage.entity.valueobject.NetworkAddress;
 import ch.dissem.bitmessage.utils.Encode;
 import ch.dissem.bitmessage.utils.UnixTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -30,6 +32,7 @@ import java.nio.ByteBuffer;
  */
 public class Version implements MessagePayload {
     private static final long serialVersionUID = 7219240857343176567L;
+    private static final Logger LOG = LoggerFactory.getLogger(Version.class);
 
     /**
      * Identifies protocol version being used by the node. Should equal 3. Nodes should disconnect if the remote node's
@@ -93,6 +96,10 @@ public class Version implements MessagePayload {
         return services;
     }
 
+    public boolean provides(Service service) {
+        return service != null && service.isEnabled(services);
+    }
+
     public long getTimestamp() {
         return timestamp;
     }
@@ -159,7 +166,7 @@ public class Version implements MessagePayload {
 
         public Builder defaults(long clientNonce) {
             version = BitmessageContext.CURRENT_VERSION;
-            services = 1;
+            services = Service.getServiceFlag(Service.NODE_NETWORK);
             timestamp = UnixTime.now();
             userAgent = "/Jabit:0.0.1/";
             streamNumbers = new long[]{1};
@@ -172,6 +179,11 @@ public class Version implements MessagePayload {
             return this;
         }
 
+        public Builder services(Service... services) {
+            this.services = Service.getServiceFlag(services);
+            return this;
+        }
+
         public Builder services(long services) {
             this.services = services;
             return this;
@@ -211,4 +223,27 @@ public class Version implements MessagePayload {
             return new Version(this);
         }
     }
+
+    public enum Service {
+        NODE_NETWORK(1);
+// TODO: NODE_SSL(2);
+
+        long flag;
+
+        Service(long flag) {
+            this.flag = flag;
+        }
+
+        public boolean isEnabled(long flag) {
+            return (flag & this.flag) != 0;
+        }
+
+        public static long getServiceFlag(Service... services) {
+            long flag = 0;
+            for (Service service : services) {
+                flag |= service.flag;
+            }
+            return flag;
+        }
+    }
 }

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/ExtendedEncoding.java

@@ -0,0 +1,76 @@
+package ch.dissem.bitmessage.entity.valueobject;
+
+import ch.dissem.bitmessage.exception.ApplicationException;
+import ch.dissem.msgpack.types.MPMap;
+import ch.dissem.msgpack.types.MPString;
+import ch.dissem.msgpack.types.MPType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.zip.DeflaterOutputStream;
+
+/**
+ * Extended encoding message object.
+ */
+public class ExtendedEncoding implements Serializable {
+    private static final long serialVersionUID = 3876871488247305200L;
+    private static final Logger LOG = LoggerFactory.getLogger(ExtendedEncoding.class);
+
+    private ExtendedType content;
+
+    public ExtendedEncoding(ExtendedType content) {
+        this.content = content;
+    }
+
+    public String getType() {
+        if (content == null) {
+            return null;
+        } else {
+            return content.getType();
+        }
+    }
+
+    public ExtendedType getContent() {
+        return content;
+    }
+
+    public byte[] zip() {
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (DeflaterOutputStream zipper = new DeflaterOutputStream(out)) {
+                content.pack().pack(zipper);
+            }
+            return out.toByteArray();
+        } catch (IOException e) {
+            throw new ApplicationException(e);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ExtendedEncoding that = (ExtendedEncoding) o;
+        return Objects.equals(content, that.content);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(content);
+    }
+
+    public interface Unpacker<T extends ExtendedType> {
+        String getType();
+
+        T unpack(MPMap<MPString, MPType<?>> map);
+    }
+
+    public interface ExtendedType extends Serializable {
+        String getType();
+
+        MPMap<MPString, MPType<?>> pack() throws IOException;
+    }
+}

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/NetworkAddress.java

@@ -17,6 +17,7 @@
 package ch.dissem.bitmessage.entity.valueobject;
 
 import ch.dissem.bitmessage.entity.Streamable;
+import ch.dissem.bitmessage.entity.Version;
 import ch.dissem.bitmessage.exception.ApplicationException;
 import ch.dissem.bitmessage.utils.Encode;
 import ch.dissem.bitmessage.utils.UnixTime;
@@ -75,6 +76,13 @@ public class NetworkAddress implements Streamable {
         return services;
     }
 
+    public boolean provides(Version.Service service) {
+        if (service == null) {
+            return false;
+        }
+        return service.isEnabled(services);
+    }
+
     public long getStream() {
         return stream;
     }
@@ -194,20 +202,20 @@ public class NetworkAddress implements Streamable {
                             int p08, int p09, int p10, int p11,
                             int p12, int p13, int p14, int p15) {
             this.ipv6 = new byte[]{
-                    (byte) p00, (byte) p01, (byte) p02, (byte) p03,
+                (byte) p00, (byte) p01, (byte) p02, (byte) p03,
-                    (byte) p04, (byte) p05, (byte) p06, (byte) p07,
+                (byte) p04, (byte) p05, (byte) p06, (byte) p07,
-                    (byte) p08, (byte) p09, (byte) p10, (byte) p11,
+                (byte) p08, (byte) p09, (byte) p10, (byte) p11,
-                    (byte) p12, (byte) p13, (byte) p14, (byte) p15
+                (byte) p12, (byte) p13, (byte) p14, (byte) p15
             };
             return this;
         }
 
         public Builder ipv4(int p00, int p01, int p02, int p03) {
             this.ipv6 = new byte[]{
-                    (byte) 0, (byte) 0, (byte) 0x00, (byte) 0x00,
+                (byte) 0, (byte) 0, (byte) 0x00, (byte) 0x00,
-                    (byte) 0, (byte) 0, (byte) 0x00, (byte) 0x00,
+                (byte) 0, (byte) 0, (byte) 0x00, (byte) 0x00,
-                    (byte) 0, (byte) 0, (byte) 0xff, (byte) 0xff,
+                (byte) 0, (byte) 0, (byte) 0xff, (byte) 0xff,
-                    (byte) p00, (byte) p01, (byte) p02, (byte) p03
+                (byte) p00, (byte) p01, (byte) p02, (byte) p03
             };
             return this;
         }

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/extended/Attachment.java

@@ -0,0 +1,100 @@
+package ch.dissem.bitmessage.entity.valueobject.extended;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A "file" attachment as used by extended encoding type messages. Could either be an attachment,
+ * or used inline to be used by a HTML message, for example.
+ */
+public class Attachment implements Serializable {
+    private static final long serialVersionUID = 7319139427666943189L;
+
+    private String name;
+    private byte[] data;
+    private String type;
+    private Disposition disposition;
+
+    public String getName() {
+        return name;
+    }
+
+    public byte[] getData() {
+        return data;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public Disposition getDisposition() {
+        return disposition;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Attachment that = (Attachment) o;
+        return Objects.equals(name, that.name) &&
+            Arrays.equals(data, that.data) &&
+            Objects.equals(type, that.type) &&
+            disposition == that.disposition;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, data, type, disposition);
+    }
+
+    public enum Disposition {
+        inline, attachment
+    }
+
+    public static final class Builder {
+        private String name;
+        private byte[] data;
+        private String type;
+        private Disposition disposition;
+
+        public Builder name(String name) {
+            this.name = name;
+            return this;
+        }
+
+        public Builder data(byte[] data) {
+            this.data = data;
+            return this;
+        }
+
+        public Builder type(String type) {
+            this.type = type;
+            return this;
+        }
+
+        public Builder inline() {
+            this.disposition = Disposition.inline;
+            return this;
+        }
+
+        public Builder attachment() {
+            this.disposition = Disposition.attachment;
+            return this;
+        }
+
+        public Builder disposition(Disposition disposition) {
+            this.disposition = disposition;
+            return this;
+        }
+
+        public Attachment build() {
+            Attachment attachment = new Attachment();
+            attachment.type = this.type;
+            attachment.disposition = this.disposition;
+            attachment.data = this.data;
+            attachment.name = this.name;
+            return attachment;
+        }
+    }
+}

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/extended/Message.java

@@ -0,0 +1,219 @@
+package ch.dissem.bitmessage.entity.valueobject.extended;
+
+import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
+import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
+import ch.dissem.msgpack.types.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URLConnection;
+import java.nio.file.Files;
+import java.util.*;
+
+import static ch.dissem.bitmessage.entity.valueobject.extended.Attachment.Disposition.attachment;
+import static ch.dissem.bitmessage.utils.Strings.str;
+import static ch.dissem.msgpack.types.Utils.mp;
+
+/**
+ * Extended encoding type 'message'. Properties 'parents' and 'files' not yet supported by PyBitmessage, so they might not work
+ * properly with future PyBitmessage implementations.
+ */
+public class Message implements ExtendedEncoding.ExtendedType {
+    private static final long serialVersionUID = -2724977231484285467L;
+    private static final Logger LOG = LoggerFactory.getLogger(Message.class);
+
+    public static final String TYPE = "message";
+
+    private String subject;
+    private String body;
+    private List<InventoryVector> parents;
+    private List<Attachment> files;
+
+    private Message(Builder builder) {
+        subject = builder.subject;
+        body = builder.body;
+        parents = Collections.unmodifiableList(builder.parents);
+        files = Collections.unmodifiableList(builder.files);
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public String getSubject() {
+        return subject;
+    }
+
+    public String getBody() {
+        return body;
+    }
+
+    public List<InventoryVector> getParents() {
+        return parents;
+    }
+
+    public List<Attachment> getFiles() {
+        return files;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Message message = (Message) o;
+        return Objects.equals(subject, message.subject) &&
+            Objects.equals(body, message.body) &&
+            Objects.equals(parents, message.parents) &&
+            Objects.equals(files, message.files);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(subject, body, parents, files);
+    }
+
+    @Override
+    public MPMap<MPString, MPType<?>> pack() throws IOException {
+        MPMap<MPString, MPType<?>> result = new MPMap<>();
+        result.put(mp(""), mp(TYPE));
+        result.put(mp("subject"), mp(subject));
+        result.put(mp("body"), mp(body));
+
+        if (!files.isEmpty()) {
+            MPArray<MPMap<MPString, MPType<?>>> items = new MPArray<>();
+            result.put(mp("files"), items);
+            for (Attachment file : files) {
+                MPMap<MPString, MPType<?>> item = new MPMap<>();
+                item.put(mp("name"), mp(file.getName()));
+                item.put(mp("data"), mp(file.getData()));
+                item.put(mp("type"), mp(file.getType()));
+                item.put(mp("disposition"), mp(file.getDisposition().name()));
+                items.add(item);
+            }
+        }
+        if (!parents.isEmpty()) {
+            MPArray<MPBinary> items = new MPArray<>();
+            result.put(mp("parents"), items);
+            for (InventoryVector parent : parents) {
+                items.add(mp(parent.getHash()));
+            }
+        }
+        return result;
+    }
+
+    public static class Builder {
+        private String subject;
+        private String body;
+        private List<InventoryVector> parents = new LinkedList<>();
+        private List<Attachment> files = new LinkedList<>();
+
+        public Builder subject(String subject) {
+            this.subject = subject;
+            return this;
+        }
+
+        public Builder body(String body) {
+            this.body = body;
+            return this;
+        }
+
+        public Builder addParent(Plaintext parent) {
+            if (parent != null) {
+                InventoryVector iv = parent.getInventoryVector();
+                if (iv == null) {
+                    LOG.debug("Ignored parent without IV");
+                } else {
+                    parents.add(iv);
+                }
+            }
+            return this;
+        }
+
+        public Builder addParent(InventoryVector iv) {
+            if (iv != null) {
+                parents.add(iv);
+            }
+            return this;
+        }
+
+        public Builder addFile(File file, Attachment.Disposition disposition) {
+            if (file != null) {
+                try {
+                    files.add(new Attachment.Builder()
+                        .name(file.getName())
+                        .disposition(disposition)
+                        .type(URLConnection.guessContentTypeFromStream(new FileInputStream(file)))
+                        .data(Files.readAllBytes(file.toPath()))
+                        .build());
+                } catch (IOException e) {
+                    LOG.error(e.getMessage(), e);
+                }
+            }
+            return this;
+        }
+
+        public Builder addFile(Attachment file) {
+            if (file != null) {
+                files.add(file);
+            }
+            return this;
+        }
+
+        public ExtendedEncoding build() {
+            return new ExtendedEncoding(new Message(this));
+        }
+    }
+
+    public static class Unpacker implements ExtendedEncoding.Unpacker<Message> {
+        @Override
+        public String getType() {
+            return TYPE;
+        }
+
+        @Override
+        public Message unpack(MPMap<MPString, MPType<?>> map) {
+            Message.Builder builder = new Message.Builder();
+            builder.subject(str(map.get(mp("subject"))));
+            builder.body(str(map.get(mp("body"))));
+            @SuppressWarnings("unchecked")
+            MPArray<MPBinary> parents = (MPArray<MPBinary>) map.get(mp("parents"));
+            if (parents != null) {
+                for (MPBinary parent : parents) {
+                    builder.addParent(new InventoryVector(parent.getValue()));
+                }
+            }
+            @SuppressWarnings("unchecked")
+            MPArray<MPMap<MPString, MPType<?>>> files = (MPArray<MPMap<MPString, MPType<?>>>) map.get(mp("files"));
+            if (files != null) {
+                for (MPMap<MPString, MPType<?>> item : files) {
+                    Attachment.Builder b = new Attachment.Builder();
+                    b.name(str(item.get(mp("name"))));
+                    b.data(bin(item.get(mp("data"))));
+                    b.type(str(item.get(mp("type"))));
+                    String disposition = str(item.get(mp("disposition")));
+                    if ("inline".equals(disposition)) {
+                        b.inline();
+                    } else if ("attachment".equals(disposition)) {
+                        b.attachment();
+                    }
+                    builder.addFile(b.build());
+                }
+            }
+
+            return new Message(builder);
+        }
+
+        private byte[] bin(MPType data) {
+            if (data instanceof MPBinary) {
+                return ((MPBinary) data).getValue();
+            } else {
+                return null;
+            }
+        }
+    }
+}

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/extended/Vote.java

@@ -0,0 +1,114 @@
+package ch.dissem.bitmessage.entity.valueobject.extended;
+
+import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
+import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
+import ch.dissem.msgpack.types.*;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static ch.dissem.bitmessage.utils.Strings.str;
+import static ch.dissem.msgpack.types.Utils.mp;
+
+/**
+ * Extended encoding type 'vote'. Specification still outstanding, so this will need some work.
+ */
+public class Vote implements ExtendedEncoding.ExtendedType {
+    private static final long serialVersionUID = -8427038604209964837L;
+
+    public static final String TYPE = "vote";
+
+    private InventoryVector msgId;
+    private String vote;
+
+    private Vote(Builder builder) {
+        msgId = builder.msgId;
+        vote = builder.vote;
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public InventoryVector getMsgId() {
+        return msgId;
+    }
+
+    public String getVote() {
+        return vote;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Vote vote1 = (Vote) o;
+        return Objects.equals(msgId, vote1.msgId) &&
+            Objects.equals(vote, vote1.vote);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(msgId, vote);
+    }
+
+    @Override
+    public MPMap<MPString, MPType<?>> pack() throws IOException {
+        MPMap<MPString, MPType<?>> result = new MPMap<>();
+        result.put(mp(""), mp(TYPE));
+        result.put(mp("msgId"), mp(msgId.getHash()));
+        result.put(mp("vote"), mp(vote));
+        return result;
+    }
+
+    public static class Builder {
+        private InventoryVector msgId;
+        private String vote;
+
+        public ExtendedEncoding up(Plaintext message) {
+            msgId = message.getInventoryVector();
+            vote = "1";
+            return new ExtendedEncoding(new Vote(this));
+        }
+
+        public ExtendedEncoding down(Plaintext message) {
+            msgId = message.getInventoryVector();
+            vote = "1";
+            return new ExtendedEncoding(new Vote(this));
+        }
+
+        public Builder msgId(InventoryVector iv) {
+            this.msgId = iv;
+            return this;
+        }
+
+        public Builder vote(String vote) {
+            this.vote = vote;
+            return this;
+        }
+
+        public ExtendedEncoding build() {
+            return new ExtendedEncoding(new Vote(this));
+        }
+    }
+
+    public static class Unpacker implements ExtendedEncoding.Unpacker<Vote> {
+        @Override
+        public String getType() {
+            return TYPE;
+        }
+
+        @Override
+        public Vote unpack(MPMap<MPString, MPType<?>> map) {
+            Vote.Builder builder = new Vote.Builder();
+            MPType<?> msgId = map.get(mp("msgId"));
+            if (msgId instanceof MPBinary) {
+                builder.msgId(new InventoryVector(((MPBinary) msgId).getValue()));
+            }
+            builder.vote(str(map.get(mp("vote"))));
+            return new Vote(builder);
+        }
+    }
+}

core/src/main/java/ch/dissem/bitmessage/factory/ExtendedEncodingFactory.java

@@ -0,0 +1,62 @@
+package ch.dissem.bitmessage.factory;
+
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
+import ch.dissem.bitmessage.entity.valueobject.extended.Message;
+import ch.dissem.bitmessage.entity.valueobject.extended.Vote;
+import ch.dissem.bitmessage.exception.ApplicationException;
+import ch.dissem.msgpack.Reader;
+import ch.dissem.msgpack.types.MPMap;
+import ch.dissem.msgpack.types.MPString;
+import ch.dissem.msgpack.types.MPType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.zip.InflaterInputStream;
+
+import static ch.dissem.bitmessage.utils.Strings.str;
+
+/**
+ * Factory that creates {@link ExtendedEncoding} objects from byte arrays. You can register your own types by adding a
+ * {@link ExtendedEncoding.Unpacker} using {@link #registerFactory(ExtendedEncoding.Unpacker)}.
+ */
+public class ExtendedEncodingFactory {
+    private static final Logger LOG = LoggerFactory.getLogger(ExtendedEncodingFactory.class);
+    private static final ExtendedEncodingFactory INSTANCE = new ExtendedEncodingFactory();
+    private static final MPString KEY_MESSAGE_TYPE = new MPString("");
+    private Map<String, ExtendedEncoding.Unpacker<?>> factories = new HashMap<>();
+
+    private ExtendedEncodingFactory() {
+        registerFactory(new Message.Unpacker());
+        registerFactory(new Vote.Unpacker());
+    }
+
+    public void registerFactory(ExtendedEncoding.Unpacker<?> factory) {
+        factories.put(factory.getType(), factory);
+    }
+
+
+    public ExtendedEncoding unzip(byte[] zippedData) {
+        try (InflaterInputStream unzipper = new InflaterInputStream(new ByteArrayInputStream(zippedData))) {
+            Reader reader = Reader.getInstance();
+            @SuppressWarnings("unchecked")
+            MPMap<MPString, MPType<?>> map = (MPMap<MPString, MPType<?>>) reader.read(unzipper);
+            MPType<?> messageType = map.get(KEY_MESSAGE_TYPE);
+            if (messageType == null) {
+                LOG.error("Missing message type");
+                return null;
+            }
+            ExtendedEncoding.Unpacker<?> factory = factories.get(str(messageType));
+            return new ExtendedEncoding(factory.unpack(map));
+        } catch (ClassCastException | IOException e) {
+            throw new ApplicationException(e);
+        }
+    }
+
+    public static ExtendedEncodingFactory getInstance() {
+        return INSTANCE;
+    }
+}

core/src/main/java/ch/dissem/bitmessage/factory/V3MessageFactory.java

@@ -24,6 +24,7 @@ import ch.dissem.bitmessage.entity.valueobject.NetworkAddress;
 import ch.dissem.bitmessage.exception.NodeException;
 import ch.dissem.bitmessage.utils.AccessCounter;
 import ch.dissem.bitmessage.utils.Decode;
+import ch.dissem.bitmessage.utils.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -103,6 +104,7 @@ class V3MessageFactory {
             payload = Factory.getObjectPayload(objectType, version, stream, dataStream, data.length);
         } catch (Exception e) {
             LOG.trace("Could not parse object payload - using generic payload instead", e);
+            LOG.info(Strings.hex(data).toString());
             payload = new GenericPayload(version, stream, data);
         }
 

core/src/main/java/ch/dissem/bitmessage/ports/AbstractCryptography.java

@@ -31,10 +31,7 @@ import javax.crypto.Mac;
 import javax.crypto.spec.SecretKeySpec;
 import java.io.IOException;
 import java.math.BigInteger;
-import java.security.GeneralSecurityException;
+import java.security.*;
-import java.security.MessageDigest;
-import java.security.Provider;
-import java.security.SecureRandom;
 
 import static ch.dissem.bitmessage.InternalContext.NETWORK_EXTRA_BYTES;
 import static ch.dissem.bitmessage.InternalContext.NETWORK_NONCE_TRIALS_PER_BYTE;
@@ -50,6 +47,10 @@ public abstract class AbstractCryptography implements Cryptography, InternalCont
     private static final BigInteger TWO_POW_64 = TWO.pow(64);
     private static final BigInteger TWO_POW_16 = TWO.pow(16);
 
+    protected static final String ALGORITHM_ECDSA = "ECDSA";
+    protected static final String ALGORITHM_ECDSA_SHA1 = "SHA1withECDSA";
+    protected static final String ALGORITHM_EVP_SHA256 = "SHA256withECDSA";
+
     protected final Provider provider;
     private InternalContext context;
 
@@ -127,6 +128,27 @@ public abstract class AbstractCryptography implements Cryptography, InternalCont
         }
     }
 
+    protected byte[] doSign(byte[] data, java.security.PrivateKey privKey) throws GeneralSecurityException {
+        // TODO: change this to ALGORITHM_EVP_SHA256 once it's generally used in the network
+        Signature sig = Signature.getInstance(ALGORITHM_ECDSA_SHA1, provider);
+        sig.initSign(privKey);
+        sig.update(data);
+        return sig.sign();
+    }
+
+
+    protected boolean doCheckSignature(byte[] data, byte[] signature, PublicKey publicKey) throws GeneralSecurityException {
+        for (String algorithm : new String[]{ALGORITHM_ECDSA_SHA1, ALGORITHM_EVP_SHA256}) {
+            Signature sig = Signature.getInstance(algorithm, provider);
+            sig.initVerify(publicKey);
+            sig.update(data);
+            if (sig.verify(signature)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override
     public byte[] getInitialHash(ObjectMessage object) {
         return sha512(object.getPayloadBytesWithoutNonce());

core/src/main/java/ch/dissem/bitmessage/ports/AbstractMessageRepository.java

@@ -19,13 +19,16 @@ package ch.dissem.bitmessage.ports;
 import ch.dissem.bitmessage.InternalContext;
 import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.exception.ApplicationException;
 import ch.dissem.bitmessage.utils.Strings;
 import ch.dissem.bitmessage.utils.UnixTime;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.UUID;
 
 import static ch.dissem.bitmessage.utils.SqlStrings.join;
 
@@ -37,15 +40,28 @@ public abstract class AbstractMessageRepository implements MessageRepository, In
         this.ctx = context;
     }
 
+    /**
+     * @deprecated use {@link #saveContactIfNecessary(BitmessageAddress)} instead.
+     */
+    @Deprecated
     protected void safeSenderIfNecessary(Plaintext message) {
         if (message.getId() == null) {
-            BitmessageAddress savedAddress = ctx.getAddressRepository().getAddress(message.getFrom().getAddress());
+            saveContactIfNecessary(message.getFrom());
+        }
+    }
+
+    protected void saveContactIfNecessary(BitmessageAddress contact) {
+        if (contact != null) {
+            BitmessageAddress savedAddress = ctx.getAddressRepository().getAddress(contact.getAddress());
             if (savedAddress == null) {
-                ctx.getAddressRepository().save(message.getFrom());
+                ctx.getAddressRepository().save(contact);
-            } else if (savedAddress.getPubkey() == null && message.getFrom().getPubkey() != null) {
+            } else if (savedAddress.getPubkey() == null && contact.getPubkey() != null) {
-                savedAddress.setPubkey(message.getFrom().getPubkey());
+                savedAddress.setPubkey(contact.getPubkey());
                 ctx.getAddressRepository().save(savedAddress);
             }
+            if (savedAddress != null) {
+                contact.setAlias(savedAddress.getAlias());
+            }
         }
     }
 
@@ -58,6 +74,11 @@ public abstract class AbstractMessageRepository implements MessageRepository, In
         }
     }
 
+    @Override
+    public Plaintext getMessage(InventoryVector iv) {
+        return single(find("iv=X'" + Strings.hex(iv.getHash()) + "'"));
+    }
+
     @Override
     public Plaintext getMessage(byte[] initialHash) {
         return single(find("initial_hash=X'" + Strings.hex(initialHash) + "'"));
@@ -95,7 +116,21 @@ public abstract class AbstractMessageRepository implements MessageRepository, In
     @Override
     public List<Plaintext> findMessagesToResend() {
         return find("status='" + Plaintext.Status.SENT.name() + "'" +
-                " AND next_try < " + UnixTime.now());
+            " AND next_try < " + UnixTime.now());
+    }
+
+    @Override
+    public List<Plaintext> findResponses(Plaintext parent) {
+        if (parent.getInventoryVector() == null) {
+            return Collections.emptyList();
+        }
+        return find("iv IN (SELECT child FROM Message_Parent"
+            + " WHERE parent=X'" + Strings.hex(parent.getInventoryVector().getHash()) + "')");
+    }
+
+    @Override
+    public List<Plaintext> getConversation(UUID conversationId) {
+        return find("conversation=X'" + conversationId.toString().replace("-", "") + "'");
     }
 
     @Override
@@ -119,7 +154,7 @@ public abstract class AbstractMessageRepository implements MessageRepository, In
                 return collection.iterator().next();
             default:
                 throw new ApplicationException("This shouldn't happen, found " + collection.size() +
-                        " items, one or none was expected");
+                    " items, one or none was expected");
         }
     }
 

core/src/main/java/ch/dissem/bitmessage/ports/AddressRepository.java

@@ -52,6 +52,11 @@ public interface AddressRepository {
      */
     List<BitmessageAddress> getContacts();
 
+    /**
+     * Implementations must not delete cryptographic keys if they're not provided by <code>address</code>.
+     *
+     * @param address to save or update
+     */
     void save(BitmessageAddress address);
 
     void remove(BitmessageAddress address);

core/src/main/java/ch/dissem/bitmessage/ports/MessageRepository.java

@@ -19,9 +19,12 @@ package ch.dissem.bitmessage.ports;
 import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.entity.Plaintext.Status;
+import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 
+import java.util.Collection;
 import java.util.List;
+import java.util.UUID;
 
 public interface MessageRepository {
     List<Label> getLabels();
@@ -32,10 +35,18 @@ public interface MessageRepository {
 
     Plaintext getMessage(Object id);
 
+    Plaintext getMessage(InventoryVector iv);
+
     Plaintext getMessage(byte[] initialHash);
 
     Plaintext getMessageForAck(byte[] ackData);
 
+    /**
+     * @param label to search for
+     * @return a distinct list of all conversations that have at least one message with the given label.
+     */
+    List<UUID> findConversations(Label label);
+
     List<Plaintext> findMessages(Label label);
 
     List<Plaintext> findMessages(Status status);
@@ -44,9 +55,21 @@ public interface MessageRepository {
 
     List<Plaintext> findMessages(BitmessageAddress sender);
 
+    List<Plaintext> findResponses(Plaintext parent);
+
     List<Plaintext> findMessagesToResend();
 
     void save(Plaintext message);
 
     void remove(Plaintext message);
+
+    /**
+     * Returns all messages with this conversation ID. The returned messages aren't sorted in any way,
+     * so you may prefer to use {@link ch.dissem.bitmessage.utils.ConversationService#getConversation(UUID)}
+     * instead.
+     *
+     * @param conversationId ID of the requested conversation
+     * @return all messages with the given conversation ID
+     */
+    Collection<Plaintext> getConversation(UUID conversationId);
 }

core/src/main/java/ch/dissem/bitmessage/ports/NodeRegistry.java

@@ -24,6 +24,12 @@ import java.util.List;
  * Stores and provides known peers.
  */
 public interface NodeRegistry {
+    /**
+     * Removes all known nodes from registry. This should work around connection issues
+     * when there are many invalid nodes in the registry.
+     */
+    void clear();
+
     List<NetworkAddress> getKnownAddresses(int limit, long... streams);
 
     void offerAddresses(List<NetworkAddress> addresses);

core/src/main/java/ch/dissem/bitmessage/utils/ConversationService.java

@@ -0,0 +1,142 @@
+/*
+ * Copyright 2017 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.utils;
+
+import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
+import ch.dissem.bitmessage.ports.MessageRepository;
+
+import java.util.*;
+import java.util.Collections;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
+
+/**
+ * Service that helps with conversations.
+ */
+public class ConversationService {
+    private final MessageRepository messageRepository;
+
+    private final Pattern SUBJECT_PREFIX = Pattern.compile("^(re|fwd?):", CASE_INSENSITIVE);
+
+    public ConversationService(MessageRepository messageRepository) {
+        this.messageRepository = messageRepository;
+    }
+
+    /**
+     * Retrieve the whole conversation from one single message. If the message isn't part
+     * of a conversation, a singleton list containing the given message is returned. Otherwise
+     * it's the same as {@link #getConversation(UUID)}
+     *
+     * @param message
+     * @return a list of messages that belong to the same conversation.
+     */
+    public List<Plaintext> getConversation(Plaintext message) {
+        if (message.getConversationId() == null) {
+            return Collections.singletonList(message);
+        }
+        return getConversation(message.getConversationId());
+    }
+
+    private LinkedList<Plaintext> sorted(Collection<Plaintext> collection) {
+        LinkedList<Plaintext> result = new LinkedList<>(collection);
+        Collections.sort(result, new Comparator<Plaintext>() {
+            @Override
+            public int compare(Plaintext o1, Plaintext o2) {
+                //noinspection NumberEquality - if both are null (if both are the same, it's a bonus)
+                if (o1.getReceived() == o2.getReceived()) {
+                    return 0;
+                }
+                if (o1.getReceived() == null) {
+                    return -1;
+                }
+                if (o2.getReceived() == null) {
+                    return 1;
+                }
+                return -o1.getReceived().compareTo(o2.getReceived());
+            }
+        });
+        return result;
+    }
+
+    public List<Plaintext> getConversation(UUID conversationId) {
+        LinkedList<Plaintext> messages = sorted(messageRepository.getConversation(conversationId));
+        Map<InventoryVector, Plaintext> map = new HashMap<>(messages.size());
+        for (Plaintext message : messages) {
+            if (message.getInventoryVector() != null) {
+                map.put(message.getInventoryVector(), message);
+            }
+        }
+
+        LinkedList<Plaintext> result = new LinkedList<>();
+        while (!messages.isEmpty()) {
+            Plaintext last = messages.poll();
+            int pos = lastParentPosition(last, result);
+            result.add(pos, last);
+            addAncestors(last, result, messages, map);
+        }
+        return result;
+    }
+
+    public String getSubject(List<Plaintext> conversation) {
+        if (conversation.isEmpty()) {
+            return null;
+        }
+        // TODO: this has room for improvement
+        String subject = conversation.get(0).getSubject();
+        Matcher matcher = SUBJECT_PREFIX.matcher(subject);
+        if (matcher.find()) {
+            return subject.substring(matcher.end()).trim();
+        }
+        return subject.trim();
+    }
+
+    private int lastParentPosition(Plaintext child, LinkedList<Plaintext> messages) {
+        Iterator<Plaintext> plaintextIterator = messages.descendingIterator();
+        int i = 0;
+        while (plaintextIterator.hasNext()) {
+            Plaintext next = plaintextIterator.next();
+            if (isParent(next, child)) {
+                break;
+            }
+            i++;
+        }
+        return messages.size() - i;
+    }
+
+    private boolean isParent(Plaintext item, Plaintext child) {
+        for (InventoryVector parentId : child.getParents()) {
+            if (parentId.equals(item.getInventoryVector())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private void addAncestors(Plaintext message, LinkedList<Plaintext> result, LinkedList<Plaintext> messages, Map<InventoryVector, Plaintext> map) {
+        for (InventoryVector parentKey : message.getParents()) {
+            Plaintext parent = map.remove(parentKey);
+            if (parent != null) {
+                messages.remove(parent);
+                result.addFirst(parent);
+                addAncestors(parent, result, messages, map);
+            }
+        }
+    }
+}

core/src/main/java/ch/dissem/bitmessage/utils/Decode.java

@@ -138,7 +138,7 @@ public class Decode {
 
     public static String varString(InputStream in, AccessCounter counter) throws IOException {
         int length = (int) varInt(in, counter);
-        // FIXME: technically, it says the length in characters, but I think this one might be correct
+        // technically, it says the length in characters, but I think this one might be correct
         // otherwise it will get complicated, as we'll need to read UTF-8 char by char...
         return new String(bytes(in, length, counter), "utf-8");
     }

core/src/main/java/ch/dissem/bitmessage/utils/Strings.java

@@ -37,4 +37,8 @@ public class Strings {
         }
         return hex;
     }
+
+    public static String str(Object o) {
+        return o == null ? null : o.toString();
+    }
 }

core/src/main/java/ch/dissem/bitmessage/utils/TTL.java

@@ -3,9 +3,8 @@ package ch.dissem.bitmessage.utils;
 import static ch.dissem.bitmessage.utils.UnixTime.DAY;
 
 /**
- * Stores times to live for different object types. Usually this shouldn't be messed with,
+ * Stores times to live in seconds for different object types. Usually this shouldn't be messed with, but for tests
- * but for tests it might be a good idea to reduce it to a minimum, and on mobile clients
+ * it might be a good idea to reduce it to a minimum, and on mobile clients you might want to optimize it as well.
- * you might want to optimize it as well.
  *
  * @author Christian Basler
  */
@@ -19,7 +18,7 @@ public class TTL {
     }
 
     public static void msg(long msg) {
-        TTL.msg = msg;
+        TTL.msg = validate(msg);
     }
 
     public static long getpubkey() {
@@ -27,7 +26,7 @@ public class TTL {
     }
 
     public static void getpubkey(long getpubkey) {
-        TTL.getpubkey = getpubkey;
+        TTL.getpubkey = validate(getpubkey);
     }
 
     public static long pubkey() {
@@ -35,6 +34,11 @@ public class TTL {
     }
 
     public static void pubkey(long pubkey) {
-        TTL.pubkey = pubkey;
+        TTL.pubkey = validate(pubkey);
+    }
+
+    private static long validate(long ttl) {
+        if (ttl < 0 || ttl > 28 * DAY) throw new IllegalArgumentException("TTL must be between 0 seconds and 28 days");
+        return ttl;
     }
 }

core/src/test/java/ch/dissem/bitmessage/BitmessageContextTest.java

@@ -25,12 +25,11 @@ import ch.dissem.bitmessage.entity.payload.ObjectType;
 import ch.dissem.bitmessage.entity.payload.Pubkey;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.ports.*;
+import ch.dissem.bitmessage.testutils.TestInventory;
 import ch.dissem.bitmessage.utils.MessageMatchers;
 import ch.dissem.bitmessage.utils.Singleton;
 import ch.dissem.bitmessage.utils.TTL;
 import ch.dissem.bitmessage.utils.TestUtils;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -51,70 +50,74 @@ import static org.mockito.Mockito.*;
 public class BitmessageContextTest {
     private BitmessageContext ctx;
     private BitmessageContext.Listener listener;
+    private TestInventory testInventory;
 
     @Before
     public void setUp() throws Exception {
         Singleton.initialize(null);
         listener = mock(BitmessageContext.Listener.class);
+        Singleton.initialize(new BouncyCryptography());
+        testInventory = new TestInventory();
         ctx = new BitmessageContext.Builder()
-                .addressRepo(mock(AddressRepository.class))
+            .addressRepo(mock(AddressRepository.class))
-                .cryptography(new BouncyCryptography())
+            .cryptography(cryptography())
-                .inventory(mock(Inventory.class))
+            .inventory(spy(testInventory))
-                .listener(listener)
+            .listener(listener)
-                .messageRepo(mock(MessageRepository.class))
+            .messageRepo(mock(MessageRepository.class))
-                .networkHandler(mock(NetworkHandler.class))
+            .networkHandler(mock(NetworkHandler.class))
-                .nodeRegistry(mock(NodeRegistry.class))
+            .nodeRegistry(mock(NodeRegistry.class))
-                .labeler(spy(new DefaultLabeler()))
+            .labeler(spy(new DefaultLabeler()))
-                .powRepo(spy(new ProofOfWorkRepository() {
+            .powRepo(spy(new ProofOfWorkRepository() {
-                    Map<InventoryVector, Item> items = new HashMap<>();
+                Map<InventoryVector, Item> items = new HashMap<>();
 
-                    @Override
+                @Override
-                    public Item getItem(byte[] initialHash) {
+                public Item getItem(byte[] initialHash) {
-                        return items.get(new InventoryVector(initialHash));
+                    return items.get(new InventoryVector(initialHash));
-                    }
+                }
 
-                    @Override
+                @Override
-                    public List<byte[]> getItems() {
+                public List<byte[]> getItems() {
-                        List<byte[]> result = new LinkedList<>();
+                    List<byte[]> result = new LinkedList<>();
-                        for (InventoryVector iv : items.keySet()) {
+                    for (InventoryVector iv : items.keySet()) {
-                            result.add(iv.getHash());
+                        result.add(iv.getHash());
-                        }
-                        return result;
                     }
+                    return result;
+                }
 
-                    @Override
+                @Override
-                    public void putObject(Item item) {
+                public void putObject(Item item) {
-                        items.put(new InventoryVector(cryptography().getInitialHash(item.object)), item);
+                    items.put(new InventoryVector(cryptography().getInitialHash(item.object)), item);
-                    }
+                }
 
-                    @Override
+                @Override
-                    public void putObject(ObjectMessage object, long nonceTrialsPerByte, long extraBytes) {
+                public void putObject(ObjectMessage object, long nonceTrialsPerByte, long extraBytes) {
-                        items.put(new InventoryVector(cryptography().getInitialHash(object)), new Item(object, nonceTrialsPerByte, extraBytes));
+                    items.put(new InventoryVector(cryptography().getInitialHash(object)), new Item(object, nonceTrialsPerByte, extraBytes));
-                    }
+                }
 
-                    @Override
+                @Override
-                    public void removeObject(byte[] initialHash) {
+                public void removeObject(byte[] initialHash) {
-                        items.remove(initialHash);
+                    items.remove(initialHash);
-                    }
+                }
-                }))
+            }))
-                .proofOfWorkEngine(spy(new ProofOfWorkEngine() {
+            .proofOfWorkEngine(spy(new ProofOfWorkEngine() {
-                    @Override
+                @Override
-                    public void calculateNonce(byte[] initialHash, byte[] target, Callback callback) {
+                public void calculateNonce(byte[] initialHash, byte[] target, Callback callback) {
-                        callback.onNonceCalculated(initialHash, new byte[8]);
+                    callback.onNonceCalculated(initialHash, new byte[8]);
-                    }
+                }
-                }))
+            }))
-                .build();
+            .build();
         TTL.msg(2 * MINUTE);
     }
 
     @Test
     public void ensureContactIsSavedAndPubkeyRequested() {
         BitmessageAddress contact = new BitmessageAddress("BM-opWQhvk9xtMFvQA2Kvetedpk8LkbraWHT");
+        when(ctx.addresses().getAddress(contact.getAddress())).thenReturn(contact);
         ctx.addContact(contact);
 
-        verify(ctx.addresses(), times(2)).save(contact);
+        verify(ctx.addresses(), timeout(1000).atLeastOnce()).save(contact);
-        verify(ctx.internals().getProofOfWorkEngine())
+        verify(ctx.internals().getProofOfWorkEngine(), timeout(1000))
-                .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
+            .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
     }
 
     @Test
@@ -128,65 +131,68 @@ public class BitmessageContextTest {
 
         verify(ctx.addresses(), times(1)).save(contact);
         verify(ctx.internals().getProofOfWorkEngine(), never())
-                .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
+            .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
     }
 
     @Test
     public void ensureV2PubkeyIsNotRequestedIfItExistsInInventory() throws Exception {
+        testInventory.init(
+            "V1Msg.payload",
+            "V2GetPubkey.payload",
+            "V2Pubkey.payload",
+            "V3GetPubkey.payload",
+            "V3Pubkey.payload",
+            "V4Broadcast.payload",
+            "V4GetPubkey.payload",
+            "V4Pubkey.payload",
+            "V5Broadcast.payload"
+        );
         BitmessageAddress contact = new BitmessageAddress("BM-opWQhvk9xtMFvQA2Kvetedpk8LkbraWHT");
-        when(ctx.internals().getInventory().getObjects(anyLong(), anyLong(), any(ObjectType.class)))
+
-                .thenReturn(Collections.singletonList(
+        when(ctx.addresses().getAddress(contact.getAddress())).thenReturn(contact);
-                        TestUtils.loadObjectMessage(2, "V2Pubkey.payload")
-                ));
 
         ctx.addContact(contact);
 
         verify(ctx.addresses(), atLeastOnce()).save(contact);
         verify(ctx.internals().getProofOfWorkEngine(), never())
-                .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
+            .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
     }
 
     @Test
     public void ensureV4PubkeyIsNotRequestedIfItExistsInInventory() throws Exception {
+        testInventory.init(
+            "V1Msg.payload",
+            "V2GetPubkey.payload",
+            "V2Pubkey.payload",
+            "V3GetPubkey.payload",
+            "V3Pubkey.payload",
+            "V4Broadcast.payload",
+            "V4GetPubkey.payload",
+            "V4Pubkey.payload",
+            "V5Broadcast.payload"
+        );
         BitmessageAddress contact = new BitmessageAddress("BM-2cXxfcSetKnbHJX2Y85rSkaVpsdNUZ5q9h");
-        when(ctx.internals().getInventory().getObjects(anyLong(), anyLong(), any(ObjectType.class)))
-                .thenReturn(Collections.singletonList(
-                        TestUtils.loadObjectMessage(2, "V4Pubkey.payload")
-                ));
         final BitmessageAddress stored = new BitmessageAddress(contact.getAddress());
         stored.setAlias("Test");
         when(ctx.addresses().getAddress(contact.getAddress())).thenReturn(stored);
 
         ctx.addContact(contact);
 
-        verify(ctx.addresses(), atLeastOnce()).save(argThat(new BaseMatcher<BitmessageAddress>() {
+        verify(ctx.addresses(), atLeastOnce()).save(any(BitmessageAddress.class));
-            @Override
-            public boolean matches(Object item) {
-                return item instanceof BitmessageAddress
-                        && ((BitmessageAddress) item).getPubkey() != null
-                        && stored.getAlias().equals(((BitmessageAddress) item).getAlias());
-            }
-
-            @Override
-            public void describeTo(Description description) {
-                description.appendText("pubkey must not be null and alias must be ").appendValue(stored.getAlias());
-            }
-        }));
         verify(ctx.internals().getProofOfWorkEngine(), never())
-                .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
+            .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
     }
 
     @Test
     public void ensureSubscriptionIsAddedAndExistingBroadcastsRetrieved() throws Exception {
         BitmessageAddress address = new BitmessageAddress("BM-2D9Vc5rFxxR5vTi53T9gkLfemViHRMVLQZ");
 
-        List<ObjectMessage> objects = new LinkedList<>();
+        testInventory.init(
-        objects.add(TestUtils.loadObjectMessage(4, "V4Broadcast.payload"));
+            "V4Broadcast.payload",
-        objects.add(TestUtils.loadObjectMessage(5, "V5Broadcast.payload"));
+            "V5Broadcast.payload"
-        when(ctx.internals().getInventory().getObjects(eq(address.getStream()), anyLong(), any(ObjectType.class)))
+        );
-                .thenReturn(objects);
-        when(ctx.addresses().getSubscriptions(anyLong())).thenReturn(Collections.singletonList(address));
 
+        when(ctx.addresses().getSubscriptions(anyLong())).thenReturn(Collections.singletonList(address));
         ctx.addSubscribtion(address);
 
         verify(ctx.addresses(), atLeastOnce()).save(address);
@@ -203,48 +209,48 @@ public class BitmessageContextTest {
     @Test
     public void ensureMessageIsSent() throws Exception {
         ctx.send(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"), TestUtils.loadContact(),
-                "Subject", "Message");
+            "Subject", "Message");
         assertEquals(2, ctx.internals().getProofOfWorkRepository().getItems().size());
         verify(ctx.internals().getProofOfWorkRepository(), timeout(10000).atLeastOnce())
-                .putObject(object(MSG), eq(1000L), eq(1000L));
+            .putObject(object(MSG), eq(1000L), eq(1000L));
         verify(ctx.messages(), timeout(10000).atLeastOnce()).save(MessageMatchers.plaintext(Type.MSG));
     }
 
     @Test
     public void ensurePubkeyIsRequestedIfItIsMissing() throws Exception {
         ctx.send(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"),
-                new BitmessageAddress("BM-opWQhvk9xtMFvQA2Kvetedpk8LkbraWHT"),
+            new BitmessageAddress("BM-opWQhvk9xtMFvQA2Kvetedpk8LkbraWHT"),
-                "Subject", "Message");
+            "Subject", "Message");
         verify(ctx.internals().getProofOfWorkRepository(), timeout(10000).atLeastOnce())
-                .putObject(object(GET_PUBKEY), eq(1000L), eq(1000L));
+            .putObject(object(GET_PUBKEY), eq(1000L), eq(1000L));
         verify(ctx.messages(), timeout(10000).atLeastOnce()).save(MessageMatchers.plaintext(Type.MSG));
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void ensureSenderMustBeIdentity() {
         ctx.send(new BitmessageAddress("BM-opWQhvk9xtMFvQA2Kvetedpk8LkbraWHT"),
-                new BitmessageAddress("BM-opWQhvk9xtMFvQA2Kvetedpk8LkbraWHT"),
+            new BitmessageAddress("BM-opWQhvk9xtMFvQA2Kvetedpk8LkbraWHT"),
-                "Subject", "Message");
+            "Subject", "Message");
     }
 
     @Test
     public void ensureBroadcastIsSent() throws Exception {
         ctx.broadcast(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"),
-                "Subject", "Message");
+            "Subject", "Message");
         verify(ctx.internals().getProofOfWorkRepository(), timeout(10000).atLeastOnce())
-                .putObject(object(BROADCAST), eq(1000L), eq(1000L));
+            .putObject(object(BROADCAST), eq(1000L), eq(1000L));
         verify(ctx.internals().getProofOfWorkEngine())
-                .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
+            .calculateNonce(any(byte[].class), any(byte[].class), any(ProofOfWorkEngine.Callback.class));
         verify(ctx.messages(), timeout(10000).atLeastOnce())
-                .save(MessageMatchers.plaintext(Type.BROADCAST));
+            .save(MessageMatchers.plaintext(Type.BROADCAST));
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void ensureSenderWithoutPrivateKeyThrowsException() {
         Plaintext msg = new Plaintext.Builder(Type.BROADCAST)
-                .from(new BitmessageAddress("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .from(new BitmessageAddress("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .message("Subject", "Message")
+            .message("Subject", "Message")
-                .build();
+            .build();
         ctx.send(msg);
     }
 
@@ -301,11 +307,11 @@ public class BitmessageContextTest {
     @Test
     public void ensureUnacknowledgedMessageIsResent() throws Exception {
         Plaintext plaintext = new Plaintext.Builder(Type.MSG)
-                .ttl(1)
+            .ttl(1)
-                .message("subject", "message")
+            .message("subject", "message")
-                .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .to(TestUtils.loadContact())
+            .to(TestUtils.loadContact())
-                .build();
+            .build();
         assertTrue(plaintext.getTo().has(Pubkey.Feature.DOES_ACK));
         when(ctx.messages().findMessagesToResend()).thenReturn(Collections.singletonList(plaintext));
         when(ctx.messages().getMessage(any(byte[].class))).thenReturn(plaintext);

core/src/test/java/ch/dissem/bitmessage/entity/SerializationTest.java

@@ -19,6 +19,7 @@ package ch.dissem.bitmessage.entity;
 import ch.dissem.bitmessage.entity.payload.*;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
+import ch.dissem.bitmessage.entity.valueobject.extended.Message;
 import ch.dissem.bitmessage.factory.Factory;
 import ch.dissem.bitmessage.utils.TestBase;
 import ch.dissem.bitmessage.utils.TestUtils;
@@ -30,7 +31,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 
 import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
-import static ch.dissem.bitmessage.utils.Singleton.cryptography;
+import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.*;
 
 public class SerializationTest extends TestBase {
@@ -78,57 +79,82 @@ public class SerializationTest extends TestBase {
 
     @Test
     public void ensurePlaintextIsSerializedAndDeserializedCorrectly() throws Exception {
-        Plaintext p1 = new Plaintext.Builder(MSG)
+        Plaintext expected = new Plaintext.Builder(MSG)
-                .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .to(TestUtils.loadContact())
+            .to(TestUtils.loadContact())
-                .message("Subject", "Message")
+            .message("Subject", "Message")
-                .ackData("ackMessage".getBytes())
+            .ackData("ackMessage".getBytes())
-                .signature(new byte[0])
+            .signature(new byte[0])
-                .build();
+            .build();
         ByteArrayOutputStream out = new ByteArrayOutputStream();
-        p1.write(out);
+        expected.write(out);
         ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Plaintext p2 = Plaintext.read(MSG, in);
+        Plaintext actual = Plaintext.read(MSG, in);
 
-        // Received is automatically set on deserialization, so we'll need to set it to 0
+        // Received is automatically set on deserialization, so we'll need to set it to null
         Field received = Plaintext.class.getDeclaredField("received");
         received.setAccessible(true);
-        received.set(p2, 0L);
+        received.set(actual, null);
 
-        assertEquals(p1, p2);
+        assertThat(expected, is(actual));
+    }
+
+    @Test
+    public void ensurePlaintextWithExtendedEncodingIsSerializedAndDeserializedCorrectly() throws Exception {
+        Plaintext expected = new Plaintext.Builder(MSG)
+            .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .to(TestUtils.loadContact())
+            .message(new Message.Builder()
+                .subject("Subject")
+                .body("Message")
+                .build())
+            .ackData("ackMessage".getBytes())
+            .signature(new byte[0])
+            .build();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        expected.write(out);
+        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+        Plaintext actual = Plaintext.read(MSG, in);
+
+        // Received is automatically set on deserialization, so we'll need to set it to null
+        Field received = Plaintext.class.getDeclaredField("received");
+        received.setAccessible(true);
+        received.set(actual, null);
+
+        assertEquals(expected, actual);
     }
 
     @Test
     public void ensurePlaintextWithAckMessageIsSerializedAndDeserializedCorrectly() throws Exception {
-        Plaintext p1 = new Plaintext.Builder(MSG)
+        Plaintext expected = new Plaintext.Builder(MSG)
-                .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .to(TestUtils.loadContact())
+            .to(TestUtils.loadContact())
-                .message("Subject", "Message")
+            .message("Subject", "Message")
-                .ackData("ackMessage".getBytes())
+            .ackData("ackMessage".getBytes())
-                .signature(new byte[0])
+            .signature(new byte[0])
-                .build();
+            .build();
-        ObjectMessage ackMessage1 = p1.getAckMessage();
+        ObjectMessage ackMessage1 = expected.getAckMessage();
         assertNotNull(ackMessage1);
 
         ByteArrayOutputStream out = new ByteArrayOutputStream();
-        p1.write(out);
+        expected.write(out);
         ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Plaintext p2 = Plaintext.read(MSG, in);
+        Plaintext actual = Plaintext.read(MSG, in);
 
-        // Received is automatically set on deserialization, so we'll need to set it to 0
+        // Received is automatically set on deserialization, so we'll need to set it to null
         Field received = Plaintext.class.getDeclaredField("received");
         received.setAccessible(true);
-        received.set(p2, 0L);
+        received.set(actual, null);
 
-        assertEquals(p1, p2);
+        assertEquals(expected, actual);
-        assertEquals(ackMessage1, p2.getAckMessage());
+        assertEquals(ackMessage1, actual.getAckMessage());
     }
 
     @Test
     public void ensureNetworkMessageIsSerializedAndDeserializedCorrectly() throws Exception {
         ArrayList<InventoryVector> ivs = new ArrayList<>(50000);
         for (int i = 0; i < 50000; i++) {
-            ivs.add(new InventoryVector(cryptography().randomBytes(32)));
+            ivs.add(TestUtils.randomInventoryVector());
         }
 
         Inv inv = new Inv.Builder().inventory(ivs).build();
@@ -156,11 +182,11 @@ public class SerializationTest extends TestBase {
     @Test
     public void ensureSystemSerializationWorks() throws Exception {
         Plaintext plaintext = new Plaintext.Builder(MSG)
-                .from(TestUtils.loadContact())
+            .from(TestUtils.loadContact())
-                .to(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .to(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .labels(Collections.singletonList(new Label("Test", Label.Type.INBOX, 0)))
+            .labels(Collections.singletonList(new Label("Test", Label.Type.INBOX, 0)))
-                .message("Test", "Test Test.\nTest")
+            .message("Test", "Test Test.\nTest")
-                .build();
+            .build();
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         ObjectOutputStream oos = new ObjectOutputStream(out);
         oos.writeObject(plaintext);

core/src/test/java/ch/dissem/bitmessage/testutils/TestInventory.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package ch.dissem.bitmessage.networking;
+package ch.dissem.bitmessage.testutils;
 
 import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.payload.ObjectType;

core/src/test/java/ch/dissem/bitmessage/utils/ConversationServiceTest.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright 2017 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.utils;
+
+import ch.dissem.bitmessage.cryptography.bc.BouncyCryptography;
+import ch.dissem.bitmessage.entity.BitmessageAddress;
+import ch.dissem.bitmessage.entity.Plaintext;
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
+import ch.dissem.bitmessage.entity.valueobject.extended.Message;
+import ch.dissem.bitmessage.ports.MessageRepository;
+import org.junit.Test;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+
+import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
+import static ch.dissem.bitmessage.utils.TestUtils.RANDOM;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ConversationServiceTest {
+    private BitmessageAddress alice = new BitmessageAddress("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8");
+    private BitmessageAddress bob = new BitmessageAddress("BM-2cTtkBnb4BUYDndTKun6D9PjtueP2h1bQj");
+
+    private MessageRepository messageRepository = mock(MessageRepository.class);
+    private ConversationService conversationService = new ConversationService(messageRepository);
+
+    static {
+        Singleton.initialize(new BouncyCryptography());
+    }
+
+    @Test
+    public void ensureConversationIsSortedProperly() {
+        List<Plaintext> expected = getConversation();
+
+        when(conversationService.getConversation(any(UUID.class))).thenReturn(expected);
+        List<Plaintext> actual = conversationService.getConversation(UUID.randomUUID());
+        assertThat(actual, is(expected));
+    }
+
+    private List<Plaintext> getConversation() {
+        List<Plaintext> result = new LinkedList<>();
+
+        Plaintext older = plaintext(alice, bob,
+            new Message.Builder()
+                .subject("hey there")
+                .body("does it work?")
+                .build(),
+            Plaintext.Status.SENT);
+        result.add(older);
+
+        Plaintext root = plaintext(alice, bob,
+            new Message.Builder()
+                .subject("new test")
+                .body("There's a new test in town!")
+                .build(),
+            Plaintext.Status.SENT);
+        result.add(root);
+
+        result.add(
+            plaintext(bob, alice,
+                new Message.Builder()
+                    .subject("Re: new test (1a)")
+                    .body("Nice!")
+                    .addParent(root)
+                    .build(),
+                Plaintext.Status.RECEIVED)
+        );
+
+        Plaintext latest = plaintext(bob, alice,
+            new Message.Builder()
+                .subject("Re: new test (2b)")
+                .body("PS: it did work!")
+                .addParent(root)
+                .addParent(older)
+                .build(),
+            Plaintext.Status.RECEIVED);
+        result.add(latest);
+
+        result.add(
+            plaintext(alice, bob,
+                new Message.Builder()
+                    .subject("Re: new test (2)")
+                    .body("")
+                    .addParent(latest)
+                    .build(),
+                Plaintext.Status.DRAFT)
+        );
+
+        return result;
+    }
+
+    private int timer = 2;
+
+    private Plaintext plaintext(BitmessageAddress from, BitmessageAddress to,
+                                ExtendedEncoding content, Plaintext.Status status) {
+        Plaintext.Builder builder = new Plaintext.Builder(MSG)
+            .IV(TestUtils.randomInventoryVector())
+            .from(from)
+            .to(to)
+            .message(content)
+            .status(status);
+        if (status != Plaintext.Status.DRAFT && status != Plaintext.Status.DOING_PROOF_OF_WORK) {
+            builder.received(5L * ++timer - RANDOM.nextInt(10));
+        }
+        return builder.build();
+    }
+}

core/src/test/java/ch/dissem/bitmessage/utils/MessageMatchers.java

@@ -19,38 +19,27 @@ package ch.dissem.bitmessage.utils;
 import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.entity.payload.ObjectType;
-import org.hamcrest.BaseMatcher;
+import org.mockito.ArgumentMatcher;
-import org.hamcrest.Description;
+import org.mockito.ArgumentMatchers;
-import org.mockito.Matchers;
 
 /**
  * @author Christian Basler
  */
 public class MessageMatchers {
     public static Plaintext plaintext(final Plaintext.Type type) {
-        return Matchers.argThat(new BaseMatcher<Plaintext>() {
+        return ArgumentMatchers.argThat(new ArgumentMatcher<Plaintext>() {
             @Override
-            public boolean matches(Object item) {
+            public boolean matches(Plaintext item) {
-                return item instanceof Plaintext && ((Plaintext) item).getType() == type;
+                return item != null && item.getType() == type;
-            }
-
-            @Override
-            public void describeTo(Description description) {
-                description.appendText("type should be ").appendValue(type);
             }
         });
     }
 
     public static ObjectMessage object(final ObjectType type) {
-        return Matchers.argThat(new BaseMatcher<ObjectMessage>() {
+        return ArgumentMatchers.argThat(new ArgumentMatcher<ObjectMessage>() {
             @Override
-            public boolean matches(Object item) {
+            public boolean matches(ObjectMessage item) {
-                return item instanceof ObjectMessage && ((ObjectMessage) item).getPayload().getType() == type;
+                return item != null && item.getPayload().getType() == type;
-            }
-
-            @Override
-            public void describeTo(Description description) {
-                description.appendText("payload type should be ").appendValue(type);
             }
         });
     }

core/src/test/java/ch/dissem/bitmessage/utils/TestUtils.java

@@ -20,6 +20,7 @@ import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.payload.Pubkey;
 import ch.dissem.bitmessage.entity.payload.V4Pubkey;
+import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
 import ch.dissem.bitmessage.exception.DecryptionFailedException;
 import ch.dissem.bitmessage.factory.Factory;
@@ -28,6 +29,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
 
@@ -35,6 +37,8 @@ import static org.junit.Assert.assertEquals;
  * If there's ever a need for this in production code, it should be rewritten to be more efficient.
  */
 public class TestUtils {
+    public static final Random RANDOM = new Random();
+
     public static byte[] int16(int number) throws IOException {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         Encode.int16(number, out);
@@ -59,6 +63,12 @@ public class TestUtils {
         return out.toByteArray();
     }
 
+    public static InventoryVector randomInventoryVector() {
+        byte[] bytes = new byte[32];
+        RANDOM.nextBytes(bytes);
+        return new InventoryVector(bytes);
+    }
+
     public static InputStream getResource(String resourceName) {
         return TestUtils.class.getClassLoader().getResourceAsStream(resourceName);
     }

cryptography-bc/build.gradle

@@ -12,7 +12,7 @@ uploadArchives {
 
 dependencies {
     compile project(':core')
-    compile 'org.bouncycastle:bcprov-jdk15on:1.52'
+    compile 'org.bouncycastle:bcprov-jdk15on:1.56'
     testCompile 'junit:junit:4.12'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
 }

cryptography-bc/src/main/java/ch/dissem/bitmessage/cryptography/bc/BouncyCryptography.java

@@ -38,10 +38,7 @@ import org.bouncycastle.jce.spec.ECPublicKeySpec;
 import org.bouncycastle.math.ec.ECPoint;
 
 import java.math.BigInteger;
-import java.security.GeneralSecurityException;
+import java.security.*;
-import java.security.KeyFactory;
-import java.security.PublicKey;
-import java.security.Signature;
 import java.security.spec.KeySpec;
 import java.util.Arrays;
 
@@ -51,7 +48,6 @@ import java.util.Arrays;
  */
 public class BouncyCryptography extends AbstractCryptography {
     private static final X9ECParameters EC_CURVE_PARAMETERS = CustomNamedCurves.getByName("secp256k1");
-    private static final String ALGORITHM_ECDSA = "ECDSA";
 
     public BouncyCryptography() {
         super(new BouncyCastleProvider());
@@ -106,10 +102,7 @@ public class BouncyCryptography extends AbstractCryptography {
             KeySpec keySpec = new ECPublicKeySpec(Q, spec);
             PublicKey publicKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider).generatePublic(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
+            return doCheckSignature(data, signature, publicKey);
-            sig.initVerify(publicKey);
-            sig.update(data);
-            return sig.verify(signature);
         } catch (GeneralSecurityException e) {
             throw new ApplicationException(e);
         }
@@ -131,10 +124,7 @@ public class BouncyCryptography extends AbstractCryptography {
             java.security.PrivateKey privKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider)
                 .generatePrivate(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
+            return doSign(data, privKey);
-            sig.initSign(privKey);
-            sig.update(data);
-            return sig.sign();
         } catch (GeneralSecurityException e) {
             throw new ApplicationException(e);
         }

cryptography-sc/build.gradle

@@ -12,7 +12,7 @@ uploadArchives {
 
 dependencies {
     compile project(':core')
-    compile 'com.madgag.spongycastle:prov:1.52.0.0'
+    compile 'com.madgag.spongycastle:prov:1.54.0.0'
     testCompile 'junit:junit:4.12'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
 }

cryptography-sc/src/main/java/ch/dissem/bitmessage/cryptography/sc/SpongyCryptography.java

@@ -41,7 +41,6 @@ import java.math.BigInteger;
 import java.security.GeneralSecurityException;
 import java.security.KeyFactory;
 import java.security.PublicKey;
-import java.security.Signature;
 import java.security.spec.KeySpec;
 import java.util.Arrays;
 
@@ -51,7 +50,6 @@ import java.util.Arrays;
  */
 public class SpongyCryptography extends AbstractCryptography {
     private static final X9ECParameters EC_CURVE_PARAMETERS = CustomNamedCurves.getByName("secp256k1");
-    private static final String ALGORITHM_ECDSA = "ECDSA";
 
     public SpongyCryptography() {
         super(new BouncyCastleProvider());
@@ -106,10 +104,7 @@ public class SpongyCryptography extends AbstractCryptography {
             KeySpec keySpec = new ECPublicKeySpec(Q, spec);
             PublicKey publicKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider).generatePublic(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
+            return doCheckSignature(data, signature, publicKey);
-            sig.initVerify(publicKey);
-            sig.update(data);
-            return sig.verify(signature);
         } catch (GeneralSecurityException e) {
             throw new ApplicationException(e);
         }
@@ -131,10 +126,7 @@ public class SpongyCryptography extends AbstractCryptography {
             java.security.PrivateKey privKey = KeyFactory.getInstance(ALGORITHM_ECDSA, provider)
                 .generatePrivate(keySpec);
 
-            Signature sig = Signature.getInstance(ALGORITHM_ECDSA, provider);
+            return doSign(data, privKey);
-            sig.initSign(privKey);
-            sig.update(data);
-            return sig.sign();
         } catch (GeneralSecurityException e) {
             throw new ApplicationException(e);
         }

demo/build.gradle

@@ -1,5 +1,5 @@
 plugins {
-    id "us.kirchmeier.capsule" version "1.0-rc1"
+    id "us.kirchmeier.capsule" version "1.0.2"
 }
 
 uploadArchives {
@@ -28,10 +28,10 @@ dependencies {
     compile project(':repositories')
     compile project(':cryptography-bc')
     compile project(':wif')
-    compile 'org.slf4j:slf4j-simple:1.7.12'
+    compile 'org.slf4j:slf4j-simple:1.7.25'
-    compile 'args4j:args4j:2.32'
+    compile 'args4j:args4j:2.33'
-    compile 'com.h2database:h2:1.4.192'
+    compile 'com.h2database:h2:1.4.194'
-    compile 'org.apache.commons:commons-lang3:3.4'
+    compile 'org.apache.commons:commons-lang3:3.5'
     testCompile 'junit:junit:4.12'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
 }

demo/src/main/java/ch/dissem/bitmessage/demo/Application.java

@@ -21,22 +21,26 @@ import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.Plaintext;
 import ch.dissem.bitmessage.entity.payload.Pubkey;
 import ch.dissem.bitmessage.entity.valueobject.Label;
+import ch.dissem.bitmessage.entity.valueobject.extended.Message;
 import org.apache.commons.lang3.text.WordUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.net.InetAddress;
 import java.util.List;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static ch.dissem.bitmessage.demo.CommandLine.COMMAND_BACK;
 import static ch.dissem.bitmessage.demo.CommandLine.ERROR_UNKNOWN_COMMAND;
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
 
 /**
  * A simple command line Bitmessage application
  */
 public class Application {
     private final static Logger LOG = LoggerFactory.getLogger(Application.class);
+    private final static Pattern RESPONSE_PATTERN = Pattern.compile("^RE:.*$", CASE_INSENSITIVE);
     private final CommandLine commandLine;
 
     private BitmessageContext ctx;
@@ -342,7 +346,7 @@ public class Application {
             System.out.println();
             System.out.println("c) compose message");
             System.out.println("s) compose broadcast");
-            if (label.getType() == Label.Type.TRASH) {
+            if (label != null && label.getType() == Label.Type.TRASH) {
                 System.out.println("e) empty trash");
             }
             System.out.println(COMMAND_BACK);
@@ -392,7 +396,7 @@ public class Application {
             command = commandLine.nextCommand();
             switch (command) {
                 case "r":
-                    compose(message.getTo(), message.getFrom(), "RE: " + message.getSubject());
+                    compose(message.getTo(), message.getFrom(), message);
                     break;
                 case "d":
                     ctx.labeler().delete(message);
@@ -442,14 +446,20 @@ public class Application {
         return commandLine.selectAddress(addresses, "To:");
     }
 
-    private void compose(BitmessageAddress from, BitmessageAddress to, String subject) {
+    private void compose(BitmessageAddress from, BitmessageAddress to, Plaintext parent) {
         boolean broadcast = (to == null);
+        String subject;
         System.out.println();
         System.out.println("From:    " + from);
         if (!broadcast) {
             System.out.println("To:      " + to);
         }
-        if (subject != null) {
+        if (parent != null) {
+            if (RESPONSE_PATTERN.matcher(parent.getSubject()).matches()) {
+                subject = parent.getSubject();
+            } else {
+                subject = "RE: " + parent.getSubject();
+            }
             System.out.println("Subject: " + subject);
         } else {
             System.out.print("Subject: ");
@@ -462,10 +472,20 @@ public class Application {
             line = commandLine.nextLine();
             message.append(line).append('\n');
         } while (line.length() > 0 || !commandLine.yesNo("Send message?"));
-        if (broadcast) {
+        Plaintext.Type type = broadcast ? Plaintext.Type.BROADCAST : Plaintext.Type.MSG;
-            ctx.broadcast(from, subject, message.toString());
+        Plaintext.Builder builder = new Plaintext.Builder(type);
+        builder.from(from);
+        builder.to(to);
+        if (commandLine.yesNo("Use extended encoding?")) {
+            Message.Builder extended = new Message.Builder();
+            extended.subject(subject).body(message.toString());
+            if (parent != null) {
+                extended.addParent(parent);
+            }
+            builder.message(extended.build());
         } else {
-            ctx.send(from, to, subject, message.toString());
+            builder.message(subject, message.toString());
         }
+        ctx.send(builder.build());
     }
 }

demo/src/main/java/ch/dissem/bitmessage/demo/Main.java

@@ -65,6 +65,11 @@ public class Main {
             .port(48444);
         if (options.localPort != null) {
             ctxBuilder.nodeRegistry(new NodeRegistry() {
+                @Override
+                public void clear() {
+                    // NO OP
+                }
+
                 @Override
                 public List<NetworkAddress> getKnownAddresses(int limit, long... streams) {
                     return Arrays.stream(streams)

demo/src/test/java/ch/dissem/bitmessage/SystemTest.java

@@ -28,7 +28,7 @@ import static ch.dissem.bitmessage.entity.payload.Pubkey.Feature.DOES_ACK;
 import static ch.dissem.bitmessage.utils.UnixTime.MINUTE;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.junit.Assert.assertThat;
-import static org.mockito.Matchers.any;
+import static org.mockito.ArgumentMatchers.any;
 
 /**
  * @author Christian Basler
@@ -54,10 +54,11 @@ public class SystemTest {
     }
 
     @Parameterized.Parameters
+    @SuppressWarnings("deprecation")
     public static List<Object[]> parameters() {
         return Arrays.asList(new Object[][]{
-                {new NioNetworkHandler(), new DefaultNetworkHandler()},
+            {new NioNetworkHandler(), new DefaultNetworkHandler()},
-                {new NioNetworkHandler(), new NioNetworkHandler()}
+            {new NioNetworkHandler(), new NioNetworkHandler()}
         });
     }
 
@@ -70,33 +71,33 @@ public class SystemTest {
         TTL.pubkey(5 * MINUTE);
         JdbcConfig aliceDB = new JdbcConfig("jdbc:h2:mem:alice;DB_CLOSE_DELAY=-1", "sa", "");
         alice = new BitmessageContext.Builder()
-                .addressRepo(new JdbcAddressRepository(aliceDB))
+            .addressRepo(new JdbcAddressRepository(aliceDB))
-                .inventory(new JdbcInventory(aliceDB))
+            .inventory(new JdbcInventory(aliceDB))
-                .messageRepo(new JdbcMessageRepository(aliceDB))
+            .messageRepo(new JdbcMessageRepository(aliceDB))
-                .powRepo(new JdbcProofOfWorkRepository(aliceDB))
+            .powRepo(new JdbcProofOfWorkRepository(aliceDB))
-                .port(alicePort)
+            .port(alicePort)
-                .nodeRegistry(new TestNodeRegistry(bobPort))
+            .nodeRegistry(new TestNodeRegistry(bobPort))
-                .networkHandler(aliceNetworkHandler)
+            .networkHandler(aliceNetworkHandler)
-                .cryptography(new BouncyCryptography())
+            .cryptography(new BouncyCryptography())
-                .listener(aliceListener)
+            .listener(aliceListener)
-                .labeler(aliceLabeler)
+            .labeler(aliceLabeler)
-                .build();
+            .build();
         alice.startup();
         aliceIdentity = alice.createIdentity(false, DOES_ACK);
 
         JdbcConfig bobDB = new JdbcConfig("jdbc:h2:mem:bob;DB_CLOSE_DELAY=-1", "sa", "");
         bob = new BitmessageContext.Builder()
-                .addressRepo(new JdbcAddressRepository(bobDB))
+            .addressRepo(new JdbcAddressRepository(bobDB))
-                .inventory(new JdbcInventory(bobDB))
+            .inventory(new JdbcInventory(bobDB))
-                .messageRepo(new JdbcMessageRepository(bobDB))
+            .messageRepo(new JdbcMessageRepository(bobDB))
-                .powRepo(new JdbcProofOfWorkRepository(bobDB))
+            .powRepo(new JdbcProofOfWorkRepository(bobDB))
-                .port(bobPort)
+            .port(bobPort)
-                .nodeRegistry(new TestNodeRegistry(alicePort))
+            .nodeRegistry(new TestNodeRegistry(alicePort))
-                .networkHandler(bobNetworkHandler)
+            .networkHandler(bobNetworkHandler)
-                .cryptography(new BouncyCryptography())
+            .cryptography(new BouncyCryptography())
-                .listener(bobListener)
+            .listener(bobListener)
-                .labeler(new DebugLabeler("Bob"))
+            .labeler(new DebugLabeler("Bob"))
-                .build();
+            .build();
         bob.startup();
         bobIdentity = bob.createIdentity(false, DOES_ACK);
 
@@ -121,7 +122,7 @@ public class SystemTest {
         assertThat(plaintext.getText(), equalTo(originalMessage));
 
         Mockito.verify(aliceLabeler, Mockito.timeout(TimeUnit.MINUTES.toMillis(15)).atLeastOnce())
-                .markAsAcknowledged(any());
+            .markAsAcknowledged(any());
     }
 
     @Test(timeout = 30_000)

demo/src/test/java/ch/dissem/bitmessage/TestNodeRegistry.java

@@ -39,6 +39,11 @@ class TestNodeRegistry implements NodeRegistry {
         }
     }
 
+    @Override
+    public void clear() {
+        // NO OP
+    }
+
     @Override
     public List<NetworkAddress> getKnownAddresses(int limit, long... streams) {
         return nodes;

extensions/build.gradle

@@ -29,8 +29,8 @@ uploadArchives {
 dependencies {
     compile project(':core')
     testCompile 'junit:junit:4.12'
-    testCompile 'org.slf4j:slf4j-simple:1.7.12'
+    testCompile 'org.slf4j:slf4j-simple:1.7.25'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
     testCompile project(path: ':core', configuration: 'testArtifacts')
     testCompile project(':cryptography-bc')
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Fri Aug 12 11:52:04 CEST 2016
+#Mon Apr 03 17:55:37 CEST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.4.1-all.zip

gradlew

@@ -6,12 +6,30 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+# Attempt to set APP_HOME
-DEFAULT_JVM_OPTS=""
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,26 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -85,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then

gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,7 +46,7 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
 if "%@eval[2+2]" == "4" goto 4NT_args

networking/build.gradle

@@ -13,8 +13,8 @@ uploadArchives {
 dependencies {
     compile project(':core')
     testCompile 'junit:junit:4.12'
-    testCompile 'org.slf4j:slf4j-simple:1.7.12'
+    testCompile 'org.slf4j:slf4j-simple:1.7.25'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
     testCompile project(path: ':core', configuration: 'testArtifacts')
     testCompile project(':cryptography-bc')
-}
+}

networking/src/main/java/ch/dissem/bitmessage/networking/nio/NioNetworkHandler.java

@@ -254,7 +254,7 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
                                         }
                                     }
                                 } catch (CancelledKeyException e) {
-                                    LOG.error(e.getMessage(), e);
+                                    LOG.debug(e.getMessage(), e);
                                 }
                             } else {
                                 // handle read/write
@@ -287,11 +287,15 @@ public class NioNetworkHandler implements NetworkHandler, InternalContext.Contex
                         }
                         // set interest ops
                         for (Map.Entry<ConnectionInfo, SelectionKey> e : connections.entrySet()) {
-                            if (e.getValue().isValid()
+                            try {
-                                && (e.getValue().interestOps() & OP_WRITE) == 0
+                                if (e.getValue().isValid()
-                                && (e.getValue().interestOps() & OP_CONNECT) == 0
+                                    && (e.getValue().interestOps() & OP_WRITE) == 0
-                                && !e.getKey().getSendingQueue().isEmpty()) {
+                                    && (e.getValue().interestOps() & OP_CONNECT) == 0
-                                e.getValue().interestOps(OP_READ | OP_WRITE);
+                                    && !e.getKey().getSendingQueue().isEmpty()) {
+                                    e.getValue().interestOps(OP_READ | OP_WRITE);
+                                }
+                            } catch (CancelledKeyException x) {
+                                e.getKey().disconnect();
                             }
                         }
                         // start new connections

networking/src/test/java/ch/dissem/bitmessage/networking/NetworkHandlerTest.java

@@ -24,6 +24,7 @@ import ch.dissem.bitmessage.entity.valueobject.NetworkAddress;
 import ch.dissem.bitmessage.exception.NodeException;
 import ch.dissem.bitmessage.networking.nio.NioNetworkHandler;
 import ch.dissem.bitmessage.ports.*;
+import ch.dissem.bitmessage.testutils.TestInventory;
 import ch.dissem.bitmessage.utils.Property;
 import org.junit.After;
 import org.junit.Before;

networking/src/test/java/ch/dissem/bitmessage/networking/TestNodeRegistry.java

@@ -32,6 +32,11 @@ class TestNodeRegistry implements NodeRegistry {
         this.nodes = Arrays.asList(nodes);
     }
 
+    @Override
+    public void clear() {
+        // no op
+    }
+
     @Override
     public List<NetworkAddress> getKnownAddresses(int limit, long... streams) {
         return nodes;

repositories/build.gradle

@@ -14,10 +14,10 @@ sourceCompatibility = 1.8
 
 dependencies {
     compile project(':core')
-    compile 'org.flywaydb:flyway-core:4.0.3'
+    compile 'org.flywaydb:flyway-core:4.1.2'
     testCompile 'junit:junit:4.12'
-    testCompile 'com.h2database:h2:1.4.192'
+    testCompile 'com.h2database:h2:1.4.194'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
     testCompile project(path: ':core', configuration: 'testArtifacts')
     testCompile project(':cryptography-bc')
 }

repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcMessageRepository.java

@@ -31,6 +31,7 @@ import java.sql.*;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.UUID;
 
 import static ch.dissem.bitmessage.repository.JdbcHelper.writeBlob;
 
@@ -46,7 +47,7 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
     @Override
     protected List<Label> findLabels(String where) {
         try (
-                Connection connection = config.getConnection()
+            Connection connection = config.getConnection()
         ) {
             return findLabels(connection, where);
         } catch (SQLException e) {
@@ -76,12 +77,12 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
             where = "id IN (SELECT message_id FROM Message_Label WHERE label_id=" + label.getId() + ") AND ";
         }
         where += "id IN (SELECT message_id FROM Message_Label WHERE label_id IN (" +
-                "SELECT id FROM Label WHERE type = '" + Label.Type.UNREAD.name() + "'))";
+            "SELECT id FROM Label WHERE type = '" + Label.Type.UNREAD.name() + "'))";
 
         try (
-                Connection connection = config.getConnection();
+            Connection connection = config.getConnection();
-                Statement stmt = connection.createStatement();
+            Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery("SELECT count(*) FROM Message WHERE " + where)
+            ResultSet rs = stmt.executeQuery("SELECT count(*) FROM Message WHERE " + where)
         ) {
             if (rs.next()) {
                 return rs.getInt(1);
@@ -96,11 +97,11 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
     protected List<Plaintext> find(String where) {
         List<Plaintext> result = new LinkedList<>();
         try (
-                Connection connection = config.getConnection();
+            Connection connection = config.getConnection();
-                Statement stmt = connection.createStatement();
+            Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery(
+            ResultSet rs = stmt.executeQuery(
-                        "SELECT id, iv, type, sender, recipient, data, ack_data, sent, received, initial_hash, status, ttl, retries, next_try " +
+                "SELECT id, iv, type, sender, recipient, data, ack_data, sent, received, initial_hash, status, ttl, retries, next_try, conversation " +
-                                "FROM Message WHERE " + where)
+                    "FROM Message WHERE " + where)
         ) {
             while (rs.next()) {
                 byte[] iv = rs.getBytes("iv");
@@ -113,14 +114,15 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
                 builder.from(ctx.getAddressRepository().getAddress(rs.getString("sender")));
                 builder.to(ctx.getAddressRepository().getAddress(rs.getString("recipient")));
                 builder.ackData(rs.getBytes("ack_data"));
-                builder.sent(rs.getLong("sent"));
+                builder.sent(rs.getObject("sent", Long.class));
-                builder.received(rs.getLong("received"));
+                builder.received(rs.getObject("received", Long.class));
                 builder.status(Plaintext.Status.valueOf(rs.getString("status")));
                 builder.ttl(rs.getLong("ttl"));
                 builder.retries(rs.getInt("retries"));
-                builder.nextTry(rs.getLong("next_try"));
+                builder.nextTry(rs.getObject("next_try", Long.class));
+                builder.conversation(rs.getObject("conversation", UUID.class));
                 builder.labels(findLabels(connection,
-                        "id IN (SELECT label_id FROM Message_Label WHERE message_id=" + id + ") ORDER BY ord"));
+                    "id IN (SELECT label_id FROM Message_Label WHERE message_id=" + id + ") ORDER BY ord"));
                 Plaintext message = builder.build();
                 message.setInitialHash(rs.getBytes("initial_hash"));
                 result.add(message);
@@ -134,8 +136,8 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
     private List<Label> findLabels(Connection connection, String where) {
         List<Label> result = new ArrayList<>();
         try (
-                Statement stmt = connection.createStatement();
+            Statement stmt = connection.createStatement();
-                ResultSet rs = stmt.executeQuery("SELECT id, label, type, color FROM Label WHERE " + where)
+            ResultSet rs = stmt.executeQuery("SELECT id, label, type, color FROM Label WHERE " + where)
         ) {
             while (rs.next()) {
                 result.add(getLabel(rs));
@@ -148,12 +150,14 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
 
     @Override
     public void save(Plaintext message) {
-        safeSenderIfNecessary(message);
+        saveContactIfNecessary(message.getFrom());
+        saveContactIfNecessary(message.getTo());
 
         try (Connection connection = config.getConnection()) {
             try {
                 connection.setAutoCommit(false);
                 save(connection, message);
+                updateParents(connection, message);
                 updateLabels(connection, message);
                 connection.commit();
             } catch (IOException | SQLException e) {
@@ -180,7 +184,7 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
         }
         // save new labels
         try (PreparedStatement ps = connection.prepareStatement("INSERT INTO Message_Label VALUES (" +
-                message.getId() + ", ?)")) {
+            message.getId() + ", ?)")) {
             for (Label label : message.getLabels()) {
                 ps.setLong(1, (Long) label.getId());
                 ps.executeUpdate();
@@ -188,12 +192,39 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
         }
     }
 
+    private void updateParents(Connection connection, Plaintext message) throws SQLException {
+        if (message.getInventoryVector() == null || message.getParents().isEmpty()) {
+            // There are no parents to save yet (they are saved in the extended data, that's enough for now)
+            return;
+        }
+        // remove existing parents
+        try (PreparedStatement ps = connection.prepareStatement("DELETE FROM Message_Parent WHERE child=?")) {
+            ps.setBytes(1, message.getInitialHash());
+            ps.executeUpdate();
+        }
+        byte[] childIV = message.getInventoryVector().getHash();
+        // save new parents
+        int order = 0;
+        try (PreparedStatement ps = connection.prepareStatement("INSERT INTO Message_Parent VALUES (?, ?, ?, ?)")) {
+            for (InventoryVector parentIV : message.getParents()) {
+                Plaintext parent = getMessage(parentIV);
+                mergeConversations(connection, parent.getConversationId(), message.getConversationId());
+                order++;
+                ps.setBytes(1, parentIV.getHash());
+                ps.setBytes(2, childIV);
+                ps.setInt(3, order); // FIXME: this might not be necessary
+                ps.setObject(4, message.getConversationId());
+                ps.executeUpdate();
+            }
+        }
+    }
+
     private void insert(Connection connection, Plaintext message) throws SQLException, IOException {
         try (PreparedStatement ps = connection.prepareStatement(
-                "INSERT INTO Message (iv, type, sender, recipient, data, ack_data, sent, received, " +
+            "INSERT INTO Message (iv, type, sender, recipient, data, ack_data, sent, received, " +
-                        "status, initial_hash, ttl, retries, next_try) " +
+                "status, initial_hash, ttl, retries, next_try, conversation) " +
-                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
-                Statement.RETURN_GENERATED_KEYS)
+            Statement.RETURN_GENERATED_KEYS)
         ) {
             ps.setBytes(1, message.getInventoryVector() == null ? null : message.getInventoryVector().getHash());
             ps.setString(2, message.getType().name());
@@ -201,13 +232,14 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
             ps.setString(4, message.getTo() == null ? null : message.getTo().getAddress());
             writeBlob(ps, 5, message);
             ps.setBytes(6, message.getAckData());
-            ps.setLong(7, message.getSent());
+            ps.setObject(7, message.getSent());
-            ps.setLong(8, message.getReceived());
+            ps.setObject(8, message.getReceived());
             ps.setString(9, message.getStatus() == null ? null : message.getStatus().name());
             ps.setBytes(10, message.getInitialHash());
             ps.setLong(11, message.getTTL());
             ps.setInt(12, message.getRetries());
             ps.setObject(13, message.getNextTry());
+            ps.setObject(14, message.getConversationId());
 
             ps.executeUpdate();
             // get generated id
@@ -220,17 +252,17 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
 
     private void update(Connection connection, Plaintext message) throws SQLException, IOException {
         try (PreparedStatement ps = connection.prepareStatement(
-                "UPDATE Message SET iv=?, type=?, sender=?, recipient=?, data=?, ack_data=?, sent=?, received=?, " +
+            "UPDATE Message SET iv=?, type=?, sender=?, recipient=?, data=?, ack_data=?, sent=?, received=?, " +
-                        "status=?, initial_hash=?, ttl=?, retries=?, next_try=? " +
+                "status=?, initial_hash=?, ttl=?, retries=?, next_try=? " +
-                        "WHERE id=?")) {
+                "WHERE id=?")) {
             ps.setBytes(1, message.getInventoryVector() == null ? null : message.getInventoryVector().getHash());
             ps.setString(2, message.getType().name());
             ps.setString(3, message.getFrom().getAddress());
             ps.setString(4, message.getTo() == null ? null : message.getTo().getAddress());
             writeBlob(ps, 5, message);
             ps.setBytes(6, message.getAckData());
-            ps.setLong(7, message.getSent());
+            ps.setObject(7, message.getSent());
-            ps.setLong(8, message.getReceived());
+            ps.setObject(8, message.getReceived());
             ps.setString(9, message.getStatus() == null ? null : message.getStatus().name());
             ps.setBytes(10, message.getInitialHash());
             ps.setLong(11, message.getTTL());
@@ -261,4 +293,52 @@ public class JdbcMessageRepository extends AbstractMessageRepository implements
             LOG.error(e.getMessage(), e);
         }
     }
+
+    @Override
+    public List<UUID> findConversations(Label label) {
+        String where;
+        if (label == null) {
+            where = "id NOT IN (SELECT message_id FROM Message_Label)";
+        } else {
+            where = "id IN (SELECT message_id FROM Message_Label WHERE label_id=" + label.getId() + ")";
+        }
+        List<UUID> result = new LinkedList<>();
+        try (
+            Connection connection = config.getConnection();
+            Statement stmt = connection.createStatement();
+            ResultSet rs = stmt.executeQuery(
+                "SELECT DISTINCT conversation FROM Message WHERE " + where)
+        ) {
+            while (rs.next()) {
+                result.add((UUID) rs.getObject(1));
+            }
+        } catch (SQLException e) {
+            LOG.error(e.getMessage(), e);
+        }
+        return result;
+    }
+
+    /**
+     * Replaces every occurrence of the source conversation ID with the target ID
+     *
+     * @param source ID of the conversation to be merged
+     * @param target ID of the merge target
+     */
+    private void mergeConversations(Connection connection, UUID source, UUID target) {
+        try (
+            PreparedStatement ps1 = connection.prepareStatement(
+                "UPDATE Message SET conversation=? WHERE conversation=?");
+            PreparedStatement ps2 = connection.prepareStatement(
+                "UPDATE Message_Parent SET conversation=? WHERE conversation=?")
+        ) {
+            ps1.setObject(1, target);
+            ps1.setObject(2, source);
+            ps1.executeUpdate();
+            ps2.setObject(1, target);
+            ps2.setObject(2, source);
+            ps2.executeUpdate();
+        } catch (SQLException e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
 }

repositories/src/main/java/ch/dissem/bitmessage/repository/JdbcNodeRegistry.java

@@ -69,6 +69,19 @@ public class JdbcNodeRegistry extends JdbcHelper implements NodeRegistry {
         }
     }
 
+    @Override
+    public void clear() {
+        try (
+            Connection connection = config.getConnection();
+            PreparedStatement ps = connection.prepareStatement(
+                "DELETE FROM Node")
+        ) {
+            ps.executeUpdate();
+        } catch (SQLException e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
+
     @Override
     public List<NetworkAddress> getKnownAddresses(int limit, long... streams) {
         List<NetworkAddress> result = new LinkedList<>();
@@ -109,6 +122,11 @@ public class JdbcNodeRegistry extends JdbcHelper implements NodeRegistry {
                     result.add(Collections.selectRandom(nodes));
                 }
             }
+            if (result.isEmpty()) {
+                // There might have been an error resolving domain names due to a missing internet exception.
+                // Try to load the stable nodes again next time.
+                stableNodes = null;
+            }
         }
         return result;
     }

repositories/src/main/resources/db/migration/V4.0__Create_table_message_parent.sql

@@ -0,0 +1,11 @@
+ALTER TABLE Message ADD COLUMN conversation UUID;
+
+CREATE TABLE Message_Parent (
+    parent       BINARY(64) NOT NULL,
+    child        BINARY(64) NOT NULL,
+    pos          INT NOT NULL,
+    conversation UUID NOT NULL,
+
+    PRIMARY KEY (parent, child),
+    FOREIGN KEY (child) REFERENCES Message (iv)
+);

repositories/src/test/java/ch/dissem/bitmessage/repository/JdbcMessageRepositoryTest.java

@@ -21,17 +21,23 @@ import ch.dissem.bitmessage.InternalContext;
 import ch.dissem.bitmessage.entity.BitmessageAddress;
 import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.Plaintext;
-import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.entity.valueobject.PrivateKey;
+import ch.dissem.bitmessage.entity.valueobject.extended.Message;
 import ch.dissem.bitmessage.ports.AddressRepository;
 import ch.dissem.bitmessage.ports.MessageRepository;
+import ch.dissem.bitmessage.utils.TestUtils;
 import ch.dissem.bitmessage.utils.UnixTime;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.UUID;
 
 import static ch.dissem.bitmessage.entity.Plaintext.Type.MSG;
 import static ch.dissem.bitmessage.entity.payload.Pubkey.Feature.DOES_ACK;
@@ -48,6 +54,7 @@ public class JdbcMessageRepositoryTest extends TestBase {
     private MessageRepository repo;
 
     private Label inbox;
+    private Label sent;
     private Label drafts;
     private Label unread;
 
@@ -58,9 +65,9 @@ public class JdbcMessageRepositoryTest extends TestBase {
         AddressRepository addressRepo = new JdbcAddressRepository(config);
         repo = new JdbcMessageRepository(config);
         new InternalContext(new BitmessageContext.Builder()
-                .cryptography(cryptography())
+            .cryptography(cryptography())
-                .addressRepo(addressRepo)
+            .addressRepo(addressRepo)
-                .messageRepo(repo)
+            .messageRepo(repo)
         );
 
         BitmessageAddress tmp = new BitmessageAddress(new PrivateKey(false, 1, 1000, 1000, DOES_ACK));
@@ -74,6 +81,7 @@ public class JdbcMessageRepositoryTest extends TestBase {
         addressRepo.save(identity);
 
         inbox = repo.getLabels(Label.Type.INBOX).get(0);
+        sent = repo.getLabels(Label.Type.SENT).get(0);
         drafts = repo.getLabels(Label.Type.DRAFT).get(0);
         unread = repo.getLabels(Label.Type.UNREAD).get(0);
 
@@ -162,12 +170,12 @@ public class JdbcMessageRepositoryTest extends TestBase {
     @Test
     public void testSave() throws Exception {
         Plaintext message = new Plaintext.Builder(MSG)
-                .IV(new InventoryVector(cryptography().randomBytes(32)))
+            .IV(TestUtils.randomInventoryVector())
-                .from(identity)
+            .from(identity)
-                .to(contactA)
+            .to(contactA)
-                .message("Subject", "Message")
+            .message("Subject", "Message")
-                .status(Plaintext.Status.DOING_PROOF_OF_WORK)
+            .status(Plaintext.Status.DOING_PROOF_OF_WORK)
-                .build();
+            .build();
         repo.save(message);
 
         assertNotNull(message.getId());
@@ -185,7 +193,7 @@ public class JdbcMessageRepositoryTest extends TestBase {
     public void testUpdate() throws Exception {
         List<Plaintext> messages = repo.findMessages(Plaintext.Status.DRAFT, contactA);
         Plaintext message = messages.get(0);
-        message.setInventoryVector(new InventoryVector(cryptography().randomBytes(32)));
+        message.setInventoryVector(TestUtils.randomInventoryVector());
         repo.save(message);
 
         messages = repo.findMessages(Plaintext.Status.DRAFT, contactA);
@@ -206,19 +214,20 @@ public class JdbcMessageRepositoryTest extends TestBase {
     @Test
     public void ensureUnacknowledgedMessagesAreFoundForResend() throws Exception {
         Plaintext message = new Plaintext.Builder(MSG)
-                .IV(new InventoryVector(cryptography().randomBytes(32)))
+            .IV(TestUtils.randomInventoryVector())
-                .from(identity)
+            .from(identity)
-                .to(contactA)
+            .to(contactA)
-                .message("Subject", "Message")
+            .message("Subject", "Message")
-                .status(Plaintext.Status.SENT)
+            .sent(UnixTime.now())
-                .ttl(2)
+            .status(Plaintext.Status.SENT)
-                .build();
+            .ttl(2)
+            .build();
         message.updateNextTry();
         assertThat(message.getRetries(), is(1));
         assertThat(message.getNextTry(), greaterThan(UnixTime.now()));
         assertThat(message.getNextTry(), lessThanOrEqualTo(UnixTime.now(+2)));
         repo.save(message);
-        Thread.sleep(4100);
+        Thread.sleep(4100); // somewhat longer than 2*TTL
         List<Plaintext> messagesToResend = repo.findMessagesToResend();
         assertThat(messagesToResend, hasSize(1));
 
@@ -230,14 +239,105 @@ public class JdbcMessageRepositoryTest extends TestBase {
         assertThat(messagesToResend, empty());
     }
 
-    private void addMessage(BitmessageAddress from, BitmessageAddress to, Plaintext.Status status, Label... labels) {
+    @Test
-        Plaintext message = new Plaintext.Builder(MSG)
+    public void ensureParentsAreSaved() {
-                .from(from)
+        Plaintext parent = storeConversation();
-                .to(to)
+
-                .message("Subject", "Message")
+        List<Plaintext> responses = repo.findResponses(parent);
-                .status(status)
+        assertThat(responses, hasSize(2));
-                .labels(Arrays.asList(labels))
+        assertThat(responses, hasItem(hasMessage("Re: new test", "Nice!")));
-                .build();
+        assertThat(responses, hasItem(hasMessage("Re: new test", "PS: it did work!")));
-        repo.save(message);
     }
-}
+
+    @Test
+    public void ensureConversationCanBeRetrieved() {
+        Plaintext root = storeConversation();
+        List<UUID> conversations = repo.findConversations(inbox);
+        assertThat(conversations, hasSize(2));
+        assertThat(conversations, hasItem(root.getConversationId()));
+    }
+
+    private Plaintext addMessage(BitmessageAddress from, BitmessageAddress to, Plaintext.Status status, Label... labels) {
+        ExtendedEncoding content = new Message.Builder()
+            .subject("Subject")
+            .body("Message")
+            .build();
+        return addMessage(from, to, content, status, labels);
+    }
+
+    private Plaintext addMessage(BitmessageAddress from, BitmessageAddress to,
+                                 ExtendedEncoding content, Plaintext.Status status, Label... labels) {
+        Plaintext message = new Plaintext.Builder(MSG)
+            .IV(TestUtils.randomInventoryVector())
+            .from(from)
+            .to(to)
+            .message(content)
+            .status(status)
+            .labels(Arrays.asList(labels))
+            .build();
+        repo.save(message);
+        return message;
+    }
+
+    private Plaintext storeConversation() {
+        Plaintext older = addMessage(identity, contactA,
+            new Message.Builder()
+                .subject("hey there")
+                .body("does it work?")
+                .build(),
+            Plaintext.Status.SENT, sent);
+
+        Plaintext root = addMessage(identity, contactA,
+            new Message.Builder()
+                .subject("new test")
+                .body("There's a new test in town!")
+                .build(),
+            Plaintext.Status.SENT, sent);
+
+        addMessage(contactA, identity,
+            new Message.Builder()
+                .subject("Re: new test")
+                .body("Nice!")
+                .addParent(root)
+                .build(),
+            Plaintext.Status.RECEIVED, inbox);
+
+        addMessage(contactA, identity,
+            new Message.Builder()
+                .subject("Re: new test")
+                .body("PS: it did work!")
+                .addParent(root)
+                .addParent(older)
+                .build(),
+            Plaintext.Status.RECEIVED, inbox);
+
+        return repo.getMessage(root.getId());
+    }
+
+    private Matcher<Plaintext> hasMessage(String subject, String body) {
+        return new BaseMatcher<Plaintext>() {
+            @Override
+            public void describeTo(Description description) {
+                description.appendText("Subject: ").appendText(subject);
+                description.appendText(", ");
+                description.appendText("Body: ").appendText(body);
+            }
+
+            @Override
+            public boolean matches(Object item) {
+                if (item instanceof Plaintext) {
+                    Plaintext message = (Plaintext) item;
+                    if (subject != null && !subject.equals(message.getSubject())) {
+                        return false;
+                    }
+                    if (body != null && !body.equals(message.getText())) {
+                        return false;
+                    }
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+        };
+    }
+}

repositories/src/test/java/ch/dissem/bitmessage/repository/TestJdbcConfig.java

@@ -16,11 +16,27 @@
 
 package ch.dissem.bitmessage.repository;
 
+import org.h2.tools.Server;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+
 /**
  * JdbcConfig to be used for tests. Uses an in-memory database and adds a useful {@link #reset()} method resetting
  * the database.
  */
 public class TestJdbcConfig extends JdbcConfig {
+    private static final Logger LOG = LoggerFactory.getLogger(TestJdbcConfig.class);
+
+    static {
+        try {
+            Server.createTcpServer().start();
+        } catch (SQLException e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
+
     public TestJdbcConfig() {
         super("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", "sa", null);
     }

wif/build.gradle

@@ -14,6 +14,6 @@ dependencies {
     compile project(':core')
     compile 'org.ini4j:ini4j:0.5.4'
     testCompile 'junit:junit:4.12'
-    testCompile 'org.mockito:mockito-core:1.10.19'
+    testCompile 'org.mockito:mockito-core:2.7.21'
     testCompile project(':cryptography-bc')
 }