Extended encoding basics

works for basic subject/body messages, no attachments and no other features supported yet

core/build.gradle

@@ -25,6 +25,7 @@ artifacts {
 
 dependencies {
     compile 'org.slf4j:slf4j-api:1.7.12'
+    compile 'org.msgpack:msgpack-core:0.8.11'
     testCompile 'junit:junit:4.12'
     testCompile 'org.hamcrest:hamcrest-library:1.3'
     testCompile 'org.mockito:mockito-core:1.10.19'

core/src/main/java/ch/dissem/bitmessage/entity/Plaintext.java

@@ -18,17 +18,23 @@ package ch.dissem.bitmessage.entity;
 
 import ch.dissem.bitmessage.entity.payload.Msg;
 import ch.dissem.bitmessage.entity.payload.Pubkey.Feature;
+import ch.dissem.bitmessage.entity.valueobject.Attachment;
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.exception.ApplicationException;
 import ch.dissem.bitmessage.factory.Factory;
-import ch.dissem.bitmessage.utils.*;
+import ch.dissem.bitmessage.utils.Decode;
+import ch.dissem.bitmessage.utils.Encode;
+import ch.dissem.bitmessage.utils.TTL;
+import ch.dissem.bitmessage.utils.UnixTime;
 
 import java.io.*;
 import java.nio.ByteBuffer;
 import java.util.*;
-import java.util.Collections;
 
+import static ch.dissem.bitmessage.entity.Plaintext.Encoding.EXTENDED;
+import static ch.dissem.bitmessage.entity.Plaintext.Encoding.SIMPLE;
 import static ch.dissem.bitmessage.utils.Singleton.cryptography;
 
 /**
@@ -42,6 +48,7 @@ public class Plaintext implements Streamable {
     private final long encoding;
     private final byte[] message;
     private final byte[] ackData;
+    private ExtendedEncoding extendedData;
     private ObjectMessage ackMessage;
     private Object id;
     private InventoryVector inventoryVector;
@@ -143,6 +150,10 @@ public class Plaintext implements Streamable {
         return labels;
     }
 
+    public Encoding getEncoding() {
+        return Encoding.fromCode(encoding);
+    }
+
     public long getStream() {
         return from.getStream();
     }
@@ -299,7 +310,13 @@ public class Plaintext implements Streamable {
     public String getSubject() {
         Scanner s = new Scanner(new ByteArrayInputStream(message), "UTF-8");
         String firstLine = s.nextLine();
-        if (encoding == 2) {
+        if (encoding == EXTENDED.code) {
+            if (getExtendedData().getMessage() == null) {
+                return null;
+            } else {
+                return extendedData.getMessage().getSubject();
+            }
+        } else if (encoding == SIMPLE.code) {
             return firstLine.substring("Subject:".length()).trim();
         } else if (firstLine.length() > 50) {
             return firstLine.substring(0, 50).trim() + "...";
@@ -309,14 +326,46 @@ public class Plaintext implements Streamable {
     }
 
     public String getText() {
-        try {
+        if (encoding == EXTENDED.code) {
-            String text = new String(message, "UTF-8");
+            if (getExtendedData().getMessage() == null) {
-            if (encoding == 2) {
+                return null;
-                return text.substring(text.indexOf("\nBody:") + 6);
+            } else {
+                return extendedData.getMessage().getBody();
             }
-            return text;
+        } else {
-        } catch (UnsupportedEncodingException e) {
+            try {
-            throw new ApplicationException(e);
+                String text = new String(message, "UTF-8");
+                if (encoding == SIMPLE.code) {
+                    return text.substring(text.indexOf("\nBody:") + 6);
+                }
+                return text;
+            } catch (UnsupportedEncodingException e) {
+                throw new ApplicationException(e);
+            }
+        }
+    }
+
+    protected ExtendedEncoding getExtendedData() {
+        if (extendedData == null && encoding == EXTENDED.code) {
+            // TODO: make sure errors are properly handled
+            extendedData = ExtendedEncoding.unzip(message);
+        }
+        return extendedData;
+    }
+
+    public List<InventoryVector> getParents() {
+        if (getExtendedData() == null || extendedData.getMessage() == null) {
+            return Collections.emptyList();
+        } else {
+            return extendedData.getMessage().getParents();
+        }
+    }
+
+    public List<Attachment> getFiles() {
+        if (getExtendedData() == null || extendedData.getMessage() == null) {
+            return Collections.emptyList();
+        } else {
+            return extendedData.getMessage().getFiles();
         }
     }
 
@@ -386,7 +435,7 @@ public class Plaintext implements Streamable {
     }
 
     public enum Encoding {
-        IGNORE(0), TRIVIAL(1), SIMPLE(2);
+        IGNORE(0), TRIVIAL(1), SIMPLE(2), EXTENDED(3);
 
         long code;
 
@@ -397,6 +446,15 @@ public class Plaintext implements Streamable {
         public long getCode() {
             return code;
         }
+
+        public static Encoding fromCode(long code) {
+            for (Encoding e : values()) {
+                if (e.getCode() == code) {
+                    return e;
+                }
+            }
+            return null;
+        }
     }
 
     public enum Status {
@@ -517,9 +575,15 @@ public class Plaintext implements Streamable {
             return this;
         }
 
+        public Builder message(ExtendedEncoding message) {
+            this.encoding = EXTENDED.getCode();
+            this.message = message.zip();
+            return this;
+        }
+
         public Builder message(String subject, String message) {
             try {
-                this.encoding = Encoding.SIMPLE.getCode();
+                this.encoding = SIMPLE.getCode();
                 this.message = ("Subject:" + subject + '\n' + "Body:" + message).getBytes("UTF-8");
             } catch (UnsupportedEncodingException e) {
                 throw new ApplicationException(e);

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/Attachment.java

@@ -0,0 +1,95 @@
+package ch.dissem.bitmessage.entity.valueobject;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A "file" attachment as used by extended encoding type messages. Could either be an attachment,
+ * or used inline to be used by a HTML message, for example.
+ */
+public class Attachment implements Serializable {
+    private static final long serialVersionUID = 7319139427666943189L;
+
+    private String name;
+    private byte[] data;
+    private String type;
+    private Disposition disposition;
+
+    public String getName() {
+        return name;
+    }
+
+    public byte[] getData() {
+        return data;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public Disposition getDisposition() {
+        return disposition;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Attachment that = (Attachment) o;
+        return Objects.equals(name, that.name) &&
+            Arrays.equals(data, that.data) &&
+            Objects.equals(type, that.type) &&
+            disposition == that.disposition;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, data, type, disposition);
+    }
+
+    private enum Disposition {
+        inline, attachment
+    }
+
+    public static final class Builder {
+        private String name;
+        private byte[] data;
+        private String type;
+        private Disposition disposition;
+
+        public Builder name(String name) {
+            this.name = name;
+            return this;
+        }
+
+        public Builder data(byte[] data) {
+            this.data = data;
+            return this;
+        }
+
+        public Builder type(String type) {
+            this.type = type;
+            return this;
+        }
+
+        public Builder inline() {
+            this.disposition = Disposition.inline;
+            return this;
+        }
+
+        public Builder attachment() {
+            this.disposition = Disposition.attachment;
+            return this;
+        }
+
+        public Attachment build() {
+            Attachment attachment = new Attachment();
+            attachment.type = this.type;
+            attachment.disposition = this.disposition;
+            attachment.data = this.data;
+            attachment.name = this.name;
+            return attachment;
+        }
+    }
+}

core/src/main/java/ch/dissem/bitmessage/entity/valueobject/ExtendedEncoding.java

@@ -0,0 +1,203 @@
+package ch.dissem.bitmessage.entity.valueobject;
+
+import ch.dissem.bitmessage.exception.ApplicationException;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessagePacker;
+import org.msgpack.core.MessageUnpacker;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.zip.DeflaterOutputStream;
+import java.util.zip.InflaterInputStream;
+
+/**
+ * Extended encoding message object.
+ */
+public class ExtendedEncoding implements Serializable {
+    private static final long serialVersionUID = 3876871488247305200L;
+
+    private Message message;
+
+    public ExtendedEncoding(Message message) {
+        this.message = message;
+    }
+
+    private ExtendedEncoding() {
+    }
+
+    public Message getMessage() {
+        return message;
+    }
+
+    public byte[] zip() {
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream();
+             DeflaterOutputStream zipper = new DeflaterOutputStream(out)) {
+
+            MessagePacker packer = MessagePack.newDefaultPacker(zipper);
+            // FIXME: this should work for trivial cases
+            if (message != null) {
+                message.pack(packer);
+            }
+            packer.close();
+            zipper.close();
+            return out.toByteArray();
+        } catch (IOException e) {
+            throw new ApplicationException(e);
+        }
+    }
+
+    public static ExtendedEncoding unzip(byte[] zippedData) {
+        ExtendedEncoding result = new ExtendedEncoding();
+        try (InflaterInputStream unzipper = new InflaterInputStream(new ByteArrayInputStream(zippedData))) {
+            MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(unzipper);
+            int mapSize = unpacker.unpackMapHeader();
+            for (int i = 0; i < mapSize; i++) {
+                String key = unpacker.unpackString();
+                switch (key) {
+                    case "":
+                        switch (unpacker.unpackString()) {
+                            case "message":
+                                result.message = new Message();
+                                break;
+                        }
+                        break;
+                    case "subject":
+                        result.message.subject = unpacker.unpackString();
+                        break;
+                    case "body":
+                        result.message.body = unpacker.unpackString();
+                        break;
+                    default:
+                        break;
+                }
+            }
+        } catch (IOException e) {
+            throw new ApplicationException(e);
+        }
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ExtendedEncoding that = (ExtendedEncoding) o;
+        return Objects.equals(message, that.message);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(message);
+    }
+
+    public static class Message implements Serializable {
+        private static final long serialVersionUID = -2724977231484285467L;
+
+        private String subject;
+        private String body;
+        private List<InventoryVector> parents;
+        private List<Attachment> files;
+
+        private Message() {
+            parents = Collections.emptyList();
+            files = Collections.emptyList();
+        }
+
+        private Message(Builder builder) {
+            subject = builder.subject;
+            body = builder.body;
+            parents = Collections.unmodifiableList(builder.parents);
+            files = Collections.unmodifiableList(builder.files);
+        }
+
+        public String getSubject() {
+            return subject;
+        }
+
+        public String getBody() {
+            return body;
+        }
+
+        public List<InventoryVector> getParents() {
+            return parents;
+        }
+
+        public List<Attachment> getFiles() {
+            return files;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Message message = (Message) o;
+            return Objects.equals(subject, message.subject) &&
+                Objects.equals(body, message.body) &&
+                Objects.equals(parents, message.parents) &&
+                Objects.equals(files, message.files);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(subject, body, parents, files);
+        }
+
+        public void pack(MessagePacker packer) throws IOException {
+            packer.packMapHeader(3);
+            packer.packString("");
+            packer.packString("message");
+            packer.packString("subject");
+            packer.packString(subject);
+            packer.packString("body");
+            packer.packString(body);
+        }
+
+        public static class Builder {
+            private String subject;
+            private String body;
+            private List<InventoryVector> parents = new LinkedList<>();
+            private List<Attachment> files = new LinkedList<>();
+
+            private Builder() {
+            }
+
+            public Builder subject(String subject) {
+                this.subject = subject;
+                return this;
+            }
+
+            public Builder body(String body) {
+                this.body = body;
+                return this;
+            }
+
+            public Builder addParent(InventoryVector iv) {
+                parents.add(iv);
+                return this;
+            }
+
+            public Builder addFile(Attachment file) {
+                files.add(file);
+                return this;
+            }
+
+            public ExtendedEncoding build() {
+                return new ExtendedEncoding(new Message(this));
+            }
+        }
+    }
+
+    public static class Builder {
+        public Message.Builder message() {
+            return new Message.Builder();
+        }
+
+        // TODO: vote (etc.?)
+    }
+}

core/src/test/java/ch/dissem/bitmessage/entity/SerializationTest.java

@@ -17,6 +17,7 @@
 package ch.dissem.bitmessage.entity;
 
 import ch.dissem.bitmessage.entity.payload.*;
+import ch.dissem.bitmessage.entity.valueobject.ExtendedEncoding;
 import ch.dissem.bitmessage.entity.valueobject.InventoryVector;
 import ch.dissem.bitmessage.entity.valueobject.Label;
 import ch.dissem.bitmessage.factory.Factory;
@@ -79,12 +80,37 @@ public class SerializationTest extends TestBase {
     @Test
     public void ensurePlaintextIsSerializedAndDeserializedCorrectly() throws Exception {
         Plaintext p1 = new Plaintext.Builder(MSG)
-                .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .to(TestUtils.loadContact())
+            .to(TestUtils.loadContact())
-                .message("Subject", "Message")
+            .message("Subject", "Message")
-                .ackData("ackMessage".getBytes())
+            .ackData("ackMessage".getBytes())
-                .signature(new byte[0])
+            .signature(new byte[0])
-                .build();
+            .build();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        p1.write(out);
+        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+        Plaintext p2 = Plaintext.read(MSG, in);
+
+        // Received is automatically set on deserialization, so we'll need to set it to 0
+        Field received = Plaintext.class.getDeclaredField("received");
+        received.setAccessible(true);
+        received.set(p2, 0L);
+
+        assertEquals(p1, p2);
+    }
+
+    @Test
+    public void ensurePlaintextWithExtendedEncodingIsSerializedAndDeserializedCorrectly() throws Exception {
+        Plaintext p1 = new Plaintext.Builder(MSG)
+            .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .to(TestUtils.loadContact())
+            .message(new ExtendedEncoding.Builder().message()
+                .subject("Subject")
+                .body("Message")
+                .build())
+            .ackData("ackMessage".getBytes())
+            .signature(new byte[0])
+            .build();
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         p1.write(out);
         ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
@@ -101,12 +127,12 @@ public class SerializationTest extends TestBase {
     @Test
     public void ensurePlaintextWithAckMessageIsSerializedAndDeserializedCorrectly() throws Exception {
         Plaintext p1 = new Plaintext.Builder(MSG)
-                .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .from(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .to(TestUtils.loadContact())
+            .to(TestUtils.loadContact())
-                .message("Subject", "Message")
+            .message("Subject", "Message")
-                .ackData("ackMessage".getBytes())
+            .ackData("ackMessage".getBytes())
-                .signature(new byte[0])
+            .signature(new byte[0])
-                .build();
+            .build();
         ObjectMessage ackMessage1 = p1.getAckMessage();
         assertNotNull(ackMessage1);
 
@@ -156,11 +182,11 @@ public class SerializationTest extends TestBase {
     @Test
     public void ensureSystemSerializationWorks() throws Exception {
         Plaintext plaintext = new Plaintext.Builder(MSG)
-                .from(TestUtils.loadContact())
+            .from(TestUtils.loadContact())
-                .to(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
+            .to(TestUtils.loadIdentity("BM-2cSqjfJ8xK6UUn5Rw3RpdGQ9RsDkBhWnS8"))
-                .labels(Collections.singletonList(new Label("Test", Label.Type.INBOX, 0)))
+            .labels(Collections.singletonList(new Label("Test", Label.Type.INBOX, 0)))
-                .message("Test", "Test Test.\nTest")
+            .message("Test", "Test Test.\nTest")
-                .build();
+            .build();
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         ObjectOutputStream oos = new ObjectOutputStream(out);
         oos.writeObject(plaintext);