Improved POW to use all cores. Interestingly, speed up factor seems to be greater than the number of cores (but still quite slow for real word application)

domain/src/main/java/ch/dissem/bitmessage/ports/MultiThreadedPOWEngine.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2015 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.ports;
+
+import ch.dissem.bitmessage.utils.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static ch.dissem.bitmessage.utils.Bytes.inc;
+
+/**
+ * Created by chris on 14.04.15.
+ */
+public class MultiThreadedPOWEngine implements ProofOfWorkEngine {
+    private static Logger LOG = LoggerFactory.getLogger(MultiThreadedPOWEngine.class);
+
+    @Override
+    public byte[] calculateNonce(byte[] initialHash, byte[] target) {
+        int cores = Runtime.getRuntime().availableProcessors();
+        if (cores > 255) cores = 255;
+        LOG.info("Doing POW using " + cores + " cores");
+        long time = System.currentTimeMillis();
+        List<Worker> workers = new ArrayList<>(cores);
+        for (int i = 0; i < cores; i++) {
+            Worker w = new Worker(workers, (byte) cores, i, initialHash, target);
+            workers.add(w);
+        }
+        for (Worker w : workers) {
+            w.start();
+        }
+        for (Worker w : workers) {
+            try {
+                w.join();
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            if (w.isSuccessful()) {
+                LOG.info("Nonce calculated in " + ((System.currentTimeMillis() - time) / 1000) + " seconds");
+                return w.getNonce();
+            }
+        }
+        throw new RuntimeException("All workers ended without yielding a nonce - something is seriously broken!");
+    }
+
+    private static class Worker extends Thread {
+        private final byte numberOfCores;
+        private final List<Worker> workers;
+        private final byte[] initialHash;
+        private final byte[] target;
+        private final MessageDigest mda;
+        private final byte[] nonce = new byte[8];
+        private boolean successful = false;
+
+        public Worker(List<Worker> workers, byte numberOfCores, int core, byte[] initialHash, byte[] target) {
+            this.numberOfCores = numberOfCores;
+            this.workers = workers;
+            this.initialHash = initialHash;
+            this.target = target;
+            this.nonce[7] = (byte) core;
+            try {
+                mda = MessageDigest.getInstance("SHA-512");
+            } catch (NoSuchAlgorithmException e) {
+                LOG.error(e.getMessage(), e);
+                throw new RuntimeException(e);
+            }
+        }
+
+        public boolean isSuccessful() {
+            return successful;
+        }
+
+        public byte[] getNonce() {
+            return nonce;
+        }
+
+        @Override
+        public void run() {
+            do {
+                inc(nonce, numberOfCores);
+                mda.update(nonce);
+                mda.update(initialHash);
+                if (!Bytes.lt(target, mda.digest(mda.digest()), 8)) {
+                    successful = true;
+                    for (Worker w : workers) {
+                        w.interrupt();
+                    }
+                    return;
+                }
+            } while (!Thread.interrupted());
+        }
+    }
+}

domain/src/main/java/ch/dissem/bitmessage/ports/ProofOfWorkEngine.java

@@ -26,9 +26,7 @@ public interface ProofOfWorkEngine {
      *
      * @param initialHash        the SHA-512 hash of the object to send, sans nonce
      * @param target             the target, representing an unsigned long
-     * @param nonceTrialsPerByte
-     * @param extraBytes
      * @return 8 bytes nonce
      */
-    byte[] calculateNonce(byte[] initialHash, byte[] target, long nonceTrialsPerByte, long extraBytes);
+    byte[] calculateNonce(byte[] initialHash, byte[] target);
 }

domain/src/main/java/ch/dissem/bitmessage/ports/SimplePOWEngine.java

@@ -19,7 +19,6 @@ package ch.dissem.bitmessage.ports;
 import ch.dissem.bitmessage.utils.Bytes;
 
 import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 
 import static ch.dissem.bitmessage.utils.Bytes.inc;
 
@@ -28,12 +27,12 @@ import static ch.dissem.bitmessage.utils.Bytes.inc;
  */
 public class SimplePOWEngine implements ProofOfWorkEngine {
     @Override
-    public byte[] calculateNonce(byte[] initialHash, byte[] target, long nonceTrialsPerByte, long extraBytes) {
+    public byte[] calculateNonce(byte[] initialHash, byte[] target) {
         byte[] nonce = new byte[8];
         MessageDigest mda;
         try {
             mda = MessageDigest.getInstance("SHA-512");
-        } catch (NoSuchAlgorithmException e) {
+        } catch (Exception e) {
             throw new RuntimeException(e);
         }
         do {

domain/src/main/java/ch/dissem/bitmessage/utils/Bytes.java

@@ -27,6 +27,20 @@ public class Bytes {
         }
     }
 
+    public static void inc(byte[] nonce, byte value) {
+        int i = nonce.length - 1;
+        nonce[i] += value;
+        if (value > 0 && (nonce[i] < 0 || nonce[i] >= value))
+            return;
+        if (value < 0 && (nonce[i] < 0 && nonce[i] >= value))
+            return;
+
+        for (i = i - 1; i >= 0; i--) {
+            nonce[i]++;
+            if (nonce[i] != 0) break;
+        }
+    }
+
     public static boolean lt(byte[] a, byte[] b) {
         byte[] max = (a.length > b.length ? a : b);
         byte[] min = (max == a ? b : a);

domain/src/main/java/ch/dissem/bitmessage/utils/Decode.java

@@ -112,9 +112,7 @@ public class Decode {
     public static String varString(InputStream stream) throws IOException {
         int length = (int) varInt(stream);
         // FIXME: technically, it says the length in characters, but I think this one might be correct
-        byte[] bytes = new byte[length];
+        // otherwise it will get complicated, as we'll need to read UTF-8 char by char...
-        // FIXME: I'm also not quite sure if this works, maybe the read return value needs to be handled properly
+        return new String(bytes(stream, length), "utf-8");
-        stream.read(bytes);
-        return new String(bytes, "utf-8");
     }
 }

domain/src/main/java/ch/dissem/bitmessage/utils/Encode.java

@@ -106,6 +106,7 @@ public class Encode {
     public static void varString(String value, OutputStream stream) throws IOException {
         byte[] bytes = value.getBytes("utf-8");
         // FIXME: technically, it says the length in characters, but I think this one might be correct
+        // see also Decode#varString()
         varInt(bytes.length, stream);
         stream.write(bytes);
     }

domain/src/main/java/ch/dissem/bitmessage/utils/Security.java

@@ -71,7 +71,7 @@ public class Security {
 
         byte[] target = getProofOfWorkTarget(object, nonceTrialsPerByte, extraBytes);
 
-        byte[] nonce = worker.calculateNonce(initialHash, target, nonceTrialsPerByte, extraBytes);
+        byte[] nonce = worker.calculateNonce(initialHash, target);
         object.setNonce(nonce);
     }
 

domain/src/test/java/ch/dissem/bitmessage/ports/ProofOfWorkEngineTest.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.ports;
+
+import ch.dissem.bitmessage.utils.Bytes;
+import ch.dissem.bitmessage.utils.Security;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Created by chris on 17.04.15.
+ */
+public class ProofOfWorkEngineTest {
+    @Test
+    public void testSimplePOWEngine() {
+        testPOW(new SimplePOWEngine());
+    }
+
+    @Test
+    public void testThreadedPOWEngine() {
+        testPOW(new MultiThreadedPOWEngine());
+    }
+
+    private void testPOW(ProofOfWorkEngine engine) {
+        long time = System.currentTimeMillis();
+        byte[] initialHash = Security.sha512(new byte[]{1, 3, 6, 4});
+        byte[] target = {0, 0, 0, -1, -1, -1, -1, -1};
+
+        byte[] nonce = engine.calculateNonce(initialHash, target);
+        System.out.println("Calculating nonce took " + (System.currentTimeMillis() - time) + "ms");
+        assertTrue(Bytes.lt(Security.doubleSha512(nonce, initialHash), target, 8));
+    }
+}

domain/src/test/java/ch/dissem/bitmessage/utils/BytesTest.java

@@ -18,7 +18,6 @@ package ch.dissem.bitmessage.utils;
 
 import org.junit.Test;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.math.BigInteger;
 import java.util.Random;
@@ -30,19 +29,28 @@ import static org.junit.Assert.assertEquals;
  * Created by chris on 10.04.15.
  */
 public class BytesTest {
+    public static final Random rnd = new Random();
+
     @Test
     public void testIncrement() throws IOException {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        Encode.int16(256, out);
-
         byte[] bytes = {0, -1};
         Bytes.inc(bytes);
-        assertArrayEquals(out.toByteArray(), bytes);
+        assertArrayEquals(TestUtils.int16(256), bytes);
+    }
+
+    @Test
+    public void testIncrementByValue() throws IOException {
+        for (int v = 0; v < 256; v++) {
+            for (int i = 1; i < 256; i++) {
+                byte[] bytes = {0, (byte) v};
+                Bytes.inc(bytes, (byte) i);
+                assertArrayEquals("value = " + v + "; inc = " + i + "; expected = " + (v + i), TestUtils.int16(v + i), bytes);
+            }
+        }
     }
 
     @Test
     public void testLowerThan() {
-        Random rnd = new Random();
         for (int i = 0; i < 1000; i++) {
             BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();
             BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();
@@ -53,7 +61,6 @@ public class BytesTest {
 
     @Test
     public void testLowerThanBounded() {
-        Random rnd = new Random();
         for (int i = 0; i < 1000; i++) {
             BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();
             BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();

domain/src/test/java/ch/dissem/bitmessage/utils/SecurityTest.java

@@ -18,7 +18,7 @@ package ch.dissem.bitmessage.utils;
 
 import ch.dissem.bitmessage.entity.ObjectMessage;
 import ch.dissem.bitmessage.entity.payload.GenericPayload;
-import ch.dissem.bitmessage.ports.SimplePOWEngine;
+import ch.dissem.bitmessage.ports.MultiThreadedPOWEngine;
 import org.junit.Test;
 
 import javax.xml.bind.DatatypeConverter;
@@ -86,8 +86,8 @@ public class SecurityTest {
                 .expiresTime(expires.getTimeInMillis() / 1000)
                 .payload(new GenericPayload(1, new byte[0]))
                 .build();
-        Security.doProofOfWork(objectMessage, new SimplePOWEngine(), 10, 10);
+        Security.doProofOfWork(objectMessage, new MultiThreadedPOWEngine(), 1000, 1000);
-        Security.checkProofOfWork(objectMessage, 10, 10);
+        Security.checkProofOfWork(objectMessage, 1000, 1000);
     }
 
     @Test

domain/src/test/java/ch/dissem/bitmessage/utils/TestUtils.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2015 Christian Basler
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.dissem.bitmessage.utils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+/**
+ * If there's ever a need for this in production code, it should be rewritten to be more efficient.
+ */
+public class TestUtils {
+    public static byte[] int16(int number) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        Encode.int16(number, out);
+        return out.toByteArray();
+    }
+}