From 8d7b9f64573cf10d8e3aec30a9670d610926f35e Mon Sep 17 00:00:00 2001 From: Christian Basler Date: Fri, 17 Apr 2015 11:17:39 +0200 Subject: [PATCH] Improved POW to use all cores. Interestingly, speed up factor seems to be greater than the number of cores (but still quite slow for real word application) --- .../ports/MultiThreadedPOWEngine.java | 111 ++++++++++++++++++ .../bitmessage/ports/ProofOfWorkEngine.java | 4 +- .../bitmessage/ports/SimplePOWEngine.java | 5 +- .../ch/dissem/bitmessage/utils/Bytes.java | 14 +++ .../ch/dissem/bitmessage/utils/Decode.java | 6 +- .../ch/dissem/bitmessage/utils/Encode.java | 1 + .../ch/dissem/bitmessage/utils/Security.java | 2 +- .../ports/ProofOfWorkEngineTest.java | 48 ++++++++ .../ch/dissem/bitmessage/utils/BytesTest.java | 21 ++-- .../dissem/bitmessage/utils/SecurityTest.java | 6 +- .../ch/dissem/bitmessage/utils/TestUtils.java | 31 +++++ 11 files changed, 228 insertions(+), 21 deletions(-) create mode 100644 domain/src/main/java/ch/dissem/bitmessage/ports/MultiThreadedPOWEngine.java create mode 100644 domain/src/test/java/ch/dissem/bitmessage/ports/ProofOfWorkEngineTest.java create mode 100644 domain/src/test/java/ch/dissem/bitmessage/utils/TestUtils.java diff --git a/domain/src/main/java/ch/dissem/bitmessage/ports/MultiThreadedPOWEngine.java b/domain/src/main/java/ch/dissem/bitmessage/ports/MultiThreadedPOWEngine.java new file mode 100644 index 0000000..98bdceb --- /dev/null +++ b/domain/src/main/java/ch/dissem/bitmessage/ports/MultiThreadedPOWEngine.java @@ -0,0 +1,111 @@ +/* + * Copyright 2015 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.ports; + +import ch.dissem.bitmessage.utils.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; + +import static ch.dissem.bitmessage.utils.Bytes.inc; + +/** + * Created by chris on 14.04.15. + */ +public class MultiThreadedPOWEngine implements ProofOfWorkEngine { + private static Logger LOG = LoggerFactory.getLogger(MultiThreadedPOWEngine.class); + + @Override + public byte[] calculateNonce(byte[] initialHash, byte[] target) { + int cores = Runtime.getRuntime().availableProcessors(); + if (cores > 255) cores = 255; + LOG.info("Doing POW using " + cores + " cores"); + long time = System.currentTimeMillis(); + List workers = new ArrayList<>(cores); + for (int i = 0; i < cores; i++) { + Worker w = new Worker(workers, (byte) cores, i, initialHash, target); + workers.add(w); + } + for (Worker w : workers) { + w.start(); + } + for (Worker w : workers) { + try { + w.join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (w.isSuccessful()) { + LOG.info("Nonce calculated in " + ((System.currentTimeMillis() - time) / 1000) + " seconds"); + return w.getNonce(); + } + } + throw new RuntimeException("All workers ended without yielding a nonce - something is seriously broken!"); + } + + private static class Worker extends Thread { + private final byte numberOfCores; + private final List workers; + private final byte[] initialHash; + private final byte[] target; + private final MessageDigest mda; + private final byte[] nonce = new byte[8]; + private boolean successful = false; + + public Worker(List workers, byte numberOfCores, int core, byte[] initialHash, byte[] target) { + this.numberOfCores = numberOfCores; + this.workers = workers; + this.initialHash = initialHash; + this.target = target; + this.nonce[7] = (byte) core; + try { + mda = MessageDigest.getInstance("SHA-512"); + } catch (NoSuchAlgorithmException e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + public boolean isSuccessful() { + return successful; + } + + public byte[] getNonce() { + return nonce; + } + + @Override + public void run() { + do { + inc(nonce, numberOfCores); + mda.update(nonce); + mda.update(initialHash); + if (!Bytes.lt(target, mda.digest(mda.digest()), 8)) { + successful = true; + for (Worker w : workers) { + w.interrupt(); + } + return; + } + } while (!Thread.interrupted()); + } + } +} diff --git a/domain/src/main/java/ch/dissem/bitmessage/ports/ProofOfWorkEngine.java b/domain/src/main/java/ch/dissem/bitmessage/ports/ProofOfWorkEngine.java index 0c14a86..31f6657 100644 --- a/domain/src/main/java/ch/dissem/bitmessage/ports/ProofOfWorkEngine.java +++ b/domain/src/main/java/ch/dissem/bitmessage/ports/ProofOfWorkEngine.java @@ -26,9 +26,7 @@ public interface ProofOfWorkEngine { * * @param initialHash the SHA-512 hash of the object to send, sans nonce * @param target the target, representing an unsigned long - * @param nonceTrialsPerByte - * @param extraBytes * @return 8 bytes nonce */ - byte[] calculateNonce(byte[] initialHash, byte[] target, long nonceTrialsPerByte, long extraBytes); + byte[] calculateNonce(byte[] initialHash, byte[] target); } diff --git a/domain/src/main/java/ch/dissem/bitmessage/ports/SimplePOWEngine.java b/domain/src/main/java/ch/dissem/bitmessage/ports/SimplePOWEngine.java index 85edb27..0d9d392 100644 --- a/domain/src/main/java/ch/dissem/bitmessage/ports/SimplePOWEngine.java +++ b/domain/src/main/java/ch/dissem/bitmessage/ports/SimplePOWEngine.java @@ -19,7 +19,6 @@ package ch.dissem.bitmessage.ports; import ch.dissem.bitmessage.utils.Bytes; import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import static ch.dissem.bitmessage.utils.Bytes.inc; @@ -28,12 +27,12 @@ import static ch.dissem.bitmessage.utils.Bytes.inc; */ public class SimplePOWEngine implements ProofOfWorkEngine { @Override - public byte[] calculateNonce(byte[] initialHash, byte[] target, long nonceTrialsPerByte, long extraBytes) { + public byte[] calculateNonce(byte[] initialHash, byte[] target) { byte[] nonce = new byte[8]; MessageDigest mda; try { mda = MessageDigest.getInstance("SHA-512"); - } catch (NoSuchAlgorithmException e) { + } catch (Exception e) { throw new RuntimeException(e); } do { diff --git a/domain/src/main/java/ch/dissem/bitmessage/utils/Bytes.java b/domain/src/main/java/ch/dissem/bitmessage/utils/Bytes.java index b83be90..0da0eb6 100644 --- a/domain/src/main/java/ch/dissem/bitmessage/utils/Bytes.java +++ b/domain/src/main/java/ch/dissem/bitmessage/utils/Bytes.java @@ -27,6 +27,20 @@ public class Bytes { } } + public static void inc(byte[] nonce, byte value) { + int i = nonce.length - 1; + nonce[i] += value; + if (value > 0 && (nonce[i] < 0 || nonce[i] >= value)) + return; + if (value < 0 && (nonce[i] < 0 && nonce[i] >= value)) + return; + + for (i = i - 1; i >= 0; i--) { + nonce[i]++; + if (nonce[i] != 0) break; + } + } + public static boolean lt(byte[] a, byte[] b) { byte[] max = (a.length > b.length ? a : b); byte[] min = (max == a ? b : a); diff --git a/domain/src/main/java/ch/dissem/bitmessage/utils/Decode.java b/domain/src/main/java/ch/dissem/bitmessage/utils/Decode.java index 20a18f7..4567d5a 100644 --- a/domain/src/main/java/ch/dissem/bitmessage/utils/Decode.java +++ b/domain/src/main/java/ch/dissem/bitmessage/utils/Decode.java @@ -112,9 +112,7 @@ public class Decode { public static String varString(InputStream stream) throws IOException { int length = (int) varInt(stream); // FIXME: technically, it says the length in characters, but I think this one might be correct - byte[] bytes = new byte[length]; - // FIXME: I'm also not quite sure if this works, maybe the read return value needs to be handled properly - stream.read(bytes); - return new String(bytes, "utf-8"); + // otherwise it will get complicated, as we'll need to read UTF-8 char by char... + return new String(bytes(stream, length), "utf-8"); } } diff --git a/domain/src/main/java/ch/dissem/bitmessage/utils/Encode.java b/domain/src/main/java/ch/dissem/bitmessage/utils/Encode.java index 6669e3a..5ea2814 100644 --- a/domain/src/main/java/ch/dissem/bitmessage/utils/Encode.java +++ b/domain/src/main/java/ch/dissem/bitmessage/utils/Encode.java @@ -106,6 +106,7 @@ public class Encode { public static void varString(String value, OutputStream stream) throws IOException { byte[] bytes = value.getBytes("utf-8"); // FIXME: technically, it says the length in characters, but I think this one might be correct + // see also Decode#varString() varInt(bytes.length, stream); stream.write(bytes); } diff --git a/domain/src/main/java/ch/dissem/bitmessage/utils/Security.java b/domain/src/main/java/ch/dissem/bitmessage/utils/Security.java index 556ebc9..5ea507d 100644 --- a/domain/src/main/java/ch/dissem/bitmessage/utils/Security.java +++ b/domain/src/main/java/ch/dissem/bitmessage/utils/Security.java @@ -71,7 +71,7 @@ public class Security { byte[] target = getProofOfWorkTarget(object, nonceTrialsPerByte, extraBytes); - byte[] nonce = worker.calculateNonce(initialHash, target, nonceTrialsPerByte, extraBytes); + byte[] nonce = worker.calculateNonce(initialHash, target); object.setNonce(nonce); } diff --git a/domain/src/test/java/ch/dissem/bitmessage/ports/ProofOfWorkEngineTest.java b/domain/src/test/java/ch/dissem/bitmessage/ports/ProofOfWorkEngineTest.java new file mode 100644 index 0000000..1d25229 --- /dev/null +++ b/domain/src/test/java/ch/dissem/bitmessage/ports/ProofOfWorkEngineTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2015 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.ports; + +import ch.dissem.bitmessage.utils.Bytes; +import ch.dissem.bitmessage.utils.Security; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Created by chris on 17.04.15. + */ +public class ProofOfWorkEngineTest { + @Test + public void testSimplePOWEngine() { + testPOW(new SimplePOWEngine()); + } + + @Test + public void testThreadedPOWEngine() { + testPOW(new MultiThreadedPOWEngine()); + } + + private void testPOW(ProofOfWorkEngine engine) { + long time = System.currentTimeMillis(); + byte[] initialHash = Security.sha512(new byte[]{1, 3, 6, 4}); + byte[] target = {0, 0, 0, -1, -1, -1, -1, -1}; + + byte[] nonce = engine.calculateNonce(initialHash, target); + System.out.println("Calculating nonce took " + (System.currentTimeMillis() - time) + "ms"); + assertTrue(Bytes.lt(Security.doubleSha512(nonce, initialHash), target, 8)); + } +} diff --git a/domain/src/test/java/ch/dissem/bitmessage/utils/BytesTest.java b/domain/src/test/java/ch/dissem/bitmessage/utils/BytesTest.java index 8a4f25c..797e1e6 100644 --- a/domain/src/test/java/ch/dissem/bitmessage/utils/BytesTest.java +++ b/domain/src/test/java/ch/dissem/bitmessage/utils/BytesTest.java @@ -18,7 +18,6 @@ package ch.dissem.bitmessage.utils; import org.junit.Test; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigInteger; import java.util.Random; @@ -30,19 +29,28 @@ import static org.junit.Assert.assertEquals; * Created by chris on 10.04.15. */ public class BytesTest { + public static final Random rnd = new Random(); + @Test public void testIncrement() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Encode.int16(256, out); - byte[] bytes = {0, -1}; Bytes.inc(bytes); - assertArrayEquals(out.toByteArray(), bytes); + assertArrayEquals(TestUtils.int16(256), bytes); + } + + @Test + public void testIncrementByValue() throws IOException { + for (int v = 0; v < 256; v++) { + for (int i = 1; i < 256; i++) { + byte[] bytes = {0, (byte) v}; + Bytes.inc(bytes, (byte) i); + assertArrayEquals("value = " + v + "; inc = " + i + "; expected = " + (v + i), TestUtils.int16(v + i), bytes); + } + } } @Test public void testLowerThan() { - Random rnd = new Random(); for (int i = 0; i < 1000; i++) { BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); @@ -53,7 +61,6 @@ public class BytesTest { @Test public void testLowerThanBounded() { - Random rnd = new Random(); for (int i = 0; i < 1000; i++) { BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); diff --git a/domain/src/test/java/ch/dissem/bitmessage/utils/SecurityTest.java b/domain/src/test/java/ch/dissem/bitmessage/utils/SecurityTest.java index 756017c..ca68dd8 100644 --- a/domain/src/test/java/ch/dissem/bitmessage/utils/SecurityTest.java +++ b/domain/src/test/java/ch/dissem/bitmessage/utils/SecurityTest.java @@ -18,7 +18,7 @@ package ch.dissem.bitmessage.utils; import ch.dissem.bitmessage.entity.ObjectMessage; import ch.dissem.bitmessage.entity.payload.GenericPayload; -import ch.dissem.bitmessage.ports.SimplePOWEngine; +import ch.dissem.bitmessage.ports.MultiThreadedPOWEngine; import org.junit.Test; import javax.xml.bind.DatatypeConverter; @@ -86,8 +86,8 @@ public class SecurityTest { .expiresTime(expires.getTimeInMillis() / 1000) .payload(new GenericPayload(1, new byte[0])) .build(); - Security.doProofOfWork(objectMessage, new SimplePOWEngine(), 10, 10); - Security.checkProofOfWork(objectMessage, 10, 10); + Security.doProofOfWork(objectMessage, new MultiThreadedPOWEngine(), 1000, 1000); + Security.checkProofOfWork(objectMessage, 1000, 1000); } @Test diff --git a/domain/src/test/java/ch/dissem/bitmessage/utils/TestUtils.java b/domain/src/test/java/ch/dissem/bitmessage/utils/TestUtils.java new file mode 100644 index 0000000..6a7f244 --- /dev/null +++ b/domain/src/test/java/ch/dissem/bitmessage/utils/TestUtils.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Christian Basler + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.dissem.bitmessage.utils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * If there's ever a need for this in production code, it should be rewritten to be more efficient. + */ +public class TestUtils { + public static byte[] int16(int number) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Encode.int16(number, out); + return out.toByteArray(); + } +}