Improved POW to use all cores. Interestingly, speed up factor seems to be greater than the number of cores (but still quite slow for real word application)

This commit is contained in:
Christian Basler 2015-04-17 11:17:39 +02:00
parent 388a10fe8a
commit 8d7b9f6457
11 changed files with 228 additions and 21 deletions

View File

@ -0,0 +1,111 @@
/*
* Copyright 2015 Christian Basler
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.dissem.bitmessage.ports;
import ch.dissem.bitmessage.utils.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import static ch.dissem.bitmessage.utils.Bytes.inc;
/**
* Created by chris on 14.04.15.
*/
public class MultiThreadedPOWEngine implements ProofOfWorkEngine {
private static Logger LOG = LoggerFactory.getLogger(MultiThreadedPOWEngine.class);
@Override
public byte[] calculateNonce(byte[] initialHash, byte[] target) {
int cores = Runtime.getRuntime().availableProcessors();
if (cores > 255) cores = 255;
LOG.info("Doing POW using " + cores + " cores");
long time = System.currentTimeMillis();
List<Worker> workers = new ArrayList<>(cores);
for (int i = 0; i < cores; i++) {
Worker w = new Worker(workers, (byte) cores, i, initialHash, target);
workers.add(w);
}
for (Worker w : workers) {
w.start();
}
for (Worker w : workers) {
try {
w.join();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (w.isSuccessful()) {
LOG.info("Nonce calculated in " + ((System.currentTimeMillis() - time) / 1000) + " seconds");
return w.getNonce();
}
}
throw new RuntimeException("All workers ended without yielding a nonce - something is seriously broken!");
}
private static class Worker extends Thread {
private final byte numberOfCores;
private final List<Worker> workers;
private final byte[] initialHash;
private final byte[] target;
private final MessageDigest mda;
private final byte[] nonce = new byte[8];
private boolean successful = false;
public Worker(List<Worker> workers, byte numberOfCores, int core, byte[] initialHash, byte[] target) {
this.numberOfCores = numberOfCores;
this.workers = workers;
this.initialHash = initialHash;
this.target = target;
this.nonce[7] = (byte) core;
try {
mda = MessageDigest.getInstance("SHA-512");
} catch (NoSuchAlgorithmException e) {
LOG.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
public boolean isSuccessful() {
return successful;
}
public byte[] getNonce() {
return nonce;
}
@Override
public void run() {
do {
inc(nonce, numberOfCores);
mda.update(nonce);
mda.update(initialHash);
if (!Bytes.lt(target, mda.digest(mda.digest()), 8)) {
successful = true;
for (Worker w : workers) {
w.interrupt();
}
return;
}
} while (!Thread.interrupted());
}
}
}

View File

@ -26,9 +26,7 @@ public interface ProofOfWorkEngine {
* *
* @param initialHash the SHA-512 hash of the object to send, sans nonce * @param initialHash the SHA-512 hash of the object to send, sans nonce
* @param target the target, representing an unsigned long * @param target the target, representing an unsigned long
* @param nonceTrialsPerByte
* @param extraBytes
* @return 8 bytes nonce * @return 8 bytes nonce
*/ */
byte[] calculateNonce(byte[] initialHash, byte[] target, long nonceTrialsPerByte, long extraBytes); byte[] calculateNonce(byte[] initialHash, byte[] target);
} }

View File

@ -19,7 +19,6 @@ package ch.dissem.bitmessage.ports;
import ch.dissem.bitmessage.utils.Bytes; import ch.dissem.bitmessage.utils.Bytes;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import static ch.dissem.bitmessage.utils.Bytes.inc; import static ch.dissem.bitmessage.utils.Bytes.inc;
@ -28,12 +27,12 @@ import static ch.dissem.bitmessage.utils.Bytes.inc;
*/ */
public class SimplePOWEngine implements ProofOfWorkEngine { public class SimplePOWEngine implements ProofOfWorkEngine {
@Override @Override
public byte[] calculateNonce(byte[] initialHash, byte[] target, long nonceTrialsPerByte, long extraBytes) { public byte[] calculateNonce(byte[] initialHash, byte[] target) {
byte[] nonce = new byte[8]; byte[] nonce = new byte[8];
MessageDigest mda; MessageDigest mda;
try { try {
mda = MessageDigest.getInstance("SHA-512"); mda = MessageDigest.getInstance("SHA-512");
} catch (NoSuchAlgorithmException e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
do { do {

View File

@ -27,6 +27,20 @@ public class Bytes {
} }
} }
public static void inc(byte[] nonce, byte value) {
int i = nonce.length - 1;
nonce[i] += value;
if (value > 0 && (nonce[i] < 0 || nonce[i] >= value))
return;
if (value < 0 && (nonce[i] < 0 && nonce[i] >= value))
return;
for (i = i - 1; i >= 0; i--) {
nonce[i]++;
if (nonce[i] != 0) break;
}
}
public static boolean lt(byte[] a, byte[] b) { public static boolean lt(byte[] a, byte[] b) {
byte[] max = (a.length > b.length ? a : b); byte[] max = (a.length > b.length ? a : b);
byte[] min = (max == a ? b : a); byte[] min = (max == a ? b : a);

View File

@ -112,9 +112,7 @@ public class Decode {
public static String varString(InputStream stream) throws IOException { public static String varString(InputStream stream) throws IOException {
int length = (int) varInt(stream); int length = (int) varInt(stream);
// FIXME: technically, it says the length in characters, but I think this one might be correct // FIXME: technically, it says the length in characters, but I think this one might be correct
byte[] bytes = new byte[length]; // otherwise it will get complicated, as we'll need to read UTF-8 char by char...
// FIXME: I'm also not quite sure if this works, maybe the read return value needs to be handled properly return new String(bytes(stream, length), "utf-8");
stream.read(bytes);
return new String(bytes, "utf-8");
} }
} }

View File

@ -106,6 +106,7 @@ public class Encode {
public static void varString(String value, OutputStream stream) throws IOException { public static void varString(String value, OutputStream stream) throws IOException {
byte[] bytes = value.getBytes("utf-8"); byte[] bytes = value.getBytes("utf-8");
// FIXME: technically, it says the length in characters, but I think this one might be correct // FIXME: technically, it says the length in characters, but I think this one might be correct
// see also Decode#varString()
varInt(bytes.length, stream); varInt(bytes.length, stream);
stream.write(bytes); stream.write(bytes);
} }

View File

@ -71,7 +71,7 @@ public class Security {
byte[] target = getProofOfWorkTarget(object, nonceTrialsPerByte, extraBytes); byte[] target = getProofOfWorkTarget(object, nonceTrialsPerByte, extraBytes);
byte[] nonce = worker.calculateNonce(initialHash, target, nonceTrialsPerByte, extraBytes); byte[] nonce = worker.calculateNonce(initialHash, target);
object.setNonce(nonce); object.setNonce(nonce);
} }

View File

@ -0,0 +1,48 @@
/*
* Copyright 2015 Christian Basler
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.dissem.bitmessage.ports;
import ch.dissem.bitmessage.utils.Bytes;
import ch.dissem.bitmessage.utils.Security;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
/**
* Created by chris on 17.04.15.
*/
public class ProofOfWorkEngineTest {
@Test
public void testSimplePOWEngine() {
testPOW(new SimplePOWEngine());
}
@Test
public void testThreadedPOWEngine() {
testPOW(new MultiThreadedPOWEngine());
}
private void testPOW(ProofOfWorkEngine engine) {
long time = System.currentTimeMillis();
byte[] initialHash = Security.sha512(new byte[]{1, 3, 6, 4});
byte[] target = {0, 0, 0, -1, -1, -1, -1, -1};
byte[] nonce = engine.calculateNonce(initialHash, target);
System.out.println("Calculating nonce took " + (System.currentTimeMillis() - time) + "ms");
assertTrue(Bytes.lt(Security.doubleSha512(nonce, initialHash), target, 8));
}
}

View File

@ -18,7 +18,6 @@ package ch.dissem.bitmessage.utils;
import org.junit.Test; import org.junit.Test;
import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.math.BigInteger; import java.math.BigInteger;
import java.util.Random; import java.util.Random;
@ -30,19 +29,28 @@ import static org.junit.Assert.assertEquals;
* Created by chris on 10.04.15. * Created by chris on 10.04.15.
*/ */
public class BytesTest { public class BytesTest {
public static final Random rnd = new Random();
@Test @Test
public void testIncrement() throws IOException { public void testIncrement() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Encode.int16(256, out);
byte[] bytes = {0, -1}; byte[] bytes = {0, -1};
Bytes.inc(bytes); Bytes.inc(bytes);
assertArrayEquals(out.toByteArray(), bytes); assertArrayEquals(TestUtils.int16(256), bytes);
}
@Test
public void testIncrementByValue() throws IOException {
for (int v = 0; v < 256; v++) {
for (int i = 1; i < 256; i++) {
byte[] bytes = {0, (byte) v};
Bytes.inc(bytes, (byte) i);
assertArrayEquals("value = " + v + "; inc = " + i + "; expected = " + (v + i), TestUtils.int16(v + i), bytes);
}
}
} }
@Test @Test
public void testLowerThan() { public void testLowerThan() {
Random rnd = new Random();
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();
BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();
@ -53,7 +61,6 @@ public class BytesTest {
@Test @Test
public void testLowerThanBounded() { public void testLowerThanBounded() {
Random rnd = new Random();
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); BigInteger a = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();
BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs(); BigInteger b = BigInteger.valueOf(rnd.nextLong()).pow((rnd.nextInt(5) + 1)).abs();

View File

@ -18,7 +18,7 @@ package ch.dissem.bitmessage.utils;
import ch.dissem.bitmessage.entity.ObjectMessage; import ch.dissem.bitmessage.entity.ObjectMessage;
import ch.dissem.bitmessage.entity.payload.GenericPayload; import ch.dissem.bitmessage.entity.payload.GenericPayload;
import ch.dissem.bitmessage.ports.SimplePOWEngine; import ch.dissem.bitmessage.ports.MultiThreadedPOWEngine;
import org.junit.Test; import org.junit.Test;
import javax.xml.bind.DatatypeConverter; import javax.xml.bind.DatatypeConverter;
@ -86,8 +86,8 @@ public class SecurityTest {
.expiresTime(expires.getTimeInMillis() / 1000) .expiresTime(expires.getTimeInMillis() / 1000)
.payload(new GenericPayload(1, new byte[0])) .payload(new GenericPayload(1, new byte[0]))
.build(); .build();
Security.doProofOfWork(objectMessage, new SimplePOWEngine(), 10, 10); Security.doProofOfWork(objectMessage, new MultiThreadedPOWEngine(), 1000, 1000);
Security.checkProofOfWork(objectMessage, 10, 10); Security.checkProofOfWork(objectMessage, 1000, 1000);
} }
@Test @Test

View File

@ -0,0 +1,31 @@
/*
* Copyright 2015 Christian Basler
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.dissem.bitmessage.utils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* If there's ever a need for this in production code, it should be rewritten to be more efficient.
*/
public class TestUtils {
public static byte[] int16(int number) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Encode.int16(number, out);
return out.toByteArray();
}
}