Added MultiStagedKeyServer, fixed Segfault

- KTConcierge optional in KeyTreeServer
- KSFactory chooses "best" KeyServer implementation for workload
- Implemented MultiStagedKS based on multiple KeyTreeServer
- Fixed wrong batch_size calculation for checkBatch

bfclient/src/bf/Brainflyer.cpp

@@ -83,9 +83,7 @@ void Brainflyer::performWork(comm::Work& work) {
     keys += 4096;
     memcpy(priv, priv_batch[4096 - 1], 32);  // set next starting key
 
-    size_t ch_batch_size =
-        work.total >= keys ? 4096 : work.total - (keys - 4096);
-    int i = checkBatch(pub_batch, ch_batch_size);
+    int i = checkBatch(pub_batch, 4096);
     if (i != -1) {
       work.key = util::bytestr(priv_batch[i], 32);
       log("Key found", util::bytestr(priv_batch[i], 32));
@@ -100,8 +98,6 @@ void Brainflyer::performWork(comm::Work& work) {
         break;
       }  // equal: work_end reached
     }
-
-    log(keys, "checked");
   }
 
   log(keys, "checked");

bfclient/src/bfclient.cpp

@@ -35,7 +35,7 @@ int main() {
   thread* threads = new thread[numThreads];
   cout << "Starting " << numThreads << " threads" << endl;
 
-  numThreads = 1;
+  //numThreads = 1;
   for (unsigned int i = 0; i < numThreads; i++) {
     threads[i] = thread{bf::Brainflyer(i, "127.0.0.1", 26765, key)};
   }

cnc/src/main/java/org/btcollider/cnc/CnC.java

@@ -1,7 +1,10 @@
 package org.btcollider.cnc;
 
+import java.util.concurrent.TimeUnit;
+
 import org.btcollider.cnc.comm.CommException;
 import org.btcollider.cnc.comm.CommServer;
+import org.btcollider.cnc.keysrv.KSFactory;
 import org.btcollider.cnc.keysrv.KeyServer;
 import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer;
 import org.slf4j.Logger;
@@ -19,7 +22,7 @@ public class CnC {
 	public static final int MAX_BITS = 62;
 
 	public static void main(String[] args) {
-		KeyServer kts = new KeyTreeServer(54, 27, 18000000000l);
+		KeyServer kts = KSFactory.build(54, 30, 90, TimeUnit.MINUTES);
 		CommServer server = new CommServer(PORT, kts);
 		try {
 			server.listen();

cnc/src/main/java/org/btcollider/cnc/comm/CommServer.java

@@ -19,6 +19,7 @@ public class CommServer {
 
 	public CommServer(int port, KeyServer keyServer) {
 		this.port = port;
+		this.keyServer = keyServer;
 	}
 
 	public void listen() throws CommException {

cnc/src/main/java/org/btcollider/cnc/keysrv/KSFactory.java

@@ -0,0 +1,16 @@
+package org.btcollider.cnc.keysrv;
+
+import java.util.concurrent.TimeUnit;
+
+import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer;
+import org.btcollider.cnc.keysrv.impl.multistage.MultiStagedServer;
+
+public class KSFactory {
+	public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit) {
+		if (depth == 25) {
+			return new KeyTreeServer(index, depth, timeUnit.toMillis(maxWorkSpan));
+		} else {
+			return new MultiStagedServer(index, depth, timeUnit.toMillis(maxWorkSpan));
+		}
+	}
+}

cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java

@@ -29,15 +29,20 @@ public class KeyTreeServer implements KeyServer {
 		this.index = index;
 		this.root = generateKeyTree(depth);
 
-		log.info("Starting KeyTree concierge");
-		ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor();
-		es.scheduleWithFixedDelay(new KTConcierge(root, maxWorkSpan), 0, 30, TimeUnit.SECONDS);
+		if (maxWorkSpan > 0) {
+			log.info("Starting KeyTree concierge");
+			ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor();
+			es.scheduleWithFixedDelay(new KTConcierge(root, maxWorkSpan), 0, 30, TimeUnit.SECONDS);
+		} else {
+			log.info("KeyTree concierge disabled");
+		}
 	}
 
 	public KeyRange getRange() {
 		BitSet keyStart = new BitSet(CnC.MAX_BITS);
 
-		if(!recCollectKey(root, keyStart, index)) return null;
+		if (!recCollectKey(root, keyStart, index))
+			return null;
 
 		BitSet keyEnd = new BitSet(CnC.MAX_BITS);
 		keyEnd.or(keyStart); // set keyEnd to keyStart
@@ -64,7 +69,8 @@ public class KeyTreeServer implements KeyServer {
 	}
 
 	public void setInWork(KeyRange keyRange) {
-		if (keyRange == null) return;
+		if (keyRange == null)
+			return;
 
 		markInWork(BitSet.valueOf(new long[] { keyRange.getStart() }));
 	}
@@ -224,8 +230,8 @@ public class KeyTreeServer implements KeyServer {
 		if (left.isLeaf() || right.isLeaf()) {
 			assert left.isLeaf() && right.isLeaf(); // fully balanced binary tree by construction
 
-
-			if (!right.isFree() && ! left.isFree()) return false;
+			if (!right.isFree() && !left.isFree())
+				return false;
 
 			// No choice finish
 			if (!right.isFree())
@@ -235,8 +241,10 @@ public class KeyTreeServer implements KeyServer {
 			else if (left.isFree() && right.isFree()) { // Randomized finish
 				Random r = new Random();
 				boolean leftFirst = r.nextBoolean();
-				if(leftFirst) keyStart.set(curIndex - 1, left.getValue());
-				else keyStart.set(curIndex - 1, right.getValue());
+				if (leftFirst)
+					keyStart.set(curIndex - 1, left.getValue());
+				else
+					keyStart.set(curIndex - 1, right.getValue());
 			}
 
 			return true;

cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java

@@ -0,0 +1,92 @@
+package org.btcollider.cnc.keysrv.impl.multistage;
+
+import org.btcollider.cnc.dto.KeyRange;
+import org.btcollider.cnc.keysrv.KeyServer;
+import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Not entirely random KeyServer that handles very large trees in multiple stages
+ * to reduce memory and processing time consumption
+ * 
+ * @author armin
+ *
+ */
+public class MultiStagedServer implements KeyServer {
+	final Logger log = LoggerFactory.getLogger(MultiStagedServer.class);
+	private static final int PARTITION_DEPTH = 15;
+
+	private KeyTreeServer primKTS;
+	private KeyTreeServer secKTS;
+	private KeyRange partition;
+	private int remainingDepth;
+	private int subIndex;
+	private long maxWorkSpan;
+	
+	public MultiStagedServer(int index, int depth, long maxWorkSpan) {
+		this.remainingDepth = depth - PARTITION_DEPTH;
+		this.subIndex = index - PARTITION_DEPTH;
+		this.maxWorkSpan = maxWorkSpan;
+
+		// Build a reasonably sized KeyTreeServer with disabled KTConcierge service
+		// This represents the first partitioning of the key space
+		this.primKTS = new KeyTreeServer(index, PARTITION_DEPTH, 0);
+		// Get a random starting range from the primary KTS, this range is still too huge
+		this.partition = primKTS.getRange();
+		// Start a secondary KTS to partition remaining key space in currentRange
+		// This represents the actual work-horse KeyTreeServer for the current partition with
+		// enabled KTConcierge service.
+		// Needs subindex+1 because starting bit always set to 1
+		this.secKTS = new KeyTreeServer(subIndex+1, remainingDepth, maxWorkSpan);
+	}
+
+	@Override
+	public KeyRange getRange() {
+		if (partition == null) return null; // We are finished searching the whole space
+		
+		KeyRange kr = secKTS.getRange(); // Get a random range from the subspace
+		
+		if (kr == null) { // sub-range finished
+			log.info("Subrange {} finished. Opening new partition.", partition);
+			primKTS.setSearched(partition);
+			partition = primKTS.getRange();
+			secKTS = new KeyTreeServer(subIndex+1, remainingDepth, maxWorkSpan);
+			return getRange();
+		}
+		
+		// strip out starting bit of subrange
+		long krStart = kr.getStart() & ~(1 << subIndex+1);
+		// extract set tail bits
+		long krEnd = kr.getStart() ^ kr.getEnd();
+
+		// Combine partition with sub-range
+		krStart = partition.getStart() | krStart;
+		krEnd = krStart | krEnd;
+		
+		return new KeyRange(krStart, krEnd);
+	}
+
+	@Override
+	public void setInWork(KeyRange keyRange) {
+		// strip out partition bits -> get subspace KeyRange only
+		long mask = (long) Math.pow(2, (subIndex+1)) - 1;
+		// add leading bit
+		long lb = (long) Math.pow(2, (subIndex+1));
+
+		KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb);
+		secKTS.setInWork(secKR);
+	}
+
+	@Override
+	public void setSearched(KeyRange keyRange) {
+		// strip out partition bits -> get subspace KeyRange only
+		long mask = (long) Math.pow(2, (subIndex+1)) - 1;
+		// add leading bit
+		long lb = (long) Math.pow(2, (subIndex+1));
+
+		KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb);
+		secKTS.setSearched(secKR);
+	}
+
+}

cnc/src/test/java/org/btcollider/cnc/keysrv/KeyServerTest.java

@@ -2,14 +2,20 @@ package org.btcollider.cnc.keysrv;
 
 import static org.junit.jupiter.api.Assertions.*;
 
+import java.util.concurrent.TimeUnit;
+
+import org.btcollider.cnc.dto.KeyRange;
 import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer;
 import org.junit.jupiter.api.Test;
 
 class KeyServerTest {
 
 	@Test
-	void testInitEmptyTree() {
-		assertThrows(AssertionError.class, () -> KeyTreeServer.init(0, 0, 0));
+	void testMultiStagedServer() {
+		KeyServer ks = KSFactory.build(40, 30, 0, TimeUnit.SECONDS);
+		KeyRange kr = ks.getRange();
+		ks.setInWork(kr);
+		ks.setSearched(kr);
 	}
 
 }