Fixed key merge in multi staged, added dump file loading and

- Added cmdline argument to read dmp file in CnC
- Fixed multiple bugs when merging keys and loading buffered keys in
multi stage

.gitignore

@@ -1,5 +1,7 @@
 *.blf
-.sonarlint/
+.sonarlint
+logs
+dmps
 
 # copied from brainflyer gitignore
 

btadmin

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+case $1 in
+	cnc)
+		setsid scp /home/armin/dev/btcollider/cnc/CnC/cnc.jar btcollider:~
+		;;
+	doc)
+		cd docker && make
+		;;
+	log)
+		scp -r btcollider:~/logs /home/armin/dev/btcollider/logs/logs-$(date -Iseconds)
+		;;
+	dmp)
+		scp btcollider:~/dmp /home/armin/dev/btcollider/dmps/dmp-$(date -Iseconds)	
+		;;
+	*)
+		echo "USAGE: ./btadmin <command>"
+		echo
+		echo "Commands:"
+		echo "cnc			upload cnc to 35.196.141.41"
+		echo "doc			build and upload docker image to gitlab"
+		echo "log			download log files from 35.196.141.41"
+		echo "dmp			download dump file from 35.196.141.41"
+		;;
+esac

cnc/src/main/java/org/btcollider/cnc/CnC.java

@@ -1,5 +1,9 @@
 package org.btcollider.cnc;
 
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.concurrent.TimeUnit;
 import org.btcollider.cnc.comm.CommException;
 import org.btcollider.cnc.comm.CommServer;
@@ -13,15 +17,16 @@ import org.slf4j.LoggerFactory;
  *
  */
 public class CnC {
-  static final Logger log = LoggerFactory.getLogger(CnC.class);
+  private static final Logger log = LoggerFactory.getLogger(CnC.class);
 
   public static final int PORT = 26765;
   // Invariant: positive long (2^63-1) must be able to hold all bits handled here
   public static final int MAX_BITS = 62;
 
   public static void main(String[] args) {
-    if (args.length != 2) {
+    if (args.length < 2 || args.length > 3) {
-      System.out.println("USAGE: java -jar cnc.jar <succFile Path> <dumpFile Path>");
+      System.out
+          .println("USAGE: java -jar cnc.jar <succFile Path> <dumpFile Path> [loadDump Path]");
       System.exit(-1);
     }
 
@@ -29,6 +34,17 @@ public class CnC {
     String dumpFile = args[1];
 
     KeyServer kts = KSFactory.build(54, 30, 90, TimeUnit.MINUTES, 0.65f);
+    if (args.length == 3) {
+      log.info("Undumping from {}", args[2]);
+      try (InputStream is = new FileInputStream(args[2])) {
+        kts.undump(is);
+      } catch (FileNotFoundException e) {
+        log.error("Dump file {} not found", args[2], e);
+      } catch (IOException e) {
+        log.error("Undumping from {} failed", args[2], e);
+      }
+    }
+
     CommServer server = new CommServer(PORT, kts, succFile, dumpFile);
     try {
       server.listen();

cnc/src/main/java/org/btcollider/cnc/keysrv/KSFactory.java

@@ -3,8 +3,12 @@ package org.btcollider.cnc.keysrv;
 import java.util.concurrent.TimeUnit;
 import org.btcollider.cnc.keysrv.keytree.KeyTreeServer;
 import org.btcollider.cnc.keysrv.multistage.MultiStagedServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class KSFactory {
+  private static final Logger log = LoggerFactory.getLogger(KSFactory.class);
+
   private KSFactory() {}
 
   public static KeyServer build(int index, int depth, long maxWorkSpan) {
@@ -30,6 +34,10 @@ public class KSFactory {
 
   public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit,
       long pruneKey) {
+
+    log.info("Starting KeyServer with index={}, depth={}, maxWorkSpan={}s, pruneKey={}", index,
+        depth, timeUnit.toSeconds(maxWorkSpan), pruneKey);
+
     if (depth <= 25) {
       return new KeyTreeServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey);
     } else {

cnc/src/main/java/org/btcollider/cnc/keysrv/keytree/KeyTreeServer.java

@@ -312,6 +312,7 @@ public class KeyTreeServer implements KeyServer {
   public void dump(OutputStream out) throws IOException {
     // Dump the KeyTree depth first
     synchronized (this) {
+      log.info("Dumping key state");
       recDump(root, out);
     }
   }
@@ -332,6 +333,7 @@ public class KeyTreeServer implements KeyServer {
   public void undump(InputStream in) throws IOException {
     // Read dump depth first
     synchronized (this) {
+      log.info("Undumping key state");
       recUndump(root, in);
     }
   }

cnc/src/main/java/org/btcollider/cnc/keysrv/multistage/KeyRangeBuffer.java

@@ -29,8 +29,8 @@ public class KeyRangeBuffer {
     this.keyRanges.put(keyRange, System.currentTimeMillis());
   }
 
-  public void setSearched(KeyRange keyRange) {
+  public boolean setSearched(KeyRange keyRange) {
-    keyRanges.computeIfPresent(keyRange, (kr, iw) -> null);
+    return (keyRanges.computeIfPresent(keyRange, (kr, iw) -> null) == null);
   }
 
   public KeyRange getRange() {

cnc/src/main/java/org/btcollider/cnc/keysrv/multistage/MultiStagedServer.java

@@ -60,22 +60,25 @@ public class MultiStagedServer implements KeyServer {
     if (partition == null)
       return null; // We are finished searching the whole space
 
-    KeyRange kr = null;
+    KeyRange kr = keyRangeBuffer.getRange();
 
-    if ((kr = keyRangeBuffer.getRange()) == null) { // Get retracted KR from buffer if one exists
+    if (kr != null) { // Get retracted KR from buffer if one exists
-      kr = secKTS.getRange(); // else: get a random range from the subspace
+      log.info("Got retracted range {} from buffer", kr);
+      return kr;
     }
 
-    if (kr == null) { // sub-range finished
+    kr = secKTS.getRange(); // else: get a random range from the subspace
+
+    if (kr == null) { // kr still null: sub-range finished
       log.info("Subrange {} finished. Opening new partition.", partition);
       primKTS.setSearched(partition);
       partition = primKTS.getRange();
       secKTS = new KeyTreeServer(subIndex + 1, remainingDepth, maxWorkSpan, 0);
-      return getRange();
+      return getRange(); // try again
     }
 
     // strip out starting bit of subrange
-    long krStart = kr.getStart() & ~(1 << subIndex + 1);
+    long krStart = kr.getStart() & ~((long) 1 << (subIndex + 1));
     // extract set tail bits
     long krEnd = kr.getStart() ^ kr.getEnd();
 
@@ -103,13 +106,22 @@ public class MultiStagedServer implements KeyServer {
 
   @Override
   public void setSearched(KeyRange keyRange) {
-
     keyRangeBuffer.setSearched(keyRange);
+
     // strip out partition bits -> get subspace KeyRange only
     long mask = ((long) 1 << subIndex + 1) - 1;
     // add leading bit
     long lb = ((long) 1 << subIndex + 1);
 
+    long nmask = ~mask;
+    Long partitionStart = keyRange.getStart() & nmask;
+    // Check if this is a left-over range from an older partition
+    if (!partitionStart.equals(this.partition.getStart())) {
+      log.debug("Searched key form old partition: {}, current partition: {}", partitionStart,
+          this.partition.getStart());
+      return;
+    }
+
     KeyRange secKR =
         new KeyRange(((keyRange.getStart() & mask) | lb), ((keyRange.getEnd() & mask) | lb));
 
@@ -119,6 +131,7 @@ public class MultiStagedServer implements KeyServer {
   @Override
   public void dump(OutputStream out) throws IOException {
     synchronized (this) {
+      log.info("Dumping key state");
       ByteArrayOutputStream primDump = new ByteArrayOutputStream();
       primKTS.dump(primDump);
       primDump.flush();
@@ -147,6 +160,7 @@ public class MultiStagedServer implements KeyServer {
   @Override
   public void undump(InputStream in) throws IOException {
     synchronized (this) {
+      log.info("Undumping key state");
       ByteArrayOutputStream bos = new ByteArrayOutputStream();
       for (int b = in.read(); b != 0xFF; b = in.read()) {
         bos.write(b);
@@ -162,7 +176,7 @@ public class MultiStagedServer implements KeyServer {
       secKTS.undump(bis);
 
       BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.US_ASCII));
-      br.readLine();
+      br.readLine(); // go to next line
       String[] partitionDump = br.readLine().split(",");
       this.partition =
           new KeyRange(Long.parseLong(partitionDump[0]), Long.parseLong(partitionDump[1]));

docker/Dockerfile

@@ -10,4 +10,4 @@ RUN apt-get update \
 	&& apt-get install -fy \
 	libssl-dev libgmp-dev
 
-CMD ["./bfclient", "35.196.141.41", "26765", "16"]
+CMD ["./bfclient", "35.196.141.41", "26765", "21"]