From 7c283aa827d3cfc50ee47f224772811542e5d968 Mon Sep 17 00:00:00 2001 From: armin Date: Sat, 28 Apr 2018 16:10:13 +0200 Subject: [PATCH] Formatted with Google Java Style --- .../4680e865da2ad46d274693738207d7b9d80f04ae | 0 .sonarlint/modules/bfclient/issues/index.pb | 3 + .../136c36f2bf1690ff10397e07c3d82baa9b298770 | 3 + .../3893e371f649cb23d5f6428b3ca5eeae3bd4979f | 0 .../7c3c3c10537a28026398df21221528f81672fc33 | 23 + .../ba1dbf788a980366658d348281a337f09d5317fc | 0 .../ecc7f16c71ca2f68bf6a65ea47837815c0ff6322 | 3 + .../fb0de740b71d2edb598c4f3deb39da1c59c1ec4f | 8 + .sonarlint/modules/cnc/issues/index.pb | 13 + cnc/src/main/java/org/btcollider/cnc/CnC.java | 34 +- .../org/btcollider/cnc/comm/ClientWorker.java | 247 ++++---- .../btcollider/cnc/comm/CommException.java | 35 +- .../org/btcollider/cnc/comm/CommServer.java | 91 ++- .../java/org/btcollider/cnc/dto/KeyRange.java | 115 ++-- .../org/btcollider/cnc/keysrv/KSFactory.java | 43 +- .../org/btcollider/cnc/keysrv/KeyServer.java | 54 +- .../cnc/keysrv/impl/keytree/KTConcierge.java | 76 +-- .../cnc/keysrv/impl/keytree/KeyTree.java | 134 ++-- .../keysrv/impl/keytree/KeyTreeServer.java | 577 +++++++++--------- .../impl/multistage/KeyRangeBuffer.java | 72 ++- .../impl/multistage/MultiStagedServer.java | 157 ++--- 21 files changed, 872 insertions(+), 816 deletions(-) create mode 100644 .sonarlint/modules/bfclient/issues/4/6/4680e865da2ad46d274693738207d7b9d80f04ae create mode 100644 .sonarlint/modules/bfclient/issues/index.pb create mode 100644 .sonarlint/modules/cnc/issues/1/3/136c36f2bf1690ff10397e07c3d82baa9b298770 create mode 100644 .sonarlint/modules/cnc/issues/3/8/3893e371f649cb23d5f6428b3ca5eeae3bd4979f create mode 100644 .sonarlint/modules/cnc/issues/7/c/7c3c3c10537a28026398df21221528f81672fc33 create mode 100644 .sonarlint/modules/cnc/issues/b/a/ba1dbf788a980366658d348281a337f09d5317fc create mode 100644 .sonarlint/modules/cnc/issues/e/c/ecc7f16c71ca2f68bf6a65ea47837815c0ff6322 create mode 100644 .sonarlint/modules/cnc/issues/f/b/fb0de740b71d2edb598c4f3deb39da1c59c1ec4f create mode 100644 .sonarlint/modules/cnc/issues/index.pb diff --git a/.sonarlint/modules/bfclient/issues/4/6/4680e865da2ad46d274693738207d7b9d80f04ae b/.sonarlint/modules/bfclient/issues/4/6/4680e865da2ad46d274693738207d7b9d80f04ae new file mode 100644 index 0000000..e69de29 diff --git a/.sonarlint/modules/bfclient/issues/index.pb b/.sonarlint/modules/bfclient/issues/index.pb new file mode 100644 index 0000000..3c08fb3 --- /dev/null +++ b/.sonarlint/modules/bfclient/issues/index.pb @@ -0,0 +1,3 @@ + +@ +src/bfclient.cpp,4/6/4680e865da2ad46d274693738207d7b9d80f04ae \ No newline at end of file diff --git a/.sonarlint/modules/cnc/issues/1/3/136c36f2bf1690ff10397e07c3d82baa9b298770 b/.sonarlint/modules/cnc/issues/1/3/136c36f2bf1690ff10397e07c3d82baa9b298770 new file mode 100644 index 0000000..e54b106 --- /dev/null +++ b/.sonarlint/modules/cnc/issues/1/3/136c36f2bf1690ff10397e07c3d82baa9b298770 @@ -0,0 +1,3 @@ + +m squid:S1118":Add a private constructor to hide the implicit public one.(HӕRMAJORZ +CODE_SMELL \ No newline at end of file diff --git a/.sonarlint/modules/cnc/issues/3/8/3893e371f649cb23d5f6428b3ca5eeae3bd4979f b/.sonarlint/modules/cnc/issues/3/8/3893e371f649cb23d5f6428b3ca5eeae3bd4979f new file mode 100644 index 0000000..e69de29 diff --git a/.sonarlint/modules/cnc/issues/7/c/7c3c3c10537a28026398df21221528f81672fc33 b/.sonarlint/modules/cnc/issues/7/c/7c3c3c10537a28026398df21221528f81672fc33 new file mode 100644 index 0000000..af235a6 --- /dev/null +++ b/.sonarlint/modules/cnc/issues/7/c/7c3c3c10537a28026398df21221528f81672fc33 @@ -0,0 +1,23 @@ + + +squid:S135"YReduce the total number of break and continue statements in this loop to use at most one.(рHȕRMINORZ +CODE_SMELL + +squid:S135"YReduce the total number of break and continue statements in this loop to use at most one.(рHɕRMINORZ +CODE_SMELL + squid:S14885"ZImmediately return this expression instead of assigning it to the temporary variable "kr".(HʕRMINORZ +CODE_SMELL +wsquid:HiddenFieldCheck"8Rename "root" which hides the field declared at line 20.(˷H˕RMAJORZ +CODE_SMELL + squid:S3776"RRefactor this method to reduce its Cognitive Complexity from 21 to the 15 allowed.(H̕RCRITICALZ +CODE_SMELL + squid:S1149Q"_Replace the synchronized class "StringBuffer" by an unsynchronized one such as "StringBuilder".(H͕RMAJORZ +CODE_SMELL +S squid:S3457#"%String contains no format specifiers.(ϷHΕRMAJORZ +CODE_SMELL +S squid:S3457'"%String contains no format specifiers.(HϕRMAJORZ +CODE_SMELL +Y squid:S3457"%String contains no format specifiers.(HЕRMAJORZ +CODE_SMELL +T squid:S3457"%String contains no format specifiers.(̉HѕRMAJORZ +CODE_SMELL \ No newline at end of file diff --git a/.sonarlint/modules/cnc/issues/b/a/ba1dbf788a980366658d348281a337f09d5317fc b/.sonarlint/modules/cnc/issues/b/a/ba1dbf788a980366658d348281a337f09d5317fc new file mode 100644 index 0000000..e69de29 diff --git a/.sonarlint/modules/cnc/issues/e/c/ecc7f16c71ca2f68bf6a65ea47837815c0ff6322 b/.sonarlint/modules/cnc/issues/e/c/ecc7f16c71ca2f68bf6a65ea47837815c0ff6322 new file mode 100644 index 0000000..fc66e0b --- /dev/null +++ b/.sonarlint/modules/cnc/issues/e/c/ecc7f16c71ca2f68bf6a65ea47837815c0ff6322 @@ -0,0 +1,3 @@ + +squid:UselessImportCheck "QRemove this unused import 'org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer'.(ֆHRMINORZ +CODE_SMELL \ No newline at end of file diff --git a/.sonarlint/modules/cnc/issues/f/b/fb0de740b71d2edb598c4f3deb39da1c59c1ec4f b/.sonarlint/modules/cnc/issues/f/b/fb0de740b71d2edb598c4f3deb39da1c59c1ec4f new file mode 100644 index 0000000..3ab314d --- /dev/null +++ b/.sonarlint/modules/cnc/issues/f/b/fb0de740b71d2edb598c4f3deb39da1c59c1ec4f @@ -0,0 +1,8 @@ + +j"squid:SwitchLastCaseIsDefaultChecky""Add a default case to this switch.(HݕRCRITICALZ +CODE_SMELL +X squid:S3457."%String contains no format specifiers.(HߕRMAJORZ +CODE_SMELL +o squid:S2259y"AA "NullPointerException" could be thrown; "suc" is nullable here.(8,HRMAJORZBUG +y squid:S1301y"KReplace this "switch" statement by "if" statements to increase readability.(HޕRMINORZ +CODE_SMELL \ No newline at end of file diff --git a/.sonarlint/modules/cnc/issues/index.pb b/.sonarlint/modules/cnc/issues/index.pb new file mode 100644 index 0000000..1ed4c1b --- /dev/null +++ b/.sonarlint/modules/cnc/issues/index.pb @@ -0,0 +1,13 @@ + +g +7src/main/java/org/btcollider/cnc/comm/ClientWorker.java,f/b/fb0de740b71d2edb598c4f3deb39da1c59c1ec4f +Y +)src/main/java/org/btcollider/cnc/CnC.java,e/c/ecc7f16c71ca2f68bf6a65ea47837815c0ff6322 +w +Gsrc/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java,7/c/7c3c3c10537a28026398df21221528f81672fc33 +f +6src/main/java/org/btcollider/cnc/keysrv/KSFactory.java,1/3/136c36f2bf1690ff10397e07c3d82baa9b298770 +q +Asrc/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTree.java,b/a/ba1dbf788a980366658d348281a337f09d5317fc +~ +Nsrc/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java,3/8/3893e371f649cb23d5f6428b3ca5eeae3bd4979f \ No newline at end of file diff --git a/cnc/src/main/java/org/btcollider/cnc/CnC.java b/cnc/src/main/java/org/btcollider/cnc/CnC.java index d704157..ed1bebc 100644 --- a/cnc/src/main/java/org/btcollider/cnc/CnC.java +++ b/cnc/src/main/java/org/btcollider/cnc/CnC.java @@ -1,12 +1,10 @@ package org.btcollider.cnc; import java.util.concurrent.TimeUnit; - import org.btcollider.cnc.comm.CommException; import org.btcollider.cnc.comm.CommServer; import org.btcollider.cnc.keysrv.KSFactory; import org.btcollider.cnc.keysrv.KeyServer; -import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,21 +13,21 @@ import org.slf4j.LoggerFactory; * */ public class CnC { - static final Logger log = LoggerFactory.getLogger(CnC.class); - - public static final int PORT = 26765; - // Invariant: positive long (2^63-1) must be able to hold all bits handled here - public static final int MAX_BITS = 62; + static final Logger log = LoggerFactory.getLogger(CnC.class); - public static void main(String[] args) { - KeyServer kts = KSFactory.build(54, 30, 90, TimeUnit.MINUTES, 0.62f); - CommServer server = new CommServer(PORT, kts); - try { - server.listen(); - } catch (CommException e) { - log.error("CommServer couldn't be started", e); - } finally { - server.stop(); - } - } + public static final int PORT = 26765; + // Invariant: positive long (2^63-1) must be able to hold all bits handled here + public static final int MAX_BITS = 62; + + public static void main(String[] args) { + KeyServer kts = KSFactory.build(54, 30, 90, TimeUnit.MINUTES, 0.62f); + CommServer server = new CommServer(PORT, kts); + try { + server.listen(); + } catch (CommException e) { + log.error("CommServer couldn't be started", e); + } finally { + server.stop(); + } + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/comm/ClientWorker.java b/cnc/src/main/java/org/btcollider/cnc/comm/ClientWorker.java index 6e59ac6..8e77a43 100644 --- a/cnc/src/main/java/org/btcollider/cnc/comm/ClientWorker.java +++ b/cnc/src/main/java/org/btcollider/cnc/comm/ClientWorker.java @@ -8,147 +8,154 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.net.Socket; - import org.btcollider.cnc.dto.KeyRange; import org.btcollider.cnc.keysrv.KeyServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ClientWorker implements Runnable { - final Logger log = LoggerFactory.getLogger(ClientWorker.class); + final Logger log = LoggerFactory.getLogger(ClientWorker.class); - private Socket clientSocket; - private KeyServer keyServer; - private BufferedReader in; - private PrintWriter out; + private Socket clientSocket; + private KeyServer keyServer; + private BufferedReader in; + private PrintWriter out; - public ClientWorker(Socket clientSocket, KeyServer keyServer) { - this.clientSocket = clientSocket; - this.keyServer = keyServer; - } + public ClientWorker(Socket clientSocket, KeyServer keyServer) { + this.clientSocket = clientSocket; + this.keyServer = keyServer; + } - @Override - public void run() { - try { - in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); - out = new PrintWriter(new OutputStreamWriter(clientSocket.getOutputStream()), true); - } catch (IOException e) { - log.error("Couldn't open communication channel", e); - closeCommChannel(); - return; - } + @Override + public void run() { + try { + in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); + out = new PrintWriter(new OutputStreamWriter(clientSocket.getOutputStream()), true); + } catch (IOException e) { + log.error("Couldn't open communication channel", e); + closeCommChannel(); + return; + } - while (!clientSocket.isClosed()) { - String cmd; - try { - cmd = in.readLine(); - } catch (IOException e) { - log.error("Couldn't read command from communication channel"); - closeCommChannel(); - return; - } + while (!clientSocket.isClosed()) { + String cmd; + try { + cmd = in.readLine(); + } catch (IOException e) { + log.error("Couldn't read command from communication channel"); + closeCommChannel(); + return; + } - if (cmd == null) { - log.info("Connection to {} closed by client", clientSocket.getInetAddress()); - shutDown(); - break; - } + if (cmd == null) { + log.info("Connection to {} closed by client", clientSocket.getInetAddress()); + shutDown(); + break; + } - switch (cmd) { - case "WRK": - sendWork(); - break; - case "RES": - retrieveResult(); - break; - default: - assert false; - } - } - } + switch (cmd) { + case "WRK": + sendWork(); + break; + case "RES": + retrieveResult(); + break; + default: + assert false; + } + } + } - private void closeCommChannel() { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - log.error("Couldn't close communication channel cleanly", e); - } - } - if (out != null) { - out.flush(); - out.close(); - } - } + private void closeCommChannel() { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + log.error("Couldn't close communication channel cleanly", e); + } + } + if (out != null) { + out.flush(); + out.close(); + } + } - private void sendWork() { - KeyRange kr; + private void sendWork() { + KeyRange kr; - synchronized (keyServer) { - kr = keyServer.getRange(); - keyServer.setInWork(kr); - } + synchronized (keyServer) { + kr = keyServer.getRange(); + keyServer.setInWork(kr); + } - if (kr == null) { - out.println("NIL"); - log.debug("Stop work sent to {}", clientSocket.getInetAddress()); - } - else { - String work = Long.toHexString(kr.getStart()) + " " + Long.toHexString(kr.getEnd()) + " " + kr.getTotal(); - out.println(work); - log.debug("Work <{}> sent to {}", work, clientSocket.getInetAddress()); - } - } + if (kr == null) { + out.println("NIL"); + log.debug("Stop work sent to {}", clientSocket.getInetAddress()); + } else { + String work = Long.toHexString(kr.getStart()) + " " + Long.toHexString(kr.getEnd()) + " " + + kr.getTotal(); + out.println(work); + log.debug("Work <{}> sent to {}", work, clientSocket.getInetAddress()); + } + } - private void retrieveResult() { - try { - String[] result = in.readLine().split(" "); - KeyRange kr = new KeyRange(Long.parseLong(result[0], 16), Long.parseLong(result[1], 16)); + private void retrieveResult() { + try { + String[] result = in.readLine().split(" "); + KeyRange kr = new KeyRange(Long.parseLong(result[0], 16), Long.parseLong(result[1], 16)); - synchronized (keyServer) { - keyServer.setSearched(kr); - } + synchronized (keyServer) { + keyServer.setSearched(kr); + } - log.debug("Keyrange {} searched", kr); + log.debug("Keyrange {} searched", kr); - out.println("ACK"); + out.println("ACK"); - String suc = in.readLine(); - switch (suc) { - case "NIL": - log.debug("Keyrange {} unsuccessful", kr); - break; - case "SUC": - log.info("Keyrange {} successful", kr); - out.println("ACK"); - suc = in.readLine(); - log.info("Found private key {}", suc); - writeKey(suc); - out.println("ACK"); - break; - } + String suc = in.readLine(); + if (suc == null) { + log.warn("Result retrieved from {} for {} but couldn't read success status", + clientSocket.getRemoteSocketAddress(), kr); + return; + } - } catch (IOException e) { - log.error("Couldn't read result", e); - } - } - - private void writeKey(String key) { - try(BufferedWriter writer = new BufferedWriter(new FileWriter("/home/armin/Desktop/cwkey"))){ - writer.write(key); - }catch(IOException e) { - log.error("Couldn't write key {} to keyfile", key, e); - } - } + switch (suc) { + case "NIL": + log.debug("Keyrange {} unsuccessful", kr); + break; + case "SUC": + log.info("Keyrange {} successful", kr); + out.println("ACK"); + suc = in.readLine(); + log.info("Found private key {}", suc); + writeKey(suc); + out.println("ACK"); + break; + default: + assert (false); + } - private void shutDown() { - closeCommChannel(); - if (!clientSocket.isClosed()) { - try { - clientSocket.close(); - } catch (IOException e) { - log.error("Couldn't close client socket", e); - } - } - } + } catch (IOException e) { + log.error("Couldn't read result", e); + } + } + + private void writeKey(String key) { + try (BufferedWriter writer = new BufferedWriter(new FileWriter("/home/armin/Desktop/cwkey"))) { + writer.write(key); + } catch (IOException e) { + log.error("Couldn't write key {} to keyfile", key, e); + } + } + + private void shutDown() { + closeCommChannel(); + if (!clientSocket.isClosed()) { + try { + clientSocket.close(); + } catch (IOException e) { + log.error("Couldn't close client socket", e); + } + } + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/comm/CommException.java b/cnc/src/main/java/org/btcollider/cnc/comm/CommException.java index fa830ed..865ac96 100644 --- a/cnc/src/main/java/org/btcollider/cnc/comm/CommException.java +++ b/cnc/src/main/java/org/btcollider/cnc/comm/CommException.java @@ -2,28 +2,27 @@ package org.btcollider.cnc.comm; public class CommException extends Exception { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - public CommException() { - super(); - } + public CommException() { + super(); + } - public CommException(String message) { - super(message); - } + public CommException(String message) { + super(message); + } - public CommException(String message, Throwable cause) { - super(message, cause); - } + public CommException(String message, Throwable cause) { + super(message, cause); + } - public CommException(Throwable cause) { - super(cause); - } + public CommException(Throwable cause) { + super(cause); + } - protected CommException(String message, Throwable cause, - boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } + protected CommException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/comm/CommServer.java b/cnc/src/main/java/org/btcollider/cnc/comm/CommServer.java index feb4c92..059a0d9 100644 --- a/cnc/src/main/java/org/btcollider/cnc/comm/CommServer.java +++ b/cnc/src/main/java/org/btcollider/cnc/comm/CommServer.java @@ -6,64 +6,63 @@ import java.net.Socket; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.btcollider.cnc.keysrv.KeyServer; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class CommServer { - final Logger log = LoggerFactory.getLogger(CommServer.class); + final Logger log = LoggerFactory.getLogger(CommServer.class); - private int port; - private ServerSocket serverSocket; - private KeyServer keyServer; + private int port; + private ServerSocket serverSocket; + private KeyServer keyServer; - public CommServer(int port, KeyServer keyServer) { - this.port = port; - this.keyServer = keyServer; - } + public CommServer(int port, KeyServer keyServer) { + this.port = port; + this.keyServer = keyServer; + } - public void listen() throws CommException { - if(serverSocket == null || serverSocket.isClosed()) { - openSocket(); - } - - ExecutorService es = Executors.newCachedThreadPool(); + public void listen() throws CommException { + if (serverSocket == null || serverSocket.isClosed()) { + openSocket(); + } - while (true) { - try { - Socket socket = serverSocket.accept(); - log.debug("Got connection from " + socket.getInetAddress()); - es.execute(new ClientWorker(socket, keyServer)); + ExecutorService es = Executors.newCachedThreadPool(); - } catch (IOException e) { - log.error("Couldn't establish connections on socket", e); - throw new CommException(e); - } - } - } + while (true) { + try { + Socket socket = serverSocket.accept(); + log.debug("Got connection from " + socket.getInetAddress()); + es.execute(new ClientWorker(socket, keyServer)); - public void stop() { - closeSocket(); - } + } catch (IOException e) { + log.error("Couldn't establish connections on socket", e); + throw new CommException(e); + } + } + } - private void openSocket() throws CommException { - log.debug("Opening socket on port {}", port); + public void stop() { + closeSocket(); + } - try { - this.serverSocket = new ServerSocket(port); - } catch (IOException e) { - log.error("Couldn't allocate server socket", e); + private void openSocket() throws CommException { + log.debug("Opening socket on port {}", port); - } - } + try { + this.serverSocket = new ServerSocket(port); + } catch (IOException e) { + log.error("Couldn't allocate server socket", e); - private void closeSocket() { - if (this.serverSocket != null && !this.serverSocket.isClosed()) { - try { - serverSocket.close(); - } catch (IOException e) { - log.error("Couldn't close server socket cleanly", e); - } - } - } + } + } + + private void closeSocket() { + if (this.serverSocket != null && !this.serverSocket.isClosed()) { + try { + serverSocket.close(); + } catch (IOException e) { + log.error("Couldn't close server socket cleanly", e); + } + } + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/dto/KeyRange.java b/cnc/src/main/java/org/btcollider/cnc/dto/KeyRange.java index ece1e43..a4250a5 100644 --- a/cnc/src/main/java/org/btcollider/cnc/dto/KeyRange.java +++ b/cnc/src/main/java/org/btcollider/cnc/dto/KeyRange.java @@ -1,73 +1,72 @@ package org.btcollider.cnc.dto; import java.util.BitSet; - import org.btcollider.cnc.CnC; public class KeyRange { - private Long start; - private Long end; + private Long start; + private Long end; - public KeyRange(BitSet start, BitSet end) { - assert start.length() <= CnC.MAX_BITS && end.length() <= CnC.MAX_BITS; + public KeyRange(BitSet start, BitSet end) { + assert start.length() <= CnC.MAX_BITS && end.length() <= CnC.MAX_BITS; - this.start = start.toLongArray()[0]; - this.end = end.toLongArray()[0]; - } - - public KeyRange(Long start, Long end) { - this.start = start; - this.end = end; - } - - public Long getStart() { - return start; - } + this.start = start.toLongArray()[0]; + this.end = end.toLongArray()[0]; + } - public Long getEnd() { - return end; - } + public KeyRange(Long start, Long end) { + this.start = start; + this.end = end; + } - public Long getTotal() { - // asserted that <= MAX_BITS, first long contains all significant bits - Long total = getEnd() - getStart() + 1; // +1 bc start is included too + public Long getStart() { + return start; + } - return total; - } + public Long getEnd() { + return end; + } - @Override - public String toString() { - return "KeyRange [start=" + start + ", end=" + end + ", total=" + getTotal() +"]"; - } + public Long getTotal() { + // asserted that <= MAX_BITS, first long contains all significant bits + Long total = getEnd() - getStart() + 1; // +1 bc start is included too - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((end == null) ? 0 : end.hashCode()); - result = prime * result + ((start == null) ? 0 : start.hashCode()); - return result; - } + return total; + } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - KeyRange other = (KeyRange) obj; - if (end == null) { - if (other.end != null) - return false; - } else if (!end.equals(other.end)) - return false; - if (start == null) { - if (other.start != null) - return false; - } else if (!start.equals(other.start)) - return false; - return true; - } + @Override + public String toString() { + return "KeyRange [start=" + start + ", end=" + end + ", total=" + getTotal() + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((end == null) ? 0 : end.hashCode()); + result = prime * result + ((start == null) ? 0 : start.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + KeyRange other = (KeyRange) obj; + if (end == null) { + if (other.end != null) + return false; + } else if (!end.equals(other.end)) + return false; + if (start == null) { + if (other.start != null) + return false; + } else if (!start.equals(other.start)) + return false; + return true; + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/keysrv/KSFactory.java b/cnc/src/main/java/org/btcollider/cnc/keysrv/KSFactory.java index e8f3d28..47ea106 100644 --- a/cnc/src/main/java/org/btcollider/cnc/keysrv/KSFactory.java +++ b/cnc/src/main/java/org/btcollider/cnc/keysrv/KSFactory.java @@ -1,35 +1,36 @@ package org.btcollider.cnc.keysrv; import java.util.concurrent.TimeUnit; - import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer; import org.btcollider.cnc.keysrv.impl.multistage.MultiStagedServer; public class KSFactory { - public static KeyServer build(int index, int depth, long maxWorkSpan) { - return build(index, depth, maxWorkSpan, TimeUnit.MILLISECONDS); - } + public static KeyServer build(int index, int depth, long maxWorkSpan) { + return build(index, depth, maxWorkSpan, TimeUnit.MILLISECONDS); + } - public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit) { - return build(index, depth, maxWorkSpan, timeUnit, 0); - } + public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit) { + return build(index, depth, maxWorkSpan, timeUnit, 0); + } - public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit, float pruneFraction) { - assert (pruneFraction >= 0.0 && pruneFraction <= 1.0); - - long maxKey = ((long) 1 << index) | (( (long) 1 << index) - 1); - long pruneKey = (long) Math.floor(maxKey * pruneFraction); + public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit, + float pruneFraction) { + assert (pruneFraction >= 0.0 && pruneFraction <= 1.0); - return build(index, depth, maxWorkSpan, timeUnit, pruneKey); - } + long maxKey = ((long) 1 << index) | (((long) 1 << index) - 1); + long pruneKey = (long) Math.floor(maxKey * pruneFraction); - public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit, long pruneKey) { - if (depth == 25) { - return new KeyTreeServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey); - } else { - return new MultiStagedServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey); - } + return build(index, depth, maxWorkSpan, timeUnit, pruneKey); + } - } + public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit, + long pruneKey) { + if (depth == 25) { + return new KeyTreeServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey); + } else { + return new MultiStagedServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey); + } + + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/keysrv/KeyServer.java b/cnc/src/main/java/org/btcollider/cnc/keysrv/KeyServer.java index cef3e7d..fb78a7a 100644 --- a/cnc/src/main/java/org/btcollider/cnc/keysrv/KeyServer.java +++ b/cnc/src/main/java/org/btcollider/cnc/keysrv/KeyServer.java @@ -3,34 +3,30 @@ package org.btcollider.cnc.keysrv; import org.btcollider.cnc.dto.KeyRange; public interface KeyServer { - /** - * Gets a free (= not setInWork and not setSearched) - * KeyRange from the pool of possible key ranges - * - * @return A free KeyRange or null if none such exists - */ - KeyRange getRange(); - - /** - * Sets a KeyRange as being currently worked on. - * - * As long as a KeyRange is not setSearched, there - * is no guarantee that an inWork KeyRange will be finished - * in any specific time frame. - * The KeyServer has to clean up dangling inWorks in good - * faith. Unexpected setSearched of already cleaned-up inWorks - * must not be of any harm. - * - * @param keyRange The KeyRange to be set as being worked on, not null - */ - void setInWork(KeyRange keyRange); + /** + * Gets a free (= not setInWork and not setSearched) KeyRange from the pool of possible key ranges + * + * @return A free KeyRange or null if none such exists + */ + KeyRange getRange(); + + /** + * Sets a KeyRange as being currently worked on. + * + * As long as a KeyRange is not setSearched, there is no guarantee that an inWork KeyRange will be + * finished in any specific time frame. The KeyServer has to clean up dangling inWorks in good + * faith. Unexpected setSearched of already cleaned-up inWorks must not be of any harm. + * + * @param keyRange The KeyRange to be set as being worked on, not null + */ + void setInWork(KeyRange keyRange); + + /** + * Sets a KeyRange as being searched, finalizing this range A searched KeyRange need not be + * processed any more + * + * @param keyRange The KeyRange to be set as searched, not null + */ + void setSearched(KeyRange keyRange); - /** - * Sets a KeyRange as being searched, finalizing this range - * A searched KeyRange need not be processed any more - * - * @param keyRange The KeyRange to be set as searched, not null - */ - void setSearched(KeyRange keyRange); - } diff --git a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KTConcierge.java b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KTConcierge.java index 822cd4a..7cd4a13 100644 --- a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KTConcierge.java +++ b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KTConcierge.java @@ -4,42 +4,44 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class KTConcierge implements Runnable { - final Logger log = LoggerFactory.getLogger(KeyTreeServer.class); - - private KeyTree keyTree; - private long maxWorkSpan; - - public KTConcierge(KeyTree keyTree, long maxWorkSpan) { - this.keyTree = keyTree; - this.maxWorkSpan = maxWorkSpan; - } - - @Override - public void run() { - recCheckInWork(keyTree); - } + final Logger log = LoggerFactory.getLogger(KeyTreeServer.class); - private void recCheckInWork(KeyTree node) { - if (node == null) return; - - if (node.inWork()) { - long inWorkSpan = System.currentTimeMillis() - node.inWorkSince(); - if (inWorkSpan >= maxWorkSpan) { - if (node.isLeaf()) { - StringBuilder key = new StringBuilder(); - key.append(node.getValue() ? "1": "0"); - KeyTree parent = node.getParent(); - while (parent != null) { - key.insert(0, parent.getValue() ? "1": "0"); - parent = parent.getParent(); - } - log.debug("{}: Retracting Work for key {}, inWorkSince: {}", System.currentTimeMillis(), key, node.inWorkSince()); - } - node.clearInWork(); - } - } - - recCheckInWork(node.getLeft()); - recCheckInWork(node.getRight()); - } + private KeyTree keyTree; + private long maxWorkSpan; + + public KTConcierge(KeyTree keyTree, long maxWorkSpan) { + this.keyTree = keyTree; + this.maxWorkSpan = maxWorkSpan; + } + + @Override + public void run() { + recCheckInWork(keyTree); + } + + private void recCheckInWork(KeyTree node) { + if (node == null) + return; + + if (node.inWork()) { + long inWorkSpan = System.currentTimeMillis() - node.inWorkSince(); + if (inWorkSpan >= maxWorkSpan) { + if (node.isLeaf()) { + StringBuilder key = new StringBuilder(); + key.append(node.getValue() ? "1" : "0"); + KeyTree parent = node.getParent(); + while (parent != null) { + key.insert(0, parent.getValue() ? "1" : "0"); + parent = parent.getParent(); + } + log.debug("{}: Retracting Work for key {}, inWorkSince: {}", System.currentTimeMillis(), + key, node.inWorkSince()); + } + node.clearInWork(); + } + } + + recCheckInWork(node.getLeft()); + recCheckInWork(node.getRight()); + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTree.java b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTree.java index 6154c7e..c076c88 100644 --- a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTree.java +++ b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTree.java @@ -1,88 +1,88 @@ package org.btcollider.cnc.keysrv.impl.keytree; public class KeyTree { - private boolean value; - private boolean searched; + private boolean value; + private boolean searched; - private long inWorkSince; + private long inWorkSince; - private KeyTree parent; + private KeyTree parent; - private KeyTree left; - private KeyTree right; + private KeyTree left; + private KeyTree right; - public KeyTree(boolean value, KeyTree parent) { - this.value = value; - this.parent = parent; - this.searched = false; - this.inWorkSince = -1; - } - - public boolean isFree() { - return !isSearched() && !inWork(); - } + public KeyTree(boolean value, KeyTree parent) { + this.value = value; + this.parent = parent; + this.searched = false; + this.inWorkSince = -1; + } - public boolean isLeaf() { - return left == null && right == null; - } + public boolean isFree() { + return !isSearched() && !inWork(); + } - public boolean getValue() { - return value; - } + public boolean isLeaf() { + return left == null && right == null; + } - public KeyTree getParent() { - return parent; - } + public boolean getValue() { + return value; + } - public KeyTree getLeft() { - return left; - } + public KeyTree getParent() { + return parent; + } - public boolean hasLeft() { - return left != null; - } + public KeyTree getLeft() { + return left; + } - public void setLeft(KeyTree left) { - this.left = left; - } - - public KeyTree getRight() { - return right; - } - - public boolean hasRight() { - return right != null; - } + public boolean hasLeft() { + return left != null; + } - public void setRight(KeyTree right) { - this.right = right; - } + public void setLeft(KeyTree left) { + this.left = left; + } - public Boolean isSearched() { - return searched; - } + public KeyTree getRight() { + return right; + } - public void setSearched(boolean searched) { - this.searched = searched; - } + public boolean hasRight() { + return right != null; + } - public void setInWork(long since) { - this.inWorkSince = since; - } + public void setRight(KeyTree right) { + this.right = right; + } - public void setInWork() { - this.inWorkSince = System.currentTimeMillis(); - } - - public void clearInWork() { - this.inWorkSince = -1; - } + public Boolean isSearched() { + return searched; + } - public boolean inWork() { - return inWorkSince != -1; - } + public void setSearched(boolean searched) { + this.searched = searched; + } - public long inWorkSince() { - return this.inWorkSince; - } + public void setInWork(long since) { + this.inWorkSince = since; + } + + public void setInWork() { + this.inWorkSince = System.currentTimeMillis(); + } + + public void clearInWork() { + this.inWorkSince = -1; + } + + public boolean inWork() { + return inWorkSince != -1; + } + + public long inWorkSince() { + return this.inWorkSince; + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java index dc874d1..0365a78 100644 --- a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java +++ b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java @@ -7,7 +7,6 @@ import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - import org.btcollider.cnc.CnC; import org.btcollider.cnc.dto.KeyRange; import org.btcollider.cnc.keysrv.KeyServer; @@ -15,293 +14,297 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class KeyTreeServer implements KeyServer { - final Logger log = LoggerFactory.getLogger(KeyTreeServer.class); + final Logger log = LoggerFactory.getLogger(KeyTreeServer.class); - private KeyTree root; - private int index; - private int depth; + private KeyTree root; + private int index; + private int depth; - public KeyTreeServer(int index, int depth, long maxWorkSpan, long pruneKey) { - assert Math.pow(2, CnC.MAX_BITS) <= Long.MAX_VALUE; - assert index < CnC.MAX_BITS && depth <= CnC.MAX_BITS; - - this.depth = depth; - this.index = index; - this.root = generateKeyTree(depth); - - if (pruneKey > 0) prune(pruneKey); - - if (maxWorkSpan > 0) { - log.info("Starting KeyTree concierge"); - ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); - es.scheduleWithFixedDelay(new KTConcierge(root, maxWorkSpan), 0, 30, TimeUnit.SECONDS); - } else { - log.info("KeyTree concierge disabled"); - } - } - - public KeyRange getRange() { - BitSet keyStart = new BitSet(CnC.MAX_BITS); - - if (!recCollectKey(root, keyStart, index)) - return null; - - BitSet keyEnd = new BitSet(CnC.MAX_BITS); - keyEnd.or(keyStart); // set keyEnd to keyStart - keyEnd.flip(0, index + 1 - depth); // reverse last bits - - KeyRange kr = new KeyRange(keyStart, keyEnd); - - return kr; - } - - public void setSearched(Long from, Long to) { - assert from >= 0 && to >= 0 && from <= to; - - long step = (long) Math.pow(2, ((index + 1) - depth)); - - for (long i = from; i < to; i += step) { - BitSet searchedKey = BitSet.valueOf(new long[] { i }); - markSearched(searchedKey); - } - } - - public void setSearched(KeyRange keyRange) { - markSearched(BitSet.valueOf(new long[] { keyRange.getStart() })); - } - - public void setInWork(KeyRange keyRange) { - if (keyRange == null) - return; - - markInWork(BitSet.valueOf(new long[] { keyRange.getStart() })); - } - - public String printDot() { - StringBuffer buffer = new StringBuffer(); - buffer.append("strict graph { \n"); - - Queue frontier = new LinkedList<>(); - frontier.add(root); - - while (!frontier.isEmpty()) { - KeyTree node = frontier.poll(); - buffer.append(dotNodeName(node) + ";\n"); - - if (node.getParent() != null) { - buffer.append(dotNodeName(node.getParent()) + " -- " + dotNodeName(node) + ";\n"); - } - - if (node.getLeft() != null) - frontier.add(node.getLeft()); - if (node.getRight() != null) - frontier.add(node.getRight()); - } - - buffer.append("\n}"); - return buffer.toString(); - } - - private String dotNodeName(KeyTree node) { - String nodeName = "\"" + node.hashCode() + "_" + node.getValue(); - - if (node.isSearched()) - nodeName += "_s"; - if (node.inWork()) - nodeName += "_w"; - - nodeName += "\""; - - return nodeName; - } - - /** - * Prunes every branch strictly smaller than pruneKey - * - * @param prune - * The smallest valid key in the keyspace Must be at least as large - * as the smallest key in the tree Must be at most as large as the - * largest key in the tree - */ - private void prune(Long pruneKey) { - BitSet pkBS = BitSet.valueOf(new long[] { pruneKey }); - int curIdx = pkBS.length() - 1; - assert (curIdx == this.index); // at least the smallest, at most the largest key in the tree - // -> pos of highest bits is equal - - recPrune(pkBS, this.root, curIdx); - } - - private void recPrune(BitSet pruneKey, KeyTree node, int curIdx) { - if (node.getValue() && !pruneKey.get(curIdx)) { // node[ix] = 0 and pruneK[ix] = 1 -> path smaller - node.setSearched(true); - return; - } else if (!node.getValue() && pruneKey.get(curIdx)) { // node[ix] = 1 and pruneK[ix] = 0 -> path - return; - } - - if (node.isLeaf()) { // end reached - assert (curIdx == 0); - return; - } - - recPrune(pruneKey, node.getLeft(), curIdx-1); - recPrune(pruneKey, node.getRight(), curIdx-1); - } - - private void markInWork(BitSet key) { - KeyTree keyTreePointer = root; - - int curIdx = key.length() - 2; // ignore root bit - while (curIdx >= 0) { - if (key.get(curIdx)) - keyTreePointer = keyTreePointer.getRight(); - else - keyTreePointer = keyTreePointer.getLeft(); - - if (keyTreePointer.inWork()) { - break; // path already marked as inWork - } - - if (keyTreePointer.isLeaf()) { // end of tree reached - keyTreePointer.setInWork(); - recMarkInWorkParents(keyTreePointer); - break; - } + public KeyTreeServer(int index, int depth, long maxWorkSpan, long pruneKey) { + assert Math.pow(2, CnC.MAX_BITS) <= Long.MAX_VALUE; + assert index < CnC.MAX_BITS && depth <= CnC.MAX_BITS; - curIdx--; - } - } - - private void recMarkInWorkParents(KeyTree child) { - if (child.getParent() == null) - return; - - KeyTree parent = child.getParent(); - - if (!parent.getLeft().inWork() || !parent.getRight().inWork()) { - return; - } - - // Mark parent inWork too, check next level - parent.setInWork(Long.max(parent.getLeft().inWorkSince(), parent.getRight().inWorkSince())); - - recMarkInWorkParents(parent); - } - - private void markSearched(BitSet key) { - KeyTree keyTreePointer = root; - - int curIdx = key.length() - 2; // ignore root bit - while (curIdx >= 0) { - if (key.get(curIdx)) - keyTreePointer = keyTreePointer.getRight(); - else - keyTreePointer = keyTreePointer.getLeft(); - - if (keyTreePointer.isSearched()) - break; // path already marked as searched - - if (keyTreePointer.isLeaf()) { // end of tree reached - keyTreePointer.setSearched(true); - recMarkSearchedParents(keyTreePointer); - break; - } - - curIdx--; - } - } - - private void recMarkSearchedParents(KeyTree child) { - if (child.getParent() == null) - return; - - KeyTree parent = child.getParent(); - - if (!parent.getLeft().isSearched() || !parent.getRight().isSearched()) { - return; - } - - // Mark parent searched too, check next level - parent.setSearched(true); - recMarkSearchedParents(parent); - } - - private KeyTree generateKeyTree(int depth) { - assert depth > 0; - - KeyTree root = new KeyTree(true, null); - log.info("Starting recGenTree"); - recGenKeyTree(root, depth - 1); - log.info("Ending recGenTree"); - - return root; - } - - private void recGenKeyTree(KeyTree kt, int depth) { - assert depth >= 0; - - if (depth == 0) - return; - - KeyTree left = new KeyTree(false, kt); - KeyTree right = new KeyTree(true, kt); - - kt.setLeft(left); - kt.setRight(right); - - recGenKeyTree(left, depth - 1); - recGenKeyTree(right, depth - 1); - } - - private boolean recCollectKey(KeyTree kt, BitSet keyStart, int curIndex) { - assert curIndex > 0; - - keyStart.set(curIndex, kt.getValue()); - - KeyTree left = kt.getLeft(); - KeyTree right = kt.getRight(); - - // Finish criteria - if (left.isLeaf() || right.isLeaf()) { - assert left.isLeaf() && right.isLeaf(); // fully balanced binary tree by construction - - if (!right.isFree() && !left.isFree()) - return false; - - // No choice finish - if (!right.isFree()) - keyStart.set(curIndex - 1, left.getValue()); - else if (!left.isFree()) - keyStart.set(curIndex - 1, right.getValue()); - else if (left.isFree() && right.isFree()) { // Randomized finish - Random r = new Random(); - boolean leftFirst = r.nextBoolean(); - if (leftFirst) - keyStart.set(curIndex - 1, left.getValue()); - else - keyStart.set(curIndex - 1, right.getValue()); - } - - return true; - } - - // No choice depth-first traversal - if (!left.isFree()) - return recCollectKey(right, keyStart, curIndex - 1); - else if (!right.isFree()) - return recCollectKey(left, keyStart, curIndex - 1); - else { // Randomized depth-first traversal - Random r = new Random(); - boolean leftFirst = r.nextBoolean(); - if (leftFirst) - return recCollectKey(left, keyStart, curIndex - 1); - else - return recCollectKey(right, keyStart, curIndex - 1); - } - } - - protected KeyTree getKeyTree() { - return this.root; - } + this.depth = depth; + this.index = index; + this.root = generateKeyTree(depth); + + if (pruneKey > 0) + prune(pruneKey); + + if (maxWorkSpan > 0) { + log.info("Starting KeyTree concierge"); + ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor(); + es.scheduleWithFixedDelay(new KTConcierge(root, maxWorkSpan), 0, 30, TimeUnit.SECONDS); + } else { + log.info("KeyTree concierge disabled"); + } + } + + @Override + public KeyRange getRange() { + BitSet keyStart = new BitSet(CnC.MAX_BITS); + + if (!recCollectKey(root, keyStart, index)) + return null; + + BitSet keyEnd = new BitSet(CnC.MAX_BITS); + keyEnd.or(keyStart); // set keyEnd to keyStart + keyEnd.flip(0, index + 1 - depth); // reverse last bits + + KeyRange kr = new KeyRange(keyStart, keyEnd); + + return kr; + } + + public void setSearched(Long from, Long to) { + assert from >= 0 && to >= 0 && from <= to; + + long step = (long) Math.pow(2, ((index + 1) - depth)); + + for (long i = from; i < to; i += step) { + BitSet searchedKey = BitSet.valueOf(new long[] {i}); + markSearched(searchedKey); + } + } + + @Override + public void setSearched(KeyRange keyRange) { + markSearched(BitSet.valueOf(new long[] {keyRange.getStart()})); + } + + @Override + public void setInWork(KeyRange keyRange) { + if (keyRange == null) + return; + + markInWork(BitSet.valueOf(new long[] {keyRange.getStart()})); + } + + public String printDot() { + StringBuffer buffer = new StringBuffer(); + buffer.append("strict graph { \n"); + + Queue frontier = new LinkedList<>(); + frontier.add(root); + + while (!frontier.isEmpty()) { + KeyTree node = frontier.poll(); + buffer.append(dotNodeName(node) + ";\n"); + + if (node.getParent() != null) { + buffer.append(dotNodeName(node.getParent()) + " -- " + dotNodeName(node) + ";\n"); + } + + if (node.getLeft() != null) + frontier.add(node.getLeft()); + if (node.getRight() != null) + frontier.add(node.getRight()); + } + + buffer.append("\n}"); + return buffer.toString(); + } + + private String dotNodeName(KeyTree node) { + String nodeName = "\"" + node.hashCode() + "_" + node.getValue(); + + if (node.isSearched()) + nodeName += "_s"; + if (node.inWork()) + nodeName += "_w"; + + nodeName += "\""; + + return nodeName; + } + + /** + * Prunes every branch strictly smaller than pruneKey + * + * @param prune The smallest valid key in the keyspace Must be at least as large as the smallest + * key in the tree Must be at most as large as the largest key in the tree + */ + private void prune(Long pruneKey) { + BitSet pkBS = BitSet.valueOf(new long[] {pruneKey}); + int curIdx = pkBS.length() - 1; + assert (curIdx == this.index); // at least the smallest, at most the largest key in the tree + // -> pos of highest bits is equal + + recPrune(pkBS, this.root, curIdx); + } + + private void recPrune(BitSet pruneKey, KeyTree node, int curIdx) { + if (node.getValue() && !pruneKey.get(curIdx)) { // node[ix] = 0 and pruneK[ix] = 1 -> path + // smaller + node.setSearched(true); + return; + } else if (!node.getValue() && pruneKey.get(curIdx)) { // node[ix] = 1 and pruneK[ix] = 0 -> + // path + return; + } + + if (node.isLeaf()) { // end reached + assert (curIdx == 0); + return; + } + + recPrune(pruneKey, node.getLeft(), curIdx - 1); + recPrune(pruneKey, node.getRight(), curIdx - 1); + } + + private void markInWork(BitSet key) { + KeyTree keyTreePointer = root; + + int curIdx = key.length() - 2; // ignore root bit + while (curIdx >= 0) { + if (key.get(curIdx)) + keyTreePointer = keyTreePointer.getRight(); + else + keyTreePointer = keyTreePointer.getLeft(); + + if (keyTreePointer.inWork()) { + break; // path already marked as inWork + } + + if (keyTreePointer.isLeaf()) { // end of tree reached + keyTreePointer.setInWork(); + recMarkInWorkParents(keyTreePointer); + break; + } + + curIdx--; + } + } + + private void recMarkInWorkParents(KeyTree child) { + if (child.getParent() == null) + return; + + KeyTree parent = child.getParent(); + + if (!parent.getLeft().inWork() || !parent.getRight().inWork()) { + return; + } + + // Mark parent inWork too, check next level + parent.setInWork(Long.max(parent.getLeft().inWorkSince(), parent.getRight().inWorkSince())); + + recMarkInWorkParents(parent); + } + + private void markSearched(BitSet key) { + KeyTree keyTreePointer = root; + + int curIdx = key.length() - 2; // ignore root bit + while (curIdx >= 0) { + if (key.get(curIdx)) + keyTreePointer = keyTreePointer.getRight(); + else + keyTreePointer = keyTreePointer.getLeft(); + + if (keyTreePointer.isSearched()) + break; // path already marked as searched + + if (keyTreePointer.isLeaf()) { // end of tree reached + keyTreePointer.setSearched(true); + recMarkSearchedParents(keyTreePointer); + break; + } + + curIdx--; + } + } + + private void recMarkSearchedParents(KeyTree child) { + if (child.getParent() == null) + return; + + KeyTree parent = child.getParent(); + + if (!parent.getLeft().isSearched() || !parent.getRight().isSearched()) { + return; + } + + // Mark parent searched too, check next level + parent.setSearched(true); + recMarkSearchedParents(parent); + } + + private KeyTree generateKeyTree(int depth) { + assert depth > 0; + + KeyTree root = new KeyTree(true, null); + log.info("Starting recGenTree"); + recGenKeyTree(root, depth - 1); + log.info("Ending recGenTree"); + + return root; + } + + private void recGenKeyTree(KeyTree kt, int depth) { + assert depth >= 0; + + if (depth == 0) + return; + + KeyTree left = new KeyTree(false, kt); + KeyTree right = new KeyTree(true, kt); + + kt.setLeft(left); + kt.setRight(right); + + recGenKeyTree(left, depth - 1); + recGenKeyTree(right, depth - 1); + } + + private boolean recCollectKey(KeyTree kt, BitSet keyStart, int curIndex) { + assert curIndex > 0; + + keyStart.set(curIndex, kt.getValue()); + + KeyTree left = kt.getLeft(); + KeyTree right = kt.getRight(); + + // Finish criteria + if (left.isLeaf() || right.isLeaf()) { + assert left.isLeaf() && right.isLeaf(); // fully balanced binary tree by construction + + if (!right.isFree() && !left.isFree()) + return false; + + // No choice finish + if (!right.isFree()) + keyStart.set(curIndex - 1, left.getValue()); + else if (!left.isFree()) + keyStart.set(curIndex - 1, right.getValue()); + else if (left.isFree() && right.isFree()) { // Randomized finish + Random r = new Random(); + boolean leftFirst = r.nextBoolean(); + if (leftFirst) + keyStart.set(curIndex - 1, left.getValue()); + else + keyStart.set(curIndex - 1, right.getValue()); + } + + return true; + } + + // No choice depth-first traversal + if (!left.isFree()) + return recCollectKey(right, keyStart, curIndex - 1); + else if (!right.isFree()) + return recCollectKey(left, keyStart, curIndex - 1); + else { // Randomized depth-first traversal + Random r = new Random(); + boolean leftFirst = r.nextBoolean(); + if (leftFirst) + return recCollectKey(left, keyStart, curIndex - 1); + else + return recCollectKey(right, keyStart, curIndex - 1); + } + } + + protected KeyTree getKeyTree() { + return this.root; + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/KeyRangeBuffer.java b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/KeyRangeBuffer.java index 8fb9eea..05868d0 100644 --- a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/KeyRangeBuffer.java +++ b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/KeyRangeBuffer.java @@ -2,49 +2,47 @@ package org.btcollider.cnc.keysrv.impl.multistage; import java.util.HashMap; import java.util.Map.Entry; - import org.btcollider.cnc.dto.KeyRange; /** - * The MultiStagedServer may switch partitions while there are still KeyRanges - * from the old partition that are currently worked on. - * - * This buffer stores the state of unfinished KeyRanges from old partitions in - * order to handle them correctly. - * - * Aborted KeyRanges from this Buffer should be always preferred to getting new - * ranges. - * + * The MultiStagedServer may switch partitions while there are still KeyRanges from the old + * partition that are currently worked on. + * + * This buffer stores the state of unfinished KeyRanges from old partitions in order to handle them + * correctly. + * + * Aborted KeyRanges from this Buffer should be always preferred to getting new ranges. + * * @author armin */ public class KeyRangeBuffer { - private HashMap keyRanges; - private Long maxWorkSpan; + private HashMap keyRanges; + private Long maxWorkSpan; - public KeyRangeBuffer(Long maxWorkSpan) { - this.keyRanges = new HashMap<>(); - this.maxWorkSpan = maxWorkSpan; - } - - public void setInWork(KeyRange keyRange) { - this.keyRanges.put(keyRange, System.currentTimeMillis()); - } - - public void setSearched(KeyRange keyRange) { - keyRanges.computeIfPresent(keyRange, (kr, iw) -> null); - } + public KeyRangeBuffer(Long maxWorkSpan) { + this.keyRanges = new HashMap<>(); + this.maxWorkSpan = maxWorkSpan; + } - public KeyRange getRange() { - long curTime = System.currentTimeMillis(); - - KeyRange retracedRange = null; - for (Entry kr: keyRanges.entrySet()) { - if ( (curTime - kr.getValue()) >= maxWorkSpan ) { - retracedRange = kr.getKey(); - break; - } - } - - return retracedRange; - } + public void setInWork(KeyRange keyRange) { + this.keyRanges.put(keyRange, System.currentTimeMillis()); + } + + public void setSearched(KeyRange keyRange) { + keyRanges.computeIfPresent(keyRange, (kr, iw) -> null); + } + + public KeyRange getRange() { + long curTime = System.currentTimeMillis(); + + KeyRange retracedRange = null; + for (Entry kr : keyRanges.entrySet()) { + if ((curTime - kr.getValue()) >= maxWorkSpan) { + retracedRange = kr.getKey(); + break; + } + } + + return retracedRange; + } } diff --git a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java index 9a60446..6d2bbcd 100644 --- a/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java +++ b/cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java @@ -7,98 +7,99 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Not entirely random KeyServer that handles very large trees in multiple stages - * to reduce memory and processing time consumption - * + * Not entirely random KeyServer that handles very large trees in multiple stages to reduce memory + * and processing time consumption + * * @author armin * */ public class MultiStagedServer implements KeyServer { - final Logger log = LoggerFactory.getLogger(MultiStagedServer.class); - private static final int PARTITION_DEPTH = 15; + final Logger log = LoggerFactory.getLogger(MultiStagedServer.class); + private static final int PARTITION_DEPTH = 15; - private KeyTreeServer primKTS; - private KeyTreeServer secKTS; - private KeyRange partition; - private KeyRangeBuffer keyRangeBuffer; - private int remainingDepth; - private int subIndex; - private long maxWorkSpan; - - public MultiStagedServer(int index, int depth, long maxWorkSpan, long pruneKey) { - this.remainingDepth = depth - PARTITION_DEPTH; - this.subIndex = index - PARTITION_DEPTH; - this.maxWorkSpan = maxWorkSpan; + private KeyTreeServer primKTS; + private KeyTreeServer secKTS; + private KeyRange partition; + private KeyRangeBuffer keyRangeBuffer; + private int remainingDepth; + private int subIndex; + private long maxWorkSpan; - // Build a reasonably sized KeyTreeServer with disabled KTConcierge service - // This represents the first partitioning of the key space - this.primKTS = new KeyTreeServer(index, PARTITION_DEPTH, 0, pruneKey); - // Get a random starting range from the primary KTS, this range is still too huge - this.partition = primKTS.getRange(); - // Start a secondary KTS to partition remaining key space in currentRange - // This represents the actual work-horse KeyTreeServer for the current partition with - // enabled KTConcierge service. - // Needs subindex+1 because starting bit always set to 1 - long subPrune = ((long) 1 << subIndex+1) | ( (((long) 1 << subIndex+1) - 1) & pruneKey ); - this.secKTS = new KeyTreeServer(subIndex+1, remainingDepth, maxWorkSpan, subPrune); + public MultiStagedServer(int index, int depth, long maxWorkSpan, long pruneKey) { + this.remainingDepth = depth - PARTITION_DEPTH; + this.subIndex = index - PARTITION_DEPTH; + this.maxWorkSpan = maxWorkSpan; - this.keyRangeBuffer = new KeyRangeBuffer(maxWorkSpan); - } + // Build a reasonably sized KeyTreeServer with disabled KTConcierge service + // This represents the first partitioning of the key space + this.primKTS = new KeyTreeServer(index, PARTITION_DEPTH, 0, pruneKey); + // Get a random starting range from the primary KTS, this range is still too huge + this.partition = primKTS.getRange(); + // Start a secondary KTS to partition remaining key space in currentRange + // This represents the actual work-horse KeyTreeServer for the current partition with + // enabled KTConcierge service. + // Needs subindex+1 because starting bit always set to 1 + long subPrune = ((long) 1 << subIndex + 1) | ((((long) 1 << subIndex + 1) - 1) & pruneKey); + this.secKTS = new KeyTreeServer(subIndex + 1, remainingDepth, maxWorkSpan, subPrune); - @Override - public KeyRange getRange() { - if (partition == null) return null; // We are finished searching the whole space - - KeyRange kr = null; - - if ( (kr = keyRangeBuffer.getRange()) == null) { // Get retracted KR from buffer if one exists - kr = secKTS.getRange(); // else: get a random range from the subspace - } - - if (kr == null) { // sub-range finished - log.info("Subrange {} finished. Opening new partition.", partition); - primKTS.setSearched(partition); - partition = primKTS.getRange(); - secKTS = new KeyTreeServer(subIndex+1, remainingDepth, maxWorkSpan, 0); - return getRange(); - } - - // strip out starting bit of subrange - long krStart = kr.getStart() & ~(1 << subIndex+1); - // extract set tail bits - long krEnd = kr.getStart() ^ kr.getEnd(); + this.keyRangeBuffer = new KeyRangeBuffer(maxWorkSpan); + } - // Combine partition with sub-range - krStart = partition.getStart() | krStart; - krEnd = krStart | krEnd; - - return new KeyRange(krStart, krEnd); - } + @Override + public KeyRange getRange() { + if (partition == null) + return null; // We are finished searching the whole space - @Override - public void setInWork(KeyRange keyRange) { - keyRangeBuffer.setInWork(keyRange); + KeyRange kr = null; - // strip out partition bits -> get subspace KeyRange only - long mask = (long) Math.pow(2, (subIndex+1)) - 1; - // add leading bit - long lb = (long) Math.pow(2, (subIndex+1)); + if ((kr = keyRangeBuffer.getRange()) == null) { // Get retracted KR from buffer if one exists + kr = secKTS.getRange(); // else: get a random range from the subspace + } - KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb); - secKTS.setInWork(secKR); - } + if (kr == null) { // sub-range finished + log.info("Subrange {} finished. Opening new partition.", partition); + primKTS.setSearched(partition); + partition = primKTS.getRange(); + secKTS = new KeyTreeServer(subIndex + 1, remainingDepth, maxWorkSpan, 0); + return getRange(); + } - @Override - public void setSearched(KeyRange keyRange) { - keyRangeBuffer.setSearched(keyRange); + // strip out starting bit of subrange + long krStart = kr.getStart() & ~(1 << subIndex + 1); + // extract set tail bits + long krEnd = kr.getStart() ^ kr.getEnd(); - // strip out partition bits -> get subspace KeyRange only - long mask = (long) Math.pow(2, (subIndex+1)) - 1; - // add leading bit - long lb = (long) Math.pow(2, (subIndex+1)); + // Combine partition with sub-range + krStart = partition.getStart() | krStart; + krEnd = krStart | krEnd; - KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb); - secKTS.setSearched(secKR); - } + return new KeyRange(krStart, krEnd); + } + + @Override + public void setInWork(KeyRange keyRange) { + keyRangeBuffer.setInWork(keyRange); + + // strip out partition bits -> get subspace KeyRange only + long mask = (long) Math.pow(2, (subIndex + 1)) - 1; + // add leading bit + long lb = (long) Math.pow(2, (subIndex + 1)); + + KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb); + secKTS.setInWork(secKR); + } + + @Override + public void setSearched(KeyRange keyRange) { + keyRangeBuffer.setSearched(keyRange); + + // strip out partition bits -> get subspace KeyRange only + long mask = (long) Math.pow(2, (subIndex + 1)) - 1; + // add leading bit + long lb = (long) Math.pow(2, (subIndex + 1)); + + KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb); + secKTS.setSearched(secKR); + } }