attic/btcollider
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Formatted with Google Java Style

Browse source
This commit is contained in:
armin 2018-04-28 16:10:13 +02:00
parent 1e2b04a75c
commit 7c283aa827
21 changed files with 872 additions and 816 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
.sonarlint/modules/bfclient/issues/4/6/4680e865da2ad46d274693738207d7b9d80f04ae Normal file
View file

3
.sonarlint/modules/bfclient/issues/index.pb Normal file
View file

@ -0,0 +1,3 @@
@
src/bfclient.cpp,4/6/4680e865da2ad46d274693738207d7b9d80f04ae

3
.sonarlint/modules/cnc/issues/1/3/136c36f2bf1690ff10397e07c3d82baa9b298770 Normal file
View file

@ -0,0 +1,3 @@
m squid:S1118":Add a private constructor to hide the implicit public one.(ôÒÿùüÿÿÿÿHÓ•RMAJORZ
CODE_SMELL

0
.sonarlint/modules/cnc/issues/3/8/3893e371f649cb23d5f6428b3ca5eeae3bd4979f Normal file
View file

23
.sonarlint/modules/cnc/issues/7/c/7c3c3c10537a28026398df21221528f81672fc33 Normal file
View file

@ -0,0 +1,23 @@
Œ
squid:S135œ"YReduce the total number of break and continue statements in this loop to use at most one.(ÊÑ€ÄûÿÿÿÿHÈ•RMINORZ
CODE_SMELL
Œ
squid:S135Ä"YReduce the total number of break and continue statements in this loop to use at most one.(ÊÑ€ÄûÿÿÿÿHÉ•RMINORZ
CODE_SMELL
ˆ squid:S14885"ZImmediately return this expression instead of assigning it to the temporary variable "kr".(·“žÉHÊ•RMINORZ
CODE_SMELL
wsquid:HiddenFieldChecké"8Rename "root" which hides the field declared at line 20.(Ë·Þ÷ÿÿÿÿÿHË•RMAJORZ
CODE_SMELL
„ squid:S3776<18>"RRefactor this method to reduce its Cognitive Complexity from 21 to the 15 allowed.(¨îïƒHÌ•RCRITICALZ
CODE_SMELL
’ squid:S1149Q"_Replace the synchronized class "StringBuffer" by an unsynchronized one such as "StringBuilder".(¨âà<C3A2>øÿÿÿÿHÍ•RMAJORZ
CODE_SMELL
S squid:S3457#"%String contains no format specifiers.(Ï·Ž®HΕRMAJORZ
CODE_SMELL
S squid:S3457'"%String contains no format specifiers.(<28>¹ÓáHÏ•RMAJORZ
CODE_SMELL
Y squid:S3457ê"%String contains no format specifiers.(óŠ¡ËþÿÿÿÿHЕRMAJORZ
CODE_SMELL
T squid:S3457ì"%String contains no format specifiers.(ÐÒ̉HÑ•RMAJORZ
CODE_SMELL

0
.sonarlint/modules/cnc/issues/b/a/ba1dbf788a980366658d348281a337f09d5317fc Normal file
View file

3
.sonarlint/modules/cnc/issues/e/c/ecc7f16c71ca2f68bf6a65ea47837815c0ff6322 Normal file
View file

@ -0,0 +1,3 @@
Œsquid:UselessImportCheck "QRemove this unused import 'org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer'.(‹Ö†’H¶•RMINORZ
CODE_SMELL

8
.sonarlint/modules/cnc/issues/f/b/fb0de740b71d2edb598c4f3deb39da1c59c1ec4f Normal file
View file

@ -0,0 +1,8 @@
j"squid:SwitchLastCaseIsDefaultChecky""Add a default case to this switch.(¬ý<C2AC>½HÝ•RCRITICALZ
CODE_SMELL
X squid:S3457."%String contains no format specifiers.(’†þ‡ùÿÿÿÿHß•RMAJORZ
CODE_SMELL
o squid:S2259y"AA "NullPointerException" could be thrown; "suc" is nullable here.(¬ý<C2AC>½8¤ð’ä°,Hà•RMAJORZBUG
y squid:S1301y"KReplace this "switch" statement by "if" statements to increase readability.(¬ý<C2AC>½HÞ•RMINORZ
CODE_SMELL

13
.sonarlint/modules/cnc/issues/index.pb Normal file
View file

@ -0,0 +1,13 @@
g
7src/main/java/org/btcollider/cnc/comm/ClientWorker.java,f/b/fb0de740b71d2edb598c4f3deb39da1c59c1ec4f
Y
)src/main/java/org/btcollider/cnc/CnC.java,e/c/ecc7f16c71ca2f68bf6a65ea47837815c0ff6322
w
Gsrc/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java,7/c/7c3c3c10537a28026398df21221528f81672fc33
f
6src/main/java/org/btcollider/cnc/keysrv/KSFactory.java,1/3/136c36f2bf1690ff10397e07c3d82baa9b298770
q
Asrc/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTree.java,b/a/ba1dbf788a980366658d348281a337f09d5317fc
~
Nsrc/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java,3/8/3893e371f649cb23d5f6428b3ca5eeae3bd4979f

34
cnc/src/main/java/org/btcollider/cnc/CnC.java
View file

@ -1,12 +1,10 @@
package org.btcollider.cnc; package org.btcollider.cnc;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.btcollider.cnc.comm.CommException; import org.btcollider.cnc.comm.CommException;
import org.btcollider.cnc.comm.CommServer; import org.btcollider.cnc.comm.CommServer;
import org.btcollider.cnc.keysrv.KSFactory; import org.btcollider.cnc.keysrv.KSFactory;
import org.btcollider.cnc.keysrv.KeyServer; import org.btcollider.cnc.keysrv.KeyServer;
import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -15,21 +13,21 @@ import org.slf4j.LoggerFactory;
* *
*/ */
public class CnC { public class CnC {
static final Logger log = LoggerFactory.getLogger(CnC.class); static final Logger log = LoggerFactory.getLogger(CnC.class);
public static final int PORT = 26765;
// Invariant: positive long (2^63-1) must be able to hold all bits handled here
public static final int MAX_BITS = 62;
public static void main(String[] args) { public static final int PORT = 26765;
KeyServer kts = KSFactory.build(54, 30, 90, TimeUnit.MINUTES, 0.62f); // Invariant: positive long (2^63-1) must be able to hold all bits handled here
CommServer server = new CommServer(PORT, kts); public static final int MAX_BITS = 62;
try {
server.listen(); public static void main(String[] args) {
} catch (CommException e) { KeyServer kts = KSFactory.build(54, 30, 90, TimeUnit.MINUTES, 0.62f);
log.error("CommServer couldn't be started", e); CommServer server = new CommServer(PORT, kts);
} finally { try {
server.stop(); server.listen();
} } catch (CommException e) {
} log.error("CommServer couldn't be started", e);
} finally {
server.stop();
}
}
} }

247
cnc/src/main/java/org/btcollider/cnc/comm/ClientWorker.java
View file

@ -8,147 +8,154 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.Socket; import java.net.Socket;
import org.btcollider.cnc.dto.KeyRange; import org.btcollider.cnc.dto.KeyRange;
import org.btcollider.cnc.keysrv.KeyServer; import org.btcollider.cnc.keysrv.KeyServer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class ClientWorker implements Runnable { public class ClientWorker implements Runnable {
final Logger log = LoggerFactory.getLogger(ClientWorker.class); final Logger log = LoggerFactory.getLogger(ClientWorker.class);
private Socket clientSocket; private Socket clientSocket;
private KeyServer keyServer; private KeyServer keyServer;
private BufferedReader in; private BufferedReader in;
private PrintWriter out; private PrintWriter out;
public ClientWorker(Socket clientSocket, KeyServer keyServer) { public ClientWorker(Socket clientSocket, KeyServer keyServer) {
this.clientSocket = clientSocket; this.clientSocket = clientSocket;
this.keyServer = keyServer; this.keyServer = keyServer;
} }
@Override @Override
public void run() { public void run() {
try { try {
in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream()));
out = new PrintWriter(new OutputStreamWriter(clientSocket.getOutputStream()), true); out = new PrintWriter(new OutputStreamWriter(clientSocket.getOutputStream()), true);
} catch (IOException e) { } catch (IOException e) {
log.error("Couldn't open communication channel", e); log.error("Couldn't open communication channel", e);
closeCommChannel(); closeCommChannel();
return; return;
} }
while (!clientSocket.isClosed()) { while (!clientSocket.isClosed()) {
String cmd; String cmd;
try { try {
cmd = in.readLine(); cmd = in.readLine();
} catch (IOException e) { } catch (IOException e) {
log.error("Couldn't read command from communication channel"); log.error("Couldn't read command from communication channel");
closeCommChannel(); closeCommChannel();
return; return;
} }
if (cmd == null) { if (cmd == null) {
log.info("Connection to {} closed by client", clientSocket.getInetAddress()); log.info("Connection to {} closed by client", clientSocket.getInetAddress());
shutDown(); shutDown();
break; break;
} }
switch (cmd) { switch (cmd) {
case "WRK": case "WRK":
sendWork(); sendWork();
break; break;
case "RES": case "RES":
retrieveResult(); retrieveResult();
break; break;
default: default:
assert false; assert false;
} }
} }
} }
private void closeCommChannel() { private void closeCommChannel() {
if (in != null) { if (in != null) {
try { try {
in.close(); in.close();
} catch (IOException e) { } catch (IOException e) {
log.error("Couldn't close communication channel cleanly", e); log.error("Couldn't close communication channel cleanly", e);
} }
} }
if (out != null) { if (out != null) {
out.flush(); out.flush();
out.close(); out.close();
} }
} }
private void sendWork() { private void sendWork() {
KeyRange kr; KeyRange kr;
synchronized (keyServer) { synchronized (keyServer) {
kr = keyServer.getRange(); kr = keyServer.getRange();
keyServer.setInWork(kr); keyServer.setInWork(kr);
} }
if (kr == null) { if (kr == null) {
out.println("NIL"); out.println("NIL");
log.debug("Stop work sent to {}", clientSocket.getInetAddress()); log.debug("Stop work sent to {}", clientSocket.getInetAddress());
} } else {
else { String work = Long.toHexString(kr.getStart()) + " " + Long.toHexString(kr.getEnd()) + " "
String work = Long.toHexString(kr.getStart()) + " " + Long.toHexString(kr.getEnd()) + " " + kr.getTotal(); + kr.getTotal();
out.println(work); out.println(work);
log.debug("Work <{}> sent to {}", work, clientSocket.getInetAddress()); log.debug("Work <{}> sent to {}", work, clientSocket.getInetAddress());
} }
} }
private void retrieveResult() { private void retrieveResult() {
try { try {
String[] result = in.readLine().split(" "); String[] result = in.readLine().split(" ");
KeyRange kr = new KeyRange(Long.parseLong(result[0], 16), Long.parseLong(result[1], 16)); KeyRange kr = new KeyRange(Long.parseLong(result[0], 16), Long.parseLong(result[1], 16));
synchronized (keyServer) { synchronized (keyServer) {
keyServer.setSearched(kr); keyServer.setSearched(kr);
} }
log.debug("Keyrange {} searched", kr); log.debug("Keyrange {} searched", kr);
out.println("ACK"); out.println("ACK");
String suc = in.readLine(); String suc = in.readLine();
switch (suc) { if (suc == null) {
case "NIL": log.warn("Result retrieved from {} for {} but couldn't read success status",
log.debug("Keyrange {} unsuccessful", kr); clientSocket.getRemoteSocketAddress(), kr);
break; return;
case "SUC": }
log.info("Keyrange {} successful", kr);
out.println("ACK");
suc = in.readLine();
log.info("Found private key {}", suc);
writeKey(suc);
out.println("ACK");
break;
}
} catch (IOException e) { switch (suc) {
log.error("Couldn't read result", e); case "NIL":
} log.debug("Keyrange {} unsuccessful", kr);
} break;
case "SUC":
private void writeKey(String key) { log.info("Keyrange {} successful", kr);
try(BufferedWriter writer = new BufferedWriter(new FileWriter("/home/armin/Desktop/cwkey"))){ out.println("ACK");
writer.write(key); suc = in.readLine();
}catch(IOException e) { log.info("Found private key {}", suc);
log.error("Couldn't write key {} to keyfile", key, e); writeKey(suc);
} out.println("ACK");
} break;
default:
assert (false);
}
private void shutDown() { } catch (IOException e) {
closeCommChannel(); log.error("Couldn't read result", e);
if (!clientSocket.isClosed()) { }
try { }
clientSocket.close();
} catch (IOException e) { private void writeKey(String key) {
log.error("Couldn't close client socket", e); try (BufferedWriter writer = new BufferedWriter(new FileWriter("/home/armin/Desktop/cwkey"))) {
} writer.write(key);
} } catch (IOException e) {
} log.error("Couldn't write key {} to keyfile", key, e);
}
}
private void shutDown() {
closeCommChannel();
if (!clientSocket.isClosed()) {
try {
clientSocket.close();
} catch (IOException e) {
log.error("Couldn't close client socket", e);
}
}
}
} }

35
cnc/src/main/java/org/btcollider/cnc/comm/CommException.java
View file

@ -2,28 +2,27 @@ package org.btcollider.cnc.comm;
public class CommException extends Exception { public class CommException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public CommException() { public CommException() {
super(); super();
} }
public CommException(String message) { public CommException(String message) {
super(message); super(message);
} }
public CommException(String message, Throwable cause) { public CommException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
public CommException(Throwable cause) { public CommException(Throwable cause) {
super(cause); super(cause);
} }
protected CommException(String message, Throwable cause, protected CommException(String message, Throwable cause, boolean enableSuppression,
boolean enableSuppression, boolean writableStackTrace) {
boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace);
super(message, cause, enableSuppression, writableStackTrace); }
}
} }

91
cnc/src/main/java/org/btcollider/cnc/comm/CommServer.java
View file

@ -6,64 +6,63 @@ import java.net.Socket;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import org.btcollider.cnc.keysrv.KeyServer; import org.btcollider.cnc.keysrv.KeyServer;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class CommServer { public class CommServer {
final Logger log = LoggerFactory.getLogger(CommServer.class); final Logger log = LoggerFactory.getLogger(CommServer.class);
private int port; private int port;
private ServerSocket serverSocket; private ServerSocket serverSocket;
private KeyServer keyServer; private KeyServer keyServer;
public CommServer(int port, KeyServer keyServer) { public CommServer(int port, KeyServer keyServer) {
this.port = port; this.port = port;
this.keyServer = keyServer; this.keyServer = keyServer;
} }
public void listen() throws CommException { public void listen() throws CommException {
if(serverSocket == null || serverSocket.isClosed()) { if (serverSocket == null || serverSocket.isClosed()) {
openSocket(); openSocket();
} }
ExecutorService es = Executors.newCachedThreadPool();
while (true) { ExecutorService es = Executors.newCachedThreadPool();
try {
Socket socket = serverSocket.accept();
log.debug("Got connection from " + socket.getInetAddress());
es.execute(new ClientWorker(socket, keyServer));
} catch (IOException e) { while (true) {
log.error("Couldn't establish connections on socket", e); try {
throw new CommException(e); Socket socket = serverSocket.accept();
} log.debug("Got connection from " + socket.getInetAddress());
} es.execute(new ClientWorker(socket, keyServer));
}
public void stop() { } catch (IOException e) {
closeSocket(); log.error("Couldn't establish connections on socket", e);
} throw new CommException(e);
}
}
}
private void openSocket() throws CommException { public void stop() {
log.debug("Opening socket on port {}", port); closeSocket();
}
try { private void openSocket() throws CommException {
this.serverSocket = new ServerSocket(port); log.debug("Opening socket on port {}", port);
} catch (IOException e) {
log.error("Couldn't allocate server socket", e);
} try {
} this.serverSocket = new ServerSocket(port);
} catch (IOException e) {
log.error("Couldn't allocate server socket", e);
private void closeSocket() { }
if (this.serverSocket != null && !this.serverSocket.isClosed()) { }
try {
serverSocket.close(); private void closeSocket() {
} catch (IOException e) { if (this.serverSocket != null && !this.serverSocket.isClosed()) {
log.error("Couldn't close server socket cleanly", e); try {
} serverSocket.close();
} } catch (IOException e) {
} log.error("Couldn't close server socket cleanly", e);
}
}
}
} }

115
cnc/src/main/java/org/btcollider/cnc/dto/KeyRange.java
View file

@ -1,73 +1,72 @@
package org.btcollider.cnc.dto; package org.btcollider.cnc.dto;
import java.util.BitSet; import java.util.BitSet;
import org.btcollider.cnc.CnC; import org.btcollider.cnc.CnC;
public class KeyRange { public class KeyRange {
private Long start; private Long start;
private Long end; private Long end;
public KeyRange(BitSet start, BitSet end) { public KeyRange(BitSet start, BitSet end) {
assert start.length() <= CnC.MAX_BITS && end.length() <= CnC.MAX_BITS; assert start.length() <= CnC.MAX_BITS && end.length() <= CnC.MAX_BITS;
this.start = start.toLongArray()[0]; this.start = start.toLongArray()[0];
this.end = end.toLongArray()[0]; this.end = end.toLongArray()[0];
} }
public KeyRange(Long start, Long end) {
this.start = start;
this.end = end;
}
public Long getStart() {
return start;
}
public Long getEnd() { public KeyRange(Long start, Long end) {
return end; this.start = start;
} this.end = end;
}
public Long getTotal() { public Long getStart() {
// asserted that <= MAX_BITS, first long contains all significant bits return start;
Long total = getEnd() - getStart() + 1; // +1 bc start is included too }
return total; public Long getEnd() {
} return end;
}
@Override public Long getTotal() {
public String toString() { // asserted that <= MAX_BITS, first long contains all significant bits
return "KeyRange [start=" + start + ", end=" + end + ", total=" + getTotal() +"]"; Long total = getEnd() - getStart() + 1; // +1 bc start is included too
}
@Override return total;
public int hashCode() { }
final int prime = 31;
int result = 1;
result = prime * result + ((end == null) ? 0 : end.hashCode());
result = prime * result + ((start == null) ? 0 : start.hashCode());
return result;
}
@Override @Override
public boolean equals(Object obj) { public String toString() {
if (this == obj) return "KeyRange [start=" + start + ", end=" + end + ", total=" + getTotal() + "]";
return true; }
if (obj == null)
return false; @Override
if (getClass() != obj.getClass()) public int hashCode() {
return false; final int prime = 31;
KeyRange other = (KeyRange) obj; int result = 1;
if (end == null) { result = prime * result + ((end == null) ? 0 : end.hashCode());
if (other.end != null) result = prime * result + ((start == null) ? 0 : start.hashCode());
return false; return result;
} else if (!end.equals(other.end)) }
return false;
if (start == null) { @Override
if (other.start != null) public boolean equals(Object obj) {
return false; if (this == obj)
} else if (!start.equals(other.start)) return true;
return false; if (obj == null)
return true; return false;
} if (getClass() != obj.getClass())
return false;
KeyRange other = (KeyRange) obj;
if (end == null) {
if (other.end != null)
return false;
} else if (!end.equals(other.end))
return false;
if (start == null) {
if (other.start != null)
return false;
} else if (!start.equals(other.start))
return false;
return true;
}
} }

43
cnc/src/main/java/org/btcollider/cnc/keysrv/KSFactory.java
View file

@ -1,35 +1,36 @@
package org.btcollider.cnc.keysrv; package org.btcollider.cnc.keysrv;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer; import org.btcollider.cnc.keysrv.impl.keytree.KeyTreeServer;
import org.btcollider.cnc.keysrv.impl.multistage.MultiStagedServer; import org.btcollider.cnc.keysrv.impl.multistage.MultiStagedServer;
public class KSFactory { public class KSFactory {
public static KeyServer build(int index, int depth, long maxWorkSpan) { public static KeyServer build(int index, int depth, long maxWorkSpan) {
return build(index, depth, maxWorkSpan, TimeUnit.MILLISECONDS); return build(index, depth, maxWorkSpan, TimeUnit.MILLISECONDS);
} }
public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit) { public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit) {
return build(index, depth, maxWorkSpan, timeUnit, 0); return build(index, depth, maxWorkSpan, timeUnit, 0);
} }
public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit, float pruneFraction) { public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit,
assert (pruneFraction >= 0.0 && pruneFraction <= 1.0); float pruneFraction) {
assert (pruneFraction >= 0.0 && pruneFraction <= 1.0);
long maxKey = ((long) 1 << index) | (( (long) 1 << index) - 1);
long pruneKey = (long) Math.floor(maxKey * pruneFraction);
return build(index, depth, maxWorkSpan, timeUnit, pruneKey); long maxKey = ((long) 1 << index) | (((long) 1 << index) - 1);
} long pruneKey = (long) Math.floor(maxKey * pruneFraction);
public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit, long pruneKey) { return build(index, depth, maxWorkSpan, timeUnit, pruneKey);
if (depth == 25) { }
return new KeyTreeServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey);
} else {
return new MultiStagedServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey);
}
} public static KeyServer build(int index, int depth, long maxWorkSpan, TimeUnit timeUnit,
long pruneKey) {
if (depth == 25) {
return new KeyTreeServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey);
} else {
return new MultiStagedServer(index, depth, timeUnit.toMillis(maxWorkSpan), pruneKey);
}
}
} }

54
cnc/src/main/java/org/btcollider/cnc/keysrv/KeyServer.java
View file

@ -3,34 +3,30 @@ package org.btcollider.cnc.keysrv;
import org.btcollider.cnc.dto.KeyRange; import org.btcollider.cnc.dto.KeyRange;
public interface KeyServer { public interface KeyServer {
/** /**
* Gets a free (= not setInWork and not setSearched) * Gets a free (= not setInWork and not setSearched) KeyRange from the pool of possible key ranges
* KeyRange from the pool of possible key ranges *
* * @return A free KeyRange or null if none such exists
* @return A free KeyRange or null if none such exists */
*/ KeyRange getRange();
KeyRange getRange();
/**
/** * Sets a KeyRange as being currently worked on.
* Sets a KeyRange as being currently worked on. *
* * As long as a KeyRange is not setSearched, there is no guarantee that an inWork KeyRange will be
* As long as a KeyRange is not setSearched, there * finished in any specific time frame. The KeyServer has to clean up dangling inWorks in good
* is no guarantee that an inWork KeyRange will be finished * faith. Unexpected setSearched of already cleaned-up inWorks must not be of any harm.
* in any specific time frame. *
* The KeyServer has to clean up dangling inWorks in good * @param keyRange The KeyRange to be set as being worked on, not null
* faith. Unexpected setSearched of already cleaned-up inWorks */
* must not be of any harm. void setInWork(KeyRange keyRange);
*
* @param keyRange The KeyRange to be set as being worked on, not null /**
*/ * Sets a KeyRange as being searched, finalizing this range A searched KeyRange need not be
void setInWork(KeyRange keyRange); * processed any more
*
* @param keyRange The KeyRange to be set as searched, not null
*/
void setSearched(KeyRange keyRange);
/**
* Sets a KeyRange as being searched, finalizing this range
* A searched KeyRange need not be processed any more
*
* @param keyRange The KeyRange to be set as searched, not null
*/
void setSearched(KeyRange keyRange);
} }

76
cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KTConcierge.java
View file

@ -4,42 +4,44 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class KTConcierge implements Runnable { public class KTConcierge implements Runnable {
final Logger log = LoggerFactory.getLogger(KeyTreeServer.class); final Logger log = LoggerFactory.getLogger(KeyTreeServer.class);
private KeyTree keyTree;
private long maxWorkSpan;
public KTConcierge(KeyTree keyTree, long maxWorkSpan) {
this.keyTree = keyTree;
this.maxWorkSpan = maxWorkSpan;
}
@Override
public void run() {
recCheckInWork(keyTree);
}
private void recCheckInWork(KeyTree node) { private KeyTree keyTree;
if (node == null) return; private long maxWorkSpan;
if (node.inWork()) { public KTConcierge(KeyTree keyTree, long maxWorkSpan) {
long inWorkSpan = System.currentTimeMillis() - node.inWorkSince(); this.keyTree = keyTree;
if (inWorkSpan >= maxWorkSpan) { this.maxWorkSpan = maxWorkSpan;
if (node.isLeaf()) { }
StringBuilder key = new StringBuilder();
key.append(node.getValue() ? "1": "0"); @Override
KeyTree parent = node.getParent(); public void run() {
while (parent != null) { recCheckInWork(keyTree);
key.insert(0, parent.getValue() ? "1": "0"); }
parent = parent.getParent();
} private void recCheckInWork(KeyTree node) {
log.debug("{}: Retracting Work for key {}, inWorkSince: {}", System.currentTimeMillis(), key, node.inWorkSince()); if (node == null)
} return;
node.clearInWork();
} if (node.inWork()) {
} long inWorkSpan = System.currentTimeMillis() - node.inWorkSince();
if (inWorkSpan >= maxWorkSpan) {
recCheckInWork(node.getLeft()); if (node.isLeaf()) {
recCheckInWork(node.getRight()); StringBuilder key = new StringBuilder();
} key.append(node.getValue() ? "1" : "0");
KeyTree parent = node.getParent();
while (parent != null) {
key.insert(0, parent.getValue() ? "1" : "0");
parent = parent.getParent();
}
log.debug("{}: Retracting Work for key {}, inWorkSince: {}", System.currentTimeMillis(),
key, node.inWorkSince());
}
node.clearInWork();
}
}
recCheckInWork(node.getLeft());
recCheckInWork(node.getRight());
}
} }

134
cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTree.java
View file

@ -1,88 +1,88 @@
package org.btcollider.cnc.keysrv.impl.keytree; package org.btcollider.cnc.keysrv.impl.keytree;
public class KeyTree { public class KeyTree {
private boolean value; private boolean value;
private boolean searched; private boolean searched;
private long inWorkSince; private long inWorkSince;
private KeyTree parent; private KeyTree parent;
private KeyTree left; private KeyTree left;
private KeyTree right; private KeyTree right;
public KeyTree(boolean value, KeyTree parent) { public KeyTree(boolean value, KeyTree parent) {
this.value = value; this.value = value;
this.parent = parent; this.parent = parent;
this.searched = false; this.searched = false;
this.inWorkSince = -1; this.inWorkSince = -1;
} }
public boolean isFree() {
return !isSearched() && !inWork();
}
public boolean isLeaf() { public boolean isFree() {
return left == null && right == null; return !isSearched() && !inWork();
} }
public boolean getValue() { public boolean isLeaf() {
return value; return left == null && right == null;
} }
public KeyTree getParent() { public boolean getValue() {
return parent; return value;
} }
public KeyTree getLeft() { public KeyTree getParent() {
return left; return parent;
} }
public boolean hasLeft() { public KeyTree getLeft() {
return left != null; return left;
} }
public void setLeft(KeyTree left) { public boolean hasLeft() {
this.left = left; return left != null;
} }
public KeyTree getRight() {
return right;
}
public boolean hasRight() {
return right != null;
}
public void setRight(KeyTree right) { public void setLeft(KeyTree left) {
this.right = right; this.left = left;
} }
public Boolean isSearched() { public KeyTree getRight() {
return searched; return right;
} }
public void setSearched(boolean searched) { public boolean hasRight() {
this.searched = searched; return right != null;
} }
public void setInWork(long since) { public void setRight(KeyTree right) {
this.inWorkSince = since; this.right = right;
} }
public void setInWork() { public Boolean isSearched() {
this.inWorkSince = System.currentTimeMillis(); return searched;
} }
public void clearInWork() {
this.inWorkSince = -1;
}
public boolean inWork() { public void setSearched(boolean searched) {
return inWorkSince != -1; this.searched = searched;
} }
public long inWorkSince() { public void setInWork(long since) {
return this.inWorkSince; this.inWorkSince = since;
} }
public void setInWork() {
this.inWorkSince = System.currentTimeMillis();
}
public void clearInWork() {
this.inWorkSince = -1;
}
public boolean inWork() {
return inWorkSince != -1;
}
public long inWorkSince() {
return this.inWorkSince;
}
} }

577
cnc/src/main/java/org/btcollider/cnc/keysrv/impl/keytree/KeyTreeServer.java
View file

@ -7,7 +7,6 @@ import java.util.Random;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.btcollider.cnc.CnC; import org.btcollider.cnc.CnC;
import org.btcollider.cnc.dto.KeyRange; import org.btcollider.cnc.dto.KeyRange;
import org.btcollider.cnc.keysrv.KeyServer; import org.btcollider.cnc.keysrv.KeyServer;
@ -15,293 +14,297 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class KeyTreeServer implements KeyServer { public class KeyTreeServer implements KeyServer {
final Logger log = LoggerFactory.getLogger(KeyTreeServer.class); final Logger log = LoggerFactory.getLogger(KeyTreeServer.class);
private KeyTree root; private KeyTree root;
private int index; private int index;
private int depth; private int depth;
public KeyTreeServer(int index, int depth, long maxWorkSpan, long pruneKey) { public KeyTreeServer(int index, int depth, long maxWorkSpan, long pruneKey) {
assert Math.pow(2, CnC.MAX_BITS) <= Long.MAX_VALUE; assert Math.pow(2, CnC.MAX_BITS) <= Long.MAX_VALUE;
assert index < CnC.MAX_BITS && depth <= CnC.MAX_BITS; assert index < CnC.MAX_BITS && depth <= CnC.MAX_BITS;
this.depth = depth;
this.index = index;
this.root = generateKeyTree(depth);
if (pruneKey > 0) prune(pruneKey);
if (maxWorkSpan > 0) {
log.info("Starting KeyTree concierge");
ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor();
es.scheduleWithFixedDelay(new KTConcierge(root, maxWorkSpan), 0, 30, TimeUnit.SECONDS);
} else {
log.info("KeyTree concierge disabled");
}
}
public KeyRange getRange() {
BitSet keyStart = new BitSet(CnC.MAX_BITS);
if (!recCollectKey(root, keyStart, index))
return null;
BitSet keyEnd = new BitSet(CnC.MAX_BITS);
keyEnd.or(keyStart); // set keyEnd to keyStart
keyEnd.flip(0, index + 1 - depth); // reverse last bits
KeyRange kr = new KeyRange(keyStart, keyEnd);
return kr;
}
public void setSearched(Long from, Long to) {
assert from >= 0 && to >= 0 && from <= to;
long step = (long) Math.pow(2, ((index + 1) - depth));
for (long i = from; i < to; i += step) {
BitSet searchedKey = BitSet.valueOf(new long[] { i });
markSearched(searchedKey);
}
}
public void setSearched(KeyRange keyRange) {
markSearched(BitSet.valueOf(new long[] { keyRange.getStart() }));
}
public void setInWork(KeyRange keyRange) {
if (keyRange == null)
return;
markInWork(BitSet.valueOf(new long[] { keyRange.getStart() }));
}
public String printDot() {
StringBuffer buffer = new StringBuffer();
buffer.append("strict graph { \n");
Queue<KeyTree> frontier = new LinkedList<>();
frontier.add(root);
while (!frontier.isEmpty()) {
KeyTree node = frontier.poll();
buffer.append(dotNodeName(node) + ";\n");
if (node.getParent() != null) {
buffer.append(dotNodeName(node.getParent()) + " -- " + dotNodeName(node) + ";\n");
}
if (node.getLeft() != null)
frontier.add(node.getLeft());
if (node.getRight() != null)
frontier.add(node.getRight());
}
buffer.append("\n}");
return buffer.toString();
}
private String dotNodeName(KeyTree node) {
String nodeName = "\"" + node.hashCode() + "_" + node.getValue();
if (node.isSearched())
nodeName += "_s";
if (node.inWork())
nodeName += "_w";
nodeName += "\"";
return nodeName;
}
/**
* Prunes every branch strictly smaller than pruneKey
*
* @param prune
* The smallest valid key in the keyspace Must be at least as large
* as the smallest key in the tree Must be at most as large as the
* largest key in the tree
*/
private void prune(Long pruneKey) {
BitSet pkBS = BitSet.valueOf(new long[] { pruneKey });
int curIdx = pkBS.length() - 1;
assert (curIdx == this.index); // at least the smallest, at most the largest key in the tree
// -> pos of highest bits is equal
recPrune(pkBS, this.root, curIdx);
}
private void recPrune(BitSet pruneKey, KeyTree node, int curIdx) {
if (node.getValue() && !pruneKey.get(curIdx)) { // node[ix] = 0 and pruneK[ix] = 1 -> path smaller
node.setSearched(true);
return;
} else if (!node.getValue() && pruneKey.get(curIdx)) { // node[ix] = 1 and pruneK[ix] = 0 -> path
return;
}
if (node.isLeaf()) { // end reached
assert (curIdx == 0);
return;
}
recPrune(pruneKey, node.getLeft(), curIdx-1);
recPrune(pruneKey, node.getRight(), curIdx-1);
}
private void markInWork(BitSet key) {
KeyTree keyTreePointer = root;
int curIdx = key.length() - 2; // ignore root bit
while (curIdx >= 0) {
if (key.get(curIdx))
keyTreePointer = keyTreePointer.getRight();
else
keyTreePointer = keyTreePointer.getLeft();
if (keyTreePointer.inWork()) {
break; // path already marked as inWork
}
if (keyTreePointer.isLeaf()) { // end of tree reached
keyTreePointer.setInWork();
recMarkInWorkParents(keyTreePointer);
break;
}
curIdx--; this.depth = depth;
} this.index = index;
} this.root = generateKeyTree(depth);
private void recMarkInWorkParents(KeyTree child) { if (pruneKey > 0)
if (child.getParent() == null) prune(pruneKey);
return;
if (maxWorkSpan > 0) {
KeyTree parent = child.getParent(); log.info("Starting KeyTree concierge");
ScheduledExecutorService es = Executors.newSingleThreadScheduledExecutor();
if (!parent.getLeft().inWork() || !parent.getRight().inWork()) { es.scheduleWithFixedDelay(new KTConcierge(root, maxWorkSpan), 0, 30, TimeUnit.SECONDS);
return; } else {
} log.info("KeyTree concierge disabled");
}
// Mark parent inWork too, check next level }
parent.setInWork(Long.max(parent.getLeft().inWorkSince(), parent.getRight().inWorkSince()));
@Override
recMarkInWorkParents(parent); public KeyRange getRange() {
} BitSet keyStart = new BitSet(CnC.MAX_BITS);
private void markSearched(BitSet key) { if (!recCollectKey(root, keyStart, index))
KeyTree keyTreePointer = root; return null;
int curIdx = key.length() - 2; // ignore root bit BitSet keyEnd = new BitSet(CnC.MAX_BITS);
while (curIdx >= 0) { keyEnd.or(keyStart); // set keyEnd to keyStart
if (key.get(curIdx)) keyEnd.flip(0, index + 1 - depth); // reverse last bits
keyTreePointer = keyTreePointer.getRight();
else KeyRange kr = new KeyRange(keyStart, keyEnd);
keyTreePointer = keyTreePointer.getLeft();
return kr;
if (keyTreePointer.isSearched()) }
break; // path already marked as searched
public void setSearched(Long from, Long to) {
if (keyTreePointer.isLeaf()) { // end of tree reached assert from >= 0 && to >= 0 && from <= to;
keyTreePointer.setSearched(true);
recMarkSearchedParents(keyTreePointer); long step = (long) Math.pow(2, ((index + 1) - depth));
break;
} for (long i = from; i < to; i += step) {
BitSet searchedKey = BitSet.valueOf(new long[] {i});
curIdx--; markSearched(searchedKey);
} }
} }
private void recMarkSearchedParents(KeyTree child) { @Override
if (child.getParent() == null) public void setSearched(KeyRange keyRange) {
return; markSearched(BitSet.valueOf(new long[] {keyRange.getStart()}));
}
KeyTree parent = child.getParent();
@Override
if (!parent.getLeft().isSearched() || !parent.getRight().isSearched()) { public void setInWork(KeyRange keyRange) {
return; if (keyRange == null)
} return;
// Mark parent searched too, check next level markInWork(BitSet.valueOf(new long[] {keyRange.getStart()}));
parent.setSearched(true); }
recMarkSearchedParents(parent);
} public String printDot() {
StringBuffer buffer = new StringBuffer();
private KeyTree generateKeyTree(int depth) { buffer.append("strict graph { \n");
assert depth > 0;
Queue<KeyTree> frontier = new LinkedList<>();
KeyTree root = new KeyTree(true, null); frontier.add(root);
log.info("Starting recGenTree");
recGenKeyTree(root, depth - 1); while (!frontier.isEmpty()) {
log.info("Ending recGenTree"); KeyTree node = frontier.poll();
buffer.append(dotNodeName(node) + ";\n");
return root;
} if (node.getParent() != null) {
buffer.append(dotNodeName(node.getParent()) + " -- " + dotNodeName(node) + ";\n");
private void recGenKeyTree(KeyTree kt, int depth) { }
assert depth >= 0;
if (node.getLeft() != null)
if (depth == 0) frontier.add(node.getLeft());
return; if (node.getRight() != null)
frontier.add(node.getRight());
KeyTree left = new KeyTree(false, kt); }
KeyTree right = new KeyTree(true, kt);
buffer.append("\n}");
kt.setLeft(left); return buffer.toString();
kt.setRight(right); }
recGenKeyTree(left, depth - 1); private String dotNodeName(KeyTree node) {
recGenKeyTree(right, depth - 1); String nodeName = "\"" + node.hashCode() + "_" + node.getValue();
}
if (node.isSearched())
private boolean recCollectKey(KeyTree kt, BitSet keyStart, int curIndex) { nodeName += "_s";
assert curIndex > 0; if (node.inWork())
nodeName += "_w";
keyStart.set(curIndex, kt.getValue());
nodeName += "\"";
KeyTree left = kt.getLeft();
KeyTree right = kt.getRight(); return nodeName;
}
// Finish criteria
if (left.isLeaf() || right.isLeaf()) { /**
assert left.isLeaf() && right.isLeaf(); // fully balanced binary tree by construction * Prunes every branch strictly smaller than pruneKey
*
if (!right.isFree() && !left.isFree()) * @param prune The smallest valid key in the keyspace Must be at least as large as the smallest
return false; * key in the tree Must be at most as large as the largest key in the tree
*/
// No choice finish private void prune(Long pruneKey) {
if (!right.isFree()) BitSet pkBS = BitSet.valueOf(new long[] {pruneKey});
keyStart.set(curIndex - 1, left.getValue()); int curIdx = pkBS.length() - 1;
else if (!left.isFree()) assert (curIdx == this.index); // at least the smallest, at most the largest key in the tree
keyStart.set(curIndex - 1, right.getValue()); // -> pos of highest bits is equal
else if (left.isFree() && right.isFree()) { // Randomized finish
Random r = new Random(); recPrune(pkBS, this.root, curIdx);
boolean leftFirst = r.nextBoolean(); }
if (leftFirst)
keyStart.set(curIndex - 1, left.getValue()); private void recPrune(BitSet pruneKey, KeyTree node, int curIdx) {
else if (node.getValue() && !pruneKey.get(curIdx)) { // node[ix] = 0 and pruneK[ix] = 1 -> path
keyStart.set(curIndex - 1, right.getValue()); // smaller
} node.setSearched(true);
return;
return true; } else if (!node.getValue() && pruneKey.get(curIdx)) { // node[ix] = 1 and pruneK[ix] = 0 ->
} // path
return;
// No choice depth-first traversal }
if (!left.isFree())
return recCollectKey(right, keyStart, curIndex - 1); if (node.isLeaf()) { // end reached
else if (!right.isFree()) assert (curIdx == 0);
return recCollectKey(left, keyStart, curIndex - 1); return;
else { // Randomized depth-first traversal }
Random r = new Random();
boolean leftFirst = r.nextBoolean(); recPrune(pruneKey, node.getLeft(), curIdx - 1);
if (leftFirst) recPrune(pruneKey, node.getRight(), curIdx - 1);
return recCollectKey(left, keyStart, curIndex - 1); }
else
return recCollectKey(right, keyStart, curIndex - 1); private void markInWork(BitSet key) {
} KeyTree keyTreePointer = root;
}
int curIdx = key.length() - 2; // ignore root bit
protected KeyTree getKeyTree() { while (curIdx >= 0) {
return this.root; if (key.get(curIdx))
} keyTreePointer = keyTreePointer.getRight();
else
keyTreePointer = keyTreePointer.getLeft();
if (keyTreePointer.inWork()) {
break; // path already marked as inWork
}
if (keyTreePointer.isLeaf()) { // end of tree reached
keyTreePointer.setInWork();
recMarkInWorkParents(keyTreePointer);
break;
}
curIdx--;
}
}
private void recMarkInWorkParents(KeyTree child) {
if (child.getParent() == null)
return;
KeyTree parent = child.getParent();
if (!parent.getLeft().inWork() || !parent.getRight().inWork()) {
return;
}
// Mark parent inWork too, check next level
parent.setInWork(Long.max(parent.getLeft().inWorkSince(), parent.getRight().inWorkSince()));
recMarkInWorkParents(parent);
}
private void markSearched(BitSet key) {
KeyTree keyTreePointer = root;
int curIdx = key.length() - 2; // ignore root bit
while (curIdx >= 0) {
if (key.get(curIdx))
keyTreePointer = keyTreePointer.getRight();
else
keyTreePointer = keyTreePointer.getLeft();
if (keyTreePointer.isSearched())
break; // path already marked as searched
if (keyTreePointer.isLeaf()) { // end of tree reached
keyTreePointer.setSearched(true);
recMarkSearchedParents(keyTreePointer);
break;
}
curIdx--;
}
}
private void recMarkSearchedParents(KeyTree child) {
if (child.getParent() == null)
return;
KeyTree parent = child.getParent();
if (!parent.getLeft().isSearched() || !parent.getRight().isSearched()) {
return;
}
// Mark parent searched too, check next level
parent.setSearched(true);
recMarkSearchedParents(parent);
}
private KeyTree generateKeyTree(int depth) {
assert depth > 0;
KeyTree root = new KeyTree(true, null);
log.info("Starting recGenTree");
recGenKeyTree(root, depth - 1);
log.info("Ending recGenTree");
return root;
}
private void recGenKeyTree(KeyTree kt, int depth) {
assert depth >= 0;
if (depth == 0)
return;
KeyTree left = new KeyTree(false, kt);
KeyTree right = new KeyTree(true, kt);
kt.setLeft(left);
kt.setRight(right);
recGenKeyTree(left, depth - 1);
recGenKeyTree(right, depth - 1);
}
private boolean recCollectKey(KeyTree kt, BitSet keyStart, int curIndex) {
assert curIndex > 0;
keyStart.set(curIndex, kt.getValue());
KeyTree left = kt.getLeft();
KeyTree right = kt.getRight();
// Finish criteria
if (left.isLeaf() || right.isLeaf()) {
assert left.isLeaf() && right.isLeaf(); // fully balanced binary tree by construction
if (!right.isFree() && !left.isFree())
return false;
// No choice finish
if (!right.isFree())
keyStart.set(curIndex - 1, left.getValue());
else if (!left.isFree())
keyStart.set(curIndex - 1, right.getValue());
else if (left.isFree() && right.isFree()) { // Randomized finish
Random r = new Random();
boolean leftFirst = r.nextBoolean();
if (leftFirst)
keyStart.set(curIndex - 1, left.getValue());
else
keyStart.set(curIndex - 1, right.getValue());
}
return true;
}
// No choice depth-first traversal
if (!left.isFree())
return recCollectKey(right, keyStart, curIndex - 1);
else if (!right.isFree())
return recCollectKey(left, keyStart, curIndex - 1);
else { // Randomized depth-first traversal
Random r = new Random();
boolean leftFirst = r.nextBoolean();
if (leftFirst)
return recCollectKey(left, keyStart, curIndex - 1);
else
return recCollectKey(right, keyStart, curIndex - 1);
}
}
protected KeyTree getKeyTree() {
return this.root;
}
} }

72
cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/KeyRangeBuffer.java
View file

@ -2,49 +2,47 @@ package org.btcollider.cnc.keysrv.impl.multistage;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.btcollider.cnc.dto.KeyRange; import org.btcollider.cnc.dto.KeyRange;
/** /**
* The MultiStagedServer may switch partitions while there are still KeyRanges * The MultiStagedServer may switch partitions while there are still KeyRanges from the old
* from the old partition that are currently worked on. * partition that are currently worked on.
* *
* This buffer stores the state of unfinished KeyRanges from old partitions in * This buffer stores the state of unfinished KeyRanges from old partitions in order to handle them
* order to handle them correctly. * correctly.
* *
* Aborted KeyRanges from this Buffer should be always preferred to getting new * Aborted KeyRanges from this Buffer should be always preferred to getting new ranges.
* ranges. *
*
* @author armin * @author armin
*/ */
public class KeyRangeBuffer { public class KeyRangeBuffer {
private HashMap<KeyRange, Long> keyRanges; private HashMap<KeyRange, Long> keyRanges;
private Long maxWorkSpan; private Long maxWorkSpan;
public KeyRangeBuffer(Long maxWorkSpan) { public KeyRangeBuffer(Long maxWorkSpan) {
this.keyRanges = new HashMap<>(); this.keyRanges = new HashMap<>();
this.maxWorkSpan = maxWorkSpan; this.maxWorkSpan = maxWorkSpan;
} }
public void setInWork(KeyRange keyRange) {
this.keyRanges.put(keyRange, System.currentTimeMillis());
}
public void setSearched(KeyRange keyRange) {
keyRanges.computeIfPresent(keyRange, (kr, iw) -> null);
}
public KeyRange getRange() { public void setInWork(KeyRange keyRange) {
long curTime = System.currentTimeMillis(); this.keyRanges.put(keyRange, System.currentTimeMillis());
}
KeyRange retracedRange = null;
for (Entry<KeyRange, Long> kr: keyRanges.entrySet()) { public void setSearched(KeyRange keyRange) {
if ( (curTime - kr.getValue()) >= maxWorkSpan ) { keyRanges.computeIfPresent(keyRange, (kr, iw) -> null);
retracedRange = kr.getKey(); }
break;
} public KeyRange getRange() {
} long curTime = System.currentTimeMillis();
return retracedRange; KeyRange retracedRange = null;
} for (Entry<KeyRange, Long> kr : keyRanges.entrySet()) {
if ((curTime - kr.getValue()) >= maxWorkSpan) {
retracedRange = kr.getKey();
break;
}
}
return retracedRange;
}
} }

157
cnc/src/main/java/org/btcollider/cnc/keysrv/impl/multistage/MultiStagedServer.java
View file

@ -7,98 +7,99 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** /**
* Not entirely random KeyServer that handles very large trees in multiple stages * Not entirely random KeyServer that handles very large trees in multiple stages to reduce memory
* to reduce memory and processing time consumption * and processing time consumption
* *
* @author armin * @author armin
* *
*/ */
public class MultiStagedServer implements KeyServer { public class MultiStagedServer implements KeyServer {
final Logger log = LoggerFactory.getLogger(MultiStagedServer.class); final Logger log = LoggerFactory.getLogger(MultiStagedServer.class);
private static final int PARTITION_DEPTH = 15; private static final int PARTITION_DEPTH = 15;
private KeyTreeServer primKTS; private KeyTreeServer primKTS;
private KeyTreeServer secKTS; private KeyTreeServer secKTS;
private KeyRange partition; private KeyRange partition;
private KeyRangeBuffer keyRangeBuffer; private KeyRangeBuffer keyRangeBuffer;
private int remainingDepth; private int remainingDepth;
private int subIndex; private int subIndex;
private long maxWorkSpan; private long maxWorkSpan;
public MultiStagedServer(int index, int depth, long maxWorkSpan, long pruneKey) {
this.remainingDepth = depth - PARTITION_DEPTH;
this.subIndex = index - PARTITION_DEPTH;
this.maxWorkSpan = maxWorkSpan;
// Build a reasonably sized KeyTreeServer with disabled KTConcierge service public MultiStagedServer(int index, int depth, long maxWorkSpan, long pruneKey) {
// This represents the first partitioning of the key space this.remainingDepth = depth - PARTITION_DEPTH;
this.primKTS = new KeyTreeServer(index, PARTITION_DEPTH, 0, pruneKey); this.subIndex = index - PARTITION_DEPTH;
// Get a random starting range from the primary KTS, this range is still too huge this.maxWorkSpan = maxWorkSpan;
this.partition = primKTS.getRange();
// Start a secondary KTS to partition remaining key space in currentRange
// This represents the actual work-horse KeyTreeServer for the current partition with
// enabled KTConcierge service.
// Needs subindex+1 because starting bit always set to 1
long subPrune = ((long) 1 << subIndex+1) | ( (((long) 1 << subIndex+1) - 1) & pruneKey );
this.secKTS = new KeyTreeServer(subIndex+1, remainingDepth, maxWorkSpan, subPrune);
this.keyRangeBuffer = new KeyRangeBuffer(maxWorkSpan); // Build a reasonably sized KeyTreeServer with disabled KTConcierge service
} // This represents the first partitioning of the key space
this.primKTS = new KeyTreeServer(index, PARTITION_DEPTH, 0, pruneKey);
// Get a random starting range from the primary KTS, this range is still too huge
this.partition = primKTS.getRange();
// Start a secondary KTS to partition remaining key space in currentRange
// This represents the actual work-horse KeyTreeServer for the current partition with
// enabled KTConcierge service.
// Needs subindex+1 because starting bit always set to 1
long subPrune = ((long) 1 << subIndex + 1) | ((((long) 1 << subIndex + 1) - 1) & pruneKey);
this.secKTS = new KeyTreeServer(subIndex + 1, remainingDepth, maxWorkSpan, subPrune);
@Override this.keyRangeBuffer = new KeyRangeBuffer(maxWorkSpan);
public KeyRange getRange() { }
if (partition == null) return null; // We are finished searching the whole space
KeyRange kr = null;
if ( (kr = keyRangeBuffer.getRange()) == null) { // Get retracted KR from buffer if one exists
kr = secKTS.getRange(); // else: get a random range from the subspace
}
if (kr == null) { // sub-range finished
log.info("Subrange {} finished. Opening new partition.", partition);
primKTS.setSearched(partition);
partition = primKTS.getRange();
secKTS = new KeyTreeServer(subIndex+1, remainingDepth, maxWorkSpan, 0);
return getRange();
}
// strip out starting bit of subrange
long krStart = kr.getStart() & ~(1 << subIndex+1);
// extract set tail bits
long krEnd = kr.getStart() ^ kr.getEnd();
// Combine partition with sub-range @Override
krStart = partition.getStart() | krStart; public KeyRange getRange() {
krEnd = krStart | krEnd; if (partition == null)
return null; // We are finished searching the whole space
return new KeyRange(krStart, krEnd);
}
@Override KeyRange kr = null;
public void setInWork(KeyRange keyRange) {
keyRangeBuffer.setInWork(keyRange);
// strip out partition bits -> get subspace KeyRange only if ((kr = keyRangeBuffer.getRange()) == null) { // Get retracted KR from buffer if one exists
long mask = (long) Math.pow(2, (subIndex+1)) - 1; kr = secKTS.getRange(); // else: get a random range from the subspace
// add leading bit }
long lb = (long) Math.pow(2, (subIndex+1));
KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb); if (kr == null) { // sub-range finished
secKTS.setInWork(secKR); log.info("Subrange {} finished. Opening new partition.", partition);
} primKTS.setSearched(partition);
partition = primKTS.getRange();
secKTS = new KeyTreeServer(subIndex + 1, remainingDepth, maxWorkSpan, 0);
return getRange();
}
@Override // strip out starting bit of subrange
public void setSearched(KeyRange keyRange) { long krStart = kr.getStart() & ~(1 << subIndex + 1);
keyRangeBuffer.setSearched(keyRange); // extract set tail bits
long krEnd = kr.getStart() ^ kr.getEnd();
// strip out partition bits -> get subspace KeyRange only // Combine partition with sub-range
long mask = (long) Math.pow(2, (subIndex+1)) - 1; krStart = partition.getStart() | krStart;
// add leading bit krEnd = krStart | krEnd;
long lb = (long) Math.pow(2, (subIndex+1));
KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb); return new KeyRange(krStart, krEnd);
secKTS.setSearched(secKR); }
}
@Override
public void setInWork(KeyRange keyRange) {
keyRangeBuffer.setInWork(keyRange);
// strip out partition bits -> get subspace KeyRange only
long mask = (long) Math.pow(2, (subIndex + 1)) - 1;
// add leading bit
long lb = (long) Math.pow(2, (subIndex + 1));
KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb);
secKTS.setInWork(secKR);
}
@Override
public void setSearched(KeyRange keyRange) {
keyRangeBuffer.setSearched(keyRange);
// strip out partition bits -> get subspace KeyRange only
long mask = (long) Math.pow(2, (subIndex + 1)) - 1;
// add leading bit
long lb = (long) Math.pow(2, (subIndex + 1));
KeyRange secKR = new KeyRange(keyRange.getStart() & mask | lb, keyRange.getEnd() & mask | lb);
secKTS.setSearched(secKR);
}
} }