Debug tool for benchmarking proof-of-work algorithms (#155)

* cmd/anubis: add a debug option for benchmarking hashrate

Having the ability to benchmark different proof-of-work implementations
is useful for extending Anubis. This adds a flag `--debug-benchmark-js`
(and its associated environment variable `DEBUG_BENCHMARK_JS`) for
serving a tool to do so.

Internally, a there is a new policy action, "DEBUG_BENCHMARK", which
serves the benchmarking tool instead of a challenge. The flag then
replaces all bot rules with a special rule matching every request
to that action. The benchmark page makes heavy use of inline styles,
because currently all global styles are shared across all pages. This
could be fixed, but I wanted to avoid major changes to the templates.

* web/js: add signal for aborting an active proof-of-work algorithm

Both proof-of-work algorithms now take an optional `AbortSignal`, which
immediately terminates all workers and returns `false` if aborted before
the challenge is complete.

* web/js: add algorithm comparison to the benchmark page

"Compare:" is added to the benchmark page for testing the relative
performance between two algorithms. Since benchmark runs generally have
high variance, it may take a while for the averages to converge on a
stable difference.

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: Xe Iaso <me@xeiaso.net>

4 files changed, 189 insertions, 6 deletions

diff --git a/web/js/bench.mjs b/web/js/bench.mjs
new file mode 100644
index 0000000..c8c69bd
--- /dev/null
+++ b/web/js/bench.mjs
@@ -0,0 +1,152 @@
+import processFast from "./proof-of-work.mjs";
+import processSlow from "./proof-of-work-slow.mjs";
+
+const defaultDifficulty = 4;
+const algorithms = {
+  fast: processFast,
+  slow: processSlow,
+};
+
+const status = document.getElementById("status");
+const difficultyInput = document.getElementById("difficulty-input");
+const algorithmSelect = document.getElementById("algorithm-select");
+const compareSelect = document.getElementById("compare-select");
+const header = document.getElementById("table-header");
+const headerCompare = document.getElementById("table-header-compare");
+const results = document.getElementById("results");
+
+const setupControls = () => {
+  difficultyInput.value = defaultDifficulty;
+  for (const alg of Object.keys(algorithms)) {
+    const option1 = document.createElement("option");
+    algorithmSelect.append(option1);
+    const option2 = document.createElement("option");
+    compareSelect.append(option2);
+    option1.value = option1.innerText = option2.value = option2.innerText = alg;
+  }
+};
+
+const benchmarkTrial = async (stats, difficulty, algorithm, signal) => {
+  if (!(difficulty >= 1)) {
+    throw new Error(`Invalid difficulty: ${difficulty}`);
+  }
+  const process = algorithms[algorithm];
+  if (process == null) {
+    throw new Error(`Unknown algorithm: ${algorithm}`);
+  }
+
+  const rawChallenge = new Uint8Array(32);
+  crypto.getRandomValues(rawChallenge);
+  const challenge = Array.from(rawChallenge)
+    .map((c) => c.toString(16).padStart(2, "0"))
+    .join("");
+
+  const t0 = performance.now();
+  const { hash, nonce } = await process(challenge, Number(difficulty), signal);
+  const t1 = performance.now();
+  console.log({ hash, nonce });
+
+  stats.time += t1 - t0;
+  stats.iters += nonce;
+
+  return { time: t1 - t0, nonce };
+};
+
+const stats = { time: 0, iters: 0 };
+const comparison = { time: 0, iters: 0 };
+const updateStatus = () => {
+  const mainRate = stats.iters / stats.time;
+  const compareRate = comparison.iters / comparison.time;
+  if (Number.isFinite(mainRate)) {
+    status.innerText = `Average hashrate: ${mainRate.toFixed(3)}kH/s`;
+    if (Number.isFinite(compareRate)) {
+      const change = ((mainRate - compareRate) / mainRate) * 100;
+      status.innerText += ` vs ${compareRate.toFixed(3)}kH/s (${change.toFixed(2)}% change)`;
+    }
+  } else {
+    status.innerText = "Benchmarking...";
+  }
+};
+
+const tableCell = (text) => {
+  const td = document.createElement("td");
+  td.innerText = text;
+  td.style.padding = "0 0.25rem";
+  return td;
+};
+
+const benchmarkLoop = async (controller) => {
+  const difficulty = difficultyInput.value;
+  const algorithm = algorithmSelect.value;
+  const compareAlgorithm = compareSelect.value;
+  updateStatus();
+
+  try {
+    const { time, nonce } = await benchmarkTrial(
+      stats,
+      difficulty,
+      algorithm,
+      controller.signal,
+    );
+
+    const tr = document.createElement("tr");
+    tr.style.display = "contents";
+    tr.append(tableCell(`${time}ms`), tableCell(nonce));
+
+    // auto-scroll to new rows
+    const atBottom =
+      results.scrollHeight - results.clientHeight <= results.scrollTop;
+    results.append(tr);
+    if (atBottom) {
+      results.scrollTop = results.scrollHeight - results.clientHeight;
+    }
+    updateStatus();
+
+    if (compareAlgorithm !== "NONE") {
+      const { time, nonce } = await benchmarkTrial(
+        comparison,
+        difficulty,
+        compareAlgorithm,
+        controller.signal,
+      );
+      tr.append(tableCell(`${time}ms`), tableCell(nonce));
+    }
+  } catch (e) {
+    if (e !== false) {
+      status.innerText = e;
+    }
+    return;
+  }
+
+  benchmarkLoop(controller);
+};
+
+let controller = null;
+const reset = () => {
+  stats.time = stats.iters = 0;
+  comparison.time = comparison.iters = 0;
+  results.innerHTML = status.innerText = "";
+
+  const table = results.parentElement;
+  if (compareSelect.value !== "NONE") {
+    table.style.gridTemplateColumns = "repeat(4,auto)";
+    header.style.display = "none";
+    headerCompare.style.display = "contents";
+  } else {
+    table.style.gridTemplateColumns = "repeat(2,auto)";
+    header.style.display = "contents";
+    headerCompare.style.display = "none";
+  }
+
+  if (controller != null) {
+    controller.abort();
+  }
+  controller = new AbortController();
+  benchmarkLoop(controller);
+};
+
+setupControls();
+difficultyInput.addEventListener("change", reset);
+algorithmSelect.addEventListener("change", reset);
+compareSelect.addEventListener("change", reset);
+reset();
\ No newline at end of file
diff --git a/web/js/main.mjs b/web/js/main.mjs
index 01f21f0..3203e4a 100644
--- a/web/js/main.mjs
+++ b/web/js/main.mjs
@@ -127,6 +127,7 @@ const dependencies = [
     const { hash, nonce } = await process(
       challenge,
       rules.difficulty,
+      null,
       (iters) => {
         const delta = Date.now() - t0;
         // only update the speed every second so it's less visually distracting
diff --git a/web/js/proof-of-work-slow.mjs b/web/js/proof-of-work-slow.mjs
index 6522c0b..0bdc146 100644
--- a/web/js/proof-of-work-slow.mjs
+++ b/web/js/proof-of-work-slow.mjs
@@ -3,6 +3,7 @@
 export default function process(
   data,
   difficulty = 5,
+  signal = null,
   progressCallback = null,
   _threads = 1,
 ) {
@@ -13,19 +14,33 @@ export default function process(
     ], { type: 'application/javascript' }));
 
     let worker = new Worker(webWorkerURL);
+    const terminate = () => {
+      worker.terminate();
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
 
     worker.onmessage = (event) => {
       if (typeof event.data === "number") {
         progressCallback?.(event.data);
       } else {
-        worker.terminate();
+        terminate();
         resolve(event.data);
       }
     };
 
     worker.onerror = (event) => {
-      worker.terminate();
-      reject();
+      terminate();
+      reject(event);
     };
 
     worker.postMessage({
diff --git a/web/js/proof-of-work.mjs b/web/js/proof-of-work.mjs
index 60d8d61..a04f5ca 100644
--- a/web/js/proof-of-work.mjs
+++ b/web/js/proof-of-work.mjs
@@ -1,6 +1,7 @@
 export default function process(
   data,
   difficulty = 5,
+  signal = null,
   progressCallback = null,
   threads = (navigator.hardwareConcurrency || 1),
 ) {
@@ -11,6 +12,20 @@ export default function process(
     ], { type: 'application/javascript' }));
 
     const workers = [];
+    const terminate = () => {
+      workers.forEach((w) => w.terminate());
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
 
     for (let i = 0; i < threads; i++) {
       let worker = new Worker(webWorkerURL);
@@ -19,14 +34,14 @@ export default function process(
         if (typeof event.data === "number") {
           progressCallback?.(event.data);
         } else {
-          workers.forEach(worker => worker.terminate());
+          terminate();
           resolve(event.data);
         }
       };
 
       worker.onerror = (event) => {
-        worker.terminate();
-        reject();
+        terminate();
+        reject(event);
       };
 
       worker.postMessage({