Debug tool for benchmarking proof-of-work algorithms (#155)

* cmd/anubis: add a debug option for benchmarking hashrate

Having the ability to benchmark different proof-of-work implementations
is useful for extending Anubis. This adds a flag `--debug-benchmark-js`
(and its associated environment variable `DEBUG_BENCHMARK_JS`) for
serving a tool to do so.

Internally, a there is a new policy action, "DEBUG_BENCHMARK", which
serves the benchmarking tool instead of a challenge. The flag then
replaces all bot rules with a special rule matching every request
to that action. The benchmark page makes heavy use of inline styles,
because currently all global styles are shared across all pages. This
could be fixed, but I wanted to avoid major changes to the templates.

* web/js: add signal for aborting an active proof-of-work algorithm

Both proof-of-work algorithms now take an optional `AbortSignal`, which
immediately terminates all workers and returns `false` if aborted before
the challenge is complete.

* web/js: add algorithm comparison to the benchmark page

"Compare:" is added to the benchmark page for testing the relative
performance between two algorithms. Since benchmark runs generally have
high variance, it may take a while for the averages to converge on a
stable difference.

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: Xe Iaso <me@xeiaso.net>

8 files changed, 303 insertions, 7 deletions

diff --git a/web/build.sh b/web/build.sh
index 70492d7..a513c59 100755
--- a/web/build.sh
+++ b/web/build.sh
@@ -7,4 +7,6 @@ cd "$(dirname "$0")"
 esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
 gzip -f -k static/js/main.mjs
 zstd -f -k --ultra -22 static/js/main.mjs
-brotli -fZk static/js/main.mjs
\ No newline at end of file
+brotli -fZk static/js/main.mjs
+
+esbuild js/bench.mjs --sourcemap --bundle --minify --outfile=static/js/bench.mjs
\ No newline at end of file
diff --git a/web/index.go b/web/index.go
index 7057cc8..6ef84b5 100644
--- a/web/index.go
+++ b/web/index.go
@@ -13,3 +13,7 @@ func Index() templ.Component {
 func ErrorPage(msg string) templ.Component {
 	return errorPage(msg)
 }
+
+func Bench() templ.Component {
+	return bench()
+}
diff --git a/web/index.templ b/web/index.templ
index 1899ae3..b43e82c 100644
--- a/web/index.templ
+++ b/web/index.templ
@@ -121,3 +121,54 @@ templ errorPage(message string) {
 		<p><a href="/">Go home</a></p>
 	</div>
 }
+
+templ bench() {
+	<div style="height:20rem;display:flex">
+		<table style="margin-top:1rem;display:grid;grid-template:auto 1fr/auto auto;gap:0 0.5rem">
+			<thead style="border-bottom:1px solid black;padding:0.25rem 0;display:grid;grid-template:1fr/subgrid;grid-column:1/-1">
+				<tr id="table-header" style="display:contents">
+					<th style="width:4.5rem">Time</th>
+					<th style="width:4rem">Iters</th>
+				</tr>
+				<tr id="table-header-compare" style="display:none">
+					<th style="width:4.5rem">Time A</th>
+					<th style="width:4rem">Iters A</th>
+					<th style="width:4.5rem">Time B</th>
+					<th style="width:4rem">Iters B</th>
+				</tr>
+			</thead>
+			<tbody id="results" style="padding-top:0.25rem;display:grid;grid-template-columns:subgrid;grid-auto-rows:min-content;grid-column:1/-1;row-gap:0.25rem;overflow-y:auto;font-variant-numeric:tabular-nums">
+			</tbody>
+		</table>
+		<div class="centered-div">
+			<img
+				id="image"
+				style="width:100%;max-width:256px;"
+				src={ "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
+	    anubis.Version }
+			/>
+			<p id="status" style="max-width:256px">Loading...</p>
+			<script async type="module" src={ "/.within.website/x/cmd/anubis/static/js/bench.mjs?cacheBuster=" + anubis.Version }></script>
+			<div id="sparkline"></div>
+			<noscript>
+				<p>Running the benchmark tool requires JavaScript to be enabled.</p>
+			</noscript>
+		</div>
+	</div>
+	<form id="controls" style="position:fixed;top:0.5rem;right:0.5rem">
+		<div style="display:flex;justify-content:end">
+			<label for="difficulty-input" style="margin-right:0.5rem">Difficulty:</label>
+			<input id="difficulty-input" type="number" name="difficulty" style="width:3rem"/>
+		</div>
+		<div style="margin-top:0.25rem;display:flex;justify-content:end">
+			<label for="algorithm-select" style="margin-right:0.5rem">Algorithm:</label>
+			<select id="algorithm-select" name="algorithm"></select>
+		</div>
+		<div style="margin-top:0.25rem;display:flex;justify-content:end">
+			<label for="compare-select" style="margin-right:0.5rem">Compare:</label>
+			<select id="compare-select" name="compare">
+			  <option value="NONE">-</option>
+			</select>
+		</div>
+	</form>
+}
diff --git a/web/index_templ.go b/web/index_templ.go
index db2e732..2e3ac49 100644
--- a/web/index_templ.go
+++ b/web/index_templ.go
@@ -222,4 +222,60 @@ func errorPage(message string) templ.Component {
 	})
 }
 
+func bench() templ.Component {
+	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
+		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
+		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
+			return templ_7745c5c3_CtxErr
+		}
+		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
+		if !templ_7745c5c3_IsBuffer {
+			defer func() {
+				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
+				if templ_7745c5c3_Err == nil {
+					templ_7745c5c3_Err = templ_7745c5c3_BufErr
+				}
+			}()
+		}
+		ctx = templ.InitializeContext(ctx)
+		templ_7745c5c3_Var12 := templ.GetChildren(ctx)
+		if templ_7745c5c3_Var12 == nil {
+			templ_7745c5c3_Var12 = templ.NopComponent
+		}
+		ctx = templ.ClearChildren(ctx)
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<div style=\"height:20rem;display:flex\"><table style=\"margin-top:1rem;display:grid;grid-template:auto 1fr/auto auto;gap:0 0.5rem\"><thead style=\"border-bottom:1px solid black;padding:0.25rem 0;display:grid;grid-template:1fr/subgrid;grid-column:1/-1\"><tr id=\"table-header\" style=\"display:contents\"><th style=\"width:4.5rem\">Time</th><th style=\"width:4rem\">Iters</th></tr><tr id=\"table-header-compare\" style=\"display:none\"><th style=\"width:4.5rem\">Time A</th><th style=\"width:4rem\">Iters A</th><th style=\"width:4.5rem\">Time B</th><th style=\"width:4rem\">Iters B</th></tr></thead> <tbody id=\"results\" style=\"padding-top:0.25rem;display:grid;grid-template-columns:subgrid;grid-auto-rows:min-content;grid-column:1/-1;row-gap:0.25rem;overflow-y:auto;font-variant-numeric:tabular-nums\"></tbody></table><div class=\"centered-div\"><img id=\"image\" style=\"width:100%;max-width:256px;\" src=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var13 string
+		templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
+			anubis.Version)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 247, Col: 19}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "\"><p id=\"status\" style=\"max-width:256px\">Loading...</p><script async type=\"module\" src=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var14 string
+		templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/js/bench.mjs?cacheBuster=" + anubis.Version)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 250, Col: 118}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\"></script><div id=\"sparkline\"></div><noscript><p>Running the benchmark tool requires JavaScript to be enabled.</p></noscript></div></div><form id=\"controls\" style=\"position:fixed;top:0.5rem;right:0.5rem\"><div style=\"display:flex;justify-content:end\"><label for=\"difficulty-input\" style=\"margin-right:0.5rem\">Difficulty:</label> <input id=\"difficulty-input\" type=\"number\" name=\"difficulty\" style=\"width:3rem\"></div><div style=\"margin-top:0.25rem;display:flex;justify-content:end\"><label for=\"algorithm-select\" style=\"margin-right:0.5rem\">Algorithm:</label> <select id=\"algorithm-select\" name=\"algorithm\"></select></div><div style=\"margin-top:0.25rem;display:flex;justify-content:end\"><label for=\"compare-select\" style=\"margin-right:0.5rem\">Compare:</label> <select id=\"compare-select\" name=\"compare\"><option value=\"NONE\">-</option></select></div></form>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		return nil
+	})
+}
+
 var _ = templruntime.GeneratedTemplate
diff --git a/web/js/bench.mjs b/web/js/bench.mjs
new file mode 100644
index 0000000..c8c69bd
--- /dev/null
+++ b/web/js/bench.mjs
@@ -0,0 +1,152 @@
+import processFast from "./proof-of-work.mjs";
+import processSlow from "./proof-of-work-slow.mjs";
+
+const defaultDifficulty = 4;
+const algorithms = {
+  fast: processFast,
+  slow: processSlow,
+};
+
+const status = document.getElementById("status");
+const difficultyInput = document.getElementById("difficulty-input");
+const algorithmSelect = document.getElementById("algorithm-select");
+const compareSelect = document.getElementById("compare-select");
+const header = document.getElementById("table-header");
+const headerCompare = document.getElementById("table-header-compare");
+const results = document.getElementById("results");
+
+const setupControls = () => {
+  difficultyInput.value = defaultDifficulty;
+  for (const alg of Object.keys(algorithms)) {
+    const option1 = document.createElement("option");
+    algorithmSelect.append(option1);
+    const option2 = document.createElement("option");
+    compareSelect.append(option2);
+    option1.value = option1.innerText = option2.value = option2.innerText = alg;
+  }
+};
+
+const benchmarkTrial = async (stats, difficulty, algorithm, signal) => {
+  if (!(difficulty >= 1)) {
+    throw new Error(`Invalid difficulty: ${difficulty}`);
+  }
+  const process = algorithms[algorithm];
+  if (process == null) {
+    throw new Error(`Unknown algorithm: ${algorithm}`);
+  }
+
+  const rawChallenge = new Uint8Array(32);
+  crypto.getRandomValues(rawChallenge);
+  const challenge = Array.from(rawChallenge)
+    .map((c) => c.toString(16).padStart(2, "0"))
+    .join("");
+
+  const t0 = performance.now();
+  const { hash, nonce } = await process(challenge, Number(difficulty), signal);
+  const t1 = performance.now();
+  console.log({ hash, nonce });
+
+  stats.time += t1 - t0;
+  stats.iters += nonce;
+
+  return { time: t1 - t0, nonce };
+};
+
+const stats = { time: 0, iters: 0 };
+const comparison = { time: 0, iters: 0 };
+const updateStatus = () => {
+  const mainRate = stats.iters / stats.time;
+  const compareRate = comparison.iters / comparison.time;
+  if (Number.isFinite(mainRate)) {
+    status.innerText = `Average hashrate: ${mainRate.toFixed(3)}kH/s`;
+    if (Number.isFinite(compareRate)) {
+      const change = ((mainRate - compareRate) / mainRate) * 100;
+      status.innerText += ` vs ${compareRate.toFixed(3)}kH/s (${change.toFixed(2)}% change)`;
+    }
+  } else {
+    status.innerText = "Benchmarking...";
+  }
+};
+
+const tableCell = (text) => {
+  const td = document.createElement("td");
+  td.innerText = text;
+  td.style.padding = "0 0.25rem";
+  return td;
+};
+
+const benchmarkLoop = async (controller) => {
+  const difficulty = difficultyInput.value;
+  const algorithm = algorithmSelect.value;
+  const compareAlgorithm = compareSelect.value;
+  updateStatus();
+
+  try {
+    const { time, nonce } = await benchmarkTrial(
+      stats,
+      difficulty,
+      algorithm,
+      controller.signal,
+    );
+
+    const tr = document.createElement("tr");
+    tr.style.display = "contents";
+    tr.append(tableCell(`${time}ms`), tableCell(nonce));
+
+    // auto-scroll to new rows
+    const atBottom =
+      results.scrollHeight - results.clientHeight <= results.scrollTop;
+    results.append(tr);
+    if (atBottom) {
+      results.scrollTop = results.scrollHeight - results.clientHeight;
+    }
+    updateStatus();
+
+    if (compareAlgorithm !== "NONE") {
+      const { time, nonce } = await benchmarkTrial(
+        comparison,
+        difficulty,
+        compareAlgorithm,
+        controller.signal,
+      );
+      tr.append(tableCell(`${time}ms`), tableCell(nonce));
+    }
+  } catch (e) {
+    if (e !== false) {
+      status.innerText = e;
+    }
+    return;
+  }
+
+  benchmarkLoop(controller);
+};
+
+let controller = null;
+const reset = () => {
+  stats.time = stats.iters = 0;
+  comparison.time = comparison.iters = 0;
+  results.innerHTML = status.innerText = "";
+
+  const table = results.parentElement;
+  if (compareSelect.value !== "NONE") {
+    table.style.gridTemplateColumns = "repeat(4,auto)";
+    header.style.display = "none";
+    headerCompare.style.display = "contents";
+  } else {
+    table.style.gridTemplateColumns = "repeat(2,auto)";
+    header.style.display = "contents";
+    headerCompare.style.display = "none";
+  }
+
+  if (controller != null) {
+    controller.abort();
+  }
+  controller = new AbortController();
+  benchmarkLoop(controller);
+};
+
+setupControls();
+difficultyInput.addEventListener("change", reset);
+algorithmSelect.addEventListener("change", reset);
+compareSelect.addEventListener("change", reset);
+reset();
\ No newline at end of file
diff --git a/web/js/main.mjs b/web/js/main.mjs
index 01f21f0..3203e4a 100644
--- a/web/js/main.mjs
+++ b/web/js/main.mjs
@@ -127,6 +127,7 @@ const dependencies = [
     const { hash, nonce } = await process(
       challenge,
       rules.difficulty,
+      null,
       (iters) => {
         const delta = Date.now() - t0;
         // only update the speed every second so it's less visually distracting
diff --git a/web/js/proof-of-work-slow.mjs b/web/js/proof-of-work-slow.mjs
index 6522c0b..0bdc146 100644
--- a/web/js/proof-of-work-slow.mjs
+++ b/web/js/proof-of-work-slow.mjs
@@ -3,6 +3,7 @@
 export default function process(
   data,
   difficulty = 5,
+  signal = null,
   progressCallback = null,
   _threads = 1,
 ) {
@@ -13,19 +14,33 @@ export default function process(
     ], { type: 'application/javascript' }));
 
     let worker = new Worker(webWorkerURL);
+    const terminate = () => {
+      worker.terminate();
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
 
     worker.onmessage = (event) => {
       if (typeof event.data === "number") {
         progressCallback?.(event.data);
       } else {
-        worker.terminate();
+        terminate();
         resolve(event.data);
       }
     };
 
     worker.onerror = (event) => {
-      worker.terminate();
-      reject();
+      terminate();
+      reject(event);
     };
 
     worker.postMessage({
diff --git a/web/js/proof-of-work.mjs b/web/js/proof-of-work.mjs
index 60d8d61..a04f5ca 100644
--- a/web/js/proof-of-work.mjs
+++ b/web/js/proof-of-work.mjs
@@ -1,6 +1,7 @@
 export default function process(
   data,
   difficulty = 5,
+  signal = null,
   progressCallback = null,
   threads = (navigator.hardwareConcurrency || 1),
 ) {
@@ -11,6 +12,20 @@ export default function process(
     ], { type: 'application/javascript' }));
 
     const workers = [];
+    const terminate = () => {
+      workers.forEach((w) => w.terminate());
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
 
     for (let i = 0; i < threads; i++) {
       let worker = new Worker(webWorkerURL);
@@ -19,14 +34,14 @@ export default function process(
         if (typeof event.data === "number") {
           progressCallback?.(event.data);
         } else {
-          workers.forEach(worker => worker.terminate());
+          terminate();
           resolve(event.data);
         }
       };
 
       worker.onerror = (event) => {
-        worker.terminate();
-        reject();
+        terminate();
+        reject(event);
       };
 
       worker.postMessage({