Debug tool for benchmarking proof-of-work algorithms (#155)

* cmd/anubis: add a debug option for benchmarking hashrate

Having the ability to benchmark different proof-of-work implementations
is useful for extending Anubis. This adds a flag `--debug-benchmark-js`
(and its associated environment variable `DEBUG_BENCHMARK_JS`) for
serving a tool to do so.

Internally, a there is a new policy action, "DEBUG_BENCHMARK", which
serves the benchmarking tool instead of a challenge. The flag then
replaces all bot rules with a special rule matching every request
to that action. The benchmark page makes heavy use of inline styles,
because currently all global styles are shared across all pages. This
could be fixed, but I wanted to avoid major changes to the templates.

* web/js: add signal for aborting an active proof-of-work algorithm

Both proof-of-work algorithms now take an optional `AbortSignal`, which
immediately terminates all workers and returns `false` if aborted before
the challenge is complete.

* web/js: add algorithm comparison to the benchmark page

"Compare:" is added to the benchmark page for testing the relative
performance between two algorithms. Since benchmark runs generally have
high variance, it may take a while for the averages to converge on a
stable difference.

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: Xe Iaso <me@xeiaso.net>

13 files changed, 331 insertions, 8 deletions

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..70e1a30
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+web/index_templ.go linguist-generated
diff --git a/cmd/anubis/main.go b/cmd/anubis/main.go
index 4cee20c..8ab370d 100644
--- a/cmd/anubis/main.go
+++ b/cmd/anubis/main.go
@@ -15,6 +15,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"regexp"
 	"strconv"
 	"sync"
 	"syscall"
@@ -23,6 +24,7 @@ import (
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/internal"
 	libanubis "github.com/TecharoHQ/anubis/lib"
+	botPolicy "github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 	"github.com/facebookgo/flagenv"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -44,6 +46,7 @@ var (
 	target               = flag.String("target", "http://localhost:3923", "target to reverse proxy to")
 	healthcheck          = flag.Bool("healthcheck", false, "run a health check against Anubis")
 	useRemoteAddress     = flag.Bool("use-remote-address", false, "read the client's IP address from the network request, useful for debugging and running Anubis on bare metal")
+	debugBenchmarkJS     = flag.Bool("debug-benchmark-js", false, "respond to every request with a challenge for benchmarking hashrate")
 )
 
 func keyFromHex(value string) (ed25519.PrivateKey, error) {
@@ -187,6 +190,16 @@ func main() {
 	}
 	fmt.Println()
 
+	// replace the bot policy rules with a single rule that always benchmarks
+	if *debugBenchmarkJS {
+		userAgent := regexp.MustCompile(".")
+		policy.Bots = []botPolicy.Bot{{
+			Name:      "",
+			UserAgent: userAgent,
+			Action:    config.RuleBenchmark,
+		}}
+	}
+
 	var priv ed25519.PrivateKey
 	if *ed25519PrivateKeyHex == "" {
 		_, priv, err = ed25519.GenerateKey(rand.Reader)
@@ -241,6 +254,7 @@ func main() {
 		"target", *target,
 		"version", anubis.Version,
 		"use-remote-address", *useRemoteAddress,
+		"debug-benchmark-js", *debugBenchmarkJS,
 	)
 
 	go func() {
diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md
index 4189e57..39407b4 100644
--- a/docs/docs/CHANGELOG.md
+++ b/docs/docs/CHANGELOG.md
@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Developer documentation has been added to the docs site
 - Show more errors when some predictable challenge page errors happen ([#150](https://github.com/TecharoHQ/anubis/issues/150))
 - Verification page now shows hash rate and a progress bar for completion probability.
+- Added the `--debug-benchmark-js` flag for testing proof-of-work performance during development.
 - Use `TrimSuffix` instead of `TrimRight` on containerbuild
 
 ## v1.15.0
diff --git a/lib/anubis.go b/lib/anubis.go
index c61b110..939262a 100644
--- a/lib/anubis.go
+++ b/lib/anubis.go
@@ -241,6 +241,10 @@ func (s *Server) MaybeReverseProxy(w http.ResponseWriter, r *http.Request) {
 		return
 	case config.RuleChallenge:
 		lg.Debug("challenge requested")
+	case config.RuleBenchmark:
+		lg.Debug("serving benchmark page")
+		s.RenderBench(w, r)
+		return
 	default:
 		s.ClearCookie(w)
 		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
@@ -334,6 +338,12 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request) {
 	handler.ServeHTTP(w, r)
 }
 
+func (s *Server) RenderBench(w http.ResponseWriter, r *http.Request) {
+	templ.Handler(
+		web.Base("Benchmarking Anubis!", web.Bench()),
+	).ServeHTTP(w, r)
+}
+
 func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
 	lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
 
diff --git a/lib/policy/config/config.go b/lib/policy/config/config.go
index b23af70..e8f5161 100644
--- a/lib/policy/config/config.go
+++ b/lib/policy/config/config.go
@@ -25,6 +25,7 @@ const (
 	RuleAllow     Rule = "ALLOW"
 	RuleDeny      Rule = "DENY"
 	RuleChallenge Rule = "CHALLENGE"
+	RuleBenchmark Rule = "DEBUG_BENCHMARK"
 )
 
 type Algorithm string
@@ -80,7 +81,7 @@ func (b BotConfig) Valid() error {
 	}
 
 	switch b.Action {
-	case RuleAllow, RuleChallenge, RuleDeny:
+	case RuleAllow, RuleBenchmark, RuleChallenge, RuleDeny:
 		// okay
 	default:
 		errs = append(errs, fmt.Errorf("%w: %q", ErrUnknownAction, b.Action))
diff --git a/web/build.sh b/web/build.sh
index 70492d7..a513c59 100755
--- a/web/build.sh
+++ b/web/build.sh
@@ -7,4 +7,6 @@ cd "$(dirname "$0")"
 esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
 gzip -f -k static/js/main.mjs
 zstd -f -k --ultra -22 static/js/main.mjs
-brotli -fZk static/js/main.mjs
\ No newline at end of file
+brotli -fZk static/js/main.mjs
+
+esbuild js/bench.mjs --sourcemap --bundle --minify --outfile=static/js/bench.mjs
\ No newline at end of file
diff --git a/web/index.go b/web/index.go
index 7057cc8..6ef84b5 100644
--- a/web/index.go
+++ b/web/index.go
@@ -13,3 +13,7 @@ func Index() templ.Component {
 func ErrorPage(msg string) templ.Component {
 	return errorPage(msg)
 }
+
+func Bench() templ.Component {
+	return bench()
+}
diff --git a/web/index.templ b/web/index.templ
index 1899ae3..b43e82c 100644
--- a/web/index.templ
+++ b/web/index.templ
@@ -121,3 +121,54 @@ templ errorPage(message string) {
 		<p><a href="/">Go home</a></p>
 	</div>
 }
+
+templ bench() {
+	<div style="height:20rem;display:flex">
+		<table style="margin-top:1rem;display:grid;grid-template:auto 1fr/auto auto;gap:0 0.5rem">
+			<thead style="border-bottom:1px solid black;padding:0.25rem 0;display:grid;grid-template:1fr/subgrid;grid-column:1/-1">
+				<tr id="table-header" style="display:contents">
+					<th style="width:4.5rem">Time</th>
+					<th style="width:4rem">Iters</th>
+				</tr>
+				<tr id="table-header-compare" style="display:none">
+					<th style="width:4.5rem">Time A</th>
+					<th style="width:4rem">Iters A</th>
+					<th style="width:4.5rem">Time B</th>
+					<th style="width:4rem">Iters B</th>
+				</tr>
+			</thead>
+			<tbody id="results" style="padding-top:0.25rem;display:grid;grid-template-columns:subgrid;grid-auto-rows:min-content;grid-column:1/-1;row-gap:0.25rem;overflow-y:auto;font-variant-numeric:tabular-nums">
+			</tbody>
+		</table>
+		<div class="centered-div">
+			<img
+				id="image"
+				style="width:100%;max-width:256px;"
+				src={ "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
+	    anubis.Version }
+			/>
+			<p id="status" style="max-width:256px">Loading...</p>
+			<script async type="module" src={ "/.within.website/x/cmd/anubis/static/js/bench.mjs?cacheBuster=" + anubis.Version }></script>
+			<div id="sparkline"></div>
+			<noscript>
+				<p>Running the benchmark tool requires JavaScript to be enabled.</p>
+			</noscript>
+		</div>
+	</div>
+	<form id="controls" style="position:fixed;top:0.5rem;right:0.5rem">
+		<div style="display:flex;justify-content:end">
+			<label for="difficulty-input" style="margin-right:0.5rem">Difficulty:</label>
+			<input id="difficulty-input" type="number" name="difficulty" style="width:3rem"/>
+		</div>
+		<div style="margin-top:0.25rem;display:flex;justify-content:end">
+			<label for="algorithm-select" style="margin-right:0.5rem">Algorithm:</label>
+			<select id="algorithm-select" name="algorithm"></select>
+		</div>
+		<div style="margin-top:0.25rem;display:flex;justify-content:end">
+			<label for="compare-select" style="margin-right:0.5rem">Compare:</label>
+			<select id="compare-select" name="compare">
+			  <option value="NONE">-</option>
+			</select>
+		</div>
+	</form>
+}
diff --git a/web/index_templ.go b/web/index_templ.go
index db2e732..2e3ac49 100644
--- a/web/index_templ.go
+++ b/web/index_templ.go
@@ -222,4 +222,60 @@ func errorPage(message string) templ.Component {
 	})
 }
 
+func bench() templ.Component {
+	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
+		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
+		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
+			return templ_7745c5c3_CtxErr
+		}
+		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
+		if !templ_7745c5c3_IsBuffer {
+			defer func() {
+				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
+				if templ_7745c5c3_Err == nil {
+					templ_7745c5c3_Err = templ_7745c5c3_BufErr
+				}
+			}()
+		}
+		ctx = templ.InitializeContext(ctx)
+		templ_7745c5c3_Var12 := templ.GetChildren(ctx)
+		if templ_7745c5c3_Var12 == nil {
+			templ_7745c5c3_Var12 = templ.NopComponent
+		}
+		ctx = templ.ClearChildren(ctx)
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<div style=\"height:20rem;display:flex\"><table style=\"margin-top:1rem;display:grid;grid-template:auto 1fr/auto auto;gap:0 0.5rem\"><thead style=\"border-bottom:1px solid black;padding:0.25rem 0;display:grid;grid-template:1fr/subgrid;grid-column:1/-1\"><tr id=\"table-header\" style=\"display:contents\"><th style=\"width:4.5rem\">Time</th><th style=\"width:4rem\">Iters</th></tr><tr id=\"table-header-compare\" style=\"display:none\"><th style=\"width:4.5rem\">Time A</th><th style=\"width:4rem\">Iters A</th><th style=\"width:4.5rem\">Time B</th><th style=\"width:4rem\">Iters B</th></tr></thead> <tbody id=\"results\" style=\"padding-top:0.25rem;display:grid;grid-template-columns:subgrid;grid-auto-rows:min-content;grid-column:1/-1;row-gap:0.25rem;overflow-y:auto;font-variant-numeric:tabular-nums\"></tbody></table><div class=\"centered-div\"><img id=\"image\" style=\"width:100%;max-width:256px;\" src=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var13 string
+		templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" +
+			anubis.Version)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 247, Col: 19}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "\"><p id=\"status\" style=\"max-width:256px\">Loading...</p><script async type=\"module\" src=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var14 string
+		templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs("/.within.website/x/cmd/anubis/static/js/bench.mjs?cacheBuster=" + anubis.Version)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `index.templ`, Line: 250, Col: 118}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\"></script><div id=\"sparkline\"></div><noscript><p>Running the benchmark tool requires JavaScript to be enabled.</p></noscript></div></div><form id=\"controls\" style=\"position:fixed;top:0.5rem;right:0.5rem\"><div style=\"display:flex;justify-content:end\"><label for=\"difficulty-input\" style=\"margin-right:0.5rem\">Difficulty:</label> <input id=\"difficulty-input\" type=\"number\" name=\"difficulty\" style=\"width:3rem\"></div><div style=\"margin-top:0.25rem;display:flex;justify-content:end\"><label for=\"algorithm-select\" style=\"margin-right:0.5rem\">Algorithm:</label> <select id=\"algorithm-select\" name=\"algorithm\"></select></div><div style=\"margin-top:0.25rem;display:flex;justify-content:end\"><label for=\"compare-select\" style=\"margin-right:0.5rem\">Compare:</label> <select id=\"compare-select\" name=\"compare\"><option value=\"NONE\">-</option></select></div></form>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		return nil
+	})
+}
+
 var _ = templruntime.GeneratedTemplate
diff --git a/web/js/bench.mjs b/web/js/bench.mjs
new file mode 100644
index 0000000..c8c69bd
--- /dev/null
+++ b/web/js/bench.mjs
@@ -0,0 +1,152 @@
+import processFast from "./proof-of-work.mjs";
+import processSlow from "./proof-of-work-slow.mjs";
+
+const defaultDifficulty = 4;
+const algorithms = {
+  fast: processFast,
+  slow: processSlow,
+};
+
+const status = document.getElementById("status");
+const difficultyInput = document.getElementById("difficulty-input");
+const algorithmSelect = document.getElementById("algorithm-select");
+const compareSelect = document.getElementById("compare-select");
+const header = document.getElementById("table-header");
+const headerCompare = document.getElementById("table-header-compare");
+const results = document.getElementById("results");
+
+const setupControls = () => {
+  difficultyInput.value = defaultDifficulty;
+  for (const alg of Object.keys(algorithms)) {
+    const option1 = document.createElement("option");
+    algorithmSelect.append(option1);
+    const option2 = document.createElement("option");
+    compareSelect.append(option2);
+    option1.value = option1.innerText = option2.value = option2.innerText = alg;
+  }
+};
+
+const benchmarkTrial = async (stats, difficulty, algorithm, signal) => {
+  if (!(difficulty >= 1)) {
+    throw new Error(`Invalid difficulty: ${difficulty}`);
+  }
+  const process = algorithms[algorithm];
+  if (process == null) {
+    throw new Error(`Unknown algorithm: ${algorithm}`);
+  }
+
+  const rawChallenge = new Uint8Array(32);
+  crypto.getRandomValues(rawChallenge);
+  const challenge = Array.from(rawChallenge)
+    .map((c) => c.toString(16).padStart(2, "0"))
+    .join("");
+
+  const t0 = performance.now();
+  const { hash, nonce } = await process(challenge, Number(difficulty), signal);
+  const t1 = performance.now();
+  console.log({ hash, nonce });
+
+  stats.time += t1 - t0;
+  stats.iters += nonce;
+
+  return { time: t1 - t0, nonce };
+};
+
+const stats = { time: 0, iters: 0 };
+const comparison = { time: 0, iters: 0 };
+const updateStatus = () => {
+  const mainRate = stats.iters / stats.time;
+  const compareRate = comparison.iters / comparison.time;
+  if (Number.isFinite(mainRate)) {
+    status.innerText = `Average hashrate: ${mainRate.toFixed(3)}kH/s`;
+    if (Number.isFinite(compareRate)) {
+      const change = ((mainRate - compareRate) / mainRate) * 100;
+      status.innerText += ` vs ${compareRate.toFixed(3)}kH/s (${change.toFixed(2)}% change)`;
+    }
+  } else {
+    status.innerText = "Benchmarking...";
+  }
+};
+
+const tableCell = (text) => {
+  const td = document.createElement("td");
+  td.innerText = text;
+  td.style.padding = "0 0.25rem";
+  return td;
+};
+
+const benchmarkLoop = async (controller) => {
+  const difficulty = difficultyInput.value;
+  const algorithm = algorithmSelect.value;
+  const compareAlgorithm = compareSelect.value;
+  updateStatus();
+
+  try {
+    const { time, nonce } = await benchmarkTrial(
+      stats,
+      difficulty,
+      algorithm,
+      controller.signal,
+    );
+
+    const tr = document.createElement("tr");
+    tr.style.display = "contents";
+    tr.append(tableCell(`${time}ms`), tableCell(nonce));
+
+    // auto-scroll to new rows
+    const atBottom =
+      results.scrollHeight - results.clientHeight <= results.scrollTop;
+    results.append(tr);
+    if (atBottom) {
+      results.scrollTop = results.scrollHeight - results.clientHeight;
+    }
+    updateStatus();
+
+    if (compareAlgorithm !== "NONE") {
+      const { time, nonce } = await benchmarkTrial(
+        comparison,
+        difficulty,
+        compareAlgorithm,
+        controller.signal,
+      );
+      tr.append(tableCell(`${time}ms`), tableCell(nonce));
+    }
+  } catch (e) {
+    if (e !== false) {
+      status.innerText = e;
+    }
+    return;
+  }
+
+  benchmarkLoop(controller);
+};
+
+let controller = null;
+const reset = () => {
+  stats.time = stats.iters = 0;
+  comparison.time = comparison.iters = 0;
+  results.innerHTML = status.innerText = "";
+
+  const table = results.parentElement;
+  if (compareSelect.value !== "NONE") {
+    table.style.gridTemplateColumns = "repeat(4,auto)";
+    header.style.display = "none";
+    headerCompare.style.display = "contents";
+  } else {
+    table.style.gridTemplateColumns = "repeat(2,auto)";
+    header.style.display = "contents";
+    headerCompare.style.display = "none";
+  }
+
+  if (controller != null) {
+    controller.abort();
+  }
+  controller = new AbortController();
+  benchmarkLoop(controller);
+};
+
+setupControls();
+difficultyInput.addEventListener("change", reset);
+algorithmSelect.addEventListener("change", reset);
+compareSelect.addEventListener("change", reset);
+reset();
\ No newline at end of file
diff --git a/web/js/main.mjs b/web/js/main.mjs
index 01f21f0..3203e4a 100644
--- a/web/js/main.mjs
+++ b/web/js/main.mjs
@@ -127,6 +127,7 @@ const dependencies = [
     const { hash, nonce } = await process(
       challenge,
       rules.difficulty,
+      null,
       (iters) => {
         const delta = Date.now() - t0;
         // only update the speed every second so it's less visually distracting
diff --git a/web/js/proof-of-work-slow.mjs b/web/js/proof-of-work-slow.mjs
index 6522c0b..0bdc146 100644
--- a/web/js/proof-of-work-slow.mjs
+++ b/web/js/proof-of-work-slow.mjs
@@ -3,6 +3,7 @@
 export default function process(
   data,
   difficulty = 5,
+  signal = null,
   progressCallback = null,
   _threads = 1,
 ) {
@@ -13,19 +14,33 @@ export default function process(
     ], { type: 'application/javascript' }));
 
     let worker = new Worker(webWorkerURL);
+    const terminate = () => {
+      worker.terminate();
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
 
     worker.onmessage = (event) => {
       if (typeof event.data === "number") {
         progressCallback?.(event.data);
       } else {
-        worker.terminate();
+        terminate();
         resolve(event.data);
       }
     };
 
     worker.onerror = (event) => {
-      worker.terminate();
-      reject();
+      terminate();
+      reject(event);
     };
 
     worker.postMessage({
diff --git a/web/js/proof-of-work.mjs b/web/js/proof-of-work.mjs
index 60d8d61..a04f5ca 100644
--- a/web/js/proof-of-work.mjs
+++ b/web/js/proof-of-work.mjs
@@ -1,6 +1,7 @@
 export default function process(
   data,
   difficulty = 5,
+  signal = null,
   progressCallback = null,
   threads = (navigator.hardwareConcurrency || 1),
 ) {
@@ -11,6 +12,20 @@ export default function process(
     ], { type: 'application/javascript' }));
 
     const workers = [];
+    const terminate = () => {
+      workers.forEach((w) => w.terminate());
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
 
     for (let i = 0; i < threads; i++) {
       let worker = new Worker(webWorkerURL);
@@ -19,14 +34,14 @@ export default function process(
         if (typeof event.data === "number") {
           progressCallback?.(event.data);
         } else {
-          workers.forEach(worker => worker.terminate());
+          terminate();
           resolve(event.data);
         }
       };
 
       worker.onerror = (event) => {
-        worker.terminate();
-        reject();
+        terminate();
+        reject(event);
       };
 
       worker.postMessage({