Refactor and split out things into cmd and lib (#77)

* Refactor anubis to split business logic into a lib, and cmd to just be direct usage.

* Post-rebase fixes.

* Update changelog, remove unnecessary one.

* lib: refactor this

This is mostly based on my personal preferences for how Go code should
be laid out. I'm not sold on the package name "lib" (I'd call it anubis
but that would stutter), but people are probably gonna import it as
libanubis so it's likely fine.

Packages have been "flattened" to centralize implementation with area of
concern. This goes against the Java-esque style that many people like,
but I think this helps make things simple.

Most notably: the dnsbl client (which is a hack) is an internal package
until it's made more generic. Then it can be made external.

I also fixed the logic such that `go generate` works and rebased on
main.

* internal/test: run tests iff npx exists and DONT_USE_NETWORK is not set

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: install deps

Signed-off-by: Xe Iaso <me@xeiaso.net>

* .github/workflows: verbose go tests?

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: sleep 2

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: nix this test so CI works

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: warmup per browser?

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: disable for now :(

Signed-off-by: Xe Iaso <me@xeiaso.net>

* lib/anubis: do not apply bot rules if address check fails

Closes #83

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: Xe Iaso <me@xeiaso.net>

51 files changed, 1116 insertions, 818 deletions

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 2a25740..54d833c 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -56,6 +56,14 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-golang-
 
+    - name: Cache playwright binaries
+      uses: actions/cache@v3
+      id: playwright-cache
+      with:
+        path: |
+          ~/.cache/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ hashFiles('**/go.sum') }}
+
     - name: Build
       run: go build ./...
 
diff --git a/anubis.go b/anubis.go
new file mode 100644
index 0000000..b184a45
--- /dev/null
+++ b/anubis.go
@@ -0,0 +1,19 @@
+// Package Anubis contains the version number of Anubis.
+package anubis
+
+// Version is the current version of Anubis.
+//
+// This variable is set at build time using the -X linker flag. If not set,
+// it defaults to "devel".
+var Version = "devel"
+
+// CookieName is the name of the cookie that Anubis uses in order to validate
+// access.
+const CookieName = "within.website-x-cmd-anubis-auth"
+
+// StaticPath is the location where all static Anubis assets are located.
+const StaticPath = "/.within.website/x/cmd/anubis/"
+
+// DefaultDifficulty is the default "difficulty" (number of leading zeroes)
+// that must be met by the client in order to pass the challenge.
+const DefaultDifficulty = 4
diff --git a/cmd/anubis/CHANGELOG.md b/cmd/anubis/CHANGELOG.md
deleted file mode 100644
index 612bec1..0000000
--- a/cmd/anubis/CHANGELOG.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# CHANGELOG
-
-## 2025-01-24
-
-- Added support for custom bot policy documentation, allowing administrators to change how Anubis works to meet their needs.
diff --git a/cmd/anubis/main.go b/cmd/anubis/main.go
index e27e02f..e493931 100644
--- a/cmd/anubis/main.go
+++ b/cmd/anubis/main.go
@@ -2,20 +2,10 @@ package main
 
 import (
 	"context"
-	"crypto/ed25519"
-	"crypto/rand"
-	"crypto/sha256"
-	"crypto/subtle"
-	"embed"
-	"encoding/hex"
-	"encoding/json"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"log/slog"
-	"math"
-	mrand "math/rand"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -29,22 +19,18 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/config"
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/dnsbl"
 	"github.com/TecharoHQ/anubis/internal"
-	"github.com/TecharoHQ/anubis/xess"
-	"github.com/a-h/templ"
+	libanubis "github.com/TecharoHQ/anubis/lib"
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+	"github.com/TecharoHQ/anubis/web"
 	"github.com/facebookgo/flagenv"
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
 	bind                = flag.String("bind", ":8923", "network address to bind HTTP to")
 	bindNetwork         = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
-	challengeDifficulty = flag.Int("difficulty", defaultDifficulty, "difficulty of the challenge")
+	challengeDifficulty = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge")
 	metricsBind         = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
 	metricsBindNetwork  = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
 	socketMode          = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
@@ -54,49 +40,8 @@ var (
 	target              = flag.String("target", "http://localhost:3923", "target to reverse proxy to")
 	healthcheck         = flag.Bool("healthcheck", false, "run a health check against Anubis")
 	debugXRealIPDefault = flag.String("debug-x-real-ip-default", "", "If set, replace empty X-Real-Ip headers with this value, useful only for debugging Anubis and running it locally")
-
-	//go:embed static botPolicies.json
-	static embed.FS
-
-	challengesIssued = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_challenges_issued",
-		Help: "The total number of challenges issued",
-	})
-
-	challengesValidated = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_challenges_validated",
-		Help: "The total number of challenges validated",
-	})
-
-	droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "anubis_dronebl_hits",
-		Help: "The total number of hits from DroneBL",
-	}, []string{"status"})
-
-	failedValidations = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_failed_validations",
-		Help: "The total number of failed validations",
-	})
-
-	timeTaken = promauto.NewHistogram(prometheus.HistogramOpts{
-		Name:    "anubis_time_taken",
-		Help:    "The time taken for a browser to generate a response (milliseconds)",
-		Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 18), 19),
-	})
 )
 
-const (
-	cookieName        = "within.website-x-cmd-anubis-auth"
-	staticPath        = "/.within.website/x/cmd/anubis/"
-	defaultDifficulty = 4
-)
-
-//go:generate go tool github.com/a-h/templ/cmd/templ generate
-//go:generate esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
-//go:generate gzip -f -k static/js/main.mjs
-//go:generate zstd -f -k --ultra -22 static/js/main.mjs
-//go:generate brotli -fZk static/js/main.mjs
-
 func doHealthCheck() error {
 	resp, err := http.Get("http://localhost" + *metricsBind + "/metrics")
 	if err != nil {
@@ -145,6 +90,34 @@ func setupListener(network string, address string) (net.Listener, string) {
 	return listener, formattedAddress
 }
 
+func makeReverseProxy(target string) (http.Handler, error) {
+	u, err := url.Parse(target)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse target URL: %w", err)
+	}
+
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+
+	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
+	if u.Scheme == "unix" {
+		// clean path up so we don't use the socket path in proxied requests
+		addr := u.Path
+		u.Path = ""
+		// tell transport how to dial unix sockets
+		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			dialer := net.Dialer{}
+			return dialer.DialContext(ctx, "unix", addr)
+		}
+		// tell transport how to handle the unix url scheme
+		transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport})
+	}
+
+	rp := httputil.NewSingleHostReverseProxy(u)
+	rp.Transport = transport
+
+	return rp, nil
+}
+
 func main() {
 	flagenv.Parse()
 	flag.Parse()
@@ -158,13 +131,18 @@ func main() {
 		return
 	}
 
-	s, err := New(*target, *policyFname)
+	rp, err := makeReverseProxy(*target)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("can't make reverse proxy: %v", err)
+	}
+
+	policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty)
+	if err != nil {
+		log.Fatalf("can't parse policy file: %v", err)
 	}
 
 	fmt.Println("Rule error IDs:")
-	for _, rule := range s.policy.Bots {
+	for _, rule := range policy.Bots {
 		if rule.Action != config.RuleDeny {
 			continue
 		}
@@ -178,25 +156,13 @@ func main() {
 	}
 	fmt.Println()
 
-	mux := http.NewServeMux()
-	xess.Mount(mux)
-
-	mux.Handle(staticPath, internal.UnchangingCache(http.StripPrefix(staticPath, http.FileServerFS(static))))
-
-	// mux.HandleFunc("GET /.within.website/x/cmd/anubis/static/js/main.mjs", serveMainJSWithBestEncoding)
-
-	mux.HandleFunc("POST /.within.website/x/cmd/anubis/api/make-challenge", s.makeChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/pass-challenge", s.passChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/test-error", s.testError)
-
-	if *robotsTxt {
-		mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, static, "static/robots.txt")
-		})
-
-		mux.HandleFunc("/.well-known/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, static, "static/robots.txt")
-		})
+	s, err := libanubis.New(libanubis.Options{
+		Next:           rp,
+		Policy:         policy,
+		ServeRobotsTXT: *robotsTxt,
+	})
+	if err != nil {
+		log.Fatalf("can't construct libanubis.Server: %v", err)
 	}
 
 	wg := new(sync.WaitGroup)
@@ -209,10 +175,8 @@ func main() {
 		go metricsServer(ctx, wg.Done)
 	}
 
-	mux.HandleFunc("/", s.maybeReverseProxy)
-
 	var h http.Handler
-	h = mux
+	h = s
 	h = internal.DefaultXRealIP(*debugXRealIPDefault, h)
 	h = internal.XForwardedForToXRealIP(h)
 
@@ -267,428 +231,6 @@ func metricsServer(ctx context.Context, done func()) {
 	}
 }
 
-func sha256sum(text string) string {
-	hash := sha256.New()
-	hash.Write([]byte(text))
-	return hex.EncodeToString(hash.Sum(nil))
-}
-
-func (s *Server) challengeFor(r *http.Request, difficulty int) string {
-	fp := sha256.Sum256(s.priv.Seed())
-
-	data := fmt.Sprintf(
-		"Accept-Language=%s,X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d",
-		r.Header.Get("Accept-Language"),
-		r.Header.Get("X-Real-Ip"),
-		r.UserAgent(),
-		time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
-		fp,
-		difficulty,
-	)
-	return sha256sum(data)
-}
-
-func New(target, policyFname string) (*Server, error) {
-	u, err := url.Parse(target)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse target URL: %w", err)
-	}
-
-	pub, priv, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate ed25519 key: %w", err)
-	}
-
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-
-	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
-	if u.Scheme == "unix" {
-		// clean path up so we don't use the socket path in proxied requests
-		addr := u.Path
-		u.Path = ""
-		// tell transport how to dial unix sockets
-		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			dialer := net.Dialer{}
-			return dialer.DialContext(ctx, "unix", addr)
-		}
-		// tell transport how to handle the unix url scheme
-		transport.RegisterProtocol("unix", unixRoundTripper{Transport: transport})
-	}
-
-	rp := httputil.NewSingleHostReverseProxy(u)
-	rp.Transport = transport
-
-	var fin io.ReadCloser
-
-	if policyFname != "" {
-		fin, err = os.Open(policyFname)
-		if err != nil {
-			return nil, fmt.Errorf("can't parse policy file %s: %w", policyFname, err)
-		}
-	} else {
-		policyFname = "(static)/botPolicies.json"
-		fin, err = static.Open("botPolicies.json")
-		if err != nil {
-			return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", policyFname, err)
-		}
-	}
-
-	defer fin.Close()
-
-	policy, err := parseConfig(fin, policyFname, *challengeDifficulty)
-	if err != nil {
-		return nil, err // parseConfig sets a fancy error for us
-	}
-
-	return &Server{
-		rp:         rp,
-		priv:       priv,
-		pub:        pub,
-		policy:     policy,
-		dnsblCache: NewDecayMap[string, dnsbl.DroneBLResponse](),
-	}, nil
-}
-
-// https://github.com/oauth2-proxy/oauth2-proxy/blob/master/pkg/upstream/http.go#L124
-type unixRoundTripper struct {
-	Transport *http.Transport
-}
-
-// set bare minimum stuff
-func (t unixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	req = req.Clone(req.Context())
-	if req.Host == "" {
-		req.Host = "localhost"
-	}
-	req.URL.Host = req.Host // proxy error: no Host in request URL
-	req.URL.Scheme = "http" // make http.Transport happy and avoid an infinite recursion
-	return t.Transport.RoundTrip(req)
-}
-
-type Server struct {
-	rp         *httputil.ReverseProxy
-	priv       ed25519.PrivateKey
-	pub        ed25519.PublicKey
-	policy     *ParsedConfig
-	dnsblCache *DecayMap[string, dnsbl.DroneBLResponse]
-}
-
-func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for",
-		r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	r.Header.Add("X-Anubis-Rule", cr.Name)
-	r.Header.Add("X-Anubis-Action", string(cr.Rule))
-	lg = lg.With("check_result", cr)
-	policyApplications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1)
-
-	ip := r.Header.Get("X-Real-Ip")
-
-	if s.policy.DNSBL && ip != "" {
-		resp, ok := s.dnsblCache.Get(ip)
-		if !ok {
-			lg.Debug("looking up ip in dnsbl")
-			resp, err := dnsbl.Lookup(ip)
-			if err != nil {
-				lg.Error("can't look up ip in dnsbl", "err", err)
-			}
-			s.dnsblCache.Set(ip, resp, 24*time.Hour)
-			droneBLHits.WithLabelValues(resp.String()).Inc()
-		}
-
-		if resp != dnsbl.AllGood {
-			lg.Info("DNSBL hit", "status", resp.String())
-			templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-			return
-		}
-	}
-
-	switch cr.Rule {
-	case config.RuleAllow:
-		lg.Debug("allowing traffic to origin (explicit)")
-		s.rp.ServeHTTP(w, r)
-		return
-	case config.RuleDeny:
-		clearCookie(w)
-		lg.Info("explicit deny")
-		if rule == nil {
-			lg.Error("rule is nil, cannot calculate checksum")
-			templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		}
-		hash, err := rule.Hash()
-		if err != nil {
-			lg.Error("can't calculate checksum of rule", "err", err)
-			templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		}
-		lg.Debug("rule hash", "hash", hash)
-		templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("Access Denied: error code %s", hash))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-		return
-	case config.RuleChallenge:
-		lg.Debug("challenge requested")
-	default:
-		clearCookie(w)
-		templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	ckie, err := r.Cookie(cookieName)
-	if err != nil {
-		lg.Debug("cookie not found", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if err := ckie.Valid(); err != nil {
-		lg.Debug("cookie is invalid", "err", err)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() {
-		lg.Debug("cookie expired", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	token, err := jwt.ParseWithClaims(ckie.Value, jwt.MapClaims{}, func(token *jwt.Token) (interface{}, error) {
-		return s.pub, nil
-	}, jwt.WithExpirationRequired(), jwt.WithStrictDecoding())
-
-	if err != nil || !token.Valid {
-		lg.Debug("invalid token", "path", r.URL.Path, "err", err)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if randomJitter() {
-		r.Header.Add("X-Anubis-Status", "PASS-BRIEF")
-		lg.Debug("cookie is not enrolled into secondary screening")
-		s.rp.ServeHTTP(w, r)
-		return
-	}
-
-	claims, ok := token.Claims.(jwt.MapClaims)
-	if !ok {
-		lg.Debug("invalid token claims type", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	if claims["challenge"] != challenge {
-		lg.Debug("invalid challenge", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	var nonce int
-
-	if v, ok := claims["nonce"].(float64); ok {
-		nonce = int(v)
-	}
-
-	calcString := fmt.Sprintf("%s%d", challenge, nonce)
-	calculated := sha256sum(calcString)
-
-	if subtle.ConstantTimeCompare([]byte(claims["response"].(string)), []byte(calculated)) != 1 {
-		lg.Debug("invalid response", "path", r.URL.Path)
-		failedValidations.Inc()
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	slog.Debug("all checks passed")
-	r.Header.Add("X-Anubis-Status", "PASS-FULL")
-	s.rp.ServeHTTP(w, r)
-}
-
-func (s *Server) renderIndex(w http.ResponseWriter, r *http.Request) {
-	templ.Handler(
-		base("Making sure you're not a bot!", index()),
-	).ServeHTTP(w, r)
-}
-
-func (s *Server) makeChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		json.NewEncoder(w).Encode(struct {
-			Error string `json:"error"`
-		}{
-			Error: "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"makeChallenge\"",
-		})
-		return
-	}
-	lg = lg.With("check_result", cr)
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	json.NewEncoder(w).Encode(struct {
-		Challenge string                 `json:"challenge"`
-		Rules     *config.ChallengeRules `json:"rules"`
-	}{
-		Challenge: challenge,
-		Rules:     rule.Challenge,
-	})
-	lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr)
-	challengesIssued.Inc()
-}
-
-func (s *Server) passChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for", r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-	lg = lg.With("check_result", cr)
-
-	nonceStr := r.FormValue("nonce")
-	if nonceStr == "" {
-		clearCookie(w)
-		lg.Debug("no nonce")
-		templ.Handler(base("Oh noes!", errorPage("missing nonce")), tem