From 57c3e9f1b2c1d685472670b8ba2660144d2ae316 Mon Sep 17 00:00:00 2001 From: Henri Vasserman Date: Fri, 28 Mar 2025 19:38:34 +0200 Subject: Change how to make Anubis work without a reverse proxy (#86) * Change how to make Anubis work without a reverse proxy * Apply suggestions from code review Co-authored-by: Xe Iaso Signed-off-by: Henri Vasserman * add support for unix sockets. * add env var docs * lib: fix tests Signed-off-by: Xe Iaso --------- Signed-off-by: Henri Vasserman Signed-off-by: Xe Iaso Co-authored-by: Xe Iaso --- cmd/anubis/main.go | 6 +++--- docs/docs/CHANGELOG.md | 1 + docs/docs/admin/installation.mdx | 1 + internal/headers.go | 26 ++++++++++++++++++++------ lib/anubis_test.go | 2 +- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/cmd/anubis/main.go b/cmd/anubis/main.go index 7f98e7b..b634676 100644 --- a/cmd/anubis/main.go +++ b/cmd/anubis/main.go @@ -45,7 +45,7 @@ var ( slogLevel = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)") target = flag.String("target", "http://localhost:3923", "target to reverse proxy to") healthcheck = flag.Bool("healthcheck", false, "run a health check against Anubis") - debugXRealIPDefault = flag.String("debug-x-real-ip-default", "", "If set, replace empty X-Real-Ip headers with this value, useful only for debugging Anubis and running it locally") + useRemoteAddress = flag.Bool("use-remote-address", false, "read the client's IP address from the network request, useful for debugging and running Anubis on bare metal") ) func keyFromHex(value string) (ed25519.PrivateKey, error) { @@ -214,7 +214,7 @@ func main() { var h http.Handler h = s - h = internal.DefaultXRealIP(*debugXRealIPDefault, h) + h = internal.RemoteXRealIP(*useRemoteAddress, *bindNetwork, h) h = internal.XForwardedForToXRealIP(h) srv := http.Server{Handler: h} @@ -226,7 +226,7 @@ func main() { "serveRobotsTXT", *robotsTxt, "target", *target, "version", anubis.Version, - "debug-x-real-ip-default", *debugXRealIPDefault, + "use-remote-address", *useRemoteAddress, ) go func() { diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index a3116b8..9898eec 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Changed `--debug-x-real-ip-default` to `--use-remote-address`, getting the IP address from the request's socket address instead. - DroneBL lookups have been disabled by default ## v1.15.0 diff --git a/docs/docs/admin/installation.mdx b/docs/docs/admin/installation.mdx index 5352683..ee17a89 100644 --- a/docs/docs/admin/installation.mdx +++ b/docs/docs/admin/installation.mdx @@ -55,6 +55,7 @@ Anubis uses these environment variables for configuration: | `POLICY_FNAME` | unset | The file containing [bot policy configuration](./policies.md). See the bot policy documentation for more details. If unset, the default bot policy configuration is used. | | `SERVE_ROBOTS_TXT` | `false` | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file. | | `TARGET` | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`. | +| `USE_REMOTE_ADDRESS` | unset | If set to `true`, Anubis will take the client's IP from the network socket. For production deployments, it is expected that a reverse proxy is used in front of Anubis, which pass the IP using headers, instead. | ### Key generation diff --git a/internal/headers.go b/internal/headers.go index 681d076..9d6ba76 100644 --- a/internal/headers.go +++ b/internal/headers.go @@ -2,6 +2,7 @@ package internal import ( "log/slog" + "net" "net/http" "github.com/TecharoHQ/anubis" @@ -21,16 +22,29 @@ func UnchangingCache(next http.Handler) http.Handler { }) } -// DefaultXRealIP sets the X-Real-Ip header to the given value if and only if -// it is not an empty string. -func DefaultXRealIP(defaultIP string, next http.Handler) http.Handler { - if defaultIP == "" { - slog.Debug("skipping middleware, defaultIP is empty") +// RemoteXRealIP sets the X-Real-Ip header to the request's real IP if +// the setting is enabled by the user. +func RemoteXRealIP(useRemoteAddress bool, bindNetwork string, next http.Handler) http.Handler { + if useRemoteAddress == false { + slog.Debug("skipping middleware, useRemoteAddress is empty") return next } + if bindNetwork == "unix" { + // For local sockets there is no real remote address but the localhost + // address should be sensible. + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Header.Set("X-Real-Ip", "127.0.0.1") + next.ServeHTTP(w, r) + }) + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r.Header.Set("X-Real-Ip", defaultIP) + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + panic(err) // this should never happen + } + r.Header.Set("X-Real-Ip", host) next.ServeHTTP(w, r) }) } diff --git a/lib/anubis_test.go b/lib/anubis_test.go index 90d2cdf..58c8834 100644 --- a/lib/anubis_test.go +++ b/lib/anubis_test.go @@ -47,7 +47,7 @@ func TestCookieSettings(t *testing.T) { CookieName: t.Name(), }) - ts := httptest.NewServer(internal.DefaultXRealIP("127.0.0.1", srv)) + ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv)) defer ts.Close() cli := &http.Client{ -- cgit v1.2.3 From 38d62eeb5676d010a08c439fdcedb4741c021bff Mon Sep 17 00:00:00 2001 From: Henri Vasserman Date: Fri, 28 Mar 2025 19:52:14 +0200 Subject: Hide directory browsing on the static content (#85) * Hide directory browsing on the static content * update changelog --- docs/docs/CHANGELOG.md | 1 + internal/headers.go | 12 ++++++++++++ lib/anubis.go | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index 9898eec..29f9416 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Hide the directory listings for Anubis' internal static content - Changed `--debug-x-real-ip-default` to `--use-remote-address`, getting the IP address from the request's socket address instead. - DroneBL lookups have been disabled by default diff --git a/internal/headers.go b/internal/headers.go index 9d6ba76..d73fa33 100644 --- a/internal/headers.go +++ b/internal/headers.go @@ -4,6 +4,7 @@ import ( "log/slog" "net" "net/http" + "strings" "github.com/TecharoHQ/anubis" "github.com/sebest/xff" @@ -62,3 +63,14 @@ func XForwardedForToXRealIP(next http.Handler) http.Handler { next.ServeHTTP(w, r) }) } + +// Do not allow browsing directory listings in paths that end with / +func NoBrowsing(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + http.NotFound(w, r) + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/lib/anubis.go b/lib/anubis.go index 83e04dd..8d5dac1 100644 --- a/lib/anubis.go +++ b/lib/anubis.go @@ -119,7 +119,7 @@ func New(opts Options) (*Server, error) { mux := http.NewServeMux() xess.Mount(mux) - mux.Handle(anubis.StaticPath, internal.UnchangingCache(http.StripPrefix(anubis.StaticPath, http.FileServerFS(web.Static)))) + mux.Handle(anubis.StaticPath, internal.UnchangingCache(internal.NoBrowsing(http.StripPrefix(anubis.StaticPath, http.FileServerFS(web.Static))))) if opts.ServeRobotsTXT { mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { -- cgit v1.2.3 From bb4f49cfd94783111e2fbed99b4ea7a2077fa0bf Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Fri, 28 Mar 2025 14:33:57 -0400 Subject: yeetfile: build debian packages Signed-off-by: Xe Iaso --- .gitignore | 1 + yeetfile.js | 36 +++++++++++++++++++----------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index b54eb8e..7debfec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +*.deb *.rpm # Go binaries and test artifacts diff --git a/yeetfile.js b/yeetfile.js index 730e7d1..ae22fbf 100644 --- a/yeetfile.js +++ b/yeetfile.js @@ -1,22 +1,24 @@ go.install(); -["amd64", "arm64"].forEach(goarch => rpm.build({ - name: "anubis", - description: "Anubis weighs the souls of incoming HTTP requests and uses a sha256 proof-of-work challenge in order to protect upstream resources from scraper bots.", - homepage: "https://xeiaso.net/blog/2025/anubis", - license: "MIT", - goarch, +["amd64", "arm64"].forEach(goarch => { + [deb, rpm].forEach(method => method.build({ + name: "anubis", + description: "Anubis weighs the souls of incoming HTTP requests and uses a sha256 proof-of-work challenge in order to protect upstream resources from scraper bots.", + homepage: "https://xeiaso.net/blog/2025/anubis", + license: "MIT", + goarch, - build: (out) => { - // install Anubis binary - go.build("-o", `${out}/usr/bin/anubis`); + build: (out) => { + // install Anubis binary + go.build("-o", `${out}/usr/bin/anubis`, "./cmd/anubis"); - // install systemd unit - yeet.run("mkdir", "-p", `${out}/usr/lib/systemd/system`); - yeet.run("cp", "run/anubis@.service", `${out}/usr/lib/systemd/system/anubis@.service`); + // install systemd unit + yeet.run("mkdir", "-p", `${out}/usr/lib/systemd/system`); + yeet.run("cp", "run/anubis@.service", `${out}/usr/lib/systemd/system/anubis@.service`); - // install default config - yeet.run("mkdir", "-p", `${out}/etc/anubis`); - yeet.run("cp", "run/anubis.env.default", `${out}/etc/anubis/anubis-default.env`); - }, -})); \ No newline at end of file + // install default config + yeet.run("mkdir", "-p", `${out}/etc/anubis`); + yeet.run("cp", "run/anubis.env.default", `${out}/etc/anubis/anubis-default.env`); + }, + })); +}); \ No newline at end of file -- cgit v1.2.3