diff options
| -rw-r--r-- | lume/src/notes/2025/anubis-testing-prod.mdx | 16 | ||||
| -rw-r--r-- | manifest/xesite/anubis/botPolicies.json | 79 | ||||
| -rw-r--r-- | manifest/xesite/deployment.yaml | 41 | ||||
| -rw-r--r-- | manifest/xesite/ingress.yaml | 2 | ||||
| -rw-r--r-- | manifest/xesite/kustomization.yaml | 8 | ||||
| -rw-r--r-- | manifest/xesite/service.yaml | 4 |
6 files changed, 148 insertions, 2 deletions
diff --git a/lume/src/notes/2025/anubis-testing-prod.mdx b/lume/src/notes/2025/anubis-testing-prod.mdx new file mode 100644 index 0000000..f9ac255 --- /dev/null +++ b/lume/src/notes/2025/anubis-testing-prod.mdx @@ -0,0 +1,16 @@ +--- +title: "I'm testing Anubis in prod" +desc: "Please let me know what URL resolvers I just broke." +date: 2025-03-20 +--- + +Hey all! + +Anubis has really been taking off to the point that it has [its own repo now](https://github.com/TecharoHQ/anubis). I'm going to be doing more work on it, but for right now what I really need is data. In order to get this data, I need you to let me know what I just broke by turning on Anubis in prod. + +What I know broke: + +- Discord link resolving (still working on fixing this, but I wanted to get this post out first) +- Twitter link resolving + +If I missed something, [contact me](/contact/). diff --git a/manifest/xesite/anubis/botPolicies.json b/manifest/xesite/anubis/botPolicies.json new file mode 100644 index 0000000..aa2cac1 --- /dev/null +++ b/manifest/xesite/anubis/botPolicies.json @@ -0,0 +1,79 @@ +{ + "bots": [ + { + "name": "amazonbot", + "user_agent_regex": "Amazonbot", + "action": "DENY" + }, + { + "name": "googlebot", + "user_agent_regex": "\\+http\\:\\/\\/www\\.google\\.com/bot\\.html", + "action": "ALLOW" + }, + { + "name": "bingbot", + "user_agent_regex": "\\+http\\:\\/\\/www\\.bing\\.com/bingbot\\.htm", + "action": "ALLOW" + }, + { + "name": "qwantbot", + "user_agent_regex": "\\+https\\:\\/\\/help\\.qwant\\.com/bot/", + "action": "ALLOW" + }, + { + "name": "discordbot", + "user_agent_regex": "Discordbot\/2\\.\\0\\; \\+https\\:\\/\\/\\discordapp\\.com", + "action": "ALLOW" + }, + { + "name": "blueskybot", + "user_agent_regex": "Bluesky Cardyb", + "action": "ALLOW" + }, + { + "name": "us-artificial-intelligence-scraper", + "user_agent_regex": "\\+https\\:\\/\\/github\\.com\\/US-Artificial-Intelligence\\/scraper", + "action": "DENY" + }, + { + "name": "well-known", + "path_regex": "^/.well-known/.*$", + "action": "ALLOW" + }, + { + "name": "favicon", + "path_regex": "^/favicon.ico$", + "action": "ALLOW" + }, + { + "name": "robots-txt", + "path_regex": "^/robots.txt$", + "action": "ALLOW" + }, + { + "name": "rss-readers", + "path_regex": ".*\\.(rss|xml|atom|json)$", + "action": "ALLOW" + }, + { + "name": "lightpanda", + "user_agent_regex": "^Lightpanda/.*$", + "action": "DENY" + }, + { + "name": "headless-chrome", + "user_agent_regex": "HeadlessChrome", + "action": "DENY" + }, + { + "name": "headless-chromium", + "user_agent_regex": "HeadlessChromium", + "action": "DENY" + }, + { + "name": "generic-browser", + "user_agent_regex": "Mozilla", + "action": "CHALLENGE" + } + ] +}
\ No newline at end of file diff --git a/manifest/xesite/deployment.yaml b/manifest/xesite/deployment.yaml index 40d8d3f..04acd6f 100644 --- a/manifest/xesite/deployment.yaml +++ b/manifest/xesite/deployment.yaml @@ -17,6 +17,9 @@ spec: claimName: xesite-cache - name: data emptyDir: {} + - name: anubis + configMap: + name: anubis-cfg containers: - name: web image: ghcr.io/xe/site/bin:latest @@ -66,3 +69,41 @@ spec: value: "Is alright, I guess" initialDelaySeconds: 30 periodSeconds: 30 + - name: anubis + image: ghcr.io/techarohq/anubis:pr-19 + imagePullPolicy: Always + env: + - name: "BIND" + value: ":8081" + - name: "DIFFICULTY" + value: "4" + - name: "METRICS_BIND" + value: ":9090" + - name: "POLICY_FNAME" + value: "/xe/cfg/anubis/botPolicies.json" + - name: "SERVE_ROBOTS_TXT" + value: "false" + - name: "TARGET" + value: "http://localhost:3000" + # - name: "SLOG_LEVEL" + # value: "debug" + volumeMounts: + - name: anubis + mountPath: /xe/cfg/anubis + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 250m + memory: 128Mi + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault
\ No newline at end of file diff --git a/manifest/xesite/ingress.yaml b/manifest/xesite/ingress.yaml index 7075631..f95edbb 100644 --- a/manifest/xesite/ingress.yaml +++ b/manifest/xesite/ingress.yaml @@ -32,4 +32,4 @@ spec: service: name: xesite port: - number: 80 + name: anubis diff --git a/manifest/xesite/kustomization.yaml b/manifest/xesite/kustomization.yaml index 58a3f8a..fc82930 100644 --- a/manifest/xesite/kustomization.yaml +++ b/manifest/xesite/kustomization.yaml @@ -6,4 +6,10 @@ resources: - machineproxy.yaml - onionservice.yaml - pod-disruption-budget.yaml - - service.yaml
\ No newline at end of file + - service.yaml + +configMapGenerator: + - name: anubis-cfg + behavior: create + files: + - ./anubis/botPolicies.json
\ No newline at end of file diff --git a/manifest/xesite/service.yaml b/manifest/xesite/service.yaml index 86b6217..87b36c4 100644 --- a/manifest/xesite/service.yaml +++ b/manifest/xesite/service.yaml @@ -16,3 +16,7 @@ spec: targetPort: 3001 protocol: TCP name: internalapi + - name: anubis + port: 8081 + targetPort: 8081 + protocol: TCP
\ No newline at end of file |