From d40b5cfdab11c62dc2ed226bde32b19ea7107f21 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Sun, 20 Apr 2025 20:09:27 -0400 Subject: lib: move config to yaml (#307) * lib: move config to yaml Signed-off-by: Xe Iaso * web: run go generate Signed-off-by: Xe Iaso * Add Haiku to known instances (#304) Signed-off-by: Asmodeus <46908100+AsmodeumX@users.noreply.github.com> * Add headers bot rule (#300) * Closes #291: add headers support to bot policy rules * Fix config validator * update docs for JSON -> YAML Signed-off-by: Xe Iaso * docs: document http header based actions Signed-off-by: Xe Iaso * lib: add missing test Signed-off-by: Xe Iaso * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Xe Iaso --------- Signed-off-by: Xe Iaso Signed-off-by: Asmodeus <46908100+AsmodeumX@users.noreply.github.com> Co-authored-by: Asmodeus <46908100+AsmodeumX@users.noreply.github.com> Co-authored-by: Neur0toxine Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- lib/anubis.go | 4 ++-- lib/anubis_test.go | 19 ++++++++++++++++++- lib/policy/config/config_test.go | 7 ++++--- lib/policy/config/testdata/bad/badregexes.yaml | 7 +++++++ lib/policy/config/testdata/bad/invalid.yaml | 1 + lib/policy/config/testdata/bad/nobots.yaml | 1 + lib/policy/config/testdata/good/allow_everyone.yaml | 6 ++++++ lib/policy/config/testdata/good/block_cf_workers.yaml | 5 +++++ lib/policy/config/testdata/good/challengemozilla.yaml | 4 ++++ .../config/testdata/good/everything_blocked.yaml | 4 ++++ lib/policy/policy.go | 6 +++--- 11 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 lib/policy/config/testdata/bad/badregexes.yaml create mode 100644 lib/policy/config/testdata/bad/invalid.yaml create mode 100644 lib/policy/config/testdata/bad/nobots.yaml create mode 100644 lib/policy/config/testdata/good/allow_everyone.yaml create mode 100644 lib/policy/config/testdata/good/block_cf_workers.yaml create mode 100644 lib/policy/config/testdata/good/challengemozilla.yaml create mode 100644 lib/policy/config/testdata/good/everything_blocked.yaml (limited to 'lib') diff --git a/lib/anubis.go b/lib/anubis.go index ba143f9..afc3d86 100644 --- a/lib/anubis.go +++ b/lib/anubis.go @@ -90,8 +90,8 @@ func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedC return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err) } } else { - fname = "(data)/botPolicies.json" - fin, err = data.BotPolicies.Open("botPolicies.json") + fname = "(data)/botPolicies.yaml" + fin, err = data.BotPolicies.Open("botPolicies.yaml") if err != nil { return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", fname, err) } diff --git a/lib/anubis_test.go b/lib/anubis_test.go index 156863d..60b7913 100644 --- a/lib/anubis_test.go +++ b/lib/anubis_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/TecharoHQ/anubis" + "github.com/TecharoHQ/anubis/data" "github.com/TecharoHQ/anubis/internal" "github.com/TecharoHQ/anubis/lib/policy" ) @@ -15,7 +16,7 @@ import ( func loadPolicies(t *testing.T, fname string) *policy.ParsedConfig { t.Helper() - anubisPolicy, err := LoadPoliciesOrDefault("", anubis.DefaultDifficulty) + anubisPolicy, err := LoadPoliciesOrDefault(fname, anubis.DefaultDifficulty) if err != nil { t.Fatal(err) } @@ -55,6 +56,22 @@ func makeChallenge(t *testing.T, ts *httptest.Server) challenge { return chall } +func TestLoadPolicies(t *testing.T) { + for _, fname := range []string{"botPolicies.json", "botPolicies.yaml"} { + t.Run(fname, func(t *testing.T) { + fin, err := data.BotPolicies.Open(fname) + if err != nil { + t.Fatal(err) + } + defer fin.Close() + + if _, err := policy.ParseConfig(fin, fname, 4); err != nil { + t.Fatal(err) + } + }) + } +} + // Regression test for CVE-2025-24369 func TestCVE2025_24369(t *testing.T) { pol := loadPolicies(t, "") diff --git a/lib/policy/config/config_test.go b/lib/policy/config/config_test.go index 0fabbb7..4176126 100644 --- a/lib/policy/config/config_test.go +++ b/lib/policy/config/config_test.go @@ -1,11 +1,12 @@ package config import ( - "encoding/json" "errors" "os" "path/filepath" "testing" + + "k8s.io/apimachinery/pkg/util/yaml" ) func p[V any](v V) *V { return &v } @@ -219,7 +220,7 @@ func TestConfigValidKnownGood(t *testing.T) { defer fin.Close() var c Config - if err := json.NewDecoder(fin).Decode(&c); err != nil { + if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil { t.Fatalf("can't decode file: %v", err) } @@ -246,7 +247,7 @@ func TestConfigValidBad(t *testing.T) { defer fin.Close() var c Config - if err := json.NewDecoder(fin).Decode(&c); err != nil { + if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil { t.Fatalf("can't decode file: %v", err) } diff --git a/lib/policy/config/testdata/bad/badregexes.yaml b/lib/policy/config/testdata/bad/badregexes.yaml new file mode 100644 index 0000000..3880e40 --- /dev/null +++ b/lib/policy/config/testdata/bad/badregexes.yaml @@ -0,0 +1,7 @@ +bots: +- name: path-bad + path_regex: "a(b" + action: DENY +- name: user-agent-bad + user_agent_regex: "a(b" + action: DENY \ No newline at end of file diff --git a/lib/policy/config/testdata/bad/invalid.yaml b/lib/policy/config/testdata/bad/invalid.yaml new file mode 100644 index 0000000..18625b6 --- /dev/null +++ b/lib/policy/config/testdata/bad/invalid.yaml @@ -0,0 +1 @@ +bots: [] \ No newline at end of file diff --git a/lib/policy/config/testdata/bad/nobots.yaml b/lib/policy/config/testdata/bad/nobots.yaml new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/lib/policy/config/testdata/bad/nobots.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/lib/policy/config/testdata/good/allow_everyone.yaml b/lib/policy/config/testdata/good/allow_everyone.yaml new file mode 100644 index 0000000..5c49534 --- /dev/null +++ b/lib/policy/config/testdata/good/allow_everyone.yaml @@ -0,0 +1,6 @@ +bots: +- name: everyones-invited + remote_addresses: + - "0.0.0.0/0" + - "::/0" + action: ALLOW \ No newline at end of file diff --git a/lib/policy/config/testdata/good/block_cf_workers.yaml b/lib/policy/config/testdata/good/block_cf_workers.yaml new file mode 100644 index 0000000..c66bade --- /dev/null +++ b/lib/policy/config/testdata/good/block_cf_workers.yaml @@ -0,0 +1,5 @@ +bots: + - name: cloudflare-workers + headers_regex: + CF-Worker: .* + action: DENY \ No newline at end of file diff --git a/lib/policy/config/testdata/good/challengemozilla.yaml b/lib/policy/config/testdata/good/challengemozilla.yaml new file mode 100644 index 0000000..15922b0 --- /dev/null +++ b/lib/policy/config/testdata/good/challengemozilla.yaml @@ -0,0 +1,4 @@ +bots: +- name: generic-browser + user_agent_regex: Mozilla + action: CHALLENGE \ No newline at end of file diff --git a/lib/policy/config/testdata/good/everything_blocked.yaml b/lib/policy/config/testdata/good/everything_blocked.yaml new file mode 100644 index 0000000..323c596 --- /dev/null +++ b/lib/policy/config/testdata/good/everything_blocked.yaml @@ -0,0 +1,4 @@ +bots: +- name: everything + user_agent_regex: .* + action: DENY diff --git a/lib/policy/policy.go b/lib/policy/policy.go index 4451b08..2d610c8 100644 --- a/lib/policy/policy.go +++ b/lib/policy/policy.go @@ -1,7 +1,6 @@ package policy import ( - "encoding/json" "errors" "fmt" "io" @@ -11,6 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/yl2chen/cidranger" + "k8s.io/apimachinery/pkg/util/yaml" "github.com/TecharoHQ/anubis/lib/policy/config" ) @@ -38,8 +38,8 @@ func NewParsedConfig(orig config.Config) *ParsedConfig { func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { var c config.Config - if err := json.NewDecoder(fin).Decode(&c); err != nil { - return nil, fmt.Errorf("can't parse policy config JSON %s: %w", fname, err) + if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil { + return nil, fmt.Errorf("can't parse policy config YAML %s: %w", fname, err) } if err := c.Valid(); err != nil { -- cgit v1.2.3