diff options
Diffstat (limited to 'data')
| -rw-r--r-- | data/botPolicies.json | 2 | ||||
| -rw-r--r-- | data/botPolicies.yaml | 2 | ||||
| -rw-r--r-- | data/bots/ai-robots-txt.yaml | 2 |
3 files changed, 3 insertions, 3 deletions
diff --git a/data/botPolicies.json b/data/botPolicies.json index dad04e8..160bbf0 100644 --- a/data/botPolicies.json +++ b/data/botPolicies.json @@ -41,7 +41,7 @@ }, { "name": "generic-browser", - "user_agent_regex": "Mozilla|Opera\n", + "user_agent_regex": "Mozilla|Opera", "action": "CHALLENGE" } ], diff --git a/data/botPolicies.yaml b/data/botPolicies.yaml index 585be15..51af499 100644 --- a/data/botPolicies.yaml +++ b/data/botPolicies.yaml @@ -43,7 +43,7 @@ bots: # Generic catchall rule - name: generic-browser - user_agent_regex: > + user_agent_regex: >- Mozilla|Opera action: CHALLENGE diff --git a/data/bots/ai-robots-txt.yaml b/data/bots/ai-robots-txt.yaml index 19cbe93..ef2790c 100644 --- a/data/bots/ai-robots-txt.yaml +++ b/data/bots/ai-robots-txt.yaml @@ -1,4 +1,4 @@ - name: "ai-robots-txt" - user_agent_regex: > + user_agent_regex: >- AI2Bot|Ai2Bot-Dolma|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Brightbot 1.0|Bytespider|CCBot|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|cohere-training-data-crawler|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|FriendlyCrawler|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo Bot|Meta-ExternalAgent|Meta-ExternalFetcher|OAI-SearchBot|omgili|omgilibot|PanguBot|Perplexity-User|PerplexityBot|PetalBot|Scrapy|SemrushBot-OCOB|SemrushBot-SWA|Sidetrade indexer bot|Timpibot|VelenPublicWebCrawler|Webzio-Extended|YouBot action: DENY
\ No newline at end of file |