1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
|
{
"bots": [
{
"import": "(data)/bots/ai-robots-txt.yaml"
},
{
"import": "(data)/bots/cloudflare-workers.yaml"
},
{
"import": "(data)/bots/headless-browsers.yaml"
},
{
"import": "(data)/bots/us-ai-scraper.yaml"
},
{
"import": "(data)/crawlers/googlebot.yaml"
},
{
"import": "(data)/crawlers/bingbot.yaml"
},
{
"import": "(data)/crawlers/duckduckbot.yaml"
},
{
"import": "(data)/crawlers/qwantbot.yaml"
},
{
"import": "(data)/crawlers/internet-archive.yaml"
},
{
"import": "(data)/crawlers/kagibot.yaml"
},
{
"import": "(data)/crawlers/marginalia.yaml"
},
{
"import": "(data)/crawlers/mojeekbot.yaml"
},
{
"import": "(data)/common/keep-internet-working.yaml"
},
{
"name": "generic-browser",
"user_agent_regex": "Mozilla|Opera",
"action": "CHALLENGE"
}
],
"dnsbl": false
}
|
