aboutsummaryrefslogtreecommitdiff
path: root/data/crawlers/googlebot.yaml
blob: f1735126902fb94f53b6bf8cca4542fcf1a843d8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
- name: googlebot
  user_agent_regex: \+http\://www\.google\.com/bot\.html
  action: ALLOW
  # https://developers.google.com/static/search/apis/ipranges/googlebot.json
  remote_addresses: [
    "2001:4860:4801:10::/64",
    "2001:4860:4801:11::/64",
    "2001:4860:4801:12::/64",
    "2001:4860:4801:13::/64",
    "2001:4860:4801:14::/64",
    "2001:4860:4801:15::/64",
    "2001:4860:4801:16::/64",
    "2001:4860:4801:17::/64",
    "2001:4860:4801:18::/64",
    "2001:4860:4801:19::/64",
    "2001:4860:4801:1a::/64",
    "2001:4860:4801:1b::/64",
    "2001:4860:4801:1c::/64",
    "2001:4860:4801:1d::/64",
    "2001:4860:4801:1e::/64",
    "2001:4860:4801:1f::/64",
    "2001:4860:4801:20::/64",
    "2001:4860:4801:21::/64",
    "2001:4860:4801:22::/64",
    "2001:4860:4801:23::/64",
    "2001:4860:4801:24::/64",
    "2001:4860:4801:25::/64",
    "2001:4860:4801:26::/64",
    "2001:4860:4801:27::/64",
    "2001:4860:4801:28::/64",
    "2001:4860:4801:29::/64",
    "2001:4860:4801:2::/64",
    "2001:4860:4801:2a::/64",
    "2001:4860:4801:2b::/64",
    "2001:4860:4801:2c::/64",
    "2001:4860:4801:2d::/64",
    "2001:4860:4801:2e::/64",
    "2001:4860:4801:2f::/64",
    "2001:4860:4801:31::/64",
    "2001:4860:4801:32::/64",
    "2001:4860:4801:33::/64",
    "2001:4860:4801:34::/64",
    "2001:4860:4801:35::/64",
    "2001:4860:4801:36::/64",
    "2001:4860:4801:37::/64",
    "2001:4860:4801:38::/64",
    "2001:4860:4801:39::/64",
    "2001:4860:4801:3a::/64",
    "2001:4860:4801:3b::/64",
    "2001:4860:4801:3c::/64",
    "2001:4860:4801:3d::/64",
    "2001:4860:4801:3e::/64",
    "2001:4860:4801:40::/64",
    "2001:4860:4801:41::/64",
    "2001:4860:4801:42::/64",
    "2001:4860:4801:43::/64",
    "2001:4860:4801:44::/64",
    "2001:4860:4801:45::/64",
    "2001:4860:4801:46::/64",
    "2001:4860:4801:47::/64",
    "2001:4860:4801:48::/64",
    "2001:4860:4801:49::/64",
    "2001:4860:4801:4a::/64",
    "2001:4860:4801:4b::/64",
    "2001:4860:4801:4c::/64",
    "2001:4860:4801:50::/64",
    "2001:4860:4801:51::/64",
    "2001:4860:4801:52::/64",
    "2001:4860:4801:53::/64",
    "2001:4860:4801:54::/64",
    "2001:4860:4801:55::/64",
    "2001:4860:4801:56::/64",
    "2001:4860:4801:60::/64",
    "2001:4860:4801:61::/64",
    "2001:4860:4801:62::/64",
    "2001:4860:4801:63::/64",
    "2001:4860:4801:64::/64",
    "2001:4860:4801:65::/64",
    "2001:4860:4801:66::/64",
    "2001:4860:4801:67::/64",
    "2001:4860:4801:68::/64",
    "2001:4860:4801:69::/64",
    "2001:4860:4801:6a::/64",
    "2001:4860:4801:6b::/64",
    "2001:4860:4801:6c::/64",
    "2001:4860:4801:6d::/64",
    "2001:4860:4801:6e::/64",
    "2001:4860:4801:6f::/64",
    "2001:4860:4801:70::/64",
    "2001:4860:4801:71::/64",
    "2001:4860:4801:72::/64",
    "2001:4860:4801:73::/64",
    "2001:4860:4801:74::/64",
    "2001:4860:4801:75::/64",
    "2001:4860:4801:76::/64",
    "2001:4860:4801:77::/64",
    "2001:4860:4801:78::/64",
    "2001:4860:4801:79::/64",
    "2001:4860:4801:80::/64",
    "2001:4860:4801:81::/64",
    "2001:4860:4801:82::/64",
    "2001:4860:4801:83::/64",
    "2001:4860:4801:84::/64",
    "2001:4860:4801:85::/64",
    "2001:4860:4801:86::/64",
    "2001:4860:4801:87::/64",
    "2001:4860:4801:88::/64",
    "2001:4860:4801:90::/64",
    "2001:4860:4801:91::/64",
    "2001:4860:4801:92::/64",
    "2001:4860:4801:93::/64",
    "2001:4860:4801:94::/64",
    "2001:4860:4801:95::/64",
    "2001:4860:4801:96::/64",
    "2001:4860:4801:a0::/64",
    "2001:4860:4801:a1::/64",
    "2001:4860:4801:a2::/64",
    "2001:4860:4801:a3::/64",
    "2001:4860:4801:a4::/64",
    "2001:4860:4801:a5::/64",
    "2001:4860:4801:c::/64",
    "2001:4860:4801:f::/64",
    "192.178.5.0/27",
    "192.178.6.0/27",
    "192.178.6.128/27",
    "192.178.6.160/27",
    "192.178.6.192/27",
    "192.178.6.32/27",
    "192.178.6.64/27",
    "192.178.6.96/27",
    "34.100.182.96/28",
    "34.101.50.144/28",
    "34.118.254.0/28",
    "34.118.66.0/28",
    "34.126.178.96/28",
    "34.146.150.144/28",
    "34.147.110.144/28",
    "34.151.74.144/28",
    "34.152.50.64/28",
    "34.154.114.144/28",
    "34.155.98.32/28",
    "34.165.18.176/28",
    "34.175.160.64/28",
    "34.176.130.16/28",
    "34.22.85.0/27",
    "34.64.82.64/28",
    "34.65.242.112/28",
    "34.80.50.80/28",
    "34.88.194.0/28",
    "34.89.10.80/28",
    "34.89.198.80/28",
    "34.96.162.48/28",
    "35.247.243.240/28",
    "66.249.64.0/27",
    "66.249.64.128/27",
    "66.249.64.160/27",
    "66.249.64.224/27",
    "66.249.64.32/27",
    "66.249.64.64/27",
    "66.249.64.96/27",
    "66.249.65.0/27",
    "66.249.65.128/27",
    "66.249.65.160/27",
    "66.249.65.192/27",
    "66.249.65.224/27",
    "66.249.65.32/27",
    "66.249.65.64/27",
    "66.249.65.96/27",
    "66.249.66.0/27",
    "66.249.66.128/27",
    "66.249.66.160/27",
    "66.249.66.192/27",
    "66.249.66.224/27",
    "66.249.66.32/27",
    "66.249.66.64/27",
    "66.249.66.96/27",
    "66.249.68.0/27",
    "66.249.68.128/27",
    "66.249.68.32/27",
    "66.249.68.64/27",
    "66.249.68.96/27",
    "66.249.69.0/27",
    "66.249.69.128/27",
    "66.249.69.160/27",
    "66.249.69.192/27",
    "66.249.69.224/27",
    "66.249.69.32/27",
    "66.249.69.64/27",
    "66.249.69.96/27",
    "66.249.70.0/27",
    "66.249.70.128/27",
    "66.249.70.160/27",
    "66.249.70.192/27",
    "66.249.70.224/27",
    "66.249.70.32/27",
    "66.249.70.64/27",
    "66.249.70.96/27",
    "66.249.71.0/27",
    "66.249.71.128/27",
    "66.249.71.160/27",
    "66.249.71.192/27",
    "66.249.71.224/27",
    "66.249.71.32/27",
    "66.249.71.64/27",
    "66.249.71.96/27",
    "66.249.72.0/27",
    "66.249.72.128/27",
    "66.249.72.160/27",
    "66.249.72.192/27",
    "66.249.72.224/27",
    "66.249.72.32/27",
    "66.249.72.64/27",
    "66.249.72.96/27",
    "66.249.73.0/27",
    "66.249.73.128/27",
    "66.249.73.160/27",
    "66.249.73.192/27",
    "66.249.73.224/27",
    "66.249.73.32/27",
    "66.249.73.64/27",
    "66.249.73.96/27",
    "66.249.74.0/27",
    "66.249.74.128/27",
    "66.249.74.160/27",
    "66.249.74.192/27",
    "66.249.74.32/27",
    "66.249.74.64/27",
    "66.249.74.96/27",
    "66.249.75.0/27",
    "66.249.75.128/27",
    "66.249.75.160/27",
    "66.249.75.192/27",
    "66.249.75.224/27",
    "66.249.75.32/27",
    "66.249.75.64/27",
    "66.249.75.96/27",
    "66.249.76.0/27",
    "66.249.76.128/27",
    "66.249.76.160/27",
    "66.249.76.192/27",
    "66.249.76.224/27",
    "66.249.76.32/27",
    "66.249.76.64/27",
    "66.249.76.96/27",
    "66.249.77.0/27",
    "66.249.77.128/27",
    "66.249.77.160/27",
    "66.249.77.192/27",
    "66.249.77.224/27",
    "66.249.77.32/27",
    "66.249.77.64/27",
    "66.249.77.96/27",
    "66.249.78.0/27",
    "66.249.78.32/27",
    "66.249.79.0/27",
    "66.249.79.128/27",
    "66.249.79.160/27",
    "66.249.79.192/27",
    "66.249.79.224/27",
    "66.249.79.32/27",
    "66.249.79.64/27",
    "66.249.79.96/27"
  ]