blob: 7f31c270b1e4c4463a1cb541db3a39091ff03024 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
|
User-agent: *
Disallow: /spartronics-slack/
Disallow: /chronology/
# See https://seirdy.one/robots.txt
## Ad Scrapers
User-agent: Adsbot
User-agent: BLEXBot
User-agent: peer39_crawler
User-agent: peer39_crawler/1.0
Disallow: /
## IP (not that IP) Scrapers
User-agent: BrandVerity/1.0
User-agent: CheckMarkNetwork/1.0
User-agent: CheckMarkNetwork/1.0 (+https://www.checkmarknetwork.com/spider.html)
User-agent: NPBot
User-agent: PiplBot
User-agent: SlySearch
User-agent: TurnitinBot
Disallow: /
## LLM Scrapers
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
Disallow: /
## (More) LLM Scrapers
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: FacebookBot
User-agent: meta-externalagent
Disallow: /
|
