diff options
Diffstat (limited to 'robots.txt')
| -rw-r--r-- | robots.txt | 24 |
1 files changed, 13 insertions, 11 deletions
@@ -4,29 +4,31 @@ Disallow: /chronology/ # See https://seirdy.one/robots.txt +## Ad Scrapers User-agent: Adsbot -Disallow: / - User-agent: BLEXBot +User-agent: peer39_crawler +User-agent: peer39_crawler/1.0 Disallow: / +## IP (not that IP) Scrapers User-agent: BrandVerity/1.0 -Disallow: / - User-agent: CheckMarkNetwork/1.0 -Disallow: / - +User-agent: CheckMarkNetwork/1.0 (+https://www.checkmarknetwork.com/spider.html) User-agent: NPBot -Disallow: / - +User-agent: PiplBot User-agent: SlySearch -Disallow: / - User-agent: TurnitinBot Disallow: / +## LLM Scrapers User-agent: ChatGPT-User +User-agent: GPTBot +User-agent: ClaudeBot Disallow: / -User-agent: GPTBot +## (More) LLM Scrapers +User-agent: Applebot-Extended +User-agent: Google-Extended +User-agent: FacebookBot Disallow: / |