summaryrefslogtreecommitdiff
path: root/robots.txt
diff options
context:
space:
mode:
Diffstat (limited to 'robots.txt')
-rw-r--r--robots.txt12
1 files changed, 9 insertions, 3 deletions
diff --git a/robots.txt b/robots.txt
index 7f31c27..933229e 100644
--- a/robots.txt
+++ b/robots.txt
@@ -19,17 +19,23 @@ User-agent: NPBot
User-agent: PiplBot
User-agent: SlySearch
User-agent: TurnitinBot
+User-Agent: AcademicBotRTU
+User-agent: PiplBot
Disallow: /
## LLM Scrapers
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
-Disallow: /
-
-## (More) LLM Scrapers
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: FacebookBot
User-agent: meta-externalagent
+User-Agent: GenAI
+User-Agent: SemrushBot-OCOB
+User-Agent: SemrushBot-FT
+User-agent: Cotoyogi
+User-agent: Webzio-extended
+User-agent: Kangaroo Bot
+User-Agent: VelenPublicWebCrawler
Disallow: /