From 1afe8601ed5c6631a58c57ab8f304f2f09cef8da Mon Sep 17 00:00:00 2001
From: Jack Levy <YOUR_GITEA_NOREPLY_EMAIL_HERE>
Date: Sun, 15 Mar 2026 21:49:05 -0400
Subject: [PATCH] =?UTF-8?q?chore:=20expand=20robots.txt=20=E2=80=94=20bloc?=
 =?UTF-8?q?k=20AI=20scrapers=20and=20SEO=20bots?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add blocks for AI training crawlers (GPTBot, CCBot, Bytespider, anthropic-ai,
Google-Extended, PerplexityBot, YouBot, cohere-ai), SEO tool bots (AhrefsBot,
SemrushBot, DotBot, MJ12bot, BLEXBot), and /_next/ static chunks. Add
Crawl-delay: 10 for well-behaved bots.

Authored by: Jack Levy
---
 frontend/public/robots.txt | 44 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/frontend/public/robots.txt b/frontend/public/robots.txt
index 8859964..c7cbd33 100644
--- a/frontend/public/robots.txt
+++ b/frontend/public/robots.txt
@@ -16,3 +16,47 @@ Disallow: /following
 Disallow: /collections
 Disallow: /alignment
 Disallow: /api/
+Disallow: /_next/
+
+Crawl-delay: 10
+
+# AI training crawlers
+User-agent: GPTBot
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: YouBot
+Disallow: /
+
+User-agent: cohere-ai
+Disallow: /
+
+# SEO tool crawlers
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: DotBot
+Disallow: /
+
+User-agent: MJ12bot
+Disallow: /
+
+User-agent: BLEXBot
+Disallow: /