From 6b07b83583c727624b1bdcd72e1ea512fd4e1317 Mon Sep 17 00:00:00 2001
From: platform-mcp <platform-mcp@service.local>
Date: Fri, 19 Jun 2026 09:35:47 +0530
Subject: [PATCH] chore(seo): open robots.txt to all major AI crawlers for GEO

Repo robots.txt already matched the deployed file (verified via diff
against https://workroot.in/robots.txt). Added commonly-cited AI agents
that were still missing, each with Allow: /:
- OpenAI: OAI-SearchBot
- Anthropic: ClaudeBot, Claude-SearchBot, Claude-User
- Perplexity: Perplexity-User
- Meta: Meta-ExternalAgent / meta-externalagent
- Apple: Applebot, Applebot-Extended
- Amazon: Amazonbot
- ByteDance: Bytespider
- Cohere: cohere-ai
- DuckDuckGo: DuckAssistBot
- You.com: YouBot
- Diffbot

Disallow list unchanged (/admin/, /private/, /_astro/) so no content
directory needed for citation is blocked. Sitemap line preserved.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 public/robots.txt | 52 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/public/robots.txt b/public/robots.txt
index ef2f5f7..5d38571 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -58,6 +58,9 @@ Allow: /
 User-agent: GPTBot
 Allow: /
 
+User-agent: OAI-SearchBot
+Allow: /
+
 # Anthropic/Claude
 User-agent: anthropic-ai
 Allow: /
@@ -65,10 +68,22 @@ Allow: /
 User-agent: Claude-Web
 Allow: /
 
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Claude-SearchBot
+Allow: /
+
+User-agent: Claude-User
+Allow: /
+
 # Perplexity AI
 User-agent: PerplexityBot
 Allow: /
 
+User-agent: Perplexity-User
+Allow: /
+
 # Google Gemini/Bard
 User-agent: Google-Extended
 Allow: /
@@ -80,3 +95,40 @@ Allow: /
 # Meta AI
 User-agent: FacebookBot
 Allow: /
+
+User-agent: Meta-ExternalAgent
+Allow: /
+
+User-agent: meta-externalagent
+Allow: /
+
+# Apple Intelligence
+User-agent: Applebot
+Allow: /
+
+User-agent: Applebot-Extended
+Allow: /
+
+# Amazon AI
+User-agent: Amazonbot
+Allow: /
+
+# ByteDance / TikTok AI
+User-agent: Bytespider
+Allow: /
+
+# Cohere AI
+User-agent: cohere-ai
+Allow: /
+
+# DuckDuckGo AI (DuckAssist)
+User-agent: DuckAssistBot
+Allow: /
+
+# You.com AI
+User-agent: YouBot
+Allow: /
+
+# Diffbot (knowledge graph / AI training)
+User-agent: Diffbot
+Allow: /