diff --git a/public/robots.txt b/public/robots.txt index ef2f5f7..5d38571 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -58,6 +58,9 @@ Allow: / User-agent: GPTBot Allow: / +User-agent: OAI-SearchBot +Allow: / + # Anthropic/Claude User-agent: anthropic-ai Allow: / @@ -65,10 +68,22 @@ Allow: / User-agent: Claude-Web Allow: / +User-agent: ClaudeBot +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: Claude-User +Allow: / + # Perplexity AI User-agent: PerplexityBot Allow: / +User-agent: Perplexity-User +Allow: / + # Google Gemini/Bard User-agent: Google-Extended Allow: / @@ -80,3 +95,40 @@ Allow: / # Meta AI User-agent: FacebookBot Allow: / + +User-agent: Meta-ExternalAgent +Allow: / + +User-agent: meta-externalagent +Allow: / + +# Apple Intelligence +User-agent: Applebot +Allow: / + +User-agent: Applebot-Extended +Allow: / + +# Amazon AI +User-agent: Amazonbot +Allow: / + +# ByteDance / TikTok AI +User-agent: Bytespider +Allow: / + +# Cohere AI +User-agent: cohere-ai +Allow: / + +# DuckDuckGo AI (DuckAssist) +User-agent: DuckAssistBot +Allow: / + +# You.com AI +User-agent: YouBot +Allow: / + +# Diffbot (knowledge graph / AI training) +User-agent: Diffbot +Allow: /