# Locusta AI — robots.txt
# Goal: full discoverability by search indexers and AI crawlers.
# - Search engines: Googlebot, Bingbot, DuckDuckBot, Applebot, etc. (covered by wildcard)
# - AI training crawlers (opt-in): GPTBot, ClaudeBot, Google-Extended, CCBot, Applebot-Extended, Meta-ExternalAgent
# - AI search/retrieval crawlers: PerplexityBot, ChatGPT-User, Claude-User, OAI-SearchBot, Perplexity-User
# - SEO/research: AhrefsBot, SemrushBot (we allow them too — they inform our own SEO audits)
# - API endpoints are disallowed everywhere (no value in indexing internal endpoints)

# === Major search indexers ===
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# === AI training crawlers (opt-in to allow model training on our public content) ===
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Diffbot
Allow: /

# === AI search / retrieval crawlers (used by ChatGPT, Perplexity, Claude, etc. for live answers) ===
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Perplexity-Plus
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

# === SEO / research crawlers ===
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

# === Default: allow everything else ===
User-agent: *
Allow: /

# === Disallow internal API endpoints (no SEO value, may surface errors) ===
Disallow: /api/
Disallow: /api/*.js

# === Sitemap and AI agent hint files ===
Sitemap: https://locusta.ai/sitemap.xml
# Reference companion file: https://locusta.ai/llms.txt
# Reference companion file: https://locusta.ai/for-ai-agents