# Standard crawlers & Googlebot disallow rules
User-agent: *
Disallow: /admin/
Disallow: /internal/
Disallow: /dashboard/
Disallow: /auth/
Disallow: /login
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /analyze
Allow: /llm.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# Search engine crawlers
User-agent: Googlebot
Disallow: /admin/
Disallow: /internal/
Disallow: /dashboard/
Disallow: /auth/
Disallow: /login
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /analyze
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

# AI search, grounding, and retrieval crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: AnthropicBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: GeminiBot
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Cohere-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: BraveBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: MistralBot
Allow: /

User-agent: Ai2Bot
Allow: /

User-agent: iaskBot
Allow: /

User-agent: PiplBot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: omgilibot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: PhindBot
Allow: /

# Sitemaps
Sitemap: https://exogram.ai/sitemap.xml

# LLM-facing documentation (llms.txt protocol)
# https://exogram.ai/llm.txt
# https://exogram.ai/llms.txt
# https://exogram.ai/llms-full.txt