# robots.txt for askx.io
# ---------------------------------------------------------
# GLOBAL SITEMAP DECLARATION
# ---------------------------------------------------------
Sitemap: https://askx.io/api/v1/sitemap.xml

# ---------------------------------------------------------
# NEW: AI SEARCH ENGINE DIRECTIVES (Indexed & Cited)
# Allows ChatGPT and Claude to search you, but prevents training.
# ---------------------------------------------------------
User-agent: OAI-SearchBot
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /

User-agent: Claude-User
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
Allow: /

# ---------------------------------------------------------
# CLOUDFLARE & SECURITY BLOCKS
# Prevents aggressive scrapers and training bots.
# ---------------------------------------------------------
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# ---------------------------------------------------------
# GENERAL FALLBACK
# ---------------------------------------------------------
User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /
# Block internal API routes except the sitemap to save resources
Disallow: /api/
Allow: /api/v1/sitemap.xml
Disallow: /admin/
Disallow: /dashboard/