# robots.txt — acipta.ai
# Allow reputable crawlers AND answer-engine crawlers (GPTBot, ClaudeBot,
# PerplexityBot, anthropic-ai) so acipta.ai content can be cited in
# ChatGPT / Claude / Perplexity answers.
# Still block training-only crawlers that don't drive citations.
# Legal pages use meta noindex (not robots.txt Disallow) so crawlers can still
# see the noindex tag — Disallow would prevent them from crawling and the
# pages could still index via link anchor text.

User-agent: *
Allow: /
# Cloudflare email-obfuscation endpoint — not a real page; was showing in GSC
# as "Not found (404)" for /cdn-cgi/l/email-protection. Block crawling.
Disallow: /cdn-cgi/
Disallow: /data/
Disallow: /*/data/
Disallow: /*/data/*
Disallow: /deploy/
Disallow: /demo/preview/
Disallow: /Archive/
Disallow: /research-artifacts/
Disallow: /*-PRE-LIVE-CHECKLIST.html$
Disallow: /*.bak-*

# Answer-engine crawlers — ALLOWED for citation visibility in ChatGPT, Claude,
# and Perplexity answers. Unblocked 2026-05-19 per Vijay (P0 site-optimization).
# GPTBot also used for OpenAI model training — accepting that trade-off in
# exchange for SearchGPT / ChatGPT citation eligibility.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

# Google-Extended gates Gemini / Vertex grounding + citation. Allowed 2026-06-18 (AI-SEO
# review): Google AI Overviews already crawl via Googlebot, so unblocking Google-Extended
# adds Gemini-app citation eligibility — same training-for-citation trade-off as GPTBot.
User-agent: Google-Extended
Allow: /

# Training-only crawlers (no citation surface) — still blocked.
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: cohere-ai
Disallow: /

Sitemap: https://acipta.ai/sitemap.xml