# https://cmtweb.com/robots.txt

# ---------------------------------------------------------------
# Everything, by default.
# ---------------------------------------------------------------
User-agent: *
Allow: /

# The form handler is not a page. /thanks is reachable only after a
# submission and carries its own noindex, so it is left crawlable.
Disallow: /api/

# ---------------------------------------------------------------
# AI answer engines.these fetch pages in order to cite them.
# Being cited is the point, so they are welcome.
# ---------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# ---------------------------------------------------------------
# User-triggered fetchers.someone pasted a link and asked about it.
# ---------------------------------------------------------------
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

# ---------------------------------------------------------------
# Model-training crawlers. Change any of these to `Disallow: /` to opt
# out of training without affecting the answer engines above.
# ---------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

# ---------------------------------------------------------------
Sitemap: https://cmtweb.com/sitemap.xml