# pawflet.com, https://pawflet.com
#
# Policy, not a ranking lever. Everything a reader needs is public HTML; only the
# operator surface and the two write endpoints are closed.
#
# Note what robots.txt actually controls: CRAWLING, not indexing. A disallowed
# URL can still appear in results if something links to it. The operator route
# also sends `X-Robots-Tag: noindex, nofollow` at the response level, which is
# the control that governs indexing.

User-agent: *
Allow: /
Disallow: /heatmaps
Disallow: /api/
Disallow: /random

# ─────────────────────────────────────────────────────────────────────────
# AI and answer engines, all ALLOWED, deliberately.
#
# The vendors document their tokens as two separate systems, and conflating
# them is the single most common robots.txt mistake:
#
#   SEARCH / ANSWER tokens  → decide whether you can appear in an answer
#   TRAINING tokens         → decide whether your text trains a model
#
# Blocking the training token does NOT remove you from answers, and allowing
# it does not help you rank. Everything below is allowed because pawflet's
# purpose is to answer readers who ask a chatbot about their cat; being absent
# from those answers while present in the results is the worst of both.
#
# Sources checked 2026-10-01:
#   OpenAI    developers.openai.com/api/docs/bots
#   Anthropic support.claude.com/en/articles/8896518
#   Perplexity docs.perplexity.ai/docs/resources/perplexity-crawlers
#   Google    developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers
# ─────────────────────────────────────────────────────────────────────────

# OpenAI, OAI-SearchBot surfaces sites in ChatGPT search answers; GPTBot is
# training only. "Sites that are opted out of OAI-SearchBot will not be shown in
# ChatGPT search answers", hence both are allowed.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic, Claude-SearchBot is search; ClaudeBot is training; Claude-User is
# user-initiated fetch. Disabling Claude-SearchBot "may reduce your site's
# visibility and accuracy in user search results."
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity, PerplexityBot is search and link-surface only, never training.
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google, Google-Extended governs Gemini training and grounding. Google states
# it "does not impact a site's inclusion in Google Search nor is it used as a
# ranking signal", so allowing it costs organic search nothing.
User-agent: Google-Extended
Allow: /

# Common Crawl, an open dataset used across the research and training
# ecosystem. CCBot is the crawl token.
User-agent: CCBot
Allow: /

Sitemap: https://pawflet.com/sitemap.xml