# Everything on this site is crawlable except the API endpoint.

User-agent: *

# The lead-capture endpoint. Not a document, and a GET returns a 405.
Disallow: /api/

# Deliberately NOT disallowed, despite being common in other robots.txt:
#
#   /_astro/   the hashed CSS and JS bundles
#   /images/   post heroes, logos, product art
#
# Blocking those is a habit carried over from stacks that leak build
# internals. It costs more than it saves here. Google renders a page
# before it ranks it, so a crawler denied the stylesheet and the scripts
# sees a broken layout and judges it accordingly, and blocked images
# cannot appear in image search. None of these paths expose anything: the
# bundles are content-hashed public assets and the images are already on
# the page.

# ── Answer engines ────────────────────────────────────────────────────────
# Named explicitly so the permission is a stated position, not an accident
# of the wildcard above. See src/pages/robots.txt.ts for why.

# OpenAI - model training, search index, user-triggered fetches
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /

# Anthropic - Claude
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Google - Gemini and Vertex AI grounding
User-agent: Google-Extended
Allow: /

# Apple - Siri and Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Common Crawl - the corpus most open models are trained on
User-agent: CCBot
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

Sitemap: https://mrscraper.com/sitemap.xml

# Machine-readable summary of the product, pricing and positioning,
# written for LLMs rather than for browsers:
# https://mrscraper.com/llms.txt

# The docs are a separate property with their own robots.txt, sitemap and
# llms.txt, and are equally open. A Sitemap: line only counts for the host
# serving it, so these are pointers rather than directives:
#   https://docs.mrscraper.com/llms.txt
#   https://docs.mrscraper.com/sitemap.xml