# LegatMatch robots.txt
# https://legatmatch.dk
#
# NOTE: a crawler obeys ONLY the most specific group that matches its name
# (RFC 9309 §2.2.1). A named group therefore replaces the "*" group outright -
# it does not add to it - so every Disallow below has to be repeated in every
# group, and ahead of "Allow: /" so that older first-match-wins parsers reach
# them at all. Leaving them out of the named groups let Googlebot, Bingbot and the
# AI crawlers straight into /admin and into the /genoptag/ links that restore
# a visitor's saved quiz answers.

# Everything not named below.
User-agent: *
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Google
User-agent: Googlebot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Bing
User-agent: Bingbot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# AI Crawlers - ChatGPT, Claude, Perplexity, etc.
User-agent: GPTBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: ChatGPT-User
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: Claude-Web
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: ClaudeBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: PerplexityBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: Applebot-Extended
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: anthropic-ai
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: cohere-ai
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: Google-Extended
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Common Crawl — the corpus most LLMs are trained on
User-agent: CCBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# ChatGPT search index (separate from GPTBot)
User-agent: OAI-SearchBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Perplexity's live fetch on a user's behalf (PerplexityBot is declared above)
User-agent: Perplexity-User
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: Applebot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: Amazonbot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: meta-externalagent
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: Bytespider
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: DuckDuckBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: YandexBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: MistralAI-User
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

User-agent: Diffbot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Google Gemini / Vertex AI grounding
User-agent: Google-CloudVertexBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Meta AI live fetch on a user's behalf
User-agent: Meta-ExternalFetcher
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# You.com
User-agent: YouBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Petal / Huawei search
User-agent: PetalBot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Kagi
User-agent: Kagibot
Disallow: /admin
Disallow: /payment-success
Disallow: /quiz-completed
Disallow: /genoptag/
Allow: /

# Sitemap and RSS feed
Sitemap: https://legatmatch.dk/sitemap.xml
Sitemap: https://legatmatch.dk/rss.xml

# AI-readable content index (full article text)
# https://legatmatch.dk/llms.txt
# https://legatmatch.dk/llms-full.txt