# ============================================================
# Whoppah robots.txt
#
# Strategy:
#  1. Welcome every general-purpose web crawler.
#  2. Name every AI / LLM crawler explicitly with Allow: / so each
#     provider's bot has an unambiguous, machine-verifiable signal
#     that Whoppah content is consentingly available for indexing
#     and AI grounding. This is the *signal*; the actual
#     unblocking is done at the WAF / CDN layer with a verified
#     User-Agent + reverse-DNS allow-list.
#  3. Disallow private account / checkout / similar-search URLs
#     that have no public-discovery value (kept identical to the
#     prior file, per language).
#  4. Reference both the main www sitemap index and the Dixa
#     support sitemap index so crawlers discover the 780+ FAQ
#     articles under support.whoppah.com without having to guess.
# ============================================================

# ---- AI / LLM crawlers (explicit welcome) ------------------
# OpenAI (training + answer-time browsing for ChatGPT)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /

# Anthropic (training + browsing for Claude.ai + claude tool use)
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /

# Perplexity (training + answer-time citation)
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Google AI Overviews / Gemini (separate from Googlebot)
User-agent: Google-Extended
Allow: /

# Apple Intelligence (separate from Applebot used by Spotlight/Siri)
User-agent: Applebot-Extended
Allow: /

# Amazon (Rufus + Alexa AI shopping)
User-agent: Amazonbot
Allow: /

# Cohere (model training + Coral)
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
Allow: /

# ByteDance (TikTok AI products)
User-agent: Bytespider
Allow: /

# Meta AI assistants (separate from Facebook scrape bots)
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Allow: /

# Genspark (AI search)
User-agent: GensparkBot
Allow: /

# You.com (AI search)
User-agent: YouBot
Allow: /

# DuckDuckGo Assist / AI features
User-agent: DuckAssistBot
Allow: /

# Diffbot (LLM grounding data provider used by multiple AI platforms)
User-agent: DiffbotBot
Allow: /

# Common Crawl (training data feed for most open-source LLMs)
User-agent: CCBot
Allow: /

# ---- All other crawlers ------------------------------------
User-Agent: *

Allow: /

Disallow: /similar
Disallow: /user-ads
Disallow: /account/ads
Disallow: /account/purchases
Disallow: /account/sales
Disallow: /create-ad/success
Disallow: /nl/similar
Disallow: /nl/user-ads
Disallow: /nl/account/ads
Disallow: /nl/account/purchases
Disallow: /nl/account/sales
Disallow: /nl/create-ad/success
Disallow: /de/similar
Disallow: /de/user-ads
Disallow: /de/account/ads
Disallow: /de/account/purchases
Disallow: /de/account/sales
Disallow: /de/create-ad/success
Disallow: /fr/similar
Disallow: /fr/user-ads
Disallow: /fr/account/ads
Disallow: /fr/account/purchases
Disallow: /fr/account/sales
Disallow: /fr/create-ad/success
Disallow: /es/similar
Disallow: /es/user-ads
Disallow: /es/account/ads
Disallow: /es/account/purchases
Disallow: /es/account/sales
Disallow: /es/create-ad/success
Disallow: /it/similar
Disallow: /it/user-ads
Disallow: /it/account/ads
Disallow: /it/account/purchases
Disallow: /it/account/sales
Disallow: /it/create-ad/success

# ---- Sitemaps ----------------------------------------------
Sitemap: https://www.whoppah.com/sitemap.xml
Sitemap: https://www.whoppah.com/website-sitemap.xml
Sitemap: https://support.whoppah.com/sitemap.xml