# Allow all bots to crawl everything
User-agent: *
Allow: /

# Block admin/auth pages and promo pages
Disallow: /admin
Disallow: /auth
Disallow: /r/

# Legacy artifact: Netlify Prerender used to serialize SPA <Navigate>
# redirects as `Location: /?url=...`, which Googlebot indexed. The source
# has been removed (all SEO redirects now live in _redirects as edge 301s,
# enforced by scripts/check-navigate-redirects.ts), but block any lingering
# variants so Google stops re-crawling them.
Disallow: /*?url=

# AI Crawlers - Explicitly allow for AI recommendations
# OAI-SearchBot is the crawler that decides inclusion in ChatGPT SEARCH results
# (GPTBot is training; ChatGPT-User is on-demand fetches and does not affect
# search eligibility - per developers.openai.com/api/docs/bots). Covered by
# User-agent: * already; listed explicitly so it can never be lost in a refactor.
User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Timesbot
Allow: /

User-agent: YouBot
Allow: /

# Sitemap location
Sitemap: https://helloregimen.com/sitemap.xml