# ⚠️ A crawler obeys the SINGLE most specific user-agent group that matches it,
# and never merges that group with `User-agent: *` (RFC 9309). So a group that
# says only `Allow: /` does not "also allow" — it REPLACES everything below,
# private paths included. This file used to carry bare `Allow: /` groups for
# Googlebot, Bingbot, GPTBot, ClaudeBot and PerplexityBot, which handed all five
# of them /dashboard, /checkout, /auth and /app.html.
#
# The rule here: any per-bot group repeats the private-path list, or it does not
# exist. Googlebot and Bingbot have no group at all — `*` already allows them.
#
# ⚠️ Cloudflare PREPENDS its own managed block to this file at the edge, and that
# block currently disallows GPTBot and ClaudeBot outright. The served file is the
# concatenation of both — check https://brevy.me/robots.txt, not this file, for
# what crawlers actually see. Cloudflare's side is configured in the dashboard
# (Security → Bots → AI Crawl Control), not here.

User-agent: *
# Allow all public pages by default
Allow: /

# Disallow private/authenticated areas and API routes
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /auth
Disallow: /checkout
Disallow: /payment-success
Disallow: /subscription
Disallow: /data-management
Disallow: /reset-password
Disallow: /password-reset-confirmation
Disallow: /design/

# The empty SPA shell that vercel.json rewrites unmatched routes to. It is
# machinery, not a page — index.html carries the real prerendered homepage.
Disallow: /app.html

# Explicitly allow important public pages for SEO
Allow: /pricing
Allow: /cv-builder
Allow: /blog
Allow: /cv/

# LLM crawlers. These groups exist to say "yes" deliberately, against the
# Cloudflare managed block above — so they must restate the private paths rather
# than collapse to a bare Allow.
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /auth
Disallow: /checkout
Disallow: /payment-success
Disallow: /subscription
Disallow: /data-management
Disallow: /reset-password
Disallow: /password-reset-confirmation
Disallow: /design/
Disallow: /app.html

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /auth
Disallow: /checkout
Disallow: /payment-success
Disallow: /subscription
Disallow: /data-management
Disallow: /reset-password
Disallow: /password-reset-confirmation
Disallow: /design/
Disallow: /app.html

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /auth
Disallow: /checkout
Disallow: /payment-success
Disallow: /subscription
Disallow: /data-management
Disallow: /reset-password
Disallow: /password-reset-confirmation
Disallow: /design/
Disallow: /app.html

Sitemap: https://brevy.me/sitemap.xml