# The Bayou Ledger - https://bayouledger.com
# Public hunter stats are crawlable; private/account/API surfaces are not.
#
# NOTE: robots.txt groups do NOT inherit. A crawler obeys only the single
# most-specific group matching its token (RFC 9309 2.2.1), so every group below
# must carry its own full Disallow list - an `Allow: /`-only group opts that
# crawler out of every exclusion here.
#
# Every private path is listed once per shipped locale prefix (bare = en, /ru,
# /zh). The universal reroute hook strips the prefix before route matching, so
# EVERY route - /api/ included - answers under every prefix, and the bare rule
# does not cover the prefixed URL. Adding a locale means extending these lists;
# tests/robots.test.ts fails when a mirror is missing.
#
# Hunter dossiers (/p/) stay crawlable ON PURPOSE - see seo-index-prune. They
# emit `noindex`, and a crawler that cannot fetch a page never reads that
# directive: a Disallow here would freeze them in the index as bare snippet-less
# links instead of removing them. Same reasoning for the payment outcome pages.
#
# /upgrade is disallowed everywhere except the link-preview group: shared
# checkout links must still unfurl into a rich card, but the page is not for
# the index (it also emits `noindex`, and is absent from sitemap.xml).
#
# The legal texts (oferta.txt, privacy-policy.txt) are disallowed for everyone.
# They carry the operator's requisites (ИНН/ОГРНИП/contact e-mail), have no
# search value, and are Russian-only. They stay reachable by humans - robots.txt
# restricts crawling, not access - so the "unrestricted access" the privacy law
# and the PSPs require is unaffected. Being static .txt, they can carry neither
# a `noindex` meta nor an `X-Robots-Tag` (svelte-adapter-bun serves the static
# dir ahead of the SvelteKit handler, so hooks.server.ts never sees them).
#
# /offline is the PWA's offline fallback page - real content for a user with no
# connection, a dead-end "You're offline" page in an index. `Disallow: /offline`
# is a prefix, so it covers /offline.html too (sirv serves both). /fonts/ holds
# the self-hosted woff2 files: not pages, and text extraction doesn't need them.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /ru/api/
Disallow: /zh/api/
Disallow: /account
Disallow: /notifications
Disallow: /watchlist
Disallow: /party-finder
Disallow: /me
Disallow: /login
Disallow: /overlay/
Disallow: /upgrade
Disallow: /oferta.txt
Disallow: /privacy-policy.txt
Disallow: /bio-links.txt
Disallow: /offline
Disallow: /fonts/
Disallow: /ru/account
Disallow: /ru/notifications
Disallow: /ru/watchlist
Disallow: /ru/party-finder
Disallow: /ru/me
Disallow: /ru/login
Disallow: /ru/overlay/
Disallow: /ru/upgrade
Disallow: /zh/account
Disallow: /zh/notifications
Disallow: /zh/watchlist
Disallow: /zh/party-finder
Disallow: /zh/me
Disallow: /zh/login
Disallow: /zh/overlay/
Disallow: /zh/upgrade

# Search engines - index public pages, same private-path exclusions as the * group
# (explicit entries document intent; keep these disallows in sync with * above)
User-agent: Googlebot
User-agent: Bingbot
User-agent: YandexBot
User-agent: DuckDuckBot
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /ru/api/
Disallow: /zh/api/
Disallow: /account
Disallow: /notifications
Disallow: /watchlist
Disallow: /party-finder
Disallow: /me
Disallow: /login
Disallow: /overlay/
Disallow: /upgrade
Disallow: /oferta.txt
Disallow: /privacy-policy.txt
Disallow: /bio-links.txt
Disallow: /offline
Disallow: /fonts/
Disallow: /ru/account
Disallow: /ru/notifications
Disallow: /ru/watchlist
Disallow: /ru/party-finder
Disallow: /ru/me
Disallow: /ru/login
Disallow: /ru/overlay/
Disallow: /ru/upgrade
Disallow: /zh/account
Disallow: /zh/notifications
Disallow: /zh/watchlist
Disallow: /zh/party-finder
Disallow: /zh/me
Disallow: /zh/login
Disallow: /zh/overlay/
Disallow: /zh/upgrade

# AI-search crawlers explicitly allowed - we want AI-answer citations.
# (SSR means our content is readable without JS execution.)
# They get the same exclusions as the * group: their own group replaces it.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /ru/api/
Disallow: /zh/api/
Disallow: /account
Disallow: /notifications
Disallow: /watchlist
Disallow: /party-finder
Disallow: /me
Disallow: /login
Disallow: /overlay/
Disallow: /upgrade
Disallow: /oferta.txt
Disallow: /privacy-policy.txt
Disallow: /bio-links.txt
Disallow: /offline
Disallow: /fonts/
Disallow: /ru/account
Disallow: /ru/notifications
Disallow: /ru/watchlist
Disallow: /ru/party-finder
Disallow: /ru/me
Disallow: /ru/login
Disallow: /ru/overlay/
Disallow: /ru/upgrade
Disallow: /zh/account
Disallow: /zh/notifications
Disallow: /zh/watchlist
Disallow: /zh/party-finder
Disallow: /zh/me
Disallow: /zh/login
Disallow: /zh/overlay/
Disallow: /zh/upgrade

# Social + messaging link-preview crawlers - shared links unfurl into rich cards
# (no api/overlay). /upgrade IS allowed here, and only here: a shared checkout
# link has to render its og: card. The legal .txt files carry no og: tags, so
# there is no card to render and nothing lost by disallowing them here too.
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: TelegramBot
User-agent: Discordbot
User-agent: Slackbot-LinkExpanding
User-agent: Slackbot
User-agent: WhatsApp
User-agent: LinkedInBot
User-agent: SkypeUriPreview
User-agent: vkShare
User-agent: redditbot
User-agent: Pinterest
Allow: /
Disallow: /api/
Disallow: /ru/api/
Disallow: /zh/api/
Disallow: /overlay/
Disallow: /ru/overlay/
Disallow: /zh/overlay/
Disallow: /oferta.txt
Disallow: /privacy-policy.txt
Disallow: /bio-links.txt

Sitemap: https://bayouledger.com/sitemap.xml