User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /watchlist
Disallow: /alerts
Disallow: /auth/callback
Disallow: /unsubscribe
Disallow: /embed/

# Answer-engine / LLM crawlers are explicitly welcome. They already inherit the
# `*` group above, but naming them makes the intent durable — a future tightening
# of `*` shouldn't silently cut off the citation channel — and it is the file
# reviewers check when asking whether a site opts in to AI training/retrieval.
#
# IMPORTANT: a crawler that matches a named group ignores the `*` group entirely,
# so the private-surface disallows MUST be repeated here. Keep this list in sync
# with the `*` group above.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: Amazonbot
User-agent: MistralAI-User
Allow: /
Disallow: /dashboard
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /watchlist
Disallow: /alerts
Disallow: /auth/callback
Disallow: /unsubscribe
Disallow: /embed/

# Crawl-delay for non-Google bots
User-agent: SemrushBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

# Declare the index only — it already lists sitemap.xml, sitemap-stocks.xml and
# sitemap-compare.xml. Declaring an index and its children side by side makes
# Search Console double-report the same URLs under two submitted sitemaps.
Sitemap: https://www.equity-rank.com/sitemap-index.xml

# LLM-oriented site summary (llms.txt convention): what the data is, how it is
# computed, and the qualifiers that must travel with any quoted figure.
# https://www.equity-rank.com/llms.txt