# maverank.com welcomes search engines and AI crawlers of every kind:
# search indexing, AI answers, and AI training are all allowed on the
# public marketing site. The signed-in product stays out of the index.
#
# Two rules govern edits to this file:
#
#   1. Exactly one `User-agent: *` group, and exactly one Content-Signal
#      line, inside it. Duplicate wildcard groups carrying conflicting
#      signals are undefined by the standard and parsers disagree about
#      which one wins.
#   2. Every named group repeats the Disallow set. Group selection is
#      most-specific-match, so a named group with a bare `Allow: /` would
#      re-open /app/ and /accounts/ to that agent alone.
#
# See seo/SITE-AUDIT.md (C1, C1b, M5) for what happens when either breaks.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/  # the Django admin
Disallow: /accounts/  # login, signup, password reset: never an entry point
Disallow: /app/  # the signed-in product is not for indexing
Disallow: /app
Disallow: /design/  # internal styleguide (dev builds only)

# The AI user agents, named and allowed explicitly. An agent that finds its
# own name here reads this group and ignores the wildcard above entirely,
# which is the point: the permission is declared, not inherited. The list
# lives in apps/web/crawlers.py.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: Amazonbot
User-agent: Bytespider
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: CloudflareBrowserRenderingCrawler
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /app/
Disallow: /app
Disallow: /design/

Sitemap: https://maverank.com/sitemap.xml