# Marketing pages (/, /why-clarity, /solutions, /pricing, /security,
# /raving-fans, /about, /contact, /help) are crawlable. Everything below is
# either token-bearing (a leaked link must not get crawled) or signed-in-only
# app shell. The server also sends X-Robots-Tag: noindex on these paths, which
# is what actually keeps an already-linked URL out of the index.
#
# Every public page is also available as markdown at the same path with a .md
# suffix, or via Accept: text/markdown. See /llms.txt.

User-agent: *
Disallow: /api/
Disallow: /report/
Disallow: /portal
Disallow: /contract-link/
Disallow: /advisor
Disallow: /agent
Disallow: /admin
Disallow: /onboarding
Disallow: /login
Disallow: /signup
Disallow: /dev/
Disallow: /dpa-breakdown
# The empty SPA shell. Real content lives at the routes themselves.
Disallow: /app-shell.html

# Assistant and answer-engine crawlers are named explicitly.
#
# Not because the wildcard group above would exclude them: it already allows
# them everything it allows anyone. It is here so the decision is recorded
# rather than inferred. The public pages are documentation we want quoted, and
# the disallowed paths hold borrower data and capability tokens, which is a
# harder line than a crawl-budget preference and should read that way.
#
# Removing a name from this list does NOT keep a model from being trained on
# the site, and nothing here is a substitute for the noindex header on the
# private paths.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: Amazonbot
User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /report/
Disallow: /portal
Disallow: /contract-link/
Disallow: /advisor
Disallow: /agent
Disallow: /admin
Disallow: /onboarding
Disallow: /login
Disallow: /signup
Disallow: /dev/
Disallow: /dpa-breakdown
Disallow: /app-shell.html

Sitemap: https://www.mortgageclarityplatform.com/sitemap.xml