# Robots.txt for Barley Medical
# Allow all search engines and AI crawlers to index public pages

User-agent: *
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

# Allow AI crawlers (LLMs, AI search engines)
User-agent: GPTBot
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: ChatGPT-User
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: Google-Extended
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: CCBot
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: anthropic-ai
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: Claude-Web
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: ClaudeBot
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: Claude-User
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: Claude-SearchBot
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: OAI-SearchBot
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: Applebot-Extended
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: meta-externalagent
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: PerplexityBot
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

User-agent: YouBot
Allow: /
Disallow: /marketing
Disallow: /flyer
Disallow: /brand

# NOTHING THAT CARRIES A noindex HEADER IS LISTED HERE, and that is the whole rule.
#
# A path blocked in this file is never fetched, so its noindex is never read — and a URL nobody may
# fetch can still be indexed from a link alone, listed bare with no title and no description. Where
# the intent is "do not index this", the Disallow is the one instruction that prevents the
# instruction from arriving.
#
# So the private questionnaire (/medicare/next-move and its sub-paths) and the delivery pages
# (/medicare/delivery/*) are deliberately absent. Both carry
# X-Robots-Tag: noindex, nofollow, noarchive in netlify.toml, on the path AND its wildcard, and the
# questionnaire's prerendered HTML carries the same as a meta tag. Those are the controls that work,
# because a crawler has to read the page to obey them.
#
# The questionnaire WAS disallowed here until 2026-09-01, alongside a noindex it could therefore
# never read — the exact mistake the delivery note below it was written to avoid, made two lines
# further up the same file. `src/lib/robots-contract.test.ts` now reads netlify.toml and fails if
# any header-noindexed path reappears in this file, so the two cannot disagree again.
#
# What IS listed is /marketing, /flyer and /brand: internal tools with no reason to be crawled at
# all. They also carry noindex headers, and blocking them costs nothing because nobody links to them.

# Sitemap location
Sitemap: https://barleymedical.com/sitemap.xml
Sitemap: https://barleymedical.com/denials/sitemap.xml