# beehiiv robots.txt
#
# Machine-readable site map for AI/LLM crawlers:
# - Curated index: https://www.beehiiv.com/llms.txt
# - Full content: https://www.beehiiv.com/llms-full.txt
# - Markdown twins: append `.md` to any page URL
# (e.g. /pricing -> /pricing.md)
#
# Markdown twins are served with `Content-Type: text/markdown` and contain
# clean, structured content with navigation/scripts/decorative chrome stripped.
# Crawling .md instead of HTML is cheaper, more accurate, and explicitly welcomed.
# *
User-agent: *
Allow: /
Allow: /*.md$
Disallow: /ads-exclusion
Disallow: /linkedinpremium
Disallow: /amazoncreatorstars
Disallow: /founder-offer
# Block literal bracket patterns (invalid URLs)
Disallow: /application/%5Bapplication_name%5D
Disallow: /application/%5Bapplication_name%5D/
Disallow: /buyers-guide/%5Bslug%5D
Disallow: /buyers-guide/%5Bslug%5D/
Disallow: /case-studies/%5Bslug%5D
Disallow: /case-studies/%5Bslug%5D/
Disallow: /experts/%5Bslug%5D
Disallow: /experts/%5Bslug%5D/
Disallow: /virtual-events/%5Bwebinar_name%5D
Disallow: /virtual-events/%5Bwebinar_name%5D/
Disallow: /features/%5Bslug%5D
Disallow: /features/%5Bslug%5D/
Disallow: /connect-with/%5Bslug%5D
Disallow: /connect-with/%5Bslug%5D/
Disallow: /partners/%5Bpartner_name%5D
Disallow: /partners/%5Bpartner_name%5D/
Disallow: /overview/%5Bslug%5D
Disallow: /splash-newsletter/%5Bslug%5D
Disallow: /splash-short/%5Bslug%5D
Disallow: /splash/%5Bslug%5D
Disallow: /report_abuse/%5B%5B...token%5D%5D
Disallow: /templates/%5Bslug%5D
Disallow: /templates/%5Bslug%5D/
Disallow: /resources/%5Blead_magnet%5D
Disallow: /i-am-a/%5Bslug%5D
Disallow: /i-am-a/%5Bslug%5D/
Disallow: /i-want-to/%5Bslug%5D
Disallow: /i-want-to/%5Bslug%5D/
# Block internal/registration routes
Disallow: /application/*/mfm
Disallow: /virtual-events/*/attendx
Disallow: /virtual-events/*/studio/
# Block Prismic dev tools
Disallow: /slice-simulator
Disallow: /slice-library
# Block internal support editor
Disallow: /kb-editor
Disallow: /kb-editor/
# OpenAI OAI-SearchBot — ChatGPT / AI-powered search (explicit allow; same path rules as *)
# https://developers.openai.com/api/docs/bots
User-agent: OAI-SearchBot
Allow: /
Allow: /*.md$
Disallow: /ads-exclusion
Disallow: /linkedinpremium
Disallow: /amazoncreatorstars
Disallow: /founder-offer
# Block literal bracket patterns (invalid URLs)
Disallow: /application/%5Bapplication_name%5D
Disallow: /application/%5Bapplication_name%5D/
Disallow: /buyers-guide/%5Bslug%5D
Disallow: /buyers-guide/%5Bslug%5D/
Disallow: /case-studies/%5Bslug%5D
Disallow: /case-studies/%5Bslug%5D/
Disallow: /experts/%5Bslug%5D
Disallow: /experts/%5Bslug%5D/
Disallow: /virtual-events/%5Bwebinar_name%5D
Disallow: /virtual-events/%5Bwebinar_name%5D/
Disallow: /features/%5Bslug%5D
Disallow: /features/%5Bslug%5D/
Disallow: /connect-with/%5Bslug%5D
Disallow: /connect-with/%5Bslug%5D/
Disallow: /partners/%5Bpartner_name%5D
Disallow: /partners/%5Bpartner_name%5D/
Disallow: /overview/%5Bslug%5D
Disallow: /splash-newsletter/%5Bslug%5D
Disallow: /splash-short/%5Bslug%5D
Disallow: /splash/%5Bslug%5D
Disallow: /report_abuse/%5B%5B...token%5D%5D
Disallow: /templates/%5Bslug%5D
Disallow: /templates/%5Bslug%5D/
Disallow: /resources/%5Blead_magnet%5D
Disallow: /i-am-a/%5Bslug%5D
Disallow: /i-am-a/%5Bslug%5D/
Disallow: /i-want-to/%5Bslug%5D
Disallow: /i-want-to/%5Bslug%5D/
# Block internal/registration routes
Disallow: /application/*/mfm
Disallow: /virtual-events/*/attendx
Disallow: /virtual-events/*/studio/
# Block Prismic dev tools
Disallow: /slice-simulator
Disallow: /slice-library
# Block internal support editor
Disallow: /kb-editor
Disallow: /kb-editor/
# Host
Host: https://www.beehiiv.com
# Sitemaps
Sitemap: https://www.beehiiv.com/sitemap-index.xml
Sitemap: https://www.beehiiv.com/sitemap-llm.xml