# ============================================
# IdeaProof.io - Robots.txt
# Last Updated: January 17, 2026
# ============================================
# Allow all crawlers by default
User-agent: *
Allow: /
Allow: /lists/
Allow: /guides/
Allow: /tools/
Allow: /versus/
Allow: /questions/
Allow: /calculators/
Allow: /blog/
Allow: /assets/
Allow: /*.js
Allow: /*.css
Crawl-delay: 1
# Protected areas - block from all crawlers
Disallow: /admin/
Disallow: /dashboard/
Disallow: /auth/
Disallow: /api/internal/
Disallow: /*.json$
Disallow: /*?*apikey*
Disallow: /*?*token*
Disallow: /.env
# ============================================
# LLM & AI CRAWLERS (Priority Access)
# ============================================
# OpenAI (GPT, ChatGPT)
User-agent: GPTBot
Allow: /
Allow: /blog/
Allow: /lists/
Allow: /features
Allow: /pricing
Allow: /about
Crawl-delay: 1
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 1
# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /
Allow: /lists/
Crawl-delay: 1
User-agent: ClaudeBot
Allow: /
Allow: /blog/
Allow: /lists/
Crawl-delay: 1
User-agent: Claude-Web
Allow: /
Crawl-delay: 1
# Google (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /
Crawl-delay: 1
User-agent: GoogleOther
Allow: /
Crawl-delay: 2
User-agent: Googlebot
Allow: /
Crawl-delay: 1
# Perplexity
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1
# Meta (Llama AI)
User-agent: FacebookBot
Allow: /
Crawl-delay: 2
User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 2
# Amazon (Alexa)
User-agent: Amazonbot
Allow: /
Crawl-delay: 2
# Apple (Siri, Apple Intelligence)
User-agent: Applebot
Allow: /
Crawl-delay: 1
User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Crawl-delay: 1
# Microsoft (Copilot, Bing)
User-agent: Bingbot
Allow: /
Crawl-delay: 1
User-agent: msnbot
Allow: /
Crawl-delay: 2
# Cohere AI
User-agent: cohere-ai
Allow: /
Crawl-delay: 2
# You.com
User-agent: YouBot
Allow: /
Crawl-delay: 2
# Common Crawl
User-agent: CCBot
Allow: /blog/
Allow: /lists/
Allow: /features
Allow: /pricing
Allow: /about
Crawl-delay: 5
# Diffbot
User-agent: Diffbot
Allow: /
Crawl-delay: 3
# xAI (Grok)
User-agent: Grok
Allow: /
Crawl-delay: 2
# DeepSeek
User-agent: DeepSeekBot
Allow: /
Crawl-delay: 2
# Mistral AI
User-agent: MistralBot
Allow: /
Crawl-delay: 2
# ============================================
# SEO CRAWLERS (Rate Limited)
# ============================================
User-agent: SemrushBot
Allow: /
Crawl-delay: 10
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10
User-agent: MJ12bot
Crawl-delay: 10
User-agent: DotBot
Crawl-delay: 10
# ============================================
# SITEMAPS
# ============================================
# Primary sitemap (served via rewrite from ideaproof.io/sitemap.xml)
Sitemap: https://ideaproof.io/sitemap.xml
# Backup sitemap (direct storage access)
Sitemap: https://aioenivvgvtwyhpdioue.supabase.co/storage/v1/object/public/sitemaps/sitemap.xml