# class2class.org — robots.txt
# Last updated: 2026-05-13
# Default crawler policy
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
# SEO tools we actively use
User-agent: Morningscore Bot
Allow: /
# Performance optimization tools (cloud workers that crawl to optimize delivery)
# FlyingPress Cloud Optimizer — extracts critical CSS / Used CSS per page
User-agent: FlyingPress
Allow: /
# AI assistants — explicit allow (GEO strategy)
# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /
# Google
User-agent: Google-Extended
Allow: /
# Anthropic / Claude
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
# Perplexity
User-agent: PerplexityBot
Allow: /
# Apple
User-agent: Applebot-Extended
Allow: /
# Common Crawl (used by many AI training pipelines)
User-agent: CCBot
Allow: /
# DeepSeek
User-agent: DeepSeekBot
Allow: /
# Mistral / Le Chat
User-agent: MistralAI-User
Allow: /
User-agent: MistralAI-Index
Allow: /
# ByteDance / Doubao (the user-action assistant crawler — not Bytespider)
User-agent: Doubaobot
Allow: /
# Cohere
User-agent: cohere-ai
Allow: /
User-agent: cohere-training-data-crawler
Allow: /
# You.com
User-agent: YouBot
Allow: /
# Diffbot (used by various AI products)
User-agent: Diffbot
Allow: /
# xAI / Grok (documented UAs are symbolic — actual crawler uses generic Mobile UA)
User-agent: GrokBot
Allow: /
User-agent: xAI-Grok
Allow: /
# Bytespider (ByteDance training crawler) — DISALLOWED due to abusive crawl rates
# Reportedly hits 1.4M req/day on some sites and ignores robots.txt anyway,
# but documenting intent. Doubao's user-facing crawler (Doubaobot) is allowed above.
User-agent: Bytespider
Disallow: /
# Sitemap
Sitemap: https://class2class.org/sitemap_index.xml