# Click2Buy - robots.txt
# Updated: 2026-05-04
# ============================================================
# Standard search engine crawlers
# ============================================================
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*?lightbox=
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /*?replytocom=
Disallow: /tag/
Disallow: /fr/tag/
# Content-Signal: declare how content may be used by automated systems
# Format: search (search indexing), ai-train (LLM training), ai-input (RAG/inference)
Content-Signal: search=yes, ai-train=yes, ai-input=yes
# ============================================================
# AI crawlers - explicitly allowed
# ============================================================
# Anthropic
User-agent: ClaudeBot
Content-Signal: search=yes, ai-train=yes, ai-input=yes
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
# OpenAI
User-agent: GPTBot
Content-Signal: search=yes, ai-train=yes, ai-input=yes
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /
# Google AI
User-agent: Google-Extended
Content-Signal: search=yes, ai-train=yes, ai-input=yes
Allow: /
User-agent: GoogleOther
Allow: /
# Perplexity
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes
Allow: /
User-agent: Perplexity-User
Allow: /
# Apple
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /
# Common Crawl
User-agent: CCBot
Allow: /
# Meta
User-agent: FacebookBot
Allow: /
User-agent: Meta-ExternalAgent
Allow: /
User-agent: Meta-ExternalFetcher
Allow: /
# Amazon
User-agent: Amazonbot
Allow: /
# Cohere
User-agent: cohere-ai
Allow: /
User-agent: cohere-training-data-crawler
Allow: /
# Mistral
User-agent: MistralAI-User
Allow: /
# DuckDuckGo
User-agent: DuckAssistBot
Allow: /
# You.com
User-agent: YouBot
Allow: /
# Diffbot
User-agent: Diffbot
Allow: /
# Timpi
User-agent: TimpiBot
Allow: /
# Bytespider (TikTok / Doubao)
User-agent: Bytespider
Allow: /
# ============================================================
# Sitemaps
# ============================================================
Sitemap: https://click2buy.com/sitemap_index.xml
# ============================================================
# AI agent guides (non-standard, declared for discovery)
# X-AI-Index: https://click2buy.com/llms.txt
# X-AI-Index-Full: https://click2buy.com/llms-full.txt
# ============================================================