# stainless-structurals.com Robots.txt - AI-First Configuration
# Last Updated: November 12, 2025
#
# AI CRAWLERS - FULL ACCESS WITH RESPECTFUL DELAYS
# =================================================
# ChatGPT / GPTBot (OpenAI) - Search & Citation
User-agent: GPTBot
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# ChatGPT Search (Real-time queries)
User-agent: ChatGPT-User
Allow: /
Disallow: /wp-admin/
# OAI-SearchBot (ChatGPT Search indexing)
User-agent: OAI-SearchBot
Allow: /
Disallow: /wp-admin/
# Claude / Anthropic
User-agent: anthropic-ai
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
User-agent: Claude-Web
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
User-agent: ClaudeBot
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Perplexity
User-agent: PerplexityBot
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
User-agent: Perplexity-User
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Google Gemini / Bard
User-agent: Google-Extended
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Microsoft Copilot / Bing Chat
User-agent: *Copilot*
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# You.com
User-agent: YouBot
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Cohere AI
User-agent: cohere-ai
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Meta AI
User-agent: meta-externalagent
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Amazon Alexa
User-agent: Amazonbot
Crawl-delay: 3
Allow: /
Disallow: /wp-admin/
# Apple Bot (Siri, Spotlight)
User-agent: Applebot
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# TRADITIONAL SEARCH ENGINES
# ==========================
# Googlebot
User-agent: Googlebot
Crawl-delay: 1
Allow: /
Disallow: /wp-admin/
# Googlebot
User-agent: Googlebot-Image
Crawl-delay: 1
Allow: /wp-content/uploads/
Disallow: /wp-admin/
# Mediapartners-Google
User-agent: Mediapartners-Google*
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Bingbot
User-agent: Bingbot
Crawl-delay: 1
Allow: /
Disallow: /wp-admin/
# MSNBot
User-agent: msnbot
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# DuckDuckBot
User-agent: DuckDuckBot
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Yandex
User-agent: Yandex
Crawl-delay: 2
Allow: /
Disallow: /wp-admin/
# Baidu
User-agent: Baiduspider
Crawl-delay: 3
Allow: /
Disallow: /wp-admin/
# DISALLOWED BOTS
# ==============
User-agent: 008
Disallow: /
# ALL OTHER BOTS
# ==============
User-agent: *
Crawl-delay: 3
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# SITEMAP
# =======
Sitemap: https://www.stainless-structurals.com/sitemap_index.xml
# ADDITIONAL RESOURCES
# ====================
# llms.txt: https://www.stainless-structurals.com/llms.txt
# llms-full.txt: https://www.stainless-structurals.com/llms-full.txt
# LLM-ready Knowledge Base: https://ai.stainless-structurals.com
# Last Updated: November 12, 2025