# ============================================
# robots.txt for yabupushelberg.com
# Optimized for International SEO/GEO
# Last Updated: December 2024
# ============================================
# ============================================
# MAJOR SEARCH ENGINES - ALLOWED
# Essential for international visibility
# ============================================
# Google (Global - Primary)
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
Allow: /
# Bing/Microsoft (Global - US/Europe)
User-agent: Bingbot
User-agent: msnbot
User-agent: BingPreview
Allow: /
# Baidu (CRITICAL for China market)
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Baiduspider-video
User-agent: Baiduspider-news
Allow: /
# Sogou (China - Secondary)
User-agent: Sogou
User-agent: sogou spider
Allow: /
# 360 Search (China)
User-agent: 360Spider
User-agent: HaoSouSpider
Allow: /
# Yandex (Russia/Eastern Europe)
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexImages
Allow: /
# Naver (South Korea)
User-agent: Yeti
User-agent: NaverBot
Allow: /
# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
# Yahoo/Japan (Important for Japanese market)
User-agent: Slurp
Allow: /
# Qwant (European privacy-focused)
User-agent: Qwantify
Allow: /
# Apple
User-agent: Applebot
Allow: /
# ============================================
# SOCIAL MEDIA CRAWLERS - ALLOWED
# Essential for link previews when sharing
# ============================================
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterest
User-agent: Pinterestbot
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: WhatsApp
User-agent: TelegramBot
Allow: /
# ============================================
# SEO & ANALYTICS TOOLS - ALLOWED
# Helpful for monitoring site performance
# ============================================
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: rogerbot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: serpstatbot
User-agent: SEOkicks
User-agent: Screaming Frog SEO Spider
Allow: /
# ============================================
# AI TRAINING CRAWLERS - BLOCKED
# Prevent content from training AI models
# ============================================
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: FacebookBot
Disallow: /
# ============================================
# EMAIL HARVESTERS - BLOCKED
# ============================================
User-agent: EmailCollector
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: EmailWolf
Disallow: /
User-agent: ExtractorPro
Disallow: /
# ============================================
# SITE COPIERS & SCRAPERS - BLOCKED
# ============================================
User-agent: Wget
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZip
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: Nimbostratus-Bot
Disallow: /
# ============================================
# ARCHIVE BOTS - BLOCKED (optional)
# Remove if you want site archived
# ============================================
User-agent: ia_archiver
Disallow: /
User-agent: archive.org_bot
Disallow: /
# ============================================
# DEFAULT RULE - All other bots
# ============================================
User-agent: *
# Block admin/private areas
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /private/
Disallow: /staging/
Disallow: /dev/
Disallow: /test/
Disallow: /tmp/
Disallow: /cgi-bin/
Disallow: /api/
Disallow: /search/
Disallow: /search?
Disallow: /*?*
# Allow everything else
Allow: /
# Crawl delay (seconds between requests)
Crawl-delay: 10
# ============================================
# SITEMAPS
# ============================================
Sitemap: https://yabupushelberg.com/sitemap.xml
# Future localized sitemaps (uncomment when live):
# Sitemap: https://yabupushelberg.com/zh-cn/sitemap.xml
# Sitemap: https://yabupushelberg.com/ar/sitemap.xml
# Sitemap: https://yabupushelberg.com/ja/sitemap.xml
# Sitemap: https://yabupushelberg.com/fr/sitemap.xml
# Sitemap: https://yabupushelberg.com/es/sitemap.xml
# ============================================
# SUMMARY FOR YABU PUSHELBERG
# ============================================
#
# ✅ ALLOWED:
# - Google, Bing, Baidu, Sogou, 360 (search engines)
# - Yandex, Naver, Yahoo Japan (regional search)
# - Facebook, LinkedIn, Twitter, Pinterest (social)
# - Ahrefs, SEMrush, Moz, etc. (SEO tools)
#
# ❌ BLOCKED:
# - GPTBot, CCBot, etc. (AI training)
# - Email harvesters
# - Site copiers (HTTrack, WebZip)
# - Archive.org (remove if you want archiving)
#
# TO DO:
# 1. Upload to root of yabupushelberg.com
# 2. Verify sitemap URL is correct
# 3. Add locale sitemaps when launched
# ============================================
Sitemap: https://www.yabupushelberg.com/sitemap.xml