# ---------------------------------------------------------------
# zinfi.com robots.txt
# Last Updated: April 2026
# Strategy: Maximum SEO + AEO + GEO visibility
# Open to all verified crawlers
# Block admin and low-value pages only
# Review quarterly — new AI crawlers launch regularly
# ---------------------------------------------------------------
# ---------------------------------------------------------------
# SITEMAPS
# ---------------------------------------------------------------
Sitemap: https://www.zinfi.com/sitemap_index.xml
#Sitemap: https://www.zinfi.com/post-sitemap.xml
#Sitemap: https://www.zinfi.com/page-sitemap.xml
# ---------------------------------------------------------------
# GLOBAL DEFAULT — Open to all crawlers
# Specific bot rules below override where needed
# ---------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /compare
Disallow: /tag/
Disallow: /category/
Disallow: /portfolio_category/
Disallow: /blog/success-stories/page/
Disallow: /resources/press-releases/page/
Disallow: /video-podcasts/partner-ecosystem-leaders/page/
# ---------------------------------------------------------------
# GOOGLE — Full Suite
# ---------------------------------------------------------------
# Primary web crawler — Google Search index
User-agent: Googlebot
Allow: /
# Image search crawler
User-agent: Googlebot-Image
Allow: /
# Video search crawler
User-agent: Googlebot-Video
Allow: /
# Google News crawler
User-agent: Googlebot-News
Allow: /
# Gemini AI, AI Overviews, Vertex AI training
User-agent: Google-Extended
Allow: /
# Google Search Console URL Inspection tool
User-agent: Google-InspectionTool
Allow: /
# Google AdSense quality crawler
User-agent: Mediapartners-Google
Allow: /
# Google Cloud Vertex AI crawler
User-agent: Google-CloudVertexBot
Allow: /
# Google AdsBot (landing page quality)
User-agent: AdsBot-Google
Allow: /
# Google Lighthouse (site performance)
User-agent: Chrome-Lighthouse
Allow: /
# Google Storebot (Shopping)
User-agent: Storebot-Google
Allow: /
User-agent: Google-NotebookLM
Allow: /
User-agent: Google-Agent
Allow: /
User-agent: GoogleAgent-Mariner
Allow: /
# ---------------------------------------------------------------
# CLAUDE - Full Suite
# ---------------------------------------------------------------
# Training
User-agent: ClaudeBot
Allow: /
# User-initiated querie
User-agent: Claude-User
Allow: /
# Search indexing
User-agent: Claude-SearchBot
Allow: /
# ---------------------------------------------------------------
# MICROSOFT / BING / COPILOT
# ---------------------------------------------------------------
# Primary Bing web crawler — powers Bing Search & Copilot
User-agent: Bingbot
Allow: /
# Legacy MSN bot
User-agent: msnbot
Allow: /
# MSN media crawler
User-agent: msnbot-media
Allow: /
# ---------------------------------------------------------------
# OPENAI / CHATGPT
# ---------------------------------------------------------------
# Primary training crawler — GPT-4o, GPT-5 training data
User-agent: GPTBot
Allow: /
# Real-time browsing during ChatGPT conversations
User-agent: ChatGPT-User
Allow: /
# OpenAI Search indexing (SearchGPT)
User-agent: OAI-SearchBot
Allow: /
User-agent: OAI-Operator
Allow: /
# ---------------------------------------------------------------
# ANTHROPIC / CLAUDE
# ---------------------------------------------------------------
# Claude citation and retrieval crawler
User-agent: ClaudeBot
Allow: /
# Claude web browsing crawler
User-agent: claude-web
Allow: /
# Claude user-initiated browsing
User-agent: Claude-User
Allow: /
# Claude AI search retrieval (new as of early 2026)
User-agent: Claude-SearchBot
Allow: /
# Anthropic model training crawler
User-agent: anthropic-ai
Allow: /
# ---------------------------------------------------------------
# PERPLEXITY AI
# ---------------------------------------------------------------
# Perplexity index builder — powers cited AI answers
User-agent: PerplexityBot
Allow: /
# Perplexity user-initiated browsing
User-agent: Perplexity-User
Allow: /
# ---------------------------------------------------------------
# APPLE / SIRI / APPLE INTELLIGENCE
# ---------------------------------------------------------------
# Powers Spotlight, Siri, Safari search
User-agent: Applebot
Allow: /
# Apple Intelligence AI features (opt-in required)
User-agent: Applebot-Extended
Allow: /
# ---------------------------------------------------------------
# AMAZON / ALEXA
# ---------------------------------------------------------------
# Powers Alexa answers and Amazon AI features
User-agent: Amazonbot
Allow: /
User-agent: Amazon-SearchBot
Allow: /
User-agent: Amazon-User
Allow: /
User-agent: AmazonBuyForMe
Allow: /
# ---------------------------------------------------------------
# META / FACEBOOK / INSTAGRAM
# ---------------------------------------------------------------
# Facebook/Instagram link previews
User-agent: facebookexternalhit
Allow: /
# Meta AI training crawler
User-agent: FacebookBot
Allow: /
# Meta external agent (backup fetcher)
User-agent: Meta-ExternalAgent
Allow: /
# Meta external fetcher
User-agent: Meta-ExternalFetcher
Allow: /
User-agent: Meta-WebIndexer
Allow: /
# ---------------------------------------------------------------
# BYTEDANCE / TIKTOK
# (AI training for Doubao and TikTok recommendations)
# ---------------------------------------------------------------
User-agent: Bytespider
Allow: /
User-agent: TikTokSpider
Allow: /
# ---------------------------------------------------------------
# COMMON CRAWL
# (Non-profit; dataset used to train GPT, Claude, LLaMA)
# ---------------------------------------------------------------
User-agent: CCBot
Allow: /
# ---------------------------------------------------------------
# DUCKDUCKGO
# ---------------------------------------------------------------
# Primary DuckDuckGo crawler
User-agent: DuckDuckBot
Allow: /
# DuckAssist AI answer feature
User-agent: DuckAssistBot
Allow: /
# ---------------------------------------------------------------
# YOU.COM
# ---------------------------------------------------------------
User-agent: YouBot
Allow: /
# ---------------------------------------------------------------
# COHERE AI
# ---------------------------------------------------------------
User-agent: cohere-ai
Allow: /
# ---------------------------------------------------------------
# XAI / GROK (X / TWITTER AI)
# ---------------------------------------------------------------
User-agent: xAI
Allow: /
# ---------------------------------------------------------------
# DIFFBOT
# (AI-powered web extraction; knowledge graph)
# ---------------------------------------------------------------
User-agent: Diffbot
Allow: /
# ---------------------------------------------------------------
# AI2 / ALLEN INSTITUTE FOR AI
# (Academic / research AI training)
# ---------------------------------------------------------------
User-agent: AI2Bot
Allow: /
User-agent: AI2Bot-Dolma
Allow: /
# ---------------------------------------------------------------
# PANGU / HUAWEI AI
# ---------------------------------------------------------------
User-agent: PanguBot
Allow: /
# ---------------------------------------------------------------
# TIMPI (Decentralized search engine)
# ---------------------------------------------------------------
User-agent: Timpibot
Allow: /
# ---------------------------------------------------------------
# BRAVE SEARCH
# ---------------------------------------------------------------
User-agent: Brave
Allow: /
# ---------------------------------------------------------------
# MOJEEK (Independent EU search engine)
# ---------------------------------------------------------------
User-agent: MojeekBot
Allow: /
# ---------------------------------------------------------------
# ECOSIA (Green search engine)
# ---------------------------------------------------------------
User-agent: Ecosia
Allow: /
# ---------------------------------------------------------------
# YANDEX (Russian search engine)
# ---------------------------------------------------------------
User-agent: Yandex
Allow: /
# ---------------------------------------------------------------
# BAIDU (Chinese search engine)
# ---------------------------------------------------------------
User-agent: Baiduspider
Allow: /
# ---------------------------------------------------------------
# NAVER (Korean search engine)
# ---------------------------------------------------------------
User-agent: NaverBot
Allow: /
# ---------------------------------------------------------------
# YAHOO (Uses Bing index + independent crawl)
# ---------------------------------------------------------------
User-agent: Slurp
Allow: /
# ---------------------------------------------------------------
# ICC-CRAWLER (NICT Japan — multilingual AI research)
# ---------------------------------------------------------------
User-agent: ICC-Crawler
Allow: /
# ---------------------------------------------------------------
# SOCIAL PLATFORM CRAWLERS
# (Link previews, cards, unfurling)
# ---------------------------------------------------------------
# Twitter / X link cards
User-agent: Twitterbot
Allow: /
# LinkedIn post link previews
User-agent: LinkedInBot
Allow: /
# Pinterest rich pins
User-agent: Pinterest
Allow: /
# Slack link unfurling
User-agent: Slackbot
Allow: /
# WhatsApp link previews
User-agent: WhatsApp
Allow: /
# Telegram link previews
User-agent: TelegramBot
Allow: /
# Discord link previews
User-agent: Discordbot
Allow: /
# ── Mistral ──────────────────────────────────────────────────
User-agent: MistralAI-User
Allow: /
# ── You.com ──────────────────────────────────────────────────
User-agent: YouBot
Allow: /
# ── DeepSeek ─────────────────────────────────────────────────
User-agent: DeepSeek
Allow: /
# ── Gemini Deep Research ─────────────────────────────────────
User-agent: Gemini-Deep-Research
Allow: /
# ---------------------------------------------------------------
# SEO & ANALYTICS TOOLS
# (Required for accurate competitive intelligence)
# ---------------------------------------------------------------
User-agent: AhrefsBot
Allow: /
User-agent: AhrefsSiteAudit
Allow: /
User-agent: SemrushBot
Allow: /
User-agent: SiteAuditBot
Allow: /
User-agent: MJ12bot
Allow: /
User-agent: DotBot
Allow: /
User-agent: BLEXBot
Allow: /
User-agent: RankBot
Allow: /
User-agent: DataForSeoBot
Allow: /
# ---------------------------------------------------------------
# MONITORING & UPTIME TOOLS
# ---------------------------------------------------------------
User-agent: UptimeRobot
Allow: /
User-agent: Pingdom
Allow: /
User-agent: GTmetrix
Allow: /
# ---------------------------------------------------------------
# FRIENDLY AI RESEARCH CRAWLERS
# ---------------------------------------------------------------
# Sentibot — sentiment analysis AI
User-agent: Sentibot
Allow: /
# FriendlyCrawler — academic research
User-agent: FriendlyCrawler
Allow: /
# VelenPublicWebCrawler — web research
User-agent: VelenPublicWebCrawler
Allow: /
# Webz.io AI data crawler
User-agent: Webzio-Extended
Allow: /
# Omgili / Webz.io (discussion indexing)
User-agent: omgili
Allow: /
User-agent: omgilibot
Allow: /
# Quora link preview bot
User-agent: Quora-Bot
Allow: /
# ---------------------------------------------------------------
# END OF FILE
# Next review date: July 2026
# Contact: [email protected]
# ---------------------------------------------------------------