# McClatchy DC Bureau content is made available for your personal, non-commercial
# use subject to our Terms of Service here:
# https://www.mcclatchydc.com/customer-service/terms-of-service
# Use of any device, tool, or process designed to data mine or scrape the content
# using automated means is prohibited without prior written permission from
# The McClatchy DC Bureau. Prohibited uses include but are not limited to:
# (1) text and data mining activities
# (2) the development of any software, machine learning, artificial intelligence (AI),
# and/or large language models (LLMs);
# (3) creating or providing archived or cached data sets containing our content to others; and/or
# (4) any commercial purposes.
# Global allow for Mediapartners - this is used by Google to place ads in content,
# not indexing purposes.
User-agent: Mediapartners-Google
Disallow:
# Temporary throttles to decrease load during launch
User-agent: Yandex
Crawl-delay: 3
User-agent: Baiduspider
Crawl-delay: 3
User-agent: bingbot
Crawl-delay: 2
User-agent: Slurp
Crawl-delay: 2
# Story blocks for selected bots
User-agent: archive.org_bot
Disallow: /*.html
User-agent: ia_archiver
Disallow: /*.html
User-agent: ia_archiver-web.archive.org
Disallow: /*.html
# Complete blocks of uninteresting bots
User-agent: A6-Indexer
Disallow: /
User-agent: AddSearchBot
Disallow: /
User-agent: aiHitBot
Disallow: /
User-agent: Andibot
Disallow: /
User-agent: ankiro.com
Disallow: /
User-agent: AspiegelBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: bigsur.ai
Disallow: /
User-agent: BLP_bbot
Disallow: /
User-agent: Brightbot 1.0
Disallow: /
User-agent: Buck
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: ChatGLM-Spider
Disallow: /
User-agent: CheckMarkNetwork
Disallow: /
User-agent: coccocbot-web
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: cohere-training-data-crawler
Disallow: /
User-agent: Cotoyogi
Disallow: /
User-agent: crawler4j
Disallow: /
User-agent: Crawlspace
Disallow: /
User-agent: dataprovider
Disallow: /
User-agent: Datenbank Crawler
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: EasouSpider
Disallow: /
User-agent: Echobot Bot
Disallow: /
User-agent: EchoboxBot
Disallow: /
User-agent: ecoResearch
Disallow: /
User-agent: FirecrawlAgent
Disallow: /
User-agent: Grok
Disallow: /
User-agent: GrokBot
Disallow: /
User-agent: GrokBot/1.0
Disallow: /
User-agent: Grok-DeepSearch/1.0
Disallow: /
User-agent: Genieo
Disallow: /
User-agent: https://hada.news
Disallow: /
User-agent: iaskspider/2.0
Disallow: /
User-agent: ICC-Crawler
Disallow: /
User-agent: img2dataset
Disallow: /
User-agent: JenkersBot
Disallow: /
User-agent: Jetslide
Disallow: /
User-agent: Kangaroo Bot
Disallow: /
User-agent: KunatoCrawler
Disallow: /
User-agent: legistorm.com
Disallow: /
User-agent: LivelapBot
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: Magus Bot
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: MAZBot
Disallow: /
User-agent: MistralAI-User
Disallow: /
User-agent: MistralAI-User/1.0
Disallow: /
User-agent: Mojeek
Disallow: /
User-agent: MojeekBot
Disallow: /
User-agent: MyCentralAIScraperBot
Disallow: /
User-agent: news-please
Disallow: /
User-agent: NewsNow
Disallow: /
User-agent: Netvibes
Disallow: /
User-agent: nlcrawler
Disallow: /
User-agent: NovaAct
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: omgili
Disallow: /
User-agent: Oracle Secure Enterprise Search
Disallow: /
User-agent: panscient.com
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: PanguBot
Disallow: /
User-agent: Poseidon Research Crawler
Disallow: /
User-agent: Primalbot
Disallow: /
User-agent: Riddler
Disallow: /
User-agent: QuillBot
Disallow: /
User-agent: quillbot.com
Disallow: /
User-agent: R6_CommentReader
Disallow: /
User-agent: SBIntuitionsBot
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: SeekportBot
Disallow: /
User-agent: Seekr
Disallow: /
User-agent: SeekrBot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: SeznamHomepageCrawler
Disallow: /
User-agent: Sidetrade indexer bot
Disallow: /
User-agent: Spinn3r
Disallow: /
User-agent: TaraGroup Intelligent Bot
Disallow: /
User-agent: Thinkbot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: VelenPublicWebCrawler
Disallow: /
User-agent: Verity
Disallow: /
User-agent: Verity/1.1
Disallow: /
User-agent: ViennaTinyBot
Disallow: /
User-agent: voxgov.com
Disallow: /
User-agent: webintegration.at
Disallow: /
User-agent: WebVac
Disallow: /
User-agent: Webzio-Extended
Disallow: /
User-agent: WebZip
Disallow: /
User-agent: WRTNBot
Disallow: /
User-agent: xAI
Disallow: /
User-agent: xAI-Grok/1.0
Disallow: /
User-agent: yacy
Disallow: /
User-agent: yacybot
Disallow: /
User-agent: YouBot
Disallow: /
# Block indexing of search results page
User-Agent: *
Disallow: /search
Disallow: /web
Disallow: /mrcontent
Disallow: /webapi-public
Disallow: /adperfect
Disallow: /deals-offers
Disallow: /*amp.html?*
Disallow: *.ece*
Allow: *.ece*.pdf
# Block indexing of specific sections/content at the market's request
User-Agent: *
Disallow: /news/politics-government/election/voters-guide
Sitemap: https://www.mcclatchydc.com/sitemap/story/update.xml
Sitemap: https://www.mcclatchydc.com/sitemap/story/archive.xml
Sitemap: https://www.mcclatchydc.com/sitemap/story/googlenews.xml
Sitemap: https://www.mcclatchydc.com/sitemap/blogpost/update.xml
Sitemap: https://www.mcclatchydc.com/sitemap/blogpost/archive.xml
Sitemap: https://www.mcclatchydc.com/sitemap/blogpost/googlenews.xml
Sitemap: https://www.mcclatchydc.com/sitemap/section/update.xml
Sitemap: https://www.mcclatchydc.com/sitemap/section/archive.xml
Sitemap: https://www.mcclatchydc.com/sitemap/topic/archive.xml
Sitemap: https://www.mcclatchydc.com/sitemap/topic/update.xml
Sitemap: https://www.mcclatchydc.com/sitemap/video/archive.xml
Sitemap: https://www.mcclatchydc.com/sitemap/video/update.xml