# START BOOKSWARM ROBOTS.TXT TEMPLATE (Modified)
# -----------------------------------
#
# General rules for all user agents, including AI bots.
# We allow crawling but disallow wp-admin and PDF files.
User-agent: *
Crawl-delay: 7
Disallow: /wp-admin/
Disallow: /*.pdf$
#
# ---------------------------------
# Explicitly allow major search engines (optional but good practice).
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
Allow: /
#
# ---------------------------------
# Block specific non-essential or aggressive crawlers.
# AI bots (GPTBot, CCBot, etc.) are NOT in this list and will use the default rules above.
User-agent: TinyTestBot
User-agent: SEMrushBot
User-agent: AhrefsBot
User-agent: PetalBot
User-agent: MJ12Bot
User-agent: DotBot
User-agent: MauiBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Barkrowler
User-agent: Bytespider
User-agent: WhatStuffWhereBot
User-agent: Applebot
User-agent: Sogou Pic Spider/3.0
User-agent: Sogou head spider/3.0
User-agent: Sogou web spider/4.0
User-agent: Sogou Orion spider/3.0
User-agent: Sogou-Test-Spider/4.0
User-agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
User-agent: Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
User-agent: Swiftbot
User-agent: Slurp
Disallow: /
#
# ---------------------------------
# END BOOKSWARM ROBOTS.TXT TEMPLATE
# _
# [ ]
# ( )
# |>|
# __/===\__
# //| o=o |\\
#
# \=====/
# / / | \ \
#