User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: newspaper/0.2.8
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: ltx71 - (http://ltx71.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)
Disallow: /
User-agent: SiteAuditBot
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: SemrushBot-SI
Disallow: /
User-agent: SemrushBot-SWA
Disallow: /
User-agent: SemrushBot-CT
Disallow: /
User-agent: SemrushBot-BM
Disallow: /
User-agent: SplitSignalBot
Disallow: /
User-agent: SemrushBot-COUB
Disallow: /
User-agent: Mozilla/5.0 (compatible; Dubbotbot/0.2; +http://dubbot.com)
Disallow: /
User-agent: Applebot
Disallow: /
User-agent: Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)
Disallow: /
User-agent: BLP_bbot/0.1
Disallow: /
User-agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
Disallow: /
User-agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
Disallow: /
User-agent: EyeMonIT_bot_version_0.1_(http://www.eyemon.it/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)
Disallow: /
User-agent: Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)
Disallow: /
User-agent: Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; MojeekBot/0.10; +https://www.mojeek.com/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)
Disallow: /
User-agent: zoombot (linkbot 1.0 http://suite.seozoom.it/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)
Disallow: /
User-agent: pyspider/0.3.10 (+http://pyspider.org/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; [email protected])
Disallow: /
User-agent: AcademicBotRTU (https://academicbot.rtu.lv; mailto:[email protected])
Disallow: /
User-agent: MauiBot ([email protected])
Disallow: /
User-agent: The Knowledge AI
Disallow: /
User-agent: INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/; info at inetdex dot com)
Disallow: /
User-agent: Screaming Frog SEO Spider/16.6
Disallow: /
User-agent: Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)
Disallow: /
User-agent: ZoominfoBot (zoominfobot at zoominfo dot com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)
Disallow: /
User-agent: Screaming Frog SEO Spider/16.7
Disallow: /
User-agent: Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)
Disallow: /
User-agent: AwarioSmartBot/1.0 (+https://awario.com/bots.html; [email protected])
Disallow: /
User-agent: Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)
Disallow: /
User-agent: LegiStorm Bot (http://www.legistorm.com/legibot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; InfoTigerBot/1.9; +https://infotiger.com/bot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
Disallow: /
User-agent: CCBot/2.0 (https://commoncrawl.org/faq/)
Disallow: /
User-agent: Buck/2.3.2; (+https://app.hypefactors.com/media-monitoring/about.html)
Disallow: /
User-agent: Turnitin (https://bit.ly/2UvnfoQ)
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: AwarioSmartBot/1.0 (+https://awario.com/bots.html; [email protected])
Disallow: /
User-agent: AwarioRssBot/1.0 (+https://awario.com/bots.html; [email protected])
Disallow: /
User-agent: Mozilla/5.0 (compatible; oBot/2.3.1; http://www.xforce-security.com/crawler/)
Disallow: /
User-agent: Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)
Disallow: /
User-agent: Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)
Disallow: /
User-agent: Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers' presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: [email protected]
Disallow: /
User-agent: Screaming Frog SEO Spider/17.2
Disallow: /
User-agent: linabot
Disallow: /
User-agent: Scrapy/1.7.4 (+https://scrapy.org)
Disallow: /
User-agent: python-requests/2.28.2
Disallow: /
User-agent: Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)
Disallow: /
User-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0
Disallow: /
User-agent: Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)
Disallow: /
User-agent: Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; [email protected])
Disallow: /
User-agent: Screaming Frog SEO Spider/18.4
Disallow: /
User-agent: Screaming Frog SEO Spider/19.0
Disallow: /
User-agent: Mozilla/5.0 (compatible; SemrushBot; +http://www.semrush.com/bot.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Dataprovider.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)
Disallow: /
User-agent: Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Paqlebot/2.0; +http://www.paqle.dk/about/paqlebot)
Disallow: /
User-agent: Mediatoolkitbot ([email protected])
Disallow: /
User-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)
Disallow: /
User-agent: claudebot
Disallow: /
User-agent: Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)
Disallow: /
User-agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; [email protected])
Disallow: /
User-agent: Mozilla/5.0 (compatible; OpenindexSpider; +https://www.openindex.io/saas/about-our-spider/)
Disallow: /
User-agent: DomCopBot (https://www.domcop.com/bot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)
Disallow: /