# AI robots disallowed
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: anthropic-ai
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Crawlspace
User-agent: Diffbot
User-agent: FriendlyCrawler
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: Meta-ExternalAgent
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: PanguBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: meta-webindexer
User-agent: meta-externalads
User-agent: meta-externalagent
User-agent: GPTBot
User-agent: CCBot
Disallow: /
# Same rules as general bots, except that these bots:
# can crawl /recipes and /products translated pages
# can crawl some query params
User-agent: Mediapartners-Google
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
User-agent: Google-Display-Ads-Bot
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /*/admin/
Disallow: /comment/reply/
Disallow: /*/comment/reply/
Disallow: /filter/tips
Disallow: /*/filter/tips
Disallow: /node/add/
Disallow: /*/node/add/
Disallow: /user/register
Disallow: /*/user/register
Disallow: /user/register-nd
Disallow: /*/user/register-nd
Disallow: /user/password
Disallow: /*/user/password
Disallow: /user/logout
Disallow: /*/user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed
# Query params
Allow: /*?itok=*
Allow: /*?*&itok=*
Disallow: *?page=*
Disallow: *?*&page=*
Disallow: */admin/*
Disallow: */comment/reply/*
Disallow: */filter/tips
Disallow: */node/add/*
Disallow: */user/password
Disallow: */user/logout
Disallow: */node/*/ajax-comments
Disallow: */help-center/symptom-check/*
Disallow: */node/*
User-agent: *
Crawl-delay: 1
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /*/admin/
Disallow: /comment/reply/
Disallow: /*/comment/reply/
Disallow: /filter/tips
Disallow: /*/filter/tips
Disallow: /node/add/
Disallow: /*/node/add/
Disallow: /user/register
Disallow: /*/user/register
Disallow: /user/register-nd
Disallow: /*/user/register-nd
Disallow: /user/password
Disallow: /*/user/password
Disallow: /user/login
Disallow: /*/user/login
Disallow: /user/logout
Disallow: /*/user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed
# Query params
Allow: /*?itok=*
Allow: /*?*&itok=*
Disallow: /*?
# General search
Disallow: */search/
Disallow: */search?
Disallow: */admin/*
Disallow: */comment/reply/*
Disallow: */filter/tips
Disallow: */node/add/*
Disallow: */user/password
Disallow: */user/logout
Disallow: */node/*/ajax-comments
Disallow: */help-center/symptom-check/*
Disallow: */node/*
# URLs with search components
Disallow: /en-int/products$
Disallow: /en-uk/products$
Disallow: /en-us/products$
Disallow: /en-ca/products$
Disallow: /de-de/produkte$
Disallow: /de-ch/produkte$
Disallow: /de-at/produkte$
Disallow: /it-it/prodotti$
Disallow: /fr-fr/produits$
Disallow: /fr-be/produits$
Disallow: /es-es/productos$
Disallow: /es-int/productos$
Disallow: /es-ar/productos$
Disallow: /es-mx/productos$
Disallow: /nl-nl/producten$
Disallow: /nl-be/producten$
Disallow: /sv-se/produkter$
Disallow: /nb-no/produkter$
Disallow: /da-dk/produkter$
Disallow: /fi-fi/tuotteet$
Disallow: /pl-pl/produkty$
Disallow: /cs-cz/produkty$
Disallow: /sk-sk/produkty$
Disallow: /hu-hu/termekek$
Disallow: /ru-ru/produkty$
Disallow: /pt-br/produtos$
Disallow: /pt-pt/produtos$
Disallow: /tr-tr/urunler$
Disallow: /ro-ro/produse$
Disallow: /el-gr/proionta$
Disallow: /sl-si/izdelki$
Disallow: /et-ee/tooted$
Disallow: /hr-hr/proizvodi$
Disallow: /en-int/recipes$
Disallow: /en-uk/recipes$
Disallow: /en-us/recipes$
Disallow: /en-ca/recipes$
Disallow: /de-de/rezepte$
Disallow: /de-ch/rezepte$
Disallow: /de-at/rezepte$
Disallow: /it-it/ricette$
Disallow: /fr-fr/recipes$
Disallow: /fr-be/recipes$
Disallow: /es-es/recetas$
Disallow: /es-int/recetas$
Disallow: /es-ar/recipes$
Disallow: /es-mx/recipes$
Disallow: /nl-nl/recepten$
Disallow: /nl-be/recipes$
Disallow: /sv-se/recept$
Disallow: /nb-no/oppskrifter$
Disallow: /da-dk/opskrifter$
Disallow: /fi-fi/reseptit$
Disallow: /pl-pl/przepisy$
Disallow: /cs-cz/recepty$
Disallow: /sk-sk/recepty$
Disallow: /hu-hu/receptek$
Disallow: /ru-ru/recepty$
Disallow: /pt-br/receitas$
Disallow: /pt-pt/receitas$
Disallow: /tr-tr/tarifler$
Disallow: /ro-ro/retete$
Disallow: /el-gr/syntages$
Disallow: /sl-si/recepti$
Disallow: /et-ee/retseptid$
Disallow: /hr-hr/recepti$
Sitemap: https://www.schaer.com/sitemap.xml