# We do not distinguish between the browsers.
User-agent: *
# Only allow URLs generated with frontend routing
Disallow: /*?id=*
Disallow: /*&id=*
# L=0 is the default language
Disallow: /*?L=0*
Disallow: /*&L=0*
# Should always be protected, but you know...
Disallow: /*/Private/*
Disallow: /*/Configuration/*
# Disallow all files in /typo3temp/var/
Disallow: /typo3temp/var/*
# Disallow all files in /typo3/
Disallow: /typo3/
# Disallow all kind of sql files
Disallow: *.sql
Disallow: *.sql.gz
User-agent: Scrapy
Disallow: /
User-agent: Scrapy/2.11.2
Disallow: /
User-agent: Scrapy/2.11.2 (+https://scrapy.org)
Disallow: /
User-agent: facebookexternalhit
Disallow: /
User-agent: facebookexternalhit/1.1
Disallow: /
User-agent: Facebot
Disallow: /