# =============================================
# General search engine rules
# =============================================
User-agent: *
Disallow: /getthedocs/files/
Disallow: /services/
Disallow: /search/*
Disallow: */draft/*
Disallow: */singlehtml/*
# =============================================
# AI-specific access rules for large language models
# Equal access for all major AI bots
# =============================================
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: PerplexityBot
User-agent: AppleBot
User-agent: PhindBot
User-agent: YouBot
User-agent: facebookexternalhit
# TYPO3 Core manuals (LTS, eLTS, dev)
Allow: /m/typo3/*/13.4/en-us/
Allow: /m/typo3/*/13.4/en-us/singlehtml/
Allow: /m/typo3/*/12.4/en-us/
Allow: /m/typo3/*/12.4/en-us/singlehtml/
Allow: /m/typo3/*/11.5/en-us/
Allow: /m/typo3/*/11.5/en-us/singlehtml/
Allow: /m/typo3/*/10.4/en-us/
Allow: /m/typo3/*/10.4/en-us/singlehtml/
Allow: /m/typo3/*/main/en-us/
Allow: /m/typo3/*/main/en-us/singlehtml/
# System extensions
Allow: /c/typo3/*/13.4/en-us/
Allow: /c/typo3/*/13.4/en-us/singlehtml/
Allow: /c/typo3/*/12.4/en-us/
Allow: /c/typo3/*/12.4/en-us/singlehtml/
Allow: /c/typo3/*/main/en-us/
Allow: /c/typo3/*/main/en-us/singlehtml/
# Official TYPO3 libraries (e.g. Fluid)
Allow: /other/typo3fluid/*/main/en-us/
Allow: /other/typo3fluid/*/main/en-us/singlehtml/
Allow: /other/typo3fluid/*/2.*/en-us/
Allow: /other/typo3fluid/*/2.*/en-us/singlehtml/
# Other official manuals
Allow: /other/typo3/*/13.4/en-us/
Allow: /other/typo3/*/13.4/en-us/singlehtml/
Allow: /other/typo3/*/12.4/en-us/
Allow: /other/typo3/*/12.4/en-us/singlehtml/
Allow: /other/typo3/*/main/en-us/
Allow: /other/typo3/*/main/en-us/singlehtml/
# Third-party extensions
Allow: /p/*/*/main/en-us/
Allow: /p/*/*/main/en-us/singlehtml/
Allow: /p/*/*/10.*/en-us/
Allow: /p/*/*/10.*/en-us/singlehtml/
Allow: /p/*/*/11.*/en-us/
Allow: /p/*/*/11.*/en-us/singlehtml/
# Disallow internal or irrelevant areas
Disallow: /search/
Disallow: /services/
Disallow: /getthedocs/files/
Disallow: */draft/*