Domain Name: GRANDTHEFTWIKI.COM
Registrar: NameCheap, Inc.
Domain Status: client transfer prohibited
Creation Date: 2006-07-29T13:43:29Z
Registry Expiry Date: 2027-07-29T13:43:29Z
Updated Date: 2026-06-29T07:51:35Z
Name Server: MARY.NS.CLOUDFLARE.COM
Name Server: PETE.NS.CLOUDFLARE.COM
REGISTRAR Contact: NameCheap, Inc.
>>> Last update of RDAP database: 2026-07-16T19:45:34Z
# Grand Theft Wiki - robots.txt
# Bot crawl control to prevent database overload
#
# This file is copied to the container's document root during Docker build.
# It tells search engines and bots how to crawl the site.
#
# Key settings:
# - 10 second crawl delay for most bots
# - 5 second delay for Google/Yandex
# - Block expensive operations (edit, history, API)
# - Block special pages
# - Aggressive bots get 30 second delay or full block
User-agent: *
# Crawl rate limits
Crawl-delay: 10
# Block expensive operations
Disallow: /api.php
Disallow: /index.php?
Disallow: /*action=edit
Disallow: /*action=history
Disallow: /*action=delete
Disallow: /*action=watch
Disallow: /*diff=
Disallow: /*oldid=
Disallow: /*printable=yes
Disallow: /Special:Search
Disallow: /Special:Random
Disallow: /Special:RecentChanges
Disallow: /Special:UserLogin
Disallow: /Special:UserLogout
# Allow important pages for SEO
Allow: /index.php?title=
Allow: /$
Allow: /Main_Page
# Specific aggressive bots - even stricter
User-agent: AhrefsBot
Crawl-delay: 30
Disallow: /
User-agent: SemrushBot
Crawl-delay: 30
Disallow: /
User-agent: DotBot
Crawl-delay: 30
Disallow: /
User-agent: MJ12bot
Crawl-delay: 30
Disallow: /
User-agent: BLEXBot
Crawl-delay: 30
Disallow: /
# Googlebot - allow but slower
User-agent: Googlebot
Crawl-delay: 5
Disallow: /*action=edit
Disallow: /*action=history
Disallow: /Special:
# YandexBot - allow but slower
User-agent: YandexBot
Crawl-delay: 5
Disallow: /*action=edit
Disallow: /*action=history
Disallow: /Special:
# Sitemap
Sitemap: https://www.grandtheftwiki.com/sitemap.xml