# Robots.txt for Us5.com (https://us5.com)
# Free online calculators and tools for math, finance, health, and daily use
# Updated: 2025-07-23
# === Universal Rules for All Search Engines ===
User-agent: *
Allow: /
# Explicitly allow important directories
Allow: /tools/
Allow: /calculators-and-tools
# Disallow technical and private directories
Disallow: /admin/
Disallow: /api/internal/
Disallow: /private/
Disallow: /_next/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.well-known/
Disallow: /scripts/
# Disallow sensitive file types
Disallow: /*.json$
Disallow: /*.log$
Disallow: /*.env$
Disallow: /*.config.js$
Disallow: /*.map$
# === SEO Tools with Rate Limiting ===
# Allow SEO tools but with crawl delay to reduce server load
User-agent: SemrushBot
Allow: /
Crawl-delay: 10
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10
# === Block Harmful/Aggressive Crawlers ===
User-agent: MJ12bot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: DotBot
Disallow: /
# === Sitemap Declaration ===
# Primary index sitemap
Sitemap: https://us5.com/sitemap-index.xml
# Main site sitemap (categories, tools, static pages)
Sitemap: https://us5.com/sitemap.xml
# Blog posts sitemap
Sitemap: https://us5.com/blog-sitemap.xml
# Guides sitemap
Sitemap: https://us5.com/guides-sitemap.xml
# === Additional Information ===
# Primary domain: https://us5.com
# www.us5.com redirects to us5.com (301 redirect for SEO)
# All subdomains redirect to main domain for consolidated SEO authority
#
# This website provides free online calculators and tools
# We welcome legitimate search engine crawling
# Server optimized for high-performance crawling
#
# For questions or concerns, contact: [email protected]