# START AI-FRIENDLY ROBOTS.TXT FOR WORDPRESS
# --- Standard Rules for All Crawlers ---
# These rules block crawlers from indexing non-public parts of your WordPress site,
# such as the admin area, core files, and plugins. This helps prevent sensitive
# information from appearing in search results.
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
# Explicitly disallow the login page to prevent it from being indexed.
Disallow: /wp-login.php
# Allow AJAX functionality for all crawlers, as it can be important for rendering content.
Allow: /wp-admin/admin-ajax.php
# --- AI Crawler Permissions ---
# Explicitly allow major AI crawlers to ensure they can access your content.
# This helps your site be included in their training data and search results.
# An empty "Disallow:" line means "allow everything" for that specific user-agent.
# Google AI (Gemini, etc.)
User-agent: Google-Extended
Disallow:
# OpenAI (ChatGPT)
User-agent: GPTBot
Disallow:
# Perplexity AI
User-agent: PerplexityBot
Disallow:
# Common Crawl (used by many AI models)
User-agent: CCBot
Disallow:
# Cohere AI
User-agent: cohere-ai
Disallow:
# --- Sitemap ---
# Provide the path to your sitemap. This is crucial for all search engines.
# Replace "your-website.com" with your actual domain.
# If you use an SEO plugin like Yoast or Rank Math, the URL might be different (e.g., sitemap_index.xml).
Sitemap: https://42law.com/sitemap_index.xml
# END ROBOTS.TXT