# robots.txt for https://wd.bible/
# ----------------------------------------------------------------------
# Last updated: 2026-05-12
# Owner: SEO / Marketing — Dawn Zhang ([email protected])
# Version: v1.1
# Companion: See robots-txt-companion-doc.md for design rationale,
# per-rule justification, and engineering follow-ups.
#
# IMPORTANT: This file controls CRAWLING, not indexing. To remove an
# already-indexed page from Google's index, add a tag to a CRAWLABLE page — do NOT use Disallow,
# because Google can't see a noindex tag on a Disallow'd page.
# ----------------------------------------------------------------------
# ======================================================================
# DEFAULT — applies to all compliant crawlers unless overridden below
# ======================================================================
User-agent: *
# --- Critical: explicitly allow render-required assets ---
# (Googlebot must fetch CSS/JS/images to evaluate page quality.)
Allow: /_next/
Allow: /assets/
Allow: /wd/
# --- Internal search (all locales) ---
# 50K+ views/90d on /search alone. Infinite URL space, no SEO value.
# The leading `*` matches any locale prefix (/, /en/, /tw/, /hk/).
Disallow: /search
Disallow: */search
Disallow: /*?q=
Disallow: /*?s=
# --- Authenticated / user-specific paths (all locales) ---
# Private user data; no SEO value; many include UUIDs in the URL.
Disallow: /account/
Disallow: */account/
Disallow: /plan/mine
Disallow: */plan/mine
Disallow: /notes
Disallow: */notes
Disallow: /highlights
Disallow: */highlights
Disallow: /favorites
Disallow: */favorites
Disallow: /playlist
Disallow: */playlist
Disallow: /user/
Disallow: */user/
Disallow: /migration
Disallow: */migration
Disallow: /tracker
Disallow: */tracker
Disallow: /book/
Disallow: */book/
# --- System / error / API paths (all locales) ---
Disallow: /crash # error page (42K views/90d — should never be indexed)
Disallow: */crash
Disallow: /404
Disallow: */404
Disallow: /500
Disallow: */500
Disallow: /api/ # RPC endpoints (e.g., /api/getUserInfo)
Disallow: */api/
Disallow: /link # internal redirect handler
Disallow: */link
Disallow: /record # cookie-setting redirector to /bible/*
Disallow: */record
Disallow: /browser-not-supported
Disallow: */browser-not-supported
Disallow: /help-mobile
Disallow: */help-mobile
Disallow: /cdn-cgi/ # Cloudflare-internal paths leaking to GA
Disallow: */cdn-cgi/
# --- Universal tracking parameters ---
# Standard exception per Google's URL-parameter guidance: these create
# duplicates of canonical pages for every shared/campaigned URL.
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?msclkid=
Disallow: /*?mc_cid=
Disallow: /*?mc_eid=
Disallow: /*?ref=
Disallow: /*?source=
# --- Open-redirect / security parameter ---
# /?url= forwards to external destinations. Should not be in the index.
Disallow: /*?url=
# --- Bytedance / in-app integration parameters ---
# Used only by the embedded mobile webview (TikTok in-app). Public web
# crawlers should never index these variants.
Disallow: /*?use_xbridge3=
Disallow: /*?loader_name=
Disallow: /*?need_sec_link=
Disallow: /*?sec_link_scene=
# --- Internal account integration ---
Disallow: /*?use-olympus-account=
# NOTE on UI-state parameters that ARE NOT blocked here:
# ?parallel=, ?theme=, ?fullscreen=, ?feature=, ?open_versions=,
# ?categoryId=, ?albumId=, ?language=, ?lang=, ?month=, ?type=,
# ?scene=, ?t=, ?returnUrl=
# These create soft duplicates but the proper fix is rel="canonical"
# tags pointing to the clean URL — handled in engineering follow-up,
# not here. (Disallowing them in robots.txt would prevent Google from
# crawling some marketing landing pages that are only visited with the
# parameter present, e.g. /nid/intro?theme=light.)
# ======================================================================
# AI CRAWLERS — explicit per-bot policy
# ----------------------------------------------------------------------
# Posture: ALLOW search/answer-engine retrieval bots (these can drive
# citations and traffic to scripture pages). DISALLOW training-only
# bots — IP stewardship for translation and study-note content.
# ======================================================================
# --- Allowed: search and answer-engine retrieval ---
User-agent: OAI-SearchBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: GoogleOther
Allow: /
# --- Disallowed: training-only crawlers ---
User-agent: GPTBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: FacebookBot
Disallow: /
# ======================================================================
# Sitemap declaration
# (Always use absolute URL. This is the only "positive" signal robots.txt
# can carry — it points crawlers at the canonical pages we want indexed.)
# ======================================================================
Sitemap: https://wd.bible/sitemap.xml