# robots.txt for livspace.com
# Source-of-truth: livspaceeng/{beta,prod}-deployments sales/{beta,prod}/pixel/falcon/configs/robots.txt
# Mounted into falcon's Next.js public/ via the falcon-public-files configMap.
# Previously served by bumblebee; migrated to falcon as part of ENG-20507.
#
# Policy:
# - Only the /in/ region is active. /sg/, /my/, /sa/, /sa-ar/ are deprecated
# and disallowed for crawl.
# - All major AI-assistant crawlers are welcomed for /in/ content. Per the
# robots.txt spec, a bot without its own User-agent block falls back to `*`,
# so the single `*` group below covers them all. Bots explicitly welcomed:
# Googlebot, Google-Extended, GPTBot, OAI-SearchBot, ChatGPT-User,
# ClaudeBot, Claude-User, Claude-SearchBot, PerplexityBot, bingbot,
# Applebot, Applebot-Extended, CCBot, Meta-ExternalAgent, Amazonbot.
# - Google-Extended is intentionally allowed. It only controls Gemini training
# and grounding for Gemini Apps / Vertex AI. It is NOT a Google Search
# ranking signal and blocking it does NOT remove the site from AI Overviews
# or AI Mode.
# Ref: developers.google.com/search/blog/2023/09/an-update-on-web-publisher-controls
# - Bytespider has its own group to throttle it (aggressive crawler).
#
# robots.txt semantics: a crawler uses the FIRST, MOST SPECIFIC matching
# User-agent group only; groups are NOT inherited. If you add a bot-specific
# block later, restate the Disallow rules — that bot will ignore `*`.
# This is not a security mechanism; for true exclusion use noindex or auth.
# Ref: developers.google.com/search/docs/crawling-indexing/robots/intro
# =============================================================================
# Default group — applies to all crawlers without a more specific block.
# Covers every welcomed AI assistant bot.
# =============================================================================
User-agent: *
Allow: /
# Explicit allows for AI-discovery files (defensive — future Disallow rules
# can't accidentally shadow these because longer match wins).
Allow: /llms.txt
Allow: /llms-full.txt
# Deprecated regions (active only on /in/)
Disallow: /sg/
Disallow: /my/
Disallow: /sa/
Disallow: /sa-ar/
# Duplicate / parameterized URLs
Disallow: /*tag=
Disallow: /*id_currency=
Disallow: /*id_lang=
Disallow: /*back=
Disallow: /*n=
# Auth, account, and utility paths
Disallow: /sign-in
Disallow: /reset-password
Disallow: /forget-password
Disallow: /user/change-password
Disallow: /user/send-otp
Disallow: /to/j8kIPb5e
Disallow: /oembed/
Disallow: /swinstall
# Internal / removed content
Disallow: /in/leadership
Disallow: /in/culture
Disallow: /in/about.htm
Disallow: /in/project.php
Disallow: /project.php
Disallow: /producta
Disallow: /index.html
Disallow: /projects.html
# Referral & profile flows (PII-adjacent, low SEO value)
Disallow: /in/interiors/referral/*
Disallow: /in/profile
Disallow: /in/profile/*
# Non-prod environment leaks
Disallow: /*livspace-web.dev.livspace.com
Disallow: /*livspace-web.alpha.livspace.com
Disallow: /*livspace-web.beta.livspace.com
Disallow: /*services.livspace.com
# Deprecated AMP variants
Disallow: /*/amp$
Disallow: /*/amp/
# =============================================================================
# ByteDance Bytespider — allow but throttle (aggressive crawler).
# This is the only crawler whose rules differ from `*`. Per the spec, groups
# are not inherited, so the path rules must be restated here.
# =============================================================================
User-agent: Bytespider
Crawl-delay: 10
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /sg/
Disallow: /my/
Disallow: /sa/
Disallow: /sa-ar/
Disallow: /*tag=
Disallow: /*id_currency=
Disallow: /*id_lang=
Disallow: /*back=
Disallow: /*n=
Disallow: /sign-in
Disallow: /reset-password
Disallow: /forget-password
Disallow: /user/change-password
Disallow: /user/send-otp
Disallow: /to/j8kIPb5e
Disallow: /oembed/
Disallow: /swinstall
Disallow: /in/leadership
Disallow: /in/culture
Disallow: /in/about.htm
Disallow: /in/project.php
Disallow: /project.php
Disallow: /producta
Disallow: /index.html
Disallow: /projects.html
Disallow: /in/interiors/referral/*
Disallow: /in/profile
Disallow: /in/profile/*
Disallow: /*livspace-web.dev.livspace.com
Disallow: /*livspace-web.alpha.livspace.com
Disallow: /*livspace-web.beta.livspace.com
Disallow: /*services.livspace.com
Disallow: /*/amp$
Disallow: /*/amp/
# =============================================================================
# Sitemaps — only the active /in/ region
# =============================================================================
Sitemap: https://www.livspace.com/in/sitemapIndex.xml