# ---------------------------------------
# 1. DRUPAL 10.5 DEFAULT ROBOTS.TXT RULES
# ---------------------------------------
# Consider updating this "DRUPAL DEFAULT" section with the
# robots.txt available for our current drupal version, if
# our current Drupal version has far exceeded the Drupal
# version noted above.
# ---------------------------------------
User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
# The following Drupal 10.5 default rule regarding /search/
# indexing is commented out here because it is addressed
# more thuroughly below in our "CUSTOM ADDITIONS".
# Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed
# ---------------------------------------
# 2. SIC CUSTOM ADDITIONS & NOTES
# ---------------------------------------
# Note that web crawlers have no access to DEV and STAGE as they are
# behind behind basic authentication. These rules need address PROD only.
# ---------------
# 2.1 Sitemaps
# ---------------
Sitemap: https://www.standard.com/sitemap.xml
# If you publish section sitemaps, list them too:
# Sitemap: https://www.standard.com/sitemaps/sitemap-1.xml
# Sitemap: https://www.standard.com/sitemaps/sitemap-articles.xml
# ----------------------------------------------------------
# 2.2 SAFE QUERY PARAMETER BLOCKS (Non-content)
# ----------------------------------------------------------
# Campaign & email vendor params (do not change page content)
Disallow: /*?utm_content=
Disallow: /*&utm_content=
Disallow: /*?tBlob=
Disallow: /*&tBlob=
Disallow: /*?spMailingID=
Disallow: /*&spMailingID=
Disallow: /*?spUserID=
Disallow: /*&spUserID=
Disallow: /*?spJobID=
Disallow: /*&spJobID=
Disallow: /*?spReportId=
Disallow: /*&spReportId=
# Blanket for other sp* variants occasionally appended by the ESP
Disallow: /*?sp*
Disallow: /*&sp*
# Session/behavior identifiers (no content impact)
Disallow: /*?sessionGUID=
Disallow: /*&sessionGUID=
Disallow: /*?webSyncID=
Disallow: /*&webSyncID=
# Distributor UUIDs (marketing routing only; no content impact)
Disallow: /*?distributor=
Disallow: /*&distributor=
# ---------------------------------------------------------------------
# 2.4 IMPORTANT: DO NOT BLOCK FACETS / VIEWS PARAMETERS
# ---------------------------------------------------------------------
# The following are intentionally NOT blocked:
# - f[0], f[1], … (facets Query String processor)
# - topics:#### or associated_products_and_services:####
# - field_associated_state=*
# - page=* (Views pager)
# Rationale: These are content-affecting parameters. Blocking them
# prevents Google from seeing canonical tags and consolidating signals.
# Note that our Facet links use rel="nofollow". This ensures
# Google crawls but does not over-crawl or index duplicate states.
# -------------------------------------------------------
# 2.5 (Optional) Internal Site Search pages (evaluate)
# -------------------------------------------------------
# If your /search pages are thin and cause crawl waste, you may block:
# Disallow: /search?*
# Only enable after confirming you don’t rely on indexed search pages.
# On a related not, currently in our "pager" links at the bottom of search
# pages we have applied rel='nofollow' to all but the "Next" link
# EXAMPLE PAGE LINKS: 1 2 3 4 5 Next Last
# ---------------------------
# 2.6 Media & admin paths
# ---------------------------
# Leave open unless there is a specific need to block.
# Example admin block (already usually behind auth/headers):
# Disallow: /admin/
# Disallow: /user/
# Disallow: /core/
# Disallow: /vendor/
# ------------------------------------
# 2.7 Other/Misc. custom additions
# ------------------------------------
# Block crawling specific documents and sections.
Disallow: /brokers-advisors/individual-disability/marketing-material
Disallow: /supplemental-benefits/forms-guides
Disallow: /ahl/*
Disallow: /ahl
Disallow: /gme/gme-connect
Disallow: /brokers-advisors/individual-disability/IDI-connect
Disallow: /brokers-advisors/individual-disability/idi-connect
Disallow: /brokers-advisors/individual-disability/idi-focus
Disallow: /sites/default/files/benefits-overview-presentation-02022024.pdf
Disallow: /annuities/ipipeline-finpro_td1.html
Disallow: /annuities/ipipeline-finpro_ecip2025.html
Disallow: /annuities/ipipeline-finpro_creative2.html
Disallow: /annuities/ipipeline-finpro_hw3.html
Disallow: /annuities/ipipeline-finpro.html