Sitemap: https://www.listingsproject.com/sitemap.xml
User-agent: Twitterbot
Disallow:
# AI model-training crawlers.
#
# These bots bulk-collect content to train LLMs. They MAY train on our
# editorial, marketing, and support pages (home, blog/resources, about,
# how-we-work, press, community agreements, etc.) but NOT on user-generated
# listings or our legal policies.
#
# Note: live answer-engine crawlers (e.g. ChatGPT-User, OAI-SearchBot,
# PerplexityBot, Perplexity-User, Claude-User) are intentionally NOT listed
# here. They fall through to the "User-agent: *" rules below and may fetch any
# public page to answer a user's question and cite us. Search crawlers
# (Googlebot, Bingbot, Applebot) are likewise unaffected — Google-Extended and
# Applebot-Extended are the training-only tokens and do not control search.
User-agent: GPTBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
Disallow: /manage/
Disallow: /new_admin/
Disallow: /account/
Disallow: /listings
Disallow: /real-estate
Disallow: /jobs-and-gigs
Disallow: /services
Disallow: /terms-of-use
Disallow: /privacy-policy
Disallow: /refund-policy
Disallow: /membership-terms-and-conditions
Disallow: /cookies
User-agent: *
Disallow: /manage/
Disallow: /new_admin/
Disallow: /account/