# Auth flows and internal previews — nothing here is indexable or useful to any crawler.
# Marketplace keyword search (?search=) is an open-ended URL space: kept out of the crawl here
# (Google: robots.txt, not noindex, for infinite spaces), and noindex in-page for anything that gets in.
# *search= also matches any future param ending in "search=" (e.g. research=) — robots.txt cannot
# anchor on ? or &, so never name a marketplace param that way.
User-agent: *
Disallow: /auth/
Disallow: /login
Disallow: /listings/post$
Disallow: /listings/post?
Disallow: /advertise/preview/
Disallow: /for-sale?*search=
Disallow: /for-sale/*?*search=
Disallow: /for-rent?*search=
Disallow: /for-rent/*?*search=

# Bulk AI training crawlers: clean pages only.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Amazonbot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: PerplexityBot
Disallow: /auth/
Disallow: /login
Disallow: /listings/post$
Disallow: /listings/post?
Disallow: /advertise/preview/
Disallow: /*?
Disallow: /subsales/*/roads=

Sitemap: https://list.my/sitemap.xml