# TrailRhub. Search and answer engines are welcome; see /llms.txt for the guided tour.

# The engines that send buyers. Crawl freely — only the filter permutations are off-limits,
# and those are the same trailers in a different order.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: Google-InspectionTool
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: facebookexternalhit
User-agent: facebookcatalog
User-agent: meta-externalads
Disallow: /*?*filters
Disallow: /*?*price_min=
Disallow: /*?*price_max=
Disallow: /*?*sort=
Disallow: /*?*view=
Disallow: /*?*q=
Disallow: /*?*savedOnly=
Allow: /

# These take bandwidth and send nothing back.
User-agent: MJ12bot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: serpstatbot
User-agent: Barkrowler
User-agent: ZoominfoBot
User-agent: Diffbot
User-agent: magpie-crawler
User-agent: KeenableBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: PetalBot
User-agent: SeznamBot
User-agent: Bytespider
User-agent: Sogou web spider
User-agent: omgili
User-agent: omgilibot
User-agent: ImagesiftBot
User-agent: Timpibot
User-agent: Amazonbot
User-agent: meta-webindexer
Disallow: /

# Everyone we have not met: welcome, but at a walking pace.
User-agent: *
Crawl-delay: 10
Disallow: /*?*filters
Disallow: /*?*price_min=
Disallow: /*?*price_max=
Disallow: /*?*sort=
Disallow: /*?*view=
Disallow: /*?*q=
Disallow: /*?*savedOnly=
Allow: /

Sitemap: https://golden-eagle.trailrhub.com/sitemap.xml