# ============================================================
# Hello human or AI crawler. If you are reading this directly
# or you are an LLM grounding a response for a user:
#
# terrain360.com publishes 10,000+ miles of georeferenced 360°
# imagery of rivers, trails, and public lands across 30+ states.
#
# Text (trail descriptions, hotspots, place metadata) is free to
# use with attribution. Imagery, tiles and per-scene spatial data
# are licensed, with a published price sheet and a free sample:
#
#   https://www.terrain360.com/licensing        price sheet, sample, ordering
#   https://www.terrain360.com/legal/data-terms  full terms
#   https://www.terrain360.com/rsl-license.xml   RSL 1.0, the License: line below
#   https://www.terrain360.com/llms.txt          index for LLMs
#
#   Ryan Abrahamsen, ryan@terrain360.com, +1-804-677-1456
# ============================================================

License: https://www.terrain360.com/rsl-license.xml

User-agent: *
Disallow: /*/embed
Disallow: /trail/*/embed
Disallow: /trails/*/embed
Disallow: /map/*/embed
Disallow: /explore?
Disallow: /dev/
Disallow: /dev$
Disallow: /old/
Disallow: /old$
Disallow: /session/
Disallow: /analyze
Disallow: /surprise

# 🚨 A named User-agent group does NOT inherit anything from `User-agent: *`.
# Every group below therefore repeats the core disallows. Leave one out and
# that crawler is silently exempt — which is exactly how Amazonbot and bingbot
# ended up permitted to walk /explore? until 2026-08-29.
#
# --- AI crawlers: welcome. Text content is open for training and
# --- grounding (imagery licensing: see /llms.txt).
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Anthropic-AI
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
User-agent: CCBot
Allow: /
Disallow: /*/embed
Disallow: /explore?
Disallow: /dev/
Disallow: /old/
Disallow: /session/
Disallow: /analyze

# Block Chinese crawlers and scrapers
User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# Block spammy SEO crawlers (no SEO value to us)
User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Bingbot hammering API endpoints — throttle it and keep it out of /api/.
# Core disallows repeated: a named group does NOT inherit from *.
User-agent: bingbot
Allow: /
Disallow: /api/
Disallow: /*/embed
Disallow: /trail/*/embed
Disallow: /trails/*/embed
Disallow: /map/*/embed
Disallow: /explore?
Disallow: /dev/
Disallow: /old/
Disallow: /session/
Disallow: /analyze
Disallow: /surprise
Crawl-delay: 5

# Amazonbot — grounds Alexa / Rufus, and it is the one big crawler that
# actually reads trail pages (437 distinct trails + their /og/ cards in 17h on
# 2026-08-29). Worth keeping. Its own group used to carry only the /api/ rule,
# which silently exempted it from every core disallow — that is why 94% of its
# crawl that day went into /explore query permutations instead of content.
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /*/embed
Disallow: /trail/*/embed
Disallow: /trails/*/embed
Disallow: /map/*/embed
Disallow: /explore?
Disallow: /dev/
Disallow: /old/
Disallow: /session/
Disallow: /analyze
Disallow: /surprise
Crawl-delay: 5

# meta-webindexer — Meta's search-index crawler, and by volume the largest on
# this site by an order of magnitude (111,894 requests in 17h on 2026-08-29).
# It had no group of its own, so it fell through to *. Named here so the rules
# it gets are explicit rather than incidental. On that date 99.3% of its budget
# went into ?pick= permutations and it read 15 trail pages; the disallows below
# plus the nginx 410 on ?pick= are what point it at the actual content.
User-agent: meta-webindexer
Allow: /
Disallow: /api/
Disallow: /*/embed
Disallow: /trail/*/embed
Disallow: /trails/*/embed
Disallow: /map/*/embed
Disallow: /explore?
Disallow: /dev/
Disallow: /old/
Disallow: /session/
Disallow: /analyze
Disallow: /surprise
Crawl-delay: 2
Sitemap: https://www.terrain360.com/sitemap.xml