# Per-agent groups completely REPLACE the "*" group for the agent they name -
# they do not layer on top of it. So a named group that omits Disallow: /api/
# is an invitation to crawl the API, which is what every group below used to
# be. Each group now carries its own copy. If you add an agent, copy the pair,
# not just the Allow.
#
# Googlebot, Bingbot and Applebot are deliberately allowed into /api/ and the
# other groups are not. The dashboard is a JavaScript app with no content until
# it fetches its own API, so a search renderer that cannot reach /api/ indexes
# an empty shell. Keeping the JSON out of the index is a separate job, done with
# X-Robots-Tag: noindex on the /api/ location in nginx.conf: robots.txt governs
# crawling, not indexing, and using it for the second job costs the first.
# /api/v1/layers/ and /api/v1/geo/ stay blocked for everyone: multi-megabyte
# GeoJSON with nothing in it a search result could show.
#
# An agent with no rendering engine gains nothing from /api/ and would only be
# copying the data wholesale, so those groups keep the blanket block. The one
# exception is /api/v1/openapi.json, which every group admits: llms.txt tells
# agents to read it, and a site that forbids the document it points them at is
# telling them two different things. It is one description of the API, not the
# data, so it is not the wholesale copying the block is for. A longer path wins
# over a shorter one in every parser that implements the standard, so the Allow
# beats the Disallow below it whatever the order.

User-agent: Googlebot
Allow: /
Disallow: /api/v1/layers/
Disallow: /api/v1/geo/
Disallow: /admin
Disallow: /analyst

User-agent: Bingbot
Allow: /
Disallow: /api/v1/layers/
Disallow: /api/v1/geo/
Disallow: /admin
Disallow: /analyst

User-agent: GPTBot
Allow: /
Allow: /api/v1/openapi.json
Disallow: /api/
Disallow: /admin
Disallow: /analyst

User-agent: Claude-claudebot
Allow: /
Allow: /api/v1/openapi.json
Disallow: /api/
Disallow: /admin
Disallow: /analyst

User-agent: PerplexityBot
Allow: /
Allow: /api/v1/openapi.json
Disallow: /api/
Disallow: /admin
Disallow: /analyst

User-agent: CCBot
Allow: /
Allow: /api/v1/openapi.json
Disallow: /api/
Disallow: /admin
Disallow: /analyst

User-agent: meta-externalagent
Allow: /
Allow: /api/v1/openapi.json
Disallow: /api/
Disallow: /admin
Disallow: /analyst

User-agent: Applebot
Allow: /
Disallow: /api/v1/layers/
Disallow: /api/v1/geo/
Disallow: /admin
Disallow: /analyst

User-agent: anthropic-ai
Allow: /
Allow: /api/v1/openapi.json
Disallow: /api/
Disallow: /admin
Disallow: /analyst

User-agent: *
Allow: /
Allow: /api/v1/openapi.json
# No value in indexing live API responses, and none at all in indexing the
# admin login page.
Disallow: /api/
Disallow: /admin
Disallow: /analyst

Sitemap: https://rift.news/sitemap.xml
# Generated by the backend from the fault and volcano catalogues, so it is never
# out of date with the pages that exist. Both lines are needed: a second Sitemap
# directive adds to the first, it does not replace it.
Sitemap: https://rift.news/sitemap-systems.xml