# robots.txt for letx.app
# https://letx.app
#
# Content Signals (https://contentsignals.org/) declare permission for:
#   search:    building a search index and providing search results
#   ai-input:  inputting content into AI models at request time (RAG, grounding,
#              live AI search answers)
#   ai-train:  training or fine-tuning AI models on collected content
#
# We allow search + ai-input (citations) but disallow ai-train. We want
# letx.app surfaced when users ask AI about LaTeX editors / Overleaf
# alternatives, but our content is not free training fodder.

# EVERY NAMED GROUP BELOW REPEATS THE SAME DISALLOW LIST, AND MUST. A crawler uses exactly one
# group, the most specific one that names it, and never merges it with the wildcard group
# (RFC 9309 section 2.2.1). Googlebot and Bingbot were therefore reading a group that said only
# "Allow: /" and were disallowed from nothing at all, including /api/ and /editor/. robots.txt has
# no include mechanism, so the list is copied rather than shared, and robots.test.ts fails the
# build if one copy drifts from another.

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# === Major search engines ===

User-agent: Googlebot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: Bingbot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: DuckDuckBot
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# Baidu, Yandex and Naver. Listed explicitly because the Chinese, Russian, Japanese and Korean
# catalogues exist and those four languages reach their readers through these engines rather than
# through Google. The groups are identical to the wildcard one and change nothing today; they are
# here so that the day a locale is enabled, the crawler that has to find it is already named.
# Verifying the property in each console is owner work: it needs a login we do not have.
User-agent: Baiduspider
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: YandexBot
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# Naver's crawler. The name is not a typo and is not related to Yandex.
User-agent: Yeti
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# === AI assistants, live citations (allowed) ===

User-agent: ChatGPT-User
Content-Signal: ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: OAI-SearchBot
Content-Signal: search=yes,ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# Validates ad landing pages. OpenAI states its data does not train foundation models, and we do
# not run ads, so this is here to be explicit rather than to grant anything.
User-agent: OAI-AdsBot
Content-Signal: ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: Claude-User
Content-Signal: ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: Claude-SearchBot
Content-Signal: search=yes,ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: PerplexityBot
Content-Signal: search=yes,ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: Perplexity-User
Content-Signal: ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: MistralAI-User
Content-Signal: ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: DuckAssistBot
Content-Signal: ai-input=yes
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# === AI training crawlers: citations OK, training NOT ===
# We let them crawl so they CAN cite us, but signal no-train.
# Bots that ignore Content-Signal and train anyway: revisit later.

User-agent: GPTBot
Content-Signal: ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: ClaudeBot
Content-Signal: ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: Google-Extended
Content-Signal: ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: Applebot
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

User-agent: Applebot-Extended
Content-Signal: ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# One group per agent. This file used to carry Meta-ExternalAgent twice, once allowed with a
# no-train signal and once disallowed outright as "meta-externalagent". User-agent matching is
# case-insensitive (RFC 9309), so those were two contradictory groups for the same crawler and
# which one applied was anybody's guess. Kept the allow, because the whole point of this file is
# that we want to be citable in AI answers and only object to being training data.
User-agent: Meta-ExternalAgent
Content-Signal: ai-input=yes,ai-train=no
Allow: /
Disallow: /editor/
Disallow: /projects
Disallow: /settings
Disallow: /tasks
Disallow: /admin/
Disallow: /auth/
Disallow: /authorize
Disallow: /api/

# === Aggressive scrapers and training-only: blocked ===

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

# Sitemap
Sitemap: https://letx.app/sitemap.xml