# Losko — content policy for AI/LLM crawlers.
# We OPT OUT of model-*training* crawlers (they ingest our authored articles
# into training datasets) but stay OPT-IN for AI *search/answer* crawlers (they
# cite losko.ru with a link — that's traffic, not appropriation). robots.txt is
# advisory: OpenAI, Anthropic, Google & Yandex state they honour it; ByteDance
# (Bytespider) and Perplexity have been caught ignoring it, so treat this as a
# declared boundary, not a hard wall. To flip a bot, move it between the blocks.

# — Model TRAINING crawlers: BLOCKED (do not feed our content to model training) —
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bytespider
User-agent: CCBot
Disallow: /

# — AI SEARCH / ANSWER / CITATION crawlers: ALLOWED (they surface & link to us) —
# Includes user-triggered fetches (a reader asks an assistant to open a losko.ru
# page) and Yandex's neuro/answer crawler (primary GEO channel for the RU site).
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: YandexAdditional
Allow: /

# — Everyone else (classic search engines: Googlebot, YandexBot, Bingbot…) —
# Google-Extended above opts out of Gemini training WITHOUT affecting Googlebot,
# so normal Google/Yandex search ranking is untouched.
#
# The Disallow block: dead WordPress-era URL trees, kept blocked so Webmaster's
# remove-by-prefix tool stays usable (it requires the prefix to be robots-
# blocked) and to stop re-crawl of paths Yandex had re-indexed as duplicates.
# The RU host (Timeweb) now serves real 404s for these, and on loskomag.com they
# 404 too, so the rules are belt-and-suspenders — the live 404 does the real work.
User-agent: *
Allow: /
Disallow: /tag/
Disallow: /page/
Disallow: /category/article/
Disallow: /wp-content/
# The private «Фильм недели» archive — password-protected (server.mjs) and out
# of the sitemap. Crawlers get a 401 anyway; this keeps them from trying.
Disallow: /archive/

# Old WP internal-search URLs (/?s=query) all render the homepage, which self-
# canonicals to "/". Clean-param (Yandex-only; others ignore it) folds the `s`
# param away so Yandex consolidates these onto the homepage AND carries their
# accumulated signals across. We deliberately DON'T `Disallow: /*?s=`: per Yandex
# docs Disallow OUTRANKS Clean-param, so pairing them suppresses the consolidation
# and leaves these stuck in the "GET-param duplicates" report (re-flagged
# 2026-07-20). Google has no Clean-param but honours the homepage's canonical.
Clean-param: s

Sitemap: https://losko.ru/sitemap-index.xml
# English edition (loskomag.com) — its own sitemap (honoured on the loskomag
# host; also submit it as a separate property in Search Console).
Sitemap: https://loskomag.com/en/sitemap.xml