User-agent: *
Allow: /

# Keep Google focused on public crawlable content. These routes either require
# auth, mutate data, or exist only for browser callbacks; indexing them creates
# noisy Search Console exclusions without adding public landing pages.
Disallow: /admin/
# Public read APIs advertised in /llms.txt + /openapi.json; robots-compliant AI agents must be able to call them.
Allow: /api/locations
Allow: /api/nearby
Allow: /api/place-media/
Allow: /api/place-audio/
# Agent surfaces advertised in /llms.txt, same carve-out logic as above.
Allow: /api/agent/markdown
Allow: /api/mcp
Allow: /api/a2a
# Wikimedia image proxy. Our image sitemap advertises these on-domain
# /api/wm-image?url=... URLs as <image:loc>, so blocking the path made the
# sitemap self-contradictory and produced growing "Blocked by robots.txt"
# exclusions in Search Console. The proxy is SSRF-hardened (only
# upload/commons.wikimedia.org, raster content-types, redirect-capped,
# private-IP blocked) and 7-day edge-cached, so allowing it makes the
# Wikimedia photos crawlable for Google Images without opening a real proxy.
# Longest-match wins, so this Allow beats the broad /api/ disallow below.
Allow: /api/wm-image
Disallow: /api/
Disallow: /auth/
Disallow: /tr/login
Disallow: /en/login

# AI crawlers: all welcome, citation and training alike, via the "*" group.
#
# This is a decision, not an omission (owner, 2026-08-10). The corpus is CC BY
# and published in bulk on /acik-veri, so a training block
# here protected nothing: anyone could download the data without crawling a
# single page. Worse, it contradicted our own licence: CC BY permits derivative
# use, and /ai.txt declares ai-train=yes to match. If this policy is ever
# reversed, reverse ai.txt in the same commit; the robots test enforces that
# the two files cannot disagree.
#
# CAUTION for future edits: every AI bot (OAI-SearchBot, ChatGPT-User,
# PerplexityBot, Claude-SearchBot, GPTBot, ...) matches the "*" group above.
# A bot that gets its own "User-agent:" group STOPS reading "*" entirely and
# loses the /api/ carve-outs with it, and tightening "*" tightens every AI
# citation surface at once. Measured before deciding: fully-allowed
# Claude-SearchBot crawled 35,051 paths and referred zero visitors, so expect
# no traffic change from this either way.

Sitemap: https://kahvetabela.com/sitemap.xml
# TEMPORARY until 2026-12-31: pages that went noindex in September 2026, listed
# so crawlers revisit them and see the directive. Not part of /sitemap.xml,
# which lists indexable pages only. See src/lib/seo/recrawl-sitemap.ts.
Sitemap: https://kahvetabela.com/sitemap-recrawl.xml