diff --git a/static/robots.txt b/static/robots.txt index 43e6c22..4b33e28 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -13,9 +13,9 @@ Disallow: /api/internal/ Disallow: /.git/ Disallow: /.env -# Plain text version — prevent duplicate content indexing -# Canonical HTML version at / should be indexed instead -Disallow: /text +# Plain text version — accessible but not indexed (X-Robots-Tag: noindex) +# Canonical HTML version at / is preferred for search results +# Allow: /text (crawlable for LLMs and text browsers) # ============================================================================= # SITEMAPS & AI CONTENT