fix: keep /text crawlable for LLMs, only noindex for search engines

2026-04-09 12:24:41 +01:00
parent 08b39653ba
commit cfb1817daa
1 changed files with 3 additions and 3 deletions
@@ -13,9 +13,9 @@ Disallow: /api/internal/
 Disallow: /.git/
 Disallow: /.env

-# Plain text version — prevent duplicate content indexing
-# Canonical HTML version at / should be indexed instead
-Disallow: /text
+# Plain text version — accessible but not indexed (X-Robots-Tag: noindex)
+# Canonical HTML version at / is preferred for search results
+# Allow: /text  (crawlable for LLMs and text browsers)

 # =============================================================================
 # SITEMAPS & AI CONTENT