fix: /text crawlable but noindex

2026-04-09 12:24:45 +01:00
parent af72c74799 cfb1817daa
commit 7d0c2179cd
1 changed files with 3 additions and 3 deletions
@@ -13,9 +13,9 @@ Disallow: /api/internal/
 Disallow: /.git/
 Disallow: /.env

-# Plain text version — prevent duplicate content indexing
-# Canonical HTML version at / should be indexed instead
-Disallow: /text
+# Plain text version — accessible but not indexed (X-Robots-Tag: noindex)
+# Canonical HTML version at / is preferred for search results
+# Allow: /text  (crawlable for LLMs and text browsers)

 # =============================================================================
 # SITEMAPS & AI CONTENT