From cfb1817daa3e731d39889e6fa3613db95a46cb80 Mon Sep 17 00:00:00 2001
From: juanatsap <juanatsap@gmail.com>
Date: Thu, 9 Apr 2026 12:24:41 +0100
Subject: [PATCH] fix: keep /text crawlable for LLMs, only noindex for search
 engines

---
 static/robots.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/static/robots.txt b/static/robots.txt
index 43e6c22..4b33e28 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -13,9 +13,9 @@ Disallow: /api/internal/
 Disallow: /.git/
 Disallow: /.env
 
-# Plain text version — prevent duplicate content indexing
-# Canonical HTML version at / should be indexed instead
-Disallow: /text
+# Plain text version — accessible but not indexed (X-Robots-Tag: noindex)
+# Canonical HTML version at / is preferred for search results
+# Allow: /text  (crawlable for LLMs and text browsers)
 
 # =============================================================================
 # SITEMAPS & AI CONTENT