from mac

2025-10-31 11:06:38 +00:00
parent 5d2f763d2e
commit a5804936ba
16 changed files with 3096 additions and 283 deletions
@@ -0,0 +1,36 @@
+# robots.txt for juan.andres.morenoyrubio.com
+
+# Allow all search engines
+User-agent: *
+Allow: /
+
+# Disallow admin/internal paths (if any in future)
+Disallow: /admin/
+Disallow: /api/internal/
+Disallow: /.git/
+Disallow: /.env
+
+# Sitemap location
+Sitemap: https://juan.andres.morenoyrubio.com/static/sitemap.xml
+
+# Crawl-delay (optional, helps prevent server overload)
+# Crawl-delay: 1
+
+# Allow specific search engines (explicit)
+User-agent: Googlebot
+Allow: /
+
+User-agent: Bingbot
+Allow: /
+
+User-agent: Slurp
+Allow: /
+
+User-agent: DuckDuckBot
+Allow: /
+
+User-agent: Baiduspider
+Allow: /
+
+User-agent: YandexBot
+Allow: /