juanatsap
08b39653ba
- Add X-Robots-Tag: noindex, nofollow header to /text endpoint - Add Link: canonical header pointing to HTML version - Add Disallow: /text to robots.txt - Update sitemap.xml lastmod to 2026-04-09
121 lines
2.5 KiB
Plaintext
121 lines
2.5 KiB
Plaintext
# robots.txt for juan.andres.morenorub.io
|
|
# Last Updated: 2025-11-30
|
|
|
|
# =============================================================================
|
|
# DEFAULT RULES - Allow all search engines
|
|
# =============================================================================
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# Disallow admin/internal paths
|
|
Disallow: /admin/
|
|
Disallow: /api/internal/
|
|
Disallow: /.git/
|
|
Disallow: /.env
|
|
|
|
# Plain text version — prevent duplicate content indexing
|
|
# Canonical HTML version at / should be indexed instead
|
|
Disallow: /text
|
|
|
|
# =============================================================================
|
|
# SITEMAPS & AI CONTENT
|
|
# =============================================================================
|
|
Sitemap: https://juan.andres.morenorub.io/static/sitemap.xml
|
|
|
|
# LLMs.txt for AI crawlers (standard: https://llmstxt.org/)
|
|
# Location: https://juan.andres.morenorub.io/static/llms.txt
|
|
|
|
# =============================================================================
|
|
# TRADITIONAL SEARCH ENGINES
|
|
# =============================================================================
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
|
|
User-agent: Slurp
|
|
Allow: /
|
|
|
|
User-agent: DuckDuckBot
|
|
Allow: /
|
|
|
|
User-agent: Baiduspider
|
|
Allow: /
|
|
|
|
User-agent: YandexBot
|
|
Allow: /
|
|
|
|
# =============================================================================
|
|
# AI CRAWLERS & LLM BOTS - Explicitly allowed
|
|
# =============================================================================
|
|
|
|
# OpenAI - ChatGPT, GPT-4
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
# Anthropic - Claude
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: Claude-Web
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
# Google AI - Bard, Gemini
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
# Meta AI
|
|
User-agent: FacebookBot
|
|
Allow: /
|
|
|
|
User-agent: Meta-ExternalAgent
|
|
Allow: /
|
|
|
|
User-agent: meta-externalagent
|
|
Allow: /
|
|
|
|
# Perplexity AI
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
# Cohere AI
|
|
User-agent: cohere-ai
|
|
Allow: /
|
|
|
|
# Common Crawl (used by many AI models)
|
|
User-agent: CCBot
|
|
Allow: /
|
|
|
|
# Amazon/Alexa
|
|
User-agent: Amazonbot
|
|
Allow: /
|
|
|
|
# Apple - Applebot (for Siri, Spotlight)
|
|
User-agent: Applebot
|
|
Allow: /
|
|
|
|
# Microsoft Copilot
|
|
User-agent: Copilot
|
|
Allow: /
|
|
|
|
# You.com AI
|
|
User-agent: YouBot
|
|
Allow: /
|
|
|
|
# Brave Search
|
|
User-agent: BraveBot
|
|
Allow: /
|
|
|
|
# =============================================================================
|
|
# CRAWL RATE LIMITS (Optional)
|
|
# =============================================================================
|
|
# Uncomment if needed to prevent server overload
|
|
# Crawl-delay: 1
|