User-agent: *
Disallow: /5555/
Disallow: /1017122/
#Lista de bots permitidos.
User-agent: Googlebot-Image
Disallow: 
User-agent: Googlebot-Mobile
Disallow: 
User-agent: Google-Extended
Disallow: /
User-agent: MSNBot
Disallow: 
User-agent: Slurp
Disallow: 
User-agent: Teoma
Disallow: 
User-agent: Gigabot
Disallow: 
User-agent: Robozilla
Disallow: 
User-agent: Nutch
Disallow: 
User-agent: ia_archiver
Disallow: 
User-agent: baiduspider
Disallow: 
User-agent: naverbot
Disallow: 
User-agent: yeti
Disallow: 
User-agent: yahoo-mmcrawler
Disallow: 
User-agent: psbot
Disallow: 
User-agent: yahoo-blogs/v3.9
Disallow: 


#Bloqueo de busquedas
User-agent: *
Disallow: /?s=
Disallow: /search



# Previene problemas de recursos bloqueados en Google Webmaster Tools
User-Agent: Googlebot
Allow: /*.css$
Allow: /*.js$

#Robot de publicidad,evitar problemas con la publicidad en paginaciones, búsquedas, etc…
User-agent: Mediapartners-Google
Allow: /

# AI data scrappers

User-agent: GPTBot
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: claude-web
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: AI2Bot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: DuckAssistBot
Disallow: /

# Bloqueo de bots y crawlers poco utiles
User-agent: MSIECrawler
Disallow: / 
User-agent: WebCopier 
Disallow: / 
User-agent: HTTrack 
Disallow: / 
User-agent: Microsoft.URL.Control 
Disallow: / 
User-agent: libwww 
Disallow: / 
User-agent: Orthogaffe 
Disallow: / 
User-agent: UbiCrawler 
Disallow: / 
User-agent: DOC 
Disallow: / 
User-agent: Zao 
Disallow: / 
User-agent: sitecheck.internetseer.com 
Disallow: / 
User-agent: Zealbot 
Disallow: / 
User-agent: MSIECrawler 
Disallow: / 
User-agent: SiteSnagger 
Disallow: / 
User-agent: WebStripper 
Disallow: / 
User-agent: WebCopier 
Disallow: / 
User-agent: Fetch 
Disallow: / 
User-agent: Offline Explorer 
Disallow: / 
User-agent: Teleport 
Disallow: / 
User-agent: TeleportPro 
Disallow: / 
User-agent: WebZIP 
Disallow: / 
User-agent: linko 
Disallow: / 
User-agent: HTTrack 
Disallow: / 
User-agent: Microsoft.URL.Control 
Disallow: / 
User-agent: Xenu 
Disallow: / 
User-agent: larbin 
Disallow: / 
User-agent: libwww 
Disallow: / 
User-agent: ZyBORG 
Disallow: / 
User-agent: Download Ninja 
Disallow: / 
User-agent: wget 
Disallow: / 
User-agent: grub-client 
Disallow: / 
User-agent: k2spider 
Disallow: / 
User-agent: NPBot 
Disallow: / 
User-agent: WebReaper 
Disallow: /
User-agent: MJ12bot
Disallow: /

#############################
# 1. AI AGENTS (Bots usuarios IA)
#############################
User-agent: AmazonBuyForMe
User-agent: ChatGPT-Agent
User-agent: GoogleAgent-Mariner
User-agent: Manus-User
User-agent: NovaAct
User-agent: TwinAgent
Disallow: /


#############################
# 2. AI ASSISTANTS (Bots de consulta de usuario)
#############################
User-agent: AI2Bot-DeepResearchEval
User-agent: bigsur.ai
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Devin
User-agent: DuckAssistBot
User-agent: Gemini-Deep-Research
User-agent: Google-NotebookLM
User-agent: KlaviyoAIBot
User-agent: LinerBot
User-agent: meta-externalfetcher
User-agent: MistralAI-User
User-agent: Perplexity-User
User-agent: Poggio-Citations
User-agent: QualifiedBot
Disallow: /


#############################
# 3. AI DATA SCRAPERS (Bots de entrenamiento)
#############################
User-agent: Ai2Bot-Dolma
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGLM-Spider
User-agent: ClaudeBot
User-agent: CloudVertexBot
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi
User-agent: Datenbank Crawler
User-agent: Diffbot
User-agent: FacebookBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GPTBot
User-agent: ICC-Crawler
User-agent: imageSpider
User-agent: Kangaroo Bot
User-agent: laion-huggingface-processor
User-agent: LCC
User-agent: meta-externalagent
User-agent: netEstate Imprint Crawler
User-agent: omgili
User-agent: PanguBot
User-agent: SBIntuitionsBot
User-agent: Spider
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: webzio-extended
Disallow: /


#############################
# 4. AI SEARCH CRAWLERS (Bots de buscadores con IA)
#############################
User-agent: AddSearchBot
User-agent: Amazonbot
User-agent: Anomura
User-agent: Applebot
User-agent: atlassian-bot
User-agent: Bravebot
User-agent: Channel3Bot
User-agent: Claude-SearchBot
User-agent: Cloudflare-AutoRAG
User-agent: Google-CloudVertexBot
User-agent: LinkupBot
User-agent: meta-webindexer
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: YouBot
User-agent: ZanistaBot
Disallow: /

## Sitemaps ##
Sitemap: https://www.diariodeburgos.es/sitemap.xml
Sitemap: https://www.diariodeburgos.es/sitemap.news.xml
Sitemap: https://www.diariodeburgos.es/sitemap.temas.xml