# robots.txt — technopieux.com
# Last updated: 2026-04-14
# Goal: Maximize visibility in search engines AND LLMs (ChatGPT, Claude, Perplexity, Gemini, AI Overviews)

# ---------- Default rules for all bots ----------
User-agent: *

# Block WordPress internals (keep admin-ajax.php allowed — needed for front-end features)
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Block low-value / duplicate-content URLs
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/mu-plugins/
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /comments/feed/
Disallow: /*?replytocom=
Disallow: /author/
Disallow: /tag/

# Block search, filter, and tracking parameters (crawl-budget savers)
Disallow: /*?s=
Disallow: /*?p=
Disallow: /*?orderby=
Disallow: /*?filter=
Disallow: /*?utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?mc_cid=
Disallow: /*?mc_eid=
Disallow: /*&utm_
Disallow: /*&gclid=
Disallow: /*&fbclid=

# Block feeds and embeds
Disallow: /*/feed/
Disallow: /*/embed/
Disallow: /embed/

# Allow media, CSS, JS (critical — Google needs these to render pages)
Allow: /wp-content/uploads/
Allow: /wp-includes/
Allow: /*.css$
Allow: /*.js$
Allow: /*.webp$
Allow: /*.avif$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.woff$
Allow: /*.woff2$

# ---------- Limit aggressive SEO crawlers (bandwidth control, still indexed) ----------
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# ---------- Block low-value crawlers ----------
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

# ---------- LLM & AI crawlers: EXPLICITLY ALLOWED ----------
# Goal: appear in ChatGPT, Claude, Perplexity, Gemini, AI Overviews, Copilot

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Amazonbot
Allow: /

# ---------- Sitemaps ----------
Sitemap: https://www.technopieux.com/sitemap_index.xml