# Robots.txt for Winner Winner Chicken Dinner
# Updated: October 23, 2025
# Optimized for efficient crawl budget allocation and faster indexing

# ============================================================
# GOOGLEBOT - Primary search engine (highest priority)
# ============================================================
User-agent: Googlebot
Allow: /
# Allow all important content pages
Allow: /buying-guides/
Allow: /reviews/
Allow: /explainers/
Allow: /a-vs-b/
Allow: /troubleshooting/

# Block duplicate/paginated content to save crawl budget
Disallow: /*?*
Disallow: /search
Disallow: /temp/

# Allow /_astro/ (CSS + JS + optimized images) so Google can fully render pages
Disallow: /~partytown/
Disallow: /search-index.json

# Note: Google ignores Crawl-delay directive


# ============================================================
# GOOGLEBOT-IMAGE - Image search bot
# ============================================================
User-agent: Googlebot-Image
Allow: /images/
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.webp
Allow: /*.svg
# /_astro/ holds Astro-optimized images — must stay crawlable for Image search


# ============================================================
# BINGBOT - Microsoft search engine
# ============================================================
User-agent: Bingbot
Allow: /
Allow: /buying-guides/
Allow: /reviews/
Allow: /explainers/
Allow: /a-vs-b/
Allow: /troubleshooting/
Disallow: /*?*
Disallow: /search
# Allow /_astro/ (CSS + JS + optimized images) so Bing can fully render pages
Disallow: /~partytown/
Disallow: /search-index.json
Disallow: /temp/
Crawl-delay: 2


# ============================================================
# SOCIAL MEDIA CRAWLERS - For link previews
# ============================================================
User-agent: facebookexternalhit
Allow: /
Disallow: /search
Disallow: /_astro/

User-agent: Twitterbot
Allow: /
Disallow: /search
Disallow: /_astro/

User-agent: LinkedInBot
Allow: /
Disallow: /search
Disallow: /_astro/


# ============================================================
# SEO TOOLS - Rate-limited to protect server resources
# ============================================================
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /search
Disallow: /_astro/
Disallow: /temp/

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /search
Disallow: /_astro/
Disallow: /temp/

User-agent: DotBot
Crawl-delay: 10
Disallow: /search
Disallow: /_astro/
Disallow: /temp/


# ============================================================
# AGGRESSIVE/UNWANTED BOTS - Block completely
# ============================================================
User-agent: MJ12bot
Disallow: /


# ============================================================
# AI SEARCH BOTS - Allow content pages for citation in AI search results
# ============================================================
# OpenAI ChatGPT web search
User-agent: GPTBot
Allow: /
Allow: /reviews/
Allow: /buying-guides/
Allow: /explainers/
Allow: /a-vs-b/
Allow: /troubleshooting/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /search
Disallow: /_astro/
Disallow: /~partytown/
Disallow: /temp/

# OpenAI SearchBot (search-only, no training)
User-agent: OAI-SearchBot
Allow: /
Allow: /reviews/
Allow: /buying-guides/
Allow: /explainers/
Allow: /a-vs-b/
Allow: /troubleshooting/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /search
Disallow: /_astro/
Disallow: /~partytown/
Disallow: /temp/

# Perplexity AI search
User-agent: PerplexityBot
Allow: /
Allow: /reviews/
Allow: /buying-guides/
Allow: /explainers/
Allow: /a-vs-b/
Allow: /troubleshooting/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /search
Disallow: /_astro/
Disallow: /~partytown/
Disallow: /temp/

# Anthropic Claude web search
User-agent: ClaudeBot
Allow: /
Allow: /reviews/
Allow: /buying-guides/
Allow: /explainers/
Allow: /a-vs-b/
Allow: /troubleshooting/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /search
Disallow: /_astro/
Disallow: /~partytown/
Disallow: /temp/

# ============================================================
# AI TRAINING-ONLY BOTS - Block to protect content
# ============================================================
# Google-Extended (Gemini model training only, does NOT affect AI Overviews)
User-agent: Google-Extended
Disallow: /

# Common Crawl (dataset used by many AI training pipelines)
User-agent: CCBot
Disallow: /

# Anthropic training crawler (legacy user-agent)
User-agent: anthropic-ai
Disallow: /

# Cohere AI training
User-agent: cohere-ai
Disallow: /

# Omgili/Webz.io
User-agent: omgili
Disallow: /


# ============================================================
# DEFAULT RULES - All other bots
# ============================================================
User-agent: *
Allow: /

# Block query parameters (duplicate content)
Disallow: /*?
Disallow: /*?sort=*
Disallow: /*?filter=*
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /*?page=*
Disallow: /*?category=*
Disallow: /*&*

# Block search and filters
Disallow: /search
Disallow: /search?*

# Block build artifacts (save crawl budget); /_astro/ left crawlable for render
Disallow: /~partytown/
Disallow: /node_modules/
Disallow: /src/
Disallow: /dist/
Disallow: /temp/

# Block non-indexable files
Disallow: /search-index.json
Allow: /manifest.json
Disallow: /.well-known/
Disallow: /_redirects
Allow: /ads.txt

# Block system files
Disallow: /.DS_Store


# ============================================================
# SITEMAPS
# ============================================================
# Sitemap index contains all sub-sitemaps — only one entry needed
Sitemap: https://winnerwinnerchickendinner.in/sitemap-index.xml