# ===========================================================
# robots.txt — valleytripplanner.in
# Platform: WordPress + Yoast SEO
# Optimized: SEO · Crawl Budget · AI / LLM Visibility (AEO/GEO)
# Google spec: developers.google.com/search/docs/crawling-indexing/robots
# Last Updated: July 2026
# CHANGE LOG vs previous version:
#   - Removed dead category-sitemap.xml reference (was 404ing)
#   - Sitemap list now matches live sitemap_index.xml exactly
#   - Added missing AI crawlers: Bytespider, Amazonbot, Diffbot, cohere-ai
# ===========================================================


# -----------------------------------------------------------
# SECTION 1 — GLOBAL BASELINE
# -----------------------------------------------------------
User-agent: *
Allow: /

# WordPress admin
Disallow: /wp-admin/
Allow:    /wp-admin/admin-ajax.php

# WordPress core — block parent, allow rendering assets only
Disallow: /wp-includes/
Allow:    /wp-includes/js/
Allow:    /wp-includes/css/

# Temp / system directories
Disallow: /wp-content/cache/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/backup-db/

# Allow media uploads (travel images — critical for indexing)
Allow: /wp-content/uploads/

# Security-sensitive files
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /readme.html
Disallow: /license.txt

# Thin & duplicate content
Disallow: /?s=
Disallow: /search/
Disallow: /trackback/
Disallow: /?replytocom=
Disallow: /?p=
Disallow: /?page_id=

# Feeds — no SEO value
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /*/feed/


# -----------------------------------------------------------
# SECTION 2 — GOOGLE BOTS (no Crawl-delay — never throttle Google)
# -----------------------------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Googlebot-Image
Allow: /wp-content/uploads/

User-agent: Googlebot-Video
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Google-InspectionTool
Allow: /

# Google-Extended: controls Gemini AI training on your content
# Allow = Google can train on your pages | Disallow = opt out
User-agent: Google-Extended
Allow: /


# -----------------------------------------------------------
# SECTION 3 — BING & MICROSOFT (no Crawl-delay)
# ChatGPT Search and Copilot both query the Bing index — never throttle Bingbot
# -----------------------------------------------------------
User-agent: Bingbot
Allow: /
Disallow: /wp-admin/

User-agent: MSNBot
Allow: /

User-agent: AdIdxBot
Allow: /


# -----------------------------------------------------------
# SECTION 4 — AI & LLM CRAWLERS (AEO / GEO)
# -----------------------------------------------------------

# OpenAI — training crawler
User-agent: GPTBot
Allow: /

# OpenAI — ChatGPT live web citations (separate from GPTBot)
User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude AI
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI — Llama models
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Apple — Siri, Spotlight, Safari Suggestions
User-agent: Applebot
Allow: /

# Common Crawl — open dataset used by many LLMs for training
User-agent: CCBot
Allow: /

# You.com AI search
User-agent: YouBot
Allow: /

# ByteDance / TikTok AI crawler
User-agent: Bytespider
Allow: /

# Amazon AI crawler (Alexa+, Rufus shopping assistant)
User-agent: Amazonbot
Allow: /

# Diffbot — structured data extraction used by several AI products
User-agent: Diffbot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /


# -----------------------------------------------------------
# SECTION 5 — OTHER SEARCH ENGINES
# -----------------------------------------------------------
User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /


# -----------------------------------------------------------
# SECTION 6 — SEO TOOLS
# Crawl-delay ONLY for scraper tools, never for search engines
# -----------------------------------------------------------
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /


# -----------------------------------------------------------
# SECTION 7 — SITEMAPS
# This list MUST exactly match what sitemap_index.xml actually contains.
# Verify each URL returns HTTP 200 before publishing.
# category-sitemap.xml removed — it 404s live (Yoast has category
# archives set to noindex, so no sitemap is generated for it).
# -----------------------------------------------------------
Sitemap: https://valleytripplanner.in/sitemap_index.xml
Sitemap: https://valleytripplanner.in/post-sitemap.xml
Sitemap: https://valleytripplanner.in/page-sitemap.xml


# ===========================================================
# END OF FILE
# ===========================================================