# Indiana Detox Guide crawler policy
# Canonical host: https://indianadetoxguide.com
# Updated: 2026-08-19
# robots.txt is crawl guidance, not access control.

# Public search indexing and answer-engine retrieval
User-agent: *
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
Allow: /
Allow: /assets/
Allow: /api/og
Allow: /llms.txt
Allow: /entitymap.json
Allow: /entitymap.html
Allow: /sitemap.xml
Allow: /sitemap.html

# Non-public source, build, test, and administrative paths
Disallow: /.git/
Disallow: /.github/
Disallow: /node_modules/
Disallow: /scripts/
Disallow: /tests/
Disallow: /private/
Disallow: /drafts/
Disallow: /tmp/
Disallow: /admin/
Disallow: /api/

# Duplicate tracking-parameter URLs and crawl traps
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*msclkid=
Disallow: /*?*ref=

# Model-training and bulk-dataset crawlers are not required for search/AEO
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

Sitemap: https://indianadetoxguide.com/sitemap.xml

# Human and machine-readable discovery
# HTML sitemap: https://indianadetoxguide.com/sitemap.html
# EntityMap HTML: https://indianadetoxguide.com/entitymap.html
# EntityMap JSON: https://indianadetoxguide.com/entitymap.json
# LLM narrative: https://indianadetoxguide.com/llms.txt