# Qohash robots.txt
# Last updated: 2025-11-12

# ============================================
# SPECIFIC BOT RULES
# ============================================

# OpenAI (ChatGPT, SearchGPT)
# GPTBot = training data collection
# ChatGPT-User = user-initiated searches
User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /admin/
Disallow: /internal/
Disallow: /search
Crawl-delay: 2

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /admin/
Disallow: /internal/
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /admin/
Disallow: /internal/
Crawl-delay: 2

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /admin/
Disallow: /internal/
Crawl-delay: 2

# Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /admin/
Disallow: /internal/
Crawl-delay: 1

# Google Search (allow indexing)
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /api/
Disallow: /search

# Bing Search (allow indexing)
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /api/
Crawl-delay: 1

# ============================================
# DEFAULT RULES (must come last!)
# ============================================

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /api/
Disallow: /search
Disallow: /*?*utm_
Disallow: /*?session=
Disallow: /*?token=
Crawl-delay: 1

# ============================================
# SITEMAPS
# ============================================

Sitemap: https://qohash.com/sitemap_index.xml
Sitemap: https://qohash.com/post-sitemap.xml
Sitemap: https://qohash.com/page-sitemap.xml

# ============================================
# MONITORING NOTES
# ============================================
# Some bots may ignore directives - monitor logs regularly
# Check for: Unusual user-agent strings, crawl patterns, bandwidth spikes
# Implement rate limiting at CDN/WAF level (Cloudflare, AWS WAF, etc.)
# Review and update quarterly or when launching new products/pages
# Test changes: https://www.google.com/webmasters/tools/robots-testing-tool