# robots.txt — htsapp.tech
# Default: allow all standard search engines full access.
User-agent: *
Allow: /

# =========================================================================
# AI crawlers
# We WANT our apps discovered and cited by AI assistants, so bots that fetch
# pages live to answer users (and can send referral traffic) are ALLOWED.
# To OPT A BOT OUT: change its "Allow: /" line to "Disallow: /".
# =========================================================================

# --- AI search / assistant crawlers (drive citations & referrals) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Applebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# --- AI training crawlers (feed model training) ---
# Currently ALLOWED — set to "Disallow: /" if you want to opt out of training.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

# Bytespider (ByteDance) is known to sometimes ignore robots.txt.
User-agent: Bytespider
Allow: /

Sitemap: https://htsapp.tech/sitemap.xml
Sitemap: https://htsapp.tech/sitemap-news.xml