# AI assistants and their search crawlers are welcome: the directory is built for them.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: CCBot
User-agent: GrokBot
User-agent: xAI-Grok
User-agent: Grok-DeepSearch
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: YouBot
User-agent: Applebot
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Google-CloudVertexBot
User-agent: PhindBot
User-agent: AI2Bot
User-agent: Diffbot
User-agent: Bingbot
User-agent: Googlebot
Allow: /
Disallow: /.netlify/

# ClaudeBot (Anthropic's TRAINING crawler) re-read all ~850k pages every day (~922k visits/day, 10/18-10/21) though listings
# rarely change. Teddy 2026-10-21: keep it crawling, just not the whole site daily. Crawl-delay 1 = ~86k pages/day, a full
# pass about every 10 days. Claude-SearchBot and Claude-User (what answers people's questions) are NOT limited.
User-agent: ClaudeBot
Allow: /
Disallow: /.netlify/
Crawl-delay: 1

# Applebot-Extended is Apple's TRAINING switch only (it never fetches pages; Applebot does, for Siri and Spotlight).
# Posture set 2026-10-20 (circle, Teddy GO): be found by Apple, not trained on. Disallowing it does not hide us from Siri.
User-agent: Applebot-Extended
Disallow: /

# SEO-tool crawlers (they sell backlink data; they send no customers and were the top traffic on 2026-10-13). Kept out.
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: DotBot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: DataForSeoBot
User-agent: serpstatbot
Disallow: /

# ShapBot (Parallel Web Systems, an AI-search index) was 27% of all traffic on 2026-10-16 while the
# directory shares TN's database. Blocked for now; re-allow once the directory has its own project.
User-agent: ShapBot
Disallow: /

User-agent: *
Allow: /
Disallow: /.netlify/

Sitemap: https://assistant247.net/sitemap.xml
Sitemap: https://assistant247.net/directory-sitemap.xml