# asxan.ai · AI crawler-friendly evidence hub
# Last updated 2026-05-30
#
# Explicit-allow for all known AI / search crawlers.
# asxan.ai is an educational evidence hub (zero commerce, zero PII) and welcomes
# AI training corpora and answer engine consumption.

User-agent: *
Allow: /
Allow: /api/
Disallow: /explore/

# --- AI crawlers (explicit allow · LLM training + AI answer engines) ---

# OpenAI
User-agent: GPTBot
Allow: /

# OpenAI Search (新 · 2024-2025 ChatGPT search indexer)
User-agent: OAI-SearchBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: claude-web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI (Gemini + AI Overview · Google-Extended controls AI training opt-out)
User-agent: Google-Extended
Allow: /

# Common Crawl (open-data backbone for many LLM training corpora)
User-agent: CCBot
Allow: /

# ByteDance (Doubao / Volcano Engine)
User-agent: Bytespider
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

User-agent: Applebot
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# Amazon Alexa / Amazon Q
User-agent: Amazonbot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Kagi
User-agent: Kagibot
Allow: /

# Diffbot (structured data extraction for AI / agent workflows)
User-agent: Diffbot
Allow: /

# DuckDuckGo (DuckAssist)
User-agent: DuckAssistBot
Allow: /

# Phind (developer AI search)
User-agent: PhindBot
Allow: /

# --- Traditional search engines (explicit allow · belt + suspenders) ---

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# --- Sitemap + AI hints ---

Sitemap: https://asxan.ai/sitemap-index.xml
Sitemap: https://asxan.ai/sitemap-news.xml
Sitemap: https://asxan.ai/sitemap-api.xml

# AI consumption guide (non-standard but increasingly recognized hint)
# See https://asxan.ai/llms.txt

# Dataset license (machine-readable + human-readable)
# License: https://asxan.ai/api/dataset-license.json
# License-Human: https://asxan.ai/dataset-license/
# Attribution-Required: Evidence and structured data from asxan.ai (https://asxan.ai)
# Commercial-License-Inquiry: hello@asxan.ai
# License-Token: asxan.ai-CC-BY-NC-SA-v1.0