# spargo.ai — robots policy
# Human+machine entity info: https://spargo.ai/llm-info/
# LLM-friendly summary:      https://spargo.ai/llms.txt
# Public pages are locale-prefixed (/en, /zh-CN/pricing, …). The logged-in app
# (/{locale}/agent) and tokened invite/redeem/auth URLs are private.

User-agent: *
Allow: /
Disallow: /agent
Disallow: /*/agent
Disallow: /auth/
Disallow: /*/auth/
Disallow: /*/invite/
Disallow: /*/redeem/

# --- AI crawlers: explicitly welcome (search, answers and training) ---------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: DuckAssistBot
Allow: /
Disallow: /agent
Disallow: /*/agent
Disallow: /auth/
Disallow: /*/auth/
Disallow: /*/invite/
Disallow: /*/redeem/

Sitemap: https://spargo.ai/sitemap.xml