# TokenOS v4 — robots.txt
# https://tokenos.ai
#
# We welcome every crawler that respects rules. We don't allow:
#   - the API surface (machine-readable, not for indexing)
#   - admin / wallet flows (private)
#   - the ephemeral build preview directory
# We DO want the marketing pages, docs, the marketplace, the leaderboard,
# the public contract pages, and dApp shells indexed so users can find them,
# and we welcome LLM crawlers and agents on the same terms (see /llms.txt).

User-agent: *
Allow: /
Allow: /docs
Allow: /agents
Allow: /leaderboard
Allow: /c/
Allow: /staking
Allow: /pricing
Allow: /blog
Allow: /og-cover.png
Allow: /logo-mark.png

Disallow: /api/
# …except the public, machine-readable discovery documents that llms.txt,
# the docs and the agent manifests point at. Longest-match wins, so these
# override the /api/ block above for every crawler that honours robots.txt.
Allow: /api/openapi.json
Allow: /api/agents/*/openapi.json
Allow: /api/x402-facilitator/quote
Allow: /api/x402-facilitator/supported
Allow: /api/x402-facilitator/list
Allow: /api/x402-facilitator/receipt/
Allow: /api/x402/quote
Allow: /api/mcp/apps
Allow: /api/status
Allow: /api/status/badge
Allow: /api/p2p/v1/models
Allow: /api/p2p/enterprise
Disallow: /admin
Disallow: /build
Disallow: /build-agent
Disallow: /console
Disallow: /advanced-console
Disallow: /live-ide
Disallow: /ide
Disallow: /dashboard
Disallow: /phantom-redirect
Disallow: /agents/*/try
Disallow: /vite-previews/
Disallow: /agents/dashboard
Disallow: /agents/create
Disallow: /agents/create-form
Disallow: /agents/requests
Disallow: /credits/
Disallow: /profile
Disallow: /wallet
Disallow: /*?_session=
Disallow: /*?wallet=

# Crawl-rate guard — be polite, our backend is async but every request
# touches MongoDB and Helius RPC.
Crawl-delay: 2

# Allow OpenGraph card image bots to fetch our dynamic Twitter cards.
User-agent: Twitterbot
Allow: /api/og/

User-agent: facebookexternalhit
Allow: /api/og/

User-agent: LinkedInBot
Allow: /api/og/

User-agent: Slackbot-LinkExpanding
Allow: /api/og/

User-agent: Discordbot
Allow: /api/og/

# LLM crawlers and agents (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot,
# Claude-User, Claude-SearchBot, PerplexityBot, Google-Extended, CCBot, …)
# follow the same rules as everyone else: public pages, /llms.txt,
# /llms-full.txt, the Markdown docs mirrors and the allow-listed API
# documents are open; the API surface, admin and wallet flows stay closed.

Sitemap: https://tokenos.ai/sitemap.xml