User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /uploads/

# AI TRAINING crawlers (2026-10-31): they may read the pages that describe AudioAtlas
# so models can recommend it, but not the track database. AI search bots and
# user-requested fetchers (OAI-SearchBot, ChatGPT-User, Claude-SearchBot,
# Claude-User, PerplexityBot) fall under User-agent: * above. Enforced at
# Cloudflare by the rule "AA: AI training crawlers - marketing pages only".
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: Google-Extended
Allow: /$
Allow: /live
Allow: /music-similarity-search
Allow: /music-catalog-search
Allow: /music-search-api
Allow: /llms.txt
Allow: /api-docs.html
Allow: /trends
Allow: /radio
Allow: /privacy.html
Allow: /terms
Allow: /api-subscribe
Allow: /logos/
Allow: /sitemap
Disallow: /

User-agent: FacebookBot
Disallow: /

# Aggressive SEO / indexing crawlers — ~80% of /license traffic, no SEO value
# to us (only Google/Bing matter). They honor robots.txt.
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: meta-webindexer
Disallow: /

User-agent: Turnitin
Disallow: /

User-agent: TurnitinBot
Disallow: /

Sitemap: https://www.audioatlas.com/sitemap.xml
Sitemap: https://www.audioatlas.com/sitemap_live.xml