# sugi.bio - robots.txt

# /biobtree/api/ is a combinatorial query space (gene x dataset x filter), each
# hit a real backend query — data endpoints, not content. The bots that crawl
# it (MJ12bot, SemrushBot, scrapers) have no named group, so the Disallow in
# `*` below covers them. Well-behaved crawlers (Googlebot, GPTBot, ...) never
# see API links anyway — they're rendered client-side, not in the page HTML.
# This does NOT affect API usage: only robots-respecting crawlers obey it.

User-agent: *
Allow: /
Disallow: /biobtree/api/

# Sitemap
Sitemap: https://sugi.bio/sitemap.xml

# Search Engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# AI Crawlers - welcome
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Anthropic-AI
Allow: /