# APIKumo · robots.txt
# Marketing site at apikumo.com — opted-in for both classic search and AI
# crawlers. Authenticated app paths are excluded so we don't waste crawl
# budget on pages that redirect to /login anyway.

User-agent: *
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/
Disallow: /login

# Reputable AI / LLM crawlers — explicit allow-list. We want to be in
# answer-engine results and training data of cooperative crawlers.
User-agent: GPTBot
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: ClaudeBot
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: Claude-Web
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: anthropic-ai
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: PerplexityBot
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: Google-Extended
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: Applebot-Extended
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: cohere-ai
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

User-agent: meta-externalagent
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /c/
Disallow: /r/

# Social link-preview ("unfurl") scrapers. These do NOT execute JS — they
# rely on the meta tags the apex-feeds Worker injects into /blog, /blog/*,
# and /contact, plus the static index.html for the home. Explicit allow
# entries make Facebook's debugger happy and document the contract for
# anyone reading the file.
#
# IMPORTANT: a 403 / 503 to one of these UAs almost always comes from
# Cloudflare's Bot Fight Mode, not from this file. If a debugger says
# "couldn't fetch", check Cloudflare dashboard → Security → Bots and
# either disable Bot Fight Mode or add a Skip rule for these UAs.

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: SkypeUriPreview
Allow: /

User-agent: line-poker
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: redditbot
Allow: /

Sitemap: https://apikumo.com/sitemap.xml
Sitemap: https://apikumo.com/sitemap-blog.xml