# https://jojapi.com/robots.txt
#
# NOTE: if Cloudflare's "Managed robots.txt" / AI Crawl Control is enabled for
# this zone, Cloudflare prepends its own block ABOVE this file — including
# Disallow: / for ClaudeBot, GPTBot, Google-Extended, CCBot, Bytespider,
# meta-externalagent, Amazonbot and Applebot-Extended. A user-agent-specific
# group always wins over the wildcard group below, so this file cannot undo it.
# Being cited by AI assistants requires that feature to be turned off in the
# Cloudflare dashboard.

# Only signed-in surfaces are blocked. /hub/search/* and the ?tab= / ?method=
# variants of an API page are deliberately NOT disallowed: they carry noindex
# and rel=canonical, and a crawler has to fetch a page to see either. Blocking
# them here would strand those signals and cost the internal links they carry.
#
# Disallow lines come before "Allow: /" on purpose. RFC 9309 crawlers apply the
# longest matching rule and would block these paths either way, but parsers
# still in the 1996 first-match tradition (urllib.robotparser among them) take
# the first rule that matches — and a leading "Allow: /" matches everything,
# silently voiding every Disallow under it. This order reads the same to both.
User-agent: *
Disallow: /webhook
Disallow: /auth/
Disallow: /workspace/
Disallow: /studio/
Disallow: /settings/
Allow: /

# Answer engines and AI assistants. Explicitly welcome: the marketplace catalog
# is the product, and being quoted when someone asks an assistant for a scraper
# or search API is a primary acquisition channel.
#
# The Disallow lines below are repeated verbatim from the wildcard group, and
# have to be. A crawler obeys exactly one group — the most specific one that
# matches its token — and rules are never merged across groups (RFC 9309 §2.2.1).
# Listing these agents with only "Allow: /" therefore does not extend the
# wildcard group for them, it REPLACES it, handing every one of them /auth/,
# /workspace/, /studio/ and /settings/. Those pages answer 200 with no noindex,
# so nothing downstream catches the mistake.
#
# Consecutive User-agent lines share the one rule block that follows, so adding
# an agent here means adding a single line — the rules stay in one place.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: CCBot
User-agent: cohere-ai
User-agent: Bytespider
Disallow: /webhook
Disallow: /auth/
Disallow: /workspace/
Disallow: /studio/
Disallow: /settings/
Allow: /

Sitemap: https://jojapi.com/sitemap.xml