# Optionomics robots.txt
# https://optionomics.ai
#
# The marketing surface is open. Since 2026-09-10 so is much of the app
# shell — a reader without an account gets the market overview, the
# charts and the filings, and a lock page for the rest — but the app's
# pages are live renders against the production database, so the
# disallows below are about crawl budget and load rather than secrecy.
# /overview and /calendar ask for a free account, so a crawler following
# either is sent to the lock page, which is disallowed below.

User-agent: *
Allow: /

# App surfaces — signed in, or open but expensive to render per symbol.
Disallow: /users/
Disallow: /daily/
Disallow: /locked
Disallow: /unlock
Disallow: /alerts/
Disallow: /options/
Disallow: /overview/
Disallow: /history/
Disallow: /builder/
Disallow: /build
Disallow: /backtest/
Disallow: /screener
Disallow: /scanner
Disallow: /darkpool
Disallow: /flow
Disallow: /uoa
# Anchored with `$` so it blocks the signed-in feed at /news and nothing
# below it. Rails published articles at /news/<slug>.html; those are 404
# now, and a crawler has to be able to fetch a 404 to drop the URL from
# its index. A blanket `/news` would freeze them there instead.
Disallow: /news$
Disallow: /market
Disallow: /calendar
Disallow: /chats
Disallow: /journal
Disallow: /trade_ideas/
Disallow: /disclosures
Disallow: /politicians
Disallow: /states
Disallow: /saved_views
Disallow: /notifications
Disallow: /settings
Disallow: /onboarding
Disallow: /for-you
Disallow: /terminal
Disallow: /developers/
Disallow: /tokens
Disallow: /mcp_console
Disallow: /mcp_tool_calls
Disallow: /webhook_deliveries
Disallow: /admin/
Disallow: /api/
Disallow: /subscription
Disallow: /subscriptions/
Disallow: /payments/
Disallow: /impersonation

# The public record: the track record page indexes only its most recent
# rows, so the receipts sitemap is how the rest are found.
Allow: /r/
Allow: /trade_ideas/track_record

# The one app page a reader without an account can read end to end
# (`Web.FreeTier`, since 2026-09-11), so there is something behind the
# "Try for free" buttons for a crawler too. Its sub-pages stay out: each
# is another live render per symbol, and none of them says anything
# /options-flow/ below does not say more cheaply.
Allow: /stocks/
Disallow: /stocks/*/news
Disallow: /stocks/*/insider_trades
Disallow: /stocks/*/politician_trades
Disallow: /stocks/*/earning_filings
Disallow: /stocks/*/earnings

# The public per-symbol pages. `/options-flow/` deliberately does not sit
# under `/options/` or `/flow`, both of which are disallowed above as
# signed-in app paths — a prefix disallow would have swallowed the whole
# surface silently.
Allow: /options-flow/
Allow: /earnings/
Allow: /learn

# ----------------------------------------------------------------------
# AI and LLM crawlers
# ----------------------------------------------------------------------
# Named explicitly because these bots match their own user-agent string
# and ignore the `*` group once they find themselves here. Each gets the
# marketing surface and /llms.txt, and the same app-path disallows as
# everyone else — there is no separate policy, only a separate stanza.

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/
Disallow: /news$
Disallow: /terminal

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: Claude-User
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: Claude-SearchBot
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: Perplexity-User
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: CCBot
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: Meta-ExternalAgent
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

User-agent: Bytespider
Allow: /
Allow: /llms.txt
Disallow: /users/
Disallow: /api/
Disallow: /admin/

# ----------------------------------------------------------------------
# Sitemaps
# ----------------------------------------------------------------------
# Three, and all of them matter. The marketing sitemap carries the pages a
# person could navigate to, the glossary included. The other two carry the
# surfaces nothing links to in bulk and nothing should — a few thousand
# receipt permalinks, and a page per symbol — which without these lines are
# undiscoverable however well they are written.

Sitemap: https://optionomics.ai/sitemap.xml
Sitemap: https://optionomics.ai/receipts-sitemap.xml
Sitemap: https://optionomics.ai/symbols-sitemap.xml