# 1) Default policy: allow public site, block clearly private areas
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /facts.json

# Content Signals - declare AI content usage preferences
# (IETF draft-romm-aipref-contentsignals / contentsignals.org)
# NOTE: kept as a comment because "Content-Signal" is still an IETF draft and
# strict robots.txt validators (e.g. Google Search Console) flag it as an
# unknown directive. Crawlers that already support the draft also accept it
# in a comment-prefixed form, so this preserves the signal without tripping
# the validator.
# Content-Signal: ai-train=no, search=yes, ai-input=yes

# Block obvious sensitive/back-office areas (edit to match your site)
Disallow: /admin/
Disallow: /internal/
Disallow: /private/
Disallow: /account/
Disallow: /login/
Disallow: /logout/
Disallow: /signup/
Disallow: /billing/
Disallow: /checkout/
Disallow: /cart/
Disallow: /orders/
Disallow: /settings/
Disallow: /profile/
Disallow: /api/
Disallow: /admin
Disallow: /investors
Disallow: /versions.json

# Block API routes that are not meant for indexing (edit)
Disallow: /api/
Disallow: /graphql

# Avoid infinite spaces / duplicate URLs via query params (keep conservative)
# Google supports pattern matching for robots rules.
Disallow: /*?*session=
Disallow: /*?*preview=
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*utm_

# Optional: allow well-known discovery files if you publish them
Allow: /.well-known/
Allow: /.well-known/*

# 2) Major search crawlers (explicit, but they will also follow the * group)
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Googlebot-Image
Allow: /og/
Allow: /og-image.png

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slurp
Allow: /

# 3) AI crawlers (optional but common)
# OpenAI robots.txt tokens include GPTBot and OAI-SearchBot.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google-Extended is a robots.txt control token (not a separate HTTP UA string).
User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

# Explicit AI bot allowances
User-agent: Claude-SearchBot
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: YouBot
Allow: /

User-agent: XAI-Bot
Allow: /

User-agent: GrokCrawler
Allow: /

User-agent: GrokSearchBot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: BraveBot
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: CCBot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: AdsBot-Google-Mobile-Apps
Allow: /

User-agent: Google-Adwords-Instant
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: cohere-ai
Allow: /

# 4) Crawl-delay (optional)
# Google ignores crawl-delay, some other bots may respect it.
# Crawl-delay: 1

# 5) Sitemaps (use your canonical host)
Sitemap: https://neuronest.cc/sitemap.xml

# If you have additional sitemaps, list them too:
Sitemap: https://neuronest.cc/sitemap_index.xml