User-agent: *
Allow: /
# Admin-only preview surface and API routes: never indexed. Repeated in every
# named group below: a crawler that matches a named group ignores the * group.
Disallow: /blog/preview/
Disallow: /api/

# OpenAI (developers.openai.com/api/docs/bots):
#   OAI-SearchBot = ChatGPT search index (disallowing it removes the site from
#   ChatGPT search answers). GPTBot = model training. ChatGPT-User = user fetches.
#   Robots changes take ~24h to propagate on OpenAI's side.
User-agent: OAI-SearchBot
Allow: /
Disallow: /blog/preview/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /blog/preview/
Disallow: /api/

User-agent: GPTBot
Allow: /
Disallow: /blog/preview/
Disallow: /api/

# Anthropic: Claude-SearchBot = search index; Claude-User = user fetches; ClaudeBot = training.
User-agent: Claude-SearchBot
Allow: /
Disallow: /blog/preview/
Disallow: /api/

User-agent: Claude-User
Allow: /
Disallow: /blog/preview/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /blog/preview/
Disallow: /api/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /blog/preview/
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /blog/preview/
Disallow: /api/

# Google-Extended governs Gemini training/grounding (AI Overviews use normal Googlebot)
User-agent: Google-Extended
Allow: /
Disallow: /blog/preview/
Disallow: /api/

Sitemap: https://collegepulse.com/sitemap.xml