# ScholarFlow Research robots.txt
#
# Discovery policy:
# - Search, retrieval, citation, and grounding are allowed.
# - Model training is not granted by this robots.txt.
# - Agents should prefer the machine-readable endpoints below over DOM scraping.
# - robots.txt is crawler guidance, not authentication or legal access control.

User-agent: *
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# OpenAI retrieval and user-triggered access are allowed.
User-agent: OAI-SearchBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

User-agent: ChatGPT-User
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# OpenAI model-training crawler is not granted access.
# Rationale: GPTBot is used to train foundation models. ScholarFlow permits
# search/retrieval/grounding but does not grant permission for training use.
User-agent: GPTBot
Content-Signal: ai-train=no, search=no, ai-input=no
Disallow: /

# Google Search is allowed; Google-Extended AI training/control token is not.
# Rationale: Google-Extended is used for AI model training and control.
# Search indexing via Googlebot remains fully permitted.
User-agent: Googlebot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

User-agent: Google-Extended
Content-Signal: ai-train=no, search=no, ai-input=no
Disallow: /

# Common retrieval/search agents currently allowed.
User-agent: Bingbot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

User-agent: PerplexityBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

User-agent: ClaudeBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

User-agent: CCBot
Content-Signal: ai-train=no, search=no, ai-input=no
Disallow: /

Sitemap: https://scholarflow.ca/sitemap.xml
LLMs: https://scholarflow.ca/llms.txt
LLM-Map: https://scholarflow.ca/.well-known/llm-map.json
Knowledge-Graph: https://scholarflow.ca/machine/knowledge-graph.jsonld
Artifact-Index: https://scholarflow.ca/machine/artifacts.json
Machine-Site-Map: https://scholarflow.ca/machine/site-map.json
Ingestion-Manifest: https://scholarflow.ca/machine/ingestion-manifest.json
API-Catalog: https://scholarflow.ca/.well-known/api-catalog
Agent-Skills: https://scholarflow.ca/.well-known/agent-skills/index.json