# First Step Development & Research Center (FSDRC)
# Official Robots.txt Configuration
# Optimized for: Google, Bing, Yahoo, DuckDuckGo, Baidu, Yandex
# Generative Engine Optimization (GEO) for: ChatGPT Search (OpenAI), Google Gemini, Perplexity AI, Claude (Anthropic), Apple Intelligence, Meta AI, Copilot

User-agent: *
Allow: /
Allow: /logo.png
Allow: /site.webmanifest
Allow: /sitemap.xml
Allow: /sitemap.txt
Allow: /urllist.txt
Allow: /sitemap.html
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /private/

# ==========================================================
# 1. TRADITIONAL SEARCH ENGINE CRAWLERS
# ==========================================================

# Google Web & Image Search
User-agent: Googlebot
Allow: /
Allow: /sitemap.xml

User-agent: Googlebot-Image
Allow: /
Allow: /logo.png

# Microsoft Bing Search & Copilot
User-agent: Bingbot
Allow: /
Allow: /sitemap.xml
Allow: /sitemap.txt
Allow: /llms.txt

# Yahoo! Search (Slurp)
User-agent: Slurp
Allow: /
Allow: /sitemap.xml
Allow: /urllist.txt

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Baidu (China)
User-agent: Baiduspider
Allow: /
Allow: /sitemap.txt

# Yandex (Russia & CIS)
User-agent: Yandex
Allow: /
Allow: /sitemap.xml

# Apple Spotlight & Siri Suggestions
User-agent: Applebot
Allow: /

# ==========================================================
# 2. GENERATIVE ENGINE OPTIMIZATION (GEO) & AI SEARCH BOTS
# ==========================================================

# OpenAI: ChatGPT Search, GPT-4o, Operator & Real-Time Citations
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: ChatGPT-User
Allow: /

# Perplexity AI Search & Citations
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Anthropic Claude & Claude Search
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google Gemini & Google AI Overviews Grounding
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: GoogleOther
Allow: /

User-agent: GoogleOther-Image
Allow: /

# Apple Intelligence & Siri AI
User-agent: Applebot-Extended
Allow: /

# Meta AI & Llama Web Ingestion
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# ByteDance AI Search
User-agent: Bytespider
Allow: /

# Amazon Alexa & AI
User-agent: Amazonbot
Allow: /

# Diffbot Knowledge Graph (Grounds Bing & Perplexity citations)
User-agent: Diffbot
Allow: /

# ==========================================================
# 3. CANONICAL SITEMAPS & LLM KNOWLEDGE FILES
# ==========================================================
Sitemap: https://www.fsdrc.org/sitemap.xml
Sitemap: https://www.fsdrc.org/sitemap.txt
Sitemap: https://www.fsdrc.org/urllist.txt
Sitemap: https://fsdrc.org/sitemap.xml
Sitemap: https://fsdrc.org/sitemap.txt