# kleene.ai robots.txt
# AI/LLM crawlers are explicitly listed and ALLOWED.
# An empty "Disallow:" means full-site access is permitted.
# (All of these are already covered by "User-agent: *" below — listing
#  them explicitly just makes the policy auditable and easy to flip to
#  block a specific agent later by changing its Disallow to "/".)
 
# ----- All other crawlers (search engines etc.) -----
User-agent: *
Disallow:
 
# ----- OpenAI -----
User-agent: GPTBot
Disallow:
 
User-agent: OAI-SearchBot
Disallow:
 
User-agent: ChatGPT-User
Disallow:
 
# ----- Anthropic (Claude) -----
User-agent: ClaudeBot
Disallow:
 
User-agent: Claude-User
Disallow:
 
User-agent: Claude-SearchBot
Disallow:
 
User-agent: Claude-Web
Disallow:
 
User-agent: anthropic-ai
Disallow:
 
# ----- Google AI (Gemini / Vertex training) -----
User-agent: Google-Extended
Disallow:
 
# ----- Perplexity -----
User-agent: PerplexityBot
Disallow:
 
User-agent: Perplexity-User
Disallow:
 
# ----- Apple Intelligence -----
User-agent: Applebot-Extended
Disallow:
 
# ----- Amazon -----
User-agent: Amazonbot
Disallow:
 
# ----- Meta AI -----
User-agent: Meta-ExternalAgent
Disallow:
 
User-agent: Meta-ExternalFetcher
Disallow:
 
# ----- ByteDance -----
User-agent: Bytespider
Disallow:
 
# ----- Common Crawl (feeds many open LLMs) -----
User-agent: CCBot
Disallow:
 
# ----- Other AI assistants / answer engines -----
User-agent: cohere-ai
Disallow:
 
User-agent: DuckAssistBot
Disallow:
 
User-agent: YouBot
Disallow:
 
Sitemap: https://kleene.ai/sitemap.xml

Sitemap: https://kleene.ai/sitemap.xml