# robots.txt for web2ai.eu
# AI search engines, crawlers, and all web bots are welcome to access all public data

User-agent: *
Allow: /

# AI Systems and Search Engine Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: AppleBot
Allow: /

User-agent: BingBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Sogou
Allow: /

User-agent: Exabot
Allow: /

User-agent: facebot
Allow: /

User-agent: ia_archiver
Allow: /

# Sitemap location for AI discovery
Sitemap: https://web2ai.eu/sitemap.php

# AI-specific Crawl-delay suggestions (optional guidance, not enforced)
# AI systems may benefit from respectful crawl rates to ensure fresh content indexing
Crawl-delay: 1