User-agent: *
Allow: /

# Prevent crawling of non-public or administrative files
Disallow: /config.php
Disallow: /includes/
Disallow: /vendor/
Disallow: /temp/

# Exclude specific file types from being crawled (optional, but good practice)
Disallow: /*.txt$
Disallow: /*.log$
Disallow: /*.sql$

# Sitemap location
Sitemap: https://web2ai.eu/sitemap.php

# AI.txt este accesibil doar pentru AI crawlers, nu pentru Googlebot
Disallow: /ai.txt