# ----------------------------------------
# robots.txt for bagabet.com
# ----------------------------------------

# ----------------------------------------
# Search Engine & AI Crawlers (Allowed)
# ----------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /

# Explicit AI Search & Training Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Bytespider
Allow: /

# ----------------------------------------
# Block SEO / Backlink Scrapers Only
# ----------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeekportBot
Disallow: /

# ----------------------------------------
# General rules for everyone else
# ----------------------------------------

User-agent: *
Allow: /

# Block Directories for all spiders
User-agent: *
Disallow: /app/
Disallow: /bootstrap/
Disallow: /config/
Disallow: /database/
Disallow: /vendor/
Disallow: /storage/
Disallow: /routes/
Disallow: /temp/
Disallow: /server/
Disallow: /test/
Disallow: /mail/
Disallow: /news/

Sitemap: https://www.bagabet.com/soccer/sitemap.xml
Sitemap: https://www.bagabet.com/basketball/sitemap.xml
Sitemap: https://www.bagabet.com/volleyball/sitemap.xml
Sitemap: https://www.bagabet.com/hockey/sitemap.xml
Sitemap: https://www.bagabet.com/tennis/sitemap.xml