# Allow major search engines full access
User-agent: *
Disallow: /admin/
Disallow: /search/
Allow: /

# Explicitly allow images for Google
User-agent: Googlebot-Image
Allow: /media/
Allow: /images/
Allow: /storage/

# Block aggressive/over-crawling bots
User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Block non-essential internal endpoints
Disallow: /pdf/convert
Disallow: /image-to-webp/convert
Disallow: /qr-code-generator/generate
Disallow: /subscribe

# Sitemap declarations
Sitemap: https://www.freedocumentmaker.com/sitemap.xml
Sitemap: https://www.freedocumentmaker.com/rss