# robots.txt for KikDirty (XenForo Setup)

# Block bad/irrelevant bots
User-agent: Amazonbot
Disallow: /

User-agent: Amazonbot-Image
Disallow: /

User-agent: Amazon Ad Bot
Disallow: /

User-agent: Amazon-Route53-Health-Check-Service
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
Disallow: /

# Allow good bots
User-agent: *
Disallow:




# Prevent crawling of internal XenForo folders (such as admin and install)
Disallow: /admin/
Disallow: /install/
Disallow: /core/
Disallow: /library/
Disallow: /misc/
Disallow: /dev/

# Prevent crawling of sensitive content such as deleted threads or soft-deleted posts
Disallow: /threads/delete/
Disallow: /threads/undelete/
Disallow: /posts/soft-deleted/

# Prevent indexing of private content (user profile pages, etc.), but allow public profiles
Disallow: /members/private/

# Allow indexing of 'Find New' and 'Search' pages
# No need to disallow these anymore

# Allow RSS feeds to be indexed
# No need to disallow RSS feeds anymore

# Sitemap for the site (both dynamic and static versions)
Sitemap: https://kikdirty.com/sitemap.php  
Sitemap: https://kikdirty.com/sitemap.xml