# ================================
# robots.txt for NextoolsHub.com
# ================================

# Allow all good bots
User-agent: *
Disallow:

# Block specific bad crawlers
User-agent: HTTrack
Disallow: /

User-agent: wget
Disallow: /

User-agent: curl
Disallow: /

User-agent: python
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: Java
Disallow: /

User-agent: libwww
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: OfflineExplorer
Disallow: /

User-agent: WebZIP
Disallow: /

# Prevent crawling of server/system pages
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /logs/
Disallow: /private/

# Prevent crawling of duplicate or low-value pages
Disallow: /404.php
Disallow: /*.php$

# Sitemap
Sitemap: https://nextoolshub.com/sitemap.xml