User-agent: Screaming Frog SEO Spider
Disallow: /

#User-agent: rogerbot
#Crawl-delay: 15
#Disallow: /

User-agent: Slurp
Crawl-delay: 30

User-agent: msnbot 
Crawl-delay: 30

User-agent: bingbot
Crawl-delay: 30

User-agent: *
Disallow: /app/  
Disallow: /admin/ 
Disallow: /api/   
Disallow: /cgi-bin/ 
Disallow: /downloader/ 
Disallow: /includes/ 
Disallow: /lib/ 
Disallow: /pkginfo/ 
Disallow: /var/
Disallow: /index.php/ 
Disallow: /catalog/product_compare/ 
Disallow: /catalog/category/view/ 
Disallow: /catalog/product/view/
Disallow: /catalogsearch/
Disallow: /control/
Disallow: /customer/
Disallow: /customize/
Disallow: /install/
Disallow: /private/
Disallow: /tag/
Disallow: /wishlist/
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /*.php$

# Paths (clean URLs)
Disallow: /checkout/
Disallow: /wishlist/
Disallow: /catalog/product_compare/
Disallow: /review/
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /catalogsearch/
Disallow: /priceslider/
Disallow: /customer/account/
Disallow: /paypal/express/start/
Disallow: /onepage/
Disallow: /checkout/
Disallow: /search
Disallow: /sales/
Disallow: /reward/
Disallow: /paypal/
Disallow: /onepage/
Disallow: /customer/
Disallow: /custom/
Disallow: /checkout/
Disallow: /wishlist/
Disallow: /shipping/tracking/

# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
Disallow: /*?sid=*
Disallow: /*?SID=*
Disallow: /*?id=*
Disallow: /*?ID=*

## Do not crawl sub category pages that are sorted or filtered.
Disallow: /*dir=*
Disallow: /*limit=*
Disallow: /*mode=*
Disallow: /*order=*
Disallow: /*min=*
Disallow: /*max=*