# robots.txt file for robot scanning of website

User-agent: *
Disallow: /

# This says to apply these settings to ALL search engine spiders/crawlers
User-agent: Slurp
Crawl-delay: 5
Disallow: /*.swf$
Disallow: /*.jpg$
Disallow: /*.pdf$
Disallow: /*.gif$
Disallow: /*?osCsid

User-agent: Googlebot-Image
Disallow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: baiduspider+
Disallow: /

User-agent: exabot-Images
Disallow: /

# 212.247.86.66 http://www.apptus.com)"
User-agent: ApptusBot/1.0
Disallow: /

User-agent: exabot

User-agent: Nutch
Disallow: /

User-agent: SBIder
Disallow: /

User-agent: soso
Disallow: /

#http://www.cuill.com/twiceler/robot.html
User-agent: Twiceler-0.9
Disallow: /

# www.cuill.com/twiceler/robot.html
User-agent: Twiceler 
Disallow: /


User-agent: cuill 
Disallow: /

User-agent: Sitesell
Disallow: /

User-agent: Kosmix
Disallow: /

User-agent: Msnbot
Disallow: /

# User-agent: Googlebot
# Disallow: /

# User-agent: Yahoo Slurp!
# Disallow: /

User-agent: Ia_archiver
Disallow: /

User-agent: Tagoobot
Disallow: /

User-agent: wget*
Disallow: /

User-agent: yandex bot
Disallow: /