# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: *
Disallow: /tmp
Disallow: /logs
Disallow: /xblog/wp-admin/
Disallow: /xblog/wp-contents/
Disallow: /xblog/wp-login.php
Disallow: wp-admin/
Disallow: wp-contents/
Disallow: wp-login.php
Disallow: /downloads
Disallow: /images
Disallow: /images/magazine
Disallow: /archives/

User-agent: Mediapartners-Google*
Disallow: /xblog/wp-admin/
Disallow: /xblog/wp-contents/
Disallow: /xblog/wp-login.php
Disallow: wp-admin/
Disallow: wp-contents/
Disallow: wp-login.php
Disallow: /downloads
Disallow: /images/magazine


User-agent: Googlebot*
Disallow: /xblog/wp-admin/
Disallow: /xblog/wp-contents/
Disallow: /xblog/wp-login.php
Disallow: wp-admin/
Disallow: wp-contents/
Disallow: wp-login.php
Disallow: /downloads
Disallow: /images/magazine


User-agent: MsnBot
Disallow: /images/
Disallow: /xblog/wp-admin/
Disallow: /xblog/wp-contents/
Disallow: /xblog/wp-login.php
Disallow: wp-admin/
Disallow: wp-contents/
Disallow: wp-login.php
Disallow: /downloads
Disallow: /images/magazine


User-agent: Blogslive
Disallow: /images/
Disallow: /xblog/wp-admin/
Disallow: /xblog/wp-contents/
Disallow: /xblog/wp-login.php
Disallow: wp-admin/
Disallow: wp-contents/
Disallow: wp-login.php
Disallow: /downloads
Disallow: /images/magazine


User-agent: topicblogs/0.9
Disallow: /images/
Disallow: /xblog/wp-admin/
Disallow: /xblog/wp-contents/
Disallow: /xblog/wp-login.php
Disallow: wp-admin/
Disallow: wp-contents/
Disallow: wp-login.php
Disallow: /downloads
Disallow: /images/magazine


User-agent: lycra
Disallow: /

User-agent: Teoma
Disallow: /images/
Disallow: /xblog/wp-admin/
Disallow: /xblog/wp-contents/
Disallow: /xblog/wp-login.php
Disallow: wp-admin/
Disallow: wp-contents/
Disallow: wp-login.php
Disallow: /downloads
Disallow: /images/magazine


User-agent: Scrubby
Disallow: /

User-agent: Robozilla
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: Nutch
Disallow: /