#
# robots.txt
#
# excerpt from: http://help.yahoo.com/help/us/ysearch/slurp/slurp-03.html
# You can add a "Crawl-delay: xx" instruction, where "xx" is a delay value
# between successive crawler accesses.  If the crawler rate is a problem for
# your server, you can set the delay up to 5 or 10...
User-agent: Slurp
Crawl-delay: 5
#
#
# excerpt from: http://search.msn.com/docs/siteowner.aspx?t=SEARCH_WEBMASTER_FAQ_MSNBotIndexing.htm#D
# MSNBot is crawling a site too frequently (back to top)
# When MSNBot is crawling your site, it generally does not try to access your site more frequently than one time every
# few seconds. If MSNBot determines that your site has a slow connection, it automatically adjusts the frequency.
# To specify a minimum frequency (in seconds), use the Crawl-delay parameter in the robots.txt file:
User-agent: msnbot
Crawl-delay: 5
#
User-agent: Googlebot
Crawl-delay: 5
#
#
User-agent: HagenDerRobotMirago
Disallow: /

User-agent: MagentaNews
Disallow: /

User-agent: wget
Disallow: /

User-agent: webzip
Disallow: /

User-agent: webmirror
Disallow: /

User-agent: webcopy
Disallow: /

User-agent: WebReaper
Disallow: /

#
User-agent: *
Allow: /