User-agent: msnbot Disallow: / # Yahoo = 1 page per 10 seconds User-agent: Slurp Crawl-delay: 10 # Cuil = 1 page per 15 seconds User-agent: Twiceler Crawl-delay: 15 User-agent: * Disallow: /tmp/ Disallow: /cgi-bin/journal.cgi