# robots.txt for http:/
User-agent: twengabot
Crawl-delay: 10
Request-rate: 1/10

User-agent: Slurp
Crawl-delay: 5
Request-rate: 1/5

User-agent: *
Crawl-delay: 10
Request-rate: 1/10