User-agent: Scrapy
Allow: /

User-agent: msnbot
Crawl-delay: 10

User-agent: bingbot
Crawl-delay: 10

User-agent: UptimeRobot
Crawl-delay: 1

User-agent: DotBot
Crawl-delay: 10