wiki:RobotsTxt

Version 3 (modified by Bruno Cornec, 12 years ago) (diff)

--

User-agent: msnbot-media Crawl-delay: 720 Disallow: /

User-agent: msnbot-news Crawl-delay: 720 Disallow: /

User-agent: msnbot-products Crawl-delay: 720 Disallow: /

User-agent: psbot Crawl-delay: 720 Disallow: /

User-agent: MSNBot Crawl-delay: 720 Disallow: /

User-agent: msnbot/1.1 Crawl-delay: 720 Disallow: /

User-agent: Twiceler-0.9 Crawl-delay: 720 Disallow: /

User-agent: yahoo-slurp Crawl-delay: 10

User-agent: Yahoo-Blogs/v3.9 Crawl-delay: 20

User-agent: * Disallow: /log Disallow: /newticket