# robots.txt for https://www.mnot.net/ # # Too greedy User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: BrowserShots Disallow: User-agent: ia_archiver Disallow: / User-agent: msnbot Crawl-delay: 120 Disallow: User-agent: * # just a working area, nothing to see Disallow: /tmp/