# references for how to code http://www.robotstxt.org/wc/faq.html#robotstxt, http://pageresource.com/zine/robotstxt.htm # exclude http://www.sitecheck.com from searching anything User-Agent: sitecheck.internetseer.com Disallow: / # exclude the Internet Archive’s crawler http://www.archive.org (and remove documents from the Wayback Machine) User-agent: ia_archiver Disallow: /