# robots.txt for http://www.thecmg.org/

# references for how to code http://www.robotstxt.org/wc/faq.html#robotstxt, http://pageresource.com/zine/robotstxt.htm

# exclude http://www.sitecheck.com from searching anything

User-agent: sitecheck.internetseer.com
Disallow: /

# exclude the Internet Archive’s crawler http://www.archive.org (and remove documents from the Wayback Machine)

User-agent: ia_archiver
Disallow: /

User-agent: *
Disallow: /V4/           # the OLD directory structure
Disallow: /V502/        # the OLD directory structure used for testing the new site
Disallow: /V5/css/
Disallow: /V5/forms/
Disallow: /V5/images/
Disallow: /V5/Scripts/
Disallow: /V5/SpryAssets/
Disallow: /V5/ssi/