User-agent: *              # applies to all robots
Disallow: /cgi-bin/       # disallow indexing of /cgi-bin/*
Disallow: /stats/
Disallow: /w3c/