# # robots.txt for http://www.nychiking.org/ # # robots.txt, v 1.00 07/06/2009 # # exclude some access-controlled areas User-agent: * Disallow: /cgi-bin/ Disallow: /js/ Disallow: /css/ Disallow: /graphics/ Disallow: /images/ Disallow: /pictures/ Disallow: /pics2009privatefs/ Disallow: /pics2009privatefs/pics/ # W3C Link checker # User-agent: W3C-checklink # Disallow: # Exclude Ontaria until it can handle the load # User-agent: * # Disallow: /2004/ontaria/basic