# robots.txt for http://www.d2r.org.uk/ User-agent: * # This selects all bots (* - wildcard) Disallow: /error/ Disallow: /errors/ Disallow: .htaccess Disallow: /disappearing/ #These pages move a lot #taken from http://www.plinko.net/404/howto.asp?article=6 User-agent: Googlebot # Disallow: / This would disallow any files for the google bot. # http://en.wikipedia.org/wiki/Robots_exclusion_standard # Tell All browsers to go away. (this is the client area website, d2r.org.uk) User-agent: * Disallow: /