# /robots.txt file for http://cityofexeter.com/ # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/wc/robots.html # # For syntax checking, see: # http://www.sxw.org.uk/computing/robots/check.html User-agent: * Crawl-delay: 10 # Directories Disallow: /audio/ Disallow: /content/ Disallow: /db/ Disallow: /jscript/ Disallow: /navigation/ Disallow: /pdf/ Disallow: /secure/ Disallow: /styles/ # Files Disallow: /changelog.txt # Paths (clean URLs) Disallow: /secure/images/ Disallow: /secure/jscript/ Disallow: /secure/styles/ # Paths (no clean URLs) Disallow: /?lnk=test/