# robots.txt for www.gilesfleming.com
# anything in /files is being kept for other sites and such random uses.
# No need for search engines to index it.

User-agent: *
Disallow: /files
Disallow: /dev
Disallow: /admin
Disallow: /me
Disallow: /cgi-bin/admin
Disallow: /uploads
# Dont want them indexing add_comments, since a) there's no page to view, and
# b) it causes the comments to be re-submitted. Hopefully wont happen anymore,
# since I've excluded it from the sitemap, but just to be sure
# found out why this happens - ought to be using POST for submitting the form.
# Now using POST. so 3 layers to prevent search engines from submitting comments
Disallow: /cgi-bin/add_comments.py
Disallow: /cgi-bin/galls_add_comments.py
# Dont want them indexing the upload page, either.
Disallow: /cgi-bin/upload.py

# (Everything else - not mentioned as disallowed - is allowed)