################################################################################
# WFLHD Robots Handler (robots.txt)
# This file provides access rules to compliant indexing robots/crawlers/spiders
################################################################################


# Tell some highly obnoxious and error-ridden spiders to just go away
# ------------------------------------------------------------------------------

User-agent: NuSearch Spider
Disallow: /
User-agent: DigExt
Disallow: /
User-agent: webcollage
Disallow: /
User-agent: heritrix
Disallow: /

# Tell all other bots to ignore certain directories and pages
# ------------------------------------------------------------------------------

User-agent: *
Disallow: /style.css
Disallow: /about/stats/
Disallow: /cgi-bin/
Disallow: /design/ae_comm/
Disallow: /design/fp03/
Disallow: /design/geopak/
Disallow: /design/proposed_std/
Disallow: /includes/
Disallow: /other/errors/
Disallow: /projects/fernan/includes/
Disallow: /projects/thompsonriver/includes/