# /robots.txt file 
#User-agent: webcrawler
#Disallow:

User-agent: *
Disallow: /prv
Disallow: /cgi
Disallow: /cgi/prv
Disallow: /survey
Disallow: /links
Disallow: /dept
Disallow: /images
Disallow: /resource
Disallow: /resource/docs
Disallow: /library
Disallow: /people/db
Disallow: /bb
Disallow: /archives
Disallow: /internal
Disallow: /abet

User-agent: myhtdig
Allow: /people
Allow: /internal
Allow: /resource
Allow: /links
Allow: /abet