# exclude members area from robots
User-agent: *
Disallow: /members/
Disallow: /cgi-bin/
# the next line is a spam bot trap, for grepping the logs. you should _really_ change this to something else...
Disallow: /all_our_e-mail_addresses
# stop direct access to some scripts
# only works for google at the moment
User-agent: googlebot
Disallow: *.cgi
# disallow stress test
user-agent: stress-agent
Disallow: /