# COGENT 1.1
# robot/spider exclusion 
# http://info.webcrawler.com/mak/projects/robots/robots.html
User-agent: EmailSiphon
User-agent: Kilroy             # OCLC
User-agent: Whacker            # WebWhacker
User-agent: FlashSite
User-agent: NetJet
User-agent: Snarf
User-Agent: TestApp/1.0 libwww/4.1b5
User-Agent: GETWWW-ROBOT/2.0
User-Agent: MemoWeb
User-Agent: net.Sweep/v1.2e-T1-CHI
User-Agent: FFiNet32.DLL/3.1 libwww/3.1
User-Agent: Javachapman:05/19/19-15:35
User-Agent: LaSCoW-trace/1.0
User-Agent: ExtractorPro
User-Agent: Rainspider/1.0
User-Agent: /0.5 libwww-perl/0.40
User-Agent: EmailWolf 1.00
User-Agent: EmailCollector/1.0
User-Agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
User-Agent: CherryPickerSE/1.0
User-Agent: CherryPickerElite/1.0
User-Agent: 
Disallow: /

User-agent: *
Disallow: /cgi-bin/
Disallow: /stats/
Disallow: /logs/
Disallow: /testbed/
Disallow: /membersonly/