User-agent: crawl
User-agent: slurp
User-agent: spider
User-agent: ebot
User-agent: obot
User-agent: abot
User-agent: dbot
User-agent: hbot
User-agent: kbot
User-agent: nbot
User-agent: pbot
User-agent: rbot
User-agent: sbot
User-agent: ybot
User-agent: accoona
User-agent: appie
User-agent: architext
User-agent: aspseek
User-agent: asterias
User-agent: atlocal
User-agent: atomz
User-agent: augurfind
User-agent: bannana_bot
User-agent: boitho
User-agent: booch
User-agent: dmoz
User-agent: docomo
User-agent: falcon
User-agent: findlinks
User-agent: gazz
User-agent: goforit
User-agent: grub
User-agent: gulliver
User-agent: harvest
User-agent: holmes
User-agent: homer
User-agent: ia_archiver
User-agent: ichiro
User-agent: iconsurf
User-agent: iltrovatore
User-agent: indexer
User-agent: infoseek
User-agent: ingrid
User-agent: ivia
User-agent: java/
User-agent: jetbot
User-agent: kit_fireball
User-agent: knowledge
User-agent: lachesis
User-agent: larbin
User-agent: linkwalker
User-agent: lwp
User-agent: mantraagent
User-agent: mediapartners
User-agent: mercator
User-agent: mj12
User-agent: mnogo
User-agent: moget/
User-agent: multitext
User-agent: muscatferret
User-agent: myweb
User-agent: nameprotect
User-agent: ncsa beta
User-agent: netmechanic
User-agent: netresearchserver
User-agent: ng/
User-agent: npbot
User-agent: nutch
User-agent: objectssearch
User-agent: omni
User-agent: osis-project
User-agent: pear.
User-agent: pompos
User-agent: poppelsdorf
User-agent: rambler
User-agent: scooter
User-agent: scrubby
User-agent: seeker
User-agent: shopwiki
User-agent: sidewinder
User-agent: smartwit
User-agent: sna-
User-agent: sohu
User-agent: spyder
User-agent: steeler/
User-agent: sygol
User-agent: szukacz
User-agent: t-h-u-n-d-e-r-s-t-o-n-e
User-agent: /teoma
User-agent: tutorgig
User-agent: ultraseek
User-agent: vagabondo
User-agent: volcano
User-agent: voyager/
User-agent: w3c_validator
User-agent: websitepulse
User-agent: wget
User-agent: worldlight
User-agent: worm
User-agent: zao/
User-agent: xenu
User-agent: zippp
User-agent: zyborg
User-agent: Gsitemap
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /includes/
Disallow: /downloads/
Disallow: /pub/