# Allows Google, Yahoo, MSN & AOL Bots
# Updated: September 19, 2009 Block All MJ12Bots
# Updated: March 15, 2008 Removed Google Crawl Delay
User-agent: * # applies to all robots unless blocked
Disallow: /cgi-bin/ # robots, stay away from our back-end
Disallow: /site_images/ # disallow indexing of images, may change later
Disallow: /listing # disallow indexing of PDF storage
User-agent: Googlebot
Disallow:
User-agent: Googlebot-Image
Disallow:
User-agent: Yahoo
Disallow:
User-agent: Slurp
Disallow:
User-agent: msnbot
Disallow:
User-agent: AOL
Disallow:
# Blocked Bots
User-agent: MJ12bot
Disallow: /
User-agent: Teoma
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Scrubby
Disallow: /
User-agent: Robozilla
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: Dotbot
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Twiceler
Disallow: /
User-agent: FatBot
Disallow: /
User-agent: Zeus
Disallow: /
User-agent: WWWOFFLE
Disallow: /
User-agent: Widow
Disallow: /
User-agent: Wget
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: WebWhacker
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: WebLeacher
Disallow: /
User-agent: SuperBot
Disallow: /
User-agent: NetSpider
Disallow: /
User-agent: NetAnts
Disallow: /
User-agent: Go!Zilla
Disallow: /
User-agent: VoilaBot
Disallow: /
User-agent: Surfbot
Disallow: /
User-agent: ScoutJet
Disallow: /

User-agent: *
Disallow: /