# Crawling and/or scraping Erowid.org requires written permission from Erowid Center.
#  Your use of software to read, save, or display Erowid pages requires you accept our
#   Terms of Use policies. Publishing our data or analysis of our data or website requires
#   written permission of Erowid Center
#

User-agent: *
Disallow: /cgi-bin/ 
Disallow: /archive/hyperreal/
Disallow: /archive/rhodium/chemistry/nitrostyrenes/
Disallow: /experiences/exp_pdf.php
Disallow: /wp-*
Disallow: /references/texts/
Disallow: /general/newsletter/new/

# Allow our search robot
User-agent: ht://Dig
Disallow:

User-agent: WebStripper/2.03
Disallow: /herbs/
Disallow: /

User-agent: Slurp
Crawl-delay: 1

User-aget: Vegi bot
Disallow: /