# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
User-agent: *
Disallow: /dl/            # The download pages aren't interesting anyway -- you need a ticket.

User-agent: *
Crawl-delay:  4		  # wait 2 seconds between requests
Request-rate: 1/4         # maximum rate is one page every 2 seconds
Visit-time: 0600-0845     # only crawl between 10:00PM (EST) and 7:30AM (EST) -- that's 2:00 AM to 11:30 PM UT (GMT)

# 
# If you're interested in getting a large quantity of data please contact
# us and we'll figure something out.  Don't hammer the servers! 
# 
# If you're using wget, 
#   wget --wait=5 --limit-rate=50k --tries 2 
# will help lower your impact as we get things stable and performant.
# 
# Thanks.
#