# File lib/mechanize/http/agent.rb, line 899 def robots_allowed? uri return true if uri.request_uri == '/robots.txt' webrobots.allowed? uri end