def initialize(uri, user_agent)
@last_accessed = Time.at(1)
io = Robots.get_robots_txt(uri, user_agent)
if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
io = StringIO.new("User-agent: *\nAllow: /\n")
end
@other = {}
@disallows = {}
@allows = {}
@delays = {}
agent = /.*/
io.each do |line|
next if line =~ /^\s*(#.*|$)/
arr = line.split(":")
key = arr.shift
value = arr.join(":").strip
value.strip!
case key
when "User-agent"
agent = to_regex(value)
when "Allow"
@allows[agent] ||= []
@allows[agent] << to_regex(value)
when "Disallow"
@disallows[agent] ||= []
@disallows[agent] << to_regex(value)
when "Crawl-delay"
@delays[agent] = value.to_i
else
@other[key] ||= []
@other[key] << value
end
end
@parsed = true
end