def initialize(uri, user_agent)
io = Robotex.get_robots_txt(uri, user_agent)
if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
io = StringIO.new("User-agent: *\nAllow: /\n")
end
@disallows = {}
@allows = {}
@delays = {}
agent = /.*/
io.each do |line|
next if line =~ /^\s*(#.*|$)/
arr = line.split(":")
key = arr.shift
value = arr.join(":").strip
value.strip!
case key.downcase
when "user-agent"
agent = to_regex(value)
when "allow"
@allows[agent] ||= []
@allows[agent] << to_regex(value)
when "disallow"
@disallows[agent] ||= []
@disallows[agent] << to_regex(value)
when "crawl-delay"
@delays[agent] = value.to_i
end
end
@parsed = true
end