# File lib/robotex.rb, line 18
    def initialize(uri, user_agent)
      io = Robotex.get_robots_txt(uri, user_agent)
      
      if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
        io = StringIO.new("User-agent: *\nAllow: /\n")
      end

      @disallows = {}
      @allows = {}
      @delays = {}
      agent = /.*/
      io.each do |line|
        next if line =~ /^\s*(#.*|$)/
        arr = line.split(":")
        key = arr.shift
        value = arr.join(":").strip
        value.strip!
        case key.downcase
          when "user-agent"
            agent = to_regex(value)
          when "allow"
            @allows[agent] ||= []
            @allows[agent] << to_regex(value)
          when "disallow"
            @disallows[agent] ||= []
            @disallows[agent] << to_regex(value)
          when "crawl-delay"
            @delays[agent] = value.to_i
        end
      end
      
      @parsed = true
    end