# File lib/robotex.rb, line 100
  def self.get_robots_txt(uri, user_agent)
    begin
      Timeout::timeout(Robotex.timeout) do
        io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
      end 
    rescue Timeout::Error
      STDERR.puts "robots.txt request timed out"
    end
  end