# File lib/robotex.rb, line 100 def self.get_robots_txt(uri, user_agent) begin Timeout::timeout(Robotex.timeout) do io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil end rescue Timeout::Error STDERR.puts "robots.txt request timed out" end end