# File lib/webrobots.rb, line 20 def initialize(user_agent, options = nil) @user_agent = user_agent @parser = RobotsTxt::Parser.new(user_agent) @parser_mutex = Mutex.new options ||= {} @http_get = options[:http_get] || method(:http_get) @robotstxt = create_cache() end