class Bundler::Fetcher

Handles all the fetching with the rubygems server

Constants

AUTH_ERRORS

Exceptions classes that should bypass retry attempts. If your password didn't work the first time, it's not going to the third time.

HTTP_ERRORS

Attributes

api_timeout[RW]
disable_endpoint[RW]
max_retries[RW]
redirect_limit[RW]

Public Class Methods

download_gem_from_uri(spec, uri) click to toggle source
# File lib/bundler/fetcher.rb, line 56
def download_gem_from_uri(spec, uri)
  spec.fetch_platform

  download_path = Bundler.requires_sudo? ? Bundler.tmp(spec.full_name) : Bundler.rubygems.gem_dir
  gem_path = "#{Bundler.rubygems.gem_dir}/cache/#{spec.full_name}.gem"

  FileUtils.mkdir_p("#{download_path}/cache")
  Bundler.rubygems.download_gem(spec, uri, download_path)

  if Bundler.requires_sudo?
    Bundler.mkdir_p "#{Bundler.rubygems.gem_dir}/cache"
    Bundler.sudo "mv #{download_path}/cache/#{spec.full_name}.gem #{gem_path}"
  end

  gem_path
ensure
  Bundler.rm_rf(download_path) if Bundler.requires_sudo?
end
new(remote_uri) click to toggle source
# File lib/bundler/fetcher.rb, line 106
def initialize(remote_uri)
  @redirect_limit = 5  # How many redirects to allow in one request
  @api_timeout    = 10 # How long to wait for each API call
  @max_retries    = 3  # How many retries for the API call

  @anonymizable_uri = configured_uri_for(remote_uri)

  Socket.do_not_reverse_lookup = true
  connection # create persistent connection
end
user_agent() click to toggle source
# File lib/bundler/fetcher.rb, line 75
def user_agent
  @user_agent ||= begin
    ruby = Bundler.ruby_version

    agent = "bundler/#{Bundler::VERSION}"
    agent << " rubygems/#{Gem::VERSION}"
    agent << " ruby/#{ruby.version}"
    agent << " (#{ruby.host})"
    agent << " command/#{ARGV.first}"

    if ruby.engine != "ruby"
      # engine_version raises on unknown engines
      engine_version = ruby.engine_version rescue "???"
      agent << " #{ruby.engine}/#{engine_version}"
    end

    agent << " options/#{Bundler.settings.all.join(",")}"

    # add a random ID so we can consolidate runs server-side
    agent << " " << SecureRandom.hex(8)

    # add any user agent strings set in the config
    extra_ua = Bundler.settings[:user_agent]
    agent << " " << extra_ua if extra_ua

    agent
  end
end

Public Instance Methods

connection() click to toggle source
# File lib/bundler/fetcher.rb, line 117
def connection
  @connection ||= begin
    needs_ssl = remote_uri.scheme == "https" ||
      Bundler.settings[:ssl_verify_mode] ||
      Bundler.settings[:ssl_client_cert]
    raise SSLError if needs_ssl && !defined?(OpenSSL::SSL)

    con = Net::HTTP::Persistent.new 'bundler', :ENV

    if remote_uri.scheme == "https"
      con.verify_mode = (Bundler.settings[:ssl_verify_mode] ||
        OpenSSL::SSL::VERIFY_PEER)
      con.cert_store = bundler_cert_store
    end

    if Bundler.settings[:ssl_client_cert]
      pem = File.read(Bundler.settings[:ssl_client_cert])
      con.cert = OpenSSL::X509::Certificate.new(pem)
      con.key  = OpenSSL::PKey::RSA.new(pem)
    end

    con.read_timeout = @api_timeout
    con.override_headers["User-Agent"] = self.class.user_agent
    con
  end
end
fetch_remote_specs(gem_names, full_dependency_list = [], last_spec_list = []) click to toggle source

fetch index

# File lib/bundler/fetcher.rb, line 213
def fetch_remote_specs(gem_names, full_dependency_list = [], last_spec_list = [])
  query_list = gem_names - full_dependency_list

  # only display the message on the first run
  if Bundler.ui.debug?
    Bundler.ui.debug "Query List: #{query_list.inspect}"
  else
    Bundler.ui.info ".", false
  end

  return {remote_uri => last_spec_list} if query_list.empty?

  remote_specs = Bundler::Retry.new("dependency api", AUTH_ERRORS).attempts do
    fetch_dependency_remote_specs(query_list)
  end

  spec_list, deps_list = remote_specs
  returned_gems = spec_list.map {|spec| spec.first }.uniq
  fetch_remote_specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list)
rescue HTTPError, MarshalError, GemspecError
  Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
  Bundler.ui.debug "could not fetch from the dependency API, trying the full index"
  @use_api = false
  return nil
end
fetch_spec(spec) click to toggle source

fetch a gem specification

# File lib/bundler/fetcher.rb, line 149
def fetch_spec(spec)
  spec = spec - [nil, 'ruby', '']
  spec_file_name = "#{spec.join '-'}.gemspec"

  uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
  if uri.scheme == 'file'
    Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path))
  elsif cached_spec_path = gemspec_cached_path(spec_file_name)
    Bundler.load_gemspec(cached_spec_path)
  else
    Bundler.load_marshal Gem.inflate(fetch(uri))
  end
rescue MarshalError
  raise HTTPError, "Gemspec #{spec} contained invalid data.\n"          "Your network or your gem server is probably having issues right now."
end
gemspec_cached_path(spec_file_name) click to toggle source

cached gem specification path, if one exists

# File lib/bundler/fetcher.rb, line 167
def gemspec_cached_path spec_file_name
  paths = Bundler.rubygems.spec_cache_dirs.map { |dir| File.join(dir, spec_file_name) }
  paths = paths.select {|path| File.file? path }
  paths.first
end
inspect() click to toggle source
# File lib/bundler/fetcher.rb, line 256
def inspect
  "#<#{self.class}:0x#{object_id} uri=#{uri}>"
end
specs(gem_names, source) click to toggle source

return the specs in the bundler format as an index

# File lib/bundler/fetcher.rb, line 174
def specs(gem_names, source)
  old = Bundler.rubygems.sources
  index = Index.new

  if gem_names && use_api
    specs = fetch_remote_specs(gem_names)
  end

  if specs.nil?
    # API errors mean we should treat this as a non-API source
    @use_api = false

    specs = Bundler::Retry.new("source fetch", AUTH_ERRORS).attempts do
      fetch_all_remote_specs
    end
  end

  specs[remote_uri].each do |name, version, platform, dependencies|
    next if name == 'bundler'
    spec = nil
    if dependencies
      spec = EndpointSpecification.new(name, version, platform, dependencies)
    else
      spec = RemoteSpecification.new(name, version, platform, self)
    end
    spec.source = source
    spec.source_uri = @anonymizable_uri
    index << spec
  end

  index
rescue CertificateFailureError => e
  Bundler.ui.info "" if gem_names && use_api # newline after dots
  raise e
ensure
  Bundler.rubygems.sources = old
end
uri() click to toggle source
# File lib/bundler/fetcher.rb, line 144
def uri
  @anonymizable_uri.without_credentials
end
use_api() click to toggle source
# File lib/bundler/fetcher.rb, line 239
def use_api
  return @use_api if defined?(@use_api)

  if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
    @use_api = false
  elsif fetch(dependency_api_uri)
    @use_api = true
  end
rescue NetworkDownError => e
  raise HTTPError, e.message
rescue AuthenticationRequiredError
  # We got a 401 from the server. Don't fall back to the full index, just fail.
  raise
rescue HTTPError
  @use_api = false
end

Private Instance Methods

bundler_cert_store() click to toggle source
# File lib/bundler/fetcher.rb, line 383
def bundler_cert_store
  store = OpenSSL::X509::Store.new
  if Bundler.settings[:ssl_ca_cert]
    if File.directory? Bundler.settings[:ssl_ca_cert]
      store.add_path Bundler.settings[:ssl_ca_cert]
    else
      store.add_file Bundler.settings[:ssl_ca_cert]
    end
  else
    store.set_default_paths
    certs = File.expand_path("../ssl_certs/*.pem", __FILE__)
    Dir.glob(certs).each { |c| store.add_file c }
  end
  store
end
configured_uri_for(uri) click to toggle source
# File lib/bundler/fetcher.rb, line 401
def configured_uri_for(uri)
  uri = Bundler::Source.mirror_for(uri)
  config_auth = Bundler.settings[uri.to_s] || Bundler.settings[uri.host]
  AnonymizableURI.new(uri, config_auth)
end
dependency_api_uri(gem_names = []) click to toggle source
# File lib/bundler/fetcher.rb, line 316
def dependency_api_uri(gem_names = [])
  uri = fetch_uri + "api/v1/dependencies"
  uri.query = "gems=#{URI.encode(gem_names.join(","))}" if gem_names.any?
  uri
end
fetch(uri, counter = 0) click to toggle source
# File lib/bundler/fetcher.rb, line 269
def fetch(uri, counter = 0)
  raise HTTPError, "Too many redirects" if counter >= @redirect_limit

  response = request(uri)
  Bundler.ui.debug("HTTP #{response.code} #{response.message}")

  case response
  when Net::HTTPRedirection
    new_uri = URI.parse(response["location"])
    if new_uri.host == uri.host
      new_uri.user = uri.user
      new_uri.password = uri.password
    end
    fetch(new_uri, counter + 1)
  when Net::HTTPSuccess
    response.body
  when Net::HTTPRequestEntityTooLarge
    raise FallbackError, response.body
  when Net::HTTPUnauthorized
    raise AuthenticationRequiredError, remote_uri.host
  else
    raise HTTPError, "#{response.class}: #{response.body}"
  end
end
fetch_all_remote_specs() click to toggle source

fetch from modern index: specs.4.8.gz

# File lib/bundler/fetcher.rb, line 347
def fetch_all_remote_specs
  old_sources = Bundler.rubygems.sources
  Bundler.rubygems.sources = [remote_uri.to_s]
  Bundler.rubygems.fetch_all_remote_specs
rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError => e
  case e.message
  when /certificate verify failed/
    raise CertificateFailureError.new(uri)
  when /401/
    raise AuthenticationRequiredError, remote_uri
  when /403/
    if remote_uri.userinfo
      raise BadAuthenticationError, remote_uri
    else
      raise AuthenticationRequiredError, remote_uri
    end
  else
    Bundler.ui.trace e
    raise HTTPError, "Could not fetch specs from #{uri}"
  end
ensure
  Bundler.rubygems.sources = old_sources
end
fetch_dependency_remote_specs(gem_names) click to toggle source

fetch from Gemcutter Dependency Endpoint API

# File lib/bundler/fetcher.rb, line 323
def fetch_dependency_remote_specs(gem_names)
  Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(',')}"
  gem_list = []
  deps_list = []

  gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names|
    marshalled_deps = fetch dependency_api_uri(names)
    gem_list += Bundler.load_marshal(marshalled_deps)
  end

  spec_list = gem_list.map do |s|
    dependencies = s[:dependencies].map do |name, requirement|
      dep = well_formed_dependency(name, requirement.split(", "))
      deps_list << dep.name
      dep
    end

    [s[:name], Gem::Version.new(s[:number]), s[:platform], dependencies]
  end

  [spec_list, deps_list.uniq]
end
fetch_uri() click to toggle source
# File lib/bundler/fetcher.rb, line 407
def fetch_uri
  @fetch_uri ||= begin
    if remote_uri.host == "rubygems.org"
      uri = remote_uri.dup
      uri.host = "bundler.rubygems.org"
      uri
    else
      remote_uri
    end
  end
end
remote_uri() click to toggle source
# File lib/bundler/fetcher.rb, line 419
def remote_uri
  @anonymizable_uri.original_uri
end
request(uri) click to toggle source
# File lib/bundler/fetcher.rb, line 294
def request(uri)
  Bundler.ui.debug "HTTP GET #{uri}"
  req = Net::HTTP::Get.new uri.request_uri
  if uri.user
    user = CGI.unescape(uri.user)
    password = uri.password ? CGI.unescape(uri.password) : nil
    req.basic_auth(user, password)
  end
  connection.request(uri, req)
rescue OpenSSL::SSL::SSLError
  raise CertificateFailureError.new(uri)
rescue *HTTP_ERRORS => e
  Bundler.ui.trace e
  case e.message
  when /host down:/, /getaddrinfo: nodename nor servname provided/
    raise NetworkDownError, "Could not reach host #{uri.host}. Check your network "          "connection and try again."
  else
    raise HTTPError, "Network error while fetching #{uri}"
  end
end
well_formed_dependency(name, *requirements) click to toggle source
# File lib/bundler/fetcher.rb, line 371
def well_formed_dependency(name, *requirements)
  Gem::Dependency.new(name, *requirements)
rescue ArgumentError => e
  illformed = 'Ill-formed requirement ["#<YAML::Syck::DefaultKey'
  raise e unless e.message.include?(illformed)
  puts # we shouldn't print the error message on the "fetching info" status line
  raise GemspecError,
    "Unfortunately, the gem #{s[:name]} (#{s[:number]}) has an invalid "          "gemspec. \nPlease ask the gem author to yank the bad version to fix "          "this issue. For more information, see http://bit.ly/syck-defaultkey."
end