diff --git a/lib/puppet/forge.rb b/lib/puppet/forge.rb index 1dd95c41e..b2ec0e139 100644 --- a/lib/puppet/forge.rb +++ b/lib/puppet/forge.rb @@ -1,213 +1,216 @@ require 'puppet/vendor' Puppet::Vendor.load_vendored require 'net/http' require 'tempfile' require 'uri' require 'pathname' require 'json' require 'semantic' class Puppet::Forge < Semantic::Dependency::Source require 'puppet/forge/cache' require 'puppet/forge/repository' require 'puppet/forge/errors' include Puppet::Forge::Errors USER_AGENT = "PMT/1.1.1 (v3; Net::HTTP)".freeze attr_reader :host, :repository def initialize(host = Puppet[:module_repository]) @host = host @repository = Puppet::Forge::Repository.new(host, USER_AGENT) end # Return a list of module metadata hashes that match the search query. # This return value is used by the module_tool face install search, # and displayed to on the console. # # Example return value: # # [ # { # "author" => "puppetlabs", # "name" => "bacula", # "tag_list" => ["backup", "bacula"], # "releases" => [{"version"=>"0.0.1"}, {"version"=>"0.0.2"}], # "full_name" => "puppetlabs/bacula", # "version" => "0.0.2", # "project_url" => "http://github.com/puppetlabs/puppetlabs-bacula", # "desc" => "bacula" # } # ] # # @param term [String] search term # @return [Array] modules found # @raise [Puppet::Forge::Errors::CommunicationError] if there is a network # related error # @raise [Puppet::Forge::Errors::SSLVerifyError] if there is a problem # verifying the remote SSL certificate # @raise [Puppet::Forge::Errors::ResponseError] if the repository returns a # bad HTTP response def search(term) matches = [] uri = "/v3/modules?query=#{URI.escape(term)}" if Puppet[:module_groups] uri += "&module_groups=#{Puppet[:module_groups]}" end while uri response = make_http_request(uri) if response.code == '200' result = JSON.parse(response.body) uri = result['pagination']['next'] matches.concat result['results'] else - raise ResponseError.new(:uri => URI.parse(@host).merge(uri) , :input => term, :response => response) + raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :response => response) end end matches.each do |mod| mod['author'] = mod['owner']['username'] mod['tag_list'] = mod['current_release']['tags'] mod['full_name'] = "#{mod['author']}/#{mod['name']}" mod['version'] = mod['current_release']['version'] mod['project_url'] = mod['homepage_url'] mod['desc'] = mod['current_release']['metadata']['summary'] || '' end end # Fetches {ModuleRelease} entries for each release of the named module. # # @param input [String] the module name to look up # @return [Array] a list of releases for # the given name # @see Semantic::Dependency::Source#fetch def fetch(input) name = input.tr('/', '-') uri = "/v3/releases?module=#{name}" if Puppet[:module_groups] uri += "&module_groups=#{Puppet[:module_groups]}" end releases = [] while uri response = make_http_request(uri) if response.code == '200' response = JSON.parse(response.body) else - raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :input => input, :response => response) + raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :response => response) end releases.concat(process(response['results'])) uri = response['pagination']['next'] end return releases end def make_http_request(*args) @repository.make_http_request(*args) end class ModuleRelease < Semantic::Dependency::ModuleRelease attr_reader :install_dir, :metadata def initialize(source, data) @data = data @metadata = meta = data['metadata'] name = meta['name'].tr('/', '-') version = Semantic::Version.parse(meta['version']) release = "#{name}@#{version}" if meta['dependencies'] dependencies = meta['dependencies'].collect do |dep| begin Puppet::ModuleTool::Metadata.new.add_dependency(dep['name'], dep['version_requirement'], dep['repository']) Puppet::ModuleTool.parse_module_dependency(release, dep)[0..1] rescue ArgumentError => e Puppet.debug "Malformed dependency: #{dep['name']}. Exception was: #{e}" end end else dependencies = [] end super(source, name, version, Hash[dependencies]) end def install(dir) staging_dir = self.prepare module_dir = dir + name[/-(.*)/, 1] module_dir.rmtree if module_dir.exist? # Make sure unpacked module has the same ownership as the folder we are moving it into. Puppet::ModuleTool::Applications::Unpacker.harmonize_ownership(dir, staging_dir) FileUtils.mv(staging_dir, module_dir) @install_dir = dir # Return the Pathname object representing the directory where the # module release archive was unpacked the to. return module_dir ensure staging_dir.rmtree if staging_dir.exist? end def prepare return @unpacked_into if @unpacked_into download(@data['file_uri'], tmpfile) validate_checksum(tmpfile, @data['file_md5']) unpack(tmpfile, tmpdir) @unpacked_into = Pathname.new(tmpdir) end private # Obtain a suitable temporary path for unpacking tarballs # # @return [Pathname] path to temporary unpacking location def tmpdir @dir ||= Dir.mktmpdir(name, Puppet::Forge::Cache.base_path) end def tmpfile @file ||= Tempfile.new(name, Puppet::Forge::Cache.base_path).tap do |f| f.binmode end end def download(uri, destination) - @source.make_http_request(uri, destination) + response = @source.make_http_request(uri, destination) destination.flush and destination.close + unless response.code == 200 + raise Puppet::Forge::Errors::ResponseError.new(:uri => uri, :response => response) + end end def validate_checksum(file, checksum) if Digest::MD5.file(file.path).hexdigest != checksum raise RuntimeError, "Downloaded release for #{name} did not match expected checksum" end end def unpack(file, destination) begin Puppet::ModuleTool::Applications::Unpacker.unpack(file.path, destination) rescue Puppet::ExecutionFailure => e raise RuntimeError, "Could not extract contents of module archive: #{e.message}" end end end private def process(list) list.map { |release| ModuleRelease.new(self, release) } end end diff --git a/lib/puppet/forge/errors.rb b/lib/puppet/forge/errors.rb index 211c63fe8..54041f592 100644 --- a/lib/puppet/forge/errors.rb +++ b/lib/puppet/forge/errors.rb @@ -1,113 +1,112 @@ require 'json' require 'puppet/error' require 'puppet/forge' # Puppet::Forge specific exceptions module Puppet::Forge::Errors # This exception is the parent for all Forge API errors class ForgeError < Puppet::Error # This is normally set by the child class, but if it is not this will # fall back to displaying the message as a multiline. # # @return [String] the multiline version of the error message def multiline self.message end end # This exception is raised when there is an SSL verification error when # communicating with the forge. class SSLVerifyError < ForgeError # @option options [String] :uri The URI that failed # @option options [String] :original the original exception def initialize(options) @uri = options[:uri] original = options[:original] super("Unable to verify the SSL certificate at #{@uri}", original) end # Return a multiline version of the error message # # @return [String] the multiline version of the error message def multiline <<-EOS.chomp Could not connect via HTTPS to #{@uri} Unable to verify the SSL certificate The certificate may not be signed by a valid CA The CA bundle included with OpenSSL may not be valid or up to date EOS end end # This exception is raised when there is a communication error when connecting # to the forge class CommunicationError < ForgeError # @option options [String] :uri The URI that failed # @option options [String] :original the original exception def initialize(options) @uri = options[:uri] original = options[:original] @detail = original.message message = "Unable to connect to the server at #{@uri}. Detail: #{@detail}." super(message, original) end # Return a multiline version of the error message # # @return [String] the multiline version of the error message def multiline <<-EOS.chomp Could not connect to #{@uri} There was a network communications problem The error we caught said '#{@detail}' Check your network connection and try again EOS end end # This exception is raised when there is a bad HTTP response from the forge # and optionally a message in the response. class ResponseError < ForgeError # @option options [String] :uri The URI that failed # @option options [String] :input The user's input (e.g. module name) # @option options [String] :message Error from the API response (optional) # @option options [Net::HTTPResponse] :response The original HTTP response def initialize(options) @uri = options[:uri] - @input = options[:input] @message = options[:message] response = options[:response] @response = "#{response.code} #{response.message.strip}" begin body = JSON.parse(response.body) if body['message'] @message ||= body['message'].strip end rescue JSON::ParserError end - message = "Could not execute operation for '#{@input}'. Detail: " + message = "Request to Puppet Forge failed. Detail: " message << @message << " / " if @message message << @response << "." super(message, original) end # Return a multiline version of the error message # # @return [String] the multiline version of the error message def multiline - message = <<-EOS -Could not execute operation for '#{@input}' + message = <<-EOS.chomp +Request to Puppet Forge failed. The server being queried was #{@uri} The HTTP response we received was '#{@response}' EOS - message << " The message we received said '#{@message}'\n" if @message - message << " Check the author and module names are correct." + message << "\n The message we received said '#{@message}'" if @message + message end end end diff --git a/spec/unit/forge/errors_spec.rb b/spec/unit/forge/errors_spec.rb index 057bf014a..fc1ac1a0e 100644 --- a/spec/unit/forge/errors_spec.rb +++ b/spec/unit/forge/errors_spec.rb @@ -1,82 +1,80 @@ require 'spec_helper' require 'puppet/forge' describe Puppet::Forge::Errors do describe 'SSLVerifyError' do subject { Puppet::Forge::Errors::SSLVerifyError } let(:exception) { subject.new(:uri => 'https://fake.com:1111') } it 'should return a valid single line error' do exception.message.should == 'Unable to verify the SSL certificate at https://fake.com:1111' end it 'should return a valid multiline error' do exception.multiline.should == <<-EOS.chomp Could not connect via HTTPS to https://fake.com:1111 Unable to verify the SSL certificate The certificate may not be signed by a valid CA The CA bundle included with OpenSSL may not be valid or up to date EOS end end describe 'CommunicationError' do subject { Puppet::Forge::Errors::CommunicationError } let(:socket_exception) { SocketError.new('There was a problem') } let(:exception) { subject.new(:uri => 'http://fake.com:1111', :original => socket_exception) } it 'should return a valid single line error' do exception.message.should == 'Unable to connect to the server at http://fake.com:1111. Detail: There was a problem.' end it 'should return a valid multiline error' do exception.multiline.should == <<-EOS.chomp Could not connect to http://fake.com:1111 There was a network communications problem The error we caught said 'There was a problem' Check your network connection and try again EOS end end describe 'ResponseError' do subject { Puppet::Forge::Errors::ResponseError } let(:response) { stub(:body => '{}', :code => '404', :message => "not found") } context 'without message' do let(:exception) { subject.new(:uri => 'http://fake.com:1111', :response => response, :input => 'user/module') } it 'should return a valid single line error' do - exception.message.should == 'Could not execute operation for \'user/module\'. Detail: 404 not found.' + exception.message.should == 'Request to Puppet Forge failed. Detail: 404 not found.' end it 'should return a valid multiline error' do exception.multiline.should == <<-eos.chomp -Could not execute operation for 'user/module' +Request to Puppet Forge failed. The server being queried was http://fake.com:1111 The HTTP response we received was '404 not found' - Check the author and module names are correct. eos end end context 'with message' do let(:exception) { subject.new(:uri => 'http://fake.com:1111', :response => response, :input => 'user/module', :message => 'no such module') } it 'should return a valid single line error' do - exception.message.should == 'Could not execute operation for \'user/module\'. Detail: no such module / 404 not found.' + exception.message.should == 'Request to Puppet Forge failed. Detail: no such module / 404 not found.' end it 'should return a valid multiline error' do exception.multiline.should == <<-eos.chomp -Could not execute operation for 'user/module' +Request to Puppet Forge failed. The server being queried was http://fake.com:1111 The HTTP response we received was '404 not found' The message we received said 'no such module' - Check the author and module names are correct. eos end end end end diff --git a/spec/unit/forge/module_release_spec.rb b/spec/unit/forge/module_release_spec.rb index a00d4d9c5..ea608d83f 100644 --- a/spec/unit/forge/module_release_spec.rb +++ b/spec/unit/forge/module_release_spec.rb @@ -1,215 +1,221 @@ # encoding: utf-8 require 'spec_helper' require 'puppet/forge' require 'net/http' require 'puppet/module_tool' describe Puppet::Forge::ModuleRelease do let(:agent) { "Test/1.0" } let(:repository) { Puppet::Forge::Repository.new('http://fake.com', agent) } let(:ssl_repository) { Puppet::Forge::Repository.new('https://fake.com', agent) } let(:api_version) { "v3" } let(:module_author) { "puppetlabs" } let(:module_name) { "stdlib" } let(:module_version) { "4.1.0" } let(:module_full_name) { "#{module_author}-#{module_name}" } let(:module_full_name_versioned) { "#{module_full_name}-#{module_version}" } let(:module_md5) { "bbf919d7ee9d278d2facf39c25578bf8" } let(:uri) { " "} let(:release) { Puppet::Forge::ModuleRelease.new(ssl_repository, JSON.parse(release_json)) } let(:mock_file) { mock_io = StringIO.new mock_io.stubs(:path).returns('/dev/null') mock_io } let(:mock_dir) { '/tmp' } shared_examples 'a module release' do def mock_digest_file_with_md5(md5) Digest::MD5.stubs(:file).returns(stub(:hexdigest => md5)) end describe '#prepare' do before :each do release.stubs(:tmpfile).returns(mock_file) release.stubs(:tmpdir).returns(mock_dir) end it 'should call sub methods with correct params' do release.expects(:download).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file) release.expects(:validate_checksum).with(mock_file, module_md5) release.expects(:unpack).with(mock_file, mock_dir) release.prepare end end describe '#tmpfile' do # This is impossible to test under Ruby 1.8.x, but should also occur there. it 'should be opened in binary mode', :unless => RUBY_VERSION >= '1.8.7' do Puppet::Forge::Cache.stubs(:base_path).returns(Dir.tmpdir) release.send(:tmpfile).binmode?.should be_true end end describe '#download' do it 'should call make_http_request with correct params' do # valid URI comes from file_uri in JSON blob above - ssl_repository.expects(:make_http_request).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file).returns(mock_file) + ssl_repository.expects(:make_http_request).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file).returns(stub(:body => '{}', :code => 200)) release.send(:download, "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file) end + + it 'should raise a response error when it receives an error from forge' do + ssl_repository.stubs(:make_http_request).returns(stub(:body => '{"errors": ["error"]}', :code => 500, :message => 'server error')) + expect { release.send(:download, "/some/path", mock_file)}. to raise_error Puppet::Forge::Errors::ResponseError + + end end describe '#verify_checksum' do it 'passes md5 check when valid' do # valid hash comes from file_md5 in JSON blob above mock_digest_file_with_md5(module_md5) release.send(:validate_checksum, mock_file, module_md5) end it 'fails md5 check when invalid' do mock_digest_file_with_md5('ffffffffffffffffffffffffffffffff') expect { release.send(:validate_checksum, mock_file, module_md5) }.to raise_error(RuntimeError, /did not match expected checksum/) end end describe '#unpack' do it 'should call unpacker with correct params' do Puppet::ModuleTool::Applications::Unpacker.expects(:unpack).with(mock_file.path, mock_dir).returns(true) release.send(:unpack, mock_file, mock_dir) end end end context 'standard forge module' do let(:release_json) do %Q{ { "uri": "/#{api_version}/releases/#{module_full_name_versioned}", "module": { "uri": "/#{api_version}/modules/#{module_full_name}", "name": "#{module_name}", "owner": { "uri": "/#{api_version}/users/#{module_author}", "username": "#{module_author}", "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" } }, "version": "#{module_version}", "metadata": { "types": [ ], "license": "Apache 2.0", "checksums": { }, "version": "#{module_version}", "description": "Standard Library for Puppet Modules", "source": "git://github.com/puppetlabs/puppetlabs-stdlib.git", "project_page": "https://github.com/puppetlabs/puppetlabs-stdlib", "summary": "Puppet Module Standard Library", "dependencies": [ ], "author": "#{module_author}", "name": "#{module_full_name}" }, "tags": [ "puppetlabs", "library", "stdlib", "standard", "stages" ], "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", "file_size": 67586, "file_md5": "#{module_md5}", "downloads": 610751, "readme": "", "changelog": "", "license": "", "created_at": "2013-05-13 08:31:19 -0700", "updated_at": "2013-05-13 08:31:19 -0700", "deleted_at": null } } end it_behaves_like 'a module release' end context 'forge module with no dependencies field' do let(:release_json) do %Q{ { "uri": "/#{api_version}/releases/#{module_full_name_versioned}", "module": { "uri": "/#{api_version}/modules/#{module_full_name}", "name": "#{module_name}", "owner": { "uri": "/#{api_version}/users/#{module_author}", "username": "#{module_author}", "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" } }, "version": "#{module_version}", "metadata": { "types": [ ], "license": "Apache 2.0", "checksums": { }, "version": "#{module_version}", "description": "Standard Library for Puppet Modules", "source": "git://github.com/puppetlabs/puppetlabs-stdlib.git", "project_page": "https://github.com/puppetlabs/puppetlabs-stdlib", "summary": "Puppet Module Standard Library", "author": "#{module_author}", "name": "#{module_full_name}" }, "tags": [ "puppetlabs", "library", "stdlib", "standard", "stages" ], "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", "file_size": 67586, "file_md5": "#{module_md5}", "downloads": 610751, "readme": "", "changelog": "", "license": "", "created_at": "2013-05-13 08:31:19 -0700", "updated_at": "2013-05-13 08:31:19 -0700", "deleted_at": null } } end it_behaves_like 'a module release' end context 'forge module with the minimal set of fields' do let(:release_json) do %Q{ { "uri": "/#{api_version}/releases/#{module_full_name_versioned}", "module": { "uri": "/#{api_version}/modules/#{module_full_name}", "name": "#{module_name}" }, "metadata": { "version": "#{module_version}", "name": "#{module_full_name}" }, "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", "file_size": 67586, "file_md5": "#{module_md5}" } } end it_behaves_like 'a module release' end end diff --git a/spec/unit/forge_spec.rb b/spec/unit/forge_spec.rb index 7847d86d7..fdf6a4393 100644 --- a/spec/unit/forge_spec.rb +++ b/spec/unit/forge_spec.rb @@ -1,157 +1,157 @@ require 'spec_helper' require 'puppet/forge' require 'net/http' require 'puppet/module_tool' describe Puppet::Forge do let(:http_response) do <<-EOF { "pagination": { "limit": 1, "offset": 0, "first": "/v3/modules?limit=1&offset=0", "previous": null, "current": "/v3/modules?limit=1&offset=0", "next": null, "total": 1832 }, "results": [ { "uri": "/v3/modules/puppetlabs-bacula", "name": "bacula", "downloads": 640274, "created_at": "2011-05-24 18:34:58 -0700", "updated_at": "2013-12-03 15:24:20 -0800", "owner": { "uri": "/v3/users/puppetlabs", "username": "puppetlabs", "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" }, "current_release": { "uri": "/v3/releases/puppetlabs-bacula-0.0.2", "module": { "uri": "/v3/modules/puppetlabs-bacula", "name": "bacula", "owner": { "uri": "/v3/users/puppetlabs", "username": "puppetlabs", "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" } }, "version": "0.0.2", "metadata": { "types": [], "license": "Apache 2.0", "checksums": { }, "version": "0.0.2", "source": "git://github.com/puppetlabs/puppetlabs-bacula.git", "project_page": "https://github.com/puppetlabs/puppetlabs-bacula", "summary": "bacula", "dependencies": [ ], "author": "puppetlabs", "name": "puppetlabs-bacula" }, "tags": [ "backup", "bacula" ], "file_uri": "/v3/files/puppetlabs-bacula-0.0.2.tar.gz", "file_size": 67586, "file_md5": "bbf919d7ee9d278d2facf39c25578bf8", "downloads": 565041, "readme": "", "changelog": "", "license": "", "created_at": "2013-05-13 08:31:19 -0700", "updated_at": "2013-05-13 08:31:19 -0700", "deleted_at": null }, "releases": [ { "uri": "/v3/releases/puppetlabs-bacula-0.0.2", "version": "0.0.2" }, { "uri": "/v3/releases/puppetlabs-bacula-0.0.1", "version": "0.0.1" } ], "homepage_url": "https://github.com/puppetlabs/puppetlabs-bacula", "issues_url": "https://projects.puppetlabs.com/projects/bacula/issues" } ] } EOF end let(:search_results) do JSON.parse(http_response)['results'].map do |hash| hash.merge( "author" => "puppetlabs", "name" => "bacula", "tag_list" => ["backup", "bacula"], "full_name" => "puppetlabs/bacula", "version" => "0.0.2", "project_url" => "https://github.com/puppetlabs/puppetlabs-bacula", "desc" => "bacula" ) end end let(:forge) { Puppet::Forge.new } def repository_responds_with(response) Puppet::Forge::Repository.any_instance.stubs(:make_http_request).returns(response) end it "returns a list of matches from the forge when there are matches for the search term" do repository_responds_with(stub(:body => http_response, :code => '200')) forge.search('bacula').should == search_results end context "when module_groups are defined" do let(:release_response) do releases = JSON.parse(http_response) releases['results'] = [] JSON.dump(releases) end before :each do repository_responds_with(stub(:body => release_response, :code => '200')).with {|uri| uri =~ /module_groups=foo/} Puppet[:module_groups] = "foo" end it "passes module_groups with search" do forge.search('bacula') end it "passes module_groups with fetch" do forge.fetch('puppetlabs-bacula') end end context "when the connection to the forge fails" do before :each do repository_responds_with(stub(:body => '{}', :code => '404', :message => "not found")) end it "raises an error for search" do - expect { forge.search('bacula') }.to raise_error Puppet::Forge::Errors::ResponseError, "Could not execute operation for 'bacula'. Detail: 404 not found." + expect { forge.search('bacula') }.to raise_error Puppet::Forge::Errors::ResponseError, "Request to Puppet Forge failed. Detail: 404 not found." end it "raises an error for fetch" do - expect { forge.fetch('puppetlabs/bacula') }.to raise_error Puppet::Forge::Errors::ResponseError, "Could not execute operation for 'puppetlabs/bacula'. Detail: 404 not found." + expect { forge.fetch('puppetlabs/bacula') }.to raise_error Puppet::Forge::Errors::ResponseError, "Request to Puppet Forge failed. Detail: 404 not found." end end context "when the API responds with an error" do before :each do repository_responds_with(stub(:body => '{"error":"invalid module"}', :code => '410', :message => "Gone")) end it "raises an error for fetch" do - expect { forge.fetch('puppetlabs/bacula') }.to raise_error Puppet::Forge::Errors::ResponseError, "Could not execute operation for 'puppetlabs/bacula'. Detail: 410 Gone." + expect { forge.fetch('puppetlabs/bacula') }.to raise_error Puppet::Forge::Errors::ResponseError, "Request to Puppet Forge failed. Detail: 410 Gone." end end end