diff --git a/lib/puppet/forge.rb b/lib/puppet/forge.rb index b2ec0e139..38a6efb0b 100644 --- a/lib/puppet/forge.rb +++ b/lib/puppet/forge.rb @@ -1,216 +1,216 @@ require 'puppet/vendor' Puppet::Vendor.load_vendored require 'net/http' require 'tempfile' require 'uri' require 'pathname' require 'json' require 'semantic' class Puppet::Forge < Semantic::Dependency::Source require 'puppet/forge/cache' require 'puppet/forge/repository' require 'puppet/forge/errors' include Puppet::Forge::Errors USER_AGENT = "PMT/1.1.1 (v3; Net::HTTP)".freeze attr_reader :host, :repository def initialize(host = Puppet[:module_repository]) @host = host @repository = Puppet::Forge::Repository.new(host, USER_AGENT) end # Return a list of module metadata hashes that match the search query. # This return value is used by the module_tool face install search, # and displayed to on the console. # # Example return value: # # [ # { # "author" => "puppetlabs", # "name" => "bacula", # "tag_list" => ["backup", "bacula"], # "releases" => [{"version"=>"0.0.1"}, {"version"=>"0.0.2"}], # "full_name" => "puppetlabs/bacula", # "version" => "0.0.2", # "project_url" => "http://github.com/puppetlabs/puppetlabs-bacula", # "desc" => "bacula" # } # ] # # @param term [String] search term # @return [Array] modules found # @raise [Puppet::Forge::Errors::CommunicationError] if there is a network # related error # @raise [Puppet::Forge::Errors::SSLVerifyError] if there is a problem # verifying the remote SSL certificate # @raise [Puppet::Forge::Errors::ResponseError] if the repository returns a # bad HTTP response def search(term) matches = [] uri = "/v3/modules?query=#{URI.escape(term)}" if Puppet[:module_groups] uri += "&module_groups=#{Puppet[:module_groups]}" end while uri response = make_http_request(uri) if response.code == '200' result = JSON.parse(response.body) uri = result['pagination']['next'] matches.concat result['results'] else raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :response => response) end end matches.each do |mod| mod['author'] = mod['owner']['username'] mod['tag_list'] = mod['current_release']['tags'] mod['full_name'] = "#{mod['author']}/#{mod['name']}" mod['version'] = mod['current_release']['version'] mod['project_url'] = mod['homepage_url'] mod['desc'] = mod['current_release']['metadata']['summary'] || '' end end # Fetches {ModuleRelease} entries for each release of the named module. # # @param input [String] the module name to look up # @return [Array] a list of releases for # the given name # @see Semantic::Dependency::Source#fetch def fetch(input) name = input.tr('/', '-') uri = "/v3/releases?module=#{name}" if Puppet[:module_groups] uri += "&module_groups=#{Puppet[:module_groups]}" end releases = [] while uri response = make_http_request(uri) if response.code == '200' response = JSON.parse(response.body) else raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :response => response) end releases.concat(process(response['results'])) uri = response['pagination']['next'] end return releases end def make_http_request(*args) @repository.make_http_request(*args) end class ModuleRelease < Semantic::Dependency::ModuleRelease attr_reader :install_dir, :metadata def initialize(source, data) @data = data @metadata = meta = data['metadata'] name = meta['name'].tr('/', '-') version = Semantic::Version.parse(meta['version']) release = "#{name}@#{version}" if meta['dependencies'] dependencies = meta['dependencies'].collect do |dep| begin Puppet::ModuleTool::Metadata.new.add_dependency(dep['name'], dep['version_requirement'], dep['repository']) Puppet::ModuleTool.parse_module_dependency(release, dep)[0..1] rescue ArgumentError => e Puppet.debug "Malformed dependency: #{dep['name']}. Exception was: #{e}" end end else dependencies = [] end super(source, name, version, Hash[dependencies]) end def install(dir) staging_dir = self.prepare module_dir = dir + name[/-(.*)/, 1] module_dir.rmtree if module_dir.exist? # Make sure unpacked module has the same ownership as the folder we are moving it into. Puppet::ModuleTool::Applications::Unpacker.harmonize_ownership(dir, staging_dir) FileUtils.mv(staging_dir, module_dir) @install_dir = dir # Return the Pathname object representing the directory where the # module release archive was unpacked the to. return module_dir ensure staging_dir.rmtree if staging_dir.exist? end def prepare return @unpacked_into if @unpacked_into download(@data['file_uri'], tmpfile) validate_checksum(tmpfile, @data['file_md5']) unpack(tmpfile, tmpdir) @unpacked_into = Pathname.new(tmpdir) end private # Obtain a suitable temporary path for unpacking tarballs # # @return [Pathname] path to temporary unpacking location def tmpdir @dir ||= Dir.mktmpdir(name, Puppet::Forge::Cache.base_path) end def tmpfile @file ||= Tempfile.new(name, Puppet::Forge::Cache.base_path).tap do |f| f.binmode end end def download(uri, destination) response = @source.make_http_request(uri, destination) destination.flush and destination.close - unless response.code == 200 + unless response.code == '200' raise Puppet::Forge::Errors::ResponseError.new(:uri => uri, :response => response) end end def validate_checksum(file, checksum) if Digest::MD5.file(file.path).hexdigest != checksum raise RuntimeError, "Downloaded release for #{name} did not match expected checksum" end end def unpack(file, destination) begin Puppet::ModuleTool::Applications::Unpacker.unpack(file.path, destination) rescue Puppet::ExecutionFailure => e raise RuntimeError, "Could not extract contents of module archive: #{e.message}" end end end private def process(list) list.map { |release| ModuleRelease.new(self, release) } end end diff --git a/spec/unit/forge/module_release_spec.rb b/spec/unit/forge/module_release_spec.rb index ea608d83f..b763f0833 100644 --- a/spec/unit/forge/module_release_spec.rb +++ b/spec/unit/forge/module_release_spec.rb @@ -1,221 +1,220 @@ # encoding: utf-8 require 'spec_helper' require 'puppet/forge' require 'net/http' require 'puppet/module_tool' describe Puppet::Forge::ModuleRelease do let(:agent) { "Test/1.0" } let(:repository) { Puppet::Forge::Repository.new('http://fake.com', agent) } let(:ssl_repository) { Puppet::Forge::Repository.new('https://fake.com', agent) } let(:api_version) { "v3" } let(:module_author) { "puppetlabs" } let(:module_name) { "stdlib" } let(:module_version) { "4.1.0" } let(:module_full_name) { "#{module_author}-#{module_name}" } let(:module_full_name_versioned) { "#{module_full_name}-#{module_version}" } let(:module_md5) { "bbf919d7ee9d278d2facf39c25578bf8" } let(:uri) { " "} let(:release) { Puppet::Forge::ModuleRelease.new(ssl_repository, JSON.parse(release_json)) } let(:mock_file) { mock_io = StringIO.new mock_io.stubs(:path).returns('/dev/null') mock_io } let(:mock_dir) { '/tmp' } shared_examples 'a module release' do def mock_digest_file_with_md5(md5) Digest::MD5.stubs(:file).returns(stub(:hexdigest => md5)) end describe '#prepare' do before :each do release.stubs(:tmpfile).returns(mock_file) release.stubs(:tmpdir).returns(mock_dir) end it 'should call sub methods with correct params' do release.expects(:download).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file) release.expects(:validate_checksum).with(mock_file, module_md5) release.expects(:unpack).with(mock_file, mock_dir) release.prepare end end describe '#tmpfile' do # This is impossible to test under Ruby 1.8.x, but should also occur there. it 'should be opened in binary mode', :unless => RUBY_VERSION >= '1.8.7' do Puppet::Forge::Cache.stubs(:base_path).returns(Dir.tmpdir) release.send(:tmpfile).binmode?.should be_true end end describe '#download' do it 'should call make_http_request with correct params' do # valid URI comes from file_uri in JSON blob above - ssl_repository.expects(:make_http_request).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file).returns(stub(:body => '{}', :code => 200)) + ssl_repository.expects(:make_http_request).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file).returns(stub(:body => '{}', :code => '200')) release.send(:download, "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file) end it 'should raise a response error when it receives an error from forge' do - ssl_repository.stubs(:make_http_request).returns(stub(:body => '{"errors": ["error"]}', :code => 500, :message => 'server error')) + ssl_repository.stubs(:make_http_request).returns(stub(:body => '{"errors": ["error"]}', :code => '500', :message => 'server error')) expect { release.send(:download, "/some/path", mock_file)}. to raise_error Puppet::Forge::Errors::ResponseError - end end describe '#verify_checksum' do it 'passes md5 check when valid' do # valid hash comes from file_md5 in JSON blob above mock_digest_file_with_md5(module_md5) release.send(:validate_checksum, mock_file, module_md5) end it 'fails md5 check when invalid' do mock_digest_file_with_md5('ffffffffffffffffffffffffffffffff') expect { release.send(:validate_checksum, mock_file, module_md5) }.to raise_error(RuntimeError, /did not match expected checksum/) end end describe '#unpack' do it 'should call unpacker with correct params' do Puppet::ModuleTool::Applications::Unpacker.expects(:unpack).with(mock_file.path, mock_dir).returns(true) release.send(:unpack, mock_file, mock_dir) end end end context 'standard forge module' do let(:release_json) do %Q{ { "uri": "/#{api_version}/releases/#{module_full_name_versioned}", "module": { "uri": "/#{api_version}/modules/#{module_full_name}", "name": "#{module_name}", "owner": { "uri": "/#{api_version}/users/#{module_author}", "username": "#{module_author}", "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" } }, "version": "#{module_version}", "metadata": { "types": [ ], "license": "Apache 2.0", "checksums": { }, "version": "#{module_version}", "description": "Standard Library for Puppet Modules", "source": "git://github.com/puppetlabs/puppetlabs-stdlib.git", "project_page": "https://github.com/puppetlabs/puppetlabs-stdlib", "summary": "Puppet Module Standard Library", "dependencies": [ ], "author": "#{module_author}", "name": "#{module_full_name}" }, "tags": [ "puppetlabs", "library", "stdlib", "standard", "stages" ], "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", "file_size": 67586, "file_md5": "#{module_md5}", "downloads": 610751, "readme": "", "changelog": "", "license": "", "created_at": "2013-05-13 08:31:19 -0700", "updated_at": "2013-05-13 08:31:19 -0700", "deleted_at": null } } end it_behaves_like 'a module release' end context 'forge module with no dependencies field' do let(:release_json) do %Q{ { "uri": "/#{api_version}/releases/#{module_full_name_versioned}", "module": { "uri": "/#{api_version}/modules/#{module_full_name}", "name": "#{module_name}", "owner": { "uri": "/#{api_version}/users/#{module_author}", "username": "#{module_author}", "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" } }, "version": "#{module_version}", "metadata": { "types": [ ], "license": "Apache 2.0", "checksums": { }, "version": "#{module_version}", "description": "Standard Library for Puppet Modules", "source": "git://github.com/puppetlabs/puppetlabs-stdlib.git", "project_page": "https://github.com/puppetlabs/puppetlabs-stdlib", "summary": "Puppet Module Standard Library", "author": "#{module_author}", "name": "#{module_full_name}" }, "tags": [ "puppetlabs", "library", "stdlib", "standard", "stages" ], "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", "file_size": 67586, "file_md5": "#{module_md5}", "downloads": 610751, "readme": "", "changelog": "", "license": "", "created_at": "2013-05-13 08:31:19 -0700", "updated_at": "2013-05-13 08:31:19 -0700", "deleted_at": null } } end it_behaves_like 'a module release' end context 'forge module with the minimal set of fields' do let(:release_json) do %Q{ { "uri": "/#{api_version}/releases/#{module_full_name_versioned}", "module": { "uri": "/#{api_version}/modules/#{module_full_name}", "name": "#{module_name}" }, "metadata": { "version": "#{module_version}", "name": "#{module_full_name}" }, "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", "file_size": 67586, "file_md5": "#{module_md5}" } } end it_behaves_like 'a module release' end end