diff --git a/lib/puppet/forge.rb b/lib/puppet/forge.rb index d442bdeef..1dd95c41e 100644 --- a/lib/puppet/forge.rb +++ b/lib/puppet/forge.rb @@ -1,205 +1,213 @@ require 'puppet/vendor' Puppet::Vendor.load_vendored require 'net/http' require 'tempfile' require 'uri' require 'pathname' require 'json' require 'semantic' class Puppet::Forge < Semantic::Dependency::Source require 'puppet/forge/cache' require 'puppet/forge/repository' require 'puppet/forge/errors' include Puppet::Forge::Errors USER_AGENT = "PMT/1.1.1 (v3; Net::HTTP)".freeze attr_reader :host, :repository def initialize(host = Puppet[:module_repository]) @host = host @repository = Puppet::Forge::Repository.new(host, USER_AGENT) end # Return a list of module metadata hashes that match the search query. # This return value is used by the module_tool face install search, # and displayed to on the console. # # Example return value: # # [ # { # "author" => "puppetlabs", # "name" => "bacula", # "tag_list" => ["backup", "bacula"], # "releases" => [{"version"=>"0.0.1"}, {"version"=>"0.0.2"}], # "full_name" => "puppetlabs/bacula", # "version" => "0.0.2", # "project_url" => "http://github.com/puppetlabs/puppetlabs-bacula", # "desc" => "bacula" # } # ] # # @param term [String] search term # @return [Array] modules found # @raise [Puppet::Forge::Errors::CommunicationError] if there is a network # related error # @raise [Puppet::Forge::Errors::SSLVerifyError] if there is a problem # verifying the remote SSL certificate # @raise [Puppet::Forge::Errors::ResponseError] if the repository returns a # bad HTTP response def search(term) matches = [] uri = "/v3/modules?query=#{URI.escape(term)}" if Puppet[:module_groups] uri += "&module_groups=#{Puppet[:module_groups]}" end while uri response = make_http_request(uri) if response.code == '200' result = JSON.parse(response.body) uri = result['pagination']['next'] matches.concat result['results'] else raise ResponseError.new(:uri => URI.parse(@host).merge(uri) , :input => term, :response => response) end end matches.each do |mod| mod['author'] = mod['owner']['username'] mod['tag_list'] = mod['current_release']['tags'] mod['full_name'] = "#{mod['author']}/#{mod['name']}" mod['version'] = mod['current_release']['version'] mod['project_url'] = mod['homepage_url'] mod['desc'] = mod['current_release']['metadata']['summary'] || '' end end # Fetches {ModuleRelease} entries for each release of the named module. # # @param input [String] the module name to look up # @return [Array] a list of releases for # the given name # @see Semantic::Dependency::Source#fetch def fetch(input) name = input.tr('/', '-') uri = "/v3/releases?module=#{name}" if Puppet[:module_groups] uri += "&module_groups=#{Puppet[:module_groups]}" end releases = [] while uri response = make_http_request(uri) if response.code == '200' response = JSON.parse(response.body) else raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :input => input, :response => response) end releases.concat(process(response['results'])) uri = response['pagination']['next'] end return releases end def make_http_request(*args) @repository.make_http_request(*args) end class ModuleRelease < Semantic::Dependency::ModuleRelease attr_reader :install_dir, :metadata def initialize(source, data) @data = data @metadata = meta = data['metadata'] name = meta['name'].tr('/', '-') version = Semantic::Version.parse(meta['version']) release = "#{name}@#{version}" - dependencies = (meta['dependencies'] || []) - dependencies.map! do |dep| - Puppet::ModuleTool.parse_module_dependency(release, dep)[0..1] + if meta['dependencies'] + dependencies = meta['dependencies'].collect do |dep| + begin + Puppet::ModuleTool::Metadata.new.add_dependency(dep['name'], dep['version_requirement'], dep['repository']) + Puppet::ModuleTool.parse_module_dependency(release, dep)[0..1] + rescue ArgumentError => e + Puppet.debug "Malformed dependency: #{dep['name']}. Exception was: #{e}" + end + end + else + dependencies = [] end super(source, name, version, Hash[dependencies]) end def install(dir) staging_dir = self.prepare module_dir = dir + name[/-(.*)/, 1] module_dir.rmtree if module_dir.exist? # Make sure unpacked module has the same ownership as the folder we are moving it into. Puppet::ModuleTool::Applications::Unpacker.harmonize_ownership(dir, staging_dir) FileUtils.mv(staging_dir, module_dir) @install_dir = dir # Return the Pathname object representing the directory where the # module release archive was unpacked the to. return module_dir ensure staging_dir.rmtree if staging_dir.exist? end def prepare return @unpacked_into if @unpacked_into download(@data['file_uri'], tmpfile) validate_checksum(tmpfile, @data['file_md5']) unpack(tmpfile, tmpdir) @unpacked_into = Pathname.new(tmpdir) end private # Obtain a suitable temporary path for unpacking tarballs # # @return [Pathname] path to temporary unpacking location def tmpdir @dir ||= Dir.mktmpdir(name, Puppet::Forge::Cache.base_path) end def tmpfile @file ||= Tempfile.new(name, Puppet::Forge::Cache.base_path).tap do |f| f.binmode end end def download(uri, destination) @source.make_http_request(uri, destination) destination.flush and destination.close end def validate_checksum(file, checksum) if Digest::MD5.file(file.path).hexdigest != checksum raise RuntimeError, "Downloaded release for #{name} did not match expected checksum" end end def unpack(file, destination) begin Puppet::ModuleTool::Applications::Unpacker.unpack(file.path, destination) rescue Puppet::ExecutionFailure => e raise RuntimeError, "Could not extract contents of module archive: #{e.message}" end end end private def process(list) list.map { |release| ModuleRelease.new(self, release) } end end diff --git a/lib/puppet/module_tool/dependency.rb b/lib/puppet/module_tool/dependency.rb index c0161dc0a..2f0995e58 100644 --- a/lib/puppet/module_tool/dependency.rb +++ b/lib/puppet/module_tool/dependency.rb @@ -1,48 +1,42 @@ require 'puppet/module_tool' require 'puppet/network/format_support' module Puppet::ModuleTool class Dependency include Puppet::Network::FormatSupport alias :to_json :to_pson attr_reader :full_module_name, :username, :name, :version_requirement, :repository # Instantiates a new module dependency with a +full_module_name+ (e.g. # "myuser-mymodule"), and optional +version_requirement+ (e.g. "0.0.1") and # optional repository (a URL string). def initialize(full_module_name, version_requirement = nil, repository = nil) @full_module_name = full_module_name # TODO: add error checking, the next line raises ArgumentError when +full_module_name+ is invalid @username, @name = Puppet::ModuleTool.username_and_modname_from(full_module_name) @version_requirement = version_requirement @repository = repository ? Puppet::Forge::Repository.new(repository) : nil end # We override Object's ==, eql, and hash so we can more easily find identical # dependencies. def ==(o) - @full_module_name == o.full_module_name && - @username == o.username && - @name == o.name && - @version_requirement == o.version_requirement && - @repository == o.repository + self.hash == o.hash end - def eql?(o) - self == o - end + alias :eql? :== def hash - [@full_module_name, @username, @name, @version_requirement, @repository].hash + [@full_module_name, @version_requirement, @repository].hash end def to_data_hash result = { :name => @full_module_name } result[:version_requirement] = @version_requirement if @version_requirement && ! @version_requirement.nil? result[:repository] = @repository.to_s if @repository && ! @repository.nil? result end end end diff --git a/lib/puppet/module_tool/metadata.rb b/lib/puppet/module_tool/metadata.rb index 23f6f01c3..0237555ad 100644 --- a/lib/puppet/module_tool/metadata.rb +++ b/lib/puppet/module_tool/metadata.rb @@ -1,214 +1,209 @@ require 'puppet/util/methodhelper' require 'puppet/module_tool' require 'puppet/network/format_support' require 'uri' require 'json' require 'set' module Puppet::ModuleTool # This class provides a data structure representing a module's metadata. # @api private class Metadata include Puppet::Network::FormatSupport attr_accessor :module_name DEFAULTS = { 'name' => nil, 'version' => nil, 'author' => nil, 'summary' => nil, 'license' => 'Apache 2.0', 'source' => '', 'project_page' => nil, 'issues_url' => nil, 'dependencies' => Set.new.freeze, } def initialize @data = DEFAULTS.dup @data['dependencies'] = @data['dependencies'].dup end # Returns a filesystem-friendly version of this module name. def dashed_name @data['name'].tr('/', '-') if @data['name'] end # Returns a string that uniquely represents this version of this module. def release_name return nil unless @data['name'] && @data['version'] [ dashed_name, @data['version'] ].join('-') end alias :name :module_name alias :full_module_name :dashed_name # Merges the current set of metadata with another metadata hash. This # method also handles the validation of module names and versions, in an # effort to be proactive about module publishing constraints. def update(data) process_name(data) if data['name'] process_version(data) if data['version'] process_source(data) if data['source'] - process_dependencies(data) if data['dependencies'] + merge_dependencies(data) if data['dependencies'] @data.merge!(data) return self end # Validates the name and version_requirement for a dependency, then creates # the Dependency and adds it. # Returns the Dependency that was added. def add_dependency(name, version_requirement=nil, repository=nil) validate_name(name) validate_version_range(version_requirement) if version_requirement - duplicates = @data['dependencies'].select { |d| d.full_module_name == name } - unless duplicates.empty? - duplicates.each do |dup| - if dup.version_requirement != version_requirement - raise ArgumentError, "Dependency conflict for #{full_module_name}: Dependency #{name} was given conflicting version requirements #{version_requirement} and #{dup.version_requirement}. Verify that there are no duplicates in the metadata.json or the Modulefile." - end - end + + if dup = @data['dependencies'].find { |d| d.full_module_name == name && d.version_requirement != version_requirement } + raise ArgumentError, "Dependency conflict for #{full_module_name}: Dependency #{name} was given conflicting version requirements #{version_requirement} and #{dup.version_requirement}. Verify that there are no duplicates in the metadata.json or the Modulefile." end + dep = Dependency.new(name, version_requirement, repository) @data['dependencies'].add(dep) dep end # Provides an accessor for the now defunct 'description' property. This # addresses a regression in Puppet 3.6.x where previously valid templates # refering to the 'description' property were broken. # @deprecated def description @data['description'] end def dependencies @data['dependencies'].to_a end # Returns a hash of the module's metadata. Used by Puppet's automated # serialization routines. # # @see Puppet::Network::FormatSupport#to_data_hash def to_hash @data end alias :to_data_hash :to_hash def to_json + data = @data.dup.merge('dependencies' => dependencies) + # This is used to simulate an ordered hash. In particular, some keys # are promoted to the top of the serialized hash (while others are # demoted) for human-friendliness. # # This particularly works around the lack of ordered hashes in 1.8.7. promoted_keys = %w[ name version author summary license source ] demoted_keys = %w[ dependencies ] - keys = @data.keys + keys = data.keys keys -= promoted_keys keys -= demoted_keys contents = (promoted_keys + keys + demoted_keys).map do |k| - if k == 'dependencies' - value = @data[k].to_a.to_json - else - value = (JSON.pretty_generate(@data[k]) rescue @data[k].to_json) - end + value = (JSON.pretty_generate(data[k]) rescue data[k].to_json) "#{k.to_json}: #{value}" end "{\n" + contents.join(",\n").gsub(/^/, ' ') + "\n}\n" end # Expose any metadata keys as callable reader methods. def method_missing(name, *args) return @data[name.to_s] if @data.key? name.to_s super end private # Do basic validation and parsing of the name parameter. def process_name(data) validate_name(data['name']) author, @module_name = data['name'].split(/[-\/]/, 2) data['author'] ||= author if @data['author'] == DEFAULTS['author'] end # Do basic validation on the version parameter. def process_version(data) validate_version(data['version']) end # Do basic parsing of the source parameter. If the source is hosted on # GitHub, we can predict sensible defaults for both project_page and # issues_url. def process_source(data) if data['source'] =~ %r[://] source_uri = URI.parse(data['source']) else source_uri = URI.parse("http://#{data['source']}") end if source_uri.host =~ /^(www\.)?github\.com$/ source_uri.scheme = 'https' source_uri.path.sub!(/\.git$/, '') data['project_page'] ||= @data['project_page'] || source_uri.to_s data['issues_url'] ||= @data['issues_url'] || source_uri.to_s.sub(/\/*$/, '') + '/issues' end rescue URI::Error return end # Validates and parses the dependencies. - def process_dependencies(data) + def merge_dependencies(data) data['dependencies'].each do |dep| add_dependency(dep['name'], dep['version_requirement'], dep['repository']) end # Clear dependencies so @data dependencies are not overwritten data.delete 'dependencies' end # Validates that the given module name is both namespaced and well-formed. def validate_name(name) return if name =~ /\A[a-z0-9]+[-\/][a-z][a-z0-9_]*\Z/i namespace, modname = name.split(/[-\/]/, 2) modname = :namespace_missing if namespace == '' err = case modname when nil, '', :namespace_missing "the field must be a namespaced module name" when /[^a-z0-9_]/i "the module name contains non-alphanumeric (or underscore) characters" when /^[^a-z]/i "the module name must begin with a letter" else "the namespace contains non-alphanumeric characters" end raise ArgumentError, "Invalid 'name' field in metadata.json: #{err}" end # Validates that the version string can be parsed as per SemVer. def validate_version(version) return if SemVer.valid?(version) err = "version string cannot be parsed as a valid Semantic Version" raise ArgumentError, "Invalid 'version' field in metadata.json: #{err}" end # Validates that the version range can be parsed by Semantic. def validate_version_range(version_range) Semantic::VersionRange.parse(version_range) rescue ArgumentError => e raise ArgumentError, "Invalid 'version_range' field in metadata.json: #{e}" end end end diff --git a/spec/unit/forge/module_release_spec.rb b/spec/unit/forge/module_release_spec.rb index fbf5e157a..a00d4d9c5 100644 --- a/spec/unit/forge/module_release_spec.rb +++ b/spec/unit/forge/module_release_spec.rb @@ -1,131 +1,215 @@ # encoding: utf-8 require 'spec_helper' require 'puppet/forge' require 'net/http' require 'puppet/module_tool' describe Puppet::Forge::ModuleRelease do let(:agent) { "Test/1.0" } let(:repository) { Puppet::Forge::Repository.new('http://fake.com', agent) } let(:ssl_repository) { Puppet::Forge::Repository.new('https://fake.com', agent) } - - let(:release_json) do - <<-EOF - { - "uri": "/v3/releases/puppetlabs-stdlib-4.1.0", - "module": { - "uri": "/v3/modules/puppetlabs-stdlib", - "name": "stdlib", - "owner": { - "uri": "/v3/users/puppetlabs", - "username": "puppetlabs", - "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" - } - }, - "version": "4.1.0", - "metadata": { - "types": [ ], - "license": "Apache 2.0", - "checksums": { }, - "version": "4.1.0", - "description": "Standard Library for Puppet Modules", - "source": "git://github.com/puppetlabs/puppetlabs-stdlib.git", - "project_page": "https://github.com/puppetlabs/puppetlabs-stdlib", - "summary": "Puppet Module Standard Library", - "dependencies": [ - - ], - "author": "puppetlabs", - "name": "puppetlabs-stdlib" - }, - "tags": [ - "puppetlabs", - "library", - "stdlib", - "standard", - "stages" - ], - "file_uri": "/v3/files/puppetlabs-stdlib-4.1.0.tar.gz", - "file_size": 67586, - "file_md5": "bbf919d7ee9d278d2facf39c25578bf8", - "downloads": 610751, - "readme": "", - "changelog": "", - "license": "", - "created_at": "2013-05-13 08:31:19 -0700", - "updated_at": "2013-05-13 08:31:19 -0700", - "deleted_at": null - } - EOF - end - + let(:api_version) { "v3" } + let(:module_author) { "puppetlabs" } + let(:module_name) { "stdlib" } + let(:module_version) { "4.1.0" } + let(:module_full_name) { "#{module_author}-#{module_name}" } + let(:module_full_name_versioned) { "#{module_full_name}-#{module_version}" } + let(:module_md5) { "bbf919d7ee9d278d2facf39c25578bf8" } + let(:uri) { " "} let(:release) { Puppet::Forge::ModuleRelease.new(ssl_repository, JSON.parse(release_json)) } let(:mock_file) { mock_io = StringIO.new mock_io.stubs(:path).returns('/dev/null') mock_io } let(:mock_dir) { '/tmp' } - def mock_digest_file_with_md5(md5) - Digest::MD5.stubs(:file).returns(stub(:hexdigest => md5)) - end - - describe '#prepare' do - before :each do - release.stubs(:tmpfile).returns(mock_file) - release.stubs(:tmpdir).returns(mock_dir) + shared_examples 'a module release' do + def mock_digest_file_with_md5(md5) + Digest::MD5.stubs(:file).returns(stub(:hexdigest => md5)) end - it 'should call sub methods with correct params' do - release.expects(:download).with('/v3/files/puppetlabs-stdlib-4.1.0.tar.gz', mock_file) - release.expects(:validate_checksum).with(mock_file, 'bbf919d7ee9d278d2facf39c25578bf8') - release.expects(:unpack).with(mock_file, mock_dir) + describe '#prepare' do + before :each do + release.stubs(:tmpfile).returns(mock_file) + release.stubs(:tmpdir).returns(mock_dir) + end + + it 'should call sub methods with correct params' do + release.expects(:download).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file) + release.expects(:validate_checksum).with(mock_file, module_md5) + release.expects(:unpack).with(mock_file, mock_dir) - release.prepare + release.prepare + end end - end - describe '#tmpfile' do + describe '#tmpfile' do - # This is impossible to test under Ruby 1.8.x, but should also occur there. - it 'should be opened in binary mode', :unless => RUBY_VERSION >= '1.8.7' do - Puppet::Forge::Cache.stubs(:base_path).returns(Dir.tmpdir) - release.send(:tmpfile).binmode?.should be_true + # This is impossible to test under Ruby 1.8.x, but should also occur there. + it 'should be opened in binary mode', :unless => RUBY_VERSION >= '1.8.7' do + Puppet::Forge::Cache.stubs(:base_path).returns(Dir.tmpdir) + release.send(:tmpfile).binmode?.should be_true + end end - end - describe '#download' do - it 'should call make_http_request with correct params' do - # valid URI comes from file_uri in JSON blob above - ssl_repository.expects(:make_http_request).with('/v3/files/puppetlabs-stdlib-4.1.0.tar.gz', mock_file).returns(mock_file) + describe '#download' do + it 'should call make_http_request with correct params' do + # valid URI comes from file_uri in JSON blob above + ssl_repository.expects(:make_http_request).with("/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file).returns(mock_file) - release.send(:download, '/v3/files/puppetlabs-stdlib-4.1.0.tar.gz', mock_file) + release.send(:download, "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", mock_file) + end end - end - describe '#verify_checksum' do - it 'passes md5 check when valid' do - # valid hash comes from file_md5 in JSON blob above - mock_digest_file_with_md5('bbf919d7ee9d278d2facf39c25578bf8') + describe '#verify_checksum' do + it 'passes md5 check when valid' do + # valid hash comes from file_md5 in JSON blob above + mock_digest_file_with_md5(module_md5) - release.send(:validate_checksum, mock_file, 'bbf919d7ee9d278d2facf39c25578bf8') + release.send(:validate_checksum, mock_file, module_md5) + end + + it 'fails md5 check when invalid' do + mock_digest_file_with_md5('ffffffffffffffffffffffffffffffff') + + expect { release.send(:validate_checksum, mock_file, module_md5) }.to raise_error(RuntimeError, /did not match expected checksum/) + end end - it 'fails md5 check when invalid' do - mock_digest_file_with_md5('ffffffffffffffffffffffffffffffff') + describe '#unpack' do + it 'should call unpacker with correct params' do + Puppet::ModuleTool::Applications::Unpacker.expects(:unpack).with(mock_file.path, mock_dir).returns(true) + + release.send(:unpack, mock_file, mock_dir) + end + end + end - expect { release.send(:validate_checksum, mock_file, 'bbf919d7ee9d278d2facf39c25578bf8') }.to raise_error(RuntimeError, /did not match expected checksum/) + context 'standard forge module' do + let(:release_json) do %Q{ + { + "uri": "/#{api_version}/releases/#{module_full_name_versioned}", + "module": { + "uri": "/#{api_version}/modules/#{module_full_name}", + "name": "#{module_name}", + "owner": { + "uri": "/#{api_version}/users/#{module_author}", + "username": "#{module_author}", + "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" + } + }, + "version": "#{module_version}", + "metadata": { + "types": [ ], + "license": "Apache 2.0", + "checksums": { }, + "version": "#{module_version}", + "description": "Standard Library for Puppet Modules", + "source": "git://github.com/puppetlabs/puppetlabs-stdlib.git", + "project_page": "https://github.com/puppetlabs/puppetlabs-stdlib", + "summary": "Puppet Module Standard Library", + "dependencies": [ + + ], + "author": "#{module_author}", + "name": "#{module_full_name}" + }, + "tags": [ + "puppetlabs", + "library", + "stdlib", + "standard", + "stages" + ], + "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", + "file_size": 67586, + "file_md5": "#{module_md5}", + "downloads": 610751, + "readme": "", + "changelog": "", + "license": "", + "created_at": "2013-05-13 08:31:19 -0700", + "updated_at": "2013-05-13 08:31:19 -0700", + "deleted_at": null + } + } end + + it_behaves_like 'a module release' end - describe '#unpack' do - it 'should call unpacker with correct params' do - Puppet::ModuleTool::Applications::Unpacker.expects(:unpack).with(mock_file.path, mock_dir).returns(true) + context 'forge module with no dependencies field' do + let(:release_json) do %Q{ + { + "uri": "/#{api_version}/releases/#{module_full_name_versioned}", + "module": { + "uri": "/#{api_version}/modules/#{module_full_name}", + "name": "#{module_name}", + "owner": { + "uri": "/#{api_version}/users/#{module_author}", + "username": "#{module_author}", + "gravatar_id": "fdd009b7c1ec96e088b389f773e87aec" + } + }, + "version": "#{module_version}", + "metadata": { + "types": [ ], + "license": "Apache 2.0", + "checksums": { }, + "version": "#{module_version}", + "description": "Standard Library for Puppet Modules", + "source": "git://github.com/puppetlabs/puppetlabs-stdlib.git", + "project_page": "https://github.com/puppetlabs/puppetlabs-stdlib", + "summary": "Puppet Module Standard Library", + "author": "#{module_author}", + "name": "#{module_full_name}" + }, + "tags": [ + "puppetlabs", + "library", + "stdlib", + "standard", + "stages" + ], + "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", + "file_size": 67586, + "file_md5": "#{module_md5}", + "downloads": 610751, + "readme": "", + "changelog": "", + "license": "", + "created_at": "2013-05-13 08:31:19 -0700", + "updated_at": "2013-05-13 08:31:19 -0700", + "deleted_at": null + } + } + end + + it_behaves_like 'a module release' + end - release.send(:unpack, mock_file, mock_dir) + context 'forge module with the minimal set of fields' do + let(:release_json) do %Q{ + { + "uri": "/#{api_version}/releases/#{module_full_name_versioned}", + "module": { + "uri": "/#{api_version}/modules/#{module_full_name}", + "name": "#{module_name}" + }, + "metadata": { + "version": "#{module_version}", + "name": "#{module_full_name}" + }, + "file_uri": "/#{api_version}/files/#{module_full_name_versioned}.tar.gz", + "file_size": 67586, + "file_md5": "#{module_md5}" + } + } end + + it_behaves_like 'a module release' end end