diff --git a/acceptance/tests/environment/use_environment_from_environmentpath.rb b/acceptance/tests/environment/use_environment_from_environmentpath.rb index ceb673067..c3dd576c4 100644 --- a/acceptance/tests/environment/use_environment_from_environmentpath.rb +++ b/acceptance/tests/environment/use_environment_from_environmentpath.rb @@ -1,180 +1,188 @@ test_name "Use environments from the environmentpath" testdir = master.tmpdir('use_environmentpath') def generate_environment(path_to_env, environment) env_content = <<-EOS "#{path_to_env}/#{environment}":; "#{path_to_env}/#{environment}/manifests":; "#{path_to_env}/#{environment}/modules":; EOS end def generate_module_content(module_name, options = {}) base_path = options[:base_path] environment = options[:environment] env_path = options[:env_path] path_to_module = [base_path, env_path, environment, "modules"].compact.join("/") module_info = "module-#{module_name}" module_info << "-from-#{environment}" if environment module_content = <<-EOS "#{path_to_module}/#{module_name}":; "#{path_to_module}/#{module_name}/manifests":; "#{path_to_module}/#{module_name}/files":; "#{path_to_module}/#{module_name}/templates":; "#{path_to_module}/#{module_name}/lib":; "#{path_to_module}/#{module_name}/lib/facter":; "#{path_to_module}/#{module_name}/manifests/init.pp": ensure => file, mode => 0640, content => 'class #{module_name} { notify { "template-#{module_name}": message => template("#{module_name}/our_template.erb") } file { "$agent_file_location/file-#{module_info}": source => "puppet:///modules/#{module_name}/data" } }' ; "#{path_to_module}/#{module_name}/lib/facter/environment_fact_#{module_name}.rb": ensure => file, mode => 0640, content => "Facter.add(:environment_fact_#{module_name}) { setcode { 'environment fact from #{module_info}' } }" ; "#{path_to_module}/#{module_name}/files/data": ensure => file, mode => 0640, content => "data file from #{module_info}" ; "#{path_to_module}/#{module_name}/templates/our_template.erb": ensure => file, mode => 0640, content => "<%= @environment_fact_#{module_name} %>" ; EOS end def generate_site_manifest(path_to_manifest, *modules_to_include) manifest_content = <<-EOS "#{path_to_manifest}/site.pp": ensure => file, mode => 0640, content => "#{modules_to_include.map { |m| "include #{m}" }.join("\n")}" ; EOS end apply_manifest_on(master, <<-MANIFEST, :catch_failures => true) File { ensure => directory, owner => #{master['user']}, group => #{master['group']}, mode => 0770, } file { "#{testdir}":; "#{testdir}/base":; "#{testdir}/additional":; "#{testdir}/modules":; #{generate_environment("#{testdir}/base", "shadowed")} #{generate_environment("#{testdir}/base", "onlybase")} #{generate_environment("#{testdir}/additional", "shadowed")} #{generate_module_content("atmp", :base_path => testdir, :env_path => 'base', :environment => 'shadowed')} #{generate_site_manifest("#{testdir}/base/shadowed/manifests", "atmp", "globalmod")} #{generate_module_content("atmp", :base_path => testdir, :env_path => 'base', :environment => 'onlybase')} #{generate_site_manifest("#{testdir}/base/onlybase/manifests", "atmp", "globalmod")} #{generate_module_content("atmp", :base_path => testdir, :env_path => 'additional', :environment => 'shadowed')} #{generate_site_manifest("#{testdir}/additional/shadowed/manifests", "atmp", "globalmod")} # And one global module (--modulepath setting) #{generate_module_content("globalmod", :base_path => testdir)} + "#{testdir}/additional/production":; + "#{testdir}/additional/production/manifests":; +#{generate_site_manifest("#{testdir}/additional/production/manifests", "globalmod")} } MANIFEST def run_with_environment(agent, environment, options = {}) expected_exit_code = options[:expected_exit_code] || 2 expected_strings = options[:expected_strings] step "running an agent in environment '#{environment}'" atmp = agent.tmpdir("use_environmentpath_#{environment}") agent_config = [ "-t", "--server", master, ] agent_config << '--environment' << environment if environment # This to test how the agent behaves when using the directory environment # loaders (which will not load an environment if it does not exist) agent_config << "--environmentpath='$confdir/environments'" if agent != master agent_config << { 'ENV' => { "FACTER_agent_file_location" => atmp }, } on(agent, puppet("agent", *agent_config), :acceptable_exit_codes => [expected_exit_code]) do |result| yield atmp, result end on agent, "rm -rf #{atmp}" end master_opts = { 'master' => { 'environmentpath' => "#{testdir}/additional:#{testdir}/base", 'basemodulepath' => "#{testdir}/modules", } } if master.is_pe? master_opts['master']['basemodulepath'] << ":#{master['sitemoduledir']}" end with_puppet_running_on master, master_opts, testdir do agents.each do |agent| run_with_environment(agent, "shadowed") do |tmpdir,catalog_result| ["module-atmp-from-shadowed", "module-globalmod"].each do |expected| assert_match(/environment fact from #{expected}/, catalog_result.stdout) end ["module-atmp-from-shadowed", "module-globalmod"].each do |expected| on agent, "cat #{tmpdir}/file-#{expected}" do |file_result| assert_match(/data file from #{expected}/, file_result.stdout) end end end run_with_environment(agent, "onlybase") do |tmpdir,catalog_result| ["module-atmp-from-onlybase", "module-globalmod"].each do |expected| assert_match(/environment fact from #{expected}/, catalog_result.stdout) end ["module-atmp-from-onlybase", "module-globalmod"].each do |expected| on agent, "cat #{tmpdir}/file-#{expected}" do |file_result| assert_match(/data file from #{expected}/, file_result.stdout) end end end if master.is_pe? step("This test cannot run if the production environment directory does not exist, because the fallback production environment puppet creates has an empty modulepath and PE cannot run without it's basemodulepath in /opt. PUP-2519, which implicitly creates the production environment directory should allow this to run again") else - run_with_environment(agent, nil, :expected_exit_code => 0) do |tmpdir, result| - assert_no_match(/module-atmp/, result.stdout, "module-atmp was included despite no environment being loaded") - assert_match(/Loading facts.*globalmod/, result.stdout) + run_with_environment(agent, nil, :expected_exit_code => 2) do |tmpdir, catalog_result| + assert_no_match(/module-atmp/, catalog_result.stdout, "module-atmp was included despite no environment being loaded") + + assert_match(/environment fact from module-globalmod/, catalog_result.stdout) + + on agent, "cat #{tmpdir}/file-module-globalmod" do |file_result| + assert_match(/data file from module-globalmod/, file_result.stdout) + end end end end end diff --git a/lib/puppet/module_tool/applications/builder.rb b/lib/puppet/module_tool/applications/builder.rb index 0390e4638..73a3989f3 100644 --- a/lib/puppet/module_tool/applications/builder.rb +++ b/lib/puppet/module_tool/applications/builder.rb @@ -1,107 +1,108 @@ require 'fileutils' require 'json' +require 'puppet/file_system' module Puppet::ModuleTool module Applications class Builder < Application def initialize(path, options = {}) @path = File.expand_path(path) @pkg_path = File.join(@path, 'pkg') super(options) end def run load_metadata! sanity_check create_directory copy_contents write_json Puppet.notice "Building #{@path} for release" pack relative = Pathname.new(archive_file).relative_path_from(Pathname.new(File.expand_path(Dir.pwd))) # Return the Pathname object representing the path to the release # archive just created. This return value is used by the module_tool # face build action, and displayed to on the console using the to_s # method. # # Example return value: # # # relative end private def archive_file File.join(@pkg_path, "#{metadata.release_name}.tar.gz") end def pack FileUtils.rm archive_file rescue nil tar = Puppet::ModuleTool::Tar.instance Dir.chdir(@pkg_path) do tar.pack(metadata.release_name, archive_file) end end def create_directory FileUtils.mkdir(@pkg_path) rescue nil if File.directory?(build_path) FileUtils.rm_rf(build_path, :secure => true) end FileUtils.mkdir(build_path) end def copy_contents Dir[File.join(@path, '*')].each do |path| case File.basename(path) when *Puppet::ModuleTool::ARTIFACTS next else FileUtils.cp_r path, build_path, :preserve => true end end end def sanity_check - symlinks = Dir.glob("#{@path}/**/*", File::FNM_DOTMATCH).map { |f| Pathname.new(f) }.select(&:symlink?) + symlinks = Dir.glob("#{@path}/**/*", File::FNM_DOTMATCH).map { |f| Pathname.new(f) }.select {|p| Puppet::FileSystem.symlink? p} dirpath = Pathname.new @path unless symlinks.empty? symlinks.each do |s| Puppet.warning "Symlinks in modules are unsupported. Please investigate symlink #{s.relative_path_from dirpath}->#{s.realpath.relative_path_from dirpath}." end raise Puppet::ModuleTool::Errors::ModuleToolError, "Found symlinks. Symlinks in modules are not allowed, please remove them." end end def write_json metadata_path = File.join(build_path, 'metadata.json') if metadata.to_hash.include? 'checksums' Puppet.warning "A 'checksums' field was found in metadata.json. This field will be ignored and can safely be removed." end # TODO: This may necessarily change the order in which the metadata.json # file is packaged from what was written by the user. This is a # regretable, but required for now. File.open(metadata_path, 'w') do |f| f.write(metadata.to_json) end File.open(File.join(build_path, 'checksums.json'), 'w') do |f| f.write(PSON.pretty_generate(Checksums.new(build_path))) end end def build_path @build_path ||= File.join(@pkg_path, metadata.release_name) end end end end diff --git a/lib/puppet/module_tool/applications/unpacker.rb b/lib/puppet/module_tool/applications/unpacker.rb index 65819e274..8ffef3d64 100644 --- a/lib/puppet/module_tool/applications/unpacker.rb +++ b/lib/puppet/module_tool/applications/unpacker.rb @@ -1,99 +1,100 @@ require 'pathname' require 'tmpdir' require 'json' +require 'puppet/file_system' module Puppet::ModuleTool module Applications class Unpacker < Application def self.unpack(filename, target) app = self.new(filename, :target_dir => target) app.unpack app.sanity_check app.move_into(target) end def self.harmonize_ownership(source, target) unless Puppet.features.microsoft_windows? source = Pathname.new(source) unless source.respond_to?(:stat) target = Pathname.new(target) unless target.respond_to?(:stat) FileUtils.chown_R(source.stat.uid, source.stat.gid, target) end end def initialize(filename, options = {}) @filename = Pathname.new(filename) super(options) @module_path = Pathname(options[:target_dir]) end def run unpack sanity_check module_dir = @module_path + module_name move_into(module_dir) # Return the Pathname object representing the directory where the # module release archive was unpacked the to. return module_dir end # @api private # Error on symlinks and other junk def sanity_check - symlinks = Dir.glob("#{tmpdir}/**/*", File::FNM_DOTMATCH).map { |f| Pathname.new(f) }.select(&:symlink?) + symlinks = Dir.glob("#{tmpdir}/**/*", File::FNM_DOTMATCH).map { |f| Pathname.new(f) }.select {|p| Puppet::FileSystem.symlink? p} tmpdirpath = Pathname.new tmpdir symlinks.each do |s| Puppet.warning "Symlinks in modules are unsupported. Please investigate symlink #{s.relative_path_from tmpdirpath}->#{s.realpath.relative_path_from tmpdirpath}." end end # @api private def unpack begin Puppet::ModuleTool::Tar.instance.unpack(@filename.to_s, tmpdir, [@module_path.stat.uid, @module_path.stat.gid].join(':')) rescue Puppet::ExecutionFailure => e raise RuntimeError, "Could not extract contents of module archive: #{e.message}" end end # @api private def root_dir return @root_dir if @root_dir # Grab the first directory containing a metadata.json file metadata_file = Dir["#{tmpdir}/**/metadata.json"].sort_by(&:length)[0] if metadata_file @root_dir = Pathname.new(metadata_file).dirname else raise "No valid metadata.json found!" end end # @api private def module_name metadata = JSON.parse((root_dir + 'metadata.json').read) name = metadata['name'][/-(.*)/, 1] end # @api private def move_into(dir) dir = Pathname.new(dir) dir.rmtree if dir.exist? FileUtils.mv(root_dir, dir) ensure FileUtils.rmtree(tmpdir) end # Obtain a suitable temporary path for unpacking tarballs # # @api private # @return [String] path to temporary unpacking location def tmpdir @dir ||= Dir.mktmpdir('tmp-unpacker', Puppet::Forge::Cache.base_path) end end end end diff --git a/spec/unit/module_tool/applications/builder_spec.rb b/spec/unit/module_tool/applications/builder_spec.rb index 8891b60da..c301afcf2 100644 --- a/spec/unit/module_tool/applications/builder_spec.rb +++ b/spec/unit/module_tool/applications/builder_spec.rb @@ -1,109 +1,110 @@ require 'spec_helper' +require 'puppet/file_system' require 'puppet/module_tool/applications' require 'puppet_spec/modules' describe Puppet::ModuleTool::Applications::Builder do include PuppetSpec::Files let(:path) { tmpdir("working_dir") } let(:module_name) { 'mymodule-mytarball' } let(:version) { '0.0.1' } let(:release_name) { "#{module_name}-#{version}" } let(:tarball) { File.join(path, 'pkg', release_name) + ".tar.gz" } let(:builder) { Puppet::ModuleTool::Applications::Builder.new(path) } shared_examples "a packagable module" do def target_exists?(file) File.exist?(File.join(path, "pkg", "#{module_name}-#{version}", file)) end it "packages the module in a tarball named after the module" do tarrer = mock('tarrer') Puppet::ModuleTool::Tar.expects(:instance).returns(tarrer) Dir.expects(:chdir).with(File.join(path, 'pkg')).yields tarrer.expects(:pack).with(release_name, tarball) builder.run expect(target_exists?('checksums.json')).to be true expect(target_exists?('metadata.json')).to be true end end context 'with metadata.json' do before :each do File.open(File.join(path, 'metadata.json'), 'w') do |f| f.puts({ "name" => "#{module_name}", "version" => "#{version}", "source" => "http://github.com/testing/#{module_name}", "author" => "testing", "license" => "Apache License Version 2.0", "summary" => "Puppet testing module", "description" => "This module can be used for basic testing", "project_page" => "http://github.com/testing/#{module_name}" }.to_json) end end it_behaves_like "a packagable module" - it "does not package with a symlink" do + it "does not package with a symlink", :if => Puppet.features.manages_symlinks? do FileUtils.touch(File.join(path, 'tempfile')) - File.symlink(File.join(path, 'tempfile'), File.join(path, 'tempfile2')) + Puppet::FileSystem.symlink(File.join(path, 'tempfile'), File.join(path, 'tempfile2')) expect { builder.run }.to raise_error Puppet::ModuleTool::Errors::ModuleToolError, /symlinks/i end - it "does not package with a symlink in a subdir" do + it "does not package with a symlink in a subdir", :if => Puppet.features.manages_symlinks? do FileUtils.mkdir(File.join(path, 'manifests')) FileUtils.touch(File.join(path, 'manifests/tempfile.pp')) - File.symlink(File.join(path, 'manifests/tempfile.pp'), File.join(path, 'manifests/tempfile2.pp')) + Puppet::FileSystem.symlink(File.join(path, 'manifests/tempfile.pp'), File.join(path, 'manifests/tempfile2.pp')) expect { builder.run }.to raise_error Puppet::ModuleTool::Errors::ModuleToolError, /symlinks/i end end context 'with metadata.json containing checksums' do before :each do File.open(File.join(path, 'metadata.json'), 'w') do |f| f.puts({ "name" => "#{module_name}", "version" => "#{version}", "source" => "http://github.com/testing/#{module_name}", "author" => "testing", "license" => "Apache License Version 2.0", "summary" => "Puppet testing module", "description" => "This module can be used for basic testing", "project_page" => "http://github.com/testing/#{module_name}", "checksums" => {"README.md" => "deadbeef"} }.to_json) end end it_behaves_like "a packagable module" end context 'with Modulefile' do before :each do File.open(File.join(path, 'Modulefile'), 'w') do |f| f.write <<-MODULEFILE name '#{module_name}' version '#{version}' source 'http://github.com/testing/#{module_name}' author 'testing' license 'Apache License Version 2.0' summary 'Puppet testing module' description 'This module can be used for basic testing' project_page 'http://github.com/testing/#{module_name}' MODULEFILE end end it_behaves_like "a packagable module" end end diff --git a/spec/unit/module_tool/applications/unpacker_spec.rb b/spec/unit/module_tool/applications/unpacker_spec.rb index 1ff93e180..81557df99 100644 --- a/spec/unit/module_tool/applications/unpacker_spec.rb +++ b/spec/unit/module_tool/applications/unpacker_spec.rb @@ -1,73 +1,74 @@ require 'spec_helper' require 'json' require 'puppet/module_tool/applications' +require 'puppet/file_system' require 'puppet_spec/modules' describe Puppet::ModuleTool::Applications::Unpacker do include PuppetSpec::Files let(:target) { tmpdir("unpacker") } let(:module_name) { 'myusername-mytarball' } let(:filename) { tmpdir("module") + "/module.tar.gz" } let(:working_dir) { tmpdir("working_dir") } before :each do Puppet.settings[:module_working_dir] = working_dir end it "should attempt to untar file to temporary location" do untar = mock('Tar') untar.expects(:unpack).with(filename, anything()) do |src, dest, _| FileUtils.mkdir(File.join(dest, 'extractedmodule')) File.open(File.join(dest, 'extractedmodule', 'metadata.json'), 'w+') do |file| file.puts JSON.generate('name' => module_name, 'version' => '1.0.0') end true end Puppet::ModuleTool::Tar.expects(:instance).returns(untar) Puppet::ModuleTool::Applications::Unpacker.run(filename, :target_dir => target) File.should be_directory(File.join(target, 'mytarball')) end - it "should warn about symlinks" do + it "should warn about symlinks", :if => Puppet.features.manages_symlinks? do untar = mock('Tar') untar.expects(:unpack).with(filename, anything()) do |src, dest, _| FileUtils.mkdir(File.join(dest, 'extractedmodule')) File.open(File.join(dest, 'extractedmodule', 'metadata.json'), 'w+') do |file| file.puts JSON.generate('name' => module_name, 'version' => '1.0.0') end FileUtils.touch(File.join(dest, 'extractedmodule/tempfile')) - File.symlink(File.join(dest, 'extractedmodule/tempfile'), File.join(dest, 'extractedmodule/tempfile2')) + Puppet::FileSystem.symlink(File.join(dest, 'extractedmodule/tempfile'), File.join(dest, 'extractedmodule/tempfile2')) true end Puppet::ModuleTool::Tar.expects(:instance).returns(untar) Puppet.expects(:warning).with(regexp_matches(/symlinks/i)) Puppet::ModuleTool::Applications::Unpacker.run(filename, :target_dir => target) File.should be_directory(File.join(target, 'mytarball')) end - it "should warn about symlinks in subdirectories" do + it "should warn about symlinks in subdirectories", :if => Puppet.features.manages_symlinks? do untar = mock('Tar') untar.expects(:unpack).with(filename, anything()) do |src, dest, _| FileUtils.mkdir(File.join(dest, 'extractedmodule')) File.open(File.join(dest, 'extractedmodule', 'metadata.json'), 'w+') do |file| file.puts JSON.generate('name' => module_name, 'version' => '1.0.0') end FileUtils.mkdir(File.join(dest, 'extractedmodule/manifests')) FileUtils.touch(File.join(dest, 'extractedmodule/manifests/tempfile')) - File.symlink(File.join(dest, 'extractedmodule/manifests/tempfile'), File.join(dest, 'extractedmodule/manifests/tempfile2')) + Puppet::FileSystem.symlink(File.join(dest, 'extractedmodule/manifests/tempfile'), File.join(dest, 'extractedmodule/manifests/tempfile2')) true end Puppet::ModuleTool::Tar.expects(:instance).returns(untar) Puppet.expects(:warning).with(regexp_matches(/symlinks/i)) Puppet::ModuleTool::Applications::Unpacker.run(filename, :target_dir => target) File.should be_directory(File.join(target, 'mytarball')) end end