diff --git a/acceptance/Gemfile b/acceptance/Gemfile index b2384520b..f5316fa44 100644 --- a/acceptance/Gemfile +++ b/acceptance/Gemfile @@ -1,13 +1,13 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org" -gem "beaker", "~> 1.7.0" +gem "beaker", "~> 1.10.0" gem 'rake', "~> 10.1.0" group(:test) do gem "rspec", "~> 2.11.0", :require => false gem "mocha", "~> 0.10.5", :require => false end if File.exists? "#{__FILE__}.local" eval(File.read("#{__FILE__}.local"), binding) end diff --git a/acceptance/lib/puppet/acceptance/install_utils.rb b/acceptance/lib/puppet/acceptance/install_utils.rb index 5df423574..e7ddc97cd 100644 --- a/acceptance/lib/puppet/acceptance/install_utils.rb +++ b/acceptance/lib/puppet/acceptance/install_utils.rb @@ -1,155 +1,201 @@ require 'open-uri' +require 'open3' +require 'uri' module Puppet module Acceptance module InstallUtils PLATFORM_PATTERNS = { :redhat => /fedora|el|centos/, :debian => /debian|ubuntu/, :solaris => /solaris/, :windows => /windows/, }.freeze # Installs packages on the hosts. # # @param hosts [Array] Array of hosts to install packages to. # @param package_hash [Hash{Symbol=>Array>}] # Keys should be a symbol for a platform in PLATFORM_PATTERNS. Values # should be an array of package names to install, or of two element # arrays where a[0] is the command we expect to find on the platform # and a[1] is the package name (when they are different). # @param options [Hash{Symbol=>Boolean}] # @option options [Boolean] :check_if_exists First check to see if # command is present before installing package. (Default false) # @return true def install_packages_on(hosts, package_hash, options = {}) check_if_exists = options[:check_if_exists] hosts = [hosts] unless hosts.kind_of?(Array) hosts.each do |host| package_hash.each do |platform_key,package_list| if pattern = PLATFORM_PATTERNS[platform_key] if pattern.match(host['platform']) package_list.each do |cmd_pkg| if cmd_pkg.kind_of?(Array) command, package = cmd_pkg else command = package = cmd_pkg end if !check_if_exists || !host.check_for_package(command) host.logger.notify("Installing #{package}") additional_switches = '--allow-unauthenticated' if platform_key == :debian host.install_package(package, additional_switches) end end end else raise("Unknown platform '#{platform_key}' in package_hash") end end end return true end def fetch(base_url, file_name, dst_dir) FileUtils.makedirs(dst_dir) src = "#{base_url}/#{file_name}" dst = File.join(dst_dir, file_name) if File.exists?(dst) logger.notify "Already fetched #{dst}" else logger.notify "Fetching: #{src}" logger.notify " and saving to #{dst}" open(src) do |remote| File.open(dst, "w") do |file| FileUtils.copy_stream(remote, file) end end end return dst end + def fetch_remote_dir(url, dst_dir) + logger.notify "fetch_remote_dir (url: #{url}, dst_dir #{dst_dir})" + if url[-1, 1] !~ /\// + url += '/' + end + url = URI.parse(url) + chunks = url.path.split('/') + dst = File.join(dst_dir, chunks.last) + #determine directory structure to cut + #only want to keep the last directory, thus cut total number of dirs - 2 (hostname + last dir name) + cut = chunks.length - 2 + wget_command = "wget -nv -P #{dst_dir} --reject \"index.html*\",\"*.gif\" --cut-dirs=#{cut} -np -nH --no-check-certificate -r #{url}" + + logger.notify "Fetching remote directory: #{url}" + logger.notify " and saving to #{dst}" + logger.notify " using command: #{wget_command}" + + #in ruby 1.9+ we can upgrade this to popen3 to gain access to the subprocess pid + result = `#{wget_command} 2>&1` + result.each_line do |line| + logger.debug(line) + end + if $?.to_i != 0 + raise "Failed to fetch_remote_dir '#{url}' (exit code #{$?}" + end + dst + end + def stop_firewall_on(host) case host['platform'] when /debian/ on host, 'iptables -F' when /fedora|el-7/ on host, puppet('resource', 'service', 'firewalld', 'ensure=stopped') when /el|centos/ on host, puppet('resource', 'service', 'iptables', 'ensure=stopped') when /ubuntu/ on host, puppet('resource', 'service', 'ufw', 'ensure=stopped') else logger.notify("Not sure how to clear firewall on #{host['platform']}") end end def install_repos_on(host, sha, repo_configs_dir) - platform = host['platform'] + platform = host['platform'].with_version_codename platform_configs_dir = File.join(repo_configs_dir,platform) case platform when /^(fedora|el|centos)-(\d+)-(.+)$/ variant = (($1 == 'centos') ? 'el' : $1) fedora_prefix = ((variant == 'fedora') ? 'f' : '') version = $2 arch = $3 rpm = fetch( "http://yum.puppetlabs.com", "puppetlabs-release-%s-%s.noarch.rpm" % [variant, version], platform_configs_dir ) pattern = "pl-puppet-%s-%s-%s%s-%s.repo" repo_filename = pattern % [ sha, variant, fedora_prefix, version, arch ] repo = fetch( "http://builds.puppetlabs.lan/puppet/%s/repo_configs/rpm/" % sha, repo_filename, platform_configs_dir ) - on host, "rm -rf /root/*.repo; rm -rf /root/*.rpm" + link = "http://builds.puppetlabs.lan/puppet/%s/repos/%s/%s/products/%s/" % [sha, variant, version, arch] + if not link_exists?(link) + link = "http://builds.puppetlabs.lan/puppet/%s/repos/%s/%s/devel/%s/" % [sha, variant, version, arch] + end + if not link_exists?(link) + raise "Unable to reach a repo directory at #{link}" + end + repo_dir = fetch_remote_dir(link, platform_configs_dir) + + on host, "rm -rf /root/*.repo; rm -rf /root/*.rpm; rm -rf /root/#{arch}" scp_to host, rpm, '/root' scp_to host, repo, '/root' + scp_to host, repo_dir, '/root' on host, "mv /root/*.repo /etc/yum.repos.d" + on host, "find /etc/yum.repos.d/ -name \"*.repo\" -exec sed -i \"s/baseurl\\s*=\\s*http:\\/\\/builds.puppetlabs.lan.*$/baseurl=file:\\/\\/\\/root\\/#{arch}/\" {} \\;" on host, "rpm -Uvh --force /root/*.rpm" when /^(debian|ubuntu)-([^-]+)-(.+)$/ variant = $1 version = $2 arch = $3 deb = fetch( "http://apt.puppetlabs.com/", "puppetlabs-release-%s.deb" % version, platform_configs_dir ) list = fetch( "http://builds.puppetlabs.lan/puppet/%s/repo_configs/deb/" % sha, "pl-puppet-%s-%s.list" % [sha, version], platform_configs_dir ) - on host, "rm -rf /root/*.list; rm -rf /root/*.deb" + repo_dir = fetch_remote_dir("http://builds.puppetlabs.lan/puppet/%s/repos/apt/%s" % [sha, version], platform_configs_dir) + + on host, "rm -rf /root/*.list; rm -rf /root/*.deb; rm -rf /root/#{version}" scp_to host, deb, '/root' scp_to host, list, '/root' + scp_to host, repo_dir, '/root' on host, "mv /root/*.list /etc/apt/sources.list.d" + on host, "find /etc/apt/sources.list.d/ -name \"*.list\" -exec sed -i \"s/deb\\s\\+http:\\/\\/builds.puppetlabs.lan.*$/deb file:\\/\\/\\/root\\/#{version} #{version} main/\" {} \\;" on host, "dpkg -i --force-all /root/*.deb" + on host, "apt-get update" else host.logger.notify("No repository installation step for #{platform} yet...") end end end end end diff --git a/acceptance/lib/puppet/acceptance/install_utils_spec.rb b/acceptance/lib/puppet/acceptance/install_utils_spec.rb index 9fb02547c..a124eef02 100644 --- a/acceptance/lib/puppet/acceptance/install_utils_spec.rb +++ b/acceptance/lib/puppet/acceptance/install_utils_spec.rb @@ -1,188 +1,261 @@ require File.join(File.dirname(__FILE__),'../../acceptance_spec_helper.rb') require 'puppet/acceptance/install_utils' describe 'InstallUtils' do class ATestCase include Puppet::Acceptance::InstallUtils end + class Platform < String + + def with_version_codename + self + end + end + class TestHost attr_accessor :config def initialize(config = {}) self.config = config end def [](key) config[key] end end let(:host) { TestHost.new } let(:testcase) { ATestCase.new } describe "install_packages_on" do it "raises an error if package_hash has unknown platform keys" do expect do testcase.install_packages_on(host, { :foo => 'bar'}) end.to raise_error(RuntimeError, /Unknown platform 'foo' in package_hash/) end shared_examples_for(:install_packages_on) do |platform,command,package| let(:package_hash) do { :redhat => ['rh_package'], :debian => [['db_command', 'db_package']], } end let(:additional_switches) { platform == 'debian' ? '--allow-unauthenticated' : nil } before do logger = mock('logger', :notify => nil) host.stubs(:logger).returns(logger) - host.config['platform'] = platform + host.config['platform'] = Platform.new(platform) end it "installs packages on a host" do host.expects(:check_for_package).never host.expects(:install_package).with(package, additional_switches).once testcase.install_packages_on(host, package_hash) end it "checks and installs packages on a host" do host.expects(:check_for_package).with(command).once host.expects(:install_package).with(package, additional_switches).once testcase.install_packages_on(host, package_hash, :check_if_exists => true) end end it_should_behave_like(:install_packages_on, 'fedora', 'rh_package', 'rh_package') it_should_behave_like(:install_packages_on, 'debian', 'db_command', 'db_package') end describe "fetch" do before do logger = stub('logger', :notify => nil) testcase.stubs(:logger).returns(logger) FileUtils.expects(:makedirs).with('dir') end it "does not fetch if destination file already exists" do File.expects(:exists?).with('dir/file').returns(true) testcase.expects(:open).never testcase.fetch('http://foo', 'file', 'dir') end it "fetches file from url and stores in destination directory as filename" do stream = mock('stream') file = mock('file') testcase.expects(:open).with('http://foo/file').yields(stream) File.expects(:open).with('dir/file', 'w').yields(file) FileUtils.expects(:copy_stream).with(stream, file) testcase.fetch('http://foo', 'file', 'dir') end it "returns path to destination file" do testcase.expects(:open).with('http://foo/file') expect(testcase.fetch('http://foo', 'file', 'dir')).to eql('dir/file') end end + describe "fetch_remote_dir" do + before do + logger = stub('logger', {:notify => nil, :debug => nil}) + testcase.stubs(:logger).returns(logger) + end + + it "calls wget with the right amount of cut dirs for url that ends in '/'" do + url = 'http://builds.puppetlabs.lan/puppet/7807591405af849da2ad6534c66bd2d4efff604f/repos/el/6/devel/x86_64/' + testcase.expects(:`).with("wget -nv -P dir --reject \"index.html*\",\"*.gif\" --cut-dirs=6 -np -nH --no-check-certificate -r #{url} 2>&1").returns("log") + + expect( testcase.fetch_remote_dir(url, 'dir')).to eql('dir/x86_64') + end + + it "calls wget with the right amount of cut dirs for url that doesn't end in '/'" do + url = 'http://builds.puppetlabs.lan/puppet/7807591405af849da2ad6534c66bd2d4efff604f/repos/apt/wheezy' + testcase.expects(:`).with("wget -nv -P dir --reject \"index.html*\",\"*.gif\" --cut-dirs=4 -np -nH --no-check-certificate -r #{url}/ 2>&1").returns("log") + + expect( testcase.fetch_remote_dir(url, 'dir')).to eql('dir/wheezy') + end + + end + shared_examples_for :redhat_platforms do |platform,sha,files| + before do + host.config['platform'] = Platform.new(platform) + end + it "fetches and installs repo configurations for #{platform}" do - host.config['platform'] = platform platform_configs_dir = "repo-configs/#{platform}" - rpm_file = files[:rpm] + rpm_url = files[:rpm][0] + rpm_file = files[:rpm][1] testcase.expects(:fetch).with( "http://yum.puppetlabs.com", rpm_file, platform_configs_dir ).returns("#{platform_configs_dir}/#{rpm_file}") repo_url = files[:repo][0] repo_file = files[:repo][1] testcase.expects(:fetch).with( repo_url, repo_file, platform_configs_dir ).returns("#{platform_configs_dir}/#{repo_file}") - testcase.expects(:on).with(host, regexp_matches(/rm.*repo; rm.*rpm/)) + repo_dir_url = files[:repo_dir][0] + repo_dir = files[:repo_dir][1] + testcase.expects(:link_exists?).returns( true ) + testcase.expects(:fetch_remote_dir).with( + repo_dir_url, + platform_configs_dir + ).returns("#{platform_configs_dir}/#{repo_dir}") + testcase.expects(:link_exists?).returns( true ) + + testcase.expects(:on).with(host, regexp_matches(/rm.*repo; rm.*rpm; rm.*#{repo_dir}/)) testcase.expects(:scp_to).with(host, "#{platform_configs_dir}/#{rpm_file}", '/root') testcase.expects(:scp_to).with(host, "#{platform_configs_dir}/#{repo_file}", '/root') + testcase.expects(:scp_to).with(host, "#{platform_configs_dir}/#{repo_dir}", '/root') testcase.expects(:on).with(host, regexp_matches(%r{mv.*repo /etc/yum.repos.d})) + testcase.expects(:on).with(host, regexp_matches(%r{find /etc/yum.repos.d/ -name .*})) testcase.expects(:on).with(host, regexp_matches(%r{rpm.*/root/.*rpm})) testcase.install_repos_on(host, sha, 'repo-configs') end end describe "install_repos_on" do let(:sha) { "abcdef10" } it_should_behave_like(:redhat_platforms, 'el-6-i386', 'abcdef10', { - :rpm => "puppetlabs-release-el-6.noarch.rpm", + :rpm => [ + "http://yum.puppetlabs.com", + "puppetlabs-release-el-6.noarch.rpm", + ], :repo => [ "http://builds.puppetlabs.lan/puppet/abcdef10/repo_configs/rpm/", "pl-puppet-abcdef10-el-6-i386.repo", ], + :repo_dir => [ + "http://builds.puppetlabs.lan/puppet/abcdef10/repos/el/6/products/i386/", + "i386", + ], }, ) it_should_behave_like(:redhat_platforms, 'fedora-18-x86_64', 'abcdef10', { - :rpm => "puppetlabs-release-fedora-18.noarch.rpm", + :rpm => [ + "http://yum.puppetlabs.com", + "puppetlabs-release-fedora-18.noarch.rpm", + ], :repo => [ "http://builds.puppetlabs.lan/puppet/abcdef10/repo_configs/rpm/", "pl-puppet-abcdef10-fedora-f18-x86_64.repo", ], + :repo_dir => [ + "http://builds.puppetlabs.lan/puppet/abcdef10/repos/fedora/18/products/x86_64/", + "x86_64", + ], }, ) it_should_behave_like(:redhat_platforms, 'centos-5-x86_64', 'abcdef10', { - :rpm => "puppetlabs-release-el-5.noarch.rpm", + :rpm => [ + "http://yum.puppetlabs.com", + "puppetlabs-release-el-5.noarch.rpm", + ], :repo => [ "http://builds.puppetlabs.lan/puppet/abcdef10/repo_configs/rpm/", "pl-puppet-abcdef10-el-5-x86_64.repo", ], + :repo_dir => [ + "http://builds.puppetlabs.lan/puppet/abcdef10/repos/el/5/products/x86_64/", + "x86_64", + ], }, ) it "installs on a debian host" do - host.config['platform'] = platform = 'ubuntu-precise-x86_64' + host.config['platform'] = platform = Platform.new('ubuntu-precise-x86_64') platform_configs_dir = "repo-configs/#{platform}" deb = "puppetlabs-release-precise.deb" testcase.expects(:fetch).with( "http://apt.puppetlabs.com/", deb, platform_configs_dir ).returns("#{platform_configs_dir}/#{deb}") list = "pl-puppet-#{sha}-precise.list" testcase.expects(:fetch).with( "http://builds.puppetlabs.lan/puppet/#{sha}/repo_configs/deb/", list, platform_configs_dir ).returns("#{platform_configs_dir}/#{list}") - testcase.expects(:on).with(host, regexp_matches(/rm.*list; rm.*deb/)) + testcase.expects(:fetch_remote_dir).with( + "http://builds.puppetlabs.lan/puppet/#{sha}/repos/apt/precise", + platform_configs_dir + ).returns("#{platform_configs_dir}/precise") + + testcase.expects(:on).with(host, regexp_matches(/rm.*list; rm.*deb; rm.*/)) testcase.expects(:scp_to).with(host, "#{platform_configs_dir}/#{deb}", '/root') testcase.expects(:scp_to).with(host, "#{platform_configs_dir}/#{list}", '/root') + testcase.expects(:scp_to).with(host, "#{platform_configs_dir}/precise", '/root') testcase.expects(:on).with(host, regexp_matches(%r{mv.*list /etc/apt/sources.list.d})) + testcase.expects(:on).with(host, regexp_matches(%r{find /etc/apt/sources.list.d/ -name .*})) testcase.expects(:on).with(host, regexp_matches(%r{dpkg -i.*/root/.*deb})) + testcase.expects(:on).with(host, regexp_matches(%r{apt-get update})) testcase.install_repos_on(host, sha, 'repo-configs') end end end