diff --git a/acceptance/Gemfile b/acceptance/Gemfile index f15b1caac..134559362 100644 --- a/acceptance/Gemfile +++ b/acceptance/Gemfile @@ -1,13 +1,13 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org" -gem "beaker", "~> 1.3.1" +gem "beaker", "~> 1.7.0" gem "rake" group(:test) do gem "rspec", "~> 2.11.0", :require => false gem "mocha", "~> 0.10.5", :require => false end if File.exists? "#{__FILE__}.local" eval(File.read("#{__FILE__}.local"), binding) end diff --git a/acceptance/Rakefile b/acceptance/Rakefile index 073628005..0d9b877da 100644 --- a/acceptance/Rakefile +++ b/acceptance/Rakefile @@ -1,318 +1,309 @@ require 'rake/clean' require 'pp' require 'yaml' ONE_DAY_IN_SECS = 24 * 60 * 60 REPO_CONFIGS_DIR = "repo-configs" CLEAN.include('*.tar', REPO_CONFIGS_DIR, 'merged_options.rb') module HarnessOptions DEFAULTS = { :type => 'git', :helper => ['lib/helper.rb'], :tests => ['tests'], :log_level => 'debug', :color => false, :root_keys => true, :ssh => { :keys => ["id_rsa-acceptance"], }, :xml => true, :timesync => true, :repo_proxy => true, :add_el_extras => true, + :preserve_hosts => 'onfail', } class Aggregator attr_reader :mode def initialize(mode) @mode = mode end def get_options(file_path) puts file_path if File.exists? file_path options = eval(File.read(file_path), binding) else puts "No options file found at #{File.expand_path(file_path)}" end options || {} end def get_mode_options get_options("./config/#{mode}/options.rb") end def get_local_options get_options("./local_options.rb") end def final_options(intermediary_options = {}) mode_options = get_mode_options local_overrides = get_local_options final_options = DEFAULTS.merge(mode_options) final_options.merge!(intermediary_options) final_options.merge!(local_overrides) return final_options end end def self.options(mode, options) final_options = Aggregator.new(mode).final_options(options) final_options end end def beaker_test(mode = :packages, options = {}) - final_options = HarnessOptions.options(mode, options) + delete_options = options[:__delete_options__] || [] + final_options = HarnessOptions.options(mode, + options.reject { |k,v| k == :__delete_options__ }) if mode == :git puppet_fork = ENV['FORK'] || 'puppetlabs' git_server = ENV['GIT_SERVER'] || 'github.com' final_options[:install] << "git://#{git_server}/#{puppet_fork}/puppet.git##{sha}" end + delete_options.each do |delete_me| + final_options.delete(delete_me) + end + options_file = 'merged_options.rb' File.open(options_file, 'w') do |merged| merged.puts <<-EOS # Copy this file to local_options.rb and adjust as needed if you wish to run # with some local overrides. EOS merged.puts(final_options.pretty_inspect) end - tests = ENV['TESTS'] || ENV['TEST'] + tests = ENV['TESTS'] || ENV['TEST'] tests_opt = "--tests=#{tests}" if tests config_opt = "--hosts=#{config}" if config overriding_options = ENV['OPTIONS'] args = ["--options-file", options_file, config_opt, tests_opt, overriding_options].compact + preserve_hosts = final_options[:preserve_hosts] + if md = /--preserve-hosts=?\s*['"]?(\w+)/.match(overriding_options) + preserve_hosts = md[1] + end + begin - sh("beaker", *args) + failed = false + sh("beaker", *args) { |ok,res| failed = true if !ok } ensure if (hosts_file = config || final_options[:hosts_file]) && hosts_file !~ /preserved_config/ cp(hosts_file, "log/latest/config.yml") - generate_config_for_latest_hosts if final_options[:preserve_hosts] || overriding_options =~ /--preserve-hosts/ + generate_config_for_latest_hosts if preserve_hosts = 'always' || (failed && preserve_hosts = 'onfail') end mv(options_file, "log/latest") end end def generate_config_for_latest_hosts preserved_config_hash = { 'HOSTS' => {} } config_hash = YAML.load_file('log/latest/config.yml').to_hash nodes = config_hash['HOSTS'].map do |node_label,hash| { :node_label => node_label, :platform => hash['platform'] } end pre_suite_log = File.read('log/latest/pre_suite-run.log') nodes.each do |node_info| hostname = /^(\w+) \(#{node_info[:node_label]}\)/.match(pre_suite_log)[1] fqdn = "#{hostname}.delivery.puppetlabs.net" preserved_config_hash['HOSTS'][fqdn] = { 'roles' => [ 'agent'], 'platform' => node_info[:platform], } preserved_config_hash['HOSTS'][fqdn]['roles'].unshift('master') if node_info[:node_label] =~ /master/ end pp preserved_config_hash File.open('log/latest/preserved_config.yaml', 'w') do |config_file| YAML.dump(preserved_config_hash, config_file) end rescue Errno::ENOENT => e puts "Couldn't generate log #{e}" end def list_preserved_configurations(secs_ago = ONE_DAY_IN_SECS) preserved = {} Dir.glob('log/*_*').each do |dir| preserved_config_path = "#{dir}/preserved_config.yaml" yesterday = Time.now - secs_ago.to_i if preserved_config = File.exists?(preserved_config_path) directory = File.new(dir) if directory.ctime > yesterday hosts = [] preserved_config = YAML.load_file(preserved_config_path).to_hash preserved_config['HOSTS'].each do |hostname,values| hosts << "#{hostname}: #{values['platform']}, #{values['roles']}" end preserved[hosts] = directory.to_path end end end preserved.map { |k,v| [v,k] }.sort { |a,b| a[0] <=> b[0] }.reverse end def list_preserved_hosts(secs_ago = ONE_DAY_IN_SECS) hosts = Set.new Dir.glob('log/**/pre*suite*run.log').each do |log| yesterday = Time.now - secs_ago.to_i File.open(log, 'r') do |file| if file.ctime > yesterday file.each_line do |line| matchdata = /^(\w+) \(.*?\) \$/.match(line.encode!('UTF-8', 'UTF-8', :invalid => :replace)) hosts.add(matchdata[1]) if matchdata end end end end hosts end -# Plagiarized from Beaker::Vcloud#cleanup -def destroy_preserved_hosts(hosts = nil, secs_ago = ONE_DAY_IN_SECS) +def release_hosts(hosts = nil, secs_ago = ONE_DAY_IN_SECS) secs_ago ||= ONE_DAY_IN_SECS hosts ||= list_preserved_hosts(secs_ago) - require 'beaker/hypervisor/vsphere_helper' - vsphere_credentials = VsphereHelper.load_config("#{ENV['HOME']}/.fog") - - puts "Connecting to vSphere at #{vsphere_credentials[:server]}" + - " with credentials for #{vsphere_credentials[:user]}" - - vsphere_helper = VsphereHelper.new( vsphere_credentials ) - - vm_names = hosts.to_a - pp vm_names - vms = vsphere_helper.find_vms vm_names - vm_names.each do |name| - unless vm = vms[name] - puts "Couldn't find VM #{name} in vSphere!" - next - end - - if vm.runtime.powerState == 'poweredOn' - puts "Shutting down #{vm.name}" - start = Time.now - vm.PowerOffVM_Task.wait_for_completion - puts "Spent %.2f seconds halting #{vm.name}" % (Time.now - start) - end - - start = Time.now - vm.Destroy_Task - puts "Spent %.2f seconds destroying #{vm.name}" % (Time.now - start) - end - - vsphere_helper.close + require 'beaker' + vcloud_pooled = Beaker::VcloudPooled.new(hosts.map { |h| { 'vmhostname' => h } }, + :logger => Beaker::Logger.new, + :dot_fog => "#{ENV['HOME']}/.fog", + 'pooling_api' => 'http://vcloud.delivery.puppetlabs.net' , + 'datastore' => 'not-used', + 'resourcepool' => 'not-used', + 'folder' => 'not-used') + vcloud_pooled.cleanup end def print_preserved(preserved) preserved.each_with_index do |entry,i| puts "##{i}: #{entry[0]}" entry[1].each { |h| puts " #{h}" } end end def beaker_run_type type = ENV['TYPE'] || :packages type = type.to_sym end def sha ENV['SHA'] end def config ENV['CONFIG'] end namespace :ci do task :check_env do raise(USAGE) unless sha end namespace :test do USAGE = <<-EOS Requires commit SHA to be put under test as environment variable: SHA=''. Also must set CONFIG=config/nodes/foo.yaml or include it in an options.rb for Beaker. You may set TESTS=path/to/test,and/more/tests. You may set additional Beaker OPTIONS='--more --options' If testing from git checkouts, you may optionally set the github fork to checkout from using FORK='other-puppet-fork'. You may also optionally set the git server to checkout from using GIT_SERVER='my.host.with.git.daemon', if you have set up a `git daemon` to pull local commits from. (In this case, the FORK should be set to the path to the repository, and you will need to allow the git daemon to serve the repo (see `git help daemon`)). If there is a Beaker options hash in a ./local_options.rb, it will be included. Commandline options set through the above environment variables will override settings in this file. EOS desc <<-EOS Run the acceptance tests through Beaker and install packages on the configuration targets. #{USAGE} EOS task :packages => 'ci:check_env' do beaker_test end desc <<-EOS Run the acceptance tests through Beaker and install from git on the configuration targets. #{USAGE} EOS task :git => 'ci:check_env' do beaker_test(:git) end end desc "Capture the master and agent hostname from the latest log and construct a preserved_config.yaml for re-running against preserved hosts without provisioning." task :extract_preserved_config do generate_config_for_latest_hosts end desc <<-EOS Run an acceptance test for a given node configuration and preserve the hosts. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. #{USAGE} EOS task :test_and_preserve_hosts => 'ci:check_env' do - beaker_test(beaker_run_type, :preserve_hosts => true) + beaker_test(beaker_run_type, :preserve_hosts => 'always') end desc "List acceptance runs from the past day which had hosts preserved." task :list_preserved do preserved = list_preserved_configurations print_preserved(preserved) end desc <<-EOS Shutdown and destroy any hosts that we have preserved for testing. These should be reaped daily by scripts, but this will free up resources immediately. Specify a list of comma separated HOST_NAMES if you have a set of dynamic vcloud host names you want to purge outside of what can be grepped from the logs. You can go back through the last SECS_AGO logs. Default is one day ago in secs. EOS - task :destroy_preserved_hosts do + task :release_hosts do host_names = ENV['HOST_NAMES'].split(',') if ENV['HOST_NAMES'] secs_ago = ENV['SECS_AGO'] - destroy_preserved_hosts(host_names, secs_ago) + release_hosts(host_names, secs_ago) + end + + task :destroy_preserved_hosts => 'ci:release_hosts' do + puts "Note: we are now releasing hosts back to the vcloud pooling api rather than destroying them directly. The rake task for this is ci:release_hosts" end desc <<-EOS Rerun an acceptance test using the last captured preserved_config.yaml to skip provisioning. Or specify a CONFIG_NUMBER from `rake ci:list_preserved`. -Uses the setup/rsync/pre-suite to rsync the local puppet source onto master and agent. -You may specify an RSYNC_FILTER_FILE as well. -You may skip purgeing and reinstalling puppet packages by including SKIP_PACKAGE_REINSTALL. -You may skip rsyncing local puppet files over to the tests hosts by including SKIP_RSYNC. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. EOS task :test_against_preserved_hosts do config_number = (ENV['CONFIG_NUMBER'] || 0).to_i preserved = list_preserved_configurations print_preserved(preserved) config_path = preserved[config_number][0] puts "Using ##{config_number}: #{config_path}" beaker_test(beaker_run_type, :hosts_file => "#{config_path}/preserved_config.yaml", :no_provision => true, - :preserve_hosts => true, - :pre_suite => ['setup/rsync/pre-suite'] + :preserve_hosts => 'always', + :__delete_options__ => [:pre_suite], ) end end task :default do sh('rake -T') end diff --git a/acceptance/setup/rsync/pre-suite/000_PurgeAndReinstall.rb b/acceptance/setup/rsync/pre-suite/000_PurgeAndReinstall.rb deleted file mode 100644 index aa2bdf412..000000000 --- a/acceptance/setup/rsync/pre-suite/000_PurgeAndReinstall.rb +++ /dev/null @@ -1,10 +0,0 @@ -test_name "Purge and Reinstall Packages" do - if !ENV['SKIP_PACKAGE_REINSTALL'] - hosts.each do |host| - host.uninstall_package('puppet') - host.uninstall_package('puppet-common') - additional_switches = '--allow-unauthenticated' if host['platform'] =~ /debian|ubuntu/ - host.install_package('puppet', additional_switches) - end - end -end diff --git a/acceptance/setup/rsync/pre-suite/010_RsyncSource.rb b/acceptance/setup/rsync/pre-suite/010_RsyncSource.rb deleted file mode 100644 index 1bfbda871..000000000 --- a/acceptance/setup/rsync/pre-suite/010_RsyncSource.rb +++ /dev/null @@ -1,25 +0,0 @@ -test_name "Rsync Source" do - if !ENV['SKIP_RSYNC'] - hosts.each do |host| - step "rsyncing local puppet source to #{host}" do - host.install_package('rsync') if !host.check_for_package('rsync') - filter_opt = "--filter='merge #{ENV['RSYNC_FILTER_FILE']}'" if ENV['RSYNC_FILTER_FILE'] - destination_dir = case host['platform'] - when /debian|ubuntu/ - then '/usr/lib/ruby/vendor_ruby' - when /el|centos/ - then '/usr/lib/ruby/site_ruby/1.8' - when /fedora/ - then '/usr/share/ruby/vendor_ruby' - else - raise "We should actually do some #{host['platform']} platform specific rsyncing here..." - end - cmd = "rsync -r --exclude '.*.swp' #{filter_opt} --size-only -i -e'ssh -i id_rsa-acceptance' ../lib/* root@#{host}:#{destination_dir}" - puts "RSYNC: #{cmd}" - result = `#{cmd}` - raise("Failed rsync execution:\n#{result}") if $? != 0 - puts result - end - end - end -end diff --git a/docs/acceptance_tests.md b/docs/acceptance_tests.md index 857d3aef2..4eea425cc 100644 --- a/docs/acceptance_tests.md +++ b/docs/acceptance_tests.md @@ -1,218 +1,210 @@ Running Acceptance Tests Yourself ================================= Table of Contents ----------------- * [General Notes](#general-notes) * [Running Tests on the vcloud](#running-tests-on-the-vcloud) * [Running Tests on Vagrant Boxen](#running-tests-on-vagrant-boxen) General Notes ------------- The rake tasks for running the tests are defined by the Rakefile in the acceptance test directory. These tasks come with some documentation: `rake -T` will give short descriptions, and a `rake -D` will give full descriptions with information on ENV options required and optional for the various tasks. If you are setting up a new repository for acceptance, you will need to bundle install first. This step assumes you have ruby and the bundler gem installed. ```sh cd /path/to/repo/acceptance bundle install --path=.bundle/gems ``` Running Tests on the vcloud --------------------------- In order to use the Puppet Labs vcloud, you'll need to be a Puppet Labs employee. Community members should see the [guide to running the tests on vagrant boxen](#running-tests-on-local-vagrant-boxen). ### Authentication Normally the ci tasks are called from a prepared Jenkins job. If you are running this on your laptop, you will need this ssh private key in order for beaker to be able to log into the vms created from the hosts file: https://github.com/puppetlabs/puppetlabs-modules/blob/qa/secure/jenkins/id_rsa-acceptance https://github.com/puppetlabs/puppetlabs-modules/blob/qa/secure/jenkins/id_rsa-acceptance.pub TODO fetch these files directly from github, but am running into rate limits and then would also have to cross the issue of authentication. You will also need QA credentials to vsphere in a ~/.fog file. These credentials can be found on any of the Jenkins coordinator hosts. ### Packages In order to run the tests on hosts provisioned from packages produced by Delivery, you will need to reference a Puppet commit sha that has been packaged using Delivery's pl:jenkins:uber_build task. This is the snippet used by 'Puppet Packaging' Jenkins jobs: ```sh rake --trace package:implode rake --trace package:bootstrap rake --trace pl:jenkins:uber_build ``` The above Rake tasks were run from the root of a Puppet checkout. They are quoted just for reference. Typically if you are investigating a failure, you will have a SHA from a failed jenkins run which should correspond to a successful pipeline run, and you should not need to run the pipeline manually. A finished pipeline will have repository information available at http://builds.puppetlabs.lan/puppet/ So you can also browse this list and select a recent sha which has repo_configs/ available. When executing the ci:test:packages task, you must set the SHA, and also set CONFIG to point to a valid Beaker hosts_file. Configurations used in the Jenkins jobs are available under config/nodes ```sh bundle exec rake ci:test:packages SHA=abcdef CONFIG=config/nodes/rhel.yaml ``` Optionally you may set the TEST (TEST=a/test.rb,and/another/test.rb), and may pass additional OPTIONS to beaker (OPTIONS='--opt foo'). You may also edit a ./local_options.rb hash which will override config/ options, and in turn be overriden by commandline options set in the environment variables CONFIG, TEST and OPTIONS. This file is a ruby file containing a Ruby hash with configuration expected by Beaker. See Beaker source, and examples in config/. ### Git Alternatively you may provision via git clone by calling the ci:test:git task. Currently we don't have packages for Windows or Solaris from the Delivery pipeline, and must use ci:test:git to provision and test these platforms. #### Source Checkout for Different Fork If you have a branch pushed to your fork which you wish to test prior to merging into puppetlabs/puppet, you can do so be setting the FORK environment variable. So, if I have a branch 'issue/master/wonder-if-this-explodes' pushed to my jpartlow puppet fork that I want to test on Windows, I could invoke the following: ```sh bundle exec rake ci:test:git CONFIG=config/nodes/win2008r2.yaml SHA=issue/master/wonder-if-this-explodes FORK=jpartlow ``` #### Source Checkout for Local Branch See notes on running acceptance with Vagrant for more details on using a local git daemon. TODO Fix up the Rakefile's handling of git urls so that there is a simple way to specify both a branch on a github fork, and a branch on some other git server daemon, so that you have fewer steps when serving from a local git daemon. ### Preserving Hosts -If you have local changes to puppet code (outside of acceptance/) that you don't want to repackage for time reasons, or you just want to ssh into the hosts after a test run, you can use the following sequence: +If you need to ssh into the hosts after a test run, you can use the following sequence: bundle exec rake ci:test_and_preserve_hosts CONFIG=some/config.yaml SHA=12345 TEST=a/foo_test.rb to get the initial templates provisioned, and a local log/latest/preserve_config.yaml created for them. Then you can log into the hosts, or rerun tests against them by: bundle exec rake ci:test_against_preserved_hosts TEST=a/foo_test.rb -This will use the existing hosts, uninstall and reinstall the puppet packages and rsync in any changes from your local source lib dir. To skip reinstalling the packages set SKIP_PACKAGE_REINSTALL=1. To skip rsyncing, set SKIP_RSYNC=1. To use rsync filters, create a file with your rsync filter settings and set RSYNC_FILTER_FILE to the name of that file. For example: - - include puppet - include puppet/defaults.rb - exclude * - -will ensure that only puppet/defaults.rb is copied. - -NOTE: By default these tasks provision with packages. Set TYPE=git to use source checkouts. +This will use the existing hosts. ### Cleaning Up Preserved Hosts If you run a number of jobs with --preserve_hosts or vi ci:test_and_preserve_hosts, you may eventually generate a large number of stale vms. They should be reaped automatically by qa infrastructure within a day or so, but you may also run: - bundle exec rake ci:destroy_preserved_hosts + bundle exec rake ci:release_hosts to clean them up sooner and free resources. There also may be scenarios where you want to specify the host(s) to destroy. E.g. you may want to destroy a subset of the hosts you've created. Or, if a test run terminates early, ci:destroy_preserved_hosts may not be able to derive the name of the vm to delete. In such cases you can specify host(s) to be deleted using the HOST_NAMES environment variable. E.g. HOST_NAMES=lvwwr9tdplg351u bundle exec rake ci:destroy_preserved_hosts HOST_NAMES=lvwwr9tdplg351u,ylrqjh5l6xvym4t bundle exec rake ci:destroy_preserved_hosts Running Tests on Vagrant Boxen ------------------------------ This guide assumes that you have an acceptable Ruby (i.e. 1.9+) installed along with the bundler gem, that you have the puppet repo checked out locally somewhere, and that the name of the checkout folder is `puppet`. I used Ruby 1.9.3-p484 Change to the `acceptance` directory in the root of the puppet repo: ```sh cd /path/to/repo/puppet/acceptance ``` Install the necessary gems with bundler: ```sh bundle install ``` Now you can get a list of test-related tasks you can run via rake: ```sh bundle exec rake -T ``` and view detailed information on the tasks with ```sh bundle exec rake -D ``` As an example, let's try running the acceptance tests using git as the code deployment mechanism. First, we'll have to create a beaker configuration file for a local vagrant box on which to run the tests. Here's what such a file could look like: ```yaml HOSTS: all-in-one: roles: - master - agent platform: centos-64-x64 hypervisor: vagrant ip: 192.168.80.100 box: centos-64-x64-vbox4210-nocm box_url: http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box CONFIG: ``` This defines a 64-bit CentOS 6.4 vagrant box that serves as both a puppet master and a puppet agent for the test roles. (For more information on beaker config files, see [beaker's README](https://github.com/puppetlabs/beaker/blob/master/README.md).) Save this file as `config/nodes/centos6-local.yaml`; we'll be needing it later. Since we have only provided a CentOS box, we don't have anywhere to run windows tests, therefore we'll have to skip those tests. That means we want to pass beaker a --tests argument that contains every directory and file in the `tests` directory besides the one called `windows`. We could pass this option on the command line, but it will be gigantic, so instead let's create a `local_options.rb` file that beaker will automatically read in. This file should contain a ruby hash of beaker's command-line flags to the corresponding flag arguments. Our hash will only contain the `tests` key, and its value will be a comma-seperated list of the other files and directories in `tests`. Here's an easy way to generate this file: ```sh echo "{tests: \"$(echo tests/* | sed -e 's| *tests/windows *||' -e 's/ /,/g')\"}" > local_options.rb" ``` The last thing that needs to be done before we can run the tests is to set up a way for the test box to check out our local changes for testing. We'll do this by starting a git daemon on our host. In another session, navigate to the folder that contains your checkout of the puppet repo, and then create the following symlink: ```sh ln -s . puppet.git ``` This works around the inflexible checkout path used by the test prep code. Now start the git daemon with ```sh git daemon --verbose --informative-errors --reuseaddr --export-all --base-path=. ``` after which you should see a message like `[32963] Ready to rumble` echoed to the console. Now we can finally run the tests! The rake task that we'll use is `ci:test:git`. Run ``` bundle exec rake -D ci:test:git ``` to read the full description of this task. From the description, we can see that we'll need to set a few environment variables: + CONFIG should be set to point to the CentOS beaker config file we created above. + SHA should be the SHA of the commit we want to test. + GIT_SERVER should be the IP address of the host (i.e. your machine) in the vagrant private network created for the test box. This is derived from the test box's ip by replacing the last octet with 1. For our example above, the host IP is 192.168.80.1 + FORK should be the path to a 'puppet.git' directory that points to the repo. In our case, this is the path to the symlink we created before, which is inside your puppet repo checkout, so FORK should just be the name of your checkout. We'll assume that the name is `puppet`. Putting it all together, we construct the following command-line invocation to run the tests: ```sh CONFIG=config/nodes/centos6-local.yaml SHA=#{test-commit-sha} GIT_SERVER='192.168.80.1' FORK='puppet' bundle exec rake --trace ci:test:git ``` Go ahead and run that sucker! Testing will take some time. After the testing finishes, you'll either see this line ``` systest completed successfully, thanks. ``` near the end of the output, indicating that all tests completed succesfully, or you'll see the end of a stack trace, indicating failed tests further up.