diff --git a/acceptance/Rakefile b/acceptance/Rakefile index b42ec8bce..45555a21c 100644 --- a/acceptance/Rakefile +++ b/acceptance/Rakefile @@ -1,323 +1,343 @@ require 'rake/clean' require 'pp' require 'yaml' $LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), 'lib')) require 'puppet/acceptance/git_utils' extend Puppet::Acceptance::GitUtils ONE_DAY_IN_SECS = 24 * 60 * 60 REPO_CONFIGS_DIR = "repo-configs" CLEAN.include('*.tar', REPO_CONFIGS_DIR, 'merged_options.rb') module HarnessOptions DEFAULTS = { :type => 'git', :helper => ['lib/helper.rb'], :tests => ['tests'], :log_level => 'debug', :color => false, :root_keys => true, :ssh => { :keys => ["id_rsa-acceptance"], }, :xml => true, :timesync => false, :repo_proxy => true, :add_el_extras => true, :preserve_hosts => 'onfail', :forge_host => 'forge-aio01-petest.puppetlabs.com' } class Aggregator attr_reader :mode def initialize(mode) @mode = mode end def get_options(file_path) puts file_path if File.exists? file_path options = eval(File.read(file_path), binding) else puts "No options file found at #{File.expand_path(file_path)}" end options || {} end def get_mode_options get_options("./config/#{mode}/options.rb") end def get_local_options get_options("./local_options.rb") end def final_options(intermediary_options = {}) mode_options = get_mode_options local_overrides = get_local_options final_options = DEFAULTS.merge(mode_options) final_options.merge!(intermediary_options) final_options.merge!(local_overrides) return final_options end end def self.options(mode, options) final_options = Aggregator.new(mode).final_options(options) final_options end end def beaker_test(mode = :packages, options = {}) - delete_options = options[:__delete_options__] || [] - final_options = HarnessOptions.options(mode, - options.reject { |k,v| k == :__delete_options__ }) + delete_options = options.delete(:__delete_options__) || [] + final_options = HarnessOptions.options(mode, options) + preserve_config = final_options.delete(:__preserve_config__) if mode == :git # Build up project git urls based on git server and fork env variables or defaults final_options[:install].map! do |install| if md = /^(\w+)#(\w+)$/.match(install) project, project_sha = md.captures "#{build_giturl(project)}##{project_sha}" elsif md = /^(\w+)$/.match(install) project = md[1] "#{build_giturl(project)}##{sha}" end end end delete_options.each do |delete_me| final_options.delete(delete_me) end options_file = 'merged_options.rb' File.open(options_file, 'w') do |merged| merged.puts <<-EOS # Copy this file to local_options.rb and adjust as needed if you wish to run # with some local overrides. EOS merged.puts(final_options.pretty_inspect) end tests = ENV['TESTS'] || ENV['TEST'] tests_opt = "--tests=#{tests}" if tests config_opt = "--hosts=#{config}" if config overriding_options = ENV['OPTIONS'] args = ["--options-file", options_file, config_opt, tests_opt, overriding_options].compact begin sh("beaker", *args) ensure - preserve_configuration(final_options, options_file) + preserve_configuration(final_options, options_file) if preserve_config end end def preserve_configuration(final_options, options_file) if (hosts_file = config || final_options[:hosts_file]) && hosts_file !~ /preserved_config/ cp(hosts_file, "log/latest/config.yml") generate_config_for_latest_hosts end mv(options_file, "log/latest") end def generate_config_for_latest_hosts preserved_config_hash = { 'HOSTS' => {} } - config_hash = YAML.load_file('log/latest/config.yml').to_hash - nodes = config_hash['HOSTS'].map do |node_label,hash| - { :node_label => node_label, :platform => hash['platform'] } - end + puts "\nPreserving configuration so that any preserved nodes can be tested again locally..." + + config_hash = YAML.load_file('log/latest/config.yml') + if !config_hash || !config_hash.include?('HOSTS') + puts "Warning: No HOSTS configuration found in log/latest/config.yml" + return + else + nodes = config_hash['HOSTS'].map do |node_label,hash| + { + :node_label => node_label, + :roles => hash['roles'], + :platform => hash['platform'] + } + end - pre_suite_log = File.read('log/latest/pre_suite-run.log') - nodes.each do |node_info| - hostname = /^(\w+) \(#{node_info[:node_label]}\)/.match(pre_suite_log)[1] - fqdn = "#{hostname}.delivery.puppetlabs.net" - preserved_config_hash['HOSTS'][fqdn] = { - 'roles' => [ 'agent'], - 'platform' => node_info[:platform], - } - preserved_config_hash['HOSTS'][fqdn]['roles'].unshift('master') if node_info[:node_label] =~ /master/ - end - pp preserved_config_hash + pre_suite_log = File.read('log/latest/pre_suite-run.log') + nodes.each do |node_info| + host_regex = /^([\w.]+) \(#{node_info[:node_label]}\)/ + if matched = host_regex.match(pre_suite_log) + hostname = matched[1] + fqdn = "#{hostname}.delivery.puppetlabs.net" + elsif /^#{node_info[:node_label]} /.match(pre_suite_log) + fqdn = "#{node_info[:node_label]}" + puts "* Couldn't find any log lines for #{host_regex}, assuming #{fqdn} is the fqdn" + end + if fqdn + preserved_config_hash['HOSTS'][fqdn] = { + 'roles' => node_info[:roles], + 'platform' => node_info[:platform], + } + else + puts "* Couldn't match #{node_info[:node_label]} in pre_suite-run.log" + end + end + pp preserved_config_hash - File.open('log/latest/preserved_config.yaml', 'w') do |config_file| - YAML.dump(preserved_config_hash, config_file) + File.open('log/latest/preserved_config.yaml', 'w') do |config_file| + YAML.dump(preserved_config_hash, config_file) + end end rescue Errno::ENOENT => e - puts "Couldn't generate log #{e}" + puts "Warning: Couldn't generate preserved_config.yaml #{e}" end def list_preserved_configurations(secs_ago = ONE_DAY_IN_SECS) preserved = {} Dir.glob('log/*_*').each do |dir| preserved_config_path = "#{dir}/preserved_config.yaml" yesterday = Time.now - secs_ago.to_i if preserved_config = File.exists?(preserved_config_path) directory = File.new(dir) if directory.ctime > yesterday hosts = [] preserved_config = YAML.load_file(preserved_config_path).to_hash preserved_config['HOSTS'].each do |hostname,values| hosts << "#{hostname}: #{values['platform']}, #{values['roles']}" end preserved[hosts] = directory.to_path end end end preserved.map { |k,v| [v,k] }.sort { |a,b| a[0] <=> b[0] }.reverse end def list_preserved_hosts(secs_ago = ONE_DAY_IN_SECS) hosts = Set.new Dir.glob('log/**/pre*suite*run.log').each do |log| yesterday = Time.now - secs_ago.to_i File.open(log, 'r') do |file| if file.ctime > yesterday file.each_line do |line| matchdata = /^(\w+) \(.*?\) \d\d:\d\d:\d\d\$/.match(line.encode!('UTF-8', 'UTF-8', :invalid => :replace)) hosts.add(matchdata[1]) if matchdata end end end end hosts end def release_hosts(hosts = nil, secs_ago = ONE_DAY_IN_SECS) secs_ago ||= ONE_DAY_IN_SECS hosts ||= list_preserved_hosts(secs_ago) require 'beaker' vcloud_pooled = Beaker::VcloudPooled.new(hosts.map { |h| { 'vmhostname' => h } }, :logger => Beaker::Logger.new, :dot_fog => "#{ENV['HOME']}/.fog", 'pooling_api' => 'http://vcloud.delivery.puppetlabs.net' , 'datastore' => 'not-used', 'resourcepool' => 'not-used', 'folder' => 'not-used') vcloud_pooled.cleanup end def print_preserved(preserved) preserved.each_with_index do |entry,i| puts "##{i}: #{entry[0]}" entry[1].each { |h| puts " #{h}" } end end def beaker_run_type type = ENV['TYPE'] || :packages type = type.to_sym end def sha ENV['SHA'] end def config ENV['CONFIG'] end namespace :ci do task :check_env do raise(USAGE) unless sha end namespace :test do USAGE = <<-EOS Requires commit SHA to be put under test as environment variable: SHA=''. Also must set CONFIG=config/nodes/foo.yaml or include it in an options.rb for Beaker. You may set TESTS=path/to/test,and/more/tests. You may set additional Beaker OPTIONS='--more --options' If testing from git checkouts, you may optionally set the github fork to checkout from using PUPPET_FORK='some-other-puppet-fork' (you may change the HIERA_FORK and FACTER_FORK as well if you wish). You may also optionally set the git server to checkout repos from using GIT_SERVER='some.git.mirror'. Or you may set PUPPET_GIT_SERVER='my.host.with.git.daemon', specifically, if you have set up a `git daemon` to pull local commits from. (You will need to allow the git daemon to serve the repo (see `git help daemon` and the docs/acceptance_tests.md for more details)). If there is a Beaker options hash in a ./local_options.rb, it will be included. Commandline options set through the above environment variables will override settings in this file. EOS desc <<-EOS Run the acceptance tests through Beaker and install packages on the configuration targets. #{USAGE} EOS task :packages => 'ci:check_env' do beaker_test end desc <<-EOS Run the acceptance tests through Beaker and install from git on the configuration targets. #{USAGE} EOS task :git => 'ci:check_env' do beaker_test(:git) end end desc "Capture the master and agent hostname from the latest log and construct a preserved_config.yaml for re-running against preserved hosts without provisioning." task :extract_preserved_config do generate_config_for_latest_hosts end desc <<-EOS Run an acceptance test for a given node configuration and preserve the hosts. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. #{USAGE} EOS task :test_and_preserve_hosts => 'ci:check_env' do - beaker_test(beaker_run_type, :preserve_hosts => 'always') + beaker_test(beaker_run_type, :preserve_hosts => 'always', :__preserve_config__ => true) end desc "List acceptance runs from the past day which had hosts preserved." task :list_preserved do preserved = list_preserved_configurations print_preserved(preserved) end desc <<-EOS Shutdown and destroy any hosts that we have preserved for testing. These should be reaped daily by scripts, but this will free up resources immediately. Specify a list of comma separated HOST_NAMES if you have a set of dynamic vcloud host names you want to purge outside of what can be grepped from the logs. You can go back through the last SECS_AGO logs. Default is one day ago in secs. EOS task :release_hosts do host_names = ENV['HOST_NAMES'].split(',') if ENV['HOST_NAMES'] secs_ago = ENV['SECS_AGO'] release_hosts(host_names, secs_ago) end task :destroy_preserved_hosts => 'ci:release_hosts' do puts "Note: we are now releasing hosts back to the vcloud pooling api rather than destroying them directly. The rake task for this is ci:release_hosts" end desc <<-EOS Rerun an acceptance test using the last captured preserved_config.yaml to skip provisioning. Or specify a CONFIG_NUMBER from `rake ci:list_preserved`. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. EOS task :test_against_preserved_hosts do config_number = (ENV['CONFIG_NUMBER'] || 0).to_i preserved = list_preserved_configurations print_preserved(preserved) config_path = preserved[config_number][0] puts "Using ##{config_number}: #{config_path}" beaker_test(beaker_run_type, :hosts_file => "#{config_path}/preserved_config.yaml", :no_provision => true, :preserve_hosts => 'always', :__delete_options__ => [:pre_suite] ) end end task :default do sh('rake -T') end task :spec do sh('rspec lib') end diff --git a/docs/acceptance_tests.md b/docs/acceptance_tests.md index baf0dd540..809ac9c43 100644 --- a/docs/acceptance_tests.md +++ b/docs/acceptance_tests.md @@ -1,237 +1,239 @@ Running Acceptance Tests Yourself ================================= Table of Contents ----------------- * [General Notes](#general-notes) * [Running Tests on the vcloud](#running-tests-on-the-vcloud) * [Running Tests on Vagrant Boxen](#running-tests-on-vagrant-boxen) General Notes ------------- The rake tasks for running the tests are defined by the Rakefile in the acceptance test directory. These tasks come with some documentation: `rake -T` will give short descriptions, and a `rake -D` will give full descriptions with information on ENV options required and optional for the various tasks. If you are setting up a new repository for acceptance, you will need to bundle install first. This step assumes you have ruby and the bundler gem installed. ```sh cd /path/to/repo/acceptance bundle install --path=.bundle/gems ``` ### Using Git Mirrors By default if you are installing from source, packages will be installed from Github, from their puppetlabs forks. This can be selectively overridden for all installed projects, or per project, by setting environment variables. GIT_SERVER => this will be the address the git server used for all installed projects. Defaults to 'github.com'. FORK => this will be the fork of the project for all installed projects. Defaults to 'puppetlabs'. To customize the server or fork for a specific project use PROJECT_NAME_GIT_SERVER and PROJECT_NAME_FORK. For example, run with these options: ```sh bundle exec rake ci:test:git CONFIG=config/nodes/win2008r2.yaml SHA=abcd PUPPET_GIT_SERVER=percival.corp.puppetlabs.net GIT_SERVER=github.delivery.puppetlabs.net ``` Beaker will install the following: ``` :install=> ["git://github.delivery.puppetlabs.net/puppetlabs-facter.git#stable", "git://github.delivery.puppetlabs.net/puppetlabs-hiera.git#stable", "git://percival.corp.puppetlabs.net/puppetlabs-puppet.git#abcd"], ``` This corresponds to installing facter and hiera stable from our internal mirror, while installing puppet SHA abcd from a git daemon on my local machine percival. See below for details on setting up a local git daemon. Running Tests on the vcloud --------------------------- In order to use the Puppet Labs vcloud, you'll need to be a Puppet Labs employee. Community members should see the [guide to running the tests on vagrant boxen](#running-tests-on-vagrant-boxen). ### Authentication Normally the ci tasks are called from a prepared Jenkins job. If you are running this on your laptop, you will need this ssh private key in order for beaker to be able to log into the vms created from the hosts file: https://github.com/puppetlabs/puppetlabs-modules/blob/production/secure/jenkins/id_rsa-acceptance https://github.com/puppetlabs/puppetlabs-modules/blob/production/secure/jenkins/id_rsa-acceptance.pub Please note in acceptance/Rakefile where the ssh key is defaulted to. It may be looking in ~/.ssh/id_rsa-acceptance, but it may want to look in the working directory (e.g. puppet/acceptance). You will also need QA credentials to vsphere in a ~/.fog file. These credentials can be found on any of the Jenkins coordinator hosts. You may want to check periodically to ensure that the credentials you have are still valid as they may change periodically. ### Packages In order to run the tests on hosts provisioned from packages produced by Delivery, you will need to reference a Puppet commit sha that has been packaged using Delivery's pl:jenkins:uber_build task. This is the snippet used by 'Puppet Packaging' Jenkins jobs: ```sh # EXAMPLE - DO NOT RUN THIS rake --trace package:implode rake --trace package:bootstrap rake --trace pl:jenkins:uber_build ``` The above Rake tasks were run from the root of a Puppet checkout. They are quoted just for reference. Typically if you are investigating a failure, you will have a SHA from a failed jenkins run which should correspond to a successful pipeline run, and you should not need to run the pipeline manually. A finished pipeline will have repository information available at http://builds.puppetlabs.lan/puppet/ So you can also browse this list and select a recent sha which has repo_configs/ available. When executing the ci:test:packages task, you must set the SHA, and also set CONFIG to point to a valid Beaker hosts_file. Configurations used in the Jenkins jobs are available under config/nodes ```sh bundle exec rake ci:test:packages SHA=abcdef CONFIG=config/nodes/rhel.yaml ``` Optionally you may set the TEST (TEST=a/test.rb,and/another/test.rb), and may pass additional OPTIONS to beaker (OPTIONS='--opt foo'). You may also edit a ./local_options.rb hash which will override config/ options, and in turn be overriden by commandline options set in the environment variables CONFIG, TEST and OPTIONS. This file is a ruby file containing a Ruby hash with configuration expected by Beaker. See Beaker source, and examples in config/. ### Git Alternatively you may provision via git clone by calling the ci:test:git task. Currently we don't have packages for Windows or Solaris from the Delivery pipeline, and must use ci:test:git to provision and test these platforms. #### Source Checkout for Different Fork If you have a branch pushed to your fork which you wish to test prior to merging into puppetlabs/puppet, you can do so be setting the FORK environment variable. So, if I have a branch 'issue/master/wonder-if-this-explodes' pushed to my jpartlow puppet fork that I want to test on Windows, I could invoke the following: ```sh bundle exec rake ci:test:git CONFIG=config/nodes/win2008r2.yaml SHA=issue/master/wonder-if-this-explodes FORK=jpartlow ``` #### Source Checkout for Local Branch See notes on running acceptance with Vagrant for more details on using a local git daemon. TODO Fix up the Rakefile's handling of git urls so that there is a simple way to specify both a branch on a github fork, and a branch on some other git server daemon, so that you have fewer steps when serving from a local git daemon. ### Preserving Hosts If you need to ssh into the hosts after a test run, you can use the following sequence: bundle exec rake ci:test_and_preserve_hosts CONFIG=some/config.yaml SHA=12345 TEST=a/foo_test.rb to get the initial templates provisioned, and a local log/latest/preserve_config.yaml created for them. Then you can log into the hosts, or rerun tests against them by: bundle exec rake ci:test_against_preserved_hosts TEST=a/foo_test.rb This will use the existing hosts. +NOTE: If you want configuration information to be preserved for all runs (potentially allowing you to run ci:test_against_preserved_hosts for any previous run that failed, and who's hosts were preserved, regardless of whether you initiated with a ci:test_and_preserve_hosts call) then you should add a ':__preserve_config__ => true' to your local_options.rb. + ### Cleaning Up Preserved Hosts If you run a number of jobs with --preserve_hosts or vi ci:test_and_preserve_hosts, you may eventually generate a large number of stale vms. They should be reaped automatically by qa infrastructure within a day or so, but you may also run: bundle exec rake ci:release_hosts to clean them up sooner and free resources. There also may be scenarios where you want to specify the host(s) to relase. E.g. you may want to relase a subset of the hosts you've created. Or, if a test run terminates early, ci:release_hosts may not be able to derive the name of the vm to delete. In such cases you can specify host(s) to be deleted using the HOST_NAMES environment variable. E.g. HOST_NAMES=lvwwr9tdplg351u bundle exec rake ci:release_hosts HOST_NAMES=lvwwr9tdplg351u,ylrqjh5l6xvym4t bundle exec rake ci:release_hosts Running Tests on Vagrant Boxen ------------------------------ This guide assumes that you have an acceptable Ruby (i.e. 1.9+) installed along with the bundler gem, that you have the puppet repo checked out locally somewhere, and that the name of the checkout folder is `puppet`. I used Ruby 1.9.3-p484 Change to the `acceptance` directory in the root of the puppet repo: ```sh cd /path/to/repo/puppet/acceptance ``` Install the necessary gems with bundler: ```sh bundle install ``` Now you can get a list of test-related tasks you can run via rake: ```sh bundle exec rake -T ``` and view detailed information on the tasks with ```sh bundle exec rake -D ``` As an example, let's try running the acceptance tests using git as the code deployment mechanism. First, we'll have to create a beaker configuration file for a local vagrant box on which to run the tests. Here's what such a file could look like: ```yaml HOSTS: all-in-one: roles: - master - agent platform: centos-64-x64 hypervisor: vagrant ip: 192.168.80.100 box: centos-64-x64-vbox4210-nocm box_url: http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box CONFIG: ``` This defines a 64-bit CentOS 6.4 vagrant box that serves as both a puppet master and a puppet agent for the test roles. (For more information on beaker config files, see [beaker's README](https://github.com/puppetlabs/beaker/blob/master/README.md).) Save this file as `config/nodes/centos6-local.yaml`; we'll be needing it later. Since we have only provided a CentOS box, we don't have anywhere to run windows tests, therefore we'll have to skip those tests. That means we want to pass beaker a --tests argument that contains every directory and file in the `tests` directory besides the one called `windows`. We could pass this option on the command line, but it will be gigantic, so instead let's create a `local_options.rb` file that beaker will automatically read in. This file should contain a ruby hash of beaker's command-line flags to the corresponding flag arguments. Our hash will only contain the `tests` key, and its value will be a comma-seperated list of the other files and directories in `tests`. Here's an easy way to generate this file: ```sh echo "{tests: \"$(echo tests/* | sed -e 's| *tests/windows *||' -e 's/ /,/g')\"}" > local_options.rb" ``` The last thing that needs to be done before we can run the tests is to set up a way for the test box to check out our local changes for testing. We'll do this by starting a git daemon on our host. In another session, navigate to the folder that contains your checkout of the puppet repo, and then create the following symlink: ```sh ln -s . puppetlabs-puppet.git ``` This works around the inflexible checkout path used by the test prep code. Now start the git daemon with ```sh git daemon --verbose --informative-errors --reuseaddr --export-all --base-path=. ``` after which you should see a message like `[32963] Ready to rumble` echoed to the console. Now we can finally run the tests! The rake task that we'll use is `ci:test:git`. Run ``` bundle exec rake -D ci:test:git ``` to read the full description of this task. From the description, we can see that we'll need to set a few environment variables: + CONFIG should be set to point to the CentOS beaker config file we created above. + SHA should be the SHA of the commit we want to test. + GIT_SERVER should be the IP address of the host (i.e. your machine) in the vagrant private network created for the test box. This is derived from the test box's ip by replacing the last octet with 1. For our example above, the host IP is 192.168.80.1 + FORK should be the path to a 'puppetlabs-puppet.git' directory that points to the repo. In our case, this is the path to the symlink we created before, which is inside your puppet repo checkout, so FORK should just be the name of your checkout. We'll assume that the name is `puppet`. Putting it all together, we construct the following command-line invocation to run the tests: ```sh CONFIG=config/nodes/centos6-local.yaml SHA=#{test-commit-sha} GIT_SERVER='192.168.80.1' FORK='puppet' bundle exec rake --trace ci:test:git ``` Go ahead and run that sucker! Testing will take some time. After the testing finishes, you'll either see this line ``` systest completed successfully, thanks. ``` near the end of the output, indicating that all tests completed succesfully, or you'll see the end of a stack trace, indicating failed tests further up.