diff --git a/acceptance/Rakefile b/acceptance/Rakefile index b47e51d72..559974bd7 100644 --- a/acceptance/Rakefile +++ b/acceptance/Rakefile @@ -1,355 +1,355 @@ require 'rake/clean' require 'pp' require 'yaml' $LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), 'lib')) require 'puppet/acceptance/git_utils' extend Puppet::Acceptance::GitUtils ONE_DAY_IN_SECS = 24 * 60 * 60 REPO_CONFIGS_DIR = "repo-configs" CLEAN.include('*.tar', REPO_CONFIGS_DIR, 'merged_options.rb') module HarnessOptions DEFAULTS = { :type => 'git', :helper => ['lib/helper.rb'], :tests => ['tests'], :log_level => 'debug', :color => false, :root_keys => true, :ssh => { :keys => ["id_rsa-acceptance"], }, :xml => true, :timesync => false, :repo_proxy => true, :add_el_extras => true, :preserve_hosts => 'onfail', :forge_host => 'forge-aio01-petest.puppetlabs.com', :'master-start-curl-retries' => 30, } class Aggregator attr_reader :mode def initialize(mode) @mode = mode end def get_options(file_path) puts file_path if File.exists? file_path options = eval(File.read(file_path), binding) else puts "No options file found at #{File.expand_path(file_path)}" end options || {} end def get_mode_options get_options("./config/#{mode}/options.rb") end def get_local_options get_options("./local_options.rb") end def final_options(intermediary_options = {}) mode_options = get_mode_options local_overrides = get_local_options final_options = DEFAULTS.merge(mode_options) final_options.merge!(intermediary_options) final_options.merge!(local_overrides) return final_options end end def self.options(mode, options) final_options = Aggregator.new(mode).final_options(options) final_options end end def beaker_test(mode = :packages, options = {}) delete_options = options.delete(:__delete_options__) || [] final_options = HarnessOptions.options(mode, options) preserve_config = final_options.delete(:__preserve_config__) if mode == :git # Build up project git urls based on git server and fork env variables or defaults final_options[:install].map! do |install| if md = /^(\w+)#(\w+)$/.match(install) project, project_sha = md.captures "#{build_giturl(project)}##{project_sha}" elsif md = /^(\w+)$/.match(install) project = md[1] "#{build_giturl(project)}##{sha}" end end end delete_options.each do |delete_me| final_options.delete(delete_me) end options_file = 'merged_options.rb' File.open(options_file, 'w') do |merged| merged.puts <<-EOS # Copy this file to local_options.rb and adjust as needed if you wish to run # with some local overrides. EOS merged.puts(final_options.pretty_inspect) end tests = ENV['TESTS'] || ENV['TEST'] tests_opt = "--tests=#{tests}" if tests config_opt = "--hosts=#{config}" if config overriding_options = ENV['OPTIONS'] args = ["--options-file", options_file, config_opt, tests_opt, overriding_options].compact begin sh("beaker", *args) ensure preserve_configuration(final_options, options_file) if preserve_config end end def preserve_configuration(final_options, options_file) if (hosts_file = config || final_options[:hosts_file]) && hosts_file !~ /preserved_config/ cp(hosts_file, "log/latest/config.yml") generate_config_for_latest_hosts end mv(options_file, "log/latest") end def generate_config_for_latest_hosts preserved_config_hash = { 'HOSTS' => {} } puts "\nPreserving configuration so that any preserved nodes can be tested again locally..." config_hash = YAML.load_file('log/latest/config.yml') if !config_hash || !config_hash.include?('HOSTS') puts "Warning: No HOSTS configuration found in log/latest/config.yml" return else nodes = config_hash['HOSTS'].map do |node_label,hash| { :node_label => node_label, :roles => hash['roles'], :platform => hash['platform'] } end pre_suite_log = File.read('log/latest/pre_suite-run.log') nodes.each do |node_info| host_regex = /^([\w.]+) \(#{node_info[:node_label]}\)/ if matched = host_regex.match(pre_suite_log) hostname = matched[1] fqdn = "#{hostname}.delivery.puppetlabs.net" elsif /^#{node_info[:node_label]} /.match(pre_suite_log) fqdn = "#{node_info[:node_label]}" puts "* Couldn't find any log lines for #{host_regex}, assuming #{fqdn} is the fqdn" end if fqdn preserved_config_hash['HOSTS'][fqdn] = { 'roles' => node_info[:roles], 'platform' => node_info[:platform], } else puts "* Couldn't match #{node_info[:node_label]} in pre_suite-run.log" end end pp preserved_config_hash File.open('log/latest/preserved_config.yaml', 'w') do |config_file| YAML.dump(preserved_config_hash, config_file) end end rescue Errno::ENOENT => e puts "Warning: Couldn't generate preserved_config.yaml #{e}" end def list_preserved_configurations(secs_ago = ONE_DAY_IN_SECS) preserved = {} Dir.glob('log/*_*').each do |dir| preserved_config_path = "#{dir}/preserved_config.yaml" yesterday = Time.now - secs_ago.to_i if preserved_config = File.exists?(preserved_config_path) directory = File.new(dir) if directory.ctime > yesterday hosts = [] preserved_config = YAML.load_file(preserved_config_path).to_hash preserved_config['HOSTS'].each do |hostname,values| hosts << "#{hostname}: #{values['platform']}, #{values['roles']}" end preserved[hosts] = directory.to_path end end end preserved.map { |k,v| [v,k] }.sort { |a,b| a[0] <=> b[0] }.reverse end def list_preserved_hosts(secs_ago = ONE_DAY_IN_SECS) hosts = Set.new Dir.glob('log/**/pre*suite*run.log').each do |log| yesterday = Time.now - secs_ago.to_i File.open(log, 'r') do |file| if file.ctime > yesterday file.each_line do |line| - matchdata = /^(\w+) \(.*?\) \d\d:\d\d:\d\d\$/.match(line.encode!('UTF-8', 'UTF-8', :invalid => :replace)) + matchdata = /^(\w+)(?:\.[\w.]+)? \(.*?\) \d\d:\d\d:\d\d\$/.match(line.encode!('UTF-8', 'UTF-8', :invalid => :replace)) hosts.add(matchdata[1]) if matchdata end end end end hosts end def release_hosts(hosts = nil, secs_ago = ONE_DAY_IN_SECS) secs_ago ||= ONE_DAY_IN_SECS hosts ||= list_preserved_hosts(secs_ago) require 'beaker' vcloud_pooled = Beaker::VcloudPooled.new(hosts.map { |h| { 'vmhostname' => h } }, :logger => Beaker::Logger.new, :dot_fog => "#{ENV['HOME']}/.fog", 'pooling_api' => 'http://vcloud.delivery.puppetlabs.net' , 'datastore' => 'not-used', 'resourcepool' => 'not-used', 'folder' => 'not-used') vcloud_pooled.cleanup end def print_preserved(preserved) preserved.each_with_index do |entry,i| puts "##{i}: #{entry[0]}" entry[1].each { |h| puts " #{h}" } end end def beaker_run_type type = ENV['TYPE'] || :packages type = type.to_sym end def sha ENV['SHA'] end def config ENV['CONFIG'] end namespace :ci do task :check_env do raise(USAGE) unless sha end namespace :test do USAGE = <<-EOS Requires commit SHA to be put under test as environment variable: SHA=''. Also must set CONFIG=config/nodes/foo.yaml or include it in an options.rb for Beaker. You may set TESTS=path/to/test,and/more/tests. You may set additional Beaker OPTIONS='--more --options' If testing from git checkouts, you may optionally set the github fork to checkout from using PUPPET_FORK='some-other-puppet-fork' (you may change the HIERA_FORK and FACTER_FORK as well if you wish). You may also optionally set the git server to checkout repos from using GIT_SERVER='some.git.mirror'. Or you may set PUPPET_GIT_SERVER='my.host.with.git.daemon', specifically, if you have set up a `git daemon` to pull local commits from. (You will need to allow the git daemon to serve the repo (see `git help daemon` and the docs/acceptance_tests.md for more details)). If there is a Beaker options hash in a ./local_options.rb, it will be included. Commandline options set through the above environment variables will override settings in this file. EOS desc <<-EOS Run the acceptance tests through Beaker and install packages on the configuration targets. #{USAGE} EOS task :packages => 'ci:check_env' do beaker_test end desc <<-EOS Run the acceptance tests through Beaker and install from git on the configuration targets. #{USAGE} EOS task :git => 'ci:check_env' do beaker_test(:git) end end desc "Capture the master and agent hostname from the latest log and construct a preserved_config.yaml for re-running against preserved hosts without provisioning." task :extract_preserved_config do generate_config_for_latest_hosts end desc <<-EOS Run an acceptance test for a given node configuration and preserve the hosts. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. #{USAGE} EOS task :test_and_preserve_hosts => 'ci:check_env' do beaker_test(beaker_run_type, :preserve_hosts => 'always', :__preserve_config__ => true) end desc "List acceptance runs from the past day which had hosts preserved." task :list_preserved do preserved = list_preserved_configurations print_preserved(preserved) end desc <<-EOS Shutdown and destroy any hosts that we have preserved for testing. These should be reaped daily by scripts, but this will free up resources immediately. Specify a list of comma separated HOST_NAMES if you have a set of dynamic vcloud host names you want to purge outside of what can be grepped from the logs. You can go back through the last SECS_AGO logs. Default is one day ago in secs. EOS task :release_hosts do host_names = ENV['HOST_NAMES'].split(',') if ENV['HOST_NAMES'] secs_ago = ENV['SECS_AGO'] release_hosts(host_names, secs_ago) end task :destroy_preserved_hosts => 'ci:release_hosts' do puts "Note: we are now releasing hosts back to the vcloud pooling api rather than destroying them directly. The rake task for this is ci:release_hosts" end desc <<-EOS Rerun an acceptance test using the last captured preserved_config.yaml to skip provisioning. Or specify a CONFIG_NUMBER from `rake ci:list_preserved`. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. EOS task :test_against_preserved_hosts do config_number = (ENV['CONFIG_NUMBER'] || 0).to_i preserved = list_preserved_configurations print_preserved(preserved) config_path = preserved[config_number][0] puts "Using ##{config_number}: #{config_path}" options = { :hosts_file => "#{config_path}/preserved_config.yaml", :no_provision => true, :preserve_hosts => 'always', } run_type = beaker_run_type if run_type == :packages options.merge!(:pre_suite => [ 'setup/packages/pre-suite/015_PackageHostsPresets.rb', 'setup/packages/pre-suite/045_EnsureMasterStartedOnPassenger.rb', ]) else options.merge!(:__delete_options__ => [:pre_suite]) end beaker_test(beaker_run_type, options) end end task :default do sh('rake -T') end task :spec do sh('rspec lib') end diff --git a/acceptance/bin/ci-rerun-pe-puppet.sh b/acceptance/bin/ci-rerun-pe-puppet.sh index 80638e87a..492fa26e9 100755 --- a/acceptance/bin/ci-rerun-pe-puppet.sh +++ b/acceptance/bin/ci-rerun-pe-puppet.sh @@ -1,124 +1,123 @@ #!/bin/bash ##!/bin/bash +e +x #source /usr/local/rvm/scripts/rvm #rvm use ruby-1.9.3-p392 umask 0002 cd acceptance if [ -z "$tests" ]; then echo "Must provide tests to run in the environment variable 'tests': got '$tests'" exit 1 fi if [ -z "$platform" ]; then - echo "'platform' not set: assuming 'el-6-x86_64'" - platform="el-6-x86_64" + echo "'platform' not set: should be 'el-6-x86_64' or 'ubuntu-1204-i386' or some other platform string acceptable to Beaker" + exit 1 fi if [ -z "$1" ]; then echo "Must provide the hostname: got '$1'" exit 1 fi if [ -z "$2" ]; then cat > hosts-immediate.cfg << EOHOSTS --- HOSTS: - ${1}: + ${1}.delivery.puppetlabs.net: roles: - agent - master - dashboard - database platform: ${platform} CONFIG: nfs_server: none consoleport: 443 datastore: instance0 folder: Delivery/Quality Assurance/Enterprise/Dynamic resourcepool: delivery/Quality Assurance/Enterprise/Dynamic pooling_api: http://vcloud.delivery.puppetlabs.net/ EOHOSTS else if [ -z "$3" ]; then cat > hosts-immediate.cfg << EOHOSTS --- HOSTS: - ${1}: + ${1}.delivery.puppetlabs.net: roles: - agent - dashboard - database - master platform: ${platform} - ${2}: + ${2}.delivery.puppetlabs.net: roles: - agent platform: ${platform} CONFIG: nfs_server: none consoleport: 443 datastore: instance0 folder: Delivery/Quality Assurance/Enterprise/Dynamic resourcepool: delivery/Quality Assurance/Enterprise/Dynamic pooling_api: http://vcloud.delivery.puppetlabs.net/ EOHOSTS else cat > hosts-immediate.cfg << EOHOSTS --- HOSTS: - ${1}: + ${1}.delivery.puppetlabs.net: roles: - - agent - master + - dashboard + - database platform: ${platform} - ${2}: + ${2}.delivery.puppetlabs.net: roles: - agent - - dashboard platform: ${platform} - ${3}: + ${3}.delivery.puppetlabs.net: roles: - agent - - database platform: ${platform} CONFIG: nfs_server: none consoleport: 443 datastore: instance0 folder: Delivery/Quality Assurance/Enterprise/Dynamic resourcepool: delivery/Quality Assurance/Enterprise/Dynamic pooling_api: http://vcloud.delivery.puppetlabs.net/ EOHOSTS fi fi export forge_host=api-forge-aio01-petest.puppetlabs.com bundle exec beaker \ --xml \ --debug \ --repo-proxy \ --config hosts-immediate.cfg \ --pre-suite setup/common/pre-suite/110_SetPEPuppetService.rb \ --tests=${tests} \ --keyfile ${HOME}/.ssh/id_rsa-acceptance \ --root-keys \ --helper lib/helper.rb \ --preserve-hosts onfail \ --no-color RESULT=$? exit $RESULT diff --git a/acceptance/lib/puppet/acceptance/classifier_utils.rb b/acceptance/lib/puppet/acceptance/classifier_utils.rb index ec8e58989..3c3e98a3b 100644 --- a/acceptance/lib/puppet/acceptance/classifier_utils.rb +++ b/acceptance/lib/puppet/acceptance/classifier_utils.rb @@ -1,215 +1,225 @@ require 'httparty' require 'tempfile' require 'stringio' require 'uuidtools' require 'json' require 'pp' module Puppet module Acceptance module ClassifierUtils DEFAULT_GROUP_ID = "00000000-0000-4000-8000-000000000000" SSL_PORT = 4433 PREFIX = "/classifier-api" # Keep track of our local tmpdirs for cleanup def self.tmpdirs @classifier_utils_tmpdirs ||= [] end # Create a Classifier Group which by default will apply to all of the passed # nodes. The Group will merge in the passed group_hash which will be converted # into the json body for a Classifier PUT /v1/groups/:id request. # # A teardown body is registered to delete the created group at the end of the test. # # @returns String the created uuid for the group. def create_group_for_nodes(nodes, group_hash) group_uuid = UUIDTools::UUID.random_create() response = nil teardown do step "Deleting group #{group_uuid}" do response = classifier_handle.delete("/v1/groups/#{group_uuid}") assert_equal(204, response.code, "Failed to delete group #{group_uuid}, #{response.code}:#{response.body}") end if response && response.code == 201 end teardown do step "Cleaning up classifier certs on test host" do cleanup_local_classifier_certs end end hostnames = nodes.map { |n| n.hostname } + step "Add group #{group_uuid} for #{hostnames.join(", ")}" rule = hostnames.inject(["or"]) do |r,name| r << ["~", "name", name] r end body = { "description" => "A classification group for the following acceptance test nodes: (#{hostnames.join(", ")})", "parent" => "#{Puppet::Acceptance::ClassifierUtils::DEFAULT_GROUP_ID}", "rule" => rule, "classes" => {} }.merge group_hash response = classifier_handle.put("/v1/groups/#{group_uuid}", :body => body.to_json) assert_equal(201, response.code, "Unexpected response code: #{response.code}, #{response.body}") return group_uuid end # Locates and then updates the 'PE MCollective' group to disable it for # all agent nodes. A teardown is registered to restore the 'PE # MCollective' group at the end of the test. def disable_pe_enterprise_mcollective_agent_classes return if !master.is_pe? + step "Get classifier groups so we can locate the PE MCollective group" response = classifier_handle.get("/v1/groups") assert_equal(200, response.code, "Unable to get classifer groups: #{response.body}") groups_json = response.body groups = JSON.parse(groups_json) pe_mcollective = groups.find { |g| g['name'] == 'PE MCollective' } assert_not_nil pe_mcollective, "Unable to find the 'PE MCollective' group in: #{groups.pretty_inspect}" + select_properties_we_can_put = lambda do |group| + group.select { |k,v| ['id','name','environment','environment_trumps','rule','description','classes','variables','parent'].include?(k) } + end + pe_mcollective = select_properties_we_can_put.call(pe_mcollective) teardown do - groups = JSON.parse(groups_json) - original_pe_mcollective = groups.find { |g| g['name'] == 'PE MCollective' } - response = classifier_handle.put("/v1/groups/#{original_pe_mcollective['id']}", :body => original_pe_mcollective.to_json) - assert_equal(201, response.code, "Unable to restore 'PE MCollective' group: #{response.code}:#{response.body}") + step "Restore original PE MCollective group" do + groups = JSON.parse(groups_json) + original_pe_mcollective = groups.find { |g| g['name'] == 'PE MCollective' } + original_pe_mcollective = select_properties_we_can_put.call(original_pe_mcollective) + response = classifier_handle.put("/v1/groups/#{original_pe_mcollective['id']}", :body => original_pe_mcollective.to_json) + assert_equal(201, response.code, "Unable to restore 'PE MCollective' group: #{response.code}:#{response.body}") + end if response.code == 201 end hostnames = agents.map { |n| n.hostname } + step "Adjust PE MCollective not to match for #{hostnames.join(", ")}" host_matching_rule = hostnames.inject(["or"]) do |r,name| r << ["~", "name", name] r end pe_mcollective['rule'] = ['and', pe_mcollective['rule'], ['not', host_matching_rule]] response = classifier_handle.put("/v1/groups/#{pe_mcollective['id']}", :body => pe_mcollective.to_json) assert_equal(201, response.code, "Unexpected response code: #{response.code}, #{response.body}") end # Creates a group which allows the given nodes to specify their own environments. # Will be torn down at the end of the test. def classify_nodes_as_agent_specified(nodes) create_group_for_nodes(nodes, { "name" => "Agent Specified Test Nodes", "environment" => "agent-specified", "environment_trumps" => true, "description" => "The following acceptance suite nodes (#{nodes.map { |n| n.hostname }.join(", ")}) expect to be able to specify their environment for tesing purposes.", }) end def classify_nodes_as_agent_specified_if_classifer_present classifier_node = false begin classifier_node = find_only_one(:classifier) rescue Beaker::DSL::Outcomes::FailTest end if classifier_node || master.is_pe? classify_nodes_as_agent_specified(agents) end end def classifier_host find_only_one(:classifier) rescue Beaker::DSL::Outcomes::FailTest # fallback to master since currently the sqautils genconfig does not recognize # a classifier role. master end def master_cert @master_cert ||= on(master, "cat `puppet config print hostcert`").stdout end def master_key @master_key ||= on(master, "cat `puppet config print hostprivkey`").stdout end def master_ca_cert_file unless @ca_cert_file ca_cert = on(master, "cat `puppet config print localcacert`").stdout cert_dir = Dir.mktmpdir("pe_classifier_certs") Puppet::Acceptance::ClassifierUtils.tmpdirs << cert_dir @ca_cert_file = File.join(cert_dir, "cacert.pem") File.open(@ca_cert_file, "w") do |f| f.write(ca_cert) end end @ca_cert_file end def cleanup_local_classifier_certs Puppet::Acceptance::ClassifierUtils.tmpdirs.each do |d| FileUtils.rm_rf(d) end end def clear_classifier_utils_cache @master_cert = nil @master_key = nil @ca_cert_file = nil @classifier_handle = nil end def classifier_handle(options = {}) unless @classifier_handle server = options[:server] || classifier_host.reachable_name port = options[:port] || SSL_PORT prefix = options[:prefix] || PREFIX cert = options[:cert] || master_cert key = options[:key] || master_key ca_cert_file = options[:ca_cert_file] || master_ca_cert_file logger = options[:logger] || self.logger # HTTParty performs a lot of configuration at the class level. # This is inconvenient for our needs because we don't have the # server/cert info available at the time the class is loaded. I'm # sidestepping this by generating an anonymous class on the fly when # the test code actually requests a handle to the classifier. @classifier_handle = Class.new do include HTTParty extend Classifier @debugout = StringIO.new @logger = logger base_uri("https://#{server}:#{port}#{prefix}") debug_output(@debugout) headers({'Content-Type' => 'application/json'}) pem(cert + key) ssl_ca_file(ca_cert_file) end end @classifier_handle end # Handle logging module Classifier [:head, :get, :post, :put, :delete].each do |method| define_method(method) do |*args, &block| log_output do super(*args, &block) end end end private # Ensure that the captured debugging output is logged to Beaker. def log_output yield ensure @debugout.rewind @debugout.each_line { |l| @logger.info(l) } @debugout.truncate(0) end end end end end diff --git a/acceptance/tests/environment/use_agent_environment_when_no_enc.rb b/acceptance/tests/environment/use_agent_environment_when_no_enc.rb index a45d573a5..6178f670f 100644 --- a/acceptance/tests/environment/use_agent_environment_when_no_enc.rb +++ b/acceptance/tests/environment/use_agent_environment_when_no_enc.rb @@ -1,50 +1,46 @@ test_name "Agent should use agent environment if there is no enc-specified environment" -require 'puppet/acceptance/classifier_utils' -extend Puppet::Acceptance::ClassifierUtils - -classify_nodes_as_agent_specified_if_classifer_present testdir = create_tmpdir_for_user master, 'use_agent_env' apply_manifest_on(master, <<-MANIFEST, :catch_failures => true) File { ensure => directory, mode => "0770", owner => #{master.puppet['user']}, group => #{master.puppet['group']}, } file { '#{testdir}/environments':; '#{testdir}/environments/production':; '#{testdir}/environments/production/manifests':; '#{testdir}/environments/more_different/':; '#{testdir}/environments/more_different/manifests':; } file { '#{testdir}/environments/production/manifests/site.pp': ensure => file, mode => "0640", content => 'notify { "production environment": }', } file { '#{testdir}/environments/more_different/manifests/more_different.pp': ensure => file, mode => "0640", content => 'notify { "more_different_string": }', } MANIFEST master_opts = { 'main' => { 'environmentpath' => "#{testdir}/environments", }, 'master' => { 'node_terminus' => 'plain' }, } with_puppet_running_on master, master_opts, testdir do agents.each do |agent| run_agent_on(agent, "--no-daemonize --onetime --server #{master} --verbose --environment more_different") assert_match(/more_different_string/, stdout, "Did not find more_different_string from \"more_different\" environment") end end diff --git a/acceptance/tests/environment/use_enc_environment.rb b/acceptance/tests/environment/use_enc_environment.rb index c28f49596..f51cc5c33 100644 --- a/acceptance/tests/environment/use_enc_environment.rb +++ b/acceptance/tests/environment/use_enc_environment.rb @@ -1,56 +1,70 @@ test_name "Agent should use environment given by ENC" +require 'puppet/acceptance/classifier_utils.rb' +extend Puppet::Acceptance::ClassifierUtils testdir = create_tmpdir_for_user master, 'use_enc_env' +if master.is_pe? + group = { + 'name' => 'Special Environment', + 'description' => 'Classify our test agent nodes in the special environment.', + 'environment' => 'special', + 'environment_trumps' => true, + } + create_group_for_nodes(agents, group) +else + create_remote_file master, "#{testdir}/enc.rb", < true) File { ensure => directory, mode => "0770", owner => #{master.puppet['user']}, group => #{master.puppet['group']}, } file { '#{testdir}/environments':; '#{testdir}/environments/production':; '#{testdir}/environments/production/manifests':; '#{testdir}/environments/special/':; '#{testdir}/environments/special/manifests':; } file { '#{testdir}/environments/production/manifests/site.pp': ensure => file, mode => "0640", content => 'notify { "production environment": }', } file { '#{testdir}/environments/special/manifests/different.pp': ensure => file, mode => "0640", content => 'notify { "expected_string": }', } MANIFEST master_opts = { 'main' => { 'environmentpath' => "#{testdir}/environments", }, - 'master' => { - 'node_terminus' => 'exec', - 'external_nodes' => "#{testdir}/enc.rb", - }, } +master_opts['master'] = { + 'node_terminus' => 'exec', + 'external_nodes' => "#{testdir}/enc.rb", +} if !master.is_pe? with_puppet_running_on master, master_opts, testdir do agents.each do |agent| run_agent_on(agent, "--no-daemonize --onetime --server #{master} --verbose") assert_match(/expected_string/, stdout, "Did not find expected_string from \"special\" environment") end end diff --git a/acceptance/tests/security/cve-2013-4761_injection_of_class_names_loading_code.rb b/acceptance/tests/security/cve-2013-4761_injection_of_class_names_loading_code.rb index 43efdd2f0..b1e12446d 100644 --- a/acceptance/tests/security/cve-2013-4761_injection_of_class_names_loading_code.rb +++ b/acceptance/tests/security/cve-2013-4761_injection_of_class_names_loading_code.rb @@ -1,77 +1,77 @@ test_name "CVE 2013-4761 Injection of bad class names causing code loading" do confine :except, :platform => 'windows' testdir = create_tmpdir_for_user master, 'class-names-injection' exploit_path = "#{testdir}/exploit.rb" exploited_path = "#{testdir}/exploited" # @return [String] path to the manifest file def create_exploit_manifest(path, exploit_path_expression) apply_manifest_on(master, <<-MANIFEST, :catch_failures => true) File { ensure => directory, mode => "0770", owner => #{master.puppet['user']}, group => #{master.puppet['group']}, } file { '#{path}/environments':; '#{path}/environments/production':; '#{path}/environments/production/manifests':; '#{path}/environments/production/manifests/site.pp': ensure => file, - content => " -$enc_data = '#{exploit_path_expression}' + content => ' +$enc_data = "#{exploit_path_expression}" include $enc_data -", +', mode => "0640", } MANIFEST end def should_not_be_able_to_exploit(environmentpath, exploited_path) master_opts = { 'main' => { 'environmentpath' => "#{environmentpath}", }, } with_puppet_running_on(master, master_opts) do agents.each do |agent| next if agent['roles'].include?('master') step "Request a catalog to trigger the exploit" do on agent, puppet('agent', '-t', "--server #{master}"), :acceptable_exit_codes => [1] end step "Check that the exploit marker was not created" do on master, "test ! -e #{exploited_path}" end end end end step "Create exploit file" do create_remote_file(master, exploit_path, <<-EXPLOIT) ::File.open('#{exploited_path}', 'w') { |f| f.puts("exploited") } EXPLOIT on(master, "chmod 777 #{exploit_path}") end step "Class name is not interpreted as an absolute path" do create_exploit_manifest(testdir, 'tmp::exploit') should_not_be_able_to_exploit("#{testdir}/environments", exploited_path) end step "Class name cannot be used for a directory traversal out of the module path" do # This is just a guess about how far back we need to go... traversal_exploit_expression = "#{'::..' * 20}#{exploit_path.gsub(File::SEPARATOR,'::')}" create_exploit_manifest(testdir, traversal_exploit_expression) should_not_be_able_to_exploit("#{testdir}/environments", exploited_path) end end diff --git a/acceptance/tests/store_configs/enc_provides_node_when_storeconfigs_enabled.rb b/acceptance/tests/store_configs/enc_provides_node_when_storeconfigs_enabled.rb index 135ebf0c4..01c764622 100644 --- a/acceptance/tests/store_configs/enc_provides_node_when_storeconfigs_enabled.rb +++ b/acceptance/tests/store_configs/enc_provides_node_when_storeconfigs_enabled.rb @@ -1,123 +1,152 @@ test_name "ENC node information is used when store configs enabled (#16698)" +require 'puppet/acceptance/classifier_utils.rb' +extend Puppet::Acceptance::ClassifierUtils confine :to, :platform => ['debian', 'ubuntu'] confine :except, :platform => 'lucid' skip_test "Test not supported on jvm" if @options[:is_puppetserver] testdir = master.tmpdir('use_enc') +apply_manifest_on(master, <<-MANIFEST, :catch_failures => true) + File { + ensure => directory, + mode => "0770", + owner => #{master.puppet['user']}, + group => #{master.puppet['group']}, + } + file { + '#{testdir}':; + '#{testdir}/environments':; + '#{testdir}/environments/production':; + '#{testdir}/environments/production/manifests':; + '#{testdir}/environments/production/manifests/site.pp': + ensure => file, + mode => "0640", + content => 'notify { $data: }'; + } +MANIFEST + +if master.is_pe? + group = { + 'name' => 'Data', + 'description' => 'A group to test that data is passed from the enc', + 'variables' => { :data => 'data from enc' } + } + create_group_for_nodes(agents, group) +else + create_remote_file master, "#{testdir}/enc.rb", < [], 'parameters' => { 'data' => 'data from enc' }, }.to_yaml) END on master, "chmod 755 #{testdir}/enc.rb" -create_remote_file(master, "#{testdir}/site.pp", 'notify { $data: }') - -on master, "chown -R #{master['user']}:#{master['group']} #{testdir}" -on master, "chmod -R g+rwX #{testdir}" - create_remote_file master, "#{testdir}/setup.pp", < $lsbmajdistrelease ? { 5 => '2.2.3', default => '3.2.16', }, default => '3.2.16', } # Trusty doesn't have a rubygems package anymore # Not sure which other Debian's might follow suit so # restricting this narrowly for now # if $lsbdistid == "Ubuntu" and $lsbdistrelease == "14.04" { package { activerecord: ensure => $active_record_version, provider => 'gem', } } else { package { rubygems: ensure => present; activerecord: ensure => $active_record_version, provider => 'gem', require => Package[rubygems]; } } if $osfamily == "Debian" { package { # This is the deb sqlite3 package sqlite3: ensure => present; libsqlite3-dev: ensure => present, require => Package[sqlite3]; } } elsif $osfamily == "RedHat" { $sqlite_gem_pkg_name = $operatingsystem ? { "Fedora" => "rubygem-sqlite3", default => "rubygem-sqlite3-ruby" } package { sqlite: ensure => present; $sqlite_gem_pkg_name: ensure => present, require => Package[sqlite] } } else { fail "Unknown OS $osfamily" } END # This is a brute force hack around PUP-1073 because the deb for the core # sqlite3 package and the rubygem for the sqlite3 driver are both named # 'sqlite3'. So we just run a second puppet apply. create_remote_file master, "#{testdir}/setup_sqlite_gem.pp", < 'sqlite3', ensure => present, provider => 'gem', } } END on master, puppet_apply("#{testdir}/setup.pp") on master, puppet_apply("#{testdir}/setup_sqlite_gem.pp") +end + master_opts = { - 'master' => { - 'node_terminus' => 'exec', - 'external_nodes' => "#{testdir}/enc.rb", - 'storeconfigs' => true, - 'dbadapter' => 'sqlite3', - 'dblocation' => "#{testdir}/store_configs.sqlite3", - 'manifest' => "#{testdir}/site.pp" - } + 'main' => { + 'environmentpath' => "#{testdir}/environments", + }, } +master_opts['master'] = { + 'node_terminus' => 'exec', + 'external_nodes' => "#{testdir}/enc.rb", + 'storeconfigs' => true, + 'dbadapter' => 'sqlite3', + 'dblocation' => "#{testdir}/store_configs.sqlite3", +} if !master.is_pe? with_puppet_running_on master, master_opts, testdir do agents.each do |agent| run_agent_on(agent, "--no-daemonize --onetime --server #{master} --verbose") assert_match(/data from enc/, stdout) end end