diff --git a/acceptance/Rakefile b/acceptance/Rakefile index b42ec8bce..45555a21c 100644 --- a/acceptance/Rakefile +++ b/acceptance/Rakefile @@ -1,323 +1,343 @@ require 'rake/clean' require 'pp' require 'yaml' $LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), 'lib')) require 'puppet/acceptance/git_utils' extend Puppet::Acceptance::GitUtils ONE_DAY_IN_SECS = 24 * 60 * 60 REPO_CONFIGS_DIR = "repo-configs" CLEAN.include('*.tar', REPO_CONFIGS_DIR, 'merged_options.rb') module HarnessOptions DEFAULTS = { :type => 'git', :helper => ['lib/helper.rb'], :tests => ['tests'], :log_level => 'debug', :color => false, :root_keys => true, :ssh => { :keys => ["id_rsa-acceptance"], }, :xml => true, :timesync => false, :repo_proxy => true, :add_el_extras => true, :preserve_hosts => 'onfail', :forge_host => 'forge-aio01-petest.puppetlabs.com' } class Aggregator attr_reader :mode def initialize(mode) @mode = mode end def get_options(file_path) puts file_path if File.exists? file_path options = eval(File.read(file_path), binding) else puts "No options file found at #{File.expand_path(file_path)}" end options || {} end def get_mode_options get_options("./config/#{mode}/options.rb") end def get_local_options get_options("./local_options.rb") end def final_options(intermediary_options = {}) mode_options = get_mode_options local_overrides = get_local_options final_options = DEFAULTS.merge(mode_options) final_options.merge!(intermediary_options) final_options.merge!(local_overrides) return final_options end end def self.options(mode, options) final_options = Aggregator.new(mode).final_options(options) final_options end end def beaker_test(mode = :packages, options = {}) - delete_options = options[:__delete_options__] || [] - final_options = HarnessOptions.options(mode, - options.reject { |k,v| k == :__delete_options__ }) + delete_options = options.delete(:__delete_options__) || [] + final_options = HarnessOptions.options(mode, options) + preserve_config = final_options.delete(:__preserve_config__) if mode == :git # Build up project git urls based on git server and fork env variables or defaults final_options[:install].map! do |install| if md = /^(\w+)#(\w+)$/.match(install) project, project_sha = md.captures "#{build_giturl(project)}##{project_sha}" elsif md = /^(\w+)$/.match(install) project = md[1] "#{build_giturl(project)}##{sha}" end end end delete_options.each do |delete_me| final_options.delete(delete_me) end options_file = 'merged_options.rb' File.open(options_file, 'w') do |merged| merged.puts <<-EOS # Copy this file to local_options.rb and adjust as needed if you wish to run # with some local overrides. EOS merged.puts(final_options.pretty_inspect) end tests = ENV['TESTS'] || ENV['TEST'] tests_opt = "--tests=#{tests}" if tests config_opt = "--hosts=#{config}" if config overriding_options = ENV['OPTIONS'] args = ["--options-file", options_file, config_opt, tests_opt, overriding_options].compact begin sh("beaker", *args) ensure - preserve_configuration(final_options, options_file) + preserve_configuration(final_options, options_file) if preserve_config end end def preserve_configuration(final_options, options_file) if (hosts_file = config || final_options[:hosts_file]) && hosts_file !~ /preserved_config/ cp(hosts_file, "log/latest/config.yml") generate_config_for_latest_hosts end mv(options_file, "log/latest") end def generate_config_for_latest_hosts preserved_config_hash = { 'HOSTS' => {} } - config_hash = YAML.load_file('log/latest/config.yml').to_hash - nodes = config_hash['HOSTS'].map do |node_label,hash| - { :node_label => node_label, :platform => hash['platform'] } - end + puts "\nPreserving configuration so that any preserved nodes can be tested again locally..." + + config_hash = YAML.load_file('log/latest/config.yml') + if !config_hash || !config_hash.include?('HOSTS') + puts "Warning: No HOSTS configuration found in log/latest/config.yml" + return + else + nodes = config_hash['HOSTS'].map do |node_label,hash| + { + :node_label => node_label, + :roles => hash['roles'], + :platform => hash['platform'] + } + end - pre_suite_log = File.read('log/latest/pre_suite-run.log') - nodes.each do |node_info| - hostname = /^(\w+) \(#{node_info[:node_label]}\)/.match(pre_suite_log)[1] - fqdn = "#{hostname}.delivery.puppetlabs.net" - preserved_config_hash['HOSTS'][fqdn] = { - 'roles' => [ 'agent'], - 'platform' => node_info[:platform], - } - preserved_config_hash['HOSTS'][fqdn]['roles'].unshift('master') if node_info[:node_label] =~ /master/ - end - pp preserved_config_hash + pre_suite_log = File.read('log/latest/pre_suite-run.log') + nodes.each do |node_info| + host_regex = /^([\w.]+) \(#{node_info[:node_label]}\)/ + if matched = host_regex.match(pre_suite_log) + hostname = matched[1] + fqdn = "#{hostname}.delivery.puppetlabs.net" + elsif /^#{node_info[:node_label]} /.match(pre_suite_log) + fqdn = "#{node_info[:node_label]}" + puts "* Couldn't find any log lines for #{host_regex}, assuming #{fqdn} is the fqdn" + end + if fqdn + preserved_config_hash['HOSTS'][fqdn] = { + 'roles' => node_info[:roles], + 'platform' => node_info[:platform], + } + else + puts "* Couldn't match #{node_info[:node_label]} in pre_suite-run.log" + end + end + pp preserved_config_hash - File.open('log/latest/preserved_config.yaml', 'w') do |config_file| - YAML.dump(preserved_config_hash, config_file) + File.open('log/latest/preserved_config.yaml', 'w') do |config_file| + YAML.dump(preserved_config_hash, config_file) + end end rescue Errno::ENOENT => e - puts "Couldn't generate log #{e}" + puts "Warning: Couldn't generate preserved_config.yaml #{e}" end def list_preserved_configurations(secs_ago = ONE_DAY_IN_SECS) preserved = {} Dir.glob('log/*_*').each do |dir| preserved_config_path = "#{dir}/preserved_config.yaml" yesterday = Time.now - secs_ago.to_i if preserved_config = File.exists?(preserved_config_path) directory = File.new(dir) if directory.ctime > yesterday hosts = [] preserved_config = YAML.load_file(preserved_config_path).to_hash preserved_config['HOSTS'].each do |hostname,values| hosts << "#{hostname}: #{values['platform']}, #{values['roles']}" end preserved[hosts] = directory.to_path end end end preserved.map { |k,v| [v,k] }.sort { |a,b| a[0] <=> b[0] }.reverse end def list_preserved_hosts(secs_ago = ONE_DAY_IN_SECS) hosts = Set.new Dir.glob('log/**/pre*suite*run.log').each do |log| yesterday = Time.now - secs_ago.to_i File.open(log, 'r') do |file| if file.ctime > yesterday file.each_line do |line| matchdata = /^(\w+) \(.*?\) \d\d:\d\d:\d\d\$/.match(line.encode!('UTF-8', 'UTF-8', :invalid => :replace)) hosts.add(matchdata[1]) if matchdata end end end end hosts end def release_hosts(hosts = nil, secs_ago = ONE_DAY_IN_SECS) secs_ago ||= ONE_DAY_IN_SECS hosts ||= list_preserved_hosts(secs_ago) require 'beaker' vcloud_pooled = Beaker::VcloudPooled.new(hosts.map { |h| { 'vmhostname' => h } }, :logger => Beaker::Logger.new, :dot_fog => "#{ENV['HOME']}/.fog", 'pooling_api' => 'http://vcloud.delivery.puppetlabs.net' , 'datastore' => 'not-used', 'resourcepool' => 'not-used', 'folder' => 'not-used') vcloud_pooled.cleanup end def print_preserved(preserved) preserved.each_with_index do |entry,i| puts "##{i}: #{entry[0]}" entry[1].each { |h| puts " #{h}" } end end def beaker_run_type type = ENV['TYPE'] || :packages type = type.to_sym end def sha ENV['SHA'] end def config ENV['CONFIG'] end namespace :ci do task :check_env do raise(USAGE) unless sha end namespace :test do USAGE = <<-EOS Requires commit SHA to be put under test as environment variable: SHA=''. Also must set CONFIG=config/nodes/foo.yaml or include it in an options.rb for Beaker. You may set TESTS=path/to/test,and/more/tests. You may set additional Beaker OPTIONS='--more --options' If testing from git checkouts, you may optionally set the github fork to checkout from using PUPPET_FORK='some-other-puppet-fork' (you may change the HIERA_FORK and FACTER_FORK as well if you wish). You may also optionally set the git server to checkout repos from using GIT_SERVER='some.git.mirror'. Or you may set PUPPET_GIT_SERVER='my.host.with.git.daemon', specifically, if you have set up a `git daemon` to pull local commits from. (You will need to allow the git daemon to serve the repo (see `git help daemon` and the docs/acceptance_tests.md for more details)). If there is a Beaker options hash in a ./local_options.rb, it will be included. Commandline options set through the above environment variables will override settings in this file. EOS desc <<-EOS Run the acceptance tests through Beaker and install packages on the configuration targets. #{USAGE} EOS task :packages => 'ci:check_env' do beaker_test end desc <<-EOS Run the acceptance tests through Beaker and install from git on the configuration targets. #{USAGE} EOS task :git => 'ci:check_env' do beaker_test(:git) end end desc "Capture the master and agent hostname from the latest log and construct a preserved_config.yaml for re-running against preserved hosts without provisioning." task :extract_preserved_config do generate_config_for_latest_hosts end desc <<-EOS Run an acceptance test for a given node configuration and preserve the hosts. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. #{USAGE} EOS task :test_and_preserve_hosts => 'ci:check_env' do - beaker_test(beaker_run_type, :preserve_hosts => 'always') + beaker_test(beaker_run_type, :preserve_hosts => 'always', :__preserve_config__ => true) end desc "List acceptance runs from the past day which had hosts preserved." task :list_preserved do preserved = list_preserved_configurations print_preserved(preserved) end desc <<-EOS Shutdown and destroy any hosts that we have preserved for testing. These should be reaped daily by scripts, but this will free up resources immediately. Specify a list of comma separated HOST_NAMES if you have a set of dynamic vcloud host names you want to purge outside of what can be grepped from the logs. You can go back through the last SECS_AGO logs. Default is one day ago in secs. EOS task :release_hosts do host_names = ENV['HOST_NAMES'].split(',') if ENV['HOST_NAMES'] secs_ago = ENV['SECS_AGO'] release_hosts(host_names, secs_ago) end task :destroy_preserved_hosts => 'ci:release_hosts' do puts "Note: we are now releasing hosts back to the vcloud pooling api rather than destroying them directly. The rake task for this is ci:release_hosts" end desc <<-EOS Rerun an acceptance test using the last captured preserved_config.yaml to skip provisioning. Or specify a CONFIG_NUMBER from `rake ci:list_preserved`. Defaults to a packages run, but you can set it to 'git' with TYPE='git'. EOS task :test_against_preserved_hosts do config_number = (ENV['CONFIG_NUMBER'] || 0).to_i preserved = list_preserved_configurations print_preserved(preserved) config_path = preserved[config_number][0] puts "Using ##{config_number}: #{config_path}" beaker_test(beaker_run_type, :hosts_file => "#{config_path}/preserved_config.yaml", :no_provision => true, :preserve_hosts => 'always', :__delete_options__ => [:pre_suite] ) end end task :default do sh('rake -T') end task :spec do sh('rspec lib') end diff --git a/docs/acceptance_tests.md b/docs/acceptance_tests.md index baf0dd540..809ac9c43 100644 --- a/docs/acceptance_tests.md +++ b/docs/acceptance_tests.md @@ -1,237 +1,239 @@ Running Acceptance Tests Yourself ================================= Table of Contents ----------------- * [General Notes](#general-notes) * [Running Tests on the vcloud](#running-tests-on-the-vcloud) * [Running Tests on Vagrant Boxen](#running-tests-on-vagrant-boxen) General Notes ------------- The rake tasks for running the tests are defined by the Rakefile in the acceptance test directory. These tasks come with some documentation: `rake -T` will give short descriptions, and a `rake -D` will give full descriptions with information on ENV options required and optional for the various tasks. If you are setting up a new repository for acceptance, you will need to bundle install first. This step assumes you have ruby and the bundler gem installed. ```sh cd /path/to/repo/acceptance bundle install --path=.bundle/gems ``` ### Using Git Mirrors By default if you are installing from source, packages will be installed from Github, from their puppetlabs forks. This can be selectively overridden for all installed projects, or per project, by setting environment variables. GIT_SERVER => this will be the address the git server used for all installed projects. Defaults to 'github.com'. FORK => this will be the fork of the project for all installed projects. Defaults to 'puppetlabs'. To customize the server or fork for a specific project use PROJECT_NAME_GIT_SERVER and PROJECT_NAME_FORK. For example, run with these options: ```sh bundle exec rake ci:test:git CONFIG=config/nodes/win2008r2.yaml SHA=abcd PUPPET_GIT_SERVER=percival.corp.puppetlabs.net GIT_SERVER=github.delivery.puppetlabs.net ``` Beaker will install the following: ``` :install=> ["git://github.delivery.puppetlabs.net/puppetlabs-facter.git#stable", "git://github.delivery.puppetlabs.net/puppetlabs-hiera.git#stable", "git://percival.corp.puppetlabs.net/puppetlabs-puppet.git#abcd"], ``` This corresponds to installing facter and hiera stable from our internal mirror, while installing puppet SHA abcd from a git daemon on my local machine percival. See below for details on setting up a local git daemon. Running Tests on the vcloud --------------------------- In order to use the Puppet Labs vcloud, you'll need to be a Puppet Labs employee. Community members should see the [guide to running the tests on vagrant boxen](#running-tests-on-vagrant-boxen). ### Authentication Normally the ci tasks are called from a prepared Jenkins job. If you are running this on your laptop, you will need this ssh private key in order for beaker to be able to log into the vms created from the hosts file: https://github.com/puppetlabs/puppetlabs-modules/blob/production/secure/jenkins/id_rsa-acceptance https://github.com/puppetlabs/puppetlabs-modules/blob/production/secure/jenkins/id_rsa-acceptance.pub Please note in acceptance/Rakefile where the ssh key is defaulted to. It may be looking in ~/.ssh/id_rsa-acceptance, but it may want to look in the working directory (e.g. puppet/acceptance). You will also need QA credentials to vsphere in a ~/.fog file. These credentials can be found on any of the Jenkins coordinator hosts. You may want to check periodically to ensure that the credentials you have are still valid as they may change periodically. ### Packages In order to run the tests on hosts provisioned from packages produced by Delivery, you will need to reference a Puppet commit sha that has been packaged using Delivery's pl:jenkins:uber_build task. This is the snippet used by 'Puppet Packaging' Jenkins jobs: ```sh # EXAMPLE - DO NOT RUN THIS rake --trace package:implode rake --trace package:bootstrap rake --trace pl:jenkins:uber_build ``` The above Rake tasks were run from the root of a Puppet checkout. They are quoted just for reference. Typically if you are investigating a failure, you will have a SHA from a failed jenkins run which should correspond to a successful pipeline run, and you should not need to run the pipeline manually. A finished pipeline will have repository information available at http://builds.puppetlabs.lan/puppet/ So you can also browse this list and select a recent sha which has repo_configs/ available. When executing the ci:test:packages task, you must set the SHA, and also set CONFIG to point to a valid Beaker hosts_file. Configurations used in the Jenkins jobs are available under config/nodes ```sh bundle exec rake ci:test:packages SHA=abcdef CONFIG=config/nodes/rhel.yaml ``` Optionally you may set the TEST (TEST=a/test.rb,and/another/test.rb), and may pass additional OPTIONS to beaker (OPTIONS='--opt foo'). You may also edit a ./local_options.rb hash which will override config/ options, and in turn be overriden by commandline options set in the environment variables CONFIG, TEST and OPTIONS. This file is a ruby file containing a Ruby hash with configuration expected by Beaker. See Beaker source, and examples in config/. ### Git Alternatively you may provision via git clone by calling the ci:test:git task. Currently we don't have packages for Windows or Solaris from the Delivery pipeline, and must use ci:test:git to provision and test these platforms. #### Source Checkout for Different Fork If you have a branch pushed to your fork which you wish to test prior to merging into puppetlabs/puppet, you can do so be setting the FORK environment variable. So, if I have a branch 'issue/master/wonder-if-this-explodes' pushed to my jpartlow puppet fork that I want to test on Windows, I could invoke the following: ```sh bundle exec rake ci:test:git CONFIG=config/nodes/win2008r2.yaml SHA=issue/master/wonder-if-this-explodes FORK=jpartlow ``` #### Source Checkout for Local Branch See notes on running acceptance with Vagrant for more details on using a local git daemon. TODO Fix up the Rakefile's handling of git urls so that there is a simple way to specify both a branch on a github fork, and a branch on some other git server daemon, so that you have fewer steps when serving from a local git daemon. ### Preserving Hosts If you need to ssh into the hosts after a test run, you can use the following sequence: bundle exec rake ci:test_and_preserve_hosts CONFIG=some/config.yaml SHA=12345 TEST=a/foo_test.rb to get the initial templates provisioned, and a local log/latest/preserve_config.yaml created for them. Then you can log into the hosts, or rerun tests against them by: bundle exec rake ci:test_against_preserved_hosts TEST=a/foo_test.rb This will use the existing hosts. +NOTE: If you want configuration information to be preserved for all runs (potentially allowing you to run ci:test_against_preserved_hosts for any previous run that failed, and who's hosts were preserved, regardless of whether you initiated with a ci:test_and_preserve_hosts call) then you should add a ':__preserve_config__ => true' to your local_options.rb. + ### Cleaning Up Preserved Hosts If you run a number of jobs with --preserve_hosts or vi ci:test_and_preserve_hosts, you may eventually generate a large number of stale vms. They should be reaped automatically by qa infrastructure within a day or so, but you may also run: bundle exec rake ci:release_hosts to clean them up sooner and free resources. There also may be scenarios where you want to specify the host(s) to relase. E.g. you may want to relase a subset of the hosts you've created. Or, if a test run terminates early, ci:release_hosts may not be able to derive the name of the vm to delete. In such cases you can specify host(s) to be deleted using the HOST_NAMES environment variable. E.g. HOST_NAMES=lvwwr9tdplg351u bundle exec rake ci:release_hosts HOST_NAMES=lvwwr9tdplg351u,ylrqjh5l6xvym4t bundle exec rake ci:release_hosts Running Tests on Vagrant Boxen ------------------------------ This guide assumes that you have an acceptable Ruby (i.e. 1.9+) installed along with the bundler gem, that you have the puppet repo checked out locally somewhere, and that the name of the checkout folder is `puppet`. I used Ruby 1.9.3-p484 Change to the `acceptance` directory in the root of the puppet repo: ```sh cd /path/to/repo/puppet/acceptance ``` Install the necessary gems with bundler: ```sh bundle install ``` Now you can get a list of test-related tasks you can run via rake: ```sh bundle exec rake -T ``` and view detailed information on the tasks with ```sh bundle exec rake -D ``` As an example, let's try running the acceptance tests using git as the code deployment mechanism. First, we'll have to create a beaker configuration file for a local vagrant box on which to run the tests. Here's what such a file could look like: ```yaml HOSTS: all-in-one: roles: - master - agent platform: centos-64-x64 hypervisor: vagrant ip: 192.168.80.100 box: centos-64-x64-vbox4210-nocm box_url: http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box CONFIG: ``` This defines a 64-bit CentOS 6.4 vagrant box that serves as both a puppet master and a puppet agent for the test roles. (For more information on beaker config files, see [beaker's README](https://github.com/puppetlabs/beaker/blob/master/README.md).) Save this file as `config/nodes/centos6-local.yaml`; we'll be needing it later. Since we have only provided a CentOS box, we don't have anywhere to run windows tests, therefore we'll have to skip those tests. That means we want to pass beaker a --tests argument that contains every directory and file in the `tests` directory besides the one called `windows`. We could pass this option on the command line, but it will be gigantic, so instead let's create a `local_options.rb` file that beaker will automatically read in. This file should contain a ruby hash of beaker's command-line flags to the corresponding flag arguments. Our hash will only contain the `tests` key, and its value will be a comma-seperated list of the other files and directories in `tests`. Here's an easy way to generate this file: ```sh echo "{tests: \"$(echo tests/* | sed -e 's| *tests/windows *||' -e 's/ /,/g')\"}" > local_options.rb" ``` The last thing that needs to be done before we can run the tests is to set up a way for the test box to check out our local changes for testing. We'll do this by starting a git daemon on our host. In another session, navigate to the folder that contains your checkout of the puppet repo, and then create the following symlink: ```sh ln -s . puppetlabs-puppet.git ``` This works around the inflexible checkout path used by the test prep code. Now start the git daemon with ```sh git daemon --verbose --informative-errors --reuseaddr --export-all --base-path=. ``` after which you should see a message like `[32963] Ready to rumble` echoed to the console. Now we can finally run the tests! The rake task that we'll use is `ci:test:git`. Run ``` bundle exec rake -D ci:test:git ``` to read the full description of this task. From the description, we can see that we'll need to set a few environment variables: + CONFIG should be set to point to the CentOS beaker config file we created above. + SHA should be the SHA of the commit we want to test. + GIT_SERVER should be the IP address of the host (i.e. your machine) in the vagrant private network created for the test box. This is derived from the test box's ip by replacing the last octet with 1. For our example above, the host IP is 192.168.80.1 + FORK should be the path to a 'puppetlabs-puppet.git' directory that points to the repo. In our case, this is the path to the symlink we created before, which is inside your puppet repo checkout, so FORK should just be the name of your checkout. We'll assume that the name is `puppet`. Putting it all together, we construct the following command-line invocation to run the tests: ```sh CONFIG=config/nodes/centos6-local.yaml SHA=#{test-commit-sha} GIT_SERVER='192.168.80.1' FORK='puppet' bundle exec rake --trace ci:test:git ``` Go ahead and run that sucker! Testing will take some time. After the testing finishes, you'll either see this line ``` systest completed successfully, thanks. ``` near the end of the output, indicating that all tests completed succesfully, or you'll see the end of a stack trace, indicating failed tests further up. diff --git a/lib/puppet/application/doc.rb b/lib/puppet/application/doc.rb index 61844b713..e45d4d0e0 100644 --- a/lib/puppet/application/doc.rb +++ b/lib/puppet/application/doc.rb @@ -1,280 +1,273 @@ require 'puppet/application' class Puppet::Application::Doc < Puppet::Application run_mode :master attr_accessor :unknown_args, :manifest def preinit {:references => [], :mode => :text, :format => :to_markdown }.each do |name,value| options[name] = value end @unknown_args = [] @manifest = false end option("--all","-a") option("--outputdir OUTPUTDIR","-o") option("--verbose","-v") option("--debug","-d") option("--charset CHARSET") option("--format FORMAT", "-f") do |arg| method = "to_#{arg}" require 'puppet/util/reference' if Puppet::Util::Reference.method_defined?(method) options[:format] = method else raise "Invalid output format #{arg}" end end option("--mode MODE", "-m") do |arg| require 'puppet/util/reference' if Puppet::Util::Reference.modes.include?(arg) or arg.intern==:rdoc options[:mode] = arg.intern else raise "Invalid output mode #{arg}" end end option("--list", "-l") do |arg| require 'puppet/util/reference' puts Puppet::Util::Reference.references.collect { |r| Puppet::Util::Reference.reference(r).doc }.join("\n") exit(0) end option("--reference REFERENCE", "-r") do |arg| options[:references] << arg.intern end def help <<-'HELP' puppet-doc(8) -- Generate Puppet documentation and references ======== SYNOPSIS -------- Generates a reference for all Puppet types. Largely meant for internal Puppet Labs use. -WARNING: RDoc support is only available under Ruby 1.8.7 and earlier. - USAGE ----- puppet doc [-a|--all] [-h|--help] [-l|--list] [-o|--outputdir ] [-m|--mode text|pdf|rdoc] [-r|--reference ] [--charset ] [] DESCRIPTION ----------- If mode is not 'rdoc', then this command generates a Markdown document describing all installed Puppet types or all allowable arguments to puppet executables. It is largely meant for internal use and is used to generate the reference document available on the Puppet Labs web site. In 'rdoc' mode, this command generates an html RDoc hierarchy describing the manifests that are in 'manifestdir' and 'modulepath' configuration directives. The generated documentation directory is doc by default but can be changed with the 'outputdir' option. If the command is run with the name of a manifest file as an argument, puppet doc will output a single manifest's documentation on stdout. -WARNING: RDoc support is only available under Ruby 1.8.7 and earlier. -The internal API used to support manifest documentation has changed -radically in newer versions, and support is not yet available for -using those versions of RDoc. - OPTIONS ------- * --all: Output the docs for all of the reference types. In 'rdoc' mode, this also outputs documentation for all resources. * --help: Print this help message * --outputdir: Used only in 'rdoc' mode. The directory to which the rdoc output should be written. * --mode: Determine the output mode. Valid modes are 'text', 'pdf' and 'rdoc'. The 'pdf' mode creates PDF formatted files in the /tmp directory. The default mode is 'text'. * --reference: Build a particular reference. Get a list of references by running 'puppet doc --list'. * --charset: Used only in 'rdoc' mode. It sets the charset used in the html files produced. * --manifestdir: Used only in 'rdoc' mode. The directory to scan for stand-alone manifests. If not supplied, puppet doc will use the manifestdir from puppet.conf. * --modulepath: Used only in 'rdoc' mode. The directory or directories to scan for modules. If not supplied, puppet doc will use the modulepath from puppet.conf. * --environment: Used only in 'rdoc' mode. The configuration environment from which to read the modulepath and manifestdir settings, when reading said settings from puppet.conf. EXAMPLE ------- $ puppet doc -r type > /tmp/type_reference.markdown or $ puppet doc --outputdir /tmp/rdoc --mode rdoc /path/to/manifests or $ puppet doc /etc/puppet/manifests/site.pp or $ puppet doc -m pdf -r configuration AUTHOR ------ Luke Kanies COPYRIGHT --------- Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License HELP end def handle_unknown( opt, arg ) @unknown_args << {:opt => opt, :arg => arg } true end def run_command return [:rdoc].include?(options[:mode]) ? send(options[:mode]) : other end def rdoc exit_code = 0 files = [] unless @manifest env = Puppet.lookup(:current_environment) files += env.modulepath files << ::File.dirname(env.manifest) if env.manifest != Puppet::Node::Environment::NO_MANIFEST end files += command_line.args Puppet.info "scanning: #{files.inspect}" Puppet.settings[:document_all] = options[:all] || false begin require 'puppet/util/rdoc' if @manifest Puppet::Util::RDoc.manifestdoc(files) else options[:outputdir] = "doc" unless options[:outputdir] Puppet::Util::RDoc.rdoc(options[:outputdir], files, options[:charset]) end rescue => detail Puppet.log_exception(detail, "Could not generate documentation: #{detail}") exit_code = 1 end exit exit_code end def other text = "" with_contents = options[:references].length <= 1 exit_code = 0 require 'puppet/util/reference' options[:references].sort { |a,b| a.to_s <=> b.to_s }.each do |name| raise "Could not find reference #{name}" unless section = Puppet::Util::Reference.reference(name) begin # Add the per-section text, but with no ToC text += section.send(options[:format], with_contents) rescue => detail Puppet.log_exception(detail, "Could not generate reference #{name}: #{detail}") exit_code = 1 next end end text += Puppet::Util::Reference.footer unless with_contents # We've only got one reference if options[:mode] == :pdf Puppet::Util::Reference.pdf(text) else puts text end exit exit_code end def setup # sole manifest documentation if command_line.args.size > 0 options[:mode] = :rdoc @manifest = true end if options[:mode] == :rdoc setup_rdoc else setup_reference end setup_logging end def setup_reference if options[:all] # Don't add dynamic references to the "all" list. require 'puppet/util/reference' options[:references] = Puppet::Util::Reference.references.reject do |ref| Puppet::Util::Reference.reference(ref).dynamic? end end options[:references] << :type if options[:references].empty? end def setup_rdoc(dummy_argument=:work_arround_for_ruby_GC_bug) # consume the unknown options # and feed them as settings if @unknown_args.size > 0 @unknown_args.each do |option| # force absolute path for modulepath when passed on commandline if option[:opt]=="--modulepath" or option[:opt] == "--manifestdir" option[:arg] = option[:arg].split(::File::PATH_SEPARATOR).collect { |p| ::File.expand_path(p) }.join(::File::PATH_SEPARATOR) end Puppet.settings.handlearg(option[:opt], option[:arg]) end end end def setup_logging # Handle the logging settings. if options[:debug] Puppet::Util::Log.level = :debug elsif options[:verbose] Puppet::Util::Log.level = :info else Puppet::Util::Log.level = :warning end Puppet::Util::Log.newdestination(:console) end end diff --git a/lib/puppet/file_system/file19.rb b/lib/puppet/file_system/file19.rb index fce9a6a82..8872ba6fc 100644 --- a/lib/puppet/file_system/file19.rb +++ b/lib/puppet/file_system/file19.rb @@ -1,5 +1,46 @@ class Puppet::FileSystem::File19 < Puppet::FileSystem::FileImpl def binread(path) path.binread end + + # Provide an encoding agnostic version of compare_stream + # + # The FileUtils implementation in Ruby 2.0+ was modified in a manner where + # it cannot properly compare File and StringIO instances. To sidestep that + # issue this method reimplements the faster 2.0 version that will correctly + # compare binary File and StringIO streams. + def compare_stream(path, stream) + open(path, 0, 'rb') do |this| + bsize = stream_blksize(this, stream) + sa = "".force_encoding('ASCII-8BIT') + sb = "".force_encoding('ASCII-8BIT') + begin + this.read(bsize, sa) + stream.read(bsize, sb) + return true if sa.empty? && sb.empty? + end while sa == sb + false + end + end + + private + def stream_blksize(*streams) + streams.each do |s| + next unless s.respond_to?(:stat) + size = blksize(s.stat) + return size if size + end + default_blksize() + end + + def blksize(st) + s = st.blksize + return nil unless s + return nil if s == 0 + s + end + + def default_blksize + 1024 + end end diff --git a/lib/puppet/network/http/connection.rb b/lib/puppet/network/http/connection.rb index 388f8ba37..8ffb1dda1 100644 --- a/lib/puppet/network/http/connection.rb +++ b/lib/puppet/network/http/connection.rb @@ -1,240 +1,240 @@ require 'net/https' require 'puppet/ssl/host' require 'puppet/ssl/configuration' require 'puppet/ssl/validator' require 'puppet/network/authentication' require 'puppet/network/http' require 'uri' module Puppet::Network::HTTP # This will be raised if too many redirects happen for a given HTTP request class RedirectionLimitExceededException < Puppet::Error ; end # This class provides simple methods for issuing various types of HTTP # requests. It's interface is intended to mirror Ruby's Net::HTTP # object, but it provides a few important bits of additional # functionality. Notably: # # * Any HTTPS requests made using this class will use Puppet's SSL # certificate configuration for their authentication, and # * Provides some useful error handling for any SSL errors that occur # during a request. # @api public class Connection include Puppet::Network::Authentication OPTION_DEFAULTS = { :use_ssl => true, :verify => nil, :redirect_limit => 10, } # Creates a new HTTP client connection to `host`:`port`. # @param host [String] the host to which this client will connect to # @param port [Fixnum] the port to which this client will connect to # @param options [Hash] options influencing the properties of the created # connection, # @option options [Boolean] :use_ssl true to connect with SSL, false # otherwise, defaults to true # @option options [#setup_connection] :verify An object that will configure # any verification to do on the connection # @option options [Fixnum] :redirect_limit the number of allowed # redirections, defaults to 10 passing any other option in the options # hash results in a Puppet::Error exception # # @note the HTTP connection itself happens lazily only when {#request}, or # one of the {#get}, {#post}, {#delete}, {#head} or {#put} is called # @note The correct way to obtain a connection is to use one of the factory # methods on {Puppet::Network::HttpPool} # @api private def initialize(host, port, options = {}) @host = host @port = port unknown_options = options.keys - OPTION_DEFAULTS.keys raise Puppet::Error, "Unrecognized option(s): #{unknown_options.map(&:inspect).sort.join(', ')}" unless unknown_options.empty? options = OPTION_DEFAULTS.merge(options) @use_ssl = options[:use_ssl] @verify = options[:verify] @redirect_limit = options[:redirect_limit] @site = Puppet::Network::HTTP::Site.new(@use_ssl ? 'https' : 'http', host, port) @pool = Puppet.lookup(:http_pool) end # @!macro [new] common_options # @param options [Hash] options influencing the request made # @option options [Hash{Symbol => String}] :basic_auth The basic auth # :username and :password to use for the request # @param path [String] # @param headers [Hash{String => String}] # @!macro common_options # @api public def get(path, headers = {}, options = {}) request_with_redirects(Net::HTTP::Get.new(path, headers), options) end # @param path [String] # @param data [String] # @param headers [Hash{String => String}] # @!macro common_options # @api public def post(path, data, headers = nil, options = {}) request = Net::HTTP::Post.new(path, headers) request.body = data request_with_redirects(request, options) end # @param path [String] # @param headers [Hash{String => String}] # @!macro common_options # @api public def head(path, headers = {}, options = {}) request_with_redirects(Net::HTTP::Head.new(path, headers), options) end # @param path [String] # @param headers [Hash{String => String}] # @!macro common_options # @api public def delete(path, headers = {'Depth' => 'Infinity'}, options = {}) request_with_redirects(Net::HTTP::Delete.new(path, headers), options) end # @param path [String] # @param data [String] # @param headers [Hash{String => String}] # @!macro common_options # @api public def put(path, data, headers = nil, options = {}) request = Net::HTTP::Put.new(path, headers) request.body = data request_with_redirects(request, options) end def request(method, *args) self.send(method, *args) end # TODO: These are proxies for the Net::HTTP#request_* methods, which are # almost the same as the "get", "post", etc. methods that we've ported above, # but they are able to accept a code block and will yield to it, which is # necessary to stream responses, e.g. file content. For now # we're not funneling these proxy implementations through our #request # method above, so they will not inherit the same error handling. In the # future we may want to refactor these so that they are funneled through # that method and do inherit the error handling. def request_get(*args, &block) with_connection(@site) do |connection| connection.request_get(*args, &block) end end def request_head(*args, &block) with_connection(@site) do |connection| connection.request_head(*args, &block) end end def request_post(*args, &block) with_connection(@site) do |connection| connection.request_post(*args, &block) end end # end of Net::HTTP#request_* proxies # The address to connect to. def address @site.host end # The port to connect to. def port @site.port end # Whether to use ssl def use_ssl? @site.use_ssl? end private def request_with_redirects(request, options) current_request = request current_site = @site response = nil 0.upto(@redirect_limit) do |redirection| return response if response with_connection(current_site) do |connection| apply_options_to(current_request, options) current_response = execute_request(connection, current_request) if [301, 302, 307].include?(current_response.code.to_i) # handle the redirection location = URI.parse(current_response['location']) current_site = current_site.move_to(location) # update to the current request path current_request = current_request.class.new(location.path) current_request.body = request.body request.each do |header, value| current_request[header] = value end else response = current_response end end # and try again... end raise RedirectionLimitExceededException, "Too many HTTP redirections for #{@host}:#{@port}" end def apply_options_to(request, options) if options[:basic_auth] request.basic_auth(options[:basic_auth][:user], options[:basic_auth][:password]) end end def execute_request(connection, request) response = connection.request(request) # Check the peer certs and warn if they're nearing expiration. warn_if_near_expiration(*@verify.peer_certs) + response + end + + def with_connection(site, &block) + response = nil + @pool.with_connection(site, @verify) do |conn| + response = yield conn + end response rescue OpenSSL::SSL::SSLError => error if error.message.include? "certificate verify failed" msg = error.message msg << ": [" + @verify.verify_errors.join('; ') + "]" raise Puppet::Error, msg, error.backtrace elsif error.message =~ /hostname.*not match.*server certificate/ leaf_ssl_cert = @verify.peer_certs.last valid_certnames = [leaf_ssl_cert.name, *leaf_ssl_cert.subject_alt_names].uniq msg = valid_certnames.length > 1 ? "one of #{valid_certnames.join(', ')}" : valid_certnames.first - msg = "Server hostname '#{connection.address}' did not match server certificate; expected #{msg}" + msg = "Server hostname '#{site.host}' did not match server certificate; expected #{msg}" raise Puppet::Error, msg, error.backtrace else raise end end - - def with_connection(site, &block) - response = nil - @pool.with_connection(site, @verify) do |conn| - response = yield conn - end - response - end end end diff --git a/spec/integration/file_bucket/file_spec.rb b/spec/integration/file_bucket/file_spec.rb index 2c411fdf7..f0dbecaa3 100644 --- a/spec/integration/file_bucket/file_spec.rb +++ b/spec/integration/file_bucket/file_spec.rb @@ -1,44 +1,65 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet/file_bucket/file' describe Puppet::FileBucket::File do describe "#indirection" do before :each do # Never connect to the network, no matter what described_class.indirection.terminus(:rest).class.any_instance.stubs(:find) end describe "when running the master application" do before :each do Puppet::Application[:master].setup_terminuses end { "md5/d41d8cd98f00b204e9800998ecf8427e" => :file, "https://puppetmaster:8140/production/file_bucket_file/md5/d41d8cd98f00b204e9800998ecf8427e" => :file, }.each do |key, terminus| it "should use the #{terminus} terminus when requesting #{key.inspect}" do described_class.indirection.terminus(terminus).class.any_instance.expects(:find) described_class.indirection.find(key) end end end describe "when running another application" do { "md5/d41d8cd98f00b204e9800998ecf8427e" => :file, "https://puppetmaster:8140/production/file_bucket_file/md5/d41d8cd98f00b204e9800998ecf8427e" => :rest, }.each do |key, terminus| it "should use the #{terminus} terminus when requesting #{key.inspect}" do described_class.indirection.terminus(terminus).class.any_instance.expects(:find) described_class.indirection.find(key) end end end end + + describe "saving binary files" do + describe "on Ruby 1.8.7", :if => RUBY_VERSION.match(/^1\.8/) do + let(:binary) { "\xD1\xF2\r\n\x81NuSc\x00" } + + it "does not error when the same contents are saved twice" do + bucket_file = Puppet::FileBucket::File.new(binary) + Puppet::FileBucket::File.indirection.save(bucket_file, bucket_file.name) + Puppet::FileBucket::File.indirection.save(bucket_file, bucket_file.name) + end + end + describe "on Ruby 1.9+", :if => RUBY_VERSION.match(/^1\.9|^2/) do + let(:binary) { "\xD1\xF2\r\n\x81NuSc\x00".force_encoding(Encoding::ASCII_8BIT) } + + it "does not error when the same contents are saved twice" do + bucket_file = Puppet::FileBucket::File.new(binary) + Puppet::FileBucket::File.indirection.save(bucket_file, bucket_file.name) + Puppet::FileBucket::File.indirection.save(bucket_file, bucket_file.name) + end + end + end end diff --git a/spec/unit/network/http/connection_spec.rb b/spec/unit/network/http/connection_spec.rb index 034f6af01..ef6ca65d6 100755 --- a/spec/unit/network/http/connection_spec.rb +++ b/spec/unit/network/http/connection_spec.rb @@ -1,289 +1,306 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet/network/http/connection' require 'puppet/network/authentication' describe Puppet::Network::HTTP::Connection do let (:host) { "me" } let (:port) { 54321 } subject { Puppet::Network::HTTP::Connection.new(host, port, :verify => Puppet::SSL::Validator.no_validator) } let (:httpok) { Net::HTTPOK.new('1.1', 200, '') } context "when providing HTTP connections" do context "when initializing http instances" do it "should return an http instance created with the passed host and port" do conn = Puppet::Network::HTTP::Connection.new(host, port, :verify => Puppet::SSL::Validator.no_validator) expect(conn.address).to eq(host) expect(conn.port).to eq(port) end it "should enable ssl on the http instance by default" do conn = Puppet::Network::HTTP::Connection.new(host, port, :verify => Puppet::SSL::Validator.no_validator) expect(conn).to be_use_ssl end it "can disable ssl using an option" do conn = Puppet::Network::HTTP::Connection.new(host, port, :use_ssl => false, :verify => Puppet::SSL::Validator.no_validator) expect(conn).to_not be_use_ssl end it "can enable ssl using an option" do conn = Puppet::Network::HTTP::Connection.new(host, port, :use_ssl => true, :verify => Puppet::SSL::Validator.no_validator) expect(conn).to be_use_ssl end it "should raise Puppet::Error when invalid options are specified" do expect { Puppet::Network::HTTP::Connection.new(host, port, :invalid_option => nil) }.to raise_error(Puppet::Error, 'Unrecognized option(s): :invalid_option') end end end context "when methods that accept a block are called with a block" do let (:host) { "my_server" } let (:port) { 8140 } let (:subject) { Puppet::Network::HTTP::Connection.new(host, port, :use_ssl => false, :verify => Puppet::SSL::Validator.no_validator) } before :each do httpok.stubs(:body).returns "" # This stubbing relies a bit more on knowledge of the internals of Net::HTTP # than I would prefer, but it works on ruby 1.8.7 and 1.9.3, and it seems # valuable enough to have tests for blocks that this is probably warranted. socket = stub_everything("socket") TCPSocket.stubs(:open).returns(socket) Net::HTTP::Post.any_instance.stubs(:exec).returns("") Net::HTTP::Head.any_instance.stubs(:exec).returns("") Net::HTTP::Get.any_instance.stubs(:exec).returns("") Net::HTTPResponse.stubs(:read_new).returns(httpok) end [:request_get, :request_head, :request_post].each do |method| context "##{method}" do it "should yield to the block" do block_executed = false subject.send(method, "/foo", {}) do |response| block_executed = true end block_executed.should == true end end end end - context "when validating HTTPS requests" do + class ConstantErrorValidator + def initialize(args) + @fails_with = args[:fails_with] + @error_string = args[:error_string] || "" + @peer_certs = args[:peer_certs] || [] + end + + def setup_connection(connection) + connection.stubs(:start).raises(OpenSSL::SSL::SSLError.new(@fails_with)) + end + + def peer_certs + @peer_certs + end + + def verify_errors + [@error_string] + end + end + + class NoProblemsValidator + def initialize(cert) + @cert = cert + end + + def setup_connection(connection) + end + + def peer_certs + [@cert] + end + + def verify_errors + [] + end + end + + shared_examples_for 'ssl verifier' do include PuppetSpec::Files let (:host) { "my_server" } let (:port) { 8140 } it "should provide a useful error message when one is available and certificate validation fails", :unless => Puppet.features.microsoft_windows? do connection = Puppet::Network::HTTP::Connection.new( host, port, :verify => ConstantErrorValidator.new(:fails_with => 'certificate verify failed', :error_string => 'shady looking signature')) expect do connection.get('request') end.to raise_error(Puppet::Error, "certificate verify failed: [shady looking signature]") end it "should provide a helpful error message when hostname was not match with server certificate", :unless => Puppet.features.microsoft_windows? do Puppet[:confdir] = tmpdir('conf') connection = Puppet::Network::HTTP::Connection.new( - host, port, - :verify => ConstantErrorValidator.new( - :fails_with => 'hostname was not match with server certificate', - :peer_certs => [Puppet::SSL::CertificateAuthority.new.generate( - 'not_my_server', :dns_alt_names => 'foo,bar,baz')])) + host, port, + :verify => ConstantErrorValidator.new( + :fails_with => 'hostname was not match with server certificate', + :peer_certs => [Puppet::SSL::CertificateAuthority.new.generate( + 'not_my_server', :dns_alt_names => 'foo,bar,baz')])) expect do connection.get('request') end.to raise_error(Puppet::Error) do |error| error.message =~ /Server hostname 'my_server' did not match server certificate; expected one of (.+)/ $1.split(', ').should =~ %w[DNS:foo DNS:bar DNS:baz DNS:not_my_server not_my_server] end end it "should pass along the error message otherwise" do connection = Puppet::Network::HTTP::Connection.new( host, port, :verify => ConstantErrorValidator.new(:fails_with => 'some other message')) expect do connection.get('request') end.to raise_error(/some other message/) end it "should check all peer certificates for upcoming expiration", :unless => Puppet.features.microsoft_windows? do Puppet[:confdir] = tmpdir('conf') cert = Puppet::SSL::CertificateAuthority.new.generate( 'server', :dns_alt_names => 'foo,bar,baz') connection = Puppet::Network::HTTP::Connection.new( host, port, :verify => NoProblemsValidator.new(cert)) + Net::HTTP.any_instance.stubs(:start) Net::HTTP.any_instance.stubs(:request).returns(httpok) connection.expects(:warn_if_near_expiration).with(cert) connection.get('request') end + end - class ConstantErrorValidator - def initialize(args) - @fails_with = args[:fails_with] - @error_string = args[:error_string] || "" - @peer_certs = args[:peer_certs] || [] - end - - def setup_connection(connection) - connection.stubs(:request).with do - true - end.raises(OpenSSL::SSL::SSLError.new(@fails_with)) - end - - def peer_certs - @peer_certs - end - - def verify_errors - [@error_string] - end + context "when using single use HTTPS connections" do + it_behaves_like 'ssl verifier' do end + end - class NoProblemsValidator - def initialize(cert) - @cert = cert - end - - def setup_connection(connection) - end - - def peer_certs - [@cert] + context "when using persistent HTTPS connections" do + around :each do |example| + pool = Puppet::Network::HTTP::Pool.new + Puppet.override(:http_pool => pool) do + example.run end + pool.close + end - def verify_errors - [] - end + it_behaves_like 'ssl verifier' do end end context "when response is a redirect" do let (:site) { Puppet::Network::HTTP::Site.new('http', 'my_server', 8140) } let (:other_site) { Puppet::Network::HTTP::Site.new('http', 'redirected', 9292) } let (:other_path) { "other-path" } let (:verify) { Puppet::SSL::Validator.no_validator } let (:subject) { Puppet::Network::HTTP::Connection.new(site.host, site.port, :use_ssl => false, :verify => verify) } let (:httpredirection) do response = Net::HTTPFound.new('1.1', 302, 'Moved Temporarily') response['location'] = "#{other_site.addr}/#{other_path}" response.stubs(:read_body).returns("This resource has moved") response end def create_connection(site, options) options[:use_ssl] = site.use_ssl? Puppet::Network::HTTP::Connection.new(site.host, site.port, options) end it "should redirect to the final resource location" do http = stub('http') http.stubs(:request).returns(httpredirection).then.returns(httpok) seq = sequence('redirection') pool = Puppet.lookup(:http_pool) pool.expects(:with_connection).with(site, anything).yields(http).in_sequence(seq) pool.expects(:with_connection).with(other_site, anything).yields(http).in_sequence(seq) conn = create_connection(site, :verify => verify) conn.get('/foo') end def expects_redirection(conn, &block) http = stub('http') http.stubs(:request).returns(httpredirection) pool = Puppet.lookup(:http_pool) pool.expects(:with_connection).with(site, anything).yields(http) pool end def expects_limit_exceeded(conn) expect { conn.get('/') }.to raise_error(Puppet::Network::HTTP::RedirectionLimitExceededException) end it "should not redirect when the limit is 0" do conn = create_connection(site, :verify => verify, :redirect_limit => 0) pool = expects_redirection(conn) pool.expects(:with_connection).with(other_site, anything).never expects_limit_exceeded(conn) end it "should redirect only once" do conn = create_connection(site, :verify => verify, :redirect_limit => 1) pool = expects_redirection(conn) pool.expects(:with_connection).with(other_site, anything).once expects_limit_exceeded(conn) end it "should raise an exception when the redirect limit is exceeded" do conn = create_connection(site, :verify => verify, :redirect_limit => 3) pool = expects_redirection(conn) pool.expects(:with_connection).with(other_site, anything).times(3) expects_limit_exceeded(conn) end end it "allows setting basic auth on get requests" do expect_request_with_basic_auth subject.get('/path', nil, :basic_auth => { :user => 'user', :password => 'password' }) end it "allows setting basic auth on post requests" do expect_request_with_basic_auth subject.post('/path', 'data', nil, :basic_auth => { :user => 'user', :password => 'password' }) end it "allows setting basic auth on head requests" do expect_request_with_basic_auth subject.head('/path', nil, :basic_auth => { :user => 'user', :password => 'password' }) end it "allows setting basic auth on delete requests" do expect_request_with_basic_auth subject.delete('/path', nil, :basic_auth => { :user => 'user', :password => 'password' }) end it "allows setting basic auth on put requests" do expect_request_with_basic_auth subject.put('/path', 'data', nil, :basic_auth => { :user => 'user', :password => 'password' }) end def expect_request_with_basic_auth Net::HTTP.any_instance.expects(:request).with do |request| expect(request['authorization']).to match(/^Basic/) end.returns(httpok) end end