diff --git a/.travis.yml b/.travis.yml index 5f5634362..8193a1496 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,14 @@ language: ruby bundler_args: --without development -script: "bundle exec rake spec" +script: "bundle exec rake \"parallel:spec[1]\"" notifications: email: false rvm: - 2.1.0 - 2.0.0 - 1.9.3 - 1.8.7 - ruby-head matrix: allow_failures: - rvm: ruby-head -env: - - SPEC_OPTS="--format documentation" \ No newline at end of file diff --git a/spec/unit/face/module/build_spec.rb b/spec/unit/face/module/build_spec.rb index d372ceb30..02bbb7e9d 100644 --- a/spec/unit/face/module/build_spec.rb +++ b/spec/unit/face/module/build_spec.rb @@ -1,68 +1,69 @@ require 'spec_helper' require 'puppet/face' +require 'puppet/module_tool' describe "puppet module build" do subject { Puppet::Face[:module, :current] } describe "when called without any options" do it "if current directory is a module root should call builder with it" do Dir.expects(:pwd).returns('/a/b/c') Puppet::ModuleTool.expects(:find_module_root).with('/a/b/c').returns('/a/b/c') Puppet::ModuleTool.expects(:set_option_defaults).returns({}) Puppet::ModuleTool::Applications::Builder.expects(:run).with('/a/b/c', {}) subject.build end it "if parent directory of current dir is a module root should call builder with it" do Dir.expects(:pwd).returns('/a/b/c') Puppet::ModuleTool.expects(:find_module_root).with('/a/b/c').returns('/a/b') Puppet::ModuleTool.expects(:set_option_defaults).returns({}) Puppet::ModuleTool::Applications::Builder.expects(:run).with('/a/b', {}) subject.build end it "if current directory or parents contain no module root, should return exception" do Dir.expects(:pwd).returns('/a/b/c') Puppet::ModuleTool.expects(:find_module_root).returns(nil) expect { subject.build }.to raise_error RuntimeError, "Unable to find module root at /a/b/c or parent directories" end end describe "when called with a path" do it "if path is a module root should call builder with it" do Puppet::ModuleTool.expects(:is_module_root?).with('/a/b/c').returns(true) Puppet::ModuleTool.expects(:set_option_defaults).returns({}) Puppet::ModuleTool::Applications::Builder.expects(:run).with('/a/b/c', {}) subject.build('/a/b/c') end it "if path is not a module root should raise exception" do Puppet::ModuleTool.expects(:is_module_root?).with('/a/b/c').returns(false) expect { subject.build('/a/b/c') }.to raise_error RuntimeError, "Unable to find module root at /a/b/c" end end describe "with options" do it "should pass through options to builder when provided" do Puppet::ModuleTool.stubs(:is_module_root?).returns(true) Puppet::ModuleTool.expects(:set_option_defaults).returns({}) Puppet::ModuleTool::Applications::Builder.expects(:run).with('/a/b/c', {:modulepath => '/x/y/z'}) subject.build('/a/b/c', :modulepath => '/x/y/z') end end describe "inline documentation" do subject { Puppet::Face[:module, :current].get_action :build } its(:summary) { should =~ /build.*module/im } its(:description) { should =~ /build.*module/im } its(:returns) { should =~ /pathname/i } its(:examples) { should_not be_empty } %w{ license copyright summary description returns examples }.each do |doc| context "of the" do its(doc.to_sym) { should_not =~ /(FIXME|REVISIT|TODO)/ } end end end end diff --git a/tasks/parallel.rake b/tasks/parallel.rake new file mode 100644 index 000000000..dbdf4cf0a --- /dev/null +++ b/tasks/parallel.rake @@ -0,0 +1,404 @@ +# encoding: utf-8 + +require 'rubygems' +require 'thread' +require 'rspec' +require 'rspec/core/formatters/helpers' +require 'facter' + +module Parallel + module RSpec + # + # Responsible for buffering the output of RSpec's progress formatter. + # + class ProgressFormatBuffer + attr_reader :pending_lines + attr_reader :failure_lines + attr_reader :examples + attr_reader :failures + attr_reader :pending + attr_reader :failed_example_lines + attr_reader :state + + module OutputState + HEADER = 1 + PROGRESS = 2 + SUMMARY = 3 + PENDING = 4 + FAILURES = 5 + DURATION = 6 + COUNTS = 7 + FAILED_EXAMPLES = 8 + end + + def initialize(io, color) + @io = io + @color = color + @state = OutputState::HEADER + @pending_lines = [] + @failure_lines = [] + @examples = 0 + @failures = 0 + @pending = 0 + @failed_example_lines = [] + end + + def color? + @color + end + + def read + # Parse and ignore the one line header + if @state == OutputState::HEADER + begin + @io.readline + rescue EOFError + return nil + end + @state = OutputState::PROGRESS + return '' + end + + # If the progress has been read, parse the summary + if @state == OutputState::SUMMARY + parse_summary + return nil + end + + # Read the progress output up to 128 bytes at a time + # 128 is a small enough number to show some progress, but not too small that + # we're constantly writing synchronized output + data = @io.read(128) + return nil unless data + + data = @remainder + data if @remainder + + # Check for the end of the progress line + if (index = data.index "\n") + @state = OutputState::SUMMARY + @remainder = data[(index+1)..-1] + data = data[0...index] + # Check for partial ANSI escape codes in colorized output + elsif @color && !data.end_with?("\e[0m") && (index = data.rindex("\e[", -6)) + @remainder = data[index..-1] + data = data[0...index] + else + @remainder = nil + end + + data + end + + private + + def parse_summary + # If there is a remainder, concat it with the next line and handle each line + unless @remainder.empty? + lines = @remainder + eof = false + begin + lines += @io.readline + rescue EOFError + eof = true + end + lines.each_line do |line| + parse_summary_line line + end + return if eof + end + + # Process the rest of the lines + begin + @io.each_line do |line| + parse_summary_line line + end + rescue EOFError + end + end + + def parse_summary_line(line) + line.chomp! + return if line.empty? + + if line == 'Pending:' + @status = OutputState::PENDING + return + elsif line == 'Failures:' + @status = OutputState::FAILURES + return + elsif line == 'Failed examples:' + @status = OutputState::FAILED_EXAMPLES + return + elsif (line.match /^Finished in ((\d+\.?\d*) minutes?)? ?(\d+\.?\d*) seconds?$/) + @status = OutputState::DURATION + return + elsif (match = line.gsub(/\e\[\d+m/, '').match /^(\d+) examples?, (\d+) failures?(, (\d+) pending)?$/) + @status = OutputState::COUNTS + @examples = match[1].to_i + @failures = match[2].to_i + @pending = (match[4] || 0).to_i + return + end + + case @status + when OutputState::PENDING + @pending_lines << line + when OutputState::FAILURES + @failure_lines << line + when OutputState::FAILED_EXAMPLES + @failed_example_lines << line + end + end + end + + # + # Responsible for parallelizing spec testing. + # + class Parallelizer + include ::RSpec::Core::Formatters::Helpers + + # Number of processes to use + attr_reader :process_count + # Approximate size of each group of tests + attr_reader :group_size + + def initialize(process_count, group_size, color) + @process_count = process_count + @group_size = group_size + @color = color + end + + def color? + @color + end + + def run + @start_time = Time.now + + groups = group_specs + fail red('error: no specs were found') if groups.length == 0 + + begin + run_specs groups + ensure + groups.each do |file| + File.unlink(file) + end + end + end + + private + + def group_specs + # Spawn the rspec_grouper utility to perform the test grouping + # We do this in a separate process to limit this processes' long-running footprint + io = IO.popen("ruby util/rspec_grouper #{@group_size}") + + header = true + spec_group_files = [] + io.each_line do |line| + line.chomp! + header = false if line.empty? + next if header || line.empty? + spec_group_files << line + end + + _, status = Process.waitpid2(io.pid) + io.close + + fail red('error: no specs were found.') unless status.success? + spec_group_files + end + + def run_specs(groups) + puts "Processing #{groups.length} spec group(s) with #{@process_count} worker(s)" + + interrupted = false + success = true + worker_threads = [] + group_index = -1 + pids = Array.new(@process_count) + mutex = Mutex.new + + # Handle SIGINT by killing child processes + original_handler = Signal.trap :SIGINT do + break if interrupted + interrupted = true + + # Can't synchronize in a trap context, so read dirty + pids.each do |pid| + begin + Process.kill(:SIGKILL, pid) if pid + rescue Errno::ESRCH + end + end + puts yellow("\nshutting down...") + end + + buffers = [] + + process_count.times do |thread_id| + worker_threads << Thread.new do + while !interrupted do + # Get the spec file for this rspec run + group = mutex.synchronize { if group_index < groups.length then groups[group_index += 1] else nil end } + break unless group && !interrupted + + # Spawn the worker process with redirected output + io = IO.popen("ruby util/rspec_runner #{group}") + pids[thread_id] = io.pid + + # TODO: make the buffer pluggable to handle other output formats like documentation + buffer = ProgressFormatBuffer.new(io, @color) + + # Process the output + while !interrupted + output = buffer.read + break unless output && !interrupted + next if output.empty? + mutex.synchronize { print output } + end + + # Kill the process if we were interrupted, just to be sure + if interrupted + begin + Process.kill(:SIGKILL, pids[thread_id]) + rescue Errno::ESRCH + end + end + + # Reap the process + result = Process.waitpid2(pids[thread_id])[1].success? + io.close + pids[thread_id] = nil + mutex.synchronize do + buffers << buffer + success &= result + end + end + end + end + + # Join all worker threads + worker_threads.each do |thread| + thread.join + end + + Signal.trap :SIGINT, original_handler + fail yellow('execution was interrupted') if interrupted + + dump_summary buffers + success + end + + def colorize(text, color_code) + if @color + "#{color_code}#{text}\e[0m" + else + text + end + end + + def red(text) + colorize(text, "\e[31m") + end + + def green(text) + colorize(text, "\e[32m") + end + + def yellow(text) + colorize(text, "\e[33m") + end + + def dump_summary(buffers) + puts + + # Print out the pending tests + print_header = true + buffers.each do |buffer| + next if buffer.pending_lines.empty? + if print_header + puts "\nPending:" + print_header = false + end + puts buffer.pending_lines + end + + # Print out the failures + print_header = true + buffers.each do |buffer| + next if buffer.failure_lines.empty? + if print_header + puts "\nFailures:" + print_header = false + end + puts + puts buffer.failure_lines + end + + # Print out the run time + puts "\nFinished in #{format_duration(Time.now - @start_time)}" + + # Count all of the examples + examples = 0 + failures = 0 + pending = 0 + buffers.each do |buffer| + examples += buffer.examples + failures += buffer.failures + pending += buffer.pending + end + if failures > 0 + puts red(summary_count_line(examples, failures, pending)) + elsif pending > 0 + puts yellow(summary_count_line(examples, failures, pending)) + else + puts green(summary_count_line(examples, failures, pending)) + end + + # Print out the failed examples + print_header = true + buffers.each do |buffer| + next if buffer.failed_example_lines.empty? + if print_header + puts "\nFailed examples:" + print_header = false + end + puts buffer.failed_example_lines + end + end + + def summary_count_line(examples, failures, pending) + summary = pluralize(examples, "example") + summary << ", " << pluralize(failures, "failure") + summary << ", #{pending} pending" if pending > 0 + summary + end + end + end +end + +namespace 'parallel' do + def color_output? + # Check with RSpec to see if color is enabled + config = ::RSpec::Core::Configuration.new + config.error_stream = $stderr + config.output_stream = $stdout + options = ::RSpec::Core::ConfigurationOptions.new [] + options.parse_options + options.configure config + config.color + end + + desc 'Runs specs in parallel.' + task 'spec', :process_count, :group_size do |_, args| + # Default group size in rspec examples + DEFAULT_GROUP_SIZE = 1000 + + process_count = [(args[:process_count] || Facter.processorcount).to_i, 1].max + group_size = [(args[:group_size] || DEFAULT_GROUP_SIZE).to_i, 1].max + + abort unless Parallel::RSpec::Parallelizer.new(process_count, group_size, color_output?).run + end +end \ No newline at end of file diff --git a/util/rspec_grouper b/util/rspec_grouper new file mode 100755 index 000000000..22260d461 --- /dev/null +++ b/util/rspec_grouper @@ -0,0 +1,117 @@ +#!/usr/bin/env ruby +require 'rubygems' +require 'rspec' + +# Disable ruby verbosity +# We need control over output so that the parallel task can parse it correctly +$VERBOSE = nil + +# Monkey patch Proc.source_location for 1.8.7 +unless Proc.method_defined? :source_location + class Proc + def source_location + match = to_s.match(/^#$/) + return unless match + [match[1], match[2]] + end + end +end + +module Parallel + module RSpec + # + # Responsible for grouping rspec examples into groups of a given size. + # + class Grouper + attr_reader :groups + attr_reader :files + attr_reader :total_examples + + def initialize(group_size) + config = ::RSpec::Core::Configuration.new + options = ::RSpec::Core::ConfigurationOptions.new((ENV['TEST'] || ENV['TESTS'] || 'spec').split(';')) + options.parse_options + options.configure config + + # This will scan and load all spec examples + config.load_spec_files + + @total_examples = 0 + + # Populate a map of spec file => example count, sorted ascending by count + # NOTE: this uses a private API of RSpec and is may break if the gem is updated + @files = ::RSpec::Core::ExampleGroup.children.inject({}) do |files, group| + file = group.metadata[:example_group_block].source_location[0] + count = count_examples(group) + files[file] = (files[file] || 0) + count + @total_examples += count + files + end.sort_by { |_, v| v} + + # Group the spec files + @groups = [] + group = nil + example_count = 0 + @files.each do |file, count| + group = [] unless group + group << file + if (example_count += count) > group_size + example_count = 0 + @groups << group + group = nil + end + end + @groups << group if group + end + + private + def count_examples(group) + return 0 unless group + # Each group can have examples as well as child groups, so recursively traverse + group.children.inject(group.examples.count) { |count, g| count + count_examples(g) } + end + end + end +end + +def print_usage + puts 'usage: rspec_grouper ' +end + +if __FILE__ == $0 + if ARGV.length != 1 + print_usage + else + group_size = ARGV[0].to_i + abort 'error: group count must be greater than zero.' if group_size < 1 + grouper = Parallel::RSpec::Grouper.new(group_size) + abort 'error: no rspec examples were found.' if grouper.total_examples == 0 + groups = grouper.groups + puts "Grouped #{grouper.total_examples} rspec example(s) into #{groups.length} group(s) from #{grouper.files.count} file(s)." + puts + + paths = [] + + begin + # Create a temp directory and write out group files + tmpdir = Dir.mktmpdir + groups.each_with_index do |group, index| + path = File.join(tmpdir, "group#{index+1}") + file = File.new(path, 'w') + paths << path + file.puts group + file.close + puts path + end + rescue Exception + # Delete all files on an exception + paths.each do |path| + begin + File.delete path + rescue Exception + end + end + raise + end + end +end \ No newline at end of file diff --git a/util/rspec_runner b/util/rspec_runner new file mode 100755 index 000000000..e4452a01c --- /dev/null +++ b/util/rspec_runner @@ -0,0 +1,53 @@ +#!/usr/bin/env ruby +require 'rubygems' +require 'rspec' +require 'rspec/core/formatters/progress_formatter' +require 'rspec/core/command_line' + +# Disable ruby verbosity +# We need control over output so that the parallel task can parse it correctly +$VERBOSE = nil + +module Parallel + module RSpec + # + # Responsible for formatting output. + # This differs from the built-in progress formatter by not appending an index to failures. + # + class Formatter < ::RSpec::Core::Formatters::ProgressFormatter + def dump_failure(example, _) + # Unlike the super class implementation, do not print the failure number + output.puts "#{short_padding}#{example.full_description}" + dump_failure_info(example) + end + end + # + # Responsible for running spec files given a spec file. + # We do it this way so that we can run very long spec file lists on Windows, since + # Windows has a limited argument length depending on method of invocation. + # + class Runner + def initialize(specs_file) + abort "error: spec list file '#{specs_file}' does not exist." unless File.exists? specs_file + @options = ['-fParallel::RSpec::Formatter'] + File.readlines(specs_file).each { |line| @options << line.chomp } + end + + def run + ::RSpec::Core::CommandLine.new(@options).run($stderr, $stdout) + end + end + end +end + +def print_usage + puts 'usage: rspec_runner ' +end + +if __FILE__ == $0 + if ARGV.length != 1 + print_usage + else + exit Parallel::RSpec::Runner.new(ARGV[0]).run + end +end \ No newline at end of file