diff --git a/lib/puppet/util/retry_action.rb b/lib/puppet/util/retry_action.rb new file mode 100644 index 000000000..fd54ade65 --- /dev/null +++ b/lib/puppet/util/retry_action.rb @@ -0,0 +1,46 @@ +module Puppet::Util::RetryAction + class RetryException < Exception; end + class RetryException::NoBlockGiven < RetryException; end + class RetryException::NoRetriesGiven < RetryException;end + class RetryException::RetriesExceeded < RetryException; end + + # Execute the supplied block retrying with exponential backoff. + # + # @param [Hash] options the retry options + # @option options [FixNum] :retries Maximum number of times to retry. + # @option options [Array] :retry_exceptions ([StandardError]) Optional array of exceptions that are allowed to be retried. + # @yield The block to be executed. + def self.retry_action(options = {}) + # Retry actions for a specified amount of time. This method will allow the final + # retry to complete even if that extends beyond the timeout period. + if !block_given? + raise RetryException::NoBlockGiven + end + + retries = options[:retries] + if retries.nil? + raise RetryException::NoRetriesGiven + end + + retry_exceptions = options[:retry_exceptions] || [StandardError] + failures = 0 + begin + yield + rescue *retry_exceptions => e + if failures >= retries + raise RetryException::RetriesExceeded, "#{retries} exceeded", e.backtrace + end + + Puppet.info("Caught exception #{e.class}:#{e} retrying") + + failures += 1 + + # Increase the amount of time that we sleep after every + # failed retry attempt. + sleep (((2 ** failures) -1) * 0.1) + + retry + + end + end +end diff --git a/spec/unit/util/retry_action_spec.rb b/spec/unit/util/retry_action_spec.rb new file mode 100755 index 000000000..287bf55e6 --- /dev/null +++ b/spec/unit/util/retry_action_spec.rb @@ -0,0 +1,85 @@ +#! /usr/bin/env ruby +require 'spec_helper' + +require 'puppet/util/retry_action' + +describe Puppet::Util::RetryAction do + let (:exceptions) { [ Puppet::Error, NameError ] } + + it "doesn't retry SystemExit" do + expect do + Puppet::Util::RetryAction.retry_action( :retries => 0 ) do + raise SystemExit + end + end.to exit_with(0) + end + + it "doesn't retry NoMemoryError" do + expect do + Puppet::Util::RetryAction.retry_action( :retries => 0 ) do + raise NoMemoryError, "OOM" + end + end.to raise_error(NoMemoryError, /OOM/) + end + + it 'should retry on any exception if no acceptable exceptions given' do + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 1) -1) * 0.1) ) + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 2) -1) * 0.1) ) + + expect do + Puppet::Util::RetryAction.retry_action( :retries => 2 ) do + raise ArgumentError, 'Fake Failure' + end + end.to raise_exception(Puppet::Util::RetryAction::RetryException::RetriesExceeded) + end + + it 'should retry on acceptable exceptions' do + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 1) -1) * 0.1) ) + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 2) -1) * 0.1) ) + + expect do + Puppet::Util::RetryAction.retry_action( :retries => 2, :retry_exceptions => exceptions) do + raise Puppet::Error, 'Fake Failure' + end + end.to raise_exception(Puppet::Util::RetryAction::RetryException::RetriesExceeded) + end + + it 'should not retry on unacceptable exceptions' do + Puppet::Util::RetryAction.expects(:sleep).never + + expect do + Puppet::Util::RetryAction.retry_action( :retries => 2, :retry_exceptions => exceptions) do + raise ArgumentError + end + end.to raise_exception(ArgumentError) + end + + it 'should succeed if nothing is raised' do + Puppet::Util::RetryAction.expects(:sleep).never + + Puppet::Util::RetryAction.retry_action( :retries => 2) do + true + end + end + + it 'should succeed if an expected exception is raised retried and succeeds' do + should_retry = nil + Puppet::Util::RetryAction.expects(:sleep).once + + Puppet::Util::RetryAction.retry_action( :retries => 2, :retry_exceptions => exceptions) do + if should_retry + true + else + should_retry = true + raise Puppet::Error, 'Fake error' + end + end + end + + it "doesn't mutate caller's arguments" do + options = { :retries => 1 }.freeze + + Puppet::Util::RetryAction.retry_action(options) do + end + end +end