diff --git a/lib/puppet/indirector/facts/inventory_active_record.rb b/lib/puppet/indirector/facts/inventory_active_record.rb index db4c63f00..93eee8a97 100644 --- a/lib/puppet/indirector/facts/inventory_active_record.rb +++ b/lib/puppet/indirector/facts/inventory_active_record.rb @@ -1,97 +1,100 @@ require 'puppet/rails' require 'puppet/rails/inventory_node' require 'puppet/rails/inventory_fact' require 'puppet/indirector/active_record' +require 'puppet/util/retryaction' class Puppet::Node::Facts::InventoryActiveRecord < Puppet::Indirector::ActiveRecord def find(request) node = Puppet::Rails::InventoryNode.find_by_name(request.key) return nil unless node facts = Puppet::Node::Facts.new(node.name, node.facts_to_hash) facts.timestamp = node.timestamp facts end def save(request) - facts = request.instance - node = Puppet::Rails::InventoryNode.find_by_name(request.key) || Puppet::Rails::InventoryNode.create(:name => request.key, :timestamp => facts.timestamp) - node.timestamp = facts.timestamp + Puppet::Util::RetryAction.retry_action :retries => 4, :retry_exceptions => {ActiveRecord::StatementInvalid => 'MySQL Error. Retrying'} do + facts = request.instance + node = Puppet::Rails::InventoryNode.find_by_name(request.key) || Puppet::Rails::InventoryNode.create(:name => request.key, :timestamp => facts.timestamp) + node.timestamp = facts.timestamp - ActiveRecord::Base.transaction do - Puppet::Rails::InventoryFact.delete_all(:node_id => node.id) - # We don't want to save internal values as facts, because those are - # metadata that belong on the node - facts.values.each do |name,value| - next if name.to_s =~ /^_/ - node.facts.build(:name => name, :value => value) + ActiveRecord::Base.transaction do + Puppet::Rails::InventoryFact.delete_all(:node_id => node.id) + # We don't want to save internal values as facts, because those are + # metadata that belong on the node + facts.values.each do |name,value| + next if name.to_s =~ /^_/ + node.facts.build(:name => name, :value => value) + end + node.save end - node.save end end def search(request) return [] unless request.options matching_nodes = [] fact_names = [] fact_filters = Hash.new {|h,k| h[k] = []} meta_filters = Hash.new {|h,k| h[k] = []} request.options.each do |key,value| type, name, operator = key.to_s.split(".") operator ||= "eq" if type == "facts" fact_filters[operator] << [name,value] elsif type == "meta" and name == "timestamp" meta_filters[operator] << [name,value] end end matching_nodes = nodes_matching_fact_filters(fact_filters) + nodes_matching_meta_filters(meta_filters) # to_a because [].inject == nil matching_nodes.inject {|nodes,this_set| nodes & this_set}.to_a.sort end private def nodes_matching_fact_filters(fact_filters) node_sets = [] fact_filters['eq'].each do |name,value| node_sets << Puppet::Rails::InventoryNode.has_fact_with_value(name,value).map {|node| node.name} end fact_filters['ne'].each do |name,value| node_sets << Puppet::Rails::InventoryNode.has_fact_without_value(name,value).map {|node| node.name} end { 'gt' => '>', 'lt' => '<', 'ge' => '>=', 'le' => '<=' }.each do |operator_name,operator| fact_filters[operator_name].each do |name,value| facts = Puppet::Rails::InventoryFact.find_by_sql(["SELECT inventory_facts.value, inventory_nodes.name AS node_name FROM inventory_facts INNER JOIN inventory_nodes ON inventory_facts.node_id = inventory_nodes.id WHERE inventory_facts.name = ?", name]) node_sets << facts.select {|fact| fact.value.to_f.send(operator, value.to_f)}.map {|fact| fact.node_name} end end node_sets end def nodes_matching_meta_filters(meta_filters) node_sets = [] { 'eq' => '=', 'ne' => '!=', 'gt' => '>', 'lt' => '<', 'ge' => '>=', 'le' => '<=' }.each do |operator_name,operator| meta_filters[operator_name].each do |name,value| node_sets << Puppet::Rails::InventoryNode.find(:all, :select => "name", :conditions => ["timestamp #{operator} ?", value]).map {|node| node.name} end end node_sets end end diff --git a/lib/puppet/util/retryaction.rb b/lib/puppet/util/retryaction.rb new file mode 100644 index 000000000..ba318ec1a --- /dev/null +++ b/lib/puppet/util/retryaction.rb @@ -0,0 +1,48 @@ +module Puppet::Util::RetryAction + class RetryException < Exception; end + class RetryException::NoBlockGiven < RetryException; end + class RetryException::NoRetriesGiven < RetryException;end + class RetryException::RetriesExceeded < RetryException; end + + def self.retry_action( parameters = { :retry_exceptions => nil, :retries => nil } ) + # Retry actions for a specified amount of time. This method will allow the final + # retry to complete even if that extends beyond the timeout period. + unless block_given? + raise RetryException::NoBlockGiven + end + + raise RetryException::NoRetriesGiven if parameters[:retries].nil? + parameters[:retry_exceptions] ||= Hash.new + + start = Time.now + failures = 0 + + begin + yield + rescue Exception => e + # If we were giving exceptions to catch, + # catch the excptions we care about and retry. + # All others fail hard + + raise RetryException::RetriesExceeded if parameters[:retries] == 0 + + if (not parameters[:retry_exceptions].keys.empty?) and parameters[:retry_exceptions].keys.include?(e.class) + Puppet.info("Caught exception #{e.class}:#{e}") + Puppet.info(parameters[:retry_exceptions][e.class]) + elsif (not parameters[:retry_exceptions].keys.empty?) + # If the exceptions is not in the list of retry_exceptions re-raise. + raise e + end + + failures += 1 + parameters[:retries] -= 1 + + # Increase the amount of time that we sleep after every + # failed retry attempt. + sleep (((2 ** failures) -1) * 0.1) + + retry + + end + end +end diff --git a/spec/unit/indirector/facts/inventory_active_record_spec.rb b/spec/unit/indirector/facts/inventory_active_record_spec.rb index 88e5e5359..43dcc7c6b 100755 --- a/spec/unit/indirector/facts/inventory_active_record_spec.rb +++ b/spec/unit/indirector/facts/inventory_active_record_spec.rb @@ -1,170 +1,180 @@ #!/usr/bin/env rspec require 'spec_helper' begin require 'sqlite3' rescue LoadError end require 'tempfile' require 'puppet/rails' describe "Puppet::Node::Facts::InventoryActiveRecord", :if => (Puppet.features.rails? and defined? SQLite3) do let(:terminus) { Puppet::Node::Facts::InventoryActiveRecord.new } before :all do require 'puppet/indirector/facts/inventory_active_record' @dbfile = Tempfile.new("testdb") @dbfile.close end after :all do Puppet::Node::Facts.indirection.reset_terminus_class @dbfile.unlink end before :each do Puppet::Node.indirection.reset_terminus_class Puppet::Node.indirection.cache_class = nil Puppet::Node::Facts.indirection.terminus_class = :inventory_active_record Puppet[:dbadapter] = 'sqlite3' Puppet[:dblocation] = @dbfile.path Puppet[:railslog] = "/dev/null" Puppet::Rails.init end after :each do Puppet::Rails.teardown ActiveRecord::Base.remove_connection end describe "#save" do + let(:node) { + Puppet::Rails::InventoryNode.new(:name => "foo", :timestamp => Time.now) + } + + let(:facts) { + Puppet::Node::Facts.new("foo", "uptime_days" => "60", "kernel" => "Darwin") + } + + it "should retry on ActiveRecord error" do + Puppet::Rails::InventoryNode.expects(:create).twice.raises(ActiveRecord::StatementInvalid).returns node + + Puppet::Node::Facts.indirection.save(facts) + end + it "should use an existing node if possible" do - node = Puppet::Rails::InventoryNode.new(:name => "foo", :timestamp => Time.now) node.save - facts = Puppet::Node::Facts.new("foo", "uptime_days" => "60", "kernel" => "Darwin") Puppet::Node::Facts.indirection.save(facts) Puppet::Rails::InventoryNode.count.should == 1 Puppet::Rails::InventoryNode.first.should == node end it "should create a new node if one can't be found" do # This test isn't valid if there are nodes to begin with Puppet::Rails::InventoryNode.count.should == 0 - facts = Puppet::Node::Facts.new("foo", "uptime_days" => "60", "kernel" => "Darwin") Puppet::Node::Facts.indirection.save(facts) Puppet::Rails::InventoryNode.count.should == 1 Puppet::Rails::InventoryNode.first.name.should == "foo" end it "should save the facts" do - facts = Puppet::Node::Facts.new("foo", "uptime_days" => "60", "kernel" => "Darwin") Puppet::Node::Facts.indirection.save(facts) Puppet::Rails::InventoryFact.all.map{|f| [f.name,f.value]}.should =~ [["uptime_days","60"],["kernel","Darwin"]] end it "should remove the previous facts for an existing node" do facts = Puppet::Node::Facts.new("foo", "uptime_days" => "30", "kernel" => "Darwin") Puppet::Node::Facts.indirection.save(facts) bar_facts = Puppet::Node::Facts.new("bar", "uptime_days" => "35", "kernel" => "Linux") foo_facts = Puppet::Node::Facts.new("foo", "uptime_days" => "60", "is_virtual" => "false") Puppet::Node::Facts.indirection.save(bar_facts) Puppet::Node::Facts.indirection.save(foo_facts) Puppet::Node::Facts.indirection.find("bar").should == bar_facts Puppet::Node::Facts.indirection.find("foo").should == foo_facts Puppet::Rails::InventoryFact.all.map{|f| [f.name,f.value]}.should_not include(["uptime_days", "30"], ["kernel", "Darwin"]) end end describe "#find" do before do @foo_facts = Puppet::Node::Facts.new("foo", "uptime_days" => "60", "kernel" => "Darwin") @bar_facts = Puppet::Node::Facts.new("bar", "uptime_days" => "30", "kernel" => "Linux") Puppet::Node::Facts.indirection.save(@foo_facts) Puppet::Node::Facts.indirection.save(@bar_facts) end it "should identify facts by node name" do Puppet::Node::Facts.indirection.find("foo").should == @foo_facts end it "should return nil if no node instance can be found" do Puppet::Node::Facts.indirection.find("non-existent node").should == nil end end describe "#search" do def search_request(conditions) Puppet::Indirector::Request.new(:facts, :search, nil, conditions) end before :each do @now = Time.now @foo = Puppet::Node::Facts.new("foo", "fact1" => "value1", "fact2" => "value2", "uptime_days" => "30") @bar = Puppet::Node::Facts.new("bar", "fact1" => "value1", "uptime_days" => "60") @baz = Puppet::Node::Facts.new("baz", "fact1" => "value2", "fact2" => "value1", "uptime_days" => "90") @bat = Puppet::Node::Facts.new("bat") @foo.timestamp = @now - 3600*1 @bar.timestamp = @now - 3600*3 @baz.timestamp = @now - 3600*5 @bat.timestamp = @now - 3600*7 [@foo, @bar, @baz, @bat].each {|facts| Puppet::Node::Facts.indirection.save(facts)} end it "should return node names that match 'equal' constraints" do request = search_request('facts.fact1.eq' => 'value1', 'facts.fact2.eq' => 'value2') terminus.search(request).should == ["foo"] end it "should return node names that match 'not equal' constraints" do request = search_request('facts.fact1.ne' => 'value2') terminus.search(request).should == ["bar","foo"] end it "should return node names that match strict inequality constraints" do request = search_request('facts.uptime_days.gt' => '20', 'facts.uptime_days.lt' => '70') terminus.search(request).should == ["bar","foo"] end it "should return node names that match non-strict inequality constraints" do request = search_request('facts.uptime_days.ge' => '30', 'facts.uptime_days.le' => '60') terminus.search(request).should == ["bar","foo"] end it "should return node names whose facts are within a given timeframe" do request = search_request('meta.timestamp.ge' => @now - 3600*5, 'meta.timestamp.le' => @now - 3600*1) terminus.search(request).should == ["bar","baz","foo"] end it "should return node names whose facts are from a specific time" do request = search_request('meta.timestamp.eq' => @now - 3600*3) terminus.search(request).should == ["bar"] end it "should return node names whose facts are not from a specific time" do request = search_request('meta.timestamp.ne' => @now - 3600*1) terminus.search(request).should == ["bar","bat","baz"] end it "should perform strict searches on nodes by timestamp" do request = search_request('meta.timestamp.gt' => @now - 3600*5, 'meta.timestamp.lt' => @now - 3600*1) terminus.search(request).should == ["bar"] end it "should search nodes based on both facts and timestamp values" do request = search_request('facts.uptime_days.gt' => '45', 'meta.timestamp.lt' => @now - 3600*4) terminus.search(request).should == ["baz"] end end end diff --git a/spec/unit/util/retryaction_spec.rb b/spec/unit/util/retryaction_spec.rb new file mode 100644 index 000000000..a003faa53 --- /dev/null +++ b/spec/unit/util/retryaction_spec.rb @@ -0,0 +1,62 @@ +#!/usr/bin/env rspec +require 'spec_helper' + +require 'puppet/util/retryaction' + +describe Puppet::Util::RetryAction do + let (:exceptions) {{ Puppet::Error => 'Puppet Error Exception' }} + + it 'should retry on any exception if no acceptable exceptions given' do + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 1) -1) * 0.1) ) + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 2) -1) * 0.1) ) + + expect do + Puppet::Util::RetryAction.retry_action( :retries => 2 ) do + raise ArgumentError, 'Fake Failure' + end + end.to raise_exception(Puppet::Util::RetryAction::RetryException::RetriesExceeded) + end + + it 'should retry on acceptable exceptions' do + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 1) -1) * 0.1) ) + Puppet::Util::RetryAction.expects(:sleep).with( (((2 ** 2) -1) * 0.1) ) + + expect do + Puppet::Util::RetryAction.retry_action( :retries => 2, :retry_exceptions => exceptions) do + raise Puppet::Error, 'Fake Failure' + end + end.to raise_exception(Puppet::Util::RetryAction::RetryException::RetriesExceeded) + end + + it 'should not retry on unacceptable exceptions' do + Puppet::Util::RetryAction.expects(:sleep).never + + expect do + Puppet::Util::RetryAction.retry_action( :retries => 2, :retry_exceptions => exceptions) do + raise ArgumentError + end + end.to raise_exception(ArgumentError) + end + + it 'should succeed if nothing is raised' do + Puppet::Util::RetryAction.expects(:sleep).never + + Puppet::Util::RetryAction.retry_action( :retries => 2) do + true + end + end + + it 'should succeed if an expected exception is raised retried and succeeds' do + should_retry = nil + Puppet::Util::RetryAction.expects(:sleep).once + + Puppet::Util::RetryAction.retry_action( :retries => 2, :retry_exceptions => exceptions) do + if should_retry + true + else + should_retry = true + raise Puppet::Error, 'Fake error' + end + end + end +end