diff --git a/lib/puppet/parser/collector.rb b/lib/puppet/parser/collector.rb index 80f7beab4..df001a232 100644 --- a/lib/puppet/parser/collector.rb +++ b/lib/puppet/parser/collector.rb @@ -1,176 +1,176 @@ # An object that collects stored objects from the central cache and returns # them to the current host, yo. class Puppet::Parser::Collector attr_accessor :type, :scope, :vquery, :equery, :form, :resources, :overrides, :collected # Call the collection method, mark all of the returned objects as # non-virtual, optionally applying parameter overrides. The collector can # also delete himself from the compiler if there is no more resources to # collect (valid only for resource fixed-set collector which get their # resources from +collect_resources+ and not from the catalog) def evaluate # Shortcut if we're not using storeconfigs and they're trying to collect # exported resources. if form == :exported and Puppet[:storeconfigs] != true Puppet.warning "Not collecting exported resources without storeconfigs" return false end if self.resources unless objects = collect_resources and ! objects.empty? return false end else method = "collect_#{@form.to_s}" objects = send(method).each do |obj| obj.virtual = false end return false if objects.empty? end # we have an override for the collected resources if @overrides and !objects.empty? # force the resource to be always child of any other resource overrides[:source].meta_def(:child_of?) do true end # tell the compiler we have some override for him unless we already # overrided those resources objects.each do |res| unless @collected.include?(res.ref) newres = Puppet::Parser::Resource. new(res.type, res.title, :parameters => overrides[:parameters], :file => overrides[:file], :line => overrides[:line], :source => overrides[:source], :scope => overrides[:scope]) scope.compiler.add_override(newres) end end end # filter out object that this collector has previously found. objects.reject! { |o| @collected.include?(o.ref) } return false if objects.empty? # keep an eye on the resources we have collected objects.inject(@collected) { |c,o| c[o.ref]=o; c } # return our newly collected resources objects end def initialize(scope, type, equery, vquery, form) @scope = scope @vquery = vquery @equery = equery # initialisation @collected = {} # Canonize the type @type = Puppet::Resource.new(type, "whatever").type unless [:exported, :virtual].include?(form) raise ArgumentError, "Invalid query form #{form}" end @form = form end # add a resource override to the soon to be exported/realized resources def add_override(hash) raise ArgumentError, "Exported resource try to override without parameters" unless hash[:parameters] # schedule an override for an upcoming collection @overrides = hash end private # Collect exported objects. def collect_exported resources = [] time = Puppet::Util.thinmark do # First get everything from the export table. Just reuse our # collect_virtual method but tell it to use 'exported? for the test. resources = collect_virtual(true).reject { |r| ! r.virtual? } # key is '#{type}/#{name}', and host and filter. found = Puppet::Resource.indirection. search(@type, :host => @scope.host, :filter => @equery, :scope => @scope) found_resources = found.map {|x| x.is_a?(Puppet::Parser::Resource) ? x : x.to_resource(@scope)} found_resources.each do |item| if existing = @scope.findresource(item.type, item.title) unless existing.collector_id == item.collector_id # unless this is the one we've already collected raise Puppet::ParseError, - "Exported resource #{item.ref} cannot override local resource" + "Another local or imported resource exists with the type and title #{item.ref}" end else item.exported = false @scope.compiler.add_resource(@scope, item) resources << item end end end scope.debug("Collected %s %s resource%s in %.2f seconds" % [resources.length, @type, resources.length == 1 ? "" : "s", time]) resources end def collect_resources @resources = [@resources] unless @resources.is_a?(Array) method = "collect_#{form.to_s}_resources" send(method) end def collect_exported_resources raise Puppet::ParseError, "realize() is not yet implemented for exported resources" end # Collect resources directly; this is the result of using 'realize', # which specifies resources, rather than using a normal collection. def collect_virtual_resources return [] unless defined?(@resources) and ! @resources.empty? result = @resources.dup.collect do |ref| if res = @scope.findresource(ref.to_s) @resources.delete(ref) res end end.reject { |r| r.nil? }.each do |res| res.virtual = false end # If there are no more resources to find, delete this from the list # of collections. @scope.compiler.delete_collection(self) if @resources.empty? result end # Collect just virtual objects, from our local compiler. def collect_virtual(exported = false) scope.compiler.resources.find_all do |resource| resource.type == @type and (exported ? resource.exported? : true) and match?(resource) end end # Does the resource match our tests? We don't yet support tests, # so it's always true at the moment. def match?(resource) if self.vquery return self.vquery.call(resource) else return true end end end diff --git a/spec/unit/parser/collector_spec.rb b/spec/unit/parser/collector_spec.rb index 633254bfb..79a6a17b9 100755 --- a/spec/unit/parser/collector_spec.rb +++ b/spec/unit/parser/collector_spec.rb @@ -1,447 +1,447 @@ #!/usr/bin/env rspec require 'spec_helper' begin require 'sqlite3' rescue LoadError end require 'puppet/rails' require 'puppet/parser/collector' describe Puppet::Parser::Collector, "when initializing" do before do @scope = mock 'scope' @resource_type = 'resource_type' @form = :exported @vquery = mock 'vquery' @equery = mock 'equery' @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @equery, @vquery, @form) end it "should require a scope" do @collector.scope.should equal(@scope) end it "should require a resource type" do @collector.type.should == 'Resource_type' end it "should only accept :virtual or :exported as the collector form" do proc { @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @vquery, @equery, :other) }.should raise_error(ArgumentError) end it "should accept an optional virtual query" do @collector.vquery.should equal(@vquery) end it "should accept an optional exported query" do @collector.equery.should equal(@equery) end it "should canonize the type name" do @collector = Puppet::Parser::Collector.new(@scope, "resource::type", @equery, @vquery, @form) @collector.type.should == "Resource::Type" end it "should accept an optional resource override" do @collector = Puppet::Parser::Collector.new(@scope, "resource::type", @equery, @vquery, @form) override = { :parameters => "whatever" } @collector.add_override(override) @collector.overrides.should equal(override) end end describe Puppet::Parser::Collector, "when collecting specific virtual resources" do before do @scope = mock 'scope' @vquery = mock 'vquery' @equery = mock 'equery' @collector = Puppet::Parser::Collector.new(@scope, "resource_type", @equery, @vquery, :virtual) end it "should not fail when it does not find any resources to collect" do @collector.resources = ["File[virtual1]", "File[virtual2]"] @scope.stubs(:findresource).returns(false) proc { @collector.evaluate }.should_not raise_error end it "should mark matched resources as non-virtual" do @collector.resources = ["File[virtual1]", "File[virtual2]"] one = stub_everything 'one' one.expects(:virtual=).with(false) @scope.stubs(:findresource).with("File[virtual1]").returns(one) @scope.stubs(:findresource).with("File[virtual2]").returns(nil) @collector.evaluate end it "should return matched resources" do @collector.resources = ["File[virtual1]", "File[virtual2]"] one = stub_everything 'one' @scope.stubs(:findresource).with("File[virtual1]").returns(one) @scope.stubs(:findresource).with("File[virtual2]").returns(nil) @collector.evaluate.should == [one] end it "should delete itself from the compile's collection list if it has found all of its resources" do @collector.resources = ["File[virtual1]"] one = stub_everything 'one' @compiler.expects(:delete_collection).with(@collector) @scope.expects(:compiler).returns(@compiler) @scope.stubs(:findresource).with("File[virtual1]").returns(one) @collector.evaluate end it "should not delete itself from the compile's collection list if it has unfound resources" do @collector.resources = ["File[virtual1]"] one = stub_everything 'one' @compiler.expects(:delete_collection).never @scope.stubs(:findresource).with("File[virtual1]").returns(nil) @collector.evaluate end end describe Puppet::Parser::Collector, "when collecting virtual and catalog resources" do before do @scope = mock 'scope' @compiler = mock 'compile' @scope.stubs(:compiler).returns(@compiler) @resource_type = "Mytype" @vquery = proc { |res| true } @collector = Puppet::Parser::Collector.new(@scope, @resource_type, nil, @vquery, :virtual) end it "should find all virtual resources matching the vquery" do one = stub_everything 'one', :type => "Mytype", :virtual? => true two = stub_everything 'two', :type => "Mytype", :virtual? => true @compiler.expects(:resources).returns([one, two]) @collector.evaluate.should == [one, two] end it "should find all non-virtual resources matching the vquery" do one = stub_everything 'one', :type => "Mytype", :virtual? => false two = stub_everything 'two', :type => "Mytype", :virtual? => false @compiler.expects(:resources).returns([one, two]) @collector.evaluate.should == [one, two] end it "should mark all matched resources as non-virtual" do one = stub_everything 'one', :type => "Mytype", :virtual? => true one.expects(:virtual=).with(false) @compiler.expects(:resources).returns([one]) @collector.evaluate end it "should return matched resources" do one = stub_everything 'one', :type => "Mytype", :virtual? => true two = stub_everything 'two', :type => "Mytype", :virtual? => true @compiler.expects(:resources).returns([one, two]) @collector.evaluate.should == [one, two] end it "should return all resources of the correct type if there is no virtual query" do one = stub_everything 'one', :type => "Mytype", :virtual? => true two = stub_everything 'two', :type => "Mytype", :virtual? => true one.expects(:virtual=).with(false) two.expects(:virtual=).with(false) @compiler.expects(:resources).returns([one, two]) @collector = Puppet::Parser::Collector.new(@scope, @resource_type, nil, nil, :virtual) @collector.evaluate.should == [one, two] end it "should not return or mark resources of a different type" do one = stub_everything 'one', :type => "Mytype", :virtual? => true two = stub_everything 'two', :type => :other, :virtual? => true one.expects(:virtual=).with(false) two.expects(:virtual=).never @compiler.expects(:resources).returns([one, two]) @collector.evaluate.should == [one] end it "should create a resource with overridden parameters" do one = stub_everything 'one', :type => "Mytype", :virtual? => true, :title => "test" param = stub 'param' @compiler.stubs(:add_override) @compiler.expects(:resources).returns([one]) @collector.add_override(:parameters => param ) Puppet::Parser::Resource.expects(:new).with { |type, title, h| h[:parameters] == param } @collector.evaluate end it "should define a new allow all child_of? on overriden resource" do one = stub_everything 'one', :type => "Mytype", :virtual? => true, :title => "test" param = stub 'param' source = stub 'source' @compiler.stubs(:add_override) @compiler.expects(:resources).returns([one]) @collector.add_override(:parameters => param, :source => source ) Puppet::Parser::Resource.stubs(:new) source.expects(:meta_def).with { |name,block| name == :child_of? } @collector.evaluate end it "should not override already overriden resources for this same collection in a previous run" do one = stub_everything 'one', :type => "Mytype", :virtual? => true, :title => "test" param = stub 'param' @compiler.stubs(:add_override) @compiler.expects(:resources).at_least(2).returns([one]) @collector.add_override(:parameters => param ) Puppet::Parser::Resource.expects(:new).once.with { |type, title, h| h[:parameters] == param } @collector.evaluate @collector.evaluate end it "should not return resources that were collected in a previous run of this collector" do one = stub_everything 'one', :type => "Mytype", :virtual? => true, :title => "test" @compiler.stubs(:resources).returns([one]) @collector.evaluate @collector.evaluate.should be_false end it "should tell the compiler about the overriden resources" do one = stub_everything 'one', :type => "Mytype", :virtual? => true, :title => "test" param = stub 'param' one.expects(:virtual=).with(false) @compiler.expects(:resources).returns([one]) @collector.add_override(:parameters => param ) Puppet::Parser::Resource.stubs(:new).returns("whatever") @compiler.expects(:add_override).with("whatever") @collector.evaluate end it "should not return or mark non-matching resources" do @collector.vquery = proc { |res| res.name == :one } one = stub_everything 'one', :name => :one, :type => "Mytype", :virtual? => true two = stub_everything 'two', :name => :two, :type => "Mytype", :virtual? => true one.expects(:virtual=).with(false) two.expects(:virtual=).never @compiler.expects(:resources).returns([one, two]) @collector.evaluate.should == [one] end end describe Puppet::Parser::Collector, "when collecting exported resources", :if => (Puppet.features.rails? and defined? SQLite3) do include PuppetSpec::Files before do @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("mynode")) @scope = Puppet::Parser::Scope.new :compiler => @compiler @resource_type = "notify" @equery = ["title", "!=", ""] @vquery = proc { |r| true } @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @equery, @vquery, :exported) end it "should just return false if :storeconfigs is not enabled" do Puppet[:storeconfigs] = false @collector.evaluate.should be_false end context "with storeconfigs enabled" do before :each do dir = Pathname(tmpdir('puppet-var')) Puppet[:vardir] = dir.to_s Puppet[:dbadapter] = 'sqlite3' Puppet[:dblocation] = (dir + 'storeconfigs.sqlite').to_s Puppet[:storeconfigs] = true Puppet[:environment] = "production" Puppet[:storeconfigs_backend] = "active_record" Puppet::Rails.init end it "should return all matching resources from the current compile and mark them non-virtual and non-exported" do one = Puppet::Parser::Resource.new('notify', 'one', :virtual => true, :exported => true, :scope => @scope) two = Puppet::Parser::Resource.new('notify', 'two', :virtual => true, :exported => true, :scope => @scope) @compiler.resources << one @compiler.resources << two @collector.evaluate.should == [one, two] one.should_not be_virtual two.should_not be_virtual # REVISIT: Apparently we never actually marked local resources as # non-exported. So, this is what the previous test asserted, and checking # what it claims to do causes test failures. --daniel 2011-08-23 end it "should mark all returned resources as not virtual" do one = Puppet::Parser::Resource.new('notify', 'one', :virtual => true, :exported => true, :scope => @scope) @compiler.resources << one @collector.evaluate.should == [one] one.should_not be_virtual end it "should convert all found resources into parser resources if necessary" do host = Puppet::Rails::Host.create!(:name => 'one.local') Puppet::Rails::Resource. create!(:host => host, :restype => 'Notify', :title => 'whammo', :exported => true) result = @collector.evaluate result.length.should == 1 result.first.should be_an_instance_of Puppet::Parser::Resource result.first.type.should == 'Notify' result.first.title.should == 'whammo' end it "should leave parser resources alone" do resource = Puppet::Parser::Resource.new(:file, "/tmp/foo", :scope => @scope) resource2 = Puppet::Parser::Resource.new(:file, "/tmp/bar", :scope => @scope) resource.expects(:to_resource).never resource2.expects(:to_resource).never resources = [resource, resource2] Puppet::Resource.indirection.stubs(:search).returns resources @collector.evaluate.should == resources end it "should override all exported collected resources if collector has an override" do host = Puppet::Rails::Host.create!(:name => 'one.local') Puppet::Rails::Resource. create!(:host => host, :restype => 'Notify', :title => 'whammo', :exported => true) param = Puppet::Parser::Resource::Param. new(:name => 'message', :value => 'howdy') @collector.add_override(:parameters => [param], :scope => @scope) got = @collector.evaluate got.first[:message].should == param.value end it "should store converted resources in the compile's resource list" do host = Puppet::Rails::Host.create!(:name => 'one.local') Puppet::Rails::Resource. create!(:host => host, :restype => 'Notify', :title => 'whammo', :exported => true) @compiler.expects(:add_resource).with do |scope, resource| scope.should be_an_instance_of Puppet::Parser::Scope resource.type.should == 'Notify' resource.title.should == 'whammo' true end @collector.evaluate end # This way one host doesn't store another host's resources as exported. it "should mark resources collected from the database as not exported" do host = Puppet::Rails::Host.create!(:name => 'one.local') Puppet::Rails::Resource. create!(:host => host, :restype => 'Notify', :title => 'whammo', :exported => true) got = @collector.evaluate got.length.should == 1 got.first.type.should == "Notify" got.first.title.should == "whammo" got.first.should_not be_exported end it "should fail if an equivalent resource already exists in the compile" do host = Puppet::Rails::Host.create!(:name => 'one.local') Puppet::Rails::Resource. create!(:host => host, :restype => 'Notify', :title => 'whammo', :exported => true) local = Puppet::Parser::Resource.new('notify', 'whammo', :scope => @scope) @compiler.add_resource(@scope, local) expect { @collector.evaluate }. - to raise_error Puppet::ParseError, /cannot override local resource/ + to raise_error Puppet::ParseError, /exists with the type and title/ end it "should ignore exported resources that match already-collected resources" do host = Puppet::Rails::Host.create!(:name => 'one.local') # One that we already collected... db = Puppet::Rails::Resource. create!(:host => host, :restype => 'Notify', :title => 'whammo', :exported => true) # ...and one we didn't. Puppet::Rails::Resource. create!(:host => host, :restype => 'Notify', :title => 'boingy-boingy', :exported => true) local = Puppet::Parser::Resource.new('notify', 'whammo', :scope => @scope, :collector_id => db.id) @compiler.add_resource(@scope, local) got = nil expect { got = @collector.evaluate }.not_to raise_error(Puppet::ParseError) got.length.should == 1 got.first.type.should == "Notify" got.first.title.should == "boingy-boingy" end end end