diff --git a/lib/puppet/pops/evaluator/collector_transformer.rb b/lib/puppet/pops/evaluator/collector_transformer.rb index 86af4c394..e15848832 100644 --- a/lib/puppet/pops/evaluator/collector_transformer.rb +++ b/lib/puppet/pops/evaluator/collector_transformer.rb @@ -1,199 +1,200 @@ class Puppet::Pops::Evaluator::CollectorTransformer def initialize @@query_visitor ||= Puppet::Pops::Visitor.new(nil, "query", 1, 1) @@match_visitor ||= Puppet::Pops::Visitor.new(nil, "match", 1, 1) @@evaluator ||= Puppet::Pops::Evaluator::EvaluatorImpl.new @@compare_operator ||= Puppet::Pops::Evaluator::CompareOperator.new() end - def query(o, scope) - @@query_visitor.visit_this_1(self, o, scope) - end - - def match(o, scope) - @@match_visitor.visit_this_1(self, o, scope) - end - def transform(o, scope) raise ArgumentError, "Expected CollectExpression" unless o.is_a? Puppet::Pops::Model::CollectExpression raise "LHS is not a type" unless o.type_expr.is_a? Puppet::Pops::Model::QualifiedReference type = o.type_expr.value().downcase() if type == 'class' fail "Classes cannot be collected" end resource_type = scope.find_resource_type(type) fail "Resource type #{type} doesn't exist" unless resource_type adapter = Puppet::Pops::Adapters::SourcePosAdapter.adapt(o) line_num = adapter.line position = adapter.pos file_path = adapter.locator.file if !o.operations.empty? overrides = { :parameters => o.operations.map{ |x| to_3x_param(x).evaluate(scope)}, :file => file_path, :line => [line_num, position], :source => scope.source, :scope => scope } end code = query_unless_nop(o.query, scope) case o.query when Puppet::Pops::Model::VirtualQuery newcoll = Puppet::Pops::Evaluator::Collectors::CatalogCollector.new(scope, resource_type.name, code, overrides) when Puppet::Pops::Model::ExportedQuery match = match_unless_nop(o.query, scope) newcoll = Puppet::Pops::Evaluator::Collectors::ExportedCollector.new(scope, resource_type.name, match, code, overrides) end scope.compiler.add_collection(newcoll) newcoll end +protected + + def query(o, scope) + @@query_visitor.visit_this_1(self, o, scope) + end + + def match(o, scope) + @@match_visitor.visit_this_1(self, o, scope) + end + def query_unless_nop(query, scope) unless query.expr.nil? || query.expr.is_a?(Puppet::Pops::Model::Nop) query(query.expr, scope) end end def match_unless_nop(query, scope) unless query.expr.nil? || query.expr.is_a?(Puppet::Pops::Model::Nop) match(query.expr, scope) end end def query_AndExpression(o, scope) left_code = query(o.left_expr, scope) right_code = query(o.right_expr, scope) proc do |resource| left_code.call(resource) && right_code.call(resource) end end def query_OrExpression(o, scope) left_code = query(o.left_expr, scope) right_code = query(o.right_expr, scope) proc do |resource| left_code.call(resource) || right_code.call(resource) end end def query_ComparisonExpression(o, scope) left_code = query(o.left_expr, scope) right_code = query(o.right_expr, scope) case o.operator when :'==' if left_code == "tag" proc do |resource| resource.tagged?(right_code) end else proc do |resource| if (tmp = resource[left_code]).is_a?(Array) @@compare_operator.include?(tmp, right_code, scope) else @@compare_operator.equals(tmp, right_code) end end end when :'!=' proc do |resource| !@@compare_operator.equals(resource[left_code], right_code) end end end def query_VariableExpression(o, scope) @@evaluator.evaluate(o, scope) end def query_LiteralBoolean(o, scope) @@evaluator.evaluate(o, scope) end def query_LiteralString(o, scope) @@evaluator.evaluate(o, scope) end def query_LiteralNumber(o, scope) @@evaluator.evaluate(o, scope) end def query_QualifiedName(o, scope) @@evaluator.evaluate(o, scope) end def query_ParenthesizedExpression(o, scope) query(o.expr, scope) end def query_Object(o, scope) raise ArgumentError, "Cannot transform object of class #{o.class}" end def match_AndExpression(o, scope) left_match = match(o.left_expr, scope) right_match = match(o.right_expr, scope) return [left_match, 'and', right_match] end def match_OrExpression(o, scope) left_match = match(o.left_expr, scope) right_match = match(o.right_expr, scope) return [left_match, 'or', right_match] end def match_ComparisonExpression(o, scope) left_match = match(o.left_expr, scope) right_match = match(o.right_expr, scope) return [left_match, o.operator.to_s, right_match] end def match_VariableExpression(o, scope) @@evaluator.evaluate(o, scope) end def match_LiteralBoolean(o, scope) @@evaluator.evaluate(o, scope) end def match_LiteralString(o, scope) @@evaluator.evaluate(o, scope) end def match_LiteralNumber(o, scope) @@evaluator.evaluate(o, scope) end def match_QualifiedName(o, scope) @@evaluator.evaluate(o, scope) end def match_ParenthesizedExpression(o, scope) match(o.expr, scope) end def match_Object(o, scope) raise ArgumentError, "Cannot transform object of class #{o.class}" end # Produces (name => expr) or (name +> expr) def to_3x_param(o) bridge = Puppet::Parser::AST::PopsBridge::Expression.new(:value => o.value_expr) args = { :value => bridge } - #TODO: May delete this line later args[:add] = true if o.operator == :'+>' args[:param] = o.attribute_name args= Puppet::Pops::Model::AstTransformer.new().merge_location(args, o) Puppet::Parser::AST::ResourceParam.new(args) end end diff --git a/lib/puppet/pops/evaluator/collectors/abstract_collector.rb b/lib/puppet/pops/evaluator/collectors/abstract_collector.rb index 00e0ac1ce..f5abe5ee6 100644 --- a/lib/puppet/pops/evaluator/collectors/abstract_collector.rb +++ b/lib/puppet/pops/evaluator/collectors/abstract_collector.rb @@ -1,84 +1,86 @@ class Puppet::Pops::Evaluator::Collectors::AbstractCollector attr_reader :scope + # The collector's hash of overrides {:parameters => params} attr_reader :overrides + # The set of collected resources attr_reader :collected # An empty array which will be returned by the unresolved_resources # method unless we have a FixSetCollector - EMPTY_RESOURCES = [].frozen + EMPTY_RESOURCES = [].freeze # Initialized the instance variables needed by the base # collector class to perform evaluation # # @param [Puppet::Parser::Scope] scope # # @param [Hash] overrides a hash of optional overrides # @options opts [Array] :parameters # @options opts [String] :file # @options opts [Array] :line # @options opts [Puppet::Resource::Type] :source # @options opts [Puppet::Parser::Scope] :scope def initialize(scope, overrides = nil) @collected = {} @scope = scope if !(overrides.nil? || overrides[:parameters]) raise ArgumentError, "Exported resource try to override without parameters" end @overrides = overrides end # Collects resources and marks collected objects as non-virtual. Also # handles overrides. # # @return [Array] the resources we have collected def evaluate objects = collect.each do |obj| obj.virtual = false end return false if objects.empty? if @overrides and !objects.empty? overrides[:source].meta_def(:child_of?) do |klass| true end objects.each do |res| unless @collected.include?(res.ref) newres = Puppet::Parser::Resource.new(res.type, res.title, @overrides) scope.compiler.add_override(newres) end end end objects.reject! { |o| @collected.include?(o.ref) } return false if objects.empty? objects.inject(@collected) { |c,o| c[o.ref]=o; c } objects end # This should only return an empty array unless we have # an FixedSetCollector, in which case it will return the # resources that have not yet been realized # # @return [Array] the resources that have not been resolved def unresolved_resources EMPTY_RESOURCES end # Collect the specified resources. The way this is done depends on which type # of collector we are dealing with. This method is implemented differently in # each of the three child classes # # @return [Array] the collected resources def collect raise NotImplementedError, "This method must be implemented by the child class" end end diff --git a/lib/puppet/pops/evaluator/collectors/catalog_collector.rb b/lib/puppet/pops/evaluator/collectors/catalog_collector.rb index 2ab819054..c684e4958 100644 --- a/lib/puppet/pops/evaluator/collectors/catalog_collector.rb +++ b/lib/puppet/pops/evaluator/collectors/catalog_collector.rb @@ -1,26 +1,25 @@ class Puppet::Pops::Evaluator::Collectors::CatalogCollector < Puppet::Pops::Evaluator::Collectors::AbstractCollector # Creates a CatalogCollector using the AbstractCollector's # constructor to set the scope and overrides # # param [Symbol] type the resource type to be collected # param [Proc] query the query which defines which resources to match def initialize(scope, type, query, overrides = nil) super(scope, overrides) @query = query - #TODO: Can this be refactored? @type = Puppet::Resource.new(type, "whatever").type end # Collects virtual resources based off a collection in a manifest def collect t = @type q = @query scope.compiler.resources.find_all do |resource| resource.type == t && (q ? q.call(resource) : true) end end end diff --git a/lib/puppet/pops/evaluator/collectors/exported_collector.rb b/lib/puppet/pops/evaluator/collectors/exported_collector.rb index ad5384cbd..e06958bb7 100644 --- a/lib/puppet/pops/evaluator/collectors/exported_collector.rb +++ b/lib/puppet/pops/evaluator/collectors/exported_collector.rb @@ -1,67 +1,66 @@ class Puppet::Pops::Evaluator::Collectors::ExportedCollector < Puppet::Pops::Evaluator::Collectors::AbstractCollector # Creates an ExportedCollector using the AbstractCollector's # constructor to set the scope and overrides # # param [Symbol] type the resource type to be collected # param [Array] equery an array representation of the query (exported query) # param [Proc] cquery a proc representation of the query (catalog query) def initialize(scope, type, equery, cquery, overrides = nil) super(scope, overrides) @equery = equery @cquery = cquery - # Canonize the type @type = Puppet::Resource.new(type, "whatever").type end # Ensures that storeconfigs is present before calling AbstractCollector's # evaluate method def evaluate if Puppet[:storeconfigs] != true Puppet.warning "Not collecting exported resources without storeconfigs" return false end super end # Collect exported resources as defined by an exported # collection. Used with PuppetDB def collect resources = [] time = Puppet::Util.thinmark do t = @type q = @cquery scope.compiler.resources.find_all do |resource| resource.type == t && resource.exported? && q.call(resource) end found = Puppet::Resource.indirection. search(@type, :host => @scope.compiler.node.name, :filter => @equery, :scope => @scope) found_resources = found.map {|x| x.is_a?(Puppet::Parser::Resource) ? x : x.to_resource(@scope)} found_resources.each do |item| if existing = @scope.findresource(item.type, item.title) unless existing.collector_id == item.collector_id raise Puppet::ParseError, "A duplicate resource was found while collecting exported resources, with the type and title #{item.ref}" end else item.exported = false @scope.compiler.add_resource(@scope, item) resources << item end end end scope.debug("Collected %s %s resource%s in %.2f seconds" % [resources.length, @type, resources.length == 1 ? "" : "s", time]) resources end end diff --git a/lib/puppet/pops/evaluator/collectors/fixed_set_collector.rb b/lib/puppet/pops/evaluator/collectors/fixed_set_collector.rb index fa949bfc4..862724584 100644 --- a/lib/puppet/pops/evaluator/collectors/fixed_set_collector.rb +++ b/lib/puppet/pops/evaluator/collectors/fixed_set_collector.rb @@ -1,36 +1,37 @@ class Puppet::Pops::Evaluator::Collectors::FixedSetCollector < Puppet::Pops::Evaluator::Collectors::AbstractCollector # Creates a FixedSetCollector using the AbstractCollector constructor # to set the scope. It is not possible for a collection to have - # overrides in this case, since we have a fixed set of resources + # overrides in this case, since we have a fixed set of resources that + # can be different types. # # @param [Array] resources the fixed set of resources we want to realize def initialize(scope, resources) super(scope) @resources = resources.is_a?(Array)? resources.dup : [resources] end # Collects a fixed set of resources and realizes them. Used # by the realize function def collect resolved = [] result = @resources.reduce([]) do |memo, ref| if res = @scope.findresource(ref.to_s) res.virtual = false memo << res resolved << ref end memo end @resources = @resources - resolved @scope.compiler.delete_collection(self) if @resources.empty? result end def unresolved_resources @resources end end diff --git a/spec/integration/parser/collector_spec.rb b/spec/integration/parser/collector_spec.rb index 55ac66a5b..cfb4f0265 100755 --- a/spec/integration/parser/collector_spec.rb +++ b/spec/integration/parser/collector_spec.rb @@ -1,276 +1,288 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet_spec/compiler' require 'puppet/parser/collector' describe Puppet::Parser::Collector do include PuppetSpec::Compiler def expect_the_message_to_be(expected_messages, code, node = Puppet::Node.new('the node')) catalog = compile_to_catalog(code, node) messages = catalog.resources.find_all { |resource| resource.type == 'Notify' }. collect { |notify| notify[:message] } messages.should include(*expected_messages) end shared_examples_for "virtual resource collection" do it "matches everything when no query given" do expect_the_message_to_be(["the other message", "the message"], <<-MANIFEST) @notify { "testing": message => "the message" } @notify { "other": message => "the other message" } Notify <| |> MANIFEST end it "matches regular resources " do expect_the_message_to_be(["changed", "changed"], <<-MANIFEST) notify { "testing": message => "the message" } notify { "other": message => "the other message" } Notify <| |> { message => "changed" } MANIFEST end it "matches on tags" do expect_the_message_to_be(["wanted"], <<-MANIFEST) @notify { "testing": tag => ["one"], message => "wanted" } @notify { "other": tag => ["two"], message => "unwanted" } Notify <| tag == one |> MANIFEST end it "matches on title" do expect_the_message_to_be(["the message"], <<-MANIFEST) @notify { "testing": message => "the message" } Notify <| title == "testing" |> MANIFEST end it "matches on other parameters" do expect_the_message_to_be(["the message"], <<-MANIFEST) @notify { "testing": message => "the message" } @notify { "other testing": message => "the wrong message" } Notify <| message == "the message" |> MANIFEST end it "matches against elements of an array valued parameter" do expect_the_message_to_be([["the", "message"]], <<-MANIFEST) @notify { "testing": message => ["the", "message"] } @notify { "other testing": message => ["not", "here"] } Notify <| message == "message" |> MANIFEST end it "allows criteria to be combined with 'and'" do expect_the_message_to_be(["the message"], <<-MANIFEST) @notify { "testing": message => "the message" } @notify { "other": message => "the message" } Notify <| title == "testing" and message == "the message" |> MANIFEST end it "allows criteria to be combined with 'or'" do expect_the_message_to_be(["the message", "other message"], <<-MANIFEST) @notify { "testing": message => "the message" } @notify { "other": message => "other message" } @notify { "yet another": message => "different message" } Notify <| title == "testing" or message == "other message" |> MANIFEST end it "allows criteria to be combined with 'or'" do expect_the_message_to_be(["the message", "other message"], <<-MANIFEST) @notify { "testing": message => "the message" } @notify { "other": message => "other message" } @notify { "yet another": message => "different message" } Notify <| title == "testing" or message == "other message" |> MANIFEST end it "allows criteria to be grouped with parens" do expect_the_message_to_be(["the message", "different message"], <<-MANIFEST) @notify { "testing": message => "different message", withpath => true } @notify { "other": message => "the message" } @notify { "yet another": message => "the message", withpath => true } Notify <| (title == "testing" or message == "the message") and withpath == true |> MANIFEST end it "does not do anything if nothing matches" do expect_the_message_to_be([], <<-MANIFEST) @notify { "testing": message => "different message" } Notify <| title == "does not exist" |> MANIFEST end it "excludes items with inequalities" do expect_the_message_to_be(["good message"], <<-MANIFEST) @notify { "testing": message => "good message" } @notify { "the wrong one": message => "bad message" } Notify <| title != "the wrong one" |> MANIFEST end it "does not exclude resources with unequal arrays" do expect_the_message_to_be(["message", ["not this message", "or this one"]], <<-MANIFEST) @notify { "testing": message => "message" } @notify { "the wrong one": message => ["not this message", "or this one"] } Notify <| message != "not this message" |> MANIFEST end it "does not exclude tags with inequalities" do expect_the_message_to_be(["wanted message", "the way it works"], <<-MANIFEST) @notify { "testing": tag => ["wanted"], message => "wanted message" } @notify { "other": tag => ["why"], message => "the way it works" } Notify <| tag != "why" |> MANIFEST end it "does not collect classes" do node = Puppet::Node.new('the node') expect do catalog = compile_to_catalog(<<-MANIFEST, node) class theclass { @notify { "testing": message => "good message" } } Class <| |> MANIFEST end.to raise_error(/Classes cannot be collected/) end + it "does not collect resources that don't exist" do + node = Puppet::Node.new('the node') + expect do + catalog = compile_to_catalog(<<-MANIFEST, node) + class theclass { + @notify { "testing": message => "good message" } + } + SomeResource <| |> + MANIFEST + end.to raise_error(/Resource type someresource doesn't exist/) + end + context "overrides" do it "modifies an existing array" do expect_the_message_to_be([["original message", "extra message"]], <<-MANIFEST) @notify { "testing": message => ["original message"] } Notify <| |> { message +> "extra message" } MANIFEST end it "converts a scalar to an array" do expect_the_message_to_be([["original message", "extra message"]], <<-MANIFEST) @notify { "testing": message => "original message" } Notify <| |> { message +> "extra message" } MANIFEST end it "collects with override when inside a class (#10963)" do expect_the_message_to_be(["overridden message"], <<-MANIFEST) @notify { "testing": message => "original message" } include collector_test class collector_test { Notify <| |> { message => "overridden message" } } MANIFEST end it "collects with override when inside a define (#10963)" do expect_the_message_to_be(["overridden message"], <<-MANIFEST) @notify { "testing": message => "original message" } collector_test { testing: } define collector_test() { Notify <| |> { message => "overridden message" } } MANIFEST end # Catches regression in implemented behavior, this is not to be taken as this is the wanted behavior # but it has been this way for a long time. it "collects and overrides user defined resources immediately (before queue is evaluated)" do expect_the_message_to_be(["overridden"], <<-MANIFEST) define foo($message) { notify { "testing": message => $message } } foo { test: message => 'given' } Foo <| |> { message => 'overridden' } MANIFEST end # Catches regression in implemented behavior, this is not to be taken as this is the wanted behavior # but it has been this way for a long time. it "collects and overrides user defined resources immediately (virtual resources not queued)" do expect_the_message_to_be(["overridden"], <<-MANIFEST) define foo($message) { @notify { "testing": message => $message } } foo { test: message => 'given' } Notify <| |> # must be collected or the assertion does not find it Foo <| |> { message => 'overridden' } MANIFEST end # Catches regression in implemented behavior, this is not to be taken as this is the wanted behavior # but it has been this way for a long time. # Note difference from none +> case where the override takes effect it "collects and overrides user defined resources with +>" do expect_the_message_to_be([["given", "overridden"]], <<-MANIFEST) define foo($message) { notify { "$name": message => $message } } foo { test: message => ['given'] } Notify <| |> { message +> ['overridden'] } MANIFEST end it "collects and overrides virtual resources multiple times using multiple collects" do expect_the_message_to_be(["overridden2"], <<-MANIFEST) @notify { "testing": message => "original" } Notify <| |> { message => 'overridden1' } Notify <| |> { message => 'overridden2' } MANIFEST end it "collects and overrides non virtual resources multiple times using multiple collects" do expect_the_message_to_be(["overridden2"], <<-MANIFEST) notify { "testing": message => "original" } Notify <| |> { message => 'overridden1' } Notify <| |> { message => 'overridden2' } MANIFEST end end end describe "in the current parser" do before :each do Puppet[:parser] = 'current' end it_behaves_like "virtual resource collection" end describe "in the future parser" do before :each do Puppet[:parser] = 'future' end it_behaves_like "virtual resource collection" end end