diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 125b8e9f2..134f47bf4 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -1,625 +1,629 @@ require 'forwardable' require 'puppet/node' require 'puppet/resource/catalog' require 'puppet/util/errors' require 'puppet/resource/type_collection_helper' # Maintain a graph of scopes, along with a bunch of data # about the individual catalog we're compiling. class Puppet::Parser::Compiler extend Forwardable include Puppet::Util include Puppet::Util::Errors include Puppet::Util::MethodHelper include Puppet::Resource::TypeCollectionHelper def self.compile(node) $env_module_directories = nil node.environment.check_for_reparse if node.environment.conflicting_manifest_settings? errmsg = [ "The 'disable_per_environment_manifest' setting is true, and this '#{node.environment}'", "has an environment.conf manifest that conflicts with the 'default_manifest' setting.", "Compilation has been halted in order to avoid running a catalog which may be using", "unexpected manifests. For more information, see", "http://docs.puppetlabs.com/puppet/latest/reference/environments.html", ] raise(Puppet::Error, errmsg.join(' ')) end new(node).compile.to_resource rescue => detail message = "#{detail} on node #{node.name}" Puppet.log_exception(detail, message) raise Puppet::Error, message, detail.backtrace end attr_reader :node, :facts, :collections, :catalog, :resources, :relationships, :topscope # The injector that provides lookup services, or nil if accessed before the compiler has started compiling and # bootstrapped. The injector is initialized and available before any manifests are evaluated. # # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services for this compiler/environment # @api public # attr_accessor :injector # Access to the configured loaders for 4x # @return [Puppet::Pops::Loader::Loaders] the configured loaders # @api private attr_reader :loaders # The injector that provides lookup services during the creation of the {#injector}. # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services during injector creation # for this compiler/environment # # @api private # attr_accessor :boot_injector # Add a collection to the global list. def_delegator :@collections, :<<, :add_collection def_delegator :@relationships, :<<, :add_relationship # Store a resource override. def add_override(override) # If possible, merge the override in immediately. if resource = @catalog.resource(override.ref) resource.merge(override) else # Otherwise, store the override for later; these # get evaluated in Resource#finish. @resource_overrides[override.ref] << override end end def add_resource(scope, resource) @resources << resource # Note that this will fail if the resource is not unique. @catalog.add_resource(resource) if not resource.class? and resource[:stage] raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage" end # Stages should not be inside of classes. They are always a # top-level container, regardless of where they appear in the # manifest. return if resource.stage? # This adds a resource to the class it lexically appears in in the # manifest. unless resource.class? return @catalog.add_edge(scope.resource, resource) end end # Do we use nodes found in the code, vs. the external node sources? def_delegator :known_resource_types, :nodes?, :ast_nodes? # Store the fact that we've evaluated a class def add_class(name) @catalog.add_class(name) unless name == "" end # Return a list of all of the defined classes. def_delegator :@catalog, :classes, :classlist # Compiler our catalog. This mostly revolves around finding and evaluating classes. # This is the main entry into our catalog. def compile Puppet.override( @context_overrides , "For compiling #{node.name}") do @catalog.environment_instance = environment # Set the client's parameters into the top scope. Puppet::Util::Profiler.profile("Compile: Set node parameters", [:compiler, :set_node_params]) { set_node_parameters } Puppet::Util::Profiler.profile("Compile: Created settings scope", [:compiler, :create_settings_scope]) { create_settings_scope } if is_binder_active? # create injector, if not already created - this is for 3x that does not trigger # lazy loading of injector via context Puppet::Util::Profiler.profile("Compile: Created injector", [:compiler, :create_injector]) { injector } end Puppet::Util::Profiler.profile("Compile: Evaluated main", [:compiler, :evaluate_main]) { evaluate_main } Puppet::Util::Profiler.profile("Compile: Evaluated AST node", [:compiler, :evaluate_ast_node]) { evaluate_ast_node } Puppet::Util::Profiler.profile("Compile: Evaluated node classes", [:compiler, :evaluate_node_classes]) { evaluate_node_classes } Puppet::Util::Profiler.profile("Compile: Evaluated generators", [:compiler, :evaluate_generators]) { evaluate_generators } Puppet::Util::Profiler.profile("Compile: Finished catalog", [:compiler, :finish_catalog]) { finish } fail_on_unevaluated @catalog end end # Constructs the overrides for the context def context_overrides() if Puppet[:parser] == 'future' require 'puppet/loaders' { :current_environment => environment, :global_scope => @topscope, # 4x placeholder for new global scope :loaders => lambda {|| loaders() }, # 4x loaders :injector => lambda {|| injector() } # 4x API - via context instead of via compiler } else { :current_environment => environment, } end end def_delegator :@collections, :delete, :delete_collection # Return the node's environment. def environment node.environment end # Evaluate all of the classes specified by the node. # Classes with parameters are evaluated as if they were declared. # Classes without parameters or with an empty set of parameters are evaluated # as if they were included. This means classes with an empty set of # parameters won't conflict even if the class has already been included. def evaluate_node_classes if @node.classes.is_a? Hash classes_with_params, classes_without_params = @node.classes.partition {|name,params| params and !params.empty?} # The results from Hash#partition are arrays of pairs rather than hashes, # so we have to convert to the forms evaluate_classes expects (Hash, and # Array of class names) classes_with_params = Hash[classes_with_params] classes_without_params.map!(&:first) else classes_with_params = {} classes_without_params = @node.classes end evaluate_classes(classes_with_params, @node_scope || topscope) evaluate_classes(classes_without_params, @node_scope || topscope) end # Evaluate each specified class in turn. If there are any classes we can't # find, raise an error. This method really just creates resource objects # that point back to the classes, and then the resources are themselves # evaluated later in the process. # # Sometimes we evaluate classes with a fully qualified name already, in which # case, we tell scope.find_hostclass we've pre-qualified the name so it # doesn't need to search its namespaces again. This gets around a weird # edge case of duplicate class names, one at top scope and one nested in our # namespace and the wrong one (or both!) getting selected. See ticket #13349 # for more detail. --jeffweiss 26 apr 2012 def evaluate_classes(classes, scope, lazy_evaluate = true, fqname = false) raise Puppet::DevError, "No source for scope passed to evaluate_classes" unless scope.source class_parameters = nil # if we are a param class, save the classes hash # and transform classes to be the keys if classes.class == Hash class_parameters = classes classes = classes.keys end hostclasses = classes.collect do |name| scope.find_hostclass(name, :assume_fqname => fqname) or raise Puppet::Error, "Could not find class #{name} for #{node.name}" end if class_parameters resources = ensure_classes_with_parameters(scope, hostclasses, class_parameters) if !lazy_evaluate resources.each(&:evaluate) end resources else already_included, newly_included = ensure_classes_without_parameters(scope, hostclasses) if !lazy_evaluate newly_included.each(&:evaluate) end already_included + newly_included end end def evaluate_relationships @relationships.each { |rel| rel.evaluate(catalog) } end # Return a resource by either its ref or its type and title. def_delegator :@catalog, :resource, :findresource def initialize(node, options = {}) @node = node set_options(options) initvars end # Create a new scope, with either a specified parent scope or # using the top scope. def newscope(parent, options = {}) parent ||= topscope scope = Puppet::Parser::Scope.new(self, options) scope.parent = parent scope end # Return any overrides for the given resource. def resource_overrides(resource) @resource_overrides[resource.ref] end def injector create_injector if @injector.nil? @injector end def loaders @loaders ||= Puppet::Pops::Loaders.new(environment) end def boot_injector create_boot_injector(nil) if @boot_injector.nil? @boot_injector end # Creates the boot injector from registered system, default, and injector config. # @return [Puppet::Pops::Binder::Injector] the created boot injector # @api private Cannot be 'private' since it is called from the BindingsComposer. # def create_boot_injector(env_boot_bindings) assert_binder_active() pb = Puppet::Pops::Binder boot_contribution = pb::SystemBindings.injector_boot_contribution(env_boot_bindings) final_contribution = pb::SystemBindings.final_contribution binder = pb::Binder.new(pb::BindingsFactory.layered_bindings(final_contribution, boot_contribution)) @boot_injector = pb::Injector.new(binder) end # Answers if Puppet Binder should be active or not, and if it should and is not active, then it is activated. # @return [Boolean] true if the Puppet Binder should be activated def is_binder_active? should_be_active = Puppet[:binder] || Puppet[:parser] == 'future' if should_be_active # TODO: this should be in a central place, not just for ParserFactory anymore... Puppet::Parser::ParserFactory.assert_rgen_installed() @@binder_loaded ||= false unless @@binder_loaded require 'puppet/pops' require 'puppetx' @@binder_loaded = true end end should_be_active end private def ensure_classes_with_parameters(scope, hostclasses, parameters) hostclasses.collect do |klass| klass.ensure_in_catalog(scope, parameters[klass.name] || {}) end end def ensure_classes_without_parameters(scope, hostclasses) already_included = [] newly_included = [] hostclasses.each do |klass| class_scope = scope.class_scope(klass) if class_scope already_included << class_scope.resource else newly_included << klass.ensure_in_catalog(scope) end end [already_included, newly_included] end # If ast nodes are enabled, then see if we can find and evaluate one. def evaluate_ast_node return unless ast_nodes? # Now see if we can find the node. astnode = nil @node.names.each do |name| break if astnode = known_resource_types.node(name.to_s.downcase) end unless (astnode ||= known_resource_types.node("default")) raise Puppet::ParseError, "Could not find default node or by name with '#{node.names.join(", ")}'" end # Create a resource to model this node, and then add it to the list # of resources. resource = astnode.ensure_in_catalog(topscope) resource.evaluate @node_scope = topscope.class_scope(astnode) end # Evaluate our collections and return true if anything returned an object. # The 'true' is used to continue a loop, so it's important. def evaluate_collections return false if @collections.empty? exceptwrap do # We have to iterate over a dup of the array because # collections can delete themselves from the list, which # changes its length and causes some collections to get missed. Puppet::Util::Profiler.profile("Evaluated collections", [:compiler, :evaluate_collections]) do found_something = false @collections.dup.each do |collection| found_something = true if collection.evaluate end found_something end end end # Make sure all of our resources have been evaluated into native resources. # We return true if any resources have, so that we know to continue the # evaluate_generators loop. def evaluate_definitions exceptwrap do Puppet::Util::Profiler.profile("Evaluated definitions", [:compiler, :evaluate_definitions]) do !unevaluated_resources.each do |resource| Puppet::Util::Profiler.profile("Evaluated resource #{resource}", [:compiler, :evaluate_resource, resource]) do resource.evaluate end end.empty? end end end # Iterate over collections and resources until we're sure that the whole # compile is evaluated. This is necessary because both collections # and defined resources can generate new resources, which themselves could # be defined resources. def evaluate_generators count = 0 loop do done = true Puppet::Util::Profiler.profile("Iterated (#{count + 1}) on generators", [:compiler, :iterate_on_generators]) do # Call collections first, then definitions. done = false if evaluate_collections done = false if evaluate_definitions end break if done count += 1 if count > 1000 raise Puppet::ParseError, "Somehow looped more than 1000 times while evaluating host catalog" end end end # Find and evaluate our main object, if possible. def evaluate_main @main = known_resource_types.find_hostclass([""], "") || known_resource_types.add(Puppet::Resource::Type.new(:hostclass, "")) @topscope.source = @main @main_resource = Puppet::Parser::Resource.new("class", :main, :scope => @topscope, :source => @main) @topscope.resource = @main_resource add_resource(@topscope, @main_resource) @main_resource.evaluate end # Make sure the entire catalog is evaluated. def fail_on_unevaluated fail_on_unevaluated_overrides fail_on_unevaluated_resource_collections end # If there are any resource overrides remaining, then we could # not find the resource they were supposed to override, so we # want to throw an exception. def fail_on_unevaluated_overrides remaining = @resource_overrides.values.flatten.collect(&:ref) if !remaining.empty? fail Puppet::ParseError, "Could not find resource(s) #{remaining.join(', ')} for overriding" end end # Make sure we don't have any remaining collections that specifically # look for resources, because we want to consider those to be # parse errors. def fail_on_unevaluated_resource_collections - remaining = @collections.collect(&:resources).flatten.compact + if Puppet[:parser] == 'future' + remaining = @collections.collect(&:unresolved_resources).flatten.compact + else + remaining = @collections.collect(&:resources).flatten.compact + end if !remaining.empty? raise Puppet::ParseError, "Failed to realize virtual resources #{remaining.join(', ')}" end end # Make sure all of our resources and such have done any last work # necessary. def finish evaluate_relationships resources.each do |resource| # Add in any resource overrides. if overrides = resource_overrides(resource) overrides.each do |over| resource.merge(over) end # Remove the overrides, so that the configuration knows there # are none left. overrides.clear end resource.finish if resource.respond_to?(:finish) end add_resource_metaparams end def add_resource_metaparams unless main = catalog.resource(:class, :main) raise "Couldn't find main" end names = Puppet::Type.metaparams.select do |name| !Puppet::Parser::Resource.relationship_parameter?(name) end data = {} catalog.walk(main, :out) do |source, target| if source_data = data[source] || metaparams_as_data(source, names) # only store anything in the data hash if we've actually got # data data[source] ||= source_data source_data.each do |param, value| target[param] = value if target[param].nil? end data[target] = source_data.merge(metaparams_as_data(target, names)) end target.tag(*(source.tags)) end end def metaparams_as_data(resource, params) data = nil params.each do |param| unless resource[param].nil? # Because we could be creating a hash for every resource, # and we actually probably don't often have any data here at all, # we're optimizing a bit by only creating a hash if there's # any data to put in it. data ||= {} data[param] = resource[param] end end data end # Set up all of our internal variables. def initvars # The list of overrides. This is used to cache overrides on objects # that don't exist yet. We store an array of each override. @resource_overrides = Hash.new do |overs, ref| overs[ref] = [] end # The list of collections that have been created. This is a global list, # but they each refer back to the scope that created them. @collections = [] # The list of relationships to evaluate. @relationships = [] # For maintaining the relationship between scopes and their resources. @catalog = Puppet::Resource::Catalog.new(@node.name, @node.environment) # MOVED HERE - SCOPE IS NEEDED (MOVE-SCOPE) # Create the initial scope, it is needed early @topscope = Puppet::Parser::Scope.new(self) # Need to compute overrides here, and remember them, because we are about to # enter the magic zone of known_resource_types and intial import. # Expensive entries in the context are bound lazily. @context_overrides = context_overrides() # This construct ensures that initial import (triggered by instantiating # the structure 'known_resource_types') has a configured context # It cannot survive the initvars method, and is later reinstated # as part of compiling... # Puppet.override( @context_overrides , "For initializing compiler") do # THE MAGIC STARTS HERE ! This triggers parsing, loading etc. @catalog.version = known_resource_types.version end @catalog.add_resource(Puppet::Parser::Resource.new("stage", :main, :scope => @topscope)) # local resource array to maintain resource ordering @resources = [] # Make sure any external node classes are in our class list if @node.classes.class == Hash @catalog.add_class(*@node.classes.keys) else @catalog.add_class(*@node.classes) end end # Set the node's parameters into the top-scope as variables. def set_node_parameters node.parameters.each do |param, value| @topscope[param.to_s] = value end # These might be nil. catalog.client_version = node.parameters["clientversion"] catalog.server_version = node.parameters["serverversion"] if Puppet[:trusted_node_data] @topscope.set_trusted(node.trusted_data) end if(Puppet[:immutable_node_data]) facts_hash = node.facts.nil? ? {} : node.facts.values @topscope.set_facts(facts_hash) end end def create_settings_scope settings_type = Puppet::Resource::Type.new :hostclass, "settings" environment.known_resource_types.add(settings_type) settings_resource = Puppet::Parser::Resource.new("class", "settings", :scope => @topscope) @catalog.add_resource(settings_resource) settings_type.evaluate_code(settings_resource) scope = @topscope.class_scope(settings_type) env = environment Puppet.settings.each do |name, setting| next if name == :name scope[name.to_s] = env[name] end end # Return an array of all of the unevaluated resources. These will be definitions, # which need to get evaluated into native resources. def unevaluated_resources # The order of these is significant for speed due to short-circuting resources.reject { |resource| resource.evaluated? or resource.virtual? or resource.builtin_type? } end # Creates the injector from bindings found in the current environment. # @return [void] # @api private # def create_injector assert_binder_active() composer = Puppet::Pops::Binder::BindingsComposer.new() layered_bindings = composer.compose(topscope) @injector = Puppet::Pops::Binder::Injector.new(Puppet::Pops::Binder::Binder.new(layered_bindings)) end def assert_binder_active unless is_binder_active? raise ArgumentError, "The Puppet Binder is only available when either '--binder true' or '--parser future' is used" end end end diff --git a/lib/puppet/parser/functions/realize.rb b/lib/puppet/parser/functions/realize.rb index 4d1567de1..2b8d3eaa6 100644 --- a/lib/puppet/parser/functions/realize.rb +++ b/lib/puppet/parser/functions/realize.rb @@ -1,14 +1,19 @@ # This is just syntactic sugar for a collection, although it will generally # be a good bit faster. Puppet::Parser::Functions::newfunction(:realize, :arity => -2, :doc => "Make a virtual object real. This is useful when you want to know the name of the virtual object and don't want to bother with a full collection. It is slightly faster than a collection, and, of course, is a bit shorter. You must pass the object using a reference; e.g.: `realize User[luke]`." ) do |vals| - coll = Puppet::Parser::Collector.new(self, :nomatter, nil, nil, :virtual) + vals = [vals] unless vals.is_a?(Array) - coll.resources = vals.flatten + if Puppet[:parser] == 'future' + coll = Puppet::Pops::Evaluator::Collectors::FixedSetCollector.new(self, vals.flatten) + else + coll = Puppet::Parser::Collector.new(self, :nomatter, nil, nil, :virtual) + coll.resources = vals.flatten + end compiler.add_collection(coll) end diff --git a/lib/puppet/parser/relationship.rb b/lib/puppet/parser/relationship.rb index a9d592cbf..a32d0ae90 100644 --- a/lib/puppet/parser/relationship.rb +++ b/lib/puppet/parser/relationship.rb @@ -1,62 +1,77 @@ class Puppet::Parser::Relationship attr_accessor :source, :target, :type PARAM_MAP = {:relationship => :before, :subscription => :notify} def arrayify(resources) - case resources - when Puppet::Parser::Collector - resources.collected.values - when Array - resources + # This if statement is needed because the 3x parser cannot load + # Puppet::Pops. This logic can be removed for 4.0 when the 3x AST + # is removed (when Pops is always used). + if !(Puppet[:parser] == 'future') + case resources + when Puppet::Parser::Collector + resources.collected.values + when Array + resources + else + [resources] + end else - [resources] + require 'puppet/pops' + case resources + when Puppet::Pops::Evaluator::Collectors::AbstractCollector + resources.collected.values + when Array + resources + else + [resources] + end end end def evaluate(catalog) arrayify(source).each do |s| arrayify(target).each do |t| mk_relationship(s, t, catalog) end end end def initialize(source, target, type) @source, @target, @type = source, target, type end def param_name PARAM_MAP[type] || raise(ArgumentError, "Invalid relationship type #{type}") end def mk_relationship(source, target, catalog) # REVISIT: In Ruby 1.8 we applied `to_s` to source and target, rather than # `join` without an argument. In 1.9 the behaviour of Array#to_s changed, # and it gives a different representation than just concat the stringified # elements. # # This restores the behaviour, but doesn't address the underlying question # of what would happen when more than one item was passed in that array. # (Hint: this will not end well.) # # See http://projects.puppetlabs.com/issues/12076 for the ticket tracking # the fact that we should dig out the sane version of this behaviour, then # implement it - where we don't risk breaking a stable release series. # --daniel 2012-01-21 source = source.is_a?(Array) ? source.join : source.to_s target = target.is_a?(Array) ? target.join : target.to_s unless source_resource = catalog.resource(source) raise ArgumentError, "Could not find resource '#{source}' for relationship on '#{target}'" end unless catalog.resource(target) raise ArgumentError, "Could not find resource '#{target}' for relationship from '#{source}'" end Puppet.debug "Adding relationship from #{source} to #{target} with '#{param_name}'" if source_resource[param_name].class != Array source_resource[param_name] = [source_resource[param_name]].compact end source_resource[param_name] << target end end diff --git a/lib/puppet/pops.rb b/lib/puppet/pops.rb index 5924f1d45..7b4c6def6 100644 --- a/lib/puppet/pops.rb +++ b/lib/puppet/pops.rb @@ -1,119 +1,125 @@ module Puppet # The Pops language system. This includes the parser, evaluator, AST model, and # Binder. # # @todo Explain how a user should use this to parse and evaluate the puppet # language. # # @note Warning: Pops is still considered experimental, as such the API may # change at any time. # # @api public module Pops require 'puppet/pops/patterns' require 'puppet/pops/utils' require 'puppet/pops/adaptable' require 'puppet/pops/adapters' require 'puppet/pops/visitable' require 'puppet/pops/visitor' require 'puppet/pops/containment' require 'puppet/pops/issues' require 'puppet/pops/semantic_error' require 'puppet/pops/label_provider' require 'puppet/pops/validation' require 'puppet/pops/issue_reporter' require 'puppet/pops/model/model' # (the Types module initializes itself) require 'puppet/pops/types/types' require 'puppet/pops/types/type_calculator' require 'puppet/pops/types/type_factory' require 'puppet/pops/types/type_parser' require 'puppet/pops/types/class_loader' require 'puppet/pops/types/enumeration' module Model require 'puppet/pops/model/tree_dumper' require 'puppet/pops/model/ast_transformer' require 'puppet/pops/model/ast_tree_dumper' require 'puppet/pops/model/factory' require 'puppet/pops/model/model_tree_dumper' require 'puppet/pops/model/model_label_provider' end module Binder module SchemeHandler # the handlers are auto loaded via bindings end module Producers require 'puppet/pops/binder/producers' end require 'puppet/pops/binder/binder' require 'puppet/pops/binder/bindings_model' require 'puppet/pops/binder/binder_issues' require 'puppet/pops/binder/bindings_checker' require 'puppet/pops/binder/bindings_factory' require 'puppet/pops/binder/bindings_label_provider' require 'puppet/pops/binder/bindings_validator_factory' require 'puppet/pops/binder/injector_entry' require 'puppet/pops/binder/key_factory' require 'puppet/pops/binder/injector' require 'puppet/pops/binder/bindings_composer' require 'puppet/pops/binder/bindings_model_dumper' require 'puppet/pops/binder/system_bindings' require 'puppet/pops/binder/bindings_loader' require 'puppet/pops/binder/lookup' module Config require 'puppet/pops/binder/config/binder_config' require 'puppet/pops/binder/config/binder_config_checker' require 'puppet/pops/binder/config/issues' require 'puppet/pops/binder/config/diagnostic_producer' end end module Parser require 'puppet/pops/parser/eparser' require 'puppet/pops/parser/parser_support' require 'puppet/pops/parser/locator' require 'puppet/pops/parser/locatable' require 'puppet/pops/parser/lexer2' require 'puppet/pops/parser/evaluating_parser' require 'puppet/pops/parser/epp_parser' end module Validation require 'puppet/pops/validation/checker4_0' require 'puppet/pops/validation/validator_factory_4_0' end module Evaluator require 'puppet/pops/evaluator/callable_signature' require 'puppet/pops/evaluator/runtime3_support' require 'puppet/pops/evaluator/evaluator_impl' require 'puppet/pops/evaluator/epp_evaluator' require 'puppet/pops/evaluator/callable_mismatch_describer' require 'puppet/pops/evaluator/collector_transformer' + module Collectors + require 'puppet/pops/evaluator/collectors/abstract_collector' + require 'puppet/pops/evaluator/collectors/fixed_set_collector' + require 'puppet/pops/evaluator/collectors/catalog_collector' + require 'puppet/pops/evaluator/collectors/exported_collector' + end end # Subsystem for puppet functions defined in ruby. # # @api public module Functions require 'puppet/pops/functions/function' require 'puppet/pops/functions/dispatch' require 'puppet/pops/functions/dispatcher' end end require 'puppet/parser/ast/pops_bridge' require 'puppet/bindings' require 'puppet/functions' end diff --git a/lib/puppet/pops/evaluator/collector_transformer.rb b/lib/puppet/pops/evaluator/collector_transformer.rb index 6aa5cff52..f7de1e830 100644 --- a/lib/puppet/pops/evaluator/collector_transformer.rb +++ b/lib/puppet/pops/evaluator/collector_transformer.rb @@ -1,193 +1,191 @@ class Puppet::Pops::Evaluator::CollectorTransformer def initialize @@query_visitor ||= Puppet::Pops::Visitor.new(nil, "query", 1, 1) @@match_visitor ||= Puppet::Pops::Visitor.new(nil, "match", 1, 1) @@evaluator ||= Puppet::Pops::Evaluator::EvaluatorImpl.new @@compare_operator ||= Puppet::Pops::Evaluator::CompareOperator.new() end def query(o, scope) @@query_visitor.visit_this(self, o, scope) end def match(o, scope) @@match_visitor.visit_this(self, o, scope) end def transform(o, scope) raise ArgumentError, "Expected CollectExpression" unless o.is_a? Puppet::Pops::Model::CollectExpression raise "LHS is not a type" unless o.type_expr.is_a? Puppet::Pops::Model::QualifiedReference type = o.type_expr.value().downcase() args = { :type => type } if type == 'class' fail "Classes cannot be collected" end resource_type = scope.find_resource_type(type) fail "Resource type #{type} doesn't exist" unless resource_type - form = - case o.query - when Puppet::Pops::Model::VirtualQuery - :virtual - when Puppet::Pops::Model::ExportedQuery - :exported - end - if o.query.expr.nil? || o.query.expr.is_a?(Puppet::Pops::Model::Nop) code = nil match = nil else code = query(o.query.expr, scope) match = match(o.query.expr, scope) end - newcoll = Puppet::Parser::Collector.new(scope, resource_type.name, match, code, form) - scope.compiler.add_collection(newcoll) - adapter = Puppet::Pops::Adapters::SourcePosAdapter.adapt(o) line_num = adapter.line position = adapter.pos file_path = adapter.locator.file # overrides if any # Evaluate all of the specified params. if !o.operations.empty? - newcoll.add_override( + overrides = { :parameters => o.operations.map{ |x| to_3x_param(x).evaluate(scope)}, :file => file_path, :line => [line_num, position], :source => scope.source, :scope => scope - ) + } end + case o.query + when Puppet::Pops::Model::VirtualQuery + newcoll = Puppet::Pops::Evaluator::Collectors::CatalogCollector.new(scope, resource_type.name, code, overrides) + when Puppet::Pops::Model::ExportedQuery + newcoll = Puppet::Pops::Evaluator::Collectors::ExportedCollector.new(scope, resource_type.name, match, code, overrides) + end + + scope.compiler.add_collection(newcoll) + newcoll end def query_AndExpression(o, scope) left_code = query(o.left_expr, scope) right_code = query(o.right_expr, scope) proc do |resource| left_code.call(resource) && right_code.call(resource) end end def query_OrExpression(o, scope) left_code = query(o.left_expr, scope) right_code = query(o.right_expr, scope) proc do |resource| left_code.call(resource) || right_code.call(resource) end end def query_ComparisonExpression(o, scope) left_code = query(o.left_expr, scope) right_code = query(o.right_expr, scope) proc do |resource| case o.operator when :'==' if left_code == "tag" resource.tagged?(right_code) else if resource[left_code].is_a?(Array) @@compare_operator.include?(resource[left_code], right_code, scope) else @@compare_operator.equals(resource[left_code], right_code) end end when :'!=' !@@compare_operator.equals(resource[left_code], right_code) end end end def query_VariableExpression(o, scope) @@evaluator.evaluate(o, scope) end def query_LiteralBoolean(o, scope) @@evaluator.evaluate(o, scope) end def query_LiteralString(o, scope) @@evaluator.evaluate(o, scope) end def query_LiteralNumber(o, scope) @@evaluator.evaluate(o, scope) end def query_QualifiedName(o, scope) @@evaluator.evaluate(o, scope) end def query_ParenthesizedExpression(o, scope) query(o.expr, scope) end def query_Object(o, scope) raise ArgumentError, "Cannot transform object of class #{o.class}" end def match_AndExpression(o, scope) left_match = match(o.left_expr, scope) right_match = match(o.right_expr, scope) return [left_match, 'and', right_match] end def match_OrExpression(o, scope) left_match = match(o.left_expr, scope) right_match = match(o.right_expr, scope) return [left_match, 'or', right_match] end def match_ComparisonExpression(o, scope) left_match = match(o.left_expr, scope) right_match = match(o.right_expr, scope) return [left_match, o.operator.to_s, right_match] end def match_VariableExpression(o, scope) @@evaluator.evaluate(o, scope) end def match_LiteralBoolean(o, scope) @@evaluator.evaluate(o, scope) end def match_LiteralString(o, scope) @@evaluator.evaluate(o, scope) end def match_LiteralNumber(o, scope) @@evaluator.evaluate(o, scope) end def match_QualifiedName(o, scope) @@evaluator.evaluate(o, scope) end def match_ParenthesizedExpression(o, scope) match(o.expr, scope) end def match_Object(o, scope) raise ArgumentError, "Cannot transform object of class #{o.class}" end # Produces (name => expr) or (name +> expr) def to_3x_param(o) bridge = Puppet::Parser::AST::PopsBridge::Expression.new(:value => o.value_expr) args = { :value => bridge } #TODO: May delete this line later args[:add] = true if o.operator == :'+>' args[:param] = o.attribute_name args= Puppet::Pops::Model::AstTransformer.new().merge_location(args, o) Puppet::Parser::AST::ResourceParam.new(args) end end diff --git a/lib/puppet/pops/evaluator/collectors/abstract_collector.rb b/lib/puppet/pops/evaluator/collectors/abstract_collector.rb new file mode 100644 index 000000000..e3bf531b4 --- /dev/null +++ b/lib/puppet/pops/evaluator/collectors/abstract_collector.rb @@ -0,0 +1,71 @@ +# Abstract class +class Puppet::Pops::Evaluator::Collectors::AbstractCollector + attr_reader :scope + # The collector's hash of overrides {:parameters => params} + attr_reader :overrides + # The set of collected resources + attr_reader :collected + + EMPTY_RESOURCES = [].frozen + + #:parameters => overrides[:parameters], + #:file => overrides[:file], + #:line => overrides[:line], + #:source => overrides[:source], + #:scope => overrides[:scope]) + def initialize(scope, overrides = nil) + @collected = {} + @scope = scope + + if !(overrides.nil? || overrides[:parameters]) + raise ArgumentError, "Exported resource try to override without parameters" + end + + @overrides = overrides + end + + def evaluate + + objects = collect.each do |obj| + obj.virtual = false + end + + return false if objects.empty? + + # we have an override for the collected resources + if @overrides and !objects.empty? + # force the resource to be always child of any other resource + overrides[:source].meta_def(:child_of?) do |klass| + true + end + + # tell the compiler we have some override for it unless we already + # overrided those resources + objects.each do |res| + unless @collected.include?(res.ref) + newres = Puppet::Parser::Resource.new(res.type, res.title, @overrides) + scope.compiler.add_override(newres) + end + end + end + + # filter out object that this collector has previously found. + objects.reject! { |o| @collected.include?(o.ref) } + + return false if objects.empty? + + # keep an eye on the resources we have collected + objects.inject(@collected) { |c,o| c[o.ref]=o; c } + + # return our newly collected resources + objects + end + + def unresolved_resources + EMPTY_RESOURCES + end + + def collect + raise NotImplementedError, "This method must be implemented by the child class" + end +end diff --git a/lib/puppet/pops/evaluator/collectors/catalog_collector.rb b/lib/puppet/pops/evaluator/collectors/catalog_collector.rb new file mode 100644 index 000000000..1af7df08e --- /dev/null +++ b/lib/puppet/pops/evaluator/collectors/catalog_collector.rb @@ -0,0 +1,22 @@ +class Puppet::Pops::Evaluator::Collectors::CatalogCollector < Puppet::Pops::Evaluator::Collectors::AbstractCollector + + def initialize(scope, type, query, overrides = nil) + super(scope, overrides) + + @query = query + + # Canonize the type + #TODO: Refactor + @type = Puppet::Resource.new(type, "whatever").type + end + + # Collect just virtual objects, from our local compiler. + def collect + t = @type + q = @query + + scope.compiler.resources.find_all do |resource| + resource.type == t && (q ? q.call(resource) : true) + end + end +end diff --git a/lib/puppet/pops/evaluator/collectors/exported_collector.rb b/lib/puppet/pops/evaluator/collectors/exported_collector.rb new file mode 100644 index 000000000..cbcd3287e --- /dev/null +++ b/lib/puppet/pops/evaluator/collectors/exported_collector.rb @@ -0,0 +1,62 @@ +class Puppet::Pops::Evaluator::Collectors::ExportedCollector < Puppet::Pops::Evaluator::Collectors::AbstractCollector + + def initialize(scope, type, equery, cquery, overrides = nil) + super(scope, overrides) + + @equery = equery + @cquery = cquery + + # Canonize the type + @type = Puppet::Resource.new(type, "whatever").type + end + + def evaluate + if Puppet[:storeconfigs] != true + Puppet.warning "Not collecting exported resources without storeconfigs" + return false + end + + super + end + + # Collect exported objects. + def collect + resources = [] + + time = Puppet::Util.thinmark do + # First get everything that is exported from the catalog + t = @type + q = @cquery + + scope.compiler.resources.find_all do |resource| + resource.type == t && resource.exported? && q.call(resource) + end + + # key is '#{type}/#{name}', and host and filter. + found = Puppet::Resource.indirection. + search(@type, :host => @scope.compiler.node.name, :filter => @equery, :scope => @scope) + + found_resources = found.map {|x| x.is_a?(Puppet::Parser::Resource) ? x : x.to_resource(@scope)} + + found_resources.each do |item| + if existing = @scope.findresource(item.type, item.title) + unless existing.collector_id == item.collector_id + # unless this is the one we've already collected + raise Puppet::ParseError, + "A duplicate resource was found while collecting exported resources, with the type and title #{item.ref}" + end + else + item.exported = false + @scope.compiler.add_resource(@scope, item) + resources << item + end + end + end + + scope.debug("Collected %s %s resource%s in %.2f seconds" % + [resources.length, @type, resources.length == 1 ? "" : "s", time]) + + resources + end + +end diff --git a/lib/puppet/pops/evaluator/collectors/fixed_set_collector.rb b/lib/puppet/pops/evaluator/collectors/fixed_set_collector.rb new file mode 100644 index 000000000..a6d8277cc --- /dev/null +++ b/lib/puppet/pops/evaluator/collectors/fixed_set_collector.rb @@ -0,0 +1,31 @@ +class Puppet::Pops::Evaluator::Collectors::FixedSetCollector < Puppet::Pops::Evaluator::Collectors::AbstractCollector + + def initialize(scope, resources) + super(scope) + @resources = resources.is_a?(Array)? resources.dup : [resources] + end + + def collect + resolved = [] + result = @resources.reduce([]) do |memo, ref| + if res = @scope.findresource(ref.to_s) + res.virtual = false + memo << res + resolved << ref + end + memo + end + + @resources = @resources - resolved + + # If there are no more resources to find, delete this from the list + # of collections. + @scope.compiler.delete_collection(self) if @resources.empty? + + result + end + + def unresolved_resources + @resources + end +end diff --git a/lib/puppet/pops/evaluator/relationship_operator.rb b/lib/puppet/pops/evaluator/relationship_operator.rb index 866afa8ef..1731870af 100644 --- a/lib/puppet/pops/evaluator/relationship_operator.rb +++ b/lib/puppet/pops/evaluator/relationship_operator.rb @@ -1,157 +1,161 @@ # The RelationshipOperator implements the semantics of the -> <- ~> <~ operators creating relationships or notification # relationships between the left and right hand side's references to resources. # # This is separate class since a second level of evaluation is required that transforms string in left or right hand # to type references. The task of "making a relationship" is delegated to the "runtime support" class that is included. # This is done to separate the concerns of the new evaluator from the 3x runtime; messy logic goes into the runtime support # module. Later when more is cleaned up this can be simplified further. # class Puppet::Pops::Evaluator::RelationshipOperator # Provides access to the Puppet 3.x runtime (scope, etc.) # This separation has been made to make it easier to later migrate the evaluator to an improved runtime. # include Puppet::Pops::Evaluator::Runtime3Support Issues = Puppet::Pops::Issues class IllegalRelationshipOperandError < RuntimeError attr_reader :operand def initialize operand @operand = operand end end class NotCatalogTypeError < RuntimeError attr_reader :type def initialize type @type = type end end def initialize @type_transformer_visitor = Puppet::Pops::Visitor.new(self, "transform", 1, 1) @type_calculator = Puppet::Pops::Types::TypeCalculator.new() @type_parser = Puppet::Pops::Types::TypeParser.new() tf = Puppet::Pops::Types::TypeFactory @catalog_type = tf.variant(tf.catalog_entry, tf.type_type(tf.catalog_entry)) end def transform(o, scope) @type_transformer_visitor.visit_this_1(self, o, scope) end # Catch all non transformable objects # @api private def transform_Object(o, scope) raise IllegalRelationshipOperandError.new(o) end # A string must be a type reference in string format # @api private def transform_String(o, scope) assert_catalog_type(@type_parser.parse(o), scope) end # A qualified name is short hand for a class with this name # @api private def transform_QualifiedName(o, scope) Puppet::Pops::Types::TypeFactory.host_class(o.value) end # Types are what they are, just check the type # @api private def transform_PAnyType(o, scope) assert_catalog_type(o, scope) end # This transforms a 3x Collector (the result of evaluating a 3x AST::Collection). # It is passed through verbatim since it is evaluated late by the compiler. At the point # where the relationship is evaluated, it is simply recorded with the compiler for later evaluation. # If one of the sides of the relationship is a Collector it is evaluated before the actual # relationship is formed. (All of this happens at a later point in time. # def transform_Collector(o, scope) o end + def transform_AbstractCollector(o, scope) + o + end + # Array content needs to be transformed def transform_Array(o, scope) o.map{|x| transform(x, scope) } end # Asserts (and returns) the type if it is a PCatalogEntryType # (A PCatalogEntryType is the base class of PHostClassType, and PResourceType). # def assert_catalog_type(o, scope) unless @type_calculator.assignable?(@catalog_type, o) raise NotCatalogTypeError.new(o) end # TODO must check if this is an abstract PResourceType (i.e. without a type_name) - which should fail ? # e.g. File -> File (and other similar constructs) - maybe the catalog protects against this since references # may be to future objects... o end RELATIONSHIP_OPERATORS = [:'->', :'~>', :'<-', :'<~'] REVERSE_OPERATORS = [:'<-', :'<~'] RELATION_TYPE = { :'->' => :relationship, :'<-' => :relationship, :'~>' => :subscription, :'<~' => :subscription } # Evaluate a relationship. # TODO: The error reporting is not fine grained since evaluation has already taken place # There is no references to the original source expressions at this point, only the overall # relationship expression. (e.g.. the expression may be ['string', func_call(), etc.] -> func_call()) # To implement this, the general evaluator needs to be able to track each evaluation result and associate # it with a corresponding expression. This structure should then be passed to the relationship operator. # def evaluate (left_right_evaluated, relationship_expression, scope) # assert operator (should have been validated, but this logic makes assumptions which would # screw things up royally). Better safe than sorry. unless RELATIONSHIP_OPERATORS.include?(relationship_expression.operator) fail(Issues::UNSUPPORTED_OPERATOR, relationship_expression, {:operator => relationship_expression.operator}) end begin # Turn each side into an array of types (this also asserts their type) # (note wrap in array first if value is not already an array) # # TODO: Later when objects are Puppet Runtime Objects and know their type, it will be more efficient to check/infer # the type first since a chained operation then does not have to visit each element again. This is not meaningful now # since inference needs to visit each object each time, and this is what the transformation does anyway). # # real is [left, right], and both the left and right may be a single value or an array. In each case all content # should be flattened, and then transformed to a type. left or right may also be a value that is transformed # into an array, and thus the resulting left and right must be flattened individually # Once flattened, the operands should be sets (to remove duplicate entries) # real = left_right_evaluated.collect {|x| [x].flatten.collect {|x| transform(x, scope) }} real[0].flatten! real[1].flatten! real[0].uniq! real[1].uniq! # reverse order if operator is Right to Left source, target = reverse_operator?(relationship_expression) ? real.reverse : real # Add the relationships to the catalog source.each {|s| target.each {|t| add_relationship(s, t, RELATION_TYPE[relationship_expression.operator], scope) }} # Produce the transformed source RHS (if this is a chain, this does not need to be done again) real.slice(1) rescue NotCatalogTypeError => e fail(Issues::ILLEGAL_RELATIONSHIP_OPERAND_TYPE, relationship_expression, {:type => @type_calculator.string(e.type)}) rescue IllegalRelationshipOperandError => e fail(Issues::ILLEGAL_RELATIONSHIP_OPERAND_TYPE, relationship_expression, {:operand => e.operand}) end end def reverse_operator?(o) REVERSE_OPERATORS.include?(o.operator) end end diff --git a/lib/puppet/pops/evaluator/runtime3_support.rb b/lib/puppet/pops/evaluator/runtime3_support.rb index a86228d33..318e11645 100644 --- a/lib/puppet/pops/evaluator/runtime3_support.rb +++ b/lib/puppet/pops/evaluator/runtime3_support.rb @@ -1,599 +1,599 @@ # A module with bindings between the new evaluator and the 3x runtime. # The intention is to separate all calls into scope, compiler, resource, etc. in this module # to make it easier to later refactor the evaluator for better implementations of the 3x classes. # # @api private module Puppet::Pops::Evaluator::Runtime3Support NAME_SPACE_SEPARATOR = '::'.freeze # Fails the evaluation of _semantic_ with a given issue. # # @param issue [Puppet::Pops::Issue] the issue to report # @param semantic [Puppet::Pops::ModelPopsObject] the object for which evaluation failed in some way. Used to determine origin. # @param options [Hash] hash of optional named data elements for the given issue # @return [!] this method does not return # @raise [Puppet::ParseError] an evaluation error initialized from the arguments (TODO: Change to EvaluationError?) # def fail(issue, semantic, options={}, except=nil) optionally_fail(issue, semantic, options, except) # an error should have been raised since fail always fails raise ArgumentError, "Internal Error: Configuration of runtime error handling wrong: should have raised exception" end # Optionally (based on severity) Fails the evaluation of _semantic_ with a given issue # If the given issue is configured to be of severity < :error it is only reported, and the function returns. # # @param issue [Puppet::Pops::Issue] the issue to report # @param semantic [Puppet::Pops::ModelPopsObject] the object for which evaluation failed in some way. Used to determine origin. # @param options [Hash] hash of optional named data elements for the given issue # @return [!] this method does not return # @raise [Puppet::ParseError] an evaluation error initialized from the arguments (TODO: Change to EvaluationError?) # def optionally_fail(issue, semantic, options={}, except=nil) if except.nil? # Want a stacktrace, and it must be passed as an exception begin raise EvaluationError.new() rescue EvaluationError => e except = e end end diagnostic_producer.accept(issue, semantic, options, except) end # Binds the given variable name to the given value in the given scope. # The reference object `o` is intended to be used for origin information - the 3x scope implementation # only makes use of location when there is an error. This is now handled by other mechanisms; first a check # is made if a variable exists and an error is raised if attempting to change an immutable value. Errors # in name, numeric variable assignment etc. have also been validated prior to this call. In the event the # scope.setvar still raises an error, the general exception handling for evaluation of the assignment # expression knows about its location. Because of this, there is no need to extract the location for each # setting (extraction is somewhat expensive since 3x requires line instead of offset). # def set_variable(name, value, o, scope) # Scope also checks this but requires that location information are passed as options. # Those are expensive to calculate and a test is instead made here to enable failing with better information. # The error is not specific enough to allow catching it - need to check the actual message text. # TODO: Improve the messy implementation in Scope. # if scope.bound?(name) if Puppet::Parser::Scope::RESERVED_VARIABLE_NAMES.include?(name) fail(Puppet::Pops::Issues::ILLEGAL_RESERVED_ASSIGNMENT, o, {:name => name} ) else fail(Puppet::Pops::Issues::ILLEGAL_REASSIGNMENT, o, {:name => name} ) end end scope.setvar(name, value) end # Returns the value of the variable (nil is returned if variable has no value, or if variable does not exist) # def get_variable_value(name, o, scope) # Puppet 3x stores all variables as strings (then converts them back to numeric with a regexp... to see if it is a match variable) # Not ideal, scope should support numeric lookup directly instead. # TODO: consider fixing scope catch(:undefined_variable) { x = scope.lookupvar(name.to_s) # Must convert :undef back to nil - this can happen when an undefined variable is used in a # parameter's default value expression - there nil must be :undef to work with the rest of 3x. # Now that the value comes back to 4x it is changed to nil. return (x == :undef) ? nil : x } # It is always ok to reference numeric variables even if they are not assigned. They are always undef # if not set by a match expression. # unless name =~ Puppet::Pops::Patterns::NUMERIC_VAR_NAME fail(Puppet::Pops::Issues::UNKNOWN_VARIABLE, o, {:name => name}) end end # Returns true if the variable of the given name is set in the given most nested scope. True is returned even if # variable is bound to nil. # def variable_bound?(name, scope) scope.bound?(name.to_s) end # Returns true if the variable is bound to a value or nil, in the scope or it's parent scopes. # def variable_exists?(name, scope) scope.exist?(name.to_s) end def set_match_data(match_data, scope) # See set_variable for rationale for not passing file and line to ephemeral_from. # NOTE: The 3x scope adds one ephemeral(match) to its internal stack per match that succeeds ! It never # clears anything. Thus a context that performs many matches will get very deep (there simply is no way to # clear the match variables without rolling back the ephemeral stack.) # This implementation does not attempt to fix this, it behaves the same bad way. unless match_data.nil? scope.ephemeral_from(match_data) end end # Creates a local scope with vairalbes set from a hash of variable name to value # def create_local_scope_from(hash, scope) # two dummy values are needed since the scope tries to give an error message (can not happen in this # case - it is just wrong, the error should be reported by the caller who knows in more detail where it # is in the source. # raise ArgumentError, "Internal error - attempt to create a local scope without a hash" unless hash.is_a?(Hash) scope.ephemeral_from(hash) end # Creates a nested match scope def create_match_scope_from(scope) # Create a transparent match scope (for future matches) scope.new_match_scope(nil) end def get_scope_nesting_level(scope) scope.ephemeral_level end def set_scope_nesting_level(scope, level) # Yup, 3x uses this method to reset the level, it also supports passing :all to destroy all # ephemeral/local scopes - which is a sure way to create havoc. # scope.unset_ephemeral_var(level) end # Adds a relationship between the given `source` and `target` of the given `relationship_type` # @param source [Puppet:Pops::Types::PCatalogEntryType] the source end of the relationship (from) # @param target [Puppet:Pops::Types::PCatalogEntryType] the target end of the relationship (to) # @param relationship_type [:relationship, :subscription] the type of the relationship # def add_relationship(source, target, relationship_type, scope) # The 3x way is to record a Puppet::Parser::Relationship that is evaluated at the end of the compilation. # This means it is not possible to detect any duplicates at this point (and signal where an attempt is made to # add a duplicate. There is also no location information to signal the original place in the logic. The user will have # to go fish. # The 3.x implementation is based on Strings :-o, so the source and target must be transformed. The resolution is # done by Catalog#resource(type, title). To do that, it creates a Puppet::Resource since it is responsible for # translating the name/type/title and create index-keys used by the catalog. The Puppet::Resource has bizarre parsing of # the type and title (scan for [] that is interpreted as type/title (but it gets it wrong). # Moreover if the type is "" or "component", the type is Class, and if the type is :main, it is :main, all other cases # undergo capitalization of name-segments (foo::bar becomes Foo::Bar). (This was earlier done in the reverse by the parser). # Further, the title undergoes the same munging !!! # # That bug infested nest of messy logic needs serious Exorcism! # # Unfortunately it is not easy to simply call more intelligent methods at a lower level as the compiler evaluates the recorded # Relationship object at a much later point, and it is responsible for invoking all the messy logic. # # TODO: Revisit the below logic when there is a sane implementation of the catalog, compiler and resource. For now # concentrate on transforming the type references to what is expected by the wacky logic. # # HOWEVER, the Compiler only records the Relationships, and the only method it calls is @relationships.each{|x| x.evaluate(catalog) } # Which means a smarter Relationship class could do this right. Instead of obtaining the resource from the catalog using # the borked resource(type, title) which creates a resource for the purpose of looking it up, it needs to instead # scan the catalog's resources # # GAAAH, it is even worse! # It starts in the parser, which parses "File['foo']" into an AST::ResourceReference with type = File, and title = foo # This AST is evaluated by looking up the type/title in the scope - causing it to be loaded if it exists, and if not, the given # type name/title is used. It does not search for resource instances, only classes and types. It returns symbolic information # [type, [title, title]]. From this, instances of Puppet::Resource are created and returned. These only have type/title information # filled out. One or an array of resources are returned. # This set of evaluated (empty reference) Resource instances are then passed to the relationship operator. It creates a # Puppet::Parser::Relationship giving it a source and a target that are (empty reference) Resource instances. These are then remembered # until the relationship is evaluated by the compiler (at the end). When evaluation takes place, the (empty reference) Resource instances # are converted to String (!?! WTF) on the simple format "#{type}[#{title}]", and the catalog is told to find a resource, by giving # it this string. If it cannot find the resource it fails, else the before/notify parameter is appended with the target. # The search for the resource begin with (you guessed it) again creating an (empty reference) resource from type and title (WTF?!?!). # The catalog now uses the reference resource to compute a key [r.type, r.title.to_s] and also gets a uniqueness key from the # resource (This is only a reference type created from title and type). If it cannot find it with the first key, it uses the # uniqueness key to lookup. # # This is probably done to allow a resource type to munge/translate the title in some way (but it is quite unclear from the long # and convoluted path of evaluation. # In order to do this in a way that is similar to 3.x two resources are created to be used as keys. # # And if that is not enough, a source/target may be a Collector (a baked query that will be evaluated by the # compiler - it is simply passed through here for processing by the compiler at the right time). # - if source.is_a?(Puppet::Parser::Collector) + if source.is_a?(Puppet::Parser::Collector) || source.is_a?(Puppet::Pops::Evaluator::Collectors::AbstractCollector) # use verbatim - behavior defined by 3x source_resource = source else # transform into the wonderful String representation in 3x type, title = catalog_type_to_split_type_title(source) source_resource = Puppet::Resource.new(type, title) end - if target.is_a?(Puppet::Parser::Collector) + if target.is_a?(Puppet::Parser::Collector) || target.is_a?(Puppet::Pops::Evaluator::Collectors::AbstractCollector) # use verbatim - behavior defined by 3x target_resource = target else # transform into the wonderful String representation in 3x type, title = catalog_type_to_split_type_title(target) target_resource = Puppet::Resource.new(type, title) end # Add the relationship to the compiler for later evaluation. scope.compiler.add_relationship(Puppet::Parser::Relationship.new(source_resource, target_resource, relationship_type)) end # Coerce value `v` to numeric or fails. # The given value `v` is coerced to Numeric, and if that fails the operation # calls {#fail}. # @param v [Object] the value to convert # @param o [Object] originating instruction # @param scope [Object] the (runtime specific) scope where evaluation of o takes place # @return [Numeric] value `v` converted to Numeric. # def coerce_numeric(v, o, scope) unless n = Puppet::Pops::Utils.to_n(v) fail(Puppet::Pops::Issues::NOT_NUMERIC, o, {:value => v}) end n end def call_function(name, args, o, scope) # Call via 4x API if the function exists there loaders = scope.compiler.loaders # find the loader that loaded the code, or use the private_environment_loader (sees env + all modules) adapter = Puppet::Pops::Utils.find_adapter(o, Puppet::Pops::Adapters::LoaderAdapter) loader = adapter.nil? ? loaders.private_environment_loader : adapter.loader if loader && func = loader.load(:function, name) return func.call(scope, *args) end # Call via 3x API if function exists there fail(Puppet::Pops::Issues::UNKNOWN_FUNCTION, o, {:name => name}) unless Puppet::Parser::Functions.function(name) # Arguments must be mapped since functions are unaware of the new and magical creatures in 4x. # NOTE: Passing an empty string last converts nil/:undef to empty string mapped_args = args.map {|a| convert(a, scope, '') } result = scope.send("function_#{name}", mapped_args) # Prevent non r-value functions from leaking their result (they are not written to care about this) Puppet::Parser::Functions.rvalue?(name) ? result : nil end # The o is used for source reference def create_resource_parameter(o, scope, name, value, operator) file, line = extract_file_line(o) Puppet::Parser::Resource::Param.new( :name => name, :value => convert(value, scope, nil), # converted to 3x since 4x supports additional objects / types :source => scope.source, :line => line, :file => file, :add => operator == :'+>' ) end CLASS_STRING = 'class'.freeze def create_resources(o, scope, virtual, exported, type_name, resource_titles, evaluated_parameters) # TODO: Unknown resource causes creation of Resource to fail with ArgumentError, should give # a proper Issue. Now the result is "Error while evaluating a Resource Statement" with the message # from the raised exception. (It may be good enough). # resolve in scope. fully_qualified_type, resource_titles = scope.resolve_type_and_titles(type_name, resource_titles) # Not 100% accurate as this is the resource expression location and each title is processed separately # The titles are however the result of evaluation and they have no location at this point (an array # of positions for the source expressions are required for this to work). # TODO: Revisit and possible improve the accuracy. # file, line = extract_file_line(o) # Build a resource for each title resource_titles.map do |resource_title| resource = Puppet::Parser::Resource.new( fully_qualified_type, resource_title, :parameters => evaluated_parameters, :file => file, :line => line, :exported => exported, :virtual => virtual, # WTF is this? Which source is this? The file? The name of the context ? :source => scope.source, :scope => scope, :strict => true ) if resource.resource_type.is_a? Puppet::Resource::Type resource.resource_type.instantiate_resource(scope, resource) end scope.compiler.add_resource(scope, resource) scope.compiler.evaluate_classes([resource_title], scope, false, true) if fully_qualified_type == CLASS_STRING # Turn the resource into a PType (a reference to a resource type) # weed out nil's resource_to_ptype(resource) end end # Defines default parameters for a type with the given name. # def create_resource_defaults(o, scope, type_name, evaluated_parameters) # Note that name must be capitalized in this 3x call # The 3x impl creates a Resource instance with a bogus title and then asks the created resource # for the type of the name. # Note, locations are available per parameter. # scope.define_settings(capitalize_qualified_name(type_name), evaluated_parameters) end # Capitalizes each segment of a qualified name # def capitalize_qualified_name(name) name.split(/::/).map(&:capitalize).join(NAME_SPACE_SEPARATOR) end # Creates resource overrides for all resource type objects in evaluated_resources. The same set of # evaluated parameters are applied to all. # def create_resource_overrides(o, scope, evaluated_resources, evaluated_parameters) # Not 100% accurate as this is the resource expression location and each title is processed separately # The titles are however the result of evaluation and they have no location at this point (an array # of positions for the source expressions are required for this to work. # TODO: Revisit and possible improve the accuracy. # file, line = extract_file_line(o) evaluated_resources.each do |r| unless r.is_a?(Puppet::Pops::Types::PResourceType) && r.type_name != 'class' fail(Puppet::Pops::Issues::ILLEGAL_OVERRIDEN_TYPE, o, {:actual => r} ) end resource = Puppet::Parser::Resource.new( r.type_name, r.title, :parameters => evaluated_parameters, :file => file, :line => line, # WTF is this? Which source is this? The file? The name of the context ? :source => scope.source, :scope => scope ) scope.compiler.add_override(resource) end end # Finds a resource given a type and a title. # def find_resource(scope, type_name, title) scope.compiler.findresource(type_name, title) end # Returns the value of a resource's parameter by first looking up the parameter in the resource # and then in the defaults for the resource. Since the resource exists (it must in order to look up its # parameters, any overrides have already been applied). Defaults are not applied to a resource until it # has been finished (which typically has not taken place when this is evaluated; hence the dual lookup). # def get_resource_parameter_value(scope, resource, parameter_name) # This gets the parameter value, or nil (for both valid parameters and parameters that do not exist). val = resource[parameter_name] # Sometimes the resource is a Puppet::Parser::Resource and sometimes it is # a Puppet::Resource. The Puppet::Resource case occurs when puppet language # is evaluated against an already completed catalog (where all instances of # Puppet::Parser::Resource are converted to Puppet::Resource instances). # Evaluating against an already completed catalog is really only found in # the language specification tests, where the puppet language is used to # test itself. if resource.is_a?(Puppet::Parser::Resource) # The defaults must be looked up in the scope where the resource was created (not in the given # scope where the lookup takes place. resource_scope = resource.scope if val.nil? && resource_scope && defaults = resource_scope.lookupdefaults(resource.type) # NOTE: 3x resource keeps defaults as hash using symbol for name as key to Parameter which (again) holds # name and value. # NOTE: meta parameters that are unset ends up here, and there are no defaults for those encoded # in the defaults, they may receive hardcoded defaults later (e.g. 'tag'). param = defaults[parameter_name.to_sym] # Some parameters (meta parameters like 'tag') does not return a param from which the value can be obtained # at all times. Instead, they return a nil param until a value has been set. val = param.nil? ? nil : param.value end end val end # Returns true, if the given name is the name of a resource parameter. # def is_parameter_of_resource?(scope, resource, name) return false unless name.is_a?(String) resource.valid_parameter?(name) end def resource_to_ptype(resource) nil if resource.nil? # inference returns the meta type since the 3x Resource is an alternate way to describe a type type_calculator.infer(resource).type end # This is the same type of "truth" as used in the current Puppet DSL. # def is_true? o # Is the value true? This allows us to control the definition of truth # in one place. case o # Support :undef since it may come from a 3x structure when :undef false else !!o end end # Utility method for TrueClass || FalseClass # @param x [Object] the object to test if it is instance of TrueClass or FalseClass def is_boolean? x x.is_a?(TrueClass) || x.is_a?(FalseClass) end def initialize @@convert_visitor ||= Puppet::Pops::Visitor.new(self, "convert", 2, 2) @@convert2_visitor ||= Puppet::Pops::Visitor.new(self, "convert2", 2, 2) end # Converts 4x supported values to 3x values. This is required because # resources and other objects do not know about the new type system, and does not support # regular expressions. Unfortunately this has to be done for array and hash as well. # A complication is that catalog types needs to be resolved against the scope. # def convert(o, scope, undef_value) @@convert_visitor.visit_this_2(self, o, scope, undef_value) end # Converts nested 4x supported values to 3x values. This is required because # resources and other objects do not know about the new type system, and does not support # regular expressions. Unfortunately this has to be done for array and hash as well. # A complication is that catalog types needs to be resolved against the scope. # def convert2(o, scope, undef_value) @@convert2_visitor.visit_this_2(self, o, scope, undef_value) end def convert_NilClass(o, scope, undef_value) undef_value end def convert2_NilClass(o, scope, undef_value) :undef end def convert_String(o, scope, undef_value) # although wasteful, needed because user code may mutate these strings in Resources o.frozen? ? o.dup : o end alias convert2_String :convert_String def convert_Object(o, scope, undef_value) o end alias :convert2_Object :convert_Object def convert_Array(o, scope, undef_value) o.map {|x| convert2(x, scope, undef_value) } end alias :convert2_Array :convert_Array def convert_Hash(o, scope, undef_value) result = {} o.each {|k,v| result[convert2(k, scope, undef_value)] = convert2(v, scope, undef_value) } result end alias :convert2_Hash :convert_Hash def convert_Regexp(o, scope, undef_value) # Puppet 3x cannot handle parameter values that are reqular expressions. Turn into regexp string in # source form o.inspect end alias :convert2_Regexp :convert_Regexp def convert_Symbol(o, scope, undef_value) case o # Support :undef since it may come from a 3x structure when :undef undef_value # 3x wants undef as either empty string or :undef else o # :default, and all others are verbatim since they are new in future evaluator end end # The :undef symbol should not be converted when nested in arrays or hashes def convert2_Symbol(o, scope, undef_value) o end def convert_PAnyType(o, scope, undef_value) o end alias :convert2_PAnyType :convert_PAnyType def convert_PCatalogEntryType(o, scope, undef_value) # Since 4x does not support dynamic scoping, all names are absolute and can be # used as is (with some check/transformation/mangling between absolute/relative form # due to Puppet::Resource's idiosyncratic behavior where some references must be # absolute and others cannot be. # Thus there is no need to call scope.resolve_type_and_titles to do dynamic lookup. Puppet::Resource.new(*catalog_type_to_split_type_title(o)) end alias :convert2_PCatalogEntryType :convert_PCatalogEntryType private # Produces an array with [type, title] from a PCatalogEntryType # This method is used to produce the arguments for creation of reference resource instances # (used when 3x is operating on a resource). # Ensures that resources are *not* absolute. # def catalog_type_to_split_type_title(catalog_type) split_type = catalog_type.is_a?(Puppet::Pops::Types::PType) ? catalog_type.type : catalog_type case split_type when Puppet::Pops::Types::PHostClassType class_name = split_type.class_name ['class', class_name.nil? ? nil : class_name.sub(/^::/, '')] when Puppet::Pops::Types::PResourceType type_name = split_type.type_name title = split_type.title if type_name =~ /^(::)?[Cc]lass/ ['class', title.nil? ? nil : title.sub(/^::/, '')] else # Ensure that title is '' if nil # Resources with absolute name always results in error because tagging does not support leading :: [type_name.nil? ? nil : type_name.sub(/^::/, ''), title.nil? ? '' : title] end else raise ArgumentError, "Cannot split the type #{catalog_type.class}, it represents neither a PHostClassType, nor a PResourceType." end end def extract_file_line(o) source_pos = Puppet::Pops::Utils.find_closest_positioned(o) return [nil, -1] unless source_pos [source_pos.locator.file, source_pos.line] end def find_closest_positioned(o) return nil if o.nil? || o.is_a?(Puppet::Pops::Model::Program) o.offset.nil? ? find_closest_positioned(o.eContainer) : Puppet::Pops::Adapters::SourcePosAdapter.adapt(o) end # Creates a diagnostic producer def diagnostic_producer Puppet::Pops::Validation::DiagnosticProducer.new( ExceptionRaisingAcceptor.new(), # Raises exception on all issues SeverityProducer.new(), # All issues are errors Puppet::Pops::Model::ModelLabelProvider.new()) end # Configure the severity of failures class SeverityProducer < Puppet::Pops::Validation::SeverityProducer Issues = Puppet::Pops::Issues def initialize super p = self # Issues triggering warning only if --debug is on if Puppet[:debug] p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :warning else p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :ignore end # Store config issues, ignore or warning p[Issues::RT_NO_STORECONFIGS_EXPORT] = Puppet[:storeconfigs] ? :ignore : :warning p[Issues::RT_NO_STORECONFIGS] = Puppet[:storeconfigs] ? :ignore : :warning end end # An acceptor of diagnostics that immediately raises an exception. class ExceptionRaisingAcceptor < Puppet::Pops::Validation::Acceptor def accept(diagnostic) super Puppet::Pops::IssueReporter.assert_and_report(self, {:message => "Evaluation Error:", :emit_warnings => true }) if errors? raise ArgumentError, "Internal Error: Configuration of runtime error handling wrong: should have raised exception" end end end class EvaluationError < StandardError end end