diff --git a/lib/puppet/loaders.rb b/lib/puppet/loaders.rb index 86bbc03c9..1880fabb2 100644 --- a/lib/puppet/loaders.rb +++ b/lib/puppet/loaders.rb @@ -1,20 +1,21 @@ module Puppet module Pops require 'puppet/pops/loaders' module Loader require 'puppet/pops/loader/loader' require 'puppet/pops/loader/base_loader' require 'puppet/pops/loader/gem_support' require 'puppet/pops/loader/module_loaders' require 'puppet/pops/loader/dependency_loader' require 'puppet/pops/loader/null_loader' require 'puppet/pops/loader/static_loader' require 'puppet/pops/loader/puppet_function_instantiator' require 'puppet/pops/loader/ruby_function_instantiator' require 'puppet/pops/loader/ruby_legacy_function_instantiator' require 'puppet/pops/loader/loader_paths' + require 'puppet/pops/loader/simple_environment_loader' end end end \ No newline at end of file diff --git a/lib/puppet/parser/ast/pops_bridge.rb b/lib/puppet/parser/ast/pops_bridge.rb index 401d6ede7..68d38142d 100644 --- a/lib/puppet/parser/ast/pops_bridge.rb +++ b/lib/puppet/parser/ast/pops_bridge.rb @@ -1,175 +1,212 @@ require 'puppet/parser/ast/top_level_construct' require 'puppet/pops' # The AST::Bridge contains classes that bridges between the new Pops based model # and the 3.x AST. This is required to be able to reuse the Puppet::Resource::Type which is # fundamental for the rest of the logic. # class Puppet::Parser::AST::PopsBridge # Bridges to one Pops Model Expression # The @value is the expression # This is used to represent the body of a class, definition, or node, and for each parameter's default value # expression. # class Expression < Puppet::Parser::AST::Leaf def initialize args super @@evaluator ||= Puppet::Pops::Parser::EvaluatingParser::Transitional.new() end def to_s Puppet::Pops::Model::ModelTreeDumper.new.dump(@value) end def evaluate(scope) @@evaluator.evaluate(scope, @value) end # Adapts to 3x where top level constructs needs to have each to iterate over children. Short circuit this # by yielding self. By adding this there is no need to wrap a pops expression inside an AST::BlockExpression # def each yield self end def sequence_with(other) if value.nil? # This happens when testing and not having a complete setup other else # When does this happen ? Ever ? raise "sequence_with called on Puppet::Parser::AST::PopsBridge::Expression - please report use case" # What should be done if the above happens (We don't want this to happen). # Puppet::Parser::AST::BlockExpression.new(:children => [self] + other.children) end end # The 3x requires code plugged in to an AST to have this in certain positions in the tree. The purpose # is to either print the content, or to look for things that needs to be defined. This implementation # cheats by always returning an empty array. (This allows simple files to not require a "Program" at the top. # def children [] end end class NilAsUndefExpression < Expression def evaluate(scope) result = super result.nil? ? :undef : result end end # Bridges the top level "Program" produced by the pops parser. # Its main purpose is to give one point where all definitions are instantiated (actually defined since the # Puppet 3x terminology is somewhat misleading - the definitions are instantiated, but instances of the created types # are not created, that happens when classes are included / required, nodes are matched and when resources are instantiated # by a resource expression (which is also used to instantiate a host class). # class Program < Puppet::Parser::AST::TopLevelConstruct attr_reader :program_model, :context def initialize(program_model, context = {}) @program_model = program_model @context = context @ast_transformer ||= Puppet::Pops::Model::AstTransformer.new(@context[:file]) @@evaluator ||= Puppet::Pops::Parser::EvaluatingParser::Transitional.new() end # This is the 3x API, the 3x AST searches through all code to find the instructions that can be instantiated. # This Pops-model based instantiation relies on the parser to build this list while parsing (which is more # efficient as it avoids one full scan of all logic via recursive enumeration/yield) # def instantiate(modname) @program_model.definitions.collect do |d| case d when Puppet::Pops::Model::HostClassDefinition instantiate_HostClassDefinition(d, modname) when Puppet::Pops::Model::ResourceTypeDefinition instantiate_ResourceTypeDefinition(d, modname) when Puppet::Pops::Model::NodeDefinition instantiate_NodeDefinition(d, modname) + when Puppet::Pops::Model::FunctionDefinition + instantiate_FunctionDefinition(d, modname) + # The 3x logic calling this will not know what to do with the result, it is compacted away at the end + next else - raise Puppet::ParseError("Internal Error: Unknown type of definition - got '#{d.class}'") + raise Puppet::ParseError, "Internal Error: Unknown type of definition - got '#{d.class}'" end - end.flatten() # flatten since node definition may have returned an array + end.flatten().compact() # flatten since node definition may have returned an array + # Compact since functions are not understood by compiler end def evaluate(scope) @@evaluator.evaluate(scope, program_model) end # Adapts to 3x where top level constructs needs to have each to iterate over children. Short circuit this # by yielding self. This means that the HostClass container will call this bridge instance with `instantiate`. # def each yield self end private def instantiate_Parameter(o) # 3x needs parameters as an array of `[name]` or `[name, value_expr]` # One problem is that the parameter evaluation takes place in the wrong context in 3x (the caller's and # can thus reference all sorts of information. Here the value expression is wrapped in an AST Bridge to a Pops # expression since the Pops side can not control the evaluation if o.value [ o.name, NilAsUndefExpression.new(:value => o.value) ] else [ o.name ] end end # Produces a hash with data for Definition and HostClass def args_from_definition(o, modname) args = { :arguments => o.parameters.collect {|p| instantiate_Parameter(p) }, :module_name => modname } unless is_nop?(o.body) args[:code] = Expression.new(:value => o.body) end @ast_transformer.merge_location(args, o) end def instantiate_HostClassDefinition(o, modname) args = args_from_definition(o, modname) args[:parent] = o.parent_class Puppet::Resource::Type.new(:hostclass, o.name, @context.merge(args)) end def instantiate_ResourceTypeDefinition(o, modname) Puppet::Resource::Type.new(:definition, o.name, @context.merge(args_from_definition(o, modname))) end def instantiate_NodeDefinition(o, modname) args = { :module_name => modname } unless is_nop?(o.body) args[:code] = Expression.new(:value => o.body) end unless is_nop?(o.parent) args[:parent] = @ast_transformer.hostname(o.parent) end host_matches = @ast_transformer.hostname(o.host_matches) @ast_transformer.merge_location(args, o) host_matches.collect do |name| Puppet::Resource::Type.new(:node, name, @context.merge(args)) end end + # Propagates a found Function to the appropriate loader. + # This is for 4x future-evaluator/loader + # + def instantiate_FunctionDefinition(function_definition, modname) + loaders = (Puppet.lookup(:loaders) { nil }) + unless loaders + raise Puppet::ParseError, "Internal Error: Puppet Context ':loaders' missing - cannot define any functions" + end + loader = + if modname.nil? || modname == "" + # TODO : Later when functions can be private, a decision is needed regarding what that means. + # A private environment loader could be used for logic outside of modules, then only that logic + # would see the function. + # + loaders.environment_loader() + else + # TODO : Later check if function is private, and then add it to + # private_loader_for_module + # + loaders.public_loader_for_module(modname) + end + unless loader + raise Puppet::ParseError, "Internal Error: did not find public loader for module: '#{modname}'" + end + + # Instantiate Function, and store it in the environment loader + typed_name, f = Puppet::Pops::Loader::PuppetFunctionInstantiator.create_from_model(function_definition, loader) + loader.set_entry(typed_name, f, Puppet::Pops::Adapters::SourcePosAdapter.adapt(function_definition).to_uri) + + nil # do not want the function to inadvertently leak into 3x + end + def code() Expression.new(:value => @value) end def is_nop?(o) @ast_transformer.is_nop?(o) end end end diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 1985d507a..2271f5da7 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -1,572 +1,594 @@ require 'forwardable' require 'puppet/node' require 'puppet/resource/catalog' require 'puppet/util/errors' require 'puppet/resource/type_collection_helper' # Maintain a graph of scopes, along with a bunch of data # about the individual catalog we're compiling. class Puppet::Parser::Compiler extend Forwardable include Puppet::Util include Puppet::Util::Errors include Puppet::Util::MethodHelper include Puppet::Resource::TypeCollectionHelper def self.compile(node) $env_module_directories = nil node.environment.check_for_reparse new(node).compile.to_resource rescue => detail message = "#{detail} on node #{node.name}" Puppet.log_exception(detail, message) raise Puppet::Error, message, detail.backtrace end attr_reader :node, :facts, :collections, :catalog, :resources, :relationships, :topscope # The injector that provides lookup services, or nil if accessed before the compiler has started compiling and # bootstrapped. The injector is initialized and available before any manifests are evaluated. # # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services for this compiler/environment # @api public # attr_accessor :injector # The injector that provides lookup services during the creation of the {#injector}. # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services during injector creation # for this compiler/environment # # @api private # attr_accessor :boot_injector # Add a collection to the global list. def_delegator :@collections, :<<, :add_collection def_delegator :@relationships, :<<, :add_relationship # Store a resource override. def add_override(override) # If possible, merge the override in immediately. if resource = @catalog.resource(override.ref) resource.merge(override) else # Otherwise, store the override for later; these # get evaluated in Resource#finish. @resource_overrides[override.ref] << override end end def add_resource(scope, resource) @resources << resource # Note that this will fail if the resource is not unique. @catalog.add_resource(resource) if not resource.class? and resource[:stage] raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage" end # Stages should not be inside of classes. They are always a # top-level container, regardless of where they appear in the # manifest. return if resource.stage? # This adds a resource to the class it lexically appears in in the # manifest. unless resource.class? return @catalog.add_edge(scope.resource, resource) end end # Do we use nodes found in the code, vs. the external node sources? def_delegator :known_resource_types, :nodes?, :ast_nodes? # Store the fact that we've evaluated a class def add_class(name) @catalog.add_class(name) unless name == "" end # Return a list of all of the defined classes. def_delegator :@catalog, :classes, :classlist # Compiler our catalog. This mostly revolves around finding and evaluating classes. # This is the main entry into our catalog. def compile - Puppet.override({ :current_environment => environment }, "For compiling #{node.name}") do +# require 'debugger'; debugger + Puppet.override( @context_overrides , "For compiling #{node.name}") do # Set the client's parameters into the top scope. Puppet::Util::Profiler.profile("Compile: Set node parameters") { set_node_parameters } Puppet::Util::Profiler.profile("Compile: Created settings scope") { create_settings_scope } - if is_binder_active? - Puppet::Util::Profiler.profile("Compile: Created injector") { create_injector } - end +# if is_binder_active? +# Puppet::Util::Profiler.profile("Compile: Created injector") { create_injector } +# end - Puppet.override(context_overrides4x) do +# Puppet.override(@context_overrides) do Puppet::Util::Profiler.profile("Compile: Evaluated main") { evaluate_main } Puppet::Util::Profiler.profile("Compile: Evaluated AST node") { evaluate_ast_node } Puppet::Util::Profiler.profile("Compile: Evaluated node classes") { evaluate_node_classes } Puppet::Util::Profiler.profile("Compile: Evaluated generators") { evaluate_generators } Puppet::Util::Profiler.profile("Compile: Finished catalog") { finish } fail_on_unevaluated @catalog - end +# end end end # Constructs the overrides for the context - def context_overrides4x() + def context_overrides() if Puppet[:parser] == 'future' require 'puppet/loaders' { - :global_scope => {}, # 4x placeholder for new global scope + :current_environment => environment, + :global_scope => @topscope, # 4x placeholder for new global scope :loaders => Puppet::Pops::Loaders.new(), # 4x loaders :injector => injector() # 4x API - via context instead of via compiler } else - {} + { + :current_environment => environment, + } end end def_delegator :@collections, :delete, :delete_collection # Return the node's environment. def environment unless node.environment.is_a? Puppet::Node::Environment raise Puppet::DevError, "node #{node} has an invalid environment!" end node.environment end # Evaluate all of the classes specified by the node. # Classes with parameters are evaluated as if they were declared. # Classes without parameters or with an empty set of parameters are evaluated # as if they were included. This means classes with an empty set of # parameters won't conflict even if the class has already been included. def evaluate_node_classes if @node.classes.is_a? Hash classes_with_params, classes_without_params = @node.classes.partition {|name,params| params and !params.empty?} # The results from Hash#partition are arrays of pairs rather than hashes, # so we have to convert to the forms evaluate_classes expects (Hash, and # Array of class names) classes_with_params = Hash[classes_with_params] classes_without_params.map!(&:first) else classes_with_params = {} classes_without_params = @node.classes end evaluate_classes(classes_without_params, @node_scope || topscope) evaluate_classes(classes_with_params, @node_scope || topscope) end # Evaluate each specified class in turn. If there are any classes we can't # find, raise an error. This method really just creates resource objects # that point back to the classes, and then the resources are themselves # evaluated later in the process. # # Sometimes we evaluate classes with a fully qualified name already, in which # case, we tell scope.find_hostclass we've pre-qualified the name so it # doesn't need to search its namespaces again. This gets around a weird # edge case of duplicate class names, one at top scope and one nested in our # namespace and the wrong one (or both!) getting selected. See ticket #13349 # for more detail. --jeffweiss 26 apr 2012 def evaluate_classes(classes, scope, lazy_evaluate = true, fqname = false) raise Puppet::DevError, "No source for scope passed to evaluate_classes" unless scope.source class_parameters = nil # if we are a param class, save the classes hash # and transform classes to be the keys if classes.class == Hash class_parameters = classes classes = classes.keys end classes.each do |name| # If we can find the class, then make a resource that will evaluate it. if klass = scope.find_hostclass(name, :assume_fqname => fqname) # If parameters are passed, then attempt to create a duplicate resource # so the appropriate error is thrown. if class_parameters resource = klass.ensure_in_catalog(scope, class_parameters[name] || {}) else next if scope.class_scope(klass) resource = klass.ensure_in_catalog(scope) end # If they've disabled lazy evaluation (which the :include function does), # then evaluate our resource immediately. resource.evaluate unless lazy_evaluate else raise Puppet::Error, "Could not find class #{name} for #{node.name}" end end end def evaluate_relationships @relationships.each { |rel| rel.evaluate(catalog) } end # Return a resource by either its ref or its type and title. def_delegator :@catalog, :resource, :findresource def initialize(node, options = {}) @node = node set_options(options) initvars end # Create a new scope, with either a specified parent scope or # using the top scope. def newscope(parent, options = {}) parent ||= topscope scope = Puppet::Parser::Scope.new(self, options) scope.parent = parent scope end # Return any overrides for the given resource. def resource_overrides(resource) @resource_overrides[resource.ref] end def injector create_injector if @injector.nil? @injector end def boot_injector create_boot_injector(nil) if @boot_injector.nil? @boot_injector end # Creates the boot injector from registered system, default, and injector config. # @return [Puppet::Pops::Binder::Injector] the created boot injector # @api private Cannot be 'private' since it is called from the BindingsComposer. # def create_boot_injector(env_boot_bindings) assert_binder_active() pb = Puppet::Pops::Binder boot_contribution = pb::SystemBindings.injector_boot_contribution(env_boot_bindings) final_contribution = pb::SystemBindings.final_contribution binder = pb::Binder.new(pb::BindingsFactory.layered_bindings(final_contribution, boot_contribution)) @boot_injector = pb::Injector.new(binder) end # Answers if Puppet Binder should be active or not, and if it should and is not active, then it is activated. # @return [Boolean] true if the Puppet Binder should be activated def is_binder_active? should_be_active = Puppet[:binder] || Puppet[:parser] == 'future' if should_be_active # TODO: this should be in a central place, not just for ParserFactory anymore... Puppet::Parser::ParserFactory.assert_rgen_installed() @@binder_loaded ||= false unless @@binder_loaded require 'puppet/pops' require 'puppetx' @@binder_loaded = true end end should_be_active end private # If ast nodes are enabled, then see if we can find and evaluate one. def evaluate_ast_node return unless ast_nodes? # Now see if we can find the node. astnode = nil @node.names.each do |name| break if astnode = known_resource_types.node(name.to_s.downcase) end unless (astnode ||= known_resource_types.node("default")) raise Puppet::ParseError, "Could not find default node or by name with '#{node.names.join(", ")}'" end # Create a resource to model this node, and then add it to the list # of resources. resource = astnode.ensure_in_catalog(topscope) resource.evaluate @node_scope = topscope.class_scope(astnode) end # Evaluate our collections and return true if anything returned an object. # The 'true' is used to continue a loop, so it's important. def evaluate_collections return false if @collections.empty? exceptwrap do # We have to iterate over a dup of the array because # collections can delete themselves from the list, which # changes its length and causes some collections to get missed. Puppet::Util::Profiler.profile("Evaluated collections") do found_something = false @collections.dup.each do |collection| found_something = true if collection.evaluate end found_something end end end # Make sure all of our resources have been evaluated into native resources. # We return true if any resources have, so that we know to continue the # evaluate_generators loop. def evaluate_definitions exceptwrap do Puppet::Util::Profiler.profile("Evaluated definitions") do !unevaluated_resources.each do |resource| Puppet::Util::Profiler.profile("Evaluated resource #{resource}") do resource.evaluate end end.empty? end end end # Iterate over collections and resources until we're sure that the whole # compile is evaluated. This is necessary because both collections # and defined resources can generate new resources, which themselves could # be defined resources. def evaluate_generators count = 0 loop do done = true Puppet::Util::Profiler.profile("Iterated (#{count + 1}) on generators") do # Call collections first, then definitions. done = false if evaluate_collections done = false if evaluate_definitions end break if done count += 1 if count > 1000 raise Puppet::ParseError, "Somehow looped more than 1000 times while evaluating host catalog" end end end # Find and evaluate our main object, if possible. def evaluate_main @main = known_resource_types.find_hostclass([""], "") || known_resource_types.add(Puppet::Resource::Type.new(:hostclass, "")) @topscope.source = @main @main_resource = Puppet::Parser::Resource.new("class", :main, :scope => @topscope, :source => @main) @topscope.resource = @main_resource add_resource(@topscope, @main_resource) @main_resource.evaluate end # Make sure the entire catalog is evaluated. def fail_on_unevaluated fail_on_unevaluated_overrides fail_on_unevaluated_resource_collections end # If there are any resource overrides remaining, then we could # not find the resource they were supposed to override, so we # want to throw an exception. def fail_on_unevaluated_overrides remaining = @resource_overrides.values.flatten.collect(&:ref) if !remaining.empty? fail Puppet::ParseError, "Could not find resource(s) #{remaining.join(', ')} for overriding" end end # Make sure we don't have any remaining collections that specifically # look for resources, because we want to consider those to be # parse errors. def fail_on_unevaluated_resource_collections remaining = @collections.collect(&:resources).flatten.compact if !remaining.empty? raise Puppet::ParseError, "Failed to realize virtual resources #{remaining.join(', ')}" end end # Make sure all of our resources and such have done any last work # necessary. def finish evaluate_relationships resources.each do |resource| # Add in any resource overrides. if overrides = resource_overrides(resource) overrides.each do |over| resource.merge(over) end # Remove the overrides, so that the configuration knows there # are none left. overrides.clear end resource.finish if resource.respond_to?(:finish) end add_resource_metaparams end def add_resource_metaparams unless main = catalog.resource(:class, :main) raise "Couldn't find main" end names = Puppet::Type.metaparams.select do |name| !Puppet::Parser::Resource.relationship_parameter?(name) end data = {} catalog.walk(main, :out) do |source, target| if source_data = data[source] || metaparams_as_data(source, names) # only store anything in the data hash if we've actually got # data data[source] ||= source_data source_data.each do |param, value| target[param] = value if target[param].nil? end data[target] = source_data.merge(metaparams_as_data(target, names)) end target.tag(*(source.tags)) end end def metaparams_as_data(resource, params) data = nil params.each do |param| unless resource[param].nil? # Because we could be creating a hash for every resource, # and we actually probably don't often have any data here at all, # we're optimizing a bit by only creating a hash if there's # any data to put in it. data ||= {} data[param] = resource[param] end end data end # Set up all of our internal variables. def initvars # The list of overrides. This is used to cache overrides on objects # that don't exist yet. We store an array of each override. @resource_overrides = Hash.new do |overs, ref| overs[ref] = [] end # The list of collections that have been created. This is a global list, # but they each refer back to the scope that created them. @collections = [] # The list of relationships to evaluate. @relationships = [] # For maintaining the relationship between scopes and their resources. @catalog = Puppet::Resource::Catalog.new(@node.name) - @catalog.version = known_resource_types.version + + # MOVED HERE - SCOPE IS NEEDED (MOVE-SCOPE) + # Create the initial scope, it is needed early + @topscope = Puppet::Parser::Scope.new(self) + + # Need to compute them here, and remember then, because we are about to + # enter the magic zone of known_resource_types + @context_overrides = context_overrides() + + # This construct ensures that initial import (triggered by instantiating + # the structure 'known_resource_types') has a configured context + # It cannot survive the initvars method, and is later reinstated + # as part of compiling... + # + Puppet.override( @context_overrides , "For initializing compiler") do + # THE MAGIC STARTS HERE ! This triggers parsing, loading etc. + @catalog.version = known_resource_types.version + end @catalog.environment = @node.environment.to_s - # Create our initial scope and a resource that will evaluate main. - @topscope = Puppet::Parser::Scope.new(self) + # ((MOVE-SCOPE) +# # Create our initial scope and a resource that will evaluate main. +# @topscope = Puppet::Parser::Scope.new(self) @catalog.add_resource(Puppet::Parser::Resource.new("stage", :main, :scope => @topscope)) # local resource array to maintain resource ordering @resources = [] # Make sure any external node classes are in our class list if @node.classes.class == Hash @catalog.add_class(*@node.classes.keys) else @catalog.add_class(*@node.classes) end end # Set the node's parameters into the top-scope as variables. def set_node_parameters node.parameters.each do |param, value| @topscope[param.to_s] = value end # These might be nil. catalog.client_version = node.parameters["clientversion"] catalog.server_version = node.parameters["serverversion"] if Puppet[:trusted_node_data] @topscope.set_trusted(node.trusted_data) end if(Puppet[:immutable_node_data]) facts_hash = node.facts.nil? ? {} : node.facts.values @topscope.set_facts(facts_hash) end end def create_settings_scope unless settings_type = environment.known_resource_types.hostclass("settings") settings_type = Puppet::Resource::Type.new :hostclass, "settings" environment.known_resource_types.add(settings_type) end settings_resource = Puppet::Parser::Resource.new("class", "settings", :scope => @topscope) @catalog.add_resource(settings_resource) settings_type.evaluate_code(settings_resource) scope = @topscope.class_scope(settings_type) Puppet.settings.each do |name, setting| next if name.to_s == "name" scope[name.to_s] = environment[name] end end # Return an array of all of the unevaluated resources. These will be definitions, # which need to get evaluated into native resources. def unevaluated_resources # The order of these is significant for speed due to short-circuting resources.reject { |resource| resource.evaluated? or resource.virtual? or resource.builtin_type? } end # Creates the injector from bindings found in the current environment. # @return [void] # @api private # def create_injector assert_binder_active() composer = Puppet::Pops::Binder::BindingsComposer.new() layered_bindings = composer.compose(topscope) @injector = Puppet::Pops::Binder::Injector.new(Puppet::Pops::Binder::Binder.new(layered_bindings)) end def assert_binder_active unless is_binder_active? raise ArgumentError, "The Puppet Binder is only available when either '--binder true' or '--parser future' is used" end end end diff --git a/lib/puppet/pops/adapters.rb b/lib/puppet/pops/adapters.rb index 17da833e7..294639172 100644 --- a/lib/puppet/pops/adapters.rb +++ b/lib/puppet/pops/adapters.rb @@ -1,101 +1,108 @@ # The Adapters module contains adapters for Documentation, Origin, SourcePosition, and Loader. # module Puppet::Pops::Adapters # A documentation adapter adapts an object with a documentation string. # (The intended use is for a source text parser to extract documentation and store this # in DocumentationAdapter instances). # class DocumentationAdapter < Puppet::Pops::Adaptable::Adapter # @return [String] The documentation associated with an object attr_accessor :documentation end # A SourcePosAdapter holds a reference to a *Positioned* object (object that has offset and length). # This somewhat complex structure makes it possible to correctly refer to a source position # in source that is embedded in some resource; a parser only sees the embedded snippet of source text # and does not know where it was embedded. It also enables lazy evaluation of source positions (they are # rarely needed - typically just when there is an error to report. # # @note It is relatively expensive to compute line and position on line - it is not something that # should be done for every token or model object. # # @see Puppet::Pops::Utils#find_adapter, Puppet::Pops::Utils#find_closest_positioned # class SourcePosAdapter < Puppet::Pops::Adaptable::Adapter attr_accessor :locator def self.create_adapter(o) new(o) end def initialize(o) @adapted = o end def locator # The locator is always the parent locator, all positioned objects are positioned within their # parent. If a positioned object also has a locator that locator is for its children! # @locator ||= find_locator(@adapted.eContainer) end def find_locator(o) if o.nil? raise ArgumentError, "InternalError: SourcePosAdapter for something that has no locator among parents" end case when o.is_a?(Puppet::Pops::Model::Program) return o.locator # TODO_HEREDOC use case of SubLocator instead when o.is_a?(Puppet::Pops::Model::SubLocatedExpression) && !(found_locator = o.locator).nil? return found_locator when adapter = self.class.get(o) return adapter.locator else find_locator(o.eContainer) end end private :find_locator def offset @adapted.offset end def length @adapted.length end # Produces the line number for the given offset. # @note This is an expensive operation # def line locator.line_for_offset(offset) end # Produces the position on the line of the given offset. # @note This is an expensive operation # def pos locator.pos_on_line(offset) end # Extracts the text represented by this source position (the string is obtained from the locator) def extract_text locator.string.slice(offset, length) end + + # Produces an URI with path?line=n&pos=n. If origin is unknown the URI is string:?line=n&pos=n + def to_uri + f = locator.file + f = 'string:' if f.nil? || f.empty? + URI("#{f}?line=#{line.to_s}&pos=#{pos.to_s}") + end end # A LoaderAdapter adapts an object with a {Puppet::Pops::Loader}. This is used to make further loading from the # perspective of the adapted object take place in the perspective of this Loader. # # It is typically enough to adapt the root of a model as a search is made towards the root of the model # until a loader is found, but there is no harm in duplicating this information provided a contained # object is adapted with the correct loader. # # @see Puppet::Pops::Utils#find_adapter # class LoaderAdapter < Puppet::Pops::Adaptable::Adapter # @return [Puppet::Pops::Loader::Loader] the loader attr_accessor :loader end end diff --git a/lib/puppet/pops/evaluator/runtime3_support.rb b/lib/puppet/pops/evaluator/runtime3_support.rb index 6fc92b4e3..c7a93780d 100644 --- a/lib/puppet/pops/evaluator/runtime3_support.rb +++ b/lib/puppet/pops/evaluator/runtime3_support.rb @@ -1,530 +1,527 @@ # A module with bindings between the new evaluator and the 3x runtime. # The intention is to separate all calls into scope, compiler, resource, etc. in this module # to make it easier to later refactor the evaluator for better implementations of the 3x classes. # # @api private module Puppet::Pops::Evaluator::Runtime3Support # Fails the evaluation of _semantic_ with a given issue. # # @param issue [Puppet::Pops::Issue] the issue to report # @param semantic [Puppet::Pops::ModelPopsObject] the object for which evaluation failed in some way. Used to determine origin. # @param options [Hash] hash of optional named data elements for the given issue # @return [!] this method does not return # @raise [Puppet::ParseError] an evaluation error initialized from the arguments (TODO: Change to EvaluationError?) # def fail(issue, semantic, options={}, except=nil) optionally_fail(issue, semantic, options, except) # an error should have been raised since fail always fails raise ArgumentError, "Internal Error: Configuration of runtime error handling wrong: should have raised exception" end # Optionally (based on severity) Fails the evaluation of _semantic_ with a given issue # If the given issue is configured to be of severity < :error it is only reported, and the function returns. # # @param issue [Puppet::Pops::Issue] the issue to report # @param semantic [Puppet::Pops::ModelPopsObject] the object for which evaluation failed in some way. Used to determine origin. # @param options [Hash] hash of optional named data elements for the given issue # @return [!] this method does not return # @raise [Puppet::ParseError] an evaluation error initialized from the arguments (TODO: Change to EvaluationError?) # def optionally_fail(issue, semantic, options={}, except=nil) if except.nil? # Want a stacktrace, and it must be passed as an exception begin raise EvaluationError.new() rescue EvaluationError => e except = e end end diagnostic_producer.accept(issue, semantic, options, except) end # Binds the given variable name to the given value in the given scope. # The reference object `o` is intended to be used for origin information - the 3x scope implementation # only makes use of location when there is an error. This is now handled by other mechanisms; first a check # is made if a variable exists and an error is raised if attempting to change an immutable value. Errors # in name, numeric variable assignment etc. have also been validated prior to this call. In the event the # scope.setvar still raises an error, the general exception handling for evaluation of the assignment # expression knows about its location. Because of this, there is no need to extract the location for each # setting (extraction is somewhat expensive since 3x requires line instead of offset). # def set_variable(name, value, o, scope) # Scope also checks this but requires that location information are passed as options. # Those are expensive to calculate and a test is instead made here to enable failing with better information. # The error is not specific enough to allow catching it - need to check the actual message text. # TODO: Improve the messy implementation in Scope. # if scope.bound?(name) if Puppet::Parser::Scope::RESERVED_VARIABLE_NAMES.include?(name) fail(Puppet::Pops::Issues::ILLEGAL_RESERVED_ASSIGNMENT, o, {:name => name} ) else fail(Puppet::Pops::Issues::ILLEGAL_REASSIGNMENT, o, {:name => name} ) end end scope.setvar(name, value) end # Returns the value of the variable (nil is returned if variable has no value, or if variable does not exist) # def get_variable_value(name, o, scope) # Puppet 3x stores all variables as strings (then converts them back to numeric with a regexp... to see if it is a match variable) # Not ideal, scope should support numeric lookup directly instead. # TODO: consider fixing scope catch(:undefined_variable) { return scope.lookupvar(name.to_s) } # It is always ok to reference numeric variables even if they are not assigned. They are always undef # if not set by a match expression. # unless name =~ Puppet::Pops::Patterns::NUMERIC_VAR_NAME fail(Puppet::Pops::Issues::UNKNOWN_VARIABLE, o, {:name => name}) end end # Returns true if the variable of the given name is set in the given most nested scope. True is returned even if # variable is bound to nil. # def variable_bound?(name, scope) scope.bound?(name.to_s) end # Returns true if the variable is bound to a value or nil, in the scope or it's parent scopes. # def variable_exists?(name, scope) scope.exist?(name.to_s) end def set_match_data(match_data, o, scope) # See set_variable for rationale for not passing file and line to ephemeral_from. # NOTE: The 3x scope adds one ephemeral(match) to its internal stack per match that succeeds ! It never # clears anything. Thus a context that performs many matches will get very deep (there simply is no way to # clear the match variables without rolling back the ephemeral stack.) # This implementation does not attempt to fix this, it behaves the same bad way. unless match_data.nil? scope.ephemeral_from(match_data) end end # Creates a local scope with vairalbes set from a hash of variable name to value # def create_local_scope_from(hash, scope) # two dummy values are needed since the scope tries to give an error message (can not happen in this # case - it is just wrong, the error should be reported by the caller who knows in more detail where it # is in the source. # raise ArgumentError, "Internal error - attempt to create a local scope without a hash" unless hash.is_a?(Hash) scope.ephemeral_from(hash) end # Creates a nested match scope def create_match_scope_from(scope) # Create a transparent match scope (for future matches) scope.new_match_scope(nil) end def get_scope_nesting_level(scope) scope.ephemeral_level end def set_scope_nesting_level(scope, level) # Yup, 3x uses this method to reset the level, it also supports passing :all to destroy all # ephemeral/local scopes - which is a sure way to create havoc. # scope.unset_ephemeral_var(level) end # Adds a relationship between the given `source` and `target` of the given `relationship_type` # @param source [Puppet:Pops::Types::PCatalogEntryType] the source end of the relationship (from) # @param target [Puppet:Pops::Types::PCatalogEntryType] the target end of the relationship (to) # @param relationship_type [:relationship, :subscription] the type of the relationship # def add_relationship(source, target, relationship_type, scope) # The 3x way is to record a Puppet::Parser::Relationship that is evaluated at the end of the compilation. # This means it is not possible to detect any duplicates at this point (and signal where an attempt is made to # add a duplicate. There is also no location information to signal the original place in the logic. The user will have # to go fish. # The 3.x implementation is based on Strings :-o, so the source and target must be transformed. The resolution is # done by Catalog#resource(type, title). To do that, it creates a Puppet::Resource since it is responsible for # translating the name/type/title and create index-keys used by the catalog. The Puppet::Resource has bizarre parsing of # the type and title (scan for [] that is interpreted as type/title (but it gets it wrong). # Moreover if the type is "" or "component", the type is Class, and if the type is :main, it is :main, all other cases # undergo capitalization of name-segments (foo::bar becomes Foo::Bar). (This was earlier done in the reverse by the parser). # Further, the title undergoes the same munging !!! # # That bug infested nest of messy logic needs serious Exorcism! # # Unfortunately it is not easy to simply call more intelligent methods at a lower level as the compiler evaluates the recorded # Relationship object at a much later point, and it is responsible for invoking all the messy logic. # # TODO: Revisit the below logic when there is a sane implementation of the catalog, compiler and resource. For now # concentrate on transforming the type references to what is expected by the wacky logic. # # HOWEVER, the Compiler only records the Relationships, and the only method it calls is @relationships.each{|x| x.evaluate(catalog) } # Which means a smarter Relationship class could do this right. Instead of obtaining the resource from the catalog using # the borked resource(type, title) which creates a resource for the purpose of looking it up, it needs to instead # scan the catalog's resources # # GAAAH, it is even worse! # It starts in the parser, which parses "File['foo']" into an AST::ResourceReference with type = File, and title = foo # This AST is evaluated by looking up the type/title in the scope - causing it to be loaded if it exists, and if not, the given # type name/title is used. It does not search for resource instances, only classes and types. It returns symbolic information # [type, [title, title]]. From this, instances of Puppet::Resource are created and returned. These only have type/title information # filled out. One or an array of resources are returned. # This set of evaluated (empty reference) Resource instances are then passed to the relationship operator. It creates a # Puppet::Parser::Relationship giving it a source and a target that are (empty reference) Resource instances. These are then remembered # until the relationship is evaluated by the compiler (at the end). When evaluation takes place, the (empty reference) Resource instances # are converted to String (!?! WTF) on the simple format "#{type}[#{title}]", and the catalog is told to find a resource, by giving # it this string. If it cannot find the resource it fails, else the before/notify parameter is appended with the target. # The search for the resource begin with (you guessed it) again creating an (empty reference) resource from type and title (WTF?!?!). # The catalog now uses the reference resource to compute a key [r.type, r.title.to_s] and also gets a uniqueness key from the # resource (This is only a reference type created from title and type). If it cannot find it with the first key, it uses the # uniqueness key to lookup. # # This is probably done to allow a resource type to munge/translate the title in some way (but it is quite unclear from the long # and convoluted path of evaluation. # In order to do this in a way that is similar to 3.x two resources are created to be used as keys. # # # TODO: logic that creates a PCatalogEntryType should resolve it to ensure it is loaded (to the best of known_resource_types knowledge). # If this is not done, the order in which things are done may be different? OTOH, it probably works anyway :-) # TODO: Not sure if references needs to be resolved via the scope? # # And if that is not enough, a source/target may be a Collector (a baked query that will be evaluated by the # compiler - it is simply passed through here for processing by the compiler at the right time). # if source.is_a?(Puppet::Parser::Collector) # use verbatim - behavior defined by 3x source_resource = source else # transform into the wonderful String representation in 3x type, title = catalog_type_to_split_type_title(source) source_resource = Puppet::Resource.new(type, title) end if target.is_a?(Puppet::Parser::Collector) # use verbatim - behavior defined by 3x target_resource = target else # transform into the wonderful String representation in 3x type, title = catalog_type_to_split_type_title(target) target_resource = Puppet::Resource.new(type, title) end # Add the relationship to the compiler for later evaluation. scope.compiler.add_relationship(Puppet::Parser::Relationship.new(source_resource, target_resource, relationship_type)) end # Coerce value `v` to numeric or fails. # The given value `v` is coerced to Numeric, and if that fails the operation # calls {#fail}. # @param v [Object] the value to convert # @param o [Object] originating instruction # @param scope [Object] the (runtime specific) scope where evaluation of o takes place # @return [Numeric] value `v` converted to Numeric. # def coerce_numeric(v, o, scope) unless n = Puppet::Pops::Utils.to_n(v) fail(Puppet::Pops::Issues::NOT_NUMERIC, o, {:value => v}) end n end def call_function(name, args, o, scope) # Call via 4x API if it is available, and the function exists # if loaders = Puppet.lookup(:loaders) {nil} - # find the loader that loaded the code, or use the system loader - adapter = Puppet::Pops::Utils.find_adapter(o, Puppet::Pops::Adapters::LoaderAdapter) - loader = adapter.nil? ? loaders.puppet_system_loader : adapter.loader - if loader && func = loader.load(:function, name) + if loaders && func = loaders.environment_loader.load(:function, name) return func.call(scope, *args) end end fail(Puppet::Pops::Issues::UNKNOWN_FUNCTION, o, {:name => name}) unless Puppet::Parser::Functions.function(name) # TODO: if Puppet[:biff] == true, then 3x functions should be called via loaders above # Arguments must be mapped since functions are unaware of the new and magical creatures in 4x. # NOTE: Passing an empty string last converts :undef to empty string mapped_args = args.map {|a| convert(a, scope, '') } result = scope.send("function_#{name}", mapped_args) # Prevent non r-value functions from leaking their result (they are not written to care about this) Puppet::Parser::Functions.rvalue?(name) ? result : nil end # The o is used for source reference def create_resource_parameter(o, scope, name, value, operator) file, line = extract_file_line(o) Puppet::Parser::Resource::Param.new( :name => name, :value => convert(value, scope, :undef), # converted to 3x since 4x supports additional objects / types :source => scope.source, :line => line, :file => file, :add => operator == :'+>' ) end def create_resources(o, scope, virtual, exported, type_name, resource_titles, evaluated_parameters) # TODO: Unknown resource causes creation of Resource to fail with ArgumentError, should give # a proper Issue. Now the result is "Error while evaluating a Resource Statement" with the message # from the raised exception. (It may be good enough). # resolve in scope. fully_qualified_type, resource_titles = scope.resolve_type_and_titles(type_name, resource_titles) # Not 100% accurate as this is the resource expression location and each title is processed separately # The titles are however the result of evaluation and they have no location at this point (an array # of positions for the source expressions are required for this to work). # TODO: Revisit and possible improve the accuracy. # file, line = extract_file_line(o) # Build a resource for each title resource_titles.map do |resource_title| resource = Puppet::Parser::Resource.new( fully_qualified_type, resource_title, :parameters => evaluated_parameters, :file => file, :line => line, :exported => exported, :virtual => virtual, # WTF is this? Which source is this? The file? The name of the context ? :source => scope.source, :scope => scope, :strict => true ) if resource.resource_type.is_a? Puppet::Resource::Type resource.resource_type.instantiate_resource(scope, resource) end scope.compiler.add_resource(scope, resource) scope.compiler.evaluate_classes([resource_title], scope, false, true) if fully_qualified_type == 'class' # Turn the resource into a PType (a reference to a resource type) # weed out nil's resource_to_ptype(resource) end end # Defines default parameters for a type with the given name. # def create_resource_defaults(o, scope, type_name, evaluated_parameters) # Note that name must be capitalized in this 3x call # The 3x impl creates a Resource instance with a bogus title and then asks the created resource # for the type of the name. # Note, locations are available per parameter. # scope.define_settings(type_name.capitalize, evaluated_parameters) end # Creates resource overrides for all resource type objects in evaluated_resources. The same set of # evaluated parameters are applied to all. # def create_resource_overrides(o, scope, evaluated_resources, evaluated_parameters) # Not 100% accurate as this is the resource expression location and each title is processed separately # The titles are however the result of evaluation and they have no location at this point (an array # of positions for the source expressions are required for this to work. # TODO: Revisit and possible improve the accuracy. # file, line = extract_file_line(o) evaluated_resources.each do |r| resource = Puppet::Parser::Resource.new( r.type_name, r.title, :parameters => evaluated_parameters, :file => file, :line => line, # WTF is this? Which source is this? The file? The name of the context ? :source => scope.source, :scope => scope ) scope.compiler.add_override(resource) end end # Finds a resource given a type and a title. # def find_resource(scope, type_name, title) scope.compiler.findresource(type_name, title) end # Returns the value of a resource's parameter by first looking up the parameter in the resource # and then in the defaults for the resource. Since the resource exists (it must in order to look up its # parameters, any overrides have already been applied). Defaults are not applied to a resource until it # has been finished (which typically has not taked place when this is evaluated; hence the dual lookup). # def get_resource_parameter_value(scope, resource, parameter_name) val = resource[parameter_name] if val.nil? && defaults = scope.lookupdefaults(resource.type) # NOTE: 3x resource keeps defaults as hash using symbol for name as key to Parameter which (again) holds # name and value. param = defaults[parameter_name.to_sym] val = param.value end val end # Returns true, if the given name is the name of a resource parameter. # def is_parameter_of_resource?(scope, resource, name) resource.valid_parameter?(name) end def resource_to_ptype(resource) nil if resource.nil? type_calculator.infer(resource) end # This is the same type of "truth" as used in the current Puppet DSL. # def is_true? o # Is the value true? This allows us to control the definition of truth # in one place. case o when '' false when :undef false else !!o end end # Utility method for TrueClass || FalseClass # @param x [Object] the object to test if it is instance of TrueClass or FalseClass def is_boolean? x x.is_a?(TrueClass) || x.is_a?(FalseClass) end def initialize @@convert_visitor ||= Puppet::Pops::Visitor.new(self, "convert", 2, 2) end # Converts 4x supported values to 3x values. This is required because # resources and other objects do not know about the new type system, and does not support # regular expressions. Unfortunately this has to be done for array and hash as well. # A complication is that catalog types needs to be resolved against the scope. # def convert(o, scope, undef_value) @@convert_visitor.visit_this_2(self, o, scope, undef_value) end def convert_NilClass(o, scope, undef_value) undef_value end def convert_Object(o, scope, undef_value) o end def convert_Array(o, scope, undef_value) o.map {|x| convert(x, scope, undef_value) } end def convert_Hash(o, scope, undef_value) result = {} o.each {|k,v| result[convert(k, scope, undef_value)] = convert(v, scope, undef_value) } result end def convert_Regexp(o, scope, undef_value) # Puppet 3x cannot handle parameter values that are reqular expressions. Turn into regexp string in # source form o.inspect end def convert_Symbol(o, scope, undef_value) case o when :undef undef_value # 3x wants :undef as empty string in function else o # :default, and all others are verbatim since they are new in future evaluator end end def convert_PAbstractType(o, scope, undef_value) o end def convert_PResourceType(o,scope, undef_value) # Needs conversion by calling scope to resolve the name and possibly return a different name # Resolution can only be called with an array, and returns an array. Here there is only one name type, titles = scope.resolve_type_and_titles(o.type_name, [o.title]) # Note: a title of nil makes Resource class throw error with information that is wrong Puppet::Resource.new(type, titles[0].nil? ? '' : titles[0] ) end def convert_PHostClassType(o, scope, undef_value) # Needs conversion by calling scope to resolve the name and possibly return a different name # Resolution can only be called with an array, and returns an array. Here there is only one name type, titles = scope.resolve_type_and_titles('class', [o.class_name]) # Note: a title of nil makes Resource class throw error with information that is wrong Puppet::Resource.new(type, titles[0].nil? ? '' : titles[0] ) end private # Produces an array with [type, title] from a PCatalogEntryType # Used to produce reference resource instances (used when 3x is operating on a resource). # def catalog_type_to_split_type_title(catalog_type) case catalog_type when Puppet::Pops::Types::PHostClassType return ['Class', catalog_type.class_name] when Puppet::Pops::Types::PResourceType return [catalog_type.type_name, catalog_type.title] else raise ArgumentError, "Cannot split the type #{catalog_type.class}, it is neither a PHostClassType, nor a PResourceClass." end end def extract_file_line(o) source_pos = Puppet::Pops::Utils.find_closest_positioned(o) return [nil, -1] unless source_pos [source_pos.locator.file, source_pos.line] end def find_closest_positioned(o) return nil if o.nil? || o.is_a?(Puppet::Pops::Model::Program) o.offset.nil? ? find_closest_positioned(o.eContainer) : Puppet::Pops::Adapters::SourcePosAdapter.adapt(o) end # Creates a diagnostic producer def diagnostic_producer Puppet::Pops::Validation::DiagnosticProducer.new( ExceptionRaisingAcceptor.new(), # Raises exception on all issues SeverityProducer.new(), # All issues are errors # Puppet::Pops::Validation::SeverityProducer.new(), # All issues are errors Puppet::Pops::Model::ModelLabelProvider.new()) end # Configure the severity of failures class SeverityProducer < Puppet::Pops::Validation::SeverityProducer Issues = Puppet::Pops::Issues def initialize super p = self # Issues triggering warning only if --debug is on if Puppet[:debug] p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :warning else p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :ignore end end end # An acceptor of diagnostics that immediately raises an exception. class ExceptionRaisingAcceptor < Puppet::Pops::Validation::Acceptor def accept(diagnostic) super Puppet::Pops::IssueReporter.assert_and_report(self, {:message => "Evaluation Error:", :emit_warnings => true }) if errors? raise ArgumentError, "Internal Error: Configuration of runtime error handling wrong: should have raised exception" end end end class EvaluationError < StandardError end end diff --git a/lib/puppet/pops/loader/base_loader.rb b/lib/puppet/pops/loader/base_loader.rb index c8f732367..f7b34ece7 100644 --- a/lib/puppet/pops/loader/base_loader.rb +++ b/lib/puppet/pops/loader/base_loader.rb @@ -1,96 +1,102 @@ # BaseLoader # === # An abstract implementation of Puppet::Pops::Loader::Loader # # A derived class should implement `find(typed_name)` and set entries, and possible handle "miss caching". # # @api private # class Puppet::Pops::Loader::BaseLoader < Puppet::Pops::Loader::Loader # The parent loader attr_reader :parent # An internal name used for debugging and error message purposes attr_reader :loader_name def initialize(parent_loader, loader_name) @parent = parent_loader # the higher priority loader to consult @named_values = {} # hash name => NamedEntry @last_name = nil # the last name asked for (optimization) @last_result = nil # the value of the last name (optimization) @loader_name = loader_name # the name of the loader (not the name-space it is a loader for) end # @api public # def load_typed(typed_name) # The check for "last queried name" is an optimization when a module searches. First it checks up its parent # chain, then itself, and then delegates to modules it depends on. # These modules are typically parented by the same # loader as the one initiating the search. It is inefficient to again try to search the same loader for # the same name. if typed_name == @last_name @last_result else @last_name = typed_name @last_result = internal_load(typed_name) end end # This method is final (subclasses should not override it) # # @api private # def get_entry(typed_name) @named_values[typed_name] end # @api private # def set_entry(typed_name, value, origin = nil) if entry = @named_values[typed_name] then fail_redefined(entry); end @named_values[typed_name] = Puppet::Pops::Loader::Loader::NamedEntry.new(typed_name, value, origin) end + # @api private + # + def add_entry(type, name, value, origin) + set_entry(Puppet::Pops::Loader::Loader::TypedName.new(type, name), value, origin) + end + # Promotes an already created entry (typically from another loader) to this loader # # @api private # def promote_entry(named_entry) typed_name = named_entry.typed_name if entry = @named_values[typed_name] then fail_redefined(entry); end @named_values[typed_name] = named_entry end private def fail_redefine(entry) origin_info = entry.origin ? " Originally set at #{origin_label(entry.origin)}." : "unknown location" raise ArgumentError, "Attempt to redefine entity '#{entry.typed_name}' originally set at #{origin_label(origin)}.#{origin_info}" end # TODO: Should not really be here?? - TODO: A Label provider ? semantics for the URI? # def origin_label(origin) if origin && origin.is_a?(URI) origin.to_s elsif origin.respond_to?(:uri) origin.uri.to_s else nil end end # loads in priority order: # 1. already loaded here # 2. load from parent # 3. find it here # 4. give up # def internal_load(typed_name) # avoid calling get_entry, by looking it up @named_values[typed_name] || parent.load_typed(typed_name) || find(typed_name) end end diff --git a/lib/puppet/pops/loader/null_loader.rb b/lib/puppet/pops/loader/null_loader.rb index 44568c99e..22076e38c 100644 --- a/lib/puppet/pops/loader/null_loader.rb +++ b/lib/puppet/pops/loader/null_loader.rb @@ -1,44 +1,44 @@ # The null loader is empty and delegates everything to its parent if it has one. # class Puppet::Pops::Loader::NullLoader < Puppet::Pops::Loader::Loader attr_reader :loader_name # Construct a NullLoader, optionally with a parent loader # def initialize(parent_loader=nil, loader_name = "null-loader") @loader_name = loader_name @parent = parent_loader end # Has parent if one was set when constructed def parent @parent end def load_typed(typed_name) if @parent.nil? nil else @parent.load_typed(typed_name) end end # Has no entries on its own - always nil def get_entry(typed_name) nil end # Finds nothing, there are no entries def find(name) nil end # Cannot store anything - def set_entry(typed_name, value) + def set_entry(typed_name, value, origin = nil) nil end def to_s() "(NullLoader '#{loader_name}')" end end \ No newline at end of file diff --git a/lib/puppet/pops/loader/puppet_function_instantiator.rb b/lib/puppet/pops/loader/puppet_function_instantiator.rb index d20561ae8..5e206c12c 100644 --- a/lib/puppet/pops/loader/puppet_function_instantiator.rb +++ b/lib/puppet/pops/loader/puppet_function_instantiator.rb @@ -1,97 +1,108 @@ # The PuppetFunctionInstantiator instantiates a Puppet::Functions::Function given a Puppet Programming language # source that when called evaluates the Puppet logic it contains. # class Puppet::Pops::Loader::PuppetFunctionInstantiator # Produces an instance of the Function class with the given typed_name, or fails with an error if the # given puppet source does not produce this instance when evaluated. # # @param loader [Puppet::Pops::Loader::Loader] The loader the function is associated with # @param typed_name [Puppet::Pops::Loader::TypedName] the type / name of the function to load # @param source_ref [URI, String] a reference to the source / origin of the puppet code to evaluate # @param pp_code_string [String] puppet code in a string # # @return [Puppet::Pops::Functions.Function] - an instantiated function with global scope closure associated with the given loader # def self.create(loader, typed_name, source_ref, pp_code_string) parser = Puppet::Pops::Parser::EvaluatingParser::Transitional.new() # parse and validate result = parser.parse_string(pp_code_string, source_ref) # Only one function is allowed (and no other definitions) case result.model.definitions.size when 0 raise ArgumentError, "The code loaded from #{source_ref} does not define the function #{typed_name.name} - it is empty." when 1 # ok else raise ArgumentError, "The code loaded from #{source_ref} must contain only the function #{typed_name.name} - it has additional definitions." end the_function_definition = result.model.definitions[0] unless the_function_definition.is_a?(Puppet::Pops::Model::FunctionDefinition) raise ArgumentError, "The code loaded from #{source_ref} does not define the function #{typed_name.name} - no function found." end unless the_function_definition.name == typed_name.name expected = typed_name.name actual = the_function_definition.name raise ArgumentError, "The code loaded from #{source_ref} produced function with the wrong name, expected #{expected}, actual #{actual}" end unless result.model().body == the_function_definition raise ArgumentError, "The code loaded from #{source_ref} contains additional logic - can only contain the function #{typed_name.name}" end # TODO: Cheating wrt. scope - assuming it is found in the context closure_scope = Puppet.lookup(:global_scope) { {} } created = create_function_class(the_function_definition, closure_scope) # create the function instance - it needs closure (scope), and loader (i.e. where it should start searching for things # when calling functions etc. # It should be bound to global scope created.new(closure_scope, loader) end + # Creates Function class and instantiates it based on a FunctionDefinition model + # @return [Array] - array of + # typed name, and an instantiated function with global scope closure associated with the given loader + # + def self.create_from_model(function_definition, loader) + closure_scope = Puppet.lookup(:global_scope) { {} } + created = create_function_class(function_definition, closure_scope) + typed_name = Puppet::Pops::Loader::Loader::TypedName.new(:function, function_definition.name) + [typed_name, created.new(closure_scope, loader)] + end + def self.create_function_class(function_definition, closure_scope) method_name = :"#{function_definition.name.split(/::/).slice(-1)}" closure = Puppet::Pops::Evaluator::Closure.new( Puppet::Pops::Evaluator::EvaluatorImpl.new(), function_definition, closure_scope) required_optional = function_definition.parameters.reduce([0, 0]) do |memo, p| if p.value.nil? memo[0] += 1 else memo[1] += 1 end memo end min_arg_count = required_optional[0] max_arg_count = required_optional[0] + required_optional[1] # Create a 4x function wrapper around the Puppet Function created_function_class = Puppet::Functions.create_function(function_definition.name) do # Define the method that is called from dispatch - this method just changes a call # with multiple unknown arguments to passing all in an array (since this is expected in the closure API. # # TODO: The closure will call the evaluator.call method which will again match args with parameters. # This can be done a better way later - unifying the two concepts - a function instance is really the same # as the current evaluator closure for lambdas, only that it also binds an evaluator. This could perhaps # be a specialization of Function... with a special dispatch # define_method(:__relay__call__) do |*args| closure.call(nil, *args) end # Define a dispatch that performs argument type/count checking # dispatch :__relay__call__ do # Use Puppet Type Object (not Optional[Object] since the 3x API passes undef as empty string). param(optional(object), 'args') # Specify arg count (transformed from FunctionDefinition.parameters, no types, or varargs yet) arg_count(min_arg_count, max_arg_count) end end created_function_class end end \ No newline at end of file diff --git a/lib/puppet/pops/loader/simple_environment_loader.rb b/lib/puppet/pops/loader/simple_environment_loader.rb new file mode 100644 index 000000000..ef9a5994f --- /dev/null +++ b/lib/puppet/pops/loader/simple_environment_loader.rb @@ -0,0 +1,18 @@ +# SimpleEnvironmentLoader +# === +# This loader does not load anything and it is populated by the bootstrapping logic that loads +# the site.pp or equivalent for an environment. It does not restrict the names of what it may contain, +# and what is loaded here overrides any child loaders (modules). +# +class Puppet::Pops::Loader::SimpleEnvironmentLoader < Puppet::Pops::Loader::BaseLoader + + # Never finds anything, everything "loaded" is set externally + def find(typed_name) + nil + end + + def to_s() + "(SimpleEnvironmentLoader '#{loader_name}')" + end + +end \ No newline at end of file diff --git a/lib/puppet/pops/loaders.rb b/lib/puppet/pops/loaders.rb index 051289ee6..1f1d43472 100644 --- a/lib/puppet/pops/loaders.rb +++ b/lib/puppet/pops/loaders.rb @@ -1,224 +1,225 @@ class Puppet::Pops::Loaders class LoaderError < Puppet::Error; end attr_reader :static_loader attr_reader :puppet_system_loader attr_reader :environment_loader def initialize() # The static loader can only be changed after a reboot @@static_loader ||= Puppet::Pops::Loader::StaticLoader.new() # Create the set of loaders # 1. Puppet, loads from the "running" puppet - i.e. bundled functions, types, extension points and extensions # Does not change without rebooting the service running puppet. # @@puppet_system_loader ||= create_puppet_system_loader() # 2. Environment loader - i.e. what is bound across the environment, may change for each setup # TODO: loaders need to work when also running in an agent doing catalog application. There is no # concept of environment the same way as when running as a master (except when doing apply). # The creation mechanisms should probably differ between the two. # @environment_loader = create_environment_loader() # 3. module loaders are set up from the create_environment_loader, they register themselves end # Clears the cached static and puppet_system loaders (to enable testing) # def self.clear @@static_loader = nil @@puppet_system_loader = nil end def static_loader @@static_loader end def puppet_system_loader @@puppet_system_loader end def self.create_loaders() self.new() end def public_loader_for_module(module_name) md = @module_resolver[module_name] || (return nil) # Note, this loader is not resolved until it is asked to load something it may contain md.public_loader end def private_loader_for_module(module_name) md = @module_resolver[module_name] || (return nil) unless md.resolved? @module_resolver.resolve(md) end md.private_loader end private def create_puppet_system_loader() module_name = nil loader_name = 'puppet_system' # Puppet system may be installed in a fixed location via RPM, installed as a Gem, via source etc. # The only way to find this across the different ways puppet can be installed is # to search up the path from this source file's __FILE__ location until it finds the parent of # lib/puppet... e.g.. dirname(__FILE__)/../../.. (i.e. /lib/puppet/pops/loaders.rb). # puppet_lib = File.join(File.dirname(__FILE__), '../../..') Puppet::Pops::Loader::ModuleLoaders::FileBased.new(static_loader, module_name, puppet_lib, loader_name) end def create_environment_loader() # This defines where to start parsing/evaluating - the "initial import" (to use 3x terminology) # Is either a reference to a single .pp file, or a directory of manifests. If the environment becomes # a module and can hold functions, types etc. then these are available across all other modules without # them declaring this dependency - it is however valuable to be able to treat it the same way # bindings and other such system related configuration. # This is further complicated by the many options available: # - The environment may not have a directory, the code comes from one appointed 'manifest' (site.pp) # - The environment may have a directory and also point to a 'manifest' # - The code to run may be set in settings (code) # Further complication is that there is nothing specifying what the visibility is into # available modules. (3x is everyone sees everything). # Puppet binder currently reads confdir/bindings - that is bad, it should be using the new environment support. current_environment = Puppet.lookup(:current_environment) # The environment is not a namespace, so give it a nil "module_name" module_name = nil loader_name = "environment:#{current_environment.name}" env_dir = Puppet[:environmentdir] if env_dir.nil? - loader = Puppet::Pops::Loader::NullLoader.new(puppet_system_loader, loader_name) + # Use an environment loader that can be populated externally + loader = Puppet::Pops::Loader::SimpleEnvironmentLoader.new(puppet_system_loader, loader_name) else envdir_path = File.join(env_dir, current_environment.name.to_s) # TODO: Representing Environment as a Module - needs something different (not all types are supported), # and it must be able to import .pp code from 3x manifest setting, or from code setting as well as from # a manifests directory under the environment's root. The below is cheating... # loader = Puppet::Pops::Loader::ModuleLoaders::FileBased(puppet_system_loader, module_name, envdir_path, loader_name) end # An environment has a module path even if it has a null loader configure_loaders_for_modules(loader, current_environment) loader end def configure_loaders_for_modules(parent_loader, current_environment) @module_resolver = mr = ModuleResolver.new() current_environment.modules.each do |puppet_module| # Create data about this module md = LoaderModuleData.new(puppet_module) mr[puppet_module.name] = md md.public_loader = Puppet::Pops::Loader::ModuleLoaders::FileBased.new(parent_loader, md.name, md.path, md.name) end end # =LoaderModuleData # Information about a Module and its loaders. # TODO: should have reference to real model element containing all module data; this is faking it # TODO: Should use Puppet::Module to get the metadata (as a hash) - a somewhat blunt instrument, but that is # what is available with a reasonable API. # class LoaderModuleData attr_accessor :state attr_accessor :public_loader attr_accessor :private_loader attr_accessor :resolutions # The Puppet::Module this LoaderModuleData represents in the loader configuration attr_reader :puppet_module # @param puppet_module [Puppet::Module] the module instance for the module being represented # def initialize(puppet_module) @state = :initial @puppet_module = puppet_module @resolutions = [] @loader = nil @private_loader = nil end def name @puppet_module.name end def version @puppet_module.version end def path @puppet_module.path end def requirements nil # FAKE: this says "wants to see everything" end def resolved? @state == :resolved end end # Resolves module loaders - resolution of model dependencies is done by Puppet::Module # class ModuleResolver def initialize() @index = {} @all_module_loaders = nil end def [](name) @index[name] end def []=(name, module_data) @index[name] = module_data end def all_module_loaders @all_module_loaders ||= @index.map {|md| md.loader } end def resolve(module_data) return if module_data.resolved? pm = module_data.puppet_module # Resolution rules # If dependencies.nil? means "see all other modules" (This to make older modules work, and modules w/o metadata) # TODO: Control via flag/feature ? module_data.private_loader = if pm.dependencies.nil? # see everything if Puppet::Util::Log.level == :debug Puppet.debug("ModuleLoader: module '#{module_data.name}' has unknown dependencies - it will have all other modules visible") end Puppet::Pops::Loader::DependencyLoader.new(module_data.loader, module_data.name, all_module_loaders()) else # If module has resolutions they must resolve - it will not see into other modules otherwise # TODO: possible give errors if there are unresolved references # i.e. !pm.unmet_dependencies.empty? (if module lacks metadata it is considered to have met all). # The face "module" can display error information. # Here, we are just giving up without explaining - the user can check with the module face (or console) # unless pm.unmet_dependencies.empty? # TODO: Exception or just warning? Puppet.warning("ModuleLoader: module '#{module_data.name}' has unresolved dependencies"+ " - it will only see those that are resolved."+ " Use 'puppet module list --tree' to see information about modules") # raise Puppet::Pops::Loader::Loader::Error, "Loader Error: Module '#{module_data.name}' has unresolved dependencies - use 'puppet module list --tree' to see information" end dependency_loaders = pm.dependencies_as_modules.map { |dep| @index[dep.name].loader } Puppet::Pops::Loader::DependencyLoader.new(module_data.loader, module_data.name, dependency_loaders) end end end end \ No newline at end of file diff --git a/spec/unit/pops/loaders/loaders_spec.rb b/spec/unit/pops/loaders/loaders_spec.rb index bb7bf6eca..d8027b360 100644 --- a/spec/unit/pops/loaders/loaders_spec.rb +++ b/spec/unit/pops/loaders/loaders_spec.rb @@ -1,83 +1,83 @@ require 'spec_helper' require 'puppet_spec/files' require 'puppet/pops' require 'puppet/loaders' describe 'loaders' do include PuppetSpec::Files def config_dir(config_name) my_fixture(config_name) end # Loaders caches the puppet_system_loader, must reset between tests # before(:each) { Puppet::Pops::Loaders.clear() } it 'creates a puppet_system loader' do loaders = Puppet::Pops::Loaders.new() expect(loaders.puppet_system_loader().class).to be(Puppet::Pops::Loader::ModuleLoaders::FileBased) end it 'creates an environment loader' do loaders = Puppet::Pops::Loaders.new() # When this test is running, there is no environments dir configured, and a NullLoader is therefore used a.t.m - expect(loaders.environment_loader().class).to be(Puppet::Pops::Loader::NullLoader) + expect(loaders.environment_loader().class).to be(Puppet::Pops::Loader::SimpleEnvironmentLoader) # The default name of the enironment is '*root*', and the loader should identify itself that way - expect(loaders.environment_loader().to_s).to eql("(NullLoader 'environment:*root*')") + expect(loaders.environment_loader().to_s).to eql("(SimpleEnvironmentLoader 'environment:*root*')") end context 'when delegating 3x to 4x' do before(:each) { Puppet[:biff] = true } it 'the puppet system loader can load 3x functions' do loaders = Puppet::Pops::Loaders.new() puppet_loader = loaders.puppet_system_loader() function = puppet_loader.load_typed(typed_name(:function, 'sprintf')).value expect(function.class.name).to eq('sprintf') expect(function.is_a?(Puppet::Functions::Function)).to eq(true) end end # TODO: LOADING OF MODULES ON MODULEPATH context 'loading from path with single module' do before do env = Puppet::Node::Environment.create(:'*test*', [File.join(config_dir('single_module'), 'modules')], '') overrides = { :current_environment => env } Puppet.push_context(overrides, "single-module-test-loaders") end after do Puppet.pop_context() end it 'can load from a module path' do loaders = Puppet::Pops::Loaders.new() modulea_loader = loaders.public_loader_for_module('modulea') expect(modulea_loader.class).to eql(Puppet::Pops::Loader::ModuleLoaders::FileBased) function = modulea_loader.load_typed(typed_name(:function, 'modulea::func_a')).value expect(function.is_a?(Puppet::Functions::Function)).to eq(true) expect(function.class.name).to eq('modulea::func_a') function = modulea_loader.load_typed(typed_name(:function, 'modulea::nested::func_a')).value expect(function.is_a?(Puppet::Functions::Function)).to eq(true) expect(function.class.name).to eq('modulea::nested::func_a') function = modulea_loader.load_typed(typed_name(:function, 'rb_func_a')).value expect(function.is_a?(Puppet::Functions::Function)).to eq(true) expect(function.class.name).to eq('rb_func_a') function = modulea_loader.load_typed(typed_name(:function, 'modulea::rb_func_a')).value expect(function.is_a?(Puppet::Functions::Function)).to eq(true) expect(function.class.name).to eq('modulea::rb_func_a') end end def typed_name(type, name) Puppet::Pops::Loader::Loader::TypedName.new(type, name) end end \ No newline at end of file