diff --git a/lib/puppet/parser/ast/pops_bridge.rb b/lib/puppet/parser/ast/pops_bridge.rb index b4177b2bf..6b1449137 100644 --- a/lib/puppet/parser/ast/pops_bridge.rb +++ b/lib/puppet/parser/ast/pops_bridge.rb @@ -1,245 +1,245 @@ require 'puppet/parser/ast/top_level_construct' require 'puppet/pops' # The AST::Bridge contains classes that bridges between the new Pops based model # and the 3.x AST. This is required to be able to reuse the Puppet::Resource::Type which is # fundamental for the rest of the logic. # class Puppet::Parser::AST::PopsBridge # Bridges to one Pops Model Expression # The @value is the expression # This is used to represent the body of a class, definition, or node, and for each parameter's default value # expression. # class Expression < Puppet::Parser::AST::Leaf def initialize args super @@evaluator ||= Puppet::Pops::Parser::EvaluatingParser::Transitional.new() end def to_s Puppet::Pops::Model::ModelTreeDumper.new.dump(@value) end def evaluate(scope) @@evaluator.evaluate(scope, @value) end # Adapts to 3x where top level constructs needs to have each to iterate over children. Short circuit this # by yielding self. By adding this there is no need to wrap a pops expression inside an AST::BlockExpression # def each yield self end def sequence_with(other) if value.nil? # This happens when testing and not having a complete setup other else # When does this happen ? Ever ? raise "sequence_with called on Puppet::Parser::AST::PopsBridge::Expression - please report use case" # What should be done if the above happens (We don't want this to happen). # Puppet::Parser::AST::BlockExpression.new(:children => [self] + other.children) end end # The 3x requires code plugged in to an AST to have this in certain positions in the tree. The purpose # is to either print the content, or to look for things that needs to be defined. This implementation # cheats by always returning an empty array. (This allows simple files to not require a "Program" at the top. # def children [] end end class NilAsUndefExpression < Expression def evaluate(scope) result = super result.nil? ? :undef : result end end # Bridges the top level "Program" produced by the pops parser. # Its main purpose is to give one point where all definitions are instantiated (actually defined since the # Puppet 3x terminology is somewhat misleading - the definitions are instantiated, but instances of the created types # are not created, that happens when classes are included / required, nodes are matched and when resources are instantiated # by a resource expression (which is also used to instantiate a host class). # class Program < Puppet::Parser::AST::TopLevelConstruct attr_reader :program_model, :context def initialize(program_model, context = {}) @program_model = program_model @context = context @ast_transformer ||= Puppet::Pops::Model::AstTransformer.new(@context[:file]) @@evaluator ||= Puppet::Pops::Parser::EvaluatingParser::Transitional.new() end # This is the 3x API, the 3x AST searches through all code to find the instructions that can be instantiated. # This Pops-model based instantiation relies on the parser to build this list while parsing (which is more # efficient as it avoids one full scan of all logic via recursive enumeration/yield) # def instantiate(modname) @program_model.definitions.collect do |d| case d when Puppet::Pops::Model::HostClassDefinition instantiate_HostClassDefinition(d, modname) when Puppet::Pops::Model::ResourceTypeDefinition instantiate_ResourceTypeDefinition(d, modname) when Puppet::Pops::Model::NodeDefinition instantiate_NodeDefinition(d, modname) else raise Puppet::ParseError, "Internal Error: Unknown type of definition - got '#{d.class}'" end end.flatten().compact() # flatten since node definition may have returned an array # Compact since functions are not understood by compiler end def evaluate(scope) @@evaluator.evaluate(scope, program_model) end # Adapts to 3x where top level constructs needs to have each to iterate over children. Short circuit this # by yielding self. This means that the HostClass container will call this bridge instance with `instantiate`. # def each yield self end private def instantiate_Parameter(o) # 3x needs parameters as an array of `[name]` or `[name, value_expr]` # One problem is that the parameter evaluation takes place in the wrong context in 3x (the caller's and # can thus reference all sorts of information. Here the value expression is wrapped in an AST Bridge to a Pops # expression since the Pops side can not control the evaluation if o.value [ o.name, NilAsUndefExpression.new(:value => o.value) ] else [ o.name ] end end def create_type_map(definition) result = {} # No need to do anything if there are no parameters return result unless definition.parameters.size > 0 # No need to do anything if there are no typed parameters typed_parameters = definition.parameters.select {|p| p.type_expr } return result if typed_parameters.empty? # If there are typed parameters, they need to be evaluated to produce the corresponding type - # instances. This evaluation requires a scope. If wed do not get one, the task is impossible to - # carry out, and a warning is logged. + # instances. This evaluation requires a scope. A scope is not available when doing deserialization + # (there is also no initialized evaluator). When running apply and test however, the environment is + # reused and we may reenter without a scope (which is fine). A debug message is then output in case + # there is the need to track down the odd corner case. See {#obtain_scope}. # return result unless scope = obtain_scope typed_parameters.each do |p| result[ p.name ] = @@evaluator.evaluate(scope, p.type_expr) if p.type_expr end result end # Obtains the scope or issues a warning if :global_scope is not bound def obtain_scope scope = Puppet.lookup(:global_scope) do - # This can occur when testing, it is not sure if this is a valid real world use case - # This may occur when deserializing a catalog - which is not a disaster - # but may result in values being given to puppet logic that has not been type checked when there is an expectancy - # to do so. If the catalog that is deserialized is produced by a version of the type where type checking - # was not in place, it may result in errors. - # - Puppet.warning("Instantiating Resource with type checked parameters - scope is missing, cannot perform type checking, please report use case") + # This occurs when testing and when applying a catalog (there is no scope available then), and + # when running tests that run a partial setup. + # This is bad if the logic is trying to compile, but a warning can not be issues since it is a normal + # use case that there is no scope when requesting the type in order to just get the parameters. + Puppet.debug("Instantiating Resource with type checked parameters - scope is missing, cannot perform type checking, please report use case") nil end scope end # Produces a hash with data for Definition and HostClass def args_from_definition(o, modname) args = { :arguments => o.parameters.collect {|p| instantiate_Parameter(p) }, :argument_types => create_type_map(o), :module_name => modname } unless is_nop?(o.body) args[:code] = Expression.new(:value => o.body) end @ast_transformer.merge_location(args, o) end def instantiate_HostClassDefinition(o, modname) args = args_from_definition(o, modname) args[:parent] = o.parent_class Puppet::Resource::Type.new(:hostclass, o.name, @context.merge(args)) end def instantiate_ResourceTypeDefinition(o, modname) Puppet::Resource::Type.new(:definition, o.name, @context.merge(args_from_definition(o, modname))) end def instantiate_NodeDefinition(o, modname) args = { :module_name => modname } unless is_nop?(o.body) args[:code] = Expression.new(:value => o.body) end unless is_nop?(o.parent) args[:parent] = @ast_transformer.hostname(o.parent) end host_matches = @ast_transformer.hostname(o.host_matches) @ast_transformer.merge_location(args, o) host_matches.collect do |name| Puppet::Resource::Type.new(:node, name, @context.merge(args)) end end # Propagates a found Function to the appropriate loader. # This is for 4x future-evaluator/loader # def instantiate_FunctionDefinition(function_definition, modname) loaders = (Puppet.lookup(:loaders) { nil }) unless loaders raise Puppet::ParseError, "Internal Error: Puppet Context ':loaders' missing - cannot define any functions" end loader = if modname.nil? || modname == "" # TODO : Later when functions can be private, a decision is needed regarding what that means. # A private environment loader could be used for logic outside of modules, then only that logic # would see the function. # # Use the private loader, this function may see the environment's dependencies (currently, all modules) loaders.private_environment_loader() else # TODO : Later check if function is private, and then add it to # private_loader_for_module # loaders.public_loader_for_module(modname) end unless loader raise Puppet::ParseError, "Internal Error: did not find public loader for module: '#{modname}'" end # Instantiate Function, and store it in the environment loader typed_name, f = Puppet::Pops::Loader::PuppetFunctionInstantiator.create_from_model(function_definition, loader) loader.set_entry(typed_name, f, Puppet::Pops::Adapters::SourcePosAdapter.adapt(function_definition).to_uri) nil # do not want the function to inadvertently leak into 3x end def code() Expression.new(:value => @value) end def is_nop?(o) @ast_transformer.is_nop?(o) end end end diff --git a/lib/puppet/resource/type.rb b/lib/puppet/resource/type.rb index e58cfd456..a1ae98830 100644 --- a/lib/puppet/resource/type.rb +++ b/lib/puppet/resource/type.rb @@ -1,413 +1,413 @@ require 'puppet/parser' require 'puppet/util/warnings' require 'puppet/util/errors' require 'puppet/util/inline_docs' require 'puppet/parser/ast/leaf' require 'puppet/parser/ast/block_expression' require 'puppet/dsl' # Puppet::Resource::Type represents nodes, classes and defined types. # # It has a standard format for external consumption, usable from the # resource_type indirection via rest and the resource_type face. See the # {file:api_docs/http_resource_type.md#Schema resource type schema # description}. # # @api public class Puppet::Resource::Type Puppet::ResourceType = self include Puppet::Util::InlineDocs include Puppet::Util::Warnings include Puppet::Util::Errors RESOURCE_KINDS = [:hostclass, :node, :definition] # Map the names used in our documentation to the names used internally RESOURCE_KINDS_TO_EXTERNAL_NAMES = { :hostclass => "class", :node => "node", :definition => "defined_type", } RESOURCE_EXTERNAL_NAMES_TO_KINDS = RESOURCE_KINDS_TO_EXTERNAL_NAMES.invert attr_accessor :file, :line, :doc, :code, :ruby_code, :parent, :resource_type_collection attr_reader :namespace, :arguments, :behaves_like, :module_name # Map from argument (aka parameter) names to Puppet Type # @return [Hash :parser def self.from_data_hash(data) name = data.delete('name') or raise ArgumentError, "Resource Type names must be specified" kind = data.delete('kind') || "definition" unless type = RESOURCE_EXTERNAL_NAMES_TO_KINDS[kind] raise ArgumentError, "Unsupported resource kind '#{kind}'" end data = data.inject({}) { |result, ary| result[ary[0].intern] = ary[1]; result } # External documentation uses "parameters" but the internal name # is "arguments" data[:arguments] = data.delete(:parameters) new(type, name, data) end def self.from_pson(data) Puppet.deprecation_warning("from_pson is being removed in favour of from_data_hash.") self.from_data_hash(data) end def to_data_hash data = [:doc, :line, :file, :parent].inject({}) do |hash, param| next hash unless (value = self.send(param)) and (value != "") hash[param.to_s] = value hash end # External documentation uses "parameters" but the internal name # is "arguments" data['parameters'] = arguments.dup unless arguments.empty? data['name'] = name unless RESOURCE_KINDS_TO_EXTERNAL_NAMES.has_key?(type) raise ArgumentError, "Unsupported resource kind '#{type}'" end data['kind'] = RESOURCE_KINDS_TO_EXTERNAL_NAMES[type] data end # Are we a child of the passed class? Do a recursive search up our # parentage tree to figure it out. def child_of?(klass) return false unless parent return(klass == parent_type ? true : parent_type.child_of?(klass)) end # Now evaluate the code associated with this class or definition. def evaluate_code(resource) static_parent = evaluate_parent_type(resource) scope = static_parent || resource.scope scope = scope.newscope(:namespace => namespace, :source => self, :resource => resource) unless resource.title == :main scope.compiler.add_class(name) unless definition? set_resource_parameters(resource, scope) resource.add_edge_to_stage if code if @match # Only bother setting up the ephemeral scope if there are match variables to add into it begin elevel = scope.ephemeral_level scope.ephemeral_from(@match, file, line) code.safeevaluate(scope) ensure scope.unset_ephemeral_var(elevel) end else code.safeevaluate(scope) end end evaluate_ruby_code(resource, scope) if ruby_code end def initialize(type, name, options = {}) @type = type.to_s.downcase.to_sym raise ArgumentError, "Invalid resource supertype '#{type}'" unless RESOURCE_KINDS.include?(@type) name = convert_from_ast(name) if name.is_a?(Puppet::Parser::AST::HostName) set_name_and_namespace(name) [:code, :doc, :line, :file, :parent].each do |param| next unless value = options[param] send(param.to_s + "=", value) end set_arguments(options[:arguments]) set_argument_types(options[:argument_types]) @match = nil @module_name = options[:module_name] end # This is only used for node names, and really only when the node name # is a regexp. def match(string) return string.to_s.downcase == name unless name_is_regex? @match = @name.match(string) end # Add code from a new instance to our code. def merge(other) fail "#{name} is not a class; cannot add code to it" unless type == :hostclass fail "#{other.name} is not a class; cannot add code from it" unless other.type == :hostclass fail "Cannot have code outside of a class/node/define because 'freeze_main' is enabled" if name == "" and Puppet.settings[:freeze_main] if parent and other.parent and parent != other.parent fail "Cannot merge classes with different parent classes (#{name} => #{parent} vs. #{other.name} => #{other.parent})" end # We know they're either equal or only one is set, so keep whichever parent is specified. self.parent ||= other.parent if other.doc self.doc ||= "" self.doc += other.doc end # This might just be an empty, stub class. return unless other.code unless self.code self.code = other.code return end self.code = Puppet::Parser::ParserFactory.code_merger.concatenate([self, other]) # self.code = self.code.sequence_with(other.code) end # Make an instance of the resource type, and place it in the catalog # if it isn't in the catalog already. This is only possible for # classes and nodes. No parameters are be supplied--if this is a # parameterized class, then all parameters take on their default # values. def ensure_in_catalog(scope, parameters=nil) type == :definition and raise ArgumentError, "Cannot create resources for defined resource types" resource_type = type == :hostclass ? :class : :node # Do nothing if the resource already exists; this makes sure we don't # get multiple copies of the class resource, which helps provide the # singleton nature of classes. # we should not do this for classes with parameters # if parameters are passed, we should still try to create the resource # even if it exists so that we can fail # this prevents us from being able to combine param classes with include if resource = scope.catalog.resource(resource_type, name) and !parameters return resource end resource = Puppet::Parser::Resource.new(resource_type, name, :scope => scope, :source => self) assign_parameter_values(parameters, resource) instantiate_resource(scope, resource) scope.compiler.add_resource(scope, resource) resource end def instantiate_resource(scope, resource) # Make sure our parent class has been evaluated, if we have one. if parent && !scope.catalog.resource(resource.type, parent) parent_type(scope).ensure_in_catalog(scope) end if ['Class', 'Node'].include? resource.type scope.catalog.tag(*resource.tags) end end def name return @name unless @name.is_a?(Regexp) @name.source.downcase.gsub(/[^-\w:.]/,'').sub(/^\.+/,'') end def name_is_regex? @name.is_a?(Regexp) end def assign_parameter_values(parameters, resource) return unless parameters # It'd be nice to assign default parameter values here, # but we can't because they often rely on local variables # created during set_resource_parameters. parameters.each do |name, value| resource.set_parameter name, value end end # MQR TODO: # # The change(s) introduced by the fix for #4270 are mostly silly & should be # removed, though we didn't realize it at the time. If it can be established/ # ensured that nodes never call parent_type and that resource_types are always # (as they should be) members of exactly one resource_type_collection the # following method could / should be replaced with: # # def parent_type # @parent_type ||= parent && ( # resource_type_collection.find_or_load([name],parent,type.to_sym) || # fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{resource_type_collection.environment}" # ) # end # # ...and then the rest of the changes around passing in scope reverted. # def parent_type(scope = nil) return nil unless parent unless @parent_type raise "Must pass scope to parent_type when called first time" unless scope unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", [name], parent) fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{scope.environment}" end end @parent_type end # Set any arguments passed by the resource as variables in the scope. def set_resource_parameters(resource, scope) set = {} resource.to_hash.each do |param, value| param = param.to_sym fail Puppet::ParseError, "#{resource.ref} does not accept attribute #{param}" unless valid_parameter?(param) exceptwrap { scope[param.to_s] = value } set[param] = true end if @type == :hostclass scope["title"] = resource.title.to_s.downcase unless set.include? :title scope["name"] = resource.name.to_s.downcase unless set.include? :name else scope["title"] = resource.title unless set.include? :title scope["name"] = resource.name unless set.include? :name end scope["module_name"] = module_name if module_name and ! set.include? :module_name if caller_name = scope.parent_module_name and ! set.include?(:caller_module_name) scope["caller_module_name"] = caller_name end scope.class_set(self.name,scope) if hostclass? or node? # Evaluate the default parameters, now that all other variables are set default_params = resource.set_default_parameters(scope) default_params.each { |param| scope[param] = resource[param] } # This has to come after the above parameters so that default values # can use their values resource.validate_complete end # Check whether a given argument is valid. def valid_parameter?(param) param = param.to_s return true if param == "name" return true if Puppet::Type.metaparam?(param) return false unless defined?(@arguments) return(arguments.include?(param) ? true : false) end def set_arguments(arguments) @arguments = {} return if arguments.nil? arguments.each do |arg, default| arg = arg.to_s warn_if_metaparam(arg, default) @arguments[arg] = default end end # Sets the argument name to Puppet Type hash used for type checking. # Names must correspond to available arguments (they must be defined first). # Arguments not mentioned will not be type-checked. Only supported when parser == "future" # def set_argument_types(name_to_type_hash) @argument_types = {} # Stop here if not running under future parser, the rest requires pops to be initialized # and that the type system is available return unless Puppet[:parser] == 'future' && name_to_type_hash name_to_type_hash.each do |name, t| # catch internal errors unless @arguments.include?(name) raise Puppet::DevError, "Parameter '#{name}' is given a type, but is not a valid parameter." end unless t.is_a? Puppet::Pops::Types::PAbstractType raise Puppet::DevError, "Parameter '#{name}' is given a type that is not a Puppet Type, got #{t.class}" end - @argument_types[name] = resulting_type + @argument_types[name] = t end end private def convert_from_ast(name) value = name.value if value.is_a?(Puppet::Parser::AST::Regex) name = value.value else name = value end end def evaluate_parent_type(resource) return unless klass = parent_type(resource.scope) and parent_resource = resource.scope.compiler.catalog.resource(:class, klass.name) || resource.scope.compiler.catalog.resource(:node, klass.name) parent_resource.evaluate unless parent_resource.evaluated? parent_scope(resource.scope, klass) end def evaluate_ruby_code(resource, scope) Puppet::DSL::ResourceAPI.new(resource, scope, ruby_code).evaluate end # Split an fq name into a namespace and name def namesplit(fullname) ary = fullname.split("::") n = ary.pop || "" ns = ary.join("::") return ns, n end def parent_scope(scope, klass) scope.class_scope(klass) || raise(Puppet::DevError, "Could not find scope for #{klass.name}") end def set_name_and_namespace(name) if name.is_a?(Regexp) @name = name @namespace = "" else @name = name.to_s.downcase # Note we're doing something somewhat weird here -- we're setting # the class's namespace to its fully qualified name. This means # anything inside that class starts looking in that namespace first. @namespace, ignored_shortname = @type == :hostclass ? [@name, ''] : namesplit(@name) end end def warn_if_metaparam(param, default) return unless Puppet::Type.metaparamclass(param) if default warnonce "#{param} is a metaparam; this value will inherit to all contained resources in the #{self.name} definition" else raise Puppet::ParseError, "#{param} is a metaparameter; please choose another parameter name in the #{self.name} definition" end end end diff --git a/spec/integration/parser/catalog_spec.rb b/spec/integration/parser/catalog_spec.rb index 41969b2a1..21f31d856 100644 --- a/spec/integration/parser/catalog_spec.rb +++ b/spec/integration/parser/catalog_spec.rb @@ -1,130 +1,125 @@ require 'spec_helper' require 'matchers/include_in_order' require 'puppet_spec/compiler' require 'puppet/indirector/catalog/compiler' describe "A catalog" do include PuppetSpec::Compiler shared_examples_for "when compiled" do context "when transmitted to the agent" do it "preserves the order in which the resources are added to the catalog" do resources_in_declaration_order = ["Class[First]", "Second[position]", "Class[Third]", "Fourth[position]"] master_catalog, agent_catalog = master_and_agent_catalogs_for(<<-EOM) define fourth() { } class third { } define second() { fourth { "position": } } class first { second { "position": } class { "third": } } include first EOM expect(resources_in(master_catalog)). to include_in_order(*resources_in_declaration_order) expect(resources_in(agent_catalog)). to include_in_order(*resources_in_declaration_order) end it "does not contain unrealized, virtual resources" do virtual_resources = ["Unrealized[unreal]", "Class[Unreal]"] master_catalog, agent_catalog = master_and_agent_catalogs_for(<<-EOM) class unreal { } define unrealized() { } class real { @unrealized { "unreal": } @class { "unreal": } } include real EOM expect(resources_in(master_catalog)).to_not include(*virtual_resources) expect(resources_in(agent_catalog)).to_not include(*virtual_resources) end it "does not contain unrealized, exported resources" do exported_resources = ["Unrealized[unreal]", "Class[Unreal]"] master_catalog, agent_catalog = master_and_agent_catalogs_for(<<-EOM) class unreal { } define unrealized() { } class real { @@unrealized { "unreal": } @@class { "unreal": } } include real EOM expect(resources_in(master_catalog)).to_not include(*exported_resources) expect(resources_in(agent_catalog)).to_not include(*exported_resources) end end it "compiles resource creation from appended array as two separate resources" do # moved here from acceptance test "jeff_append_to_array.rb" master_catalog = master_catalog_for(<<-EOM) class parent { $arr1 = [ "parent array element" ] } class parent::child inherits parent { $arr1 += ["child array element"] notify { $arr1: } } include parent::child EOM expect(resources_in(master_catalog)).to include('Notify[parent array element]', 'Notify[child array element]') end end describe 'using classic parser' do before :each do Puppet[:parser] = 'current' end it_behaves_like 'when compiled' do end end describe 'using future parser' do before :each do Puppet[:parser] = 'future' end it_behaves_like 'when compiled' do end end def master_catalog_for(manifest) master_catalog = Puppet::Resource::Catalog::Compiler.new.filter(compile_to_catalog(manifest)) end def master_and_agent_catalogs_for(manifest) - # There are issues with this sequence, the compilation via the indirector work - # as expected, it sets up the context and compiles. The second use of the catalog, - # to convert it again goes into initial_import, and performs evaluation but without - # a fully configured context. - # compiler = Puppet::Resource::Catalog::Compiler.new master_catalog = compiler.filter(compile_to_catalog(manifest)) agent_catalog = Puppet::Resource::Catalog.convert_from(:pson, master_catalog.render(:pson)) [master_catalog, agent_catalog] end def resources_in(catalog) catalog.resources.map(&:ref) end end