diff --git a/lib/puppet/resource/catalog.rb b/lib/puppet/resource/catalog.rb index 7de7df8dc..5a67f2574 100644 --- a/lib/puppet/resource/catalog.rb +++ b/lib/puppet/resource/catalog.rb @@ -1,549 +1,552 @@ require 'puppet/node' require 'puppet/indirector' require 'puppet/transaction' require 'puppet/util/pson' require 'puppet/util/tagging' require 'puppet/graph' # This class models a node catalog. It is the thing meant to be passed # from server to client, and it contains all of the information in the # catalog, including the resources and the relationships between them. # # @api public class Puppet::Resource::Catalog < Puppet::Graph::SimpleGraph class DuplicateResourceError < Puppet::Error include Puppet::ExternalFileError end extend Puppet::Indirector indirects :catalog, :terminus_setting => :catalog_terminus include Puppet::Util::Tagging extend Puppet::Util::Pson # The host name this is a catalog for. attr_accessor :name # The catalog version. Used for testing whether a catalog # is up to date. attr_accessor :version # How long this catalog took to retrieve. Used for reporting stats. attr_accessor :retrieval_duration # Whether this is a host catalog, which behaves very differently. # In particular, reports are sent, graphs are made, and state is # stored in the state database. If this is set incorrectly, then you often # end up in infinite loops, because catalogs are used to make things # that the host catalog needs. attr_accessor :host_config # Whether this catalog was retrieved from the cache, which affects # whether it is written back out again. attr_accessor :from_cache # Some metadata to help us compile and generally respond to the current state. attr_accessor :client_version, :server_version # A String representing the environment for this catalog attr_accessor :environment # The actual environment instance that was used during compilation attr_accessor :environment_instance # Add classes to our class list. def add_class(*classes) classes.each do |klass| @classes << klass end # Add the class names as tags, too. tag(*classes) end def title_key_for_ref( ref ) ref =~ /^([-\w:]+)\[(.*)\]$/m [$1, $2] end def add_resource(*resources) resources.each do |resource| add_one_resource(resource) end end # @param resource [A Resource] a resource in the catalog # @return [A Resource, nil] the resource that contains the given resource # @api public def container_of(resource) adjacent(resource, :direction => :in)[0] end def add_one_resource(resource) fail_on_duplicate_type_and_title(resource) add_resource_to_table(resource) create_resource_aliases(resource) resource.catalog = self if resource.respond_to?(:catalog=) add_resource_to_graph(resource) end private :add_one_resource def add_resource_to_table(resource) title_key = title_key_for_ref(resource.ref) @resource_table[title_key] = resource @resources << title_key end private :add_resource_to_table def add_resource_to_graph(resource) add_vertex(resource) @relationship_graph.add_vertex(resource) if @relationship_graph end private :add_resource_to_graph def create_resource_aliases(resource) if resource.respond_to?(:isomorphic?) and resource.isomorphic? and resource.name != resource.title self.alias(resource, resource.uniqueness_key) end end private :create_resource_aliases # Create an alias for a resource. def alias(resource, key) resource.ref =~ /^(.+)\[/ class_name = $1 || resource.class.name newref = [class_name, key].flatten if key.is_a? String ref_string = "#{class_name}[#{key}]" return if ref_string == resource.ref end # LAK:NOTE It's important that we directly compare the references, # because sometimes an alias is created before the resource is # added to the catalog, so comparing inside the below if block # isn't sufficient. if existing = @resource_table[newref] return if existing == resource resource_declaration = " at #{resource.file}:#{resource.line}" if resource.file and resource.line existing_declaration = " at #{existing.file}:#{existing.line}" if existing.file and existing.line msg = "Cannot alias #{resource.ref} to #{key.inspect}#{resource_declaration}; resource #{newref.inspect} already declared#{existing_declaration}" raise ArgumentError, msg end @resource_table[newref] = resource @aliases[resource.ref] ||= [] @aliases[resource.ref] << newref end # Apply our catalog to the local host. # @param options [Hash{Symbol => Object}] a hash of options # @option options [Puppet::Transaction::Report] :report # The report object to log this transaction to. This is optional, # and the resulting transaction will create a report if not # supplied. # @option options [Array[String]] :tags # Tags used to filter the transaction. If supplied then only # resources tagged with any of these tags will be evaluated. # @option options [Boolean] :ignoreschedules # Ignore schedules when evaluating resources # @option options [Boolean] :for_network_device # Whether this catalog is for a network device # # @return [Puppet::Transaction] the transaction created for this # application # # @api public def apply(options = {}) Puppet::Util::Storage.load if host_config? transaction = create_transaction(options) begin transaction.report.as_logging_destination do transaction.evaluate end rescue Puppet::Error => detail Puppet.log_exception(detail, "Could not apply complete catalog: #{detail}") rescue => detail Puppet.log_exception(detail, "Got an uncaught exception of type #{detail.class}: #{detail}") ensure # Don't try to store state unless we're a host config # too recursive. Puppet::Util::Storage.store if host_config? end yield transaction if block_given? transaction end # The relationship_graph form of the catalog. This contains all of the # dependency edges that are used for determining order. # + # @param given_prioritizer [Puppet::Graph::Prioritizer] The prioritization + # strategy to use when constructing the relationship graph. Defaults the + # being determined by the `ordering` setting. # @return [Puppet::Graph::RelationshipGraph] # @api public - def relationship_graph + def relationship_graph(given_prioritizer = nil) if @relationship_graph.nil? - @relationship_graph = Puppet::Graph::RelationshipGraph.new(prioritizer) + @relationship_graph = Puppet::Graph::RelationshipGraph.new(given_prioritizer || prioritizer) @relationship_graph.populate_from(self) end @relationship_graph end def clear(remove_resources = true) super() # We have to do this so that the resources clean themselves up. @resource_table.values.each { |resource| resource.remove } if remove_resources @resource_table.clear @resources = [] if @relationship_graph @relationship_graph.clear @relationship_graph = nil end end def classes @classes.dup end # Create a new resource and register it in the catalog. def create_resource(type, options) unless klass = Puppet::Type.type(type) raise ArgumentError, "Unknown resource type #{type}" end return unless resource = klass.new(options) add_resource(resource) resource end # Make sure all of our resources are "finished". def finalize make_default_resources @resource_table.values.each { |resource| resource.finish } write_graph(:resources) end def host_config? host_config end def initialize(name = nil, environment = Puppet::Node::Environment::NONE) super() @name = name @classes = [] @resource_table = {} @resources = [] @relationship_graph = nil @host_config = true @environment_instance = environment @environment = environment.to_s @aliases = {} if block_given? yield(self) finalize end end # Make the default objects necessary for function. def make_default_resources # We have to add the resources to the catalog, or else they won't get cleaned up after # the transaction. # First create the default scheduling objects Puppet::Type.type(:schedule).mkdefaultschedules.each { |res| add_resource(res) unless resource(res.ref) } # And filebuckets if bucket = Puppet::Type.type(:filebucket).mkdefaultbucket add_resource(bucket) unless resource(bucket.ref) end end # Remove the resource from our catalog. Notice that we also call # 'remove' on the resource, at least until resource classes no longer maintain # references to the resource instances. def remove_resource(*resources) resources.each do |resource| title_key = title_key_for_ref(resource.ref) @resource_table.delete(title_key) if aliases = @aliases[resource.ref] aliases.each { |res_alias| @resource_table.delete(res_alias) } @aliases.delete(resource.ref) end remove_vertex!(resource) if vertex?(resource) @relationship_graph.remove_vertex!(resource) if @relationship_graph and @relationship_graph.vertex?(resource) @resources.delete(title_key) resource.remove end end # Look a resource up by its reference (e.g., File[/etc/passwd]). def resource(type, title = nil) # Always create a resource reference, so that it always # canonicalizes how we are referring to them. if title res = Puppet::Resource.new(type, title) else # If they didn't provide a title, then we expect the first # argument to be of the form 'Class[name]', which our # Reference class canonicalizes for us. res = Puppet::Resource.new(nil, type) end res.catalog = self title_key = [res.type, res.title.to_s] uniqueness_key = [res.type, res.uniqueness_key].flatten @resource_table[title_key] || @resource_table[uniqueness_key] end def resource_refs resource_keys.collect{ |type, name| name.is_a?( String ) ? "#{type}[#{name}]" : nil}.compact end def resource_keys @resource_table.keys end def resources @resources.collect do |key| @resource_table[key] end end def self.from_data_hash(data) result = new(data['name'], Puppet::Node::Environment::NONE) if tags = data['tags'] result.tag(*tags) end if version = data['version'] result.version = version end if environment = data['environment'] result.environment = environment result.environment_instance = Puppet::Node::Environment.remote(environment.to_sym) end if resources = data['resources'] result.add_resource(*resources.collect do |res| Puppet::Resource.from_data_hash(res) end) end if edges = data['edges'] edges.each do |edge_hash| edge = Puppet::Relationship.from_data_hash(edge_hash) unless source = result.resource(edge.source) raise ArgumentError, "Could not intern from data: Could not find relationship source #{edge.source.inspect}" end edge.source = source unless target = result.resource(edge.target) raise ArgumentError, "Could not intern from data: Could not find relationship target #{edge.target.inspect}" end edge.target = target result.add_edge(edge) end end if classes = data['classes'] result.add_class(*classes) end result end def self.from_pson(data) Puppet.deprecation_warning("from_pson is being removed in favour of from_data_hash.") self.from_data_hash(data) end def to_data_hash { 'tags' => tags, 'name' => name, 'version' => version, 'environment' => environment.to_s, 'resources' => @resources.collect { |v| @resource_table[v].to_pson_data_hash }, 'edges' => edges. collect { |e| e.to_pson_data_hash }, 'classes' => classes } end PSON.register_document_type('Catalog',self) def to_pson_data_hash { 'document_type' => 'Catalog', 'data' => to_data_hash, 'metadata' => { 'api_version' => 1 } } end def to_pson(*args) to_pson_data_hash.to_pson(*args) end # Convert our catalog into a RAL catalog. def to_ral to_catalog :to_ral end # Convert our catalog into a catalog of Puppet::Resource instances. def to_resource to_catalog :to_resource end # filter out the catalog, applying +block+ to each resource. # If the block result is false, the resource will # be kept otherwise it will be skipped def filter(&block) to_catalog :to_resource, &block end # Store the classes in the classfile. def write_class_file ::File.open(Puppet[:classfile], "w") do |f| f.puts classes.join("\n") end rescue => detail Puppet.err "Could not create class file #{Puppet[:classfile]}: #{detail}" end # Store the list of resources we manage def write_resource_file ::File.open(Puppet[:resourcefile], "w") do |f| to_print = resources.map do |resource| next unless resource.managed? if resource.name_var "#{resource.type}[#{resource[resource.name_var]}]" else "#{resource.ref.downcase}" end end.compact f.puts to_print.join("\n") end rescue => detail Puppet.err "Could not create resource file #{Puppet[:resourcefile]}: #{detail}" end # Produce the graph files if requested. def write_graph(name) # We only want to graph the main host catalog. return unless host_config? super end private def prioritizer @prioritizer ||= case Puppet[:ordering] when "title-hash" Puppet::Graph::TitleHashPrioritizer.new when "manifest" Puppet::Graph::SequentialPrioritizer.new when "random" Puppet::Graph::RandomPrioritizer.new else raise Puppet::DevError, "Unknown ordering type #{Puppet[:ordering]}" end end def create_transaction(options) transaction = Puppet::Transaction.new(self, options[:report], prioritizer) transaction.tags = options[:tags] if options[:tags] transaction.ignoreschedules = true if options[:ignoreschedules] transaction.for_network_device = options[:network_device] transaction end # Verify that the given resource isn't declared elsewhere. def fail_on_duplicate_type_and_title(resource) # Short-circuit the common case, return unless existing_resource = @resource_table[title_key_for_ref(resource.ref)] # If we've gotten this far, it's a real conflict msg = "Duplicate declaration: #{resource.ref} is already declared" msg << " in file #{existing_resource.file}:#{existing_resource.line}" if existing_resource.file and existing_resource.line msg << "; cannot redeclare" raise DuplicateResourceError.new(msg, resource.file, resource.line) end # An abstracted method for converting one catalog into another type of catalog. # This pretty much just converts all of the resources from one class to another, using # a conversion method. def to_catalog(convert) result = self.class.new(self.name, self.environment_instance) result.version = self.version map = {} resources.each do |resource| next if virtual_not_exported?(resource) next if block_given? and yield resource newres = resource.copy_as_resource newres.catalog = result if convert != :to_resource newres = newres.to_ral end # We can't guarantee that resources don't munge their names # (like files do with trailing slashes), so we have to keep track # of what a resource got converted to. map[resource.ref] = newres result.add_resource newres end message = convert.to_s.gsub "_", " " edges.each do |edge| # Skip edges between virtual resources. next if virtual_not_exported?(edge.source) next if block_given? and yield edge.source next if virtual_not_exported?(edge.target) next if block_given? and yield edge.target unless source = map[edge.source.ref] raise Puppet::DevError, "Could not find resource #{edge.source.ref} when converting #{message} resources" end unless target = map[edge.target.ref] raise Puppet::DevError, "Could not find resource #{edge.target.ref} when converting #{message} resources" end result.add_edge(source, target, edge.label) end map.clear result.add_class(*self.classes) result.tag(*self.tags) result end def virtual_not_exported?(resource) resource.respond_to?(:virtual?) and resource.virtual? and (resource.respond_to?(:exported?) and not resource.exported?) end end diff --git a/lib/puppet/transaction.rb b/lib/puppet/transaction.rb index ab74ed385..983dbd307 100644 --- a/lib/puppet/transaction.rb +++ b/lib/puppet/transaction.rb @@ -1,340 +1,340 @@ require 'puppet' require 'puppet/util/tagging' require 'puppet/application' require 'digest/sha1' require 'set' # the class that actually walks our resource/property tree, collects the changes, # and performs them # # @api private class Puppet::Transaction require 'puppet/transaction/additional_resource_generator' require 'puppet/transaction/event' require 'puppet/transaction/event_manager' require 'puppet/transaction/resource_harness' require 'puppet/resource/status' attr_accessor :catalog, :ignoreschedules, :for_network_device # The report, once generated. attr_reader :report # Routes and stores any events and subscriptions. attr_reader :event_manager # Handles most of the actual interacting with resources attr_reader :resource_harness attr_reader :prefetched_providers include Puppet::Util include Puppet::Util::Tagging def initialize(catalog, report, prioritizer) @catalog = catalog @report = report || Puppet::Transaction::Report.new("apply", catalog.version, catalog.environment) @prioritizer = prioritizer @report.add_times(:config_retrieval, @catalog.retrieval_duration || 0) @event_manager = Puppet::Transaction::EventManager.new(self) @resource_harness = Puppet::Transaction::ResourceHarness.new(self) @prefetched_providers = Hash.new { |h,k| h[k] = {} } end # This method does all the actual work of running a transaction. It # collects all of the changes, executes them, and responds to any # necessary events. def evaluate(&block) block ||= method(:eval_resource) generator = AdditionalResourceGenerator.new(@catalog, relationship_graph, @prioritizer) @catalog.vertices.each { |resource| generator.generate_additional_resources(resource) } Puppet.info "Applying configuration version '#{catalog.version}'" if catalog.version continue_while = lambda { !stop_processing? } post_evalable_providers = Set.new pre_process = lambda do |resource| prov_class = resource.provider.class post_evalable_providers << prov_class if prov_class.respond_to?(:post_resource_eval) prefetch_if_necessary(resource) # If we generated resources, we don't know what they are now # blocking, so we opt to recompute it, rather than try to track every # change that would affect the number. relationship_graph.clear_blockers if generator.eval_generate(resource) end providerless_types = [] overly_deferred_resource_handler = lambda do |resource| # We don't automatically assign unsuitable providers, so if there # is one, it must have been selected by the user. if resource.provider resource.err "Provider #{resource.provider.class.name} is not functional on this host" else providerless_types << resource.type end resource_status(resource).failed = true end canceled_resource_handler = lambda do |resource| resource_status(resource).skipped = true resource.debug "Transaction canceled, skipping" end teardown = lambda do # Just once per type. No need to punish the user. providerless_types.uniq.each do |type| Puppet.err "Could not find a suitable provider for #{type}" end post_evalable_providers.each do |provider| begin provider.post_resource_eval rescue => detail Puppet.log_exception(detail, "post_resource_eval failed for provider #{provider}") end end end relationship_graph.traverse(:while => continue_while, :pre_process => pre_process, :overly_deferred_resource_handler => overly_deferred_resource_handler, :canceled_resource_handler => canceled_resource_handler, :teardown => teardown) do |resource| if resource.is_a?(Puppet::Type::Component) Puppet.warning "Somehow left a component in the relationship graph" else resource.info "Starting to evaluate the resource" if Puppet[:evaltrace] and @catalog.host_config? seconds = thinmark { block.call(resource) } resource.info "Evaluated in %0.2f seconds" % seconds if Puppet[:evaltrace] and @catalog.host_config? end end Puppet.debug "Finishing transaction #{object_id}" end # Wraps application run state check to flag need to interrupt processing def stop_processing? Puppet::Application.stop_requested? && catalog.host_config? end # Are there any failed resources in this transaction? def any_failed? report.resource_statuses.values.detect { |status| status.failed? } end # Find all of the changed resources. def changed? report.resource_statuses.values.find_all { |status| status.changed }.collect { |status| catalog.resource(status.resource) } end def relationship_graph - catalog.relationship_graph + catalog.relationship_graph(@prioritizer) end def resource_status(resource) report.resource_statuses[resource.to_s] || add_resource_status(Puppet::Resource::Status.new(resource)) end # The tags we should be checking. def tags self.tags = Puppet[:tags] unless defined?(@tags) super end def prefetch_if_necessary(resource) provider_class = resource.provider.class return unless provider_class.respond_to?(:prefetch) and !prefetched_providers[resource.type][provider_class.name] resources = resources_by_provider(resource.type, provider_class.name) if provider_class == resource.class.defaultprovider providerless_resources = resources_by_provider(resource.type, nil) providerless_resources.values.each {|res| res.provider = provider_class.name} resources.merge! providerless_resources end prefetch(provider_class, resources) end private # Apply all changes for a resource def apply(resource, ancestor = nil) status = resource_harness.evaluate(resource) add_resource_status(status) event_manager.queue_events(ancestor || resource, status.events) unless status.failed? rescue => detail resource.err "Could not evaluate: #{detail}" end # Evaluate a single resource. def eval_resource(resource, ancestor = nil) if skip?(resource) resource_status(resource).skipped = true else resource_status(resource).scheduled = true apply(resource, ancestor) end # Check to see if there are any events queued for this resource event_manager.process_events(resource) end def failed?(resource) s = resource_status(resource) and s.failed? end # Does this resource have any failed dependencies? def failed_dependencies?(resource) # First make sure there are no failed dependencies. To do this, # we check for failures in any of the vertexes above us. It's not # enough to check the immediate dependencies, which is why we use # a tree from the reversed graph. found_failed = false # When we introduced the :whit into the graph, to reduce the combinatorial # explosion of edges, we also ended up reporting failures for containers # like class and stage. This is undesirable; while just skipping the # output isn't perfect, it is RC-safe. --daniel 2011-06-07 suppress_report = (resource.class == Puppet::Type.type(:whit)) relationship_graph.dependencies(resource).each do |dep| next unless failed?(dep) found_failed = true # See above. --daniel 2011-06-06 unless suppress_report then resource.notice "Dependency #{dep} has failures: #{resource_status(dep).failed}" end end found_failed end # A general method for recursively generating new resources from a # resource. def generate_additional_resources(resource) return unless resource.respond_to?(:generate) begin made = resource.generate rescue => detail resource.log_exception(detail, "Failed to generate additional resources using 'generate': #{detail}") end return unless made made = [made] unless made.is_a?(Array) made.uniq.each do |res| begin res.tag(*resource.tags) @catalog.add_resource(res) res.finish add_conditional_directed_dependency(resource, res) generate_additional_resources(res) rescue Puppet::Resource::Catalog::DuplicateResourceError res.info "Duplicate generated resource; skipping" end end end # Should we ignore tags? def ignore_tags? ! @catalog.host_config? end def resources_by_provider(type_name, provider_name) unless @resources_by_provider @resources_by_provider = Hash.new { |h, k| h[k] = Hash.new { |h, k| h[k] = {} } } @catalog.vertices.each do |resource| if resource.class.attrclass(:provider) prov = resource.provider && resource.provider.class.name @resources_by_provider[resource.type][prov][resource.name] = resource end end end @resources_by_provider[type_name][provider_name] || {} end # Prefetch any providers that support it, yo. We don't support prefetching # types, just providers. def prefetch(provider_class, resources) type_name = provider_class.resource_type.name return if @prefetched_providers[type_name][provider_class.name] Puppet.debug "Prefetching #{provider_class.name} resources for #{type_name}" begin provider_class.prefetch(resources) rescue => detail Puppet.log_exception(detail, "Could not prefetch #{type_name} provider '#{provider_class.name}': #{detail}") end @prefetched_providers[type_name][provider_class.name] = true end def add_resource_status(status) report.add_resource_status(status) end # Is the resource currently scheduled? def scheduled?(resource) self.ignoreschedules or resource_harness.scheduled?(resource) end # Should this resource be skipped? def skip?(resource) if missing_tags?(resource) resource.debug "Not tagged with #{tags.join(", ")}" elsif ! scheduled?(resource) resource.debug "Not scheduled" elsif failed_dependencies?(resource) # When we introduced the :whit into the graph, to reduce the combinatorial # explosion of edges, we also ended up reporting failures for containers # like class and stage. This is undesirable; while just skipping the # output isn't perfect, it is RC-safe. --daniel 2011-06-07 unless resource.class == Puppet::Type.type(:whit) then resource.warning "Skipping because of failed dependencies" end elsif resource.virtual? resource.debug "Skipping because virtual" elsif !host_and_device_resource?(resource) && resource.appliable_to_host? && for_network_device resource.debug "Skipping host resources because running on a device" elsif !host_and_device_resource?(resource) && resource.appliable_to_device? && !for_network_device resource.debug "Skipping device resources because running on a posix host" else return false end true end def host_and_device_resource?(resource) resource.appliable_to_host? && resource.appliable_to_device? end def handle_qualified_tags( qualified ) # The default behavior of Puppet::Util::Tagging is # to split qualified tags into parts. That would cause # qualified tags to match too broadly here. return end # Is this resource tagged appropriately? def missing_tags?(resource) return false if ignore_tags? return false if tags.empty? not resource.tagged?(*tags) end end require 'puppet/transaction/report'