diff --git a/lib/puppet/resource/catalog.rb b/lib/puppet/resource/catalog.rb index a6cff9bdc..b742d283f 100644 --- a/lib/puppet/resource/catalog.rb +++ b/lib/puppet/resource/catalog.rb @@ -1,654 +1,654 @@ require 'puppet/node' require 'puppet/indirector' require 'puppet/simple_graph' require 'puppet/transaction' require 'puppet/util/cacher' require 'puppet/util/pson' require 'puppet/util/tagging' # This class models a node catalog. It is the thing # meant to be passed from server to client, and it contains all # of the information in the catalog, including the resources # and the relationships between them. class Puppet::Resource::Catalog < Puppet::SimpleGraph class DuplicateResourceError < Puppet::Error; end extend Puppet::Indirector indirects :catalog, :terminus_setting => :catalog_terminus include Puppet::Util::Tagging extend Puppet::Util::Pson include Puppet::Util::Cacher::Expirer # The host name this is a catalog for. attr_accessor :name # The catalog version. Used for testing whether a catalog # is up to date. attr_accessor :version # How long this catalog took to retrieve. Used for reporting stats. attr_accessor :retrieval_duration # Whether this is a host catalog, which behaves very differently. # In particular, reports are sent, graphs are made, and state is # stored in the state database. If this is set incorrectly, then you often # end up in infinite loops, because catalogs are used to make things # that the host catalog needs. attr_accessor :host_config # Whether this catalog was retrieved from the cache, which affects # whether it is written back out again. attr_accessor :from_cache # Some metadata to help us compile and generally respond to the current state. attr_accessor :client_version, :server_version # Add classes to our class list. def add_class(*classes) classes.each do |klass| @classes << klass end # Add the class names as tags, too. tag(*classes) end def title_key_for_ref( ref ) ref =~ /^([-\w:]+)\[(.*)\]$/m [$1, $2] end # Add a resource to our graph and to our resource table. # This is actually a relatively complicated method, because it handles multiple # aspects of Catalog behaviour: # * Add the resource to the resource table # * Add the resource to the resource graph # * Add the resource to the relationship graph # * Add any aliases that make sense for the resource (e.g., name != title) def add_resource(*resource) add_resource(*resource[0..-2]) if resource.length > 1 resource = resource.pop raise ArgumentError, "Can only add objects that respond to :ref, not instances of #{resource.class}" unless resource.respond_to?(:ref) fail_on_duplicate_type_and_title(resource) title_key = title_key_for_ref(resource.ref) - + @transient_resources << resource if applying? @resource_table[title_key] = resource # If the name and title differ, set up an alias if resource.respond_to?(:name) and resource.respond_to?(:title) and resource.respond_to?(:isomorphic?) and resource.name != resource.title self.alias(resource, resource.uniqueness_key) if resource.isomorphic? end resource.catalog = self if resource.respond_to?(:catalog=) add_vertex(resource) @relationship_graph.add_vertex(resource) if @relationship_graph end # Create an alias for a resource. def alias(resource, key) resource.ref =~ /^(.+)\[/ class_name = $1 || resource.class.name newref = [class_name, key] if key.is_a? String ref_string = "#{class_name}[#{key}]" return if ref_string == resource.ref end # LAK:NOTE It's important that we directly compare the references, # because sometimes an alias is created before the resource is # added to the catalog, so comparing inside the below if block # isn't sufficient. if existing = @resource_table[newref] return if existing == resource raise(ArgumentError, "Cannot alias #{resource.ref} to #{key.inspect}; resource #{newref.inspect} already exists") end @resource_table[newref] = resource @aliases[resource.ref] ||= [] @aliases[resource.ref] << newref end # Apply our catalog to the local host. Valid options # are: # :tags - set the tags that restrict what resources run # during the transaction # :ignoreschedules - tell the transaction to ignore schedules # when determining the resources to run def apply(options = {}) @applying = true # Expire all of the resource data -- this ensures that all # data we're operating against is entirely current. expire Puppet::Util::Storage.load if host_config? transaction = Puppet::Transaction.new(self) transaction.report = options[:report] if options[:report] transaction.tags = options[:tags] if options[:tags] transaction.ignoreschedules = true if options[:ignoreschedules] transaction.for_network_device = options[:network_device] transaction.add_times :config_retrieval => self.retrieval_duration || 0 begin transaction.evaluate rescue Puppet::Error => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Could not apply complete catalog: #{detail}" rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Got an uncaught exception of type #{detail.class}: #{detail}" ensure # Don't try to store state unless we're a host config # too recursive. Puppet::Util::Storage.store if host_config? end yield transaction if block_given? return transaction ensure @applying = false cleanup end # Are we in the middle of applying the catalog? def applying? @applying end def clear(remove_resources = true) super() # We have to do this so that the resources clean themselves up. @resource_table.values.each { |resource| resource.remove } if remove_resources @resource_table.clear if @relationship_graph @relationship_graph.clear @relationship_graph = nil end end def classes @classes.dup end # Create a new resource and register it in the catalog. def create_resource(type, options) unless klass = Puppet::Type.type(type) raise ArgumentError, "Unknown resource type #{type}" end return unless resource = klass.new(options) add_resource(resource) resource end def dependent_data_expired?(ts) if applying? return super else return true end end # Turn our catalog graph into an old-style tree of TransObjects and TransBuckets. # LAK:NOTE(20081211): This is a pre-0.25 backward compatibility method. # It can be removed as soon as xmlrpc is killed. def extract top = nil current = nil buckets = {} unless main = resource(:stage, "main") raise Puppet::DevError, "Could not find 'main' stage; cannot generate catalog" end if stages = vertices.find_all { |v| v.type == "Stage" and v.title != "main" } and ! stages.empty? Puppet.warning "Stages are not supported by 0.24.x client; stage(s) #{stages.collect { |s| s.to_s }.join(', ') } will be ignored" end bucket = nil walk(main, :out) do |source, target| # The sources are always non-builtins. unless tmp = buckets[source.to_s] if tmp = buckets[source.to_s] = source.to_trans bucket = tmp else # This is because virtual resources return nil. If a virtual # container resource contains realized resources, we still need to get # to them. So, we keep a reference to the last valid bucket # we returned and use that if the container resource is virtual. end end bucket = tmp || bucket if child = target.to_trans raise "No bucket created for #{source}" unless bucket bucket.push child # It's important that we keep a reference to any TransBuckets we've created, so # we don't create multiple buckets for children. buckets[target.to_s] = child unless target.builtin? end end # Retrieve the bucket for the top-level scope and set the appropriate metadata. unless result = buckets[main.to_s] # This only happens when the catalog is entirely empty. result = buckets[main.to_s] = main.to_trans end result.classes = classes # Clear the cache to encourage the GC buckets.clear result end # Make sure all of our resources are "finished". def finalize make_default_resources @resource_table.values.each { |resource| resource.finish } write_graph(:resources) end def host_config? host_config end def initialize(name = nil) super() @name = name if name @classes = [] @resource_table = {} @transient_resources = [] @applying = false @relationship_graph = nil @host_config = true @aliases = {} if block_given? yield(self) finalize end end # Make the default objects necessary for function. def make_default_resources # We have to add the resources to the catalog, or else they won't get cleaned up after # the transaction. # First create the default scheduling objects Puppet::Type.type(:schedule).mkdefaultschedules.each { |res| add_resource(res) unless resource(res.ref) } # And filebuckets if bucket = Puppet::Type.type(:filebucket).mkdefaultbucket add_resource(bucket) unless resource(bucket.ref) end end # Create a graph of all of the relationships in our catalog. def relationship_graph unless @relationship_graph # It's important that we assign the graph immediately, because # the debug messages below use the relationships in the # relationship graph to determine the path to the resources # spitting out the messages. If this is not set, # then we get into an infinite loop. @relationship_graph = Puppet::SimpleGraph.new # First create the dependency graph self.vertices.each do |vertex| @relationship_graph.add_vertex vertex vertex.builddepends.each do |edge| @relationship_graph.add_edge(edge) end end # Lastly, add in any autorequires @relationship_graph.vertices.each do |vertex| vertex.autorequire(self).each do |edge| unless @relationship_graph.edge?(edge.source, edge.target) # don't let automatic relationships conflict with manual ones. unless @relationship_graph.edge?(edge.target, edge.source) vertex.debug "Autorequiring #{edge.source}" @relationship_graph.add_edge(edge) else vertex.debug "Skipping automatic relationship with #{(edge.source == vertex ? edge.target : edge.source)}" end end end end @relationship_graph.write_graph(:relationships) if host_config? # Then splice in the container information splice!(@relationship_graph) @relationship_graph.write_graph(:expanded_relationships) if host_config? end @relationship_graph end - # Impose our container information on another graph by using it - # to replace any container vertices X with a pair of verticies + # Impose our container information on another graph by using it + # to replace any container vertices X with a pair of verticies # { admissible_X and completed_X } such that that # # 0) completed_X depends on admissible_X # 1) contents of X each depend on admissible_X # 2) completed_X depends on each on the contents of X # 3) everything which depended on X depens on completed_X # 4) admissible_X depends on everything X depended on # 5) the containers and their edges must be removed # # Note that this requires attention to the possible case of containers # which contain or depend on other containers, but has the advantage # that the number of new edges created scales linearly with the number - # of contained verticies regardless of how containers are related; - # alternatives such as replacing container-edges with content-edges + # of contained verticies regardless of how containers are related; + # alternatives such as replacing container-edges with content-edges # scale as the product of the number of external dependences, which is # to say geometrically in the case of nested / chained containers. # Default_label = { :callback => :refresh, :event => :ALL_EVENTS } def splice!(other) stage_class = Puppet::Type.type(:stage) whit_class = Puppet::Type.type(:whit) component_class = Puppet::Type.type(:component) containers = vertices.find_all { |v| (v.is_a?(component_class) or v.is_a?(stage_class)) and vertex?(v) } # # These two hashes comprise the aforementioned attention to the possible # case of containers that contain / depend on other containers; they map # containers to their sentinals but pass other verticies through. Thus we # can "do the right thing" for references to other verticies that may or # may not be containers. # admissible = Hash.new { |h,k| k } completed = Hash.new { |h,k| k } containers.each { |x| - admissible[x] = whit_class.new(:name => "admissible_#{x.name}", :catalog => self) - completed[x] = whit_class.new(:name => "completed_#{x.name}", :catalog => self) + admissible[x] = whit_class.new(:name => "admissible_#{x.ref}", :catalog => self) + completed[x] = whit_class.new(:name => "completed_#{x.ref}", :catalog => self) } # # Implement the six requierments listed above # containers.each { |x| contents = adjacent(x, :direction => :out) other.add_edge(admissible[x],completed[x]) if contents.empty? # (0) contents.each { |v| other.add_edge(admissible[x],admissible[v],Default_label) # (1) other.add_edge(completed[v], completed[x], Default_label) # (2) } # (3) & (5) other.adjacent(x,:direction => :in,:type => :edges).each { |e| other.add_edge(completed[e.source],admissible[x],e.label) other.remove_edge! e } # (4) & (5) other.adjacent(x,:direction => :out,:type => :edges).each { |e| other.add_edge(completed[x],admissible[e.target],e.label) other.remove_edge! e } } containers.each { |x| other.remove_vertex! x } # (5) end # Remove the resource from our catalog. Notice that we also call # 'remove' on the resource, at least until resource classes no longer maintain # references to the resource instances. def remove_resource(*resources) resources.each do |resource| @resource_table.delete(resource.ref) if aliases = @aliases[resource.ref] aliases.each { |res_alias| @resource_table.delete(res_alias) } @aliases.delete(resource.ref) end remove_vertex!(resource) if vertex?(resource) @relationship_graph.remove_vertex!(resource) if @relationship_graph and @relationship_graph.vertex?(resource) resource.remove end end # Look a resource up by its reference (e.g., File[/etc/passwd]). def resource(type, title = nil) # Always create a resource reference, so that it always canonizes how we # are referring to them. if title res = Puppet::Resource.new(type, title) else # If they didn't provide a title, then we expect the first # argument to be of the form 'Class[name]', which our # Reference class canonizes for us. res = Puppet::Resource.new(nil, type) end title_key = [res.type, res.title.to_s] uniqueness_key = [res.type, res.uniqueness_key] @resource_table[title_key] || @resource_table[uniqueness_key] end def resource_refs resource_keys.collect{ |type, name| name.is_a?( String ) ? "#{type}[#{name}]" : nil}.compact end def resource_keys @resource_table.keys end def resources @resource_table.values.uniq end def self.from_pson(data) result = new(data['name']) if tags = data['tags'] result.tag(*tags) end if version = data['version'] result.version = version end if resources = data['resources'] resources = PSON.parse(resources) if resources.is_a?(String) resources.each do |res| resource_from_pson(result, res) end end if edges = data['edges'] edges = PSON.parse(edges) if edges.is_a?(String) edges.each do |edge| edge_from_pson(result, edge) end end if classes = data['classes'] result.add_class(*classes) end result end def self.edge_from_pson(result, edge) # If no type information was presented, we manually find # the class. edge = Puppet::Relationship.from_pson(edge) if edge.is_a?(Hash) unless source = result.resource(edge.source) raise ArgumentError, "Could not convert from pson: Could not find relationship source #{edge.source.inspect}" end edge.source = source unless target = result.resource(edge.target) raise ArgumentError, "Could not convert from pson: Could not find relationship target #{edge.target.inspect}" end edge.target = target result.add_edge(edge) end def self.resource_from_pson(result, res) res = Puppet::Resource.from_pson(res) if res.is_a? Hash result.add_resource(res) end PSON.register_document_type('Catalog',self) def to_pson_data_hash { 'document_type' => 'Catalog', 'data' => { 'tags' => tags, 'name' => name, 'version' => version, 'resources' => vertices.collect { |v| v.to_pson_data_hash }, 'edges' => edges. collect { |e| e.to_pson_data_hash }, 'classes' => classes }, 'metadata' => { 'api_version' => 1 } } end def to_pson(*args) to_pson_data_hash.to_pson(*args) end # Convert our catalog into a RAL catalog. def to_ral to_catalog :to_ral end # Convert our catalog into a catalog of Puppet::Resource instances. def to_resource to_catalog :to_resource end # filter out the catalog, applying +block+ to each resource. # If the block result is false, the resource will # be kept otherwise it will be skipped def filter(&block) to_catalog :to_resource, &block end # Store the classes in the classfile. def write_class_file ::File.open(Puppet[:classfile], "w") do |f| f.puts classes.join("\n") end rescue => detail Puppet.err "Could not create class file #{Puppet[:classfile]}: #{detail}" end # Produce the graph files if requested. def write_graph(name) # We only want to graph the main host catalog. return unless host_config? super end private def cleanup # Expire any cached data the resources are keeping. expire end # Verify that the given resource isn't defined elsewhere. def fail_on_duplicate_type_and_title(resource) # Short-curcuit the common case, return unless existing_resource = @resource_table[title_key_for_ref(resource.ref)] # If we've gotten this far, it's a real conflict msg = "Duplicate definition: #{resource.ref} is already defined" msg << " in file #{existing_resource.file} at line #{existing_resource.line}" if existing_resource.file and existing_resource.line msg << "; cannot redefine" if resource.line or resource.file raise DuplicateResourceError.new(msg) end # An abstracted method for converting one catalog into another type of catalog. # This pretty much just converts all of the resources from one class to another, using # a conversion method. def to_catalog(convert) result = self.class.new(self.name) result.version = self.version map = {} vertices.each do |resource| next if virtual_not_exported?(resource) next if block_given? and yield resource #This is hackity hack for 1094 #Aliases aren't working in the ral catalog because the current instance of the resource #has a reference to the catalog being converted. . . So, give it a reference to the new one #problem solved. . . if resource.class == Puppet::Resource resource = resource.dup resource.catalog = result elsif resource.is_a?(Puppet::TransObject) resource = resource.dup resource.catalog = result elsif resource.is_a?(Puppet::Parser::Resource) resource = resource.to_resource resource.catalog = result end if resource.is_a?(Puppet::Resource) and convert.to_s == "to_resource" newres = resource else newres = resource.send(convert) end # We can't guarantee that resources don't munge their names # (like files do with trailing slashes), so we have to keep track # of what a resource got converted to. map[resource.ref] = newres result.add_resource newres end message = convert.to_s.gsub "_", " " edges.each do |edge| # Skip edges between virtual resources. next if virtual_not_exported?(edge.source) next if block_given? and yield edge.source next if virtual_not_exported?(edge.target) next if block_given? and yield edge.target unless source = map[edge.source.ref] raise Puppet::DevError, "Could not find resource #{edge.source.ref} when converting #{message} resources" end unless target = map[edge.target.ref] raise Puppet::DevError, "Could not find resource #{edge.target.ref} when converting #{message} resources" end result.add_edge(source, target, edge.label) end map.clear result.add_class(*self.classes) result.tag(*self.tags) result end def virtual_not_exported?(resource) resource.respond_to?(:virtual?) and resource.virtual? and (resource.respond_to?(:exported?) and not resource.exported?) end end diff --git a/lib/puppet/transaction/event_manager.rb b/lib/puppet/transaction/event_manager.rb index f5da870ed..8f1a695af 100644 --- a/lib/puppet/transaction/event_manager.rb +++ b/lib/puppet/transaction/event_manager.rb @@ -1,102 +1,115 @@ require 'puppet/transaction' class Puppet::Transaction::EventManager attr_reader :transaction, :events def initialize(transaction) @transaction = transaction @event_queues = {} @events = [] end def relationship_graph transaction.relationship_graph end # Respond to any queued events for this resource. def process_events(resource) restarted = false queued_events(resource) do |callback, events| r = process_callback(resource, callback, events) restarted ||= r end if restarted queue_events(resource, [resource.event(:name => :restarted, :status => "success")]) transaction.resource_status(resource).restarted = true end end # Queue events for other resources to respond to. All of these events have # to be from the same resource. def queue_events(resource, events) #@events += events # Do some basic normalization so we're not doing so many # graph queries for large sets of events. events.inject({}) do |collection, event| collection[event.name] ||= [] collection[event.name] << event collection end.collect do |name, list| # It doesn't matter which event we use - they all have the same source # and name here. event = list[0] # Collect the targets of any subscriptions to those events. We pass # the parent resource in so it will override the source in the events, # since eval_generated children can't have direct relationships. received = (event.name != :restarted) relationship_graph.matching_edges(event, resource).each do |edge| received ||= true unless edge.target.is_a?(Puppet::Type.type(:whit)) next unless method = edge.callback next unless edge.target.respond_to?(method) queue_events_for_resource(resource, edge.target, method, list) end @events << event if received queue_events_for_resource(resource, resource, :refresh, [event]) if resource.self_refresh? and ! resource.deleting? end end def queue_events_for_resource(source, target, callback, events) - source.info "Scheduling #{callback} of #{target}" + whit = Puppet::Type.type(:whit) + + # The message that a resource is refreshing the completed-whit for its own class + # is extremely counter-intuitive. Basically everything else is easy to understand, + # if you suppress the whit-lookingness of the whit resources + refreshing_c_whit = target.is_a?(whit) && target.name =~ /^completed_/ + + if refreshing_c_whit + source.debug "The container #{target} will propagate my #{callback} event" + else + source.info "Scheduling #{callback} of #{target}" + end @event_queues[target] ||= {} @event_queues[target][callback] ||= [] @event_queues[target][callback] += events end def queued_events(resource) return unless callbacks = @event_queues[resource] callbacks.each do |callback, events| yield callback, events end end private def process_callback(resource, callback, events) process_noop_events(resource, callback, events) and return false unless events.detect { |e| e.status != "noop" } resource.send(callback) - resource.notice "Triggered '#{callback}' from #{events.length} events" + if not resource.is_a?(Puppet::Type.type(:whit)) + resource.notice "Triggered '#{callback}' from #{events.length} events" + end return true rescue => detail resource.err "Failed to call #{callback}: #{detail}" transaction.resource_status(resource).failed_to_restart = true puts detail.backtrace if Puppet[:trace] return false end def process_noop_events(resource, callback, events) resource.notice "Would have triggered '#{callback}' from #{events.length} events" # And then add an event for it. queue_events(resource, [resource.event(:status => "noop", :name => :noop_restart)]) true # so the 'and if' works end end diff --git a/lib/puppet/type/whit.rb b/lib/puppet/type/whit.rb index 55ed0386e..4c77915b3 100644 --- a/lib/puppet/type/whit.rb +++ b/lib/puppet/type/whit.rb @@ -1,17 +1,23 @@ Puppet::Type.newtype(:whit) do desc "The smallest possible resource type, for when you need a resource and naught else." newparam :name do desc "The name of the whit, because it must have one." end + + # Hide the fact that we're a whit from logs def to_s - "(#{name})" + name.sub(/^completed_|^admissible_/, "") + end + + def path + to_s end def refresh # We don't do anything with them, but we need this to # show that we are "refresh aware" and not break the # chain of propogation. end end diff --git a/spec/unit/simple_graph_spec.rb b/spec/unit/simple_graph_spec.rb index c8fea3b58..472ebbd36 100755 --- a/spec/unit/simple_graph_spec.rb +++ b/spec/unit/simple_graph_spec.rb @@ -1,904 +1,908 @@ #!/usr/bin/env rspec # # Created by Luke Kanies on 2007-11-1. # Copyright (c) 2006. All rights reserved. require 'spec_helper' require 'puppet/simple_graph' describe Puppet::SimpleGraph do it "should return the number of its vertices as its length" do @graph = Puppet::SimpleGraph.new @graph.add_vertex("one") @graph.add_vertex("two") @graph.size.should == 2 end it "should consider itself a directed graph" do Puppet::SimpleGraph.new.directed?.should be_true end it "should provide a method for reversing the graph" do @graph = Puppet::SimpleGraph.new @graph.add_edge(:one, :two) @graph.reversal.edge?(:two, :one).should be_true end it "should be able to produce a dot graph" do @graph = Puppet::SimpleGraph.new @graph.add_edge(:one, :two) proc { @graph.to_dot_graph }.should_not raise_error end describe "when managing vertices" do before do @graph = Puppet::SimpleGraph.new end it "should provide a method to add a vertex" do @graph.add_vertex(:test) @graph.vertex?(:test).should be_true end it "should reset its reversed graph when vertices are added" do rev = @graph.reversal @graph.add_vertex(:test) @graph.reversal.should_not equal(rev) end it "should ignore already-present vertices when asked to add a vertex" do @graph.add_vertex(:test) proc { @graph.add_vertex(:test) }.should_not raise_error end it "should return true when asked if a vertex is present" do @graph.add_vertex(:test) @graph.vertex?(:test).should be_true end it "should return false when asked if a non-vertex is present" do @graph.vertex?(:test).should be_false end it "should return all set vertices when asked" do @graph.add_vertex(:one) @graph.add_vertex(:two) @graph.vertices.length.should == 2 @graph.vertices.should include(:one) @graph.vertices.should include(:two) end it "should remove a given vertex when asked" do @graph.add_vertex(:one) @graph.remove_vertex!(:one) @graph.vertex?(:one).should be_false end it "should do nothing when a non-vertex is asked to be removed" do proc { @graph.remove_vertex!(:one) }.should_not raise_error end end describe "when managing edges" do before do @graph = Puppet::SimpleGraph.new end it "should provide a method to test whether a given vertex pair is an edge" do @graph.should respond_to(:edge?) end it "should reset its reversed graph when edges are added" do rev = @graph.reversal @graph.add_edge(:one, :two) @graph.reversal.should_not equal(rev) end it "should provide a method to add an edge as an instance of the edge class" do edge = Puppet::Relationship.new(:one, :two) @graph.add_edge(edge) @graph.edge?(:one, :two).should be_true end it "should provide a method to add an edge by specifying the two vertices" do @graph.add_edge(:one, :two) @graph.edge?(:one, :two).should be_true end it "should provide a method to add an edge by specifying the two vertices and a label" do @graph.add_edge(:one, :two, :callback => :awesome) @graph.edge?(:one, :two).should be_true end describe "when retrieving edges between two nodes" do it "should handle the case of nodes not in the graph" do @graph.edges_between(:one, :two).should == [] end it "should handle the case of nodes with no edges between them" do @graph.add_vertex(:one) @graph.add_vertex(:two) @graph.edges_between(:one, :two).should == [] end it "should handle the case of nodes connected by a single edge" do edge = Puppet::Relationship.new(:one, :two) @graph.add_edge(edge) @graph.edges_between(:one, :two).length.should == 1 @graph.edges_between(:one, :two)[0].should equal(edge) end it "should handle the case of nodes connected by multiple edges" do edge1 = Puppet::Relationship.new(:one, :two, :callback => :foo) edge2 = Puppet::Relationship.new(:one, :two, :callback => :bar) @graph.add_edge(edge1) @graph.add_edge(edge2) Set.new(@graph.edges_between(:one, :two)).should == Set.new([edge1, edge2]) end end it "should add the edge source as a vertex if it is not already" do edge = Puppet::Relationship.new(:one, :two) @graph.add_edge(edge) @graph.vertex?(:one).should be_true end it "should add the edge target as a vertex if it is not already" do edge = Puppet::Relationship.new(:one, :two) @graph.add_edge(edge) @graph.vertex?(:two).should be_true end it "should return all edges as edge instances when asked" do one = Puppet::Relationship.new(:one, :two) two = Puppet::Relationship.new(:two, :three) @graph.add_edge(one) @graph.add_edge(two) edges = @graph.edges edges.should be_instance_of(Array) edges.length.should == 2 edges.should include(one) edges.should include(two) end it "should remove an edge when asked" do edge = Puppet::Relationship.new(:one, :two) @graph.add_edge(edge) @graph.remove_edge!(edge) @graph.edge?(edge.source, edge.target).should be_false end it "should remove all related edges when a vertex is removed" do one = Puppet::Relationship.new(:one, :two) two = Puppet::Relationship.new(:two, :three) @graph.add_edge(one) @graph.add_edge(two) @graph.remove_vertex!(:two) @graph.edge?(:one, :two).should be_false @graph.edge?(:two, :three).should be_false @graph.edges.length.should == 0 end end describe "when finding adjacent vertices" do before do @graph = Puppet::SimpleGraph.new @one_two = Puppet::Relationship.new(:one, :two) @two_three = Puppet::Relationship.new(:two, :three) @one_three = Puppet::Relationship.new(:one, :three) @graph.add_edge(@one_two) @graph.add_edge(@one_three) @graph.add_edge(@two_three) end it "should return adjacent vertices" do adj = @graph.adjacent(:one) adj.should be_include(:three) adj.should be_include(:two) end it "should default to finding :out vertices" do @graph.adjacent(:two).should == [:three] end it "should support selecting :in vertices" do @graph.adjacent(:two, :direction => :in).should == [:one] end it "should default to returning the matching vertices as an array of vertices" do @graph.adjacent(:two).should == [:three] end it "should support returning an array of matching edges" do @graph.adjacent(:two, :type => :edges).should == [@two_three] end # Bug #2111 it "should not consider a vertex adjacent just because it was asked about previously" do @graph = Puppet::SimpleGraph.new @graph.add_vertex("a") @graph.add_vertex("b") @graph.edge?("a", "b") @graph.adjacent("a").should == [] end end describe "when clearing" do before do @graph = Puppet::SimpleGraph.new one = Puppet::Relationship.new(:one, :two) two = Puppet::Relationship.new(:two, :three) @graph.add_edge(one) @graph.add_edge(two) @graph.clear end it "should remove all vertices" do @graph.vertices.should be_empty end it "should remove all edges" do @graph.edges.should be_empty end end describe "when reversing graphs" do before do @graph = Puppet::SimpleGraph.new end it "should provide a method for reversing the graph" do @graph.add_edge(:one, :two) @graph.reversal.edge?(:two, :one).should be_true end it "should add all vertices to the reversed graph" do @graph.add_edge(:one, :two) @graph.vertex?(:one).should be_true @graph.vertex?(:two).should be_true end it "should retain labels on edges" do @graph.add_edge(:one, :two, :callback => :awesome) edge = @graph.reversal.edges_between(:two, :one)[0] edge.label.should == {:callback => :awesome} end end describe "when reporting cycles in the graph" do before do @graph = Puppet::SimpleGraph.new end def add_edges(hash) hash.each do |a,b| @graph.add_edge(a, b) end end it "should fail on two-vertex loops" do add_edges :a => :b, :b => :a proc { @graph.report_cycles_in_graph }.should raise_error(Puppet::Error) end it "should fail on multi-vertex loops" do add_edges :a => :b, :b => :c, :c => :a proc { @graph.report_cycles_in_graph }.should raise_error(Puppet::Error) end it "should fail when a larger tree contains a small cycle" do add_edges :a => :b, :b => :a, :c => :a, :d => :c proc { @graph.report_cycles_in_graph }.should raise_error(Puppet::Error) end it "should succeed on trees with no cycles" do add_edges :a => :b, :b => :e, :c => :a, :d => :c proc { @graph.report_cycles_in_graph }.should_not raise_error end it "should produce the correct relationship text" do add_edges :a => :b, :b => :a # cycle detection starts from a or b randomly # so we need to check for either ordering in the error message want = %r{Found 1 dependency cycle:\n\((a => b => a|b => a => b)\)\nTry} expect { @graph.report_cycles_in_graph }.to raise_error(Puppet::Error, want) end it "cycle discovery should be the minimum cycle for a simple graph" do add_edges "a" => "b" add_edges "b" => "a" add_edges "b" => "c" cycles = nil expect { cycles = @graph.find_cycles_in_graph.sort }.should_not raise_error cycles.should be == [["a", "b"]] end it "cycle discovery should handle two distinct cycles" do add_edges "a" => "a1", "a1" => "a" add_edges "b" => "b1", "b1" => "b" cycles = nil expect { cycles = @graph.find_cycles_in_graph.sort }.should_not raise_error cycles.should be == [["a", "a1"], ["b", "b1"]] end it "cycle discovery should handle two cycles in a connected graph" do add_edges "a" => "b", "b" => "c", "c" => "d" add_edges "a" => "a1", "a1" => "a" add_edges "c" => "c1", "c1" => "c2", "c2" => "c3", "c3" => "c" cycles = nil expect { cycles = @graph.find_cycles_in_graph.sort }.should_not raise_error cycles.should be == [%w{a a1}, %w{c c1 c2 c3}] end it "cycle discovery should handle a complicated cycle" do add_edges "a" => "b", "b" => "c" add_edges "a" => "c" add_edges "c" => "c1", "c1" => "a" add_edges "c" => "c2", "c2" => "b" cycles = nil expect { cycles = @graph.find_cycles_in_graph.sort }.should_not raise_error cycles.should be == [%w{a b c c1 c2}] end it "cycle discovery should not fail with large data sets" do limit = 3000 (1..(limit - 1)).each do |n| add_edges n.to_s => (n+1).to_s end cycles = nil expect { cycles = @graph.find_cycles_in_graph.sort }.should_not raise_error cycles.should be == [] end it "path finding should work with a simple cycle" do add_edges "a" => "b", "b" => "c", "c" => "a" cycles = @graph.find_cycles_in_graph.sort paths = @graph.paths_in_cycle(cycles.first, 100) paths.should be == [%w{a b c a}] end it "path finding should work with two independent cycles" do add_edges "a" => "b1" add_edges "a" => "b2" add_edges "b1" => "a", "b2" => "a" cycles = @graph.find_cycles_in_graph.sort cycles.length.should be == 1 paths = @graph.paths_in_cycle(cycles.first, 100) paths.sort.should be == [%w{a b1 a}, %w{a b2 a}] end it "path finding should prefer shorter paths in cycles" do add_edges "a" => "b", "b" => "c", "c" => "a" add_edges "b" => "a" cycles = @graph.find_cycles_in_graph.sort cycles.length.should be == 1 paths = @graph.paths_in_cycle(cycles.first, 100) paths.should be == [%w{a b a}, %w{a b c a}] end it "path finding should respect the max_path value" do (1..20).each do |n| add_edges "a" => "b#{n}", "b#{n}" => "a" end cycles = @graph.find_cycles_in_graph.sort cycles.length.should be == 1 (1..20).each do |n| paths = @graph.paths_in_cycle(cycles.first, n) paths.length.should be == n end paths = @graph.paths_in_cycle(cycles.first, 21) paths.length.should be == 20 end end describe "when writing dot files" do before do @graph = Puppet::SimpleGraph.new @name = :test @file = File.join(Puppet[:graphdir], @name.to_s + ".dot") end it "should only write when graphing is enabled" do File.expects(:open).with(@file).never Puppet[:graph] = false @graph.write_graph(@name) end it "should write a dot file based on the passed name" do File.expects(:open).with(@file, "w").yields(stub("file", :puts => nil)) @graph.expects(:to_dot).with("name" => @name.to_s.capitalize) Puppet[:graph] = true @graph.write_graph(@name) end after do Puppet.settings.clear end end describe Puppet::SimpleGraph do before do @graph = Puppet::SimpleGraph.new end it "should correctly clear vertices and edges when asked" do @graph.add_edge("a", "b") @graph.add_vertex "c" @graph.clear @graph.vertices.should be_empty @graph.edges.should be_empty end end describe "when matching edges" do before do @graph = Puppet::SimpleGraph.new @event = Puppet::Transaction::Event.new(:name => :yay, :resource => "a") @none = Puppet::Transaction::Event.new(:name => :NONE, :resource => "a") @edges = {} @edges["a/b"] = Puppet::Relationship.new("a", "b", {:event => :yay, :callback => :refresh}) @edges["a/c"] = Puppet::Relationship.new("a", "c", {:event => :yay, :callback => :refresh}) @graph.add_edge(@edges["a/b"]) end it "should match edges whose source matches the source of the event" do @graph.matching_edges(@event).should == [@edges["a/b"]] end it "should match always match nothing when the event is :NONE" do @graph.matching_edges(@none).should be_empty end it "should match multiple edges" do @graph.add_edge(@edges["a/c"]) edges = @graph.matching_edges(@event) edges.should be_include(@edges["a/b"]) edges.should be_include(@edges["a/c"]) end end describe "when determining dependencies" do before do @graph = Puppet::SimpleGraph.new @graph.add_edge("a", "b") @graph.add_edge("a", "c") @graph.add_edge("b", "d") end it "should find all dependents when they are on multiple levels" do @graph.dependents("a").sort.should == %w{b c d}.sort end it "should find single dependents" do @graph.dependents("b").sort.should == %w{d}.sort end it "should return an empty array when there are no dependents" do @graph.dependents("c").sort.should == [].sort end it "should find all dependencies when they are on multiple levels" do @graph.dependencies("d").sort.should == %w{a b} end it "should find single dependencies" do @graph.dependencies("c").sort.should == %w{a} end it "should return an empty array when there are no dependencies" do @graph.dependencies("a").sort.should == [] end end require 'puppet/util/graph' class Container < Puppet::Type::Component include Puppet::Util::Graph include Enumerable attr_accessor :name def each @children.each do |c| yield c end end def initialize(name, ary) @name = name @children = ary end def push(*ary) ary.each { |c| @children.push(c)} end def to_s @name end + + def ref + "Container[#{self}]" + end end require "puppet/resource/catalog" describe "when splicing the graph" do def container_graph @one = Container.new("one", %w{a b}) @two = Container.new("two", ["c", "d"]) @three = Container.new("three", ["i", "j"]) @middle = Container.new("middle", ["e", "f", @two]) @top = Container.new("top", ["g", "h", @middle, @one, @three]) @empty = Container.new("empty", []) - + @whit = Puppet::Type.type(:whit) @stage = Puppet::Type.type(:stage).new(:name => "foo") @contgraph = @top.to_graph(Puppet::Resource::Catalog.new) # We have to add the container to the main graph, else it won't # be spliced in the dependency graph. @contgraph.add_vertex(@empty) end def containers @contgraph.vertices.select { |x| !x.is_a? String } end def contents_of(x) @contgraph.direct_dependents_of(x) end def dependency_graph @depgraph = Puppet::SimpleGraph.new @contgraph.vertices.each do |v| @depgraph.add_vertex(v) end # We have to specify a relationship to our empty container, else it # never makes it into the dep graph in the first place. @explicit_dependencies = {@one => @two, "f" => "c", "h" => @middle, "c" => @empty} @explicit_dependencies.each do |source, target| @depgraph.add_edge(source, target, :callback => :refresh) end end def splice @contgraph.splice!(@depgraph) end def whit_called(name) - x = @depgraph.vertices.find { |v| v.is_a?(@whit) && v.name =~ /#{name}/ } + x = @depgraph.vertices.find { |v| v.is_a?(@whit) && v.name =~ /#{Regexp.escape(name)}/ } x.should_not be_nil def x.to_s "Whit[#{name}]" end def x.inspect to_s end x end def admissible_sentinal_of(x) - @depgraph.vertex?(x) ? x : whit_called("admissible_#{x.name}") + @depgraph.vertex?(x) ? x : whit_called("admissible_#{x.ref}") end def completed_sentinal_of(x) - @depgraph.vertex?(x) ? x : whit_called("completed_#{x.name}") + @depgraph.vertex?(x) ? x : whit_called("completed_#{x.ref}") end before do container_graph dependency_graph splice end # This is the real heart of splicing -- replacing all containers X in our # relationship graph with a pair of whits { admissible_X and completed_X } # such that that # # 0) completed_X depends on admissible_X # 1) contents of X each depend on admissible_X # 2) completed_X depends on each on the contents of X # 3) everything which depended on X depends on completed_X # 4) admissible_X depends on everything X depended on # 5) the containers and their edges must be removed # # Note that this requires attention to the possible case of containers # which contain or depend on other containers. # # Point by point: # 0) completed_X depends on admissible_X # it "every container's completed sentinal should depend on its admissible sentinal" do - containers.each { |container| + containers.each { |container| @depgraph.path_between(admissible_sentinal_of(container),completed_sentinal_of(container)).should be } end # 1) contents of X each depend on admissible_X # it "all contained objects should depend on their container's admissible sentinal" do - containers.each { |container| + containers.each { |container| contents_of(container).each { |leaf| @depgraph.should be_edge(admissible_sentinal_of(container),admissible_sentinal_of(leaf)) } } end # 2) completed_X depends on each on the contents of X # it "completed sentinals should depend on their container's contents" do - containers.each { |container| + containers.each { |container| contents_of(container).each { |leaf| @depgraph.should be_edge(completed_sentinal_of(leaf),completed_sentinal_of(container)) } } end # # 3) everything which depended on X depends on completed_X # # 4) admissible_X depends on everything X depended on # 5) the containers and their edges must be removed # it "should remove all Container objects from the dependency graph" do @depgraph.vertices.find_all { |v| v.is_a?(Container) }.should be_empty end it "should remove all Stage resources from the dependency graph" do @depgraph.vertices.find_all { |v| v.is_a?(Puppet::Type.type(:stage)) }.should be_empty end it "should no longer contain anything but the non-container objects" do @depgraph.vertices.find_all { |v| ! v.is_a?(String) and ! v.is_a?(@whit)}.should be_empty end it "should retain labels on non-containment edges" do @explicit_dependencies.each { |f,t| @depgraph.edges_between(completed_sentinal_of(f),admissible_sentinal_of(t))[0].label.should == {:callback => :refresh} } end it "should not add labels to edges that have none" do @depgraph.add_edge(@two, @three) splice @depgraph.path_between("c", "i").any? {|segment| segment.all? {|e| e.label == {} }}.should be end it "should copy labels over edges that have none" do @depgraph.add_edge("c", @three, {:callback => :refresh}) splice # And make sure the label got copied. @depgraph.path_between("c", "i").flatten.select {|e| e.label == {:callback => :refresh} }.should_not be_empty end it "should not replace a label with a nil label" do # Lastly, add some new label-less edges and make sure the label stays. @depgraph.add_edge(@middle, @three) @depgraph.add_edge("c", @three, {:callback => :refresh}) splice @depgraph.path_between("c","i").flatten.select {|e| e.label == {:callback => :refresh} }.should_not be_empty end it "should copy labels to all created edges" do @depgraph.add_edge(@middle, @three) @depgraph.add_edge("c", @three, {:callback => :refresh}) splice @three.each do |child| edge = Puppet::Relationship.new("c", child) (path = @depgraph.path_between(edge.source, edge.target)).should be path.should_not be_empty path.flatten.select {|e| e.label == {:callback => :refresh} }.should_not be_empty end end end it "should serialize to YAML using the old format by default" do Puppet::SimpleGraph.use_new_yaml_format.should == false end describe "(yaml tests)" do def empty_graph(graph) end def one_vertex_graph(graph) graph.add_vertex(:a) end def graph_without_edges(graph) [:a, :b, :c].each { |x| graph.add_vertex(x) } end def one_edge_graph(graph) graph.add_edge(:a, :b) end def many_edge_graph(graph) graph.add_edge(:a, :b) graph.add_edge(:a, :c) graph.add_edge(:b, :d) graph.add_edge(:c, :d) end def labeled_edge_graph(graph) graph.add_edge(:a, :b, :callback => :foo, :event => :bar) end def overlapping_edge_graph(graph) graph.add_edge(:a, :b, :callback => :foo, :event => :bar) graph.add_edge(:a, :b, :callback => :biz, :event => :baz) end def self.all_test_graphs [:empty_graph, :one_vertex_graph, :graph_without_edges, :one_edge_graph, :many_edge_graph, :labeled_edge_graph, :overlapping_edge_graph] end def object_ids(enumerable) # Return a sorted list of the object id's of the elements of an # enumerable. enumerable.collect { |x| x.object_id }.sort end def graph_to_yaml(graph, which_format) previous_use_new_yaml_format = Puppet::SimpleGraph.use_new_yaml_format Puppet::SimpleGraph.use_new_yaml_format = (which_format == :new) ZAML.dump(graph) ensure Puppet::SimpleGraph.use_new_yaml_format = previous_use_new_yaml_format end # Test serialization of graph to YAML. [:old, :new].each do |which_format| all_test_graphs.each do |graph_to_test| it "should be able to serialize #{graph_to_test} to YAML (#{which_format} format)" do graph = Puppet::SimpleGraph.new send(graph_to_test, graph) yaml_form = graph_to_yaml(graph, which_format) # Hack the YAML so that objects in the Puppet namespace get # changed to YAML::DomainType objects. This lets us inspect # the serialized objects easily without invoking any # yaml_initialize hooks. yaml_form.gsub!('!ruby/object:Puppet::', '!hack/object:Puppet::') serialized_object = YAML.load(yaml_form) # Check that the object contains instance variables @edges and # @vertices only. @reversal is also permitted, but we don't # check it, because it is going to be phased out. serialized_object.type_id.should == 'object:Puppet::SimpleGraph' serialized_object.value.keys.reject { |x| x == 'reversal' }.sort.should == ['edges', 'vertices'] # Check edges by forming a set of tuples (source, target, # callback, event) based on the graph and the YAML and make sure # they match. edges = serialized_object.value['edges'] edges.should be_a(Array) expected_edge_tuples = graph.edges.collect { |edge| [edge.source, edge.target, edge.callback, edge.event] } actual_edge_tuples = edges.collect do |edge| edge.type_id.should == 'object:Puppet::Relationship' %w{source target}.each { |x| edge.value.keys.should include(x) } edge.value.keys.each { |x| ['source', 'target', 'callback', 'event'].should include(x) } %w{source target callback event}.collect { |x| edge.value[x] } end Set.new(actual_edge_tuples).should == Set.new(expected_edge_tuples) actual_edge_tuples.length.should == expected_edge_tuples.length # Check vertices one by one. vertices = serialized_object.value['vertices'] if which_format == :old vertices.should be_a(Hash) Set.new(vertices.keys).should == Set.new(graph.vertices) vertices.each do |key, value| value.type_id.should == 'object:Puppet::SimpleGraph::VertexWrapper' value.value.keys.sort.should == %w{adjacencies vertex} value.value['vertex'].should equal(key) adjacencies = value.value['adjacencies'] adjacencies.should be_a(Hash) Set.new(adjacencies.keys).should == Set.new([:in, :out]) [:in, :out].each do |direction| adjacencies[direction].should be_a(Hash) expected_adjacent_vertices = Set.new(graph.adjacent(key, :direction => direction, :type => :vertices)) Set.new(adjacencies[direction].keys).should == expected_adjacent_vertices adjacencies[direction].each do |adj_key, adj_value| # Since we already checked edges, just check consistency # with edges. desired_source = direction == :in ? adj_key : key desired_target = direction == :in ? key : adj_key expected_edges = edges.select do |edge| edge.value['source'] == desired_source && edge.value['target'] == desired_target end adj_value.should be_a(Set) if object_ids(adj_value) != object_ids(expected_edges) raise "For vertex #{key.inspect}, direction #{direction.inspect}: expected adjacencies #{expected_edges.inspect} but got #{adj_value.inspect}" end end end end else vertices.should be_a(Array) Set.new(vertices).should == Set.new(graph.vertices) vertices.length.should == graph.vertices.length end end end # Test deserialization of graph from YAML. This presumes the # correctness of serialization to YAML, which has already been # tested. all_test_graphs.each do |graph_to_test| it "should be able to deserialize #{graph_to_test} from YAML (#{which_format} format)" do reference_graph = Puppet::SimpleGraph.new send(graph_to_test, reference_graph) yaml_form = graph_to_yaml(reference_graph, which_format) recovered_graph = YAML.load(yaml_form) # Test that the recovered vertices match the vertices in the # reference graph. expected_vertices = reference_graph.vertices.to_a recovered_vertices = recovered_graph.vertices.to_a Set.new(recovered_vertices).should == Set.new(expected_vertices) recovered_vertices.length.should == expected_vertices.length # Test that the recovered edges match the edges in the # reference graph. expected_edge_tuples = reference_graph.edges.collect do |edge| [edge.source, edge.target, edge.callback, edge.event] end recovered_edge_tuples = recovered_graph.edges.collect do |edge| [edge.source, edge.target, edge.callback, edge.event] end Set.new(recovered_edge_tuples).should == Set.new(expected_edge_tuples) recovered_edge_tuples.length.should == expected_edge_tuples.length # We ought to test that the recovered graph is self-consistent # too. But we're not going to bother with that yet because # the internal representation of the graph is about to change. end end it "should be able to serialize a graph where the vertices contain backreferences to the graph (#{which_format} format)" do reference_graph = Puppet::SimpleGraph.new vertex = Object.new vertex.instance_eval { @graph = reference_graph } reference_graph.add_edge(vertex, :other_vertex) yaml_form = graph_to_yaml(reference_graph, which_format) recovered_graph = YAML.load(yaml_form) recovered_graph.vertices.length.should == 2 recovered_vertex = recovered_graph.vertices.reject { |x| x.is_a?(Symbol) }[0] recovered_vertex.instance_eval { @graph }.should equal(recovered_graph) recovered_graph.edges.length.should == 1 recovered_edge = recovered_graph.edges[0] recovered_edge.source.should equal(recovered_vertex) recovered_edge.target.should == :other_vertex end end it "should serialize properly when used as a base class" do class Puppet::TestDerivedClass < Puppet::SimpleGraph attr_accessor :foo end derived = Puppet::TestDerivedClass.new derived.add_edge(:a, :b) derived.foo = 1234 recovered_derived = YAML.load(YAML.dump(derived)) recovered_derived.class.should equal(Puppet::TestDerivedClass) recovered_derived.edges.length.should == 1 recovered_derived.edges[0].source.should == :a recovered_derived.edges[0].target.should == :b recovered_derived.vertices.length.should == 2 recovered_derived.foo.should == 1234 end end end diff --git a/spec/unit/type/whit_spec.rb b/spec/unit/type/whit_spec.rb index 4d0949900..9a076d1f1 100755 --- a/spec/unit/type/whit_spec.rb +++ b/spec/unit/type/whit_spec.rb @@ -1,10 +1,10 @@ #!/usr/bin/env rspec require 'spec_helper' whit = Puppet::Type.type(:whit).new(:name => "Foo::Bar") describe whit do it "should stringify in a way that users will regognise" do - whit.to_s.should == "(Foo::Bar)" + whit.to_s.should == "Foo::Bar" end end