diff --git a/lib/puppet/gratr/adjacency_graph.rb b/lib/puppet/gratr/adjacency_graph.rb index bea50e8c8..a02ed0b2f 100644 --- a/lib/puppet/gratr/adjacency_graph.rb +++ b/lib/puppet/gratr/adjacency_graph.rb @@ -1,232 +1,257 @@ #-- # Copyright (c) 2006 Shawn Patrick Garbett # Copyright (c) 2002,2004,2005 by Horst Duchene # # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice(s), # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the Shawn Garbett nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #++ require 'puppet/gratr/edge' require 'puppet/gratr/graph' require 'set' module GRATR # This provides the basic routines needed to implement the Digraph, UndirectedGraph, # PseudoGraph, DirectedPseudoGraph, MultiGraph and DirectedPseudoGraph class. module AdjacencyGraph include Graph class ArrayWithAdd < Array # :nodoc: alias add push end # Initialization parameters can include an Array of edges to add, Graphs to # copy (will merge if multiple) # :parallel_edges denotes that duplicate edges are allowed # :loops denotes that loops are allowed def initialize(*params) @vertex_dict = Hash.new raise ArgumentError if params.any? do |p| !(p.kind_of? GRATR::Graph or p.kind_of? Array or p == :parallel_edges or p == :loops) end clear_all_labels # Basic configuration of adjacency @allow_loops = params.any? {|p| p == :loops} @parallel_edges = params.any? {|p| p == :parallel_edges} @edgelist_class = @parallel_edges ? ArrayWithAdd : Set if @parallel_edges @edge_number = Hash.new @next_edge_number = 0 end # Copy any given graph into this graph params.select {|p| p.kind_of? GRATR::Graph}.each do |g| g.edges.each do |e| add_edge!(e) edge_label_set(e, edge_label(e)) if edge_label(e) end g.vertices.each do |v| vertex_label_set(v, vertex_label(v)) if vertex_label(v) end end # Add all array edges specified params.select {|p| p.kind_of? Array}.each do |a| 0.step(a.size-1, 2) {|i| add_edge!(a[i], a[i+1])} end end # Returns true if v is a vertex of this Graph # An O(1) implementation of vertex? def vertex?(v) @vertex_dict.has_key?(v); end # Returns true if [u,v] or u is an Edge # An O(1) implementation def edge?(u, v=nil) u, v = u.source, u.target if u.kind_of? GRATR::Edge vertex?(u) and @vertex_dict[u].include?(v) end # Adds a vertex to the graph with an optional label def add_vertex!(vertex, label=nil) @vertex_dict[vertex] ||= @edgelist_class.new self[vertex] = label if label self end # Adds an edge to the graph # Can be called in two basic ways, label is optional # * add_edge!(Edge[source,target], "Label") # * add_edge!(source,target, "Label") def add_edge!(u, v=nil, l=nil, n=nil) - n = u.number if u.class.include? EdgeNumber and n.nil? - u, v, l = u.source, u.target, u.label if u.kind_of? GRATR::Edge - return self if not @allow_loops and u == v - n = (@next_edge_number+=1) unless n if @parallel_edges - add_vertex!(u); add_vertex!(v) + if u.class.include? EdgeNumber and n.nil? + n = u.number + end + if u.kind_of? GRATR::Edge + edge = u + u, v, l = u.source, u.target, u.label + end + if not @allow_loops and u == v + return self + end + if @parallel_edges and ! n + n = (@next_edge_number+=1) + end + add_vertex!(u); + add_vertex!(v) @vertex_dict[u].add(v) - (@edge_number[u] ||= @edgelist_class.new).add(n) if @parallel_edges + + if @parallel_edges + (@edge_number[u] ||= @edgelist_class.new).add(n) + end unless directed? @vertex_dict[v].add(u) - (@edge_number[v] ||= @edgelist_class.new).add(n) if @parallel_edges + if @parallel_edges + (@edge_number[v] ||= @edgelist_class.new).add(n) + end end - self[n ? edge_class[u,v,n] : edge_class[u,v]] = l if l + + if l + unless edge + if n + edge = edge_class[u,v,n] + else + edge = edge_class[u,v] + end + end + self[edge] = l + end self end # Removes a given vertex from the graph def remove_vertex!(v) # FIXME This is broken for multi graphs @vertex_dict.delete(v) @vertex_dict.each_value { |adjList| adjList.delete(v) } @vertex_dict.keys.each do |u| delete_label(edge_class[u,v]) delete_label(edge_class[v,u]) end delete_label(v) self end # Removes an edge from the graph, can be called with source and target or with # and object of GRATR::Edge derivation def remove_edge!(u, v=nil) unless u.kind_of? GRATR::Edge raise ArgumentError if @parallel_edges u = edge_class[u,v] end raise ArgumentError if @parallel_edges and (u.number || 0) == 0 return self unless @vertex_dict[u.source] # It doesn't exist delete_label(u) # Get rid of label if @parallel_edges index = @edge_number[u.source].index(u.number) raise NoEdgeError unless index @vertex_dict[u.source].delete_at(index) @edge_number[u.source].delete_at(index) else @vertex_dict[u.source].delete(u.target) end self end # Returns an array of vertices that the graph has def vertices() @vertex_dict.keys; end # Returns an array of edges, most likely of class Edge or UndirectedEdge depending # upon the type of graph def edges @vertex_dict.keys.inject(Set.new) do |a,v| if @parallel_edges and @edge_number[v] @vertex_dict[v].zip(@edge_number[v]).each do |w| s,t,n = v,w[0],w[1] a.add( edge_class[ s,t,n, edge_label(s,t,n) ] ) end else @vertex_dict[v].each do |w| a.add(edge_class[v,w,edge_label(v,w)]) end end; a end.to_a end alias graph_adjacent adjacent def adjacent(x, options={}) unless @vertex_dict.has_key?(x) raise ArgumentError, "%s is not a vertex" % x end options[:direction] ||= :out if !x.kind_of?(GRATR::Edge) and (options[:direction] == :out || !directed?) if options[:type] == :edges @parallel_edges ? @vertex_dict[x].map {|v| e=edge_class[x,v,@edge_number[x][v]]; e.label = self[e]; e} : @vertex_dict[x].map {|v| e=edge_class[x,v]; e.label = self[e]; e} else @vertex_dict[x].to_a end else graph_adjacent(x,options) end end public def self.included(cl) # Shortcut for creating a Graph # # Example: GRATR::Digraph[1,2, 2,3, 2,4, 4,5].edges.to_a.to_s => # "(1-2)(2-3)(2-4)(4-5)" # # Or as a Hash for specifying lables # GRATR::Digraph[ [:a,:b] => 3, [:b,:c] => 4 ] (Note: Do not use for Multi or Pseudo graphs) def cl.[] (*a) result = new if a.size == 1 and a[0].kind_of? Hash # Convert to edge class a[0].each do |k,v| if result.edge_class.include? GRATR::EdgeNumber result.add_edge!(result.edge_class[k[0],k[1],nil,v]) else result.add_edge!(result.edge_class[k[0],k[1],v]) end end elsif a[0].kind_of? GRATR::Edge a.each{|e| result.add_edge!(e); result[e] = e.label} elsif a.size % 2 == 0 0.step(a.size-1, 2) {|i| result.add_edge!(a[i], a[i+1])} else raise ArgumentError end result end end end # Adjacency Graph end # GRATR diff --git a/lib/puppet/pgraph.rb b/lib/puppet/pgraph.rb index 56363f9a9..ce7bc52b8 100644 --- a/lib/puppet/pgraph.rb +++ b/lib/puppet/pgraph.rb @@ -1,167 +1,168 @@ #!/usr/bin/env ruby # # Created by Luke A. Kanies on 2006-11-24. # Copyright (c) 2006. All rights reserved. require 'puppet/gratr/digraph' require 'puppet/gratr/import' require 'puppet/gratr/dot' require 'puppet/relationship' # This class subclasses a graph class in order to handle relationships # among resources. class Puppet::PGraph < GRATR::Digraph # This is the type used for splicing. attr_accessor :container_type include Puppet::Util def add_edge!(*args) @reversal = nil super end def add_vertex!(*args) @reversal = nil super end def clear @vertex_dict.clear if defined? @edge_number @edge_number.clear end end # Which resources a given resource depends upon. def dependents(resource) tree_from_vertex2(resource).keys end # Which resources depend upon the given resource. def dependencies(resource) # Cache the reversal graph, because it's somewhat expensive # to create. unless defined? @reversal and @reversal @reversal = reversal end # Strangely, it's significantly faster to search a reversed # tree in the :out direction than to search a normal tree # in the :in direction. @reversal.tree_from_vertex2(resource, :out).keys #tree_from_vertex2(resource, :in).keys end # Override this method to use our class instead. def edge_class() Puppet::Relationship end # Determine all of the leaf nodes below a given vertex. def leaves(vertex, type = :dfs) tree = tree_from_vertex(vertex, type) - leaves = tree.keys.find_all { |c| adjacent(c, :direction => :out).empty? } - return leaves + l = tree.keys.find_all { |c| adjacent(c, :direction => :out).empty? } + return l end # Collect all of the edges that the passed events match. Returns # an array of edges. def matching_edges(events) events.collect do |event| source = event.source unless vertex?(source) Puppet.warning "Got an event from invalid vertex %s" % source.ref next end # Get all of the edges that this vertex should forward events # to, which is the same thing as saying all edges directly below # This vertex in the graph. adjacent(source, :direction => :out, :type => :edges).find_all do |edge| edge.match?(event.event) end.each { |edge| target = edge.target if target.respond_to?(:ref) source.info "Scheduling %s of %s" % [edge.callback, target.ref] end } end.flatten end # Take container information from another graph and use it # to replace any container vertices with their respective leaves. # This creates direct relationships where there were previously # indirect relationships through the containers. def splice!(other, type) vertices.each do |vertex| # Go through each vertex and replace the edges with edges # to the leaves instead next unless vertex.is_a?(type) - leaves = other.leaves(vertex) - if leaves.empty? + #oleaves = other.leaves(vertex) + oleaves = other.adjacent(vertex, :direction => :out) + if oleaves.empty? remove_vertex!(vertex) next end # First create new edges for each of the :in edges adjacent(vertex, :direction => :in, :type => :edges).each do |edge| - leaves.each do |leaf| + oleaves.each do |leaf| add_edge!(edge.source, leaf, edge.label) if cyclic? raise ArgumentError, "%s => %s results in a loop" % - [up, leaf] + [edge.source, leaf] end end end # Then for each of the out edges adjacent(vertex, :direction => :out, :type => :edges).each do |edge| - leaves.each do |leaf| + oleaves.each do |leaf| add_edge!(leaf, edge.target, edge.label) if cyclic? raise ArgumentError, "%s => %s results in a loop" % - [leaf, down] + [leaf, edge.target] end end end # And finally, remove the vertex entirely. remove_vertex!(vertex) end end # For some reason, unconnected vertices do not show up in # this graph. def to_jpg(path, name) gv = vertices() Dir.chdir(path) do induced_subgraph(gv).write_to_graphic_file('jpg', name) end end # A different way of walking a tree, and a much faster way than the # one that comes with GRATR. def tree_from_vertex2(start, direction = :out) predecessor={} walk(start, direction) do |parent, child| predecessor[child] = parent end predecessor end # A support method for tree_from_vertex2. Just walk the tree and pass # the parents and children. def walk(source, direction, &block) adjacent(source, :direction => direction).each do |target| yield source, target walk(target, direction, &block) end end end # $Id$ diff --git a/lib/puppet/transaction.rb b/lib/puppet/transaction.rb index 67d016c15..ff78281ad 100644 --- a/lib/puppet/transaction.rb +++ b/lib/puppet/transaction.rb @@ -1,602 +1,637 @@ # the class that actually walks our resource/state tree, collects the changes, # and performs them require 'puppet' require 'puppet/statechange' module Puppet class Transaction attr_accessor :component, :resources, :ignoreschedules, :ignoretags attr_accessor :relgraph, :sorted_resources attr_writer :tags include Puppet::Util Puppet.config.setdefaults(:transaction, :tags => ["", "Tags to use to find resources. If this is set, then only resources tagged with the specified tags will be applied. - Values must be comma-separated."] + Values must be comma-separated."], + :evaltrace => [false, "Whether each resource should log when it is + being evaluated. This allows you to interactively see exactly + what is being done."] ) # Add some additional times for reporting def addtimes(hash) hash.each do |name, num| @timemetrics[name] = num end end # Apply all changes for a resource, returning a list of the events # generated. def apply(resource) begin changes = resource.evaluate rescue => detail if Puppet[:trace] puts detail.backtrace end resource.err "Failed to retrieve current state: %s" % detail # Mark that it failed @failures[resource] += 1 # And then return return [] end # If a resource is going to be deleted but it still has dependencies, then # don't delete it unless it's implicit. - if ! resource.implicit? and deps = @relgraph.dependents(resource) and ! deps.empty? and changes.detect { |change| - change.state.name == :ensure and change.should == :absent - } - resource.warning "%s still depend%s on me -- not deleting" % - [deps.collect { |r| r.ref }.join(","), if deps.length > 1; ""; else "s"; end] - return [] + if ! resource.implicit? and deleting?(changes) + if deps = @relgraph.dependents(resource) and ! deps.empty? + resource.warning "%s still depend%s on me -- not deleting" % + [deps.collect { |r| r.ref }.join(","), if deps.length > 1; ""; else "s"; end] + return [] + end end unless changes.is_a? Array changes = [changes] end if changes.length > 0 @resourcemetrics[:out_of_sync] += 1 end resourceevents = changes.collect { |change| @changes << change @count += 1 change.transaction = self events = nil begin # use an array, so that changes can return more than one # event if they want events = [change.forward].flatten.reject { |e| e.nil? } rescue => detail if Puppet[:trace] puts detail.backtrace end change.state.err "change from %s to %s failed: %s" % [change.state.is_to_s, change.state.should_to_s, detail] @failures[resource] += 1 next # FIXME this should support using onerror to determine # behaviour; or more likely, the client calling us # should do so end # Mark that our change happened, so it can be reversed # if we ever get to that point unless events.nil? or (events.is_a?(Array) and events.empty?) change.changed = true @resourcemetrics[:applied] += 1 end events }.flatten.reject { |e| e.nil? } unless changes.empty? # Record when we last synced resource.cache(:synced, Time.now) # Flush, if appropriate if resource.respond_to?(:flush) resource.flush end end resourceevents end # Find all of the changed resources. def changed? @changes.find_all { |change| change.changed }.collect { |change| change.state.parent }.uniq end # Do any necessary cleanup. If we don't get rid of the graphs, the # contained resources might never get cleaned up. def cleanup @generated.each do |resource| resource.remove end if defined? @relgraph @relgraph.clear end @resources.clear end - + + # Copy an important relationships from the parent to the newly-generated + # child resource. + def copy_relationships(resource, children) + depthfirst = resource.depthfirst? + + # We only have to worry about dependents, because any dependencies + # already missed their chance to register an event for us. + dependents = @relgraph.adjacent(resource, + :direction => :out, :type => :edges).reject { |e| ! e.callback } + + #sources = @relgraph.adjacent(resource, + # :direction => :in, :type => :edges).reject { |e| ! e.callback } + + children.each do |gen_child| + if depthfirst + @relgraph.add_edge!(gen_child, resource) + else + @relgraph.add_edge!(resource, gen_child) + end + dependents.each do |edge| + @relgraph.add_edge!(gen_child, edge.target, edge.label) + end + #sources.each do |edge| + # @relgraph.add_edge!(edge.source, gen_child, edge.label) + #end + end + end + + # Are we deleting this resource? + def deleting?(changes) + changes.detect { |change| + change.state.name == :ensure and change.should == :absent + } + end + # See if the resource generates new resources at evaluation time. def eval_generate(resource) if resource.respond_to?(:eval_generate) if children = resource.eval_generate - depthfirst = resource.depthfirst? - dependents = @relgraph.adjacent(resource, :direction => :out, :type => :edges) - targets = @relgraph.adjacent(resource, :direction => :in, :type => :edges) - children.each do |gen_child| - if depthfirst - @relgraph.add_edge!(gen_child, resource) - else - @relgraph.add_edge!(resource, gen_child) - end - dependents.each do |edge| - @relgraph.add_edge!(gen_child, edge.target, edge.label) - end - targets.each do |edge| - @relgraph.add_edge!(edge.source, gen_child, edge.label) - end - @generated << gen_child - end + copy_relationships(resource, children) + @generated += children return children end end end # Evaluate a single resource. - def eval_resource(resource) + def eval_resource(resource, checkskip = true) events = [] if resource.is_a?(Puppet::Type::Component) raise Puppet::DevError, "Got a component to evaluate" end - if skip?(resource) + if checkskip and skip?(resource) @resourcemetrics[:skipped] += 1 else @resourcemetrics[:scheduled] += 1 # We need to generate first regardless, because the recursive # actions sometimes change how the top resource is applied. children = eval_generate(resource) if resource.depthfirst? and children children.each do |child| - events += eval_resource(child) + # The child will never be skipped when the parent isn't + events += eval_resource(child, false) end end # Perform the actual changes seconds = thinmark do events += apply(resource) end - + if ! resource.depthfirst? and children children.each do |child| events += eval_resource(child) end end # Keep track of how long we spend in each type of resource @timemetrics[resource.class.name] += seconds end # Check to see if there are any events for this resource if triggedevents = trigger(resource) events += triggedevents end # Collect the targets of any subscriptions to those events @relgraph.matching_edges(events).each do |edge| @targets[edge.target] << edge end # And return the events for collection events end # This method does all the actual work of running a transaction. It # collects all of the changes, executes them, and responds to any # necessary events. def evaluate @count = 0 # Start logging. Puppet::Log.newdestination(@report) prepare() begin allevents = @sorted_resources.collect { |resource| - eval_resource(resource) + ret = nil + seconds = thinmark do + ret = eval_resource(resource) + end + + if Puppet[:evaltrace] + resource.info "Evaluated in %0.2f seconds" % seconds + end + ret }.flatten.reject { |e| e.nil? } ensure # And then close the transaction log. Puppet::Log.close(@report) end + @relgraph.to_jpg(File.expand_path("~/tmp/graphs"), "simple_relgraph") + Puppet.debug "Finishing transaction %s with %s changes" % [self.object_id, @count] allevents end # Determine whether a given resource has failed. def failed?(obj) if @failures[obj] > 0 return @failures[obj] else return false end end # Does this resource have any failed dependencies? def failed_dependencies?(resource) # First make sure there are no failed dependencies. To do this, # we check for failures in any of the vertexes above us. It's not # enough to check the immediate dependencies, which is why we use # a tree from the reversed graph. skip = false - @relgraph.dependencies(resource).each do |dep| + deps = @relgraph.dependencies(resource) + deps.each do |dep| if fails = failed?(dep) resource.notice "Dependency %s[%s] has %s failures" % [dep.class.name, dep.name, @failures[dep]] skip = true end end return skip end # Collect any dynamically generated resources. def generate list = @resources.vertices # Store a list of all generated resources, so that we can clean them up # after the transaction closes. @generated = [] newlist = [] while ! list.empty? list.each do |resource| if resource.respond_to?(:generate) made = resource.generate next unless made unless made.is_a?(Array) made = [made] end made.uniq! made.each do |res| @resources.add_vertex!(res) newlist << res @generated << res end end end list.clear list = newlist newlist = [] end end # this should only be called by a Puppet::Type::Component resource now # and it should only receive an array def initialize(resources) @resources = resources.to_graph @resourcemetrics = { :total => @resources.vertices.length, :out_of_sync => 0, # The number of resources that had changes :applied => 0, # The number of resources fixed :skipped => 0, # The number of resources skipped :restarted => 0, # The number of resources triggered :failed_restarts => 0, # The number of resources that fail a trigger :scheduled => 0 # The number of resources scheduled } # Metrics for distributing times across the different types. @timemetrics = Hash.new(0) # The number of resources that were triggered in this run @triggered = Hash.new { |hash, key| hash[key] = Hash.new(0) } # Targets of being triggered. @targets = Hash.new do |hash, key| hash[key] = [] end # The changes we're performing @changes = [] # The resources that have failed and the number of failures each. This # is used for skipping resources because of failed dependencies. @failures = Hash.new do |h, key| h[key] = 0 end @report = Report.new end # Prefetch any providers that support it. We don't support prefetching # types, just providers. def prefetch @resources.collect { |obj| if pro = obj.provider pro.class else nil end }.reject { |o| o.nil? }.uniq.each do |klass| # XXX We need to do something special here in case of failure. if klass.respond_to?(:prefetch) klass.prefetch end end end # Prepare to evaluate the elements in a transaction. def prepare prefetch() # Now add any dynamically generated resources generate() # Create a relationship graph from our resource graph @relgraph = relationship_graph @sorted_resources = @relgraph.topsort end # Create a graph of all of the relationships in our resource graph. def relationship_graph graph = Puppet::PGraph.new # First create the dependency graph @resources.vertices.each do |vertex| graph.add_vertex!(vertex) vertex.builddepends.each do |edge| graph.add_edge!(edge) end end # Then splice in the container information graph.splice!(@resources, Puppet::Type::Component) # Lastly, add in any autorequires graph.vertices.each do |vertex| vertex.autorequire.each do |edge| unless graph.edge?(edge) graph.add_edge!(edge) end end end return graph end # Generate a transaction report. def report @resourcemetrics[:failed] = @failures.find_all do |name, num| num > 0 end.length # Get the total time spent @timemetrics[:total] = @timemetrics.inject(0) do |total, vals| total += vals[1] total end # Unfortunately, RRD does not deal well with changing lists of values, # so we have to pick a list of values and stick with it. In this case, # that means we record the total time, the config time, and that's about # it. We should probably send each type's time as a separate metric. @timemetrics.dup.each do |name, value| if Puppet::Type.type(name) @timemetrics.delete(name) end end # Add all of the metrics related to resource count and status @report.newmetric(:resources, @resourcemetrics) # Record the relative time spent in each resource. @report.newmetric(:time, @timemetrics) # Then all of the change-related metrics @report.newmetric(:changes, :total => @changes.length ) @report.time = Time.now return @report end # Roll all completed changes back. def rollback @targets.clear @triggered.clear allevents = @changes.reverse.collect { |change| # skip changes that were never actually run unless change.changed Puppet.debug "%s was not changed" % change.to_s next end begin events = change.backward rescue => detail Puppet.err("%s rollback failed: %s" % [change,detail]) if Puppet[:trace] puts detail.backtrace end next # at this point, we would normally do error handling # but i haven't decided what to do for that yet # so just record that a sync failed for a given resource #@@failures[change.state.parent] += 1 # this still could get hairy; what if file contents changed, # but a chmod failed? how would i handle that error? dern end @relgraph.matching_edges(events).each do |edge| @targets[edge.target] << edge end # Now check to see if there are any events for this child. # Kind of hackish, since going backwards goes a change at a # time, not a child at a time. trigger(change.state.parent) # And return the events for collection events }.flatten.reject { |e| e.nil? } end # Is the resource currently scheduled? def scheduled?(resource) self.ignoreschedules or resource.scheduled? end # Should this resource be skipped? def skip?(resource) skip = false if ! tagged?(resource) resource.debug "Not tagged with %s" % tags.join(", ") elsif ! scheduled?(resource) resource.debug "Not scheduled" elsif failed_dependencies?(resource) resource.warning "Skipping because of failed dependencies" else return false end return true end # The tags we should be checking. def tags # Allow the tags to be overridden unless defined? @tags @tags = Puppet[:tags] end unless defined? @processed_tags if @tags.nil? or @tags == "" @tags = [] else @tags = [@tags] unless @tags.is_a? Array @tags = @tags.collect do |tag| tag.split(/\s*,\s*/) end.flatten end @processed_tags = true end @tags end # Is this resource tagged appropriately? def tagged?(resource) self.ignoretags or tags.empty? or resource.tagged?(tags) end # Are there any edges that target this resource? def targeted?(resource) @targets[resource] end # Trigger any subscriptions to a child. This does an upwardly recursive # search -- it triggers the passed resource, but also the resource's parent # and so on up the tree. def trigger(child) obj = child callbacks = Hash.new { |hash, key| hash[key] = [] } sources = Hash.new { |hash, key| hash[key] = [] } trigged = [] while obj if @targets.include?(obj) callbacks.clear sources.clear @targets[obj].each do |edge| # Some edges don't have callbacks next unless edge.callback # Collect all of the subs for each callback callbacks[edge.callback] << edge # And collect the sources for logging sources[edge.source] << edge.callback end sources.each do |source, callbacklist| obj.debug "%s[%s] results in triggering %s" % [source.class.name, source.name, callbacklist.join(", ")] end callbacks.each do |callback, subs| message = "Triggering '%s' from %s dependencies" % [callback, subs.length] obj.notice message # At this point, just log failures, don't try to react # to them in any way. begin obj.send(callback) @resourcemetrics[:restarted] += 1 rescue => detail obj.err "Failed to call %s on %s: %s" % [callback, obj, detail] @resourcemetrics[:failed_restarts] += 1 if Puppet[:trace] puts detail.backtrace end end # And then add an event for it. trigged << Puppet::Event.new( :event => :triggered, :transaction => self, :source => obj, :message => message ) triggered(obj, callback) end end obj = obj.parent end if trigged.empty? return nil else return trigged end end def triggered(resource, method) @triggered[resource][method] += 1 end def triggered?(resource, method) @triggered[resource][method] end end end require 'puppet/transaction/report' # $Id$ diff --git a/test/other/transactions.rb b/test/other/transactions.rb index d9a173986..2ae4b1b13 100755 --- a/test/other/transactions.rb +++ b/test/other/transactions.rb @@ -1,786 +1,788 @@ #!/usr/bin/env ruby $:.unshift("../lib").unshift("../../lib") if __FILE__ =~ /\.rb$/ require 'puppet' require 'puppettest' require 'puppettest/support/resources' # $Id$ class TestTransactions < Test::Unit::TestCase include PuppetTest::FileTesting include PuppetTest::Support::Resources def mkgenerator(&block) # Create a bogus type that generates new instances with shorter type = Puppet::Type.newtype(:generator) do newparam(:name, :namevar => true) end if block type.class_eval(&block) end cleanup do Puppet::Type.rmtype(:generator) end return type end def test_reports path1 = tempfile() path2 = tempfile() objects = [] objects << Puppet::Type.newfile( :path => path1, :content => "yayness" ) objects << Puppet::Type.newfile( :path => path2, :content => "booness" ) trans = assert_events([:file_created, :file_created], *objects) report = nil assert_nothing_raised { report = trans.report } # First test the report logs assert(report.logs.length > 0, "Did not get any report logs") report.logs.each do |obj| assert_instance_of(Puppet::Log, obj) end # Then test the metrics metrics = report.metrics assert(metrics, "Did not get any metrics") assert(metrics.length > 0, "Did not get any metrics") assert(metrics.has_key?("resources"), "Did not get object metrics") assert(metrics.has_key?("changes"), "Did not get change metrics") metrics.each do |name, metric| assert_instance_of(Puppet::Metric, metric) end end def test_prefetch # Create a type just for testing prefetch name = :prefetchtesting $prefetched = false type = Puppet::Type.newtype(name) do newparam(:name) {} end cleanup do Puppet::Type.rmtype(name) end # Now create a provider type.provide(:prefetch) do def self.prefetch $prefetched = true end end # Now create an instance inst = type.create :name => "yay" # Create a transaction trans = Puppet::Transaction.new(newcomp(inst)) # Make sure prefetch works assert_nothing_raised do trans.prefetch end assert_equal(true, $prefetched, "type prefetch was not called") # Now make sure it gets called from within evaluate() $prefetched = false assert_nothing_raised do trans.evaluate end assert_equal(true, $prefetched, "evaluate did not call prefetch") end def test_refreshes_generate_events path = tempfile() firstpath = tempfile() secondpath = tempfile() file = Puppet::Type.newfile(:title => "file", :path => path, :content => "yayness") first = Puppet::Type.newexec(:title => "first", :command => "/bin/echo first > #{firstpath}", :subscribe => [:file, path], :refreshonly => true ) second = Puppet::Type.newexec(:title => "second", :command => "/bin/echo second > #{secondpath}", :subscribe => [:exec, "first"], :refreshonly => true ) assert_apply(file, first, second) assert(FileTest.exists?(secondpath), "Refresh did not generate an event") end unless %x{groups}.chomp.split(/ /).length > 1 $stderr.puts "You must be a member of more than one group to test transactions" else def ingroup(gid) require 'etc' begin group = Etc.getgrgid(gid) rescue => detail puts "Could not retrieve info for group %s: %s" % [gid, detail] return nil end return @groups.include?(group.name) end def setup super @groups = %x{groups}.chomp.split(/ /) unless @groups.length > 1 p @groups raise "You must be a member of more than one group to test this" end end def newfile(hash = {}) tmpfile = tempfile() File.open(tmpfile, "w") { |f| f.puts rand(100) } # XXX now, because os x apparently somehow allows me to make a file # owned by a group i'm not a member of, i have to verify that # the file i just created is owned by one of my groups # grrr unless ingroup(File.stat(tmpfile).gid) Puppet.info "Somehow created file in non-member group %s; fixing" % File.stat(tmpfile).gid require 'etc' firstgr = @groups[0] unless firstgr.is_a?(Integer) str = Etc.getgrnam(firstgr) firstgr = str.gid end File.chown(nil, firstgr, tmpfile) end hash[:name] = tmpfile assert_nothing_raised() { return Puppet.type(:file).create(hash) } end def newservice assert_nothing_raised() { return Puppet.type(:service).create( :name => "sleeper", :type => "init", :path => exampledir("root/etc/init.d"), :hasstatus => true, :check => [:ensure] ) } end def newexec(file) assert_nothing_raised() { return Puppet.type(:exec).create( :name => "touch %s" % file, :path => "/bin:/usr/bin:/sbin:/usr/sbin", :returns => 0 ) } end # modify a file and then roll the modifications back def test_filerollback transaction = nil file = newfile() states = {} check = [:group,:mode] file[:check] = check assert_nothing_raised() { file.retrieve } assert_nothing_raised() { check.each { |state| assert(file[state]) states[state] = file[state] } } component = newcomp("file",file) require 'etc' groupname = Etc.getgrgid(File.stat(file.name).gid).name assert_nothing_raised() { # Find a group that it's not set to group = @groups.find { |group| group != groupname } unless group raise "Could not find suitable group" end file[:group] = group file[:mode] = "755" } trans = assert_events([:file_changed, :file_changed], component) file.retrieve assert_rollback_events(trans, [:file_changed, :file_changed], "file") assert_nothing_raised() { file.retrieve } states.each { |state,value| assert_equal( value,file.is(state), "File %s remained %s" % [state, file.is(state)] ) } end # start a service, and then roll the modification back # Disabled, because it wasn't really worth the effort. def disabled_test_servicetrans transaction = nil service = newservice() component = newcomp("service",service) assert_nothing_raised() { service[:ensure] = 1 } service.retrieve assert(service.insync?, "Service did not start") system("ps -ef | grep ruby") trans = assert_events([:service_started], component) service.retrieve assert_rollback_events(trans, [:service_stopped], "service") end # test that services are correctly restarted and that work is done # in the right order def test_refreshing transaction = nil file = newfile() execfile = File.join(tmpdir(), "exectestingness") exec = newexec(execfile) states = {} check = [:group,:mode] file[:check] = check file[:group] = @groups[0] assert_apply(file) @@tmpfiles << execfile component = newcomp("both",file,exec) # 'subscribe' expects an array of arrays exec[:subscribe] = [[file.class.name,file.name]] exec[:refreshonly] = true assert_nothing_raised() { file.retrieve exec.retrieve } check.each { |state| states[state] = file[state] } assert_nothing_raised() { file[:mode] = "755" } trans = assert_events([:file_changed, :triggered], component) assert(FileTest.exists?(execfile), "Execfile does not exist") File.unlink(execfile) assert_nothing_raised() { file[:group] = @groups[1] } trans = assert_events([:file_changed, :triggered], component) assert(FileTest.exists?(execfile), "Execfile does not exist") end # Verify that one component requiring another causes the contained # resources in the requiring component to get refreshed. def test_refresh_across_two_components transaction = nil file = newfile() execfile = File.join(tmpdir(), "exectestingness2") @@tmpfiles << execfile exec = newexec(execfile) states = {} check = [:group,:mode] file[:check] = check file[:group] = @groups[0] assert_apply(file) fcomp = newcomp("file",file) ecomp = newcomp("exec",exec) component = newcomp("both",fcomp,ecomp) # 'subscribe' expects an array of arrays #component[:require] = [[file.class.name,file.name]] ecomp[:subscribe] = fcomp exec[:refreshonly] = true trans = assert_events([], component) assert_nothing_raised() { file[:group] = @groups[1] file[:mode] = "755" } trans = assert_events([:file_changed, :file_changed, :triggered], component) end # Make sure that multiple subscriptions get triggered. def test_multisubs path = tempfile() file1 = tempfile() file2 = tempfile() file = Puppet.type(:file).create( :path => path, :ensure => "file" ) exec1 = Puppet.type(:exec).create( :path => ENV["PATH"], :command => "touch %s" % file1, :refreshonly => true, :subscribe => [:file, path] ) exec2 = Puppet.type(:exec).create( :path => ENV["PATH"], :command => "touch %s" % file2, :refreshonly => true, :subscribe => [:file, path] ) assert_apply(file, exec1, exec2) assert(FileTest.exists?(file1), "File 1 did not get created") assert(FileTest.exists?(file2), "File 2 did not get created") end # Make sure that a failed trigger doesn't result in other events not # getting triggered. def test_failedrefreshes path = tempfile() newfile = tempfile() file = Puppet.type(:file).create( :path => path, :ensure => "file" ) svc = Puppet.type(:service).create( :name => "thisservicedoesnotexist", :subscribe => [:file, path] ) exec = Puppet.type(:exec).create( :path => ENV["PATH"], :command => "touch %s" % newfile, :logoutput => true, :refreshonly => true, :subscribe => [:file, path] ) assert_apply(file, svc, exec) assert(FileTest.exists?(path), "File did not get created") assert(FileTest.exists?(newfile), "Refresh file did not get created") end # Make sure that unscheduled and untagged objects still respond to events def test_unscheduled_and_untagged_response Puppet::Type.type(:schedule).mkdefaultschedules Puppet[:ignoreschedules] = false file = Puppet.type(:file).create( :name => tempfile(), :ensure => "file" ) fname = tempfile() exec = Puppet.type(:exec).create( :name => "touch %s" % fname, :path => "/usr/bin:/bin", :schedule => "monthly", :subscribe => ["file", file.name] ) comp = newcomp(file,exec) comp.finalize # Run it once assert_apply(comp) assert(FileTest.exists?(fname), "File did not get created") assert(!exec.scheduled?, "Exec is somehow scheduled") # Now remove it, so it can get created again File.unlink(fname) file[:content] = "some content" assert_events([:file_changed, :triggered], comp) assert(FileTest.exists?(fname), "File did not get recreated") # Now remove it, so it can get created again File.unlink(fname) # And tag our exec exec.tag("testrun") # And our file, so it runs file.tag("norun") Puppet[:tags] = "norun" file[:content] = "totally different content" assert(! file.insync?, "Uh, file is in sync?") assert_events([:file_changed, :triggered], comp) assert(FileTest.exists?(fname), "File did not get recreated") end def test_failed_reqs_mean_no_run exec = Puppet::Type.type(:exec).create( :command => "/bin/mkdir /this/path/cannot/possibly/exit", :title => "mkdir" ) file1 = Puppet::Type.type(:file).create( :title => "file1", :path => tempfile(), :require => exec, :ensure => :file ) file2 = Puppet::Type.type(:file).create( :title => "file2", :path => tempfile(), :require => file1, :ensure => :file ) comp = newcomp(exec, file1, file2) comp.finalize assert_apply(comp) assert(! FileTest.exists?(file1[:path]), "File got created even tho its dependency failed") assert(! FileTest.exists?(file2[:path]), "File got created even tho its deep dependency failed") end end def f(n) Puppet::Type.type(:file)["/tmp/#{n.to_s}"] end def test_relationship_graph one, two, middle, top = mktree {one => two, "f" => "c", "h" => middle}.each do |source, target| if source.is_a?(String) source = f(source) end if target.is_a?(String) target = f(target) end target[:require] = source end trans = Puppet::Transaction.new(top) graph = nil assert_nothing_raised do graph = trans.relationship_graph end assert_instance_of(Puppet::PGraph, graph, "Did not get relationship graph") # Make sure all of the components are gone comps = graph.vertices.find_all { |v| v.is_a?(Puppet::Type::Component)} assert(comps.empty?, "Deps graph still contains components") # It must be reversed because of how topsort works sorted = graph.topsort.reverse # Now make sure the appropriate edges are there and are in the right order assert(graph.dependents(f(:f)).include?(f(:c)), "c not marked a dep of f") assert(sorted.index(f(:c)) < sorted.index(f(:f)), "c is not before f") one.each do |o| two.each do |t| assert(graph.dependents(o).include?(t), "%s not marked a dep of %s" % [t.ref, o.ref]) assert(sorted.index(t) < sorted.index(o), "%s is not before %s" % [t.ref, o.ref]) end end trans.resources.leaves(middle).each do |child| assert(graph.dependents(f(:h)).include?(child), "%s not marked a dep of h" % [child.ref]) assert(sorted.index(child) < sorted.index(f(:h)), "%s is not before h" % child.ref) end # Lastly, make sure our 'g' vertex made it into the relationship # graph, since it's not involved in any relationships. assert(graph.vertex?(f(:g)), "Lost vertexes with no relations") # Now make the reversal graph and make sure all of the vertices made it into that reverse = graph.reversal %w{a b c d e f g h}.each do |letter| file = f(letter) assert(reverse.vertex?(file), "%s did not make it into reversal" % letter) end end # Test pre-evaluation generation def test_generate mkgenerator() do def generate ret = [] if title.length > 1 ret << self.class.create(:title => title[0..-2]) else return nil end ret end end yay = Puppet::Type.newgenerator :title => "yay" rah = Puppet::Type.newgenerator :title => "rah" comp = newcomp(yay, rah) trans = comp.evaluate assert_nothing_raised do trans.generate end %w{ya ra y r}.each do |name| assert(trans.resources.vertex?(Puppet::Type.type(:generator)[name]), "Generated %s was not a vertex" % name) end # Now make sure that cleanup gets rid of those generated types. assert_nothing_raised do trans.cleanup end %w{ya ra y r}.each do |name| assert(!trans.resources.vertex?(Puppet::Type.type(:generator)[name]), "Generated vertex %s was not removed from graph" % name) assert_nil(Puppet::Type.type(:generator)[name], "Generated vertex %s was not removed from class" % name) end end # Test mid-evaluation generation. def test_eval_generate $evaluated = [] type = mkgenerator() do def eval_generate ret = [] if title.length > 1 ret << self.class.create(:title => title[0..-2]) else return nil end ret end def evaluate $evaluated << self.title return [] end end yay = Puppet::Type.newgenerator :title => "yay" rah = Puppet::Type.newgenerator :title => "rah", :subscribe => yay comp = newcomp(yay, rah) trans = comp.evaluate trans.prepare # Now apply the resources, and make sure they appropriately generate # things. assert_nothing_raised("failed to apply yay") do trans.eval_resource(yay) end ya = type["ya"] assert(ya, "Did not generate ya") assert(trans.relgraph.vertex?(ya), "Did not add ya to rel_graph") # Now make sure the appropriate relationships were added assert(trans.relgraph.edge?(yay, ya), "parent was not required by child") assert(trans.relgraph.edge?(ya, rah), "rah was not subscribed to ya") # And make sure the relationship is a subscription with a callback, # not just a require. assert_equal({:callback => :refresh, :event => :ALL_EVENTS}, trans.relgraph[Puppet::Relationship.new(ya, rah)], "The label was not retained") # Now make sure it in turn eval_generates appropriately assert_nothing_raised("failed to apply yay") do trans.eval_resource(type["ya"]) end %w{y}.each do |name| res = type[name] assert(res, "Did not generate %s" % name) assert(trans.relgraph.vertex?(res), "Did not add %s to rel_graph" % name) end assert_nothing_raised("failed to eval_generate with nil response") do trans.eval_resource(type["y"]) end assert(trans.relgraph.edge?(yay, ya), "no edge was created for ya => yay") assert_nothing_raised("failed to apply rah") do trans.eval_resource(rah) end ra = type["ra"] assert(ra, "Did not generate ra") assert(trans.relgraph.vertex?(ra), "Did not add ra to rel_graph" % name) - # Now make sure this generated resource has the same relationships as the generating - # resource - assert(trans.relgraph.edge?(yay, ra), - "yay is not required by ra") + # Now make sure this generated resource has the same relationships as + # the generating resource + #assert(trans.relgraph.edge?(yay, ra), + # "yay is not required by ra") + assert(trans.relgraph.edge?(ya, rah), + "rah is not subscribed to ya") assert(trans.relgraph.edge?(ya, ra), "ra is not subscribed to ya") # And make sure the relationship is a subscription with a callback, # not just a require. assert_equal({:callback => :refresh, :event => :ALL_EVENTS}, trans.relgraph[Puppet::Relationship.new(ya, ra)], "The label was not retained") # Now make sure that cleanup gets rid of those generated types. assert_nothing_raised do trans.cleanup end %w{ya ra y r}.each do |name| assert(!trans.relgraph.vertex?(type[name]), "Generated vertex %s was not removed from graph" % name) assert_nil(type[name], "Generated vertex %s was not removed from class" % name) end # Now, start over and make sure that everything gets evaluated. trans = comp.evaluate $evaluated.clear assert_nothing_raised do trans.evaluate end assert_equal(%w{yay ya y rah ra r}, $evaluated, "Not all resources were evaluated or not in the right order") end def test_tags res = Puppet::Type.newfile :path => tempfile() comp = newcomp(res) # Make sure they default to none assert_equal([], comp.evaluate.tags) # Make sure we get the main tags Puppet[:tags] = %w{this is some tags} assert_equal(%w{this is some tags}, comp.evaluate.tags) # And make sure they get processed correctly Puppet[:tags] = ["one", "two,three", "four"] assert_equal(%w{one two three four}, comp.evaluate.tags) # lastly, make sure we can override them trans = comp.evaluate trans.tags = ["one", "two,three", "four"] assert_equal(%w{one two three four}, comp.evaluate.tags) end def test_tagged? res = Puppet::Type.newfile :path => tempfile() comp = newcomp(res) trans = comp.evaluate assert(trans.tagged?(res), "tagged? defaulted to false") # Now set some tags trans.tags = %w{some tags} # And make sure it's false assert(! trans.tagged?(res), "matched invalid tags") # Set ignoretags and make sure it sticks trans.ignoretags = true assert(trans.tagged?(res), "tags were not ignored") # Now make sure we actually correctly match tags res[:tag] = "mytag" trans.ignoretags = false trans.tags = %w{notag} assert(! trans.tagged?(res), "tags incorrectly matched") trans.tags = %w{mytag yaytag} assert(trans.tagged?(res), "tags should have matched") end # We don't want to purge resources that have relationships with other resources, # so we want our transactions to check for that. def test_required_resources_not_deleted @file = Puppet::Type.type(:file) path1 = tempfile() path2 = tempfile() # Create our first file File.open(path1, "w") { |f| f.puts "yay" } # Create a couple of related resources file1 = @file.create :title => "dependee", :path => path1, :ensure => :absent file2 = @file.create :title => "depender", :path => path2, :content => "some stuff", :require => file1 # Now make sure we don't actually delete the first file assert_apply(file1, file2) assert(FileTest.exists?(path1), "required file was deleted") end end # $Id$