diff --git a/lib/puppet/configurer.rb b/lib/puppet/configurer.rb index a39f9cda8..72e387c64 100644 --- a/lib/puppet/configurer.rb +++ b/lib/puppet/configurer.rb @@ -1,261 +1,248 @@ # The client for interacting with the puppetmaster config server. require 'sync' require 'timeout' require 'puppet/network/http_pool' require 'puppet/util' -require 'puppet/util/instrumentation' class Puppet::Configurer class CommandHookError < RuntimeError; end require 'puppet/configurer/fact_handler' require 'puppet/configurer/plugin_handler' include Puppet::Configurer::FactHandler include Puppet::Configurer::PluginHandler - include Puppet::Util::Instrumentation # For benchmarking include Puppet::Util attr_reader :compile_time # Provide more helpful strings to the logging that the Agent does def self.to_s "Puppet configuration client" end class << self # Puppetd should only have one instance running, and we need a way # to retrieve it. attr_accessor :instance include Puppet::Util end # How to lock instances of this class. def self.lockfile_path Puppet[:puppetdlockfile] end def clear @catalog.clear(true) if @catalog @catalog = nil end def execute_postrun_command execute_from_setting(:postrun_command) end def execute_prerun_command execute_from_setting(:prerun_command) end # Initialize and load storage def dostorage Puppet::Util::Storage.load @compile_time ||= Puppet::Util::Storage.cache(:configuration)[:compile_time] rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Corrupt state file #{Puppet[:statefile]}: #{detail}" begin ::File.unlink(Puppet[:statefile]) retry rescue => detail raise Puppet::Error.new("Cannot remove #{Puppet[:statefile]}: #{detail}") end end # Just so we can specify that we are "the" instance. def initialize Puppet.settings.use(:main, :ssl, :agent) self.class.instance = self @running = false @splayed = false end # Prepare for catalog retrieval. Downloads everything necessary, etc. def prepare(options) dostorage - instrument("downloading plugins") do - download_plugins unless options[:skip_plugin_download] - end + download_plugins unless options[:skip_plugin_download] - instrument("downloading facts plugins") do - download_fact_plugins unless options[:skip_plugin_download] - end + download_fact_plugins unless options[:skip_plugin_download] - instrument("executing prerun command") do - execute_prerun_command - end + execute_prerun_command end # Get the remote catalog, yo. Returns nil if no catalog can be found. def retrieve_catalog if Puppet::Resource::Catalog.indirection.terminus_class == :rest # This is a bit complicated. We need the serialized and escaped facts, # and we need to know which format they're encoded in. Thus, we # get a hash with both of these pieces of information. fact_options = facts_for_uploading else fact_options = {} end # First try it with no cache, then with the cache. unless (Puppet[:use_cached_catalog] and result = retrieve_catalog_from_cache(fact_options)) or result = retrieve_new_catalog(fact_options) if ! Puppet[:usecacheonfailure] Puppet.warning "Not using cache on failed catalog" return nil end result = retrieve_catalog_from_cache(fact_options) end return nil unless result convert_catalog(result, @duration) end # Convert a plain resource catalog into our full host catalog. def convert_catalog(result, duration) catalog = result.to_ral catalog.finalize catalog.retrieval_duration = duration catalog.write_class_file catalog end # The code that actually runs the catalog. # This just passes any options on to the catalog, # which accepts :tags and :ignoreschedules. def run(options = {}) begin prepare(options) rescue SystemExit,NoMemoryError raise rescue Exception => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Failed to prepare catalog: #{detail}" end options[:report] ||= Puppet::Transaction::Report.new("apply") report = options[:report] Puppet::Util::Log.newdestination(report) if catalog = options[:catalog] options.delete(:catalog) elsif ! catalog = retrieve_catalog Puppet.err "Could not retrieve catalog; skipping run" return end report.configuration_version = catalog.version transaction = nil begin - instrument("applying catalog") do - benchmark(:notice, "Finished catalog run") do - transaction = catalog.apply(options) - end + benchmark(:notice, "Finished catalog run") do + transaction = catalog.apply(options) end report rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Failed to apply catalog: #{detail}" return end ensure # Make sure we forget the retained module_directories of any autoload # we might have used. Thread.current[:env_module_directories] = nil # Now close all of our existing http connections, since there's no # reason to leave them lying open. Puppet::Network::HttpPool.clear_http_instances execute_postrun_command Puppet::Util::Log.close(report) - - instrument("sending report") do - send_report(report, transaction) - end + send_report(report, transaction) end def send_report(report, trans) report.finalize_report if trans puts report.summary if Puppet[:summarize] save_last_run_summary(report) if Puppet[:report] Puppet::Transaction::Report.indirection.save(report) end rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Could not send report: #{detail}" end def save_last_run_summary(report) Puppet::Util::FileLocking.writelock(Puppet[:lastrunfile], 0660) do |file| file.print YAML.dump(report.raw_summary) end rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Could not save last run local report: #{detail}" end private def self.timeout timeout = Puppet[:configtimeout] case timeout when String if timeout =~ /^\d+$/ timeout = Integer(timeout) else raise ArgumentError, "Configuration timeout must be an integer" end when Integer # nothing else raise ArgumentError, "Configuration timeout must be an integer" end timeout end def execute_from_setting(setting) return if (command = Puppet[setting]) == "" begin Puppet::Util.execute([command]) rescue => detail raise CommandHookError, "Could not run command from #{setting}: #{detail}" end end def retrieve_catalog_from_cache(fact_options) result = nil @duration = thinmark do result = Puppet::Resource::Catalog.indirection.find(Puppet[:certname], fact_options.merge(:ignore_terminus => true)) end Puppet.notice "Using cached catalog" result rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Could not retrieve catalog from cache: #{detail}" return nil end def retrieve_new_catalog(fact_options) result = nil @duration = thinmark do result = Puppet::Resource::Catalog.indirection.find(Puppet[:certname], fact_options.merge(:ignore_cache => true)) end result rescue SystemExit,NoMemoryError raise rescue Exception => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Could not retrieve catalog from remote server: #{detail}" return nil end end diff --git a/lib/puppet/network/http/handler.rb b/lib/puppet/network/http/handler.rb index aa33f82f7..2b9e81b61 100644 --- a/lib/puppet/network/http/handler.rb +++ b/lib/puppet/network/http/handler.rb @@ -1,248 +1,244 @@ module Puppet::Network::HTTP end require 'puppet/network/http/api/v1' require 'puppet/network/rest_authorization' require 'puppet/network/rights' require 'resolv' -require 'puppet/util/instrumentation' module Puppet::Network::HTTP::Handler include Puppet::Network::HTTP::API::V1 include Puppet::Network::RestAuthorization - include Puppet::Util::Instrumentation attr_reader :server, :handler # Retrieve the accept header from the http request. def accept_header(request) raise NotImplementedError end # Retrieve the Content-Type header from the http request. def content_type_header(request) raise NotImplementedError end # Which format to use when serializing our response or interpreting the request. # IF the client provided a Content-Type use this, otherwise use the Accept header # and just pick the first value. def format_to_use(request) unless header = accept_header(request) raise ArgumentError, "An Accept header must be provided to pick the right format" end format = nil header.split(/,\s*/).each do |name| next unless format = Puppet::Network::FormatHandler.format(name) next unless format.suitable? return format end raise "No specified acceptable formats (#{header}) are functional on this machine" end def request_format(request) if header = content_type_header(request) header.gsub!(/\s*;.*$/,'') # strip any charset format = Puppet::Network::FormatHandler.mime(header) raise "Client sent a mime-type (#{header}) that doesn't correspond to a format we support" if format.nil? return format.name.to_s if format.suitable? end raise "No Content-Type header was received, it isn't possible to unserialize the request" end def format_to_mime(format) format.is_a?(Puppet::Network::Format) ? format.mime : format end def initialize_for_puppet(server) @server = server end # handle an HTTP request def process(request, response) indirection, method, key, params = uri2indirection(http_method(request), path(request), params(request)) check_authorization(indirection, method, key, params) - instrument("processing #{indirection} #{key}") do - send("do_#{method}", indirection, key, params, request, response) - end + send("do_#{method}", indirection, key, params, request, response) rescue SystemExit,NoMemoryError raise rescue Exception => e return do_exception(response, e) end # Set the response up, with the body and status. def set_response(response, body, status = 200) raise NotImplementedError end # Set the specified format as the content type of the response. def set_content_type(response, format) raise NotImplementedError end def do_exception(response, exception, status=400) if exception.is_a?(Puppet::Network::AuthorizationError) # make sure we return the correct status code # for authorization issues status = 403 if status == 400 end if exception.is_a?(Exception) puts exception.backtrace if Puppet[:trace] Puppet.err(exception) end set_content_type(response, "text/plain") set_response(response, exception.to_s, status) end def model(indirection_name) raise ArgumentError, "Could not find indirection '#{indirection_name}'" unless indirection = Puppet::Indirector::Indirection.instance(indirection_name.to_sym) indirection.model end # Execute our find. def do_find(indirection_name, key, params, request, response) unless result = model(indirection_name).indirection.find(key, params) Puppet.info("Could not find #{indirection_name} for '#{key}'") return do_exception(response, "Could not find #{indirection_name} #{key}", 404) end # The encoding of the result must include the format to use, # and it needs to be used for both the rendering and as # the content type. format = format_to_use(request) set_content_type(response, format) if result.respond_to?(:render) set_response(response, result.render(format)) else set_response(response, result) end end # Execute our head. def do_head(indirection_request, request, response) unless indirection_request.model.head(indirection_request.key, indirection_request.to_hash) Puppet.info("Could not find #{indirection_request.indirection_name} for '#{indirection_request.key}'") return do_exception(response, "Could not find #{indirection_request.indirection_name} #{indirection_request.key}", 404) end # No need to set a response because no response is expected from a # HEAD request. All we need to do is not die. end # Execute our search. def do_search(indirection_name, key, params, request, response) model = self.model(indirection_name) result = model.indirection.search(key, params) if result.nil? return do_exception(response, "Could not find instances in #{indirection_name} with '#{key}'", 404) end format = format_to_use(request) set_content_type(response, format) set_response(response, model.render_multiple(format, result)) end # Execute our destroy. def do_destroy(indirection_name, key, params, request, response) result = model(indirection_name).indirection.destroy(key, params) return_yaml_response(response, result) end # Execute our save. def do_save(indirection_name, key, params, request, response) data = body(request).to_s raise ArgumentError, "No data to save" if !data or data.empty? format = request_format(request) obj = model(indirection_name).convert_from(format, data) result = model(indirection_name).indirection.save(obj, key) return_yaml_response(response, result) end # resolve node name from peer's ip address # this is used when the request is unauthenticated def resolve_node(result) begin return Resolv.getname(result[:ip]) rescue => detail Puppet.err "Could not resolve #{result[:ip]}: #{detail}" end result[:ip] end private def return_yaml_response(response, body) set_content_type(response, Puppet::Network::FormatHandler.format("yaml")) set_response(response, body.to_yaml) end def get?(request) http_method(request) == 'GET' end def put?(request) http_method(request) == 'PUT' end def delete?(request) http_method(request) == 'DELETE' end # methods to be overridden by the including web server class def http_method(request) raise NotImplementedError end def path(request) raise NotImplementedError end def request_key(request) raise NotImplementedError end def body(request) raise NotImplementedError end def params(request) raise NotImplementedError end def decode_params(params) params.inject({}) do |result, ary| param, value = ary next result if param.nil? || param.empty? param = param.to_sym # These shouldn't be allowed to be set by clients # in the query string, for security reasons. next result if param == :node next result if param == :ip value = CGI.unescape(value) if value =~ /^---/ value = YAML.load(value) else value = true if value == "true" value = false if value == "false" value = Integer(value) if value =~ /^\d+$/ value = value.to_f if value =~ /^\d+\.\d+$/ end result[param] = value result end end end diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 4301e8c0f..fdabd05c9 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -1,493 +1,489 @@ # Created by Luke A. Kanies on 2007-08-13. # Copyright (c) 2007. All rights reserved. require 'puppet/node' require 'puppet/resource/catalog' require 'puppet/util/errors' -require 'puppet/util/instrumentation' require 'puppet/resource/type_collection_helper' # Maintain a graph of scopes, along with a bunch of data # about the individual catalog we're compiling. class Puppet::Parser::Compiler include Puppet::Util include Puppet::Util::Errors include Puppet::Resource::TypeCollectionHelper - extend Puppet::Util::Instrumentation def self.compile(node) - instrument("compiling #{node.name}") do - new(node).compile.to_resource - end + new(node).compile.to_resource rescue => detail puts detail.backtrace if Puppet[:trace] raise Puppet::Error, "#{detail} on node #{node.name}" ensure # We get these from the environment and only cache them in a thread # variable for the duration of the compilation. Thread.current[:known_resource_types] = nil Thread.current[:env_module_directories] = nil end attr_reader :node, :facts, :collections, :catalog, :node_scope, :resources, :relationships # Add a collection to the global list. def add_collection(coll) @collections << coll end def add_relationship(dep) @relationships << dep end # Store a resource override. def add_override(override) # If possible, merge the override in immediately. if resource = @catalog.resource(override.ref) resource.merge(override) else # Otherwise, store the override for later; these # get evaluated in Resource#finish. @resource_overrides[override.ref] << override end end # Store a resource in our resource table. def add_resource(scope, resource) @resources << resource # Note that this will fail if the resource is not unique. @catalog.add_resource(resource) # Add our container edge. If we're a class, then we get treated specially - we can # control the stage that the class is applied in. Otherwise, we just # get added to our parent container. return if resource.type.to_s.downcase == "stage" if resource.type.to_s.downcase != "class" raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage" if resource[:stage] return @catalog.add_edge(scope.resource, resource) end unless stage = @catalog.resource(:stage, resource[:stage] || (scope && scope.resource && scope.resource[:stage]) || :main) raise ArgumentError, "Could not find stage #{resource[:stage] || :main} specified by #{resource}" end resource[:stage] ||= stage.title unless stage.title == :main @catalog.add_edge(stage, resource) end # Do we use nodes found in the code, vs. the external node sources? def ast_nodes? known_resource_types.nodes? end # Store the fact that we've evaluated a class def add_class(name) @catalog.add_class(name) unless name == "" end # Return a list of all of the defined classes. def classlist @catalog.classes end # Compiler our catalog. This mostly revolves around finding and evaluating classes. # This is the main entry into our catalog. def compile # Set the client's parameters into the top scope. set_node_parameters create_settings_scope evaluate_main evaluate_ast_node evaluate_node_classes evaluate_generators finish fail_on_unevaluated @catalog end # LAK:FIXME There are no tests for this. def delete_collection(coll) @collections.delete(coll) if @collections.include?(coll) end # Return the node's environment. def environment unless defined?(@environment) @environment = (node.environment and node.environment != "") ? node.environment : nil end Puppet::Node::Environment.current = @environment @environment end # Evaluate all of the classes specified by the node. def evaluate_node_classes evaluate_classes(@node.classes, topscope) end # Evaluate each specified class in turn. If there are any classes we can't # find, just tag the catalog and move on. This method really just # creates resource objects that point back to the classes, and then the # resources are themselves evaluated later in the process. def evaluate_classes(classes, scope, lazy_evaluate = true) raise Puppet::DevError, "No source for scope passed to evaluate_classes" unless scope.source found = [] param_classes = nil # if we are a param class, save the classes hash # and transform classes to be the keys if classes.class == Hash param_classes = classes classes = classes.keys end classes.each do |name| # If we can find the class, then make a resource that will evaluate it. if klass = scope.find_hostclass(name) if param_classes resource = klass.ensure_in_catalog(scope, param_classes[name] || {}) else found << name and next if scope.class_scope(klass) resource = klass.ensure_in_catalog(scope) end # If they've disabled lazy evaluation (which the :include function does), # then evaluate our resource immediately. resource.evaluate unless lazy_evaluate found << name else Puppet.info "Could not find class #{name} for #{node.name}" @catalog.tag(name) end end found end def evaluate_relationships @relationships.each { |rel| rel.evaluate(catalog) } end # Return a resource by either its ref or its type and title. def findresource(*args) @catalog.resource(*args) end def initialize(node, options = {}) @node = node options.each do |param, value| begin send(param.to_s + "=", value) rescue NoMethodError raise ArgumentError, "Compiler objects do not accept #{param}" end end initvars end # Create a new scope, with either a specified parent scope or # using the top scope. def newscope(parent, options = {}) parent ||= topscope options[:compiler] = self scope = Puppet::Parser::Scope.new(options) scope.parent = parent scope end # Return any overrides for the given resource. def resource_overrides(resource) @resource_overrides[resource.ref] end # The top scope is usually the top-level scope, but if we're using AST nodes, # then it is instead the node's scope. def topscope node_scope || @topscope end private # If ast nodes are enabled, then see if we can find and evaluate one. def evaluate_ast_node return unless ast_nodes? # Now see if we can find the node. astnode = nil @node.names.each do |name| break if astnode = known_resource_types.node(name.to_s.downcase) end unless (astnode ||= known_resource_types.node("default")) raise Puppet::ParseError, "Could not find default node or by name with '#{node.names.join(", ")}'" end # Create a resource to model this node, and then add it to the list # of resources. resource = astnode.ensure_in_catalog(topscope) resource.evaluate # Now set the node scope appropriately, so that :topscope can # behave differently. @node_scope = topscope.class_scope(astnode) end # Evaluate our collections and return true if anything returned an object. # The 'true' is used to continue a loop, so it's important. def evaluate_collections return false if @collections.empty? found_something = false exceptwrap do # We have to iterate over a dup of the array because # collections can delete themselves from the list, which # changes its length and causes some collections to get missed. @collections.dup.each do |collection| found_something = true if collection.evaluate end end found_something end # Make sure all of our resources have been evaluated into native resources. # We return true if any resources have, so that we know to continue the # evaluate_generators loop. def evaluate_definitions exceptwrap do !unevaluated_resources.each { |resource| resource.evaluate }.empty? end end # Iterate over collections and resources until we're sure that the whole # compile is evaluated. This is necessary because both collections # and defined resources can generate new resources, which themselves could # be defined resources. def evaluate_generators count = 0 loop do done = true # Call collections first, then definitions. done = false if evaluate_collections done = false if evaluate_definitions break if done count += 1 if count > 1000 raise Puppet::ParseError, "Somehow looped more than 1000 times while evaluating host catalog" end end end # Find and evaluate our main object, if possible. def evaluate_main @main = known_resource_types.find_hostclass([""], "") || known_resource_types.add(Puppet::Resource::Type.new(:hostclass, "")) @topscope.source = @main @main_resource = Puppet::Parser::Resource.new("class", :main, :scope => @topscope, :source => @main) @topscope.resource = @main_resource add_resource(@topscope, @main_resource) @main_resource.evaluate end # Make sure the entire catalog is evaluated. def fail_on_unevaluated fail_on_unevaluated_overrides fail_on_unevaluated_resource_collections end # If there are any resource overrides remaining, then we could # not find the resource they were supposed to override, so we # want to throw an exception. def fail_on_unevaluated_overrides remaining = [] @resource_overrides.each do |name, overrides| remaining += overrides end unless remaining.empty? fail Puppet::ParseError, "Could not find resource(s) %s for overriding" % remaining.collect { |o| o.ref }.join(", ") end end # Make sure we don't have any remaining collections that specifically # look for resources, because we want to consider those to be # parse errors. def fail_on_unevaluated_resource_collections remaining = [] @collections.each do |coll| # We're only interested in the 'resource' collections, # which result from direct calls of 'realize'. Anything # else is allowed not to return resources. # Collect all of them, so we have a useful error. if r = coll.resources if r.is_a?(Array) remaining += r else remaining << r end end end raise Puppet::ParseError, "Failed to realize virtual resources #{remaining.join(', ')}" unless remaining.empty? end # Make sure all of our resources and such have done any last work # necessary. def finish evaluate_relationships resources.each do |resource| # Add in any resource overrides. if overrides = resource_overrides(resource) overrides.each do |over| resource.merge(over) end # Remove the overrides, so that the configuration knows there # are none left. overrides.clear end resource.finish if resource.respond_to?(:finish) end add_resource_metaparams end def add_resource_metaparams unless main = catalog.resource(:class, :main) raise "Couldn't find main" end names = [] Puppet::Type.eachmetaparam do |name| next if Puppet::Parser::Resource.relationship_parameter?(name) names << name end data = {} catalog.walk(main, :out) do |source, target| if source_data = data[source] || metaparams_as_data(source, names) # only store anything in the data hash if we've actually got # data data[source] ||= source_data source_data.each do |param, value| target[param] = value if target[param].nil? end data[target] = source_data.merge(metaparams_as_data(target, names)) end target.tag(*(source.tags)) end end def metaparams_as_data(resource, params) data = nil params.each do |param| unless resource[param].nil? # Because we could be creating a hash for every resource, # and we actually probably don't often have any data here at all, # we're optimizing a bit by only creating a hash if there's # any data to put in it. data ||= {} data[param] = resource[param] end end data end # Set up all of our internal variables. def initvars # The list of objects that will available for export. @exported_resources = {} # The list of overrides. This is used to cache overrides on objects # that don't exist yet. We store an array of each override. @resource_overrides = Hash.new do |overs, ref| overs[ref] = [] end # The list of collections that have been created. This is a global list, # but they each refer back to the scope that created them. @collections = [] # The list of relationships to evaluate. @relationships = [] # For maintaining the relationship between scopes and their resources. @catalog = Puppet::Resource::Catalog.new(@node.name) @catalog.version = known_resource_types.version # Create our initial scope and a resource that will evaluate main. @topscope = Puppet::Parser::Scope.new(:compiler => self) @main_stage_resource = Puppet::Parser::Resource.new("stage", :main, :scope => @topscope) @catalog.add_resource(@main_stage_resource) # local resource array to maintain resource ordering @resources = [] # Make sure any external node classes are in our class list if @node.classes.class == Hash @catalog.add_class(*@node.classes.keys) else @catalog.add_class(*@node.classes) end end # Set the node's parameters into the top-scope as variables. def set_node_parameters node.parameters.each do |param, value| @topscope.setvar(param, value) end # These might be nil. catalog.client_version = node.parameters["clientversion"] catalog.server_version = node.parameters["serverversion"] end def create_settings_scope unless settings_type = environment.known_resource_types.hostclass("settings") settings_type = Puppet::Resource::Type.new :hostclass, "settings" environment.known_resource_types.add(settings_type) end settings_resource = Puppet::Parser::Resource.new("class", "settings", :scope => @topscope) settings_type.evaluate_code(settings_resource) @catalog.add_resource(settings_resource) scope = @topscope.class_scope(settings_type) Puppet.settings.each do |name, setting| next if name.to_s == "name" scope.setvar name.to_s, environment[name] end end # Return an array of all of the unevaluated resources. These will be definitions, # which need to get evaluated into native resources. def unevaluated_resources # The order of these is significant for speed due to short-circuting resources.reject { |resource| resource.evaluated? or resource.virtual? or resource.builtin_type? } end end diff --git a/lib/puppet/transaction.rb b/lib/puppet/transaction.rb index df4c8b27d..eba601cfe 100644 --- a/lib/puppet/transaction.rb +++ b/lib/puppet/transaction.rb @@ -1,339 +1,334 @@ # the class that actually walks our resource/property tree, collects the changes, # and performs them require 'puppet' require 'puppet/util/tagging' require 'puppet/application' -require 'puppet/util/instrumentation' class Puppet::Transaction - include Puppet::Util::Instrumentation - require 'puppet/transaction/event' require 'puppet/transaction/event_manager' require 'puppet/transaction/resource_harness' require 'puppet/resource/status' attr_accessor :component, :catalog, :ignoreschedules attr_accessor :sorted_resources, :configurator # The report, once generated. attr_accessor :report # Routes and stores any events and subscriptions. attr_reader :event_manager # Handles most of the actual interacting with resources attr_reader :resource_harness include Puppet::Util include Puppet::Util::Tagging # Wraps application run state check to flag need to interrupt processing def stop_processing? Puppet::Application.stop_requested? end # Add some additional times for reporting def add_times(hash) hash.each do |name, num| report.add_times(name, num) end end # Are there any failed resources in this transaction? def any_failed? report.resource_statuses.values.detect { |status| status.failed? } end # Apply all changes for a resource def apply(resource, ancestor = nil) status = resource_harness.evaluate(resource) add_resource_status(status) event_manager.queue_events(ancestor || resource, status.events) rescue => detail resource.err "Could not evaluate: #{detail}" end # Find all of the changed resources. def changed? report.resource_statuses.values.find_all { |status| status.changed }.collect { |status| catalog.resource(status.resource) } end # Copy an important relationships from the parent to the newly-generated # child resource. def make_parent_child_relationship(resource, children) depthfirst = resource.depthfirst? children.each do |gen_child| if depthfirst edge = [gen_child, resource] else edge = [resource, gen_child] end relationship_graph.add_vertex(gen_child) unless relationship_graph.edge?(edge[1], edge[0]) relationship_graph.add_edge(*edge) else resource.debug "Skipping automatic relationship to #{gen_child}" end end end # See if the resource generates new resources at evaluation time. def eval_generate(resource) generate_additional_resources(resource, :eval_generate) end # Evaluate a single resource. def eval_resource(resource, ancestor = nil) if skip?(resource) resource_status(resource).skipped = true else eval_children_and_apply_resource(resource, ancestor) end # Check to see if there are any events queued for this resource event_manager.process_events(resource) end def eval_children_and_apply_resource(resource, ancestor = nil) resource_status(resource).scheduled = true # We need to generate first regardless, because the recursive # actions sometimes change how the top resource is applied. children = eval_generate(resource) if ! children.empty? and resource.depthfirst? children.each do |child| # The child will never be skipped when the parent isn't eval_resource(child, ancestor || resource) end end # Perform the actual changes apply(resource, ancestor) if ! children.empty? and ! resource.depthfirst? children.each do |child| eval_resource(child, ancestor || resource) end end end # This method does all the actual work of running a transaction. It # collects all of the changes, executes them, and responds to any # necessary events. def evaluate # Start logging. Puppet::Util::Log.newdestination(@report) prepare Puppet.info "Applying configuration version '#{catalog.version}'" if catalog.version begin @sorted_resources.each do |resource| next if stop_processing? if resource.is_a?(Puppet::Type::Component) Puppet.warning "Somehow left a component in the relationship graph" next end ret = nil - instrument("evaluating #{resource}") do - seconds = thinmark do - ret = eval_resource(resource) - end + seconds = thinmark do + ret = eval_resource(resource) end resource.info "Evaluated in %0.2f seconds" % seconds if Puppet[:evaltrace] and @catalog.host_config? ret end ensure # And then close the transaction log. Puppet::Util::Log.close(@report) end Puppet.debug "Finishing transaction #{object_id}" end def events event_manager.events end def failed?(resource) s = resource_status(resource) and s.failed? end # Does this resource have any failed dependencies? def failed_dependencies?(resource) # First make sure there are no failed dependencies. To do this, # we check for failures in any of the vertexes above us. It's not # enough to check the immediate dependencies, which is why we use # a tree from the reversed graph. found_failed = false relationship_graph.dependencies(resource).each do |dep| next unless failed?(dep) resource.notice "Dependency #{dep} has failures: #{resource_status(dep).failed}" found_failed = true end found_failed end # A general method for recursively generating new resources from a # resource. def generate_additional_resources(resource, method) return [] unless resource.respond_to?(method) begin made = resource.send(method) rescue => detail puts detail.backtrace if Puppet[:trace] resource.err "Failed to generate additional resources using '#{method}': #{detail}" end return [] unless made made = [made] unless made.is_a?(Array) made.uniq.find_all do |res| begin res.tag(*resource.tags) @catalog.add_resource(res) do |r| r.finish make_parent_child_relationship(resource, [r]) # Call 'generate' recursively generate_additional_resources(r, method) end true rescue Puppet::Resource::Catalog::DuplicateResourceError res.info "Duplicate generated resource; skipping" false end end end # Collect any dynamically generated resources. This method is called # before the transaction starts. def generate list = @catalog.vertices newlist = [] while ! list.empty? list.each do |resource| newlist += generate_additional_resources(resource, :generate) end list = newlist newlist = [] end end # Should we ignore tags? def ignore_tags? ! (@catalog.host_config? or Puppet[:name] == "puppet") end # this should only be called by a Puppet::Type::Component resource now # and it should only receive an array def initialize(catalog) @catalog = catalog @report = Puppet::Transaction::Report.new("apply") @event_manager = Puppet::Transaction::EventManager.new(self) @resource_harness = Puppet::Transaction::ResourceHarness.new(self) end # Prefetch any providers that support it. We don't support prefetching # types, just providers. def prefetch prefetchers = {} @catalog.vertices.each do |resource| if provider = resource.provider and provider.class.respond_to?(:prefetch) prefetchers[provider.class] ||= {} prefetchers[provider.class][resource.name] = resource end end # Now call prefetch, passing in the resources so that the provider instances can be replaced. prefetchers.each do |provider, resources| Puppet.debug "Prefetching #{provider.name} resources for #{provider.resource_type.name}" begin provider.prefetch(resources) rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "Could not prefetch #{provider.resource_type.name} provider '#{provider.name}': #{detail}" end end end # Prepare to evaluate the resources in a transaction. def prepare # Now add any dynamically generated resources generate # Then prefetch. It's important that we generate and then prefetch, # so that any generated resources also get prefetched. prefetch # This will throw an error if there are cycles in the graph. @sorted_resources = relationship_graph.topsort end def relationship_graph catalog.relationship_graph end def add_resource_status(status) report.add_resource_status status end def resource_status(resource) report.resource_statuses[resource.to_s] || add_resource_status(Puppet::Resource::Status.new(resource)) end # Is the resource currently scheduled? def scheduled?(resource) self.ignoreschedules or resource_harness.scheduled?(resource_status(resource), resource) end # Should this resource be skipped? def skip?(resource) if missing_tags?(resource) resource.debug "Not tagged with #{tags.join(", ")}" elsif ! scheduled?(resource) resource.debug "Not scheduled" elsif failed_dependencies?(resource) resource.warning "Skipping because of failed dependencies" elsif resource.virtual? resource.debug "Skipping because virtual" else return false end true end # The tags we should be checking. def tags self.tags = Puppet[:tags] unless defined?(@tags) super end def handle_qualified_tags( qualified ) # The default behavior of Puppet::Util::Tagging is # to split qualified tags into parts. That would cause # qualified tags to match too broadly here. return end # Is this resource tagged appropriately? def missing_tags?(resource) return false if ignore_tags? return false if tags.empty? not resource.tagged?(*tags) end end require 'puppet/transaction/report' diff --git a/lib/puppet/util/instrumentation.rb b/lib/puppet/util/instrumentation.rb deleted file mode 100644 index 5981bea59..000000000 --- a/lib/puppet/util/instrumentation.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'puppet/util/instrumentation/process_name' - -module Puppet::Util::Instrumentation - - def instrument(title) - Puppet::Util::Instrumentation::ProcessName.instrument(title) do - yield - end - end - module_function :instrument - -end \ No newline at end of file diff --git a/lib/puppet/util/instrumentation/process_name.rb b/lib/puppet/util/instrumentation/process_name.rb deleted file mode 100644 index 370d29e2e..000000000 --- a/lib/puppet/util/instrumentation/process_name.rb +++ /dev/null @@ -1,129 +0,0 @@ -require 'puppet' -require 'puppet/util/instrumentation' - -module Puppet::Util::Instrumentation - class ProcessName - - # start scrolling when process name is longer than - SCROLL_LENGTH = 50 - - @active = false - class << self - attr_accessor :active, :reason - end - - trap(:QUIT) do - active? ? disable : enable - end - - def self.active? - !! @active - end - - def self.enable - mutex.synchronize do - Puppet.info("Process Name instrumentation is enabled") - @active = true - @x = 0 - setproctitle - end - end - - def self.disable - mutex.synchronize do - Puppet.info("Process Name instrumentation is disabled") - @active = false - $0 = @oldname - end - end - - def self.instrument(activity) - # inconditionnally start the scroller thread here - # because it doesn't seem possible to start a new thrad - # from the USR2 signal handler - @scroller ||= Thread.new do - loop do - scroll if active? - sleep 1 - end - end - - push_activity(Thread.current, activity) - yield - ensure - pop_activity(Thread.current) - end - - def self.setproctitle - @oldname ||= $0 - $0 = "#{base}: " + rotate(process_name,@x) if active? - end - - def self.push_activity(thread, activity) - mutex.synchronize do - @reason ||= {} - @reason[thread] ||= [] - @reason[thread].push(activity) - setproctitle - end - end - - def self.pop_activity(thread) - mutex.synchronize do - @reason[thread].pop - if @reason[thread].empty? - @reason.delete(thread) - end - setproctitle - end - end - - def self.process_name - out = (@reason || {}).inject([]) do |out, reason| - out << "#{thread_id(reason[0])} #{reason[1].join(',')}" - end - out.join(' | ') - end - - # certainly non-portable - def self.thread_id(thread) - thread.inspect.gsub(/^#<.*:0x([a-f0-9]+) .*>$/, '\1') - end - - def self.rotate(string, steps) - steps ||= 0 - if string.length > 0 && steps > 0 - steps = steps % string.length - return string[steps..string.length].concat " -- #{string[0..(steps-1)]}" - end - string - end - - def self.base - basename = case Puppet.run_mode.name - when :master - "master" - when :agent - "agent" - else - "puppet" - end - end - - def self.mutex - #Thread.exclusive { - @mutex ||= Sync.new - #} - @mutex - end - - def self.scroll - return if process_name.length < SCROLL_LENGTH - mutex.synchronize do - setproctitle - @x += 1 - end - end - - end -end \ No newline at end of file diff --git a/spec/unit/util/instrumentation/process_name_spec.rb b/spec/unit/util/instrumentation/process_name_spec.rb deleted file mode 100644 index 9cbedf2d2..000000000 --- a/spec/unit/util/instrumentation/process_name_spec.rb +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env ruby - -require File.dirname(__FILE__) + '/../../../spec_helper' - -describe Puppet::Util::Instrumentation::ProcessName do - - ProcessName = Puppet::Util::Instrumentation::ProcessName - - after(:each) do - ProcessName.reason = {} - end - - it "should be disabled by default" do - ProcessName.should_not be_active - end - - describe "when managing thread activity" do - before(:each) do - ProcessName.stubs(:setproctitle) - ProcessName.stubs(:base).returns("base") - end - - it "should be able to append activity" do - thread1 = stub 'thread1' - ProcessName.push_activity(:thread1,"activity1") - ProcessName.push_activity(:thread1,"activity2") - - ProcessName.reason[:thread1].should == ["activity1", "activity2"] - end - - it "should be able to remove activity" do - ProcessName.push_activity(:thread1,"activity1") - ProcessName.push_activity(:thread1,"activity1") - ProcessName.pop_activity(:thread1) - - ProcessName.reason[:thread1].should == ["activity1"] - end - - it "should maintain activity thread by thread" do - ProcessName.push_activity(:thread1,"activity1") - ProcessName.push_activity(:thread2,"activity2") - - ProcessName.reason[:thread1].should == ["activity1"] - ProcessName.reason[:thread2].should == ["activity2"] - end - - it "should set process title" do - ProcessName.expects(:setproctitle) - - ProcessName.push_activity("thread1","activity1") - end - end - - describe "when computing the current process name" do - before(:each) do - ProcessName.stubs(:setproctitle) - ProcessName.stubs(:base).returns("base") - end - - it "should include every running thread activity" do - thread1 = stub 'thread1', :inspect => "\#", :hash => 1 - thread2 = stub 'thread2', :inspect => "\#", :hash => 0 - - ProcessName.push_activity(thread1,"Compiling node1.domain.com") - ProcessName.push_activity(thread2,"Compiling node4.domain.com") - ProcessName.push_activity(thread1,"Parsing file site.pp") - ProcessName.push_activity(thread2,"Parsing file node.pp") - - ProcessName.process_name.should == "12344321 Compiling node4.domain.com,Parsing file node.pp | deadbeef Compiling node1.domain.com,Parsing file site.pp" - end - end - - describe "when finding base process name" do - {:master => "master", :agent => "agent", :user => "puppet"}.each do |program,base| - it "should return #{base} for #{program}" do - Puppet.run_mode.stubs(:name).returns(program) - ProcessName.base.should == base - end - end - end - - describe "when finding a thread id" do - it "should return the id from the thread inspect string" do - thread = stub 'thread', :inspect => "\#" - ProcessName.thread_id(thread).should == "1234abdc" - end - end - - describe "when scrolling the instrumentation string" do - it "should rotate the string of various step" do - ProcessName.rotate("this is a rotation", 10).should == "rotation -- this is a " - end - - it "should not rotate the string for the 0 offset" do - ProcessName.rotate("this is a rotation", 0).should == "this is a rotation" - end - end - - describe "when setting process name" do - before(:each) do - ProcessName.stubs(:process_name).returns("12345 activity") - ProcessName.stubs(:base).returns("base") - @oldname = $0 - end - - after(:each) do - $0 = @oldname - end - - it "should not do it if the feature is disabled" do - ProcessName.setproctitle - - $0.should_not == "base: 12345 activity" - end - - it "should do it if the feature is enabled" do - ProcessName.active = true - ProcessName.setproctitle - - $0.should == "base: 12345 activity" - end - end - - describe "when setting a probe" do - before(:each) do - thread = stub 'thread', :inspect => "\#" - Thread.stubs(:current).returns(thread) - Thread.stubs(:new) - ProcessName.active = true - end - - it "should start the scroller thread" do - Thread.expects(:new) - ProcessName.instrument("doing something") do - end - end - - it "should push current thread activity and execute the block" do - ProcessName.instrument("doing something") do - $0.should == "puppet: 1234abdc doing something" - end - end - - it "should finally pop the activity" do - ProcessName.instrument("doing something") do - end - $0.should == "puppet: " - end - end - - describe "when enabling" do - before do - Thread.stubs(:new) - ProcessName.stubs(:setproctitle) - end - - it "should be active" do - ProcessName.enable - ProcessName.should be_active - end - - it "should set the new process name" do - ProcessName.expects(:setproctitle) - ProcessName.enable - end - end - - describe "when disabling" do - it "should set active to false" do - ProcessName.active = true - ProcessName.disable - ProcessName.should_not be_active - end - - it "should restore the old process name" do - oldname = $0 - ProcessName.active = true - ProcessName.setproctitle - ProcessName.disable - $0.should == oldname - end - end - - describe "when scrolling" do - it "should do nothing for shorter process names" do - ProcessName.expects(:setproctitle).never - ProcessName.scroll - end - - it "should call setproctitle" do - ProcessName.stubs(:process_name).returns("x" * 60) - ProcessName.expects(:setproctitle) - ProcessName.scroll - end - - it "should increment rotation offset" do - name = "x" * 60 - ProcessName.active = true - ProcessName.stubs(:process_name).returns(name) - ProcessName.expects(:rotate).once.with(name,1).returns("") - ProcessName.expects(:rotate).once.with(name,2).returns("") - ProcessName.scroll - ProcessName.scroll - end - end - -end \ No newline at end of file