diff --git a/lib/puppet/application/apply.rb b/lib/puppet/application/apply.rb index cb5ba744a..b4a81cafe 100644 --- a/lib/puppet/application/apply.rb +++ b/lib/puppet/application/apply.rb @@ -1,290 +1,297 @@ require 'puppet/application' require 'puppet/configurer' +require 'puppet/util/profiler/aggregate' class Puppet::Application::Apply < Puppet::Application option("--debug","-d") option("--execute EXECUTE","-e") do |arg| options[:code] = arg end option("--loadclasses","-L") option("--test","-t") option("--verbose","-v") option("--use-nodes") option("--detailed-exitcodes") option("--write-catalog-summary") option("--catalog catalog", "-c catalog") do |arg| options[:catalog] = arg end option("--logdest LOGDEST", "-l") do |arg| handle_logdest_arg(arg) end option("--parseonly") do |args| puts "--parseonly has been removed. Please use 'puppet parser validate '" exit 1 end def help <<-'HELP' puppet-apply(8) -- Apply Puppet manifests locally ======== SYNOPSIS -------- Applies a standalone Puppet manifest to the local system. USAGE ----- puppet apply [-h|--help] [-V|--version] [-d|--debug] [-v|--verbose] [-e|--execute] [--detailed-exitcodes] [-L|--loadclasses] [-l|--logdest ] [--noop] [--catalog ] [--write-catalog-summary] DESCRIPTION ----------- This is the standalone puppet execution tool; use it to apply individual manifests. When provided with a modulepath, via command line or config file, puppet apply can effectively mimic the catalog that would be served by puppet master with access to the same modules, although there are some subtle differences. When combined with scheduling and an automated system for pushing manifests, this can be used to implement a serverless Puppet site. Most users should use 'puppet agent' and 'puppet master' for site-wide manifests. OPTIONS ------- Note that any setting that's valid in the configuration file is also a valid long argument. For example, 'tags' is a valid setting, so you can specify '--tags ,' as an argument. See the configuration file documentation at http://docs.puppetlabs.com/references/stable/configuration.html for the full list of acceptable parameters. A commented list of all configuration options can also be generated by running puppet with '--genconfig'. * --debug: Enable full debugging. * --detailed-exitcodes: Provide transaction information via exit codes. If this is enabled, an exit code of '2' means there were changes, an exit code of '4' means there were failures during the transaction, and an exit code of '6' means there were both changes and failures. * --help: Print this help message * --loadclasses: Load any stored classes. 'puppet agent' caches configured classes (usually at /etc/puppet/classes.txt), and setting this option causes all of those classes to be set in your puppet manifest. * --logdest: Where to send messages. Choose between syslog, the console, and a log file. Defaults to sending messages to the console. * --noop: Use 'noop' mode where Puppet runs in a no-op or dry-run mode. This is useful for seeing what changes Puppet will make without actually executing the changes. * --execute: Execute a specific piece of Puppet code * --test: Enable the most common options used for testing. These are 'verbose', 'detailed-exitcodes' and 'show_diff'. * --verbose: Print extra information. * --catalog: Apply a JSON catalog (such as one generated with 'puppet master --compile'). You can either specify a JSON file or pipe in JSON from standard input. * --write-catalog-summary After compiling the catalog saves the resource list and classes list to the node in the state directory named classes.txt and resources.txt EXAMPLE ------- $ puppet apply -l /tmp/manifest.log manifest.pp $ puppet apply --modulepath=/root/dev/modules -e "include ntpd::server" $ puppet apply --catalog catalog.json AUTHOR ------ Luke Kanies COPYRIGHT --------- Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License HELP end def app_defaults super.merge({ :default_file_terminus => :file_server, }) end def run_command if options[:catalog] apply else main end end def apply if options[:catalog] == "-" text = $stdin.read else text = ::File.read(options[:catalog]) end catalog = read_catalog(text) apply_catalog(catalog) end def main # Set our code or file to use. if options[:code] or command_line.args.length == 0 Puppet[:code] = options[:code] || STDIN.read else manifest = command_line.args.shift raise "Could not find file #{manifest}" unless Puppet::FileSystem.exist?(manifest) Puppet.warning("Only one file can be applied per run. Skipping #{command_line.args.join(', ')}") if command_line.args.size > 0 end unless Puppet[:node_name_fact].empty? # Collect our facts. unless facts = Puppet::Node::Facts.indirection.find(Puppet[:node_name_value]) raise "Could not find facts for #{Puppet[:node_name_value]}" end Puppet[:node_name_value] = facts.values[Puppet[:node_name_fact]] facts.name = Puppet[:node_name_value] end configured_environment = Puppet.lookup(:current_environment) apply_environment = manifest ? configured_environment.override_with(:manifest => manifest) : configured_environment Puppet.override(:current_environment => apply_environment) do # Find our Node unless node = Puppet::Node.indirection.find(Puppet[:node_name_value]) raise "Could not find node #{Puppet[:node_name_value]}" end # Merge in the facts. node.merge(facts.values) if facts # Allow users to load the classes that puppet agent creates. if options[:loadclasses] file = Puppet[:classfile] if Puppet::FileSystem.exist?(file) unless FileTest.readable?(file) $stderr.puts "#{file} is not readable" exit(63) end node.classes = ::File.read(file).split(/[\s\n]+/) end end begin # Compile our catalog starttime = Time.now catalog = Puppet::Resource::Catalog.indirection.find(node.name, :use_node => node) # Translate it to a RAL catalog catalog = catalog.to_ral catalog.finalize catalog.retrieval_duration = Time.now - starttime if options[:write_catalog_summary] catalog.write_class_file catalog.write_resource_file end exit_status = apply_catalog(catalog) if not exit_status exit(1) elsif options[:detailed_exitcodes] then exit(exit_status) else exit(0) end rescue => detail Puppet.log_exception(detail) exit(1) end end + + ensure + if @profiler + Puppet::Util::Profiler.remove_profiler(@profiler) + @profiler.shutdown + end end # Enable all of the most common test options. def setup_test Puppet.settings.handlearg("--show_diff") options[:verbose] = true options[:detailed_exitcodes] = true end def setup setup_test if options[:test] exit(Puppet.settings.print_configs ? 0 : 1) if Puppet.settings.print_configs? Puppet::Util::Log.newdestination(:console) unless options[:setdest] Signal.trap(:INT) do $stderr.puts "Exiting" exit(1) end # we want the last report to be persisted locally Puppet::Transaction::Report.indirection.cache_class = :yaml set_log_level if Puppet[:profile] - Puppet::Util::Profiler.add_profiler(Puppet::Util::Profiler::WallClock.new(Puppet.method(:debug), "apply")) + @profiler = Puppet::Util::Profiler.add_profiler(Puppet::Util::Profiler::Aggregate.new(Puppet.method(:debug), "apply")) end end private def read_catalog(text) begin catalog = Puppet::Resource::Catalog.convert_from(Puppet::Resource::Catalog.default_format,text) catalog = Puppet::Resource::Catalog.pson_create(catalog) unless catalog.is_a?(Puppet::Resource::Catalog) rescue => detail raise Puppet::Error, "Could not deserialize catalog from pson: #{detail}", detail.backtrace end catalog.to_ral end def apply_catalog(catalog) configurer = Puppet::Configurer.new configurer.run(:catalog => catalog, :pluginsync => false) end end diff --git a/lib/puppet/indirector/catalog/compiler.rb b/lib/puppet/indirector/catalog/compiler.rb index 0804d1819..6f4e2f3e4 100644 --- a/lib/puppet/indirector/catalog/compiler.rb +++ b/lib/puppet/indirector/catalog/compiler.rb @@ -1,182 +1,182 @@ require 'puppet/node' require 'puppet/resource/catalog' require 'puppet/indirector/code' require 'puppet/util/profiler' require 'yaml' class Puppet::Resource::Catalog::Compiler < Puppet::Indirector::Code desc "Compiles catalogs on demand using Puppet's compiler." include Puppet::Util attr_accessor :code def extract_facts_from_request(request) return unless text_facts = request.options[:facts] unless format = request.options[:facts_format] raise ArgumentError, "Facts but no fact format provided for #{request.key}" end - Puppet::Util::Profiler.profile("Found facts") do + Puppet::Util::Profiler.profile("Found facts", [:compiler, :find_facts]) do # If the facts were encoded as yaml, then the param reconstitution system # in Network::HTTP::Handler will automagically deserialize the value. if text_facts.is_a?(Puppet::Node::Facts) facts = text_facts else # We unescape here because the corresponding code in Puppet::Configurer::FactHandler escapes facts = Puppet::Node::Facts.convert_from(format, CGI.unescape(text_facts)) end unless facts.name == request.key raise Puppet::Error, "Catalog for #{request.key.inspect} was requested with fact definition for the wrong node (#{facts.name.inspect})." end facts.add_timestamp options = { :environment => request.environment, :transaction_uuid => request.options[:transaction_uuid], } Puppet::Node::Facts.indirection.save(facts, nil, options) end end # Compile a node's catalog. def find(request) extract_facts_from_request(request) node = node_from_request(request) node.trusted_data = Puppet.lookup(:trusted_information) { Puppet::Context::TrustedInformation.local(node) }.to_h if catalog = compile(node) return catalog else # This shouldn't actually happen; we should either return # a config or raise an exception. return nil end end # filter-out a catalog to remove exported resources def filter(catalog) return catalog.filter { |r| r.virtual? } if catalog.respond_to?(:filter) catalog end def initialize - Puppet::Util::Profiler.profile("Setup server facts for compiling") do + Puppet::Util::Profiler.profile("Setup server facts for compiling", [:compiler, :init_server_facts]) do set_server_facts end end # Is our compiler part of a network, or are we just local? def networked? Puppet.run_mode.master? end private # Add any extra data necessary to the node. def add_node_data(node) # Merge in our server-side facts, so they can be used during compilation. node.merge(@server_facts) end # Compile the actual catalog. def compile(node) str = "Compiled catalog for #{node.name}" str += " in environment #{node.environment}" if node.environment config = nil benchmark(:notice, str) do - Puppet::Util::Profiler.profile(str) do + Puppet::Util::Profiler.profile(str, [:compiler, :compile, node.environment, node.name]) do begin config = Puppet::Parser::Compiler.compile(node) rescue Puppet::Error => detail Puppet.err(detail.to_s) if networked? raise end end end config end # Turn our host name into a node object. def find_node(name, environment, transaction_uuid) - Puppet::Util::Profiler.profile("Found node information") do + Puppet::Util::Profiler.profile("Found node information", [:compiler, :find_node]) do node = nil begin node = Puppet::Node.indirection.find(name, :environment => environment, :transaction_uuid => transaction_uuid) rescue => detail message = "Failed when searching for node #{name}: #{detail}" Puppet.log_exception(detail, message) raise Puppet::Error, message, detail.backtrace end # Add any external data to the node. if node add_node_data(node) end node end end # Extract the node from the request, or use the request # to find the node. def node_from_request(request) if node = request.options[:use_node] if request.remote? raise Puppet::Error, "Invalid option use_node for a remote request" else return node end end # We rely on our authorization system to determine whether the connected # node is allowed to compile the catalog's node referenced by key. # By default the REST authorization system makes sure only the connected node # can compile his catalog. # This allows for instance monitoring systems or puppet-load to check several # node's catalog with only one certificate and a modification to auth.conf # If no key is provided we can only compile the currently connected node. name = request.key || request.node if node = find_node(name, request.environment, request.options[:transaction_uuid]) return node end raise ArgumentError, "Could not find node '#{name}'; cannot compile" end # Initialize our server fact hash; we add these to each client, and they # won't change while we're running, so it's safe to cache the values. def set_server_facts @server_facts = {} # Add our server version to the fact list @server_facts["serverversion"] = Puppet.version.to_s # And then add the server name and IP {"servername" => "fqdn", "serverip" => "ipaddress" }.each do |var, fact| if value = Facter.value(fact) @server_facts[var] = value else Puppet.warning "Could not retrieve fact #{fact}" end end if @server_facts["servername"].nil? host = Facter.value(:hostname) if domain = Facter.value(:domain) @server_facts["servername"] = [host, domain].join(".") else @server_facts["servername"] = host end end end end diff --git a/lib/puppet/indirector/indirection.rb b/lib/puppet/indirector/indirection.rb index a22f465ac..26a33543c 100644 --- a/lib/puppet/indirector/indirection.rb +++ b/lib/puppet/indirector/indirection.rb @@ -1,336 +1,336 @@ require 'puppet/util/docs' require 'puppet/util/profiler' require 'puppet/util/methodhelper' require 'puppet/indirector/envelope' require 'puppet/indirector/request' require 'puppet/util/instrumentation/instrumentable' # The class that connects functional classes with their different collection # back-ends. Each indirection has a set of associated terminus classes, # each of which is a subclass of Puppet::Indirector::Terminus. class Puppet::Indirector::Indirection include Puppet::Util::MethodHelper include Puppet::Util::Docs extend Puppet::Util::Instrumentation::Instrumentable attr_accessor :name, :model attr_reader :termini probe :find, :label => Proc.new { |parent, key, *args| "find_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} probe :save, :label => Proc.new { |parent, key, *args| "save_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} probe :search, :label => Proc.new { |parent, key, *args| "search_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} probe :destroy, :label => Proc.new { |parent, key, *args| "destroy_#{parent.name}_#{parent.terminus_class}" }, :data => Proc.new { |parent, key, *args| { :key => key }} @@indirections = [] # Find an indirection by name. This is provided so that Terminus classes # can specifically hook up with the indirections they are associated with. def self.instance(name) @@indirections.find { |i| i.name == name } end # Return a list of all known indirections. Used to generate the # reference. def self.instances @@indirections.collect { |i| i.name } end # Find an indirected model by name. This is provided so that Terminus classes # can specifically hook up with the indirections they are associated with. def self.model(name) return nil unless match = @@indirections.find { |i| i.name == name } match.model end # Create and return our cache terminus. def cache raise(Puppet::DevError, "Tried to cache when no cache class was set") unless cache_class terminus(cache_class) end # Should we use a cache? def cache? cache_class ? true : false end attr_reader :cache_class # Define a terminus class to be used for caching. def cache_class=(class_name) validate_terminus_class(class_name) if class_name @cache_class = class_name end # This is only used for testing. def delete @@indirections.delete(self) if @@indirections.include?(self) end # Set the time-to-live for instances created through this indirection. def ttl=(value) raise ArgumentError, "Indirection TTL must be an integer" unless value.is_a?(Fixnum) @ttl = value end # Default to the runinterval for the ttl. def ttl @ttl ||= Puppet[:runinterval] end # Calculate the expiration date for a returned instance. def expiration Time.now + ttl end # Generate the full doc string. def doc text = "" text << scrub(@doc) << "\n\n" if @doc text << "* **Indirected Class**: `#{@indirected_class}`\n"; if terminus_setting text << "* **Terminus Setting**: #{terminus_setting}\n" end text end def initialize(model, name, options = {}) @model = model @name = name @termini = {} @cache_class = nil @terminus_class = nil raise(ArgumentError, "Indirection #{@name} is already defined") if @@indirections.find { |i| i.name == @name } @@indirections << self @indirected_class = options.delete(:indirected_class) if mod = options[:extend] extend(mod) options.delete(:extend) end # This is currently only used for cache_class and terminus_class. set_options(options) end # Set up our request object. def request(*args) Puppet::Indirector::Request.new(self.name, *args) end # Return the singleton terminus for this indirection. def terminus(terminus_name = nil) # Get the name of the terminus. raise Puppet::DevError, "No terminus specified for #{self.name}; cannot redirect" unless terminus_name ||= terminus_class termini[terminus_name] ||= make_terminus(terminus_name) end # This can be used to select the terminus class. attr_accessor :terminus_setting # Determine the terminus class. def terminus_class unless @terminus_class if setting = self.terminus_setting self.terminus_class = Puppet.settings[setting] else raise Puppet::DevError, "No terminus class nor terminus setting was provided for indirection #{self.name}" end end @terminus_class end def reset_terminus_class @terminus_class = nil end # Specify the terminus class to use. def terminus_class=(klass) validate_terminus_class(klass) @terminus_class = klass end # This is used by terminus_class= and cache=. def validate_terminus_class(terminus_class) raise ArgumentError, "Invalid terminus name #{terminus_class.inspect}" unless terminus_class and terminus_class.to_s != "" unless Puppet::Indirector::Terminus.terminus_class(self.name, terminus_class) raise ArgumentError, "Could not find terminus #{terminus_class} for indirection #{self.name}" end end # Expire a cached object, if one is cached. Note that we don't actually # remove it, we expire it and write it back out to disk. This way people # can still use the expired object if they want. def expire(key, options={}) request = request(:expire, key, nil, options) return nil unless cache? return nil unless instance = cache.find(request(:find, key, nil, options)) Puppet.info "Expiring the #{self.name} cache of #{instance.name}" # Set an expiration date in the past instance.expiration = Time.now - 60 cache.save(request(:save, nil, instance, options)) end def allow_remote_requests? terminus.allow_remote_requests? end # Search for an instance in the appropriate terminus, caching the # results if caching is configured.. def find(key, options={}) request = request(:find, key, nil, options) terminus = prepare(request) result = find_in_cache(request) if not result.nil? result elsif request.ignore_terminus? nil else # Otherwise, return the result from the terminus, caching if # appropriate. result = terminus.find(request) if not result.nil? result.expiration ||= self.expiration if result.respond_to?(:expiration) if cache? Puppet.info "Caching #{self.name} for #{request.key}" cache.save request(:save, key, result, options) end filtered = result if terminus.respond_to?(:filter) - Puppet::Util::Profiler.profile("Filtered result for #{self.name} #{request.key}") do + Puppet::Util::Profiler.profile("Filtered result for #{self.name} #{request.key}", [:indirector, :filter, self.name, request.key]) do filtered = terminus.filter(result) end end filtered end end end # Search for an instance in the appropriate terminus, and return a # boolean indicating whether the instance was found. def head(key, options={}) request = request(:head, key, nil, options) terminus = prepare(request) # Look in the cache first, then in the terminus. Force the result # to be a boolean. !!(find_in_cache(request) || terminus.head(request)) end def find_in_cache(request) # See if our instance is in the cache and up to date. return nil unless cache? and ! request.ignore_cache? and cached = cache.find(request) if cached.expired? Puppet.info "Not using expired #{self.name} for #{request.key} from cache; expired at #{cached.expiration}" return nil end Puppet.debug "Using cached #{self.name} for #{request.key}" cached rescue => detail Puppet.log_exception(detail, "Cached #{self.name} for #{request.key} failed: #{detail}") nil end # Remove something via the terminus. def destroy(key, options={}) request = request(:destroy, key, nil, options) terminus = prepare(request) result = terminus.destroy(request) if cache? and cache.find(request(:find, key, nil, options)) # Reuse the existing request, since it's equivalent. cache.destroy(request) end result end # Search for more than one instance. Should always return an array. def search(key, options={}) request = request(:search, key, nil, options) terminus = prepare(request) if result = terminus.search(request) raise Puppet::DevError, "Search results from terminus #{terminus.name} are not an array" unless result.is_a?(Array) result.each do |instance| next unless instance.respond_to? :expiration instance.expiration ||= self.expiration end return result end end # Save the instance in the appropriate terminus. This method is # normally an instance method on the indirected class. def save(instance, key = nil, options={}) request = request(:save, key, instance, options) terminus = prepare(request) result = terminus.save(request) # If caching is enabled, save our document there cache.save(request) if cache? result end private # Check authorization if there's a hook available; fail if there is one # and it returns false. def check_authorization(request, terminus) # At this point, we're assuming authorization makes no sense without # client information. return unless request.node # This is only to authorize via a terminus-specific authorization hook. return unless terminus.respond_to?(:authorized?) unless terminus.authorized?(request) msg = "Not authorized to call #{request.method} on #{request}" msg += " with #{request.options.inspect}" unless request.options.empty? raise ArgumentError, msg end end # Setup a request, pick the appropriate terminus, check the request's authorization, and return it. def prepare(request) # Pick our terminus. if respond_to?(:select_terminus) unless terminus_name = select_terminus(request) raise ArgumentError, "Could not determine appropriate terminus for #{request}" end else terminus_name = terminus_class end dest_terminus = terminus(terminus_name) check_authorization(request, dest_terminus) dest_terminus.validate(request) dest_terminus end # Create a new terminus instance. def make_terminus(terminus_class) # Load our terminus class. unless klass = Puppet::Indirector::Terminus.terminus_class(self.name, terminus_class) raise ArgumentError, "Could not find terminus #{terminus_class} for indirection #{self.name}" end klass.new end end diff --git a/lib/puppet/network/http/api/v1.rb b/lib/puppet/network/http/api/v1.rb index 54fa9af23..24b7d910f 100644 --- a/lib/puppet/network/http/api/v1.rb +++ b/lib/puppet/network/http/api/v1.rb @@ -1,222 +1,222 @@ require 'puppet/network/authorization' class Puppet::Network::HTTP::API::V1 include Puppet::Network::Authorization # How we map http methods and the indirection name in the URI # to an indirection method. METHOD_MAP = { "GET" => { :plural => :search, :singular => :find }, "POST" => { :singular => :find, }, "PUT" => { :singular => :save }, "DELETE" => { :singular => :destroy }, "HEAD" => { :singular => :head } } def self.routes Puppet::Network::HTTP::Route.path(/.*/).any(new) end # handle an HTTP request def call(request, response) indirection_name, method, key, params = uri2indirection(request.method, request.path, request.params) certificate = request.client_cert check_authorization(method, "/#{indirection_name}/#{key}", params) indirection = Puppet::Indirector::Indirection.instance(indirection_name.to_sym) raise ArgumentError, "Could not find indirection '#{indirection_name}'" unless indirection if !indirection.allow_remote_requests? # TODO: should we tell the user we found an indirection but it doesn't # allow remote requests, or just pretend there's no handler at all? what # are the security implications for the former? raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new("No handler for #{indirection.name}", :NO_INDIRECTION_REMOTE_REQUESTS) end trusted = Puppet::Context::TrustedInformation.remote(params[:authenticated], params[:node], certificate) Puppet.override(:trusted_information => trusted) do send("do_#{method}", indirection, key, params, request, response) end rescue Puppet::Network::HTTP::Error::HTTPError => e return do_http_control_exception(response, e) rescue Exception => e return do_exception(response, e) end def uri2indirection(http_method, uri, params) environment, indirection, key = uri.split("/", 4)[1..-1] # the first field is always nil because of the leading slash raise ArgumentError, "The environment must be purely alphanumeric, not '#{environment}'" unless Puppet::Node::Environment.valid_name?(environment) raise ArgumentError, "The indirection name must be purely alphanumeric, not '#{indirection}'" unless indirection =~ /^\w+$/ method = indirection_method(http_method, indirection) configured_environment = Puppet.lookup(:environments).get(environment) if configured_environment.nil? raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new("Could not find environment '#{environment}'", Puppet::Network::HTTP::Issues::ENVIRONMENT_NOT_FOUND) else configured_environment = configured_environment.override_from_commandline(Puppet.settings) params[:environment] = configured_environment end params.delete(:bucket_path) raise ArgumentError, "No request key specified in #{uri}" if key == "" or key.nil? key = URI.unescape(key) [indirection, method, key, params] end private def do_http_control_exception(response, exception) msg = exception.message Puppet.info(msg) response.respond_with(exception.status, "text/plain", msg) end def do_exception(response, exception, status=400) if exception.is_a?(Puppet::Network::AuthorizationError) # make sure we return the correct status code # for authorization issues status = 403 if status == 400 end Puppet.log_exception(exception) response.respond_with(status, "text/plain", exception.to_s) end # Execute our find. def do_find(indirection, key, params, request, response) unless result = indirection.find(key, params) raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new("Could not find #{indirection.name} #{key}", Puppet::Network::HTTP::Issues::RESOURCE_NOT_FOUND) end format = accepted_response_formatter_for(indirection.model, request) rendered_result = result if result.respond_to?(:render) - Puppet::Util::Profiler.profile("Rendered result in #{format}") do + Puppet::Util::Profiler.profile("Rendered result in #{format}", [:http, :v1_render, format]) do rendered_result = result.render(format) end end - Puppet::Util::Profiler.profile("Sent response") do + Puppet::Util::Profiler.profile("Sent response", [:http, :v1_response]) do response.respond_with(200, format, rendered_result) end end # Execute our head. def do_head(indirection, key, params, request, response) unless indirection.head(key, params) raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new("Could not find #{indirection.name} #{key}", Puppet::Network::HTTP::Issues::RESOURCE_NOT_FOUND) end # No need to set a response because no response is expected from a # HEAD request. All we need to do is not die. end # Execute our search. def do_search(indirection, key, params, request, response) result = indirection.search(key, params) if result.nil? raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new("Could not find instances in #{indirection.name} with '#{key}'", Puppet::Network::HTTP::Issues::RESOURCE_NOT_FOUND) end format = accepted_response_formatter_for(indirection.model, request) response.respond_with(200, format, indirection.model.render_multiple(format, result)) end # Execute our destroy. def do_destroy(indirection, key, params, request, response) formatter = accepted_response_formatter_or_yaml_for(indirection.model, request) result = indirection.destroy(key, params) response.respond_with(200, formatter, formatter.render(result)) end # Execute our save. def do_save(indirection, key, params, request, response) formatter = accepted_response_formatter_or_yaml_for(indirection.model, request) sent_object = read_body_into_model(indirection.model, request) result = indirection.save(sent_object, key) response.respond_with(200, formatter, formatter.render(result)) end def accepted_response_formatter_for(model_class, request) accepted_formats = request.headers['accept'] or raise Puppet::Network::HTTP::Error::HTTPNotAcceptableError.new("Missing required Accept header", Puppet::Network::HTTP::Issues::MISSING_HEADER_FIELD) request.response_formatter_for(model_class.supported_formats, accepted_formats) end def accepted_response_formatter_or_yaml_for(model_class, request) accepted_formats = request.headers['accept'] || "yaml" request.response_formatter_for(model_class.supported_formats, accepted_formats) end def read_body_into_model(model_class, request) data = request.body.to_s format = request.format model_class.convert_from(format, data) end def indirection_method(http_method, indirection) raise ArgumentError, "No support for http method #{http_method}" unless METHOD_MAP[http_method] unless method = METHOD_MAP[http_method][plurality(indirection)] raise ArgumentError, "No support for plurality #{plurality(indirection)} for #{http_method} operations" end method end def self.indirection2uri(request) indirection = request.method == :search ? pluralize(request.indirection_name.to_s) : request.indirection_name.to_s "/#{request.environment.to_s}/#{indirection}/#{request.escaped_key}#{request.query_string}" end def self.request_to_uri_and_body(request) indirection = request.method == :search ? pluralize(request.indirection_name.to_s) : request.indirection_name.to_s ["/#{request.environment.to_s}/#{indirection}/#{request.escaped_key}", request.query_string.sub(/^\?/,'')] end def self.pluralize(indirection) return(indirection == "status" ? "statuses" : indirection + "s") end def plurality(indirection) # NOTE This specific hook for facts is ridiculous, but it's a *many*-line # fix to not need this, and our goal is to move away from the complication # that leads to the fix being too long. return :singular if indirection == "facts" return :singular if indirection == "status" return :singular if indirection == "certificate_status" return :plural if indirection == "inventory" result = (indirection =~ /s$|_search$/) ? :plural : :singular indirection.sub!(/s$|_search$/, '') indirection.sub!(/statuse$/, 'status') result end end diff --git a/lib/puppet/network/http/handler.rb b/lib/puppet/network/http/handler.rb index f6148f752..d20d9776f 100644 --- a/lib/puppet/network/http/handler.rb +++ b/lib/puppet/network/http/handler.rb @@ -1,185 +1,187 @@ module Puppet::Network::HTTP end require 'puppet/network/http' require 'puppet/network/http/api/v1' require 'puppet/network/authentication' require 'puppet/network/rights' require 'puppet/util/profiler' +require 'puppet/util/profiler/aggregate' require 'resolv' module Puppet::Network::HTTP::Handler include Puppet::Network::Authentication include Puppet::Network::HTTP::Issues # These shouldn't be allowed to be set by clients # in the query string, for security reasons. DISALLOWED_KEYS = ["node", "ip"] def register(routes) # There's got to be a simpler way to do this, right? dupes = {} routes.each { |r| dupes[r.path_matcher] = (dupes[r.path_matcher] || 0) + 1 } dupes = dupes.collect { |pm, count| pm if count > 1 }.compact if dupes.count > 0 raise ArgumentError, "Given multiple routes with identical path regexes: #{dupes.map{ |rgx| rgx.inspect }.join(', ')}" end @routes = routes Puppet.debug("Routes Registered:") @routes.each do |route| Puppet.debug(route.inspect) end end # Retrieve all headers from the http request, as a hash with the header names # (lower-cased) as the keys def headers(request) raise NotImplementedError end def format_to_mime(format) format.is_a?(Puppet::Network::Format) ? format.mime : format end # handle an HTTP request def process(request, response) new_response = Puppet::Network::HTTP::Response.new(self, response) request_headers = headers(request) request_params = params(request) request_method = http_method(request) request_path = path(request) new_request = Puppet::Network::HTTP::Request.new(request_headers, request_params, request_method, request_path, request_path, client_cert(request), body(request)) response[Puppet::Network::HTTP::HEADER_PUPPET_VERSION] = Puppet.version profiler = configure_profiler(request_headers, request_params) warn_if_near_expiration(new_request.client_cert) - Puppet::Util::Profiler.profile("Processed request #{request_method} #{request_path}") do + Puppet::Util::Profiler.profile("Processed request #{request_method} #{request_path}", [:http, request_method, request_path]) do if route = @routes.find { |route| route.matches?(new_request) } route.process(new_request, new_response) else raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new("No route for #{new_request.method} #{new_request.path}", HANDLER_NOT_FOUND) end end rescue Puppet::Network::HTTP::Error::HTTPError => e Puppet.info(e.message) new_response.respond_with(e.status, "application/json", e.to_json) rescue Exception => e http_e = Puppet::Network::HTTP::Error::HTTPServerError.new(e) Puppet.err(http_e.message) new_response.respond_with(http_e.status, "application/json", http_e.to_json) ensure if profiler remove_profiler(profiler) end cleanup(request) end # Set the response up, with the body and status. def set_response(response, body, status = 200) raise NotImplementedError end # Set the specified format as the content type of the response. def set_content_type(response, format) raise NotImplementedError end # resolve node name from peer's ip address # this is used when the request is unauthenticated def resolve_node(result) begin return Resolv.getname(result[:ip]) rescue => detail Puppet.err "Could not resolve #{result[:ip]}: #{detail}" end result[:ip] end private # methods to be overridden by the including web server class def http_method(request) raise NotImplementedError end def path(request) raise NotImplementedError end def request_key(request) raise NotImplementedError end def body(request) raise NotImplementedError end def params(request) raise NotImplementedError end def client_cert(request) raise NotImplementedError end def cleanup(request) # By default, there is nothing to cleanup. end def decode_params(params) params.select { |key, _| allowed_parameter?(key) }.inject({}) do |result, ary| param, value = ary result[param.to_sym] = parse_parameter_value(param, value) result end end def allowed_parameter?(name) not (name.nil? || name.empty? || DISALLOWED_KEYS.include?(name)) end def parse_parameter_value(param, value) case value when /^---/ Puppet.debug("Found YAML while processing request parameter #{param} (value: <#{value}>)") Puppet.deprecation_warning("YAML in network requests is deprecated and will be removed in a future version. See http://links.puppetlabs.com/deprecate_yaml_on_network") YAML.load(value, :safe => true, :deserialize_symbols => true) when Array value.collect { |v| parse_primitive_parameter_value(v) } else parse_primitive_parameter_value(value) end end def parse_primitive_parameter_value(value) case value when "true" true when "false" false when /^\d+$/ Integer(value) when /^\d+\.\d+$/ value.to_f else value end end def configure_profiler(request_headers, request_params) if (request_headers.has_key?(Puppet::Network::HTTP::HEADER_ENABLE_PROFILING.downcase) or Puppet[:profile]) - Puppet::Util::Profiler.add_profiler(Puppet::Util::Profiler::WallClock.new(Puppet.method(:debug), request_params.object_id)) + Puppet::Util::Profiler.add_profiler(Puppet::Util::Profiler::Aggregate.new(Puppet.method(:debug), request_params.object_id)) end end def remove_profiler(profiler) + profiler.shutdown Puppet::Util::Profiler.remove_profiler(profiler) end end diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 367a75248..f49134883 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -1,619 +1,619 @@ require 'forwardable' require 'puppet/node' require 'puppet/resource/catalog' require 'puppet/util/errors' require 'puppet/resource/type_collection_helper' # Maintain a graph of scopes, along with a bunch of data # about the individual catalog we're compiling. class Puppet::Parser::Compiler extend Forwardable include Puppet::Util include Puppet::Util::Errors include Puppet::Util::MethodHelper include Puppet::Resource::TypeCollectionHelper def self.compile(node) $env_module_directories = nil node.environment.check_for_reparse new(node).compile.to_resource rescue => detail message = "#{detail} on node #{node.name}" Puppet.log_exception(detail, message) raise Puppet::Error, message, detail.backtrace end attr_reader :node, :facts, :collections, :catalog, :resources, :relationships, :topscope # The injector that provides lookup services, or nil if accessed before the compiler has started compiling and # bootstrapped. The injector is initialized and available before any manifests are evaluated. # # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services for this compiler/environment # @api public # attr_accessor :injector # Access to the configured loaders for 4x # @return [Puppet::Pops::Loader::Loaders] the configured loaders # @api private attr_reader :loaders # The injector that provides lookup services during the creation of the {#injector}. # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services during injector creation # for this compiler/environment # # @api private # attr_accessor :boot_injector # Add a collection to the global list. def_delegator :@collections, :<<, :add_collection def_delegator :@relationships, :<<, :add_relationship # Store a resource override. def add_override(override) # If possible, merge the override in immediately. if resource = @catalog.resource(override.ref) resource.merge(override) else # Otherwise, store the override for later; these # get evaluated in Resource#finish. @resource_overrides[override.ref] << override end end def add_resource(scope, resource) @resources << resource # Note that this will fail if the resource is not unique. @catalog.add_resource(resource) if not resource.class? and resource[:stage] raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage" end # Stages should not be inside of classes. They are always a # top-level container, regardless of where they appear in the # manifest. return if resource.stage? # This adds a resource to the class it lexically appears in in the # manifest. unless resource.class? return @catalog.add_edge(scope.resource, resource) end end # Do we use nodes found in the code, vs. the external node sources? def_delegator :known_resource_types, :nodes?, :ast_nodes? # Store the fact that we've evaluated a class def add_class(name) @catalog.add_class(name) unless name == "" end # Return a list of all of the defined classes. def_delegator :@catalog, :classes, :classlist # Compiler our catalog. This mostly revolves around finding and evaluating classes. # This is the main entry into our catalog. def compile Puppet.override( @context_overrides , "For compiling #{node.name}") do @catalog.environment_instance = environment # Set the client's parameters into the top scope. - Puppet::Util::Profiler.profile("Compile: Set node parameters") { set_node_parameters } + Puppet::Util::Profiler.profile("Compile: Set node parameters", [:compiler, :set_node_params]) { set_node_parameters } - Puppet::Util::Profiler.profile("Compile: Created settings scope") { create_settings_scope } + Puppet::Util::Profiler.profile("Compile: Created settings scope", [:compiler, :create_settings_scope]) { create_settings_scope } if is_binder_active? # create injector, if not already created - this is for 3x that does not trigger # lazy loading of injector via context - Puppet::Util::Profiler.profile("Compile: Created injector") { injector } + Puppet::Util::Profiler.profile("Compile: Created injector", [:compiler, :create_injector]) { injector } end - Puppet::Util::Profiler.profile("Compile: Evaluated main") { evaluate_main } + Puppet::Util::Profiler.profile("Compile: Evaluated main", [:compiler, :evaluate_main]) { evaluate_main } - Puppet::Util::Profiler.profile("Compile: Evaluated AST node") { evaluate_ast_node } + Puppet::Util::Profiler.profile("Compile: Evaluated AST node", [:compiler, :evaluate_ast_node]) { evaluate_ast_node } - Puppet::Util::Profiler.profile("Compile: Evaluated node classes") { evaluate_node_classes } + Puppet::Util::Profiler.profile("Compile: Evaluated node classes", [:compiler, :evaluate_node_classes]) { evaluate_node_classes } - Puppet::Util::Profiler.profile("Compile: Evaluated generators") { evaluate_generators } + Puppet::Util::Profiler.profile("Compile: Evaluated generators", [:compiler, :evaluate_generators]) { evaluate_generators } - Puppet::Util::Profiler.profile("Compile: Finished catalog") { finish } + Puppet::Util::Profiler.profile("Compile: Finished catalog", [:compiler, :finish_catalog]) { finish } fail_on_unevaluated @catalog end end # Constructs the overrides for the context def context_overrides() if Puppet[:parser] == 'future' require 'puppet/loaders' { :current_environment => environment, :global_scope => @topscope, # 4x placeholder for new global scope :loaders => lambda {|| loaders() }, # 4x loaders :injector => lambda {|| injector() } # 4x API - via context instead of via compiler } else { :current_environment => environment, } end end def_delegator :@collections, :delete, :delete_collection # Return the node's environment. def environment unless node.environment.is_a? Puppet::Node::Environment raise Puppet::DevError, "node #{node} has an invalid environment!" end node.environment end # Evaluate all of the classes specified by the node. # Classes with parameters are evaluated as if they were declared. # Classes without parameters or with an empty set of parameters are evaluated # as if they were included. This means classes with an empty set of # parameters won't conflict even if the class has already been included. def evaluate_node_classes if @node.classes.is_a? Hash classes_with_params, classes_without_params = @node.classes.partition {|name,params| params and !params.empty?} # The results from Hash#partition are arrays of pairs rather than hashes, # so we have to convert to the forms evaluate_classes expects (Hash, and # Array of class names) classes_with_params = Hash[classes_with_params] classes_without_params.map!(&:first) else classes_with_params = {} classes_without_params = @node.classes end evaluate_classes(classes_without_params, @node_scope || topscope) evaluate_classes(classes_with_params, @node_scope || topscope) end # Evaluate each specified class in turn. If there are any classes we can't # find, raise an error. This method really just creates resource objects # that point back to the classes, and then the resources are themselves # evaluated later in the process. # # Sometimes we evaluate classes with a fully qualified name already, in which # case, we tell scope.find_hostclass we've pre-qualified the name so it # doesn't need to search its namespaces again. This gets around a weird # edge case of duplicate class names, one at top scope and one nested in our # namespace and the wrong one (or both!) getting selected. See ticket #13349 # for more detail. --jeffweiss 26 apr 2012 def evaluate_classes(classes, scope, lazy_evaluate = true, fqname = false) raise Puppet::DevError, "No source for scope passed to evaluate_classes" unless scope.source class_parameters = nil # if we are a param class, save the classes hash # and transform classes to be the keys if classes.class == Hash class_parameters = classes classes = classes.keys end hostclasses = classes.collect do |name| scope.find_hostclass(name, :assume_fqname => fqname) or raise Puppet::Error, "Could not find class #{name} for #{node.name}" end if class_parameters resources = ensure_classes_with_parameters(scope, hostclasses, class_parameters) if !lazy_evaluate resources.each(&:evaluate) end resources else already_included, newly_included = ensure_classes_without_parameters(scope, hostclasses) if !lazy_evaluate newly_included.each(&:evaluate) end already_included + newly_included end end def evaluate_relationships @relationships.each { |rel| rel.evaluate(catalog) } end # Return a resource by either its ref or its type and title. def_delegator :@catalog, :resource, :findresource def initialize(node, options = {}) @node = node set_options(options) initvars end # Create a new scope, with either a specified parent scope or # using the top scope. def newscope(parent, options = {}) parent ||= topscope scope = Puppet::Parser::Scope.new(self, options) scope.parent = parent scope end # Return any overrides for the given resource. def resource_overrides(resource) @resource_overrides[resource.ref] end def injector create_injector if @injector.nil? @injector end def loaders @loaders ||= Puppet::Pops::Loaders.new(environment) end def boot_injector create_boot_injector(nil) if @boot_injector.nil? @boot_injector end # Creates the boot injector from registered system, default, and injector config. # @return [Puppet::Pops::Binder::Injector] the created boot injector # @api private Cannot be 'private' since it is called from the BindingsComposer. # def create_boot_injector(env_boot_bindings) assert_binder_active() pb = Puppet::Pops::Binder boot_contribution = pb::SystemBindings.injector_boot_contribution(env_boot_bindings) final_contribution = pb::SystemBindings.final_contribution binder = pb::Binder.new(pb::BindingsFactory.layered_bindings(final_contribution, boot_contribution)) @boot_injector = pb::Injector.new(binder) end # Answers if Puppet Binder should be active or not, and if it should and is not active, then it is activated. # @return [Boolean] true if the Puppet Binder should be activated def is_binder_active? should_be_active = Puppet[:binder] || Puppet[:parser] == 'future' if should_be_active # TODO: this should be in a central place, not just for ParserFactory anymore... Puppet::Parser::ParserFactory.assert_rgen_installed() @@binder_loaded ||= false unless @@binder_loaded require 'puppet/pops' require 'puppetx' @@binder_loaded = true end end should_be_active end private def ensure_classes_with_parameters(scope, hostclasses, parameters) hostclasses.collect do |klass| klass.ensure_in_catalog(scope, parameters[klass.name] || {}) end end def ensure_classes_without_parameters(scope, hostclasses) already_included = [] newly_included = [] hostclasses.each do |klass| class_scope = scope.class_scope(klass) if class_scope already_included << class_scope.resource else newly_included << klass.ensure_in_catalog(scope) end end [already_included, newly_included] end # If ast nodes are enabled, then see if we can find and evaluate one. def evaluate_ast_node return unless ast_nodes? # Now see if we can find the node. astnode = nil @node.names.each do |name| break if astnode = known_resource_types.node(name.to_s.downcase) end unless (astnode ||= known_resource_types.node("default")) raise Puppet::ParseError, "Could not find default node or by name with '#{node.names.join(", ")}'" end # Create a resource to model this node, and then add it to the list # of resources. resource = astnode.ensure_in_catalog(topscope) resource.evaluate @node_scope = topscope.class_scope(astnode) end # Evaluate our collections and return true if anything returned an object. # The 'true' is used to continue a loop, so it's important. def evaluate_collections return false if @collections.empty? exceptwrap do # We have to iterate over a dup of the array because # collections can delete themselves from the list, which # changes its length and causes some collections to get missed. - Puppet::Util::Profiler.profile("Evaluated collections") do + Puppet::Util::Profiler.profile("Evaluated collections", [:compiler, :evaluate_collections]) do found_something = false @collections.dup.each do |collection| found_something = true if collection.evaluate end found_something end end end # Make sure all of our resources have been evaluated into native resources. # We return true if any resources have, so that we know to continue the # evaluate_generators loop. def evaluate_definitions exceptwrap do - Puppet::Util::Profiler.profile("Evaluated definitions") do + Puppet::Util::Profiler.profile("Evaluated definitions", [:compiler, :evaluate_definitions]) do !unevaluated_resources.each do |resource| - Puppet::Util::Profiler.profile("Evaluated resource #{resource}") do + Puppet::Util::Profiler.profile("Evaluated resource #{resource}", [:compiler, :evaluate_resource, resource]) do resource.evaluate end end.empty? end end end # Iterate over collections and resources until we're sure that the whole # compile is evaluated. This is necessary because both collections # and defined resources can generate new resources, which themselves could # be defined resources. def evaluate_generators count = 0 loop do done = true - Puppet::Util::Profiler.profile("Iterated (#{count + 1}) on generators") do + Puppet::Util::Profiler.profile("Iterated (#{count + 1}) on generators", [:compiler, :iterate_on_generators]) do # Call collections first, then definitions. done = false if evaluate_collections done = false if evaluate_definitions end break if done count += 1 if count > 1000 raise Puppet::ParseError, "Somehow looped more than 1000 times while evaluating host catalog" end end end # Find and evaluate our main object, if possible. def evaluate_main @main = known_resource_types.find_hostclass([""], "") || known_resource_types.add(Puppet::Resource::Type.new(:hostclass, "")) @topscope.source = @main @main_resource = Puppet::Parser::Resource.new("class", :main, :scope => @topscope, :source => @main) @topscope.resource = @main_resource add_resource(@topscope, @main_resource) @main_resource.evaluate end # Make sure the entire catalog is evaluated. def fail_on_unevaluated fail_on_unevaluated_overrides fail_on_unevaluated_resource_collections end # If there are any resource overrides remaining, then we could # not find the resource they were supposed to override, so we # want to throw an exception. def fail_on_unevaluated_overrides remaining = @resource_overrides.values.flatten.collect(&:ref) if !remaining.empty? fail Puppet::ParseError, "Could not find resource(s) #{remaining.join(', ')} for overriding" end end # Make sure we don't have any remaining collections that specifically # look for resources, because we want to consider those to be # parse errors. def fail_on_unevaluated_resource_collections remaining = @collections.collect(&:resources).flatten.compact if !remaining.empty? raise Puppet::ParseError, "Failed to realize virtual resources #{remaining.join(', ')}" end end # Make sure all of our resources and such have done any last work # necessary. def finish evaluate_relationships resources.each do |resource| # Add in any resource overrides. if overrides = resource_overrides(resource) overrides.each do |over| resource.merge(over) end # Remove the overrides, so that the configuration knows there # are none left. overrides.clear end resource.finish if resource.respond_to?(:finish) end add_resource_metaparams end def add_resource_metaparams unless main = catalog.resource(:class, :main) raise "Couldn't find main" end names = Puppet::Type.metaparams.select do |name| !Puppet::Parser::Resource.relationship_parameter?(name) end data = {} catalog.walk(main, :out) do |source, target| if source_data = data[source] || metaparams_as_data(source, names) # only store anything in the data hash if we've actually got # data data[source] ||= source_data source_data.each do |param, value| target[param] = value if target[param].nil? end data[target] = source_data.merge(metaparams_as_data(target, names)) end target.tag(*(source.tags)) end end def metaparams_as_data(resource, params) data = nil params.each do |param| unless resource[param].nil? # Because we could be creating a hash for every resource, # and we actually probably don't often have any data here at all, # we're optimizing a bit by only creating a hash if there's # any data to put in it. data ||= {} data[param] = resource[param] end end data end # Set up all of our internal variables. def initvars # The list of overrides. This is used to cache overrides on objects # that don't exist yet. We store an array of each override. @resource_overrides = Hash.new do |overs, ref| overs[ref] = [] end # The list of collections that have been created. This is a global list, # but they each refer back to the scope that created them. @collections = [] # The list of relationships to evaluate. @relationships = [] # For maintaining the relationship between scopes and their resources. @catalog = Puppet::Resource::Catalog.new(@node.name, @node.environment) # MOVED HERE - SCOPE IS NEEDED (MOVE-SCOPE) # Create the initial scope, it is needed early @topscope = Puppet::Parser::Scope.new(self) # Need to compute overrides here, and remember them, because we are about to # enter the magic zone of known_resource_types and intial import. # Expensive entries in the context are bound lazily. @context_overrides = context_overrides() # This construct ensures that initial import (triggered by instantiating # the structure 'known_resource_types') has a configured context # It cannot survive the initvars method, and is later reinstated # as part of compiling... # Puppet.override( @context_overrides , "For initializing compiler") do # THE MAGIC STARTS HERE ! This triggers parsing, loading etc. @catalog.version = known_resource_types.version end @catalog.add_resource(Puppet::Parser::Resource.new("stage", :main, :scope => @topscope)) # local resource array to maintain resource ordering @resources = [] # Make sure any external node classes are in our class list if @node.classes.class == Hash @catalog.add_class(*@node.classes.keys) else @catalog.add_class(*@node.classes) end end # Set the node's parameters into the top-scope as variables. def set_node_parameters node.parameters.each do |param, value| @topscope[param.to_s] = value end # These might be nil. catalog.client_version = node.parameters["clientversion"] catalog.server_version = node.parameters["serverversion"] if Puppet[:trusted_node_data] @topscope.set_trusted(node.trusted_data) end if(Puppet[:immutable_node_data]) facts_hash = node.facts.nil? ? {} : node.facts.values @topscope.set_facts(facts_hash) end end def create_settings_scope unless settings_type = environment.known_resource_types.hostclass("settings") settings_type = Puppet::Resource::Type.new :hostclass, "settings" environment.known_resource_types.add(settings_type) end settings_resource = Puppet::Parser::Resource.new("class", "settings", :scope => @topscope) @catalog.add_resource(settings_resource) settings_type.evaluate_code(settings_resource) scope = @topscope.class_scope(settings_type) Puppet.settings.each do |name, setting| next if name.to_s == "name" scope[name.to_s] = environment[name] end end # Return an array of all of the unevaluated resources. These will be definitions, # which need to get evaluated into native resources. def unevaluated_resources # The order of these is significant for speed due to short-circuting resources.reject { |resource| resource.evaluated? or resource.virtual? or resource.builtin_type? } end # Creates the injector from bindings found in the current environment. # @return [void] # @api private # def create_injector assert_binder_active() composer = Puppet::Pops::Binder::BindingsComposer.new() layered_bindings = composer.compose(topscope) @injector = Puppet::Pops::Binder::Injector.new(Puppet::Pops::Binder::Binder.new(layered_bindings)) end def assert_binder_active unless is_binder_active? raise ArgumentError, "The Puppet Binder is only available when either '--binder true' or '--parser future' is used" end end end diff --git a/lib/puppet/parser/functions.rb b/lib/puppet/parser/functions.rb index 6ce9c641a..45e9a1f0f 100644 --- a/lib/puppet/parser/functions.rb +++ b/lib/puppet/parser/functions.rb @@ -1,268 +1,268 @@ require 'puppet/util/autoload' require 'puppet/parser/scope' # A module for managing parser functions. Each specified function # is added to a central module that then gets included into the Scope # class. # # @api public module Puppet::Parser::Functions Environment = Puppet::Node::Environment class << self include Puppet::Util end # Reset the list of loaded functions. # # @api private def self.reset @modules = {} # Runs a newfunction to create a function for each of the log levels Puppet::Util::Log.levels.each do |level| newfunction(level, :environment => Puppet.lookup(:root_environment), :doc => "Log a message on the server at level #{level.to_s}.") do |vals| send(level, vals.join(" ")) end end end # Accessor for singleton autoloader # # @api private def self.autoloader @autoloader ||= Puppet::Util::Autoload.new( self, "puppet/parser/functions", :wrap => false ) end # Get the module that functions are mixed into corresponding to an # environment # # @api private def self.environment_module(env) @modules[env.name] ||= Module.new do @metadata = {} def self.all_function_info @metadata end def self.get_function_info(name) @metadata[name] end def self.add_function_info(name, info) @metadata[name] = info end end end # Create a new Puppet DSL function. # # **The {newfunction} method provides a public API.** # # This method is used both internally inside of Puppet to define parser # functions. For example, template() is defined in # {file:lib/puppet/parser/functions/template.rb template.rb} using the # {newfunction} method. Third party Puppet modules such as # [stdlib](https://forge.puppetlabs.com/puppetlabs/stdlib) use this method to # extend the behavior and functionality of Puppet. # # See also [Docs: Custom # Functions](http://docs.puppetlabs.com/guides/custom_functions.html) # # @example Define a new Puppet DSL Function # >> Puppet::Parser::Functions.newfunction(:double, :arity => 1, # :doc => "Doubles an object, typically a number or string.", # :type => :rvalue) {|i| i[0]*2 } # => {:arity=>1, :type=>:rvalue, # :name=>"function_double", # :doc=>"Doubles an object, typically a number or string."} # # @example Invoke the double function from irb as is done in RSpec examples: # >> require 'puppet_spec/scope' # >> scope = PuppetSpec::Scope.create_test_scope_for_node('example') # => Scope() # >> scope.function_double([2]) # => 4 # >> scope.function_double([4]) # => 8 # >> scope.function_double([]) # ArgumentError: double(): Wrong number of arguments given (0 for 1) # >> scope.function_double([4,8]) # ArgumentError: double(): Wrong number of arguments given (2 for 1) # >> scope.function_double(["hello"]) # => "hellohello" # # @param [Symbol] name the name of the function represented as a ruby Symbol. # The {newfunction} method will define a Ruby method based on this name on # the parser scope instance. # # @param [Proc] block the block provided to the {newfunction} method will be # executed when the Puppet DSL function is evaluated during catalog # compilation. The arguments to the function will be passed as an array to # the first argument of the block. The return value of the block will be # the return value of the Puppet DSL function for `:rvalue` functions. # # @option options [:rvalue, :statement] :type (:statement) the type of function. # Either `:rvalue` for functions that return a value, or `:statement` for # functions that do not return a value. # # @option options [String] :doc ('') the documentation for the function. # This string will be extracted by documentation generation tools. # # @option options [Integer] :arity (-1) the # [arity](http://en.wikipedia.org/wiki/Arity) of the function. When # specified as a positive integer the function is expected to receive # _exactly_ the specified number of arguments. When specified as a # negative number, the function is expected to receive _at least_ the # absolute value of the specified number of arguments incremented by one. # For example, a function with an arity of `-4` is expected to receive at # minimum 3 arguments. A function with the default arity of `-1` accepts # zero or more arguments. A function with an arity of 2 must be provided # with exactly two arguments, no more and no less. Added in Puppet 3.1.0. # # @option options [Puppet::Node::Environment] :environment (nil) can # explicitly pass the environment we wanted the function added to. Only used # to set logging functions in root environment # # @return [Hash] describing the function. # # @api public def self.newfunction(name, options = {}, &block) # Short circuit this call when 4x "biff" is in effect to allow the new loader system to load # and define the function a different way. # if Puppet[:biff] return Puppet::Pops::Loader::RubyLegacyFunctionInstantiator.legacy_newfunction(name, options, &block) end name = name.intern environment = options[:environment] || Puppet.lookup(:current_environment) Puppet.warning "Overwriting previous definition for function #{name}" if get_function(name, environment) arity = options[:arity] || -1 ftype = options[:type] || :statement unless ftype == :statement or ftype == :rvalue raise Puppet::DevError, "Invalid statement type #{ftype.inspect}" end # the block must be installed as a method because it may use "return", # which is not allowed from procs. real_fname = "real_function_#{name}" environment_module(environment).send(:define_method, real_fname, &block) fname = "function_#{name}" env_module = environment_module(environment) env_module.send(:define_method, fname) do |*args| - Puppet::Util::Profiler.profile("Called #{name}") do + Puppet::Util::Profiler.profile("Called #{name}", [:functions, name]) do if args[0].is_a? Array if arity >= 0 and args[0].size != arity raise ArgumentError, "#{name}(): Wrong number of arguments given (#{args[0].size} for #{arity})" elsif arity < 0 and args[0].size < (arity+1).abs raise ArgumentError, "#{name}(): Wrong number of arguments given (#{args[0].size} for minimum #{(arity+1).abs})" end self.send(real_fname, args[0]) else raise ArgumentError, "custom functions must be called with a single array that contains the arguments. For example, function_example([1]) instead of function_example(1)" end end end func = {:arity => arity, :type => ftype, :name => fname} func[:doc] = options[:doc] if options[:doc] env_module.add_function_info(name, func) func end # Determine if a function is defined # # @param [Symbol] name the function # @param [Puppet::Node::Environment] environment the environment to find the function in # # @return [Symbol, false] The name of the function if it's defined, # otherwise false. # # @api public def self.function(name, environment = Puppet.lookup(:current_environment)) name = name.intern func = nil unless func = get_function(name, environment) autoloader.load(name, environment) func = get_function(name, environment) end if func func[:name] else false end end def self.functiondocs(environment = Puppet.lookup(:current_environment)) autoloader.loadall ret = "" merged_functions(environment).sort { |a,b| a[0].to_s <=> b[0].to_s }.each do |name, hash| ret << "#{name}\n#{"-" * name.to_s.length}\n" if hash[:doc] ret << Puppet::Util::Docs.scrub(hash[:doc]) else ret << "Undocumented.\n" end ret << "\n\n- *Type*: #{hash[:type]}\n\n" end ret end # Determine whether a given function returns a value. # # @param [Symbol] name the function # @param [Puppet::Node::Environment] environment The environment to find the function in # @return [Boolean] whether it is an rvalue function # # @api public def self.rvalue?(name, environment = Puppet.lookup(:current_environment)) func = get_function(name, environment) func ? func[:type] == :rvalue : false end # Return the number of arguments a function expects. # # @param [Symbol] name the function # @param [Puppet::Node::Environment] environment The environment to find the function in # @return [Integer] The arity of the function. See {newfunction} for # the meaning of negative values. # # @api public def self.arity(name, environment = Puppet.lookup(:current_environment)) func = get_function(name, environment) func ? func[:arity] : -1 end class << self private def merged_functions(environment) root = environment_module(Puppet.lookup(:root_environment)) env = environment_module(environment) root.all_function_info.merge(env.all_function_info) end def get_function(name, environment) environment_module(environment).get_function_info(name.intern) || environment_module(Puppet.lookup(:root_environment)).get_function_info(name.intern) end end end diff --git a/lib/puppet/util/profiler.rb b/lib/puppet/util/profiler.rb index d0d373cc2..820231f27 100644 --- a/lib/puppet/util/profiler.rb +++ b/lib/puppet/util/profiler.rb @@ -1,52 +1,53 @@ require 'benchmark' # A simple profiling callback system. # # @api public module Puppet::Util::Profiler require 'puppet/util/profiler/wall_clock' require 'puppet/util/profiler/object_counts' require 'puppet/util/profiler/around_profiler' @profiler = Puppet::Util::Profiler::AroundProfiler.new # Reset the profiling system to the original state # # @api private def self.clear @profiler.clear end # Retrieve the current list of profilers # # @api private def self.current @profiler.current end # @param profiler [#profile] A profiler for the current thread # @api private def self.add_profiler(profiler) @profiler.add_profiler(profiler) end # @param profiler [#profile] A profiler to remove from the current thread # @api private def self.remove_profiler(profiler) @profiler.remove_profiler(profiler) end # Profile a block of code and log the time it took to execute. # # This outputs logs entries to the Puppet masters logging destination # providing the time it took, a message describing the profiled code # and a leaf location marking where the profile method was called # in the profiled hierachy. # # @param message [String] A description of the profiled event + # @param metric_id [Array] A list of strings making up the ID of a metric to profile # @param block [Block] The segment of code to profile # @api public - def self.profile(message, &block) - @profiler.profile(message, &block) + def self.profile(message, metric_id = nil, &block) + @profiler.profile(message, metric_id, &block) end end diff --git a/lib/puppet/util/profiler/aggregate.rb b/lib/puppet/util/profiler/aggregate.rb new file mode 100644 index 000000000..e8a4ca595 --- /dev/null +++ b/lib/puppet/util/profiler/aggregate.rb @@ -0,0 +1,85 @@ +require 'puppet/util/profiler' +require 'puppet/util/profiler/wall_clock' + +class Puppet::Util::Profiler::Aggregate < Puppet::Util::Profiler::WallClock + def initialize(logger, identifier) + super(logger, identifier) + @metrics_hash = Metric.new + end + + def shutdown() + super + @logger.call("AGGREGATE PROFILING RESULTS:") + @logger.call("----------------------------") + print_metrics(@metrics_hash, "") + @logger.call("----------------------------") + end + + def do_start(description, metric_id) + super(description, metric_id) + end + + def do_finish(context, description, metric_id) + result = super(context, description, metric_id) + update_metric(@metrics_hash, metric_id, result[:time]) + result + end + + def update_metric(metrics_hash, metric_id, time) + first, *rest = *metric_id + if first + m = metrics_hash[first] + m.increment + m.add_time(time) + if rest.count > 0 + update_metric(m, rest, time) + end + end + end + + def values + @metrics_hash + end + + def print_metrics(metrics_hash, prefix) + metrics_hash.sort_by {|k,v| v.time }.reverse.each do |k,v| + @logger.call("#{prefix}#{k}: #{v.time} ms (#{v.count} calls)") + print_metrics(metrics_hash[k], "#{prefix}#{k} -> ") + end + end + + class Metric < Hash + def initialize + super + @count = 0 + @time = 0 + end + attr_reader :count, :time + + def [](key) + if !has_key?(key) + self[key] = Metric.new + end + super(key) + end + + def increment + @count += 1 + end + + def add_time(time) + @time += time + end + end + + class Timer + def initialize + @start = Time.now + end + + def stop + Time.now - @start + end + end +end + diff --git a/lib/puppet/util/profiler/around_profiler.rb b/lib/puppet/util/profiler/around_profiler.rb index 5ee546640..0b408c6d8 100644 --- a/lib/puppet/util/profiler/around_profiler.rb +++ b/lib/puppet/util/profiler/around_profiler.rb @@ -1,66 +1,67 @@ # A Profiler that can be used to wrap around blocks of code. It is configured # with other profilers and controls them to start before the block is executed # and finish after the block is executed. # # @api private class Puppet::Util::Profiler::AroundProfiler def initialize @profilers = [] end # Reset the profiling system to the original state # # @api private def clear @profilers = [] end # Retrieve the current list of profilers # # @api private def current @profilers end # @param profiler [#profile] A profiler for the current thread # @api private def add_profiler(profiler) @profilers << profiler profiler end # @param profiler [#profile] A profiler to remove from the current thread # @api private def remove_profiler(profiler) @profilers.delete(profiler) end # Profile a block of code and log the time it took to execute. # # This outputs logs entries to the Puppet masters logging destination # providing the time it took, a message describing the profiled code # and a leaf location marking where the profile method was called # in the profiled hierachy. # # @param message [String] A description of the profiled event + # @param metric_id [Array] A list of strings making up the ID of a metric to profile # @param block [Block] The segment of code to profile # @api private - def profile(message) + def profile(message, metric_id = nil) retval = nil contexts = {} @profilers.each do |profiler| - contexts[profiler] = profiler.start(message) + contexts[profiler] = profiler.start(message, metric_id) end begin retval = yield ensure @profilers.each do |profiler| - profiler.finish(contexts[profiler], message) + profiler.finish(contexts[profiler], message, metric_id) end end retval end end diff --git a/lib/puppet/util/profiler/logging.rb b/lib/puppet/util/profiler/logging.rb index e4806c3f8..5d1fd101c 100644 --- a/lib/puppet/util/profiler/logging.rb +++ b/lib/puppet/util/profiler/logging.rb @@ -1,44 +1,48 @@ class Puppet::Util::Profiler::Logging def initialize(logger, identifier) @logger = logger @identifier = identifier @sequence = Sequence.new end - def start(description) + def start(description, metric_id) @sequence.next @sequence.down - do_start + do_start(description, metric_id) end - def finish(context, description) - profile_explanation = do_finish(context) + def finish(context, description, metric_id) + profile_explanation = do_finish(context, description, metric_id)[:msg] @sequence.up @logger.call("PROFILE [#{@identifier}] #{@sequence} #{description}: #{profile_explanation}") end + def shutdown() + # nothing to do + end + class Sequence INITIAL = 0 SEPARATOR = '.' def initialize @elements = [INITIAL] end def next @elements[-1] += 1 end def down @elements << INITIAL end def up @elements.pop end def to_s @elements.join(SEPARATOR) end end end diff --git a/lib/puppet/util/profiler/wall_clock.rb b/lib/puppet/util/profiler/wall_clock.rb index ae0a151c3..46ac095ec 100644 --- a/lib/puppet/util/profiler/wall_clock.rb +++ b/lib/puppet/util/profiler/wall_clock.rb @@ -1,34 +1,35 @@ require 'puppet/util/profiler/logging' # A profiler implementation that measures the number of seconds a segment of # code takes to execute and provides a callback with a string representation of # the profiling information. # # @api private class Puppet::Util::Profiler::WallClock < Puppet::Util::Profiler::Logging - def do_start + def do_start(description, metric_id) Timer.new end - def do_finish(context) - context.stop - "took #{context} seconds" + def do_finish(context, description, metric_id) + {:time => context.stop, + :msg => "took #{context} seconds"} end class Timer FOUR_DECIMAL_DIGITS = '%0.4f' def initialize @start = Time.now end def stop - @finish = Time.now + @time = Time.now - @start + @time end def to_s - format(FOUR_DECIMAL_DIGITS, @finish - @start) + format(FOUR_DECIMAL_DIGITS, @time) end end end diff --git a/spec/unit/network/http/handler_spec.rb b/spec/unit/network/http/handler_spec.rb index 1259b749b..caa372180 100755 --- a/spec/unit/network/http/handler_spec.rb +++ b/spec/unit/network/http/handler_spec.rb @@ -1,243 +1,241 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet/indirector_testing' require 'puppet/network/authorization' require 'puppet/network/authentication' require 'puppet/network/http' describe Puppet::Network::HTTP::Handler do before :each do Puppet::IndirectorTesting.indirection.terminus_class = :memory end let(:indirection) { Puppet::IndirectorTesting.indirection } def a_request(method = "HEAD", path = "/production/#{indirection.name}/unknown") { :accept_header => "pson", :content_type_header => "text/yaml", :http_method => method, :path => path, :params => {}, :client_cert => nil, :headers => {}, :body => nil } end let(:handler) { TestingHandler.new() } describe "the HTTP Handler" do def respond(text) lambda { |req, res| res.respond_with(200, "text/plain", text) } end it "hands the request to the first route that matches the request path" do handler = TestingHandler.new( Puppet::Network::HTTP::Route.path(%r{^/foo}).get(respond("skipped")), Puppet::Network::HTTP::Route.path(%r{^/vtest}).get(respond("used")), Puppet::Network::HTTP::Route.path(%r{^/vtest/foo}).get(respond("ignored"))) req = a_request("GET", "/vtest/foo") res = {} handler.process(req, res) expect(res[:body]).to eq("used") end it "raises an error if multiple routes with the same path regex are registered" do expect do handler = TestingHandler.new( Puppet::Network::HTTP::Route.path(%r{^/foo}).get(respond("ignored")), Puppet::Network::HTTP::Route.path(%r{^/foo}).post(respond("also ignored"))) end.to raise_error(ArgumentError) end it "raises an HTTP not found error if no routes match" do handler = TestingHandler.new req = a_request("GET", "/vtest/foo") res = {} handler.process(req, res) res_body = JSON(res[:body]) expect(res[:content_type_header]).to eq("application/json") expect(res_body["issue_kind"]).to eq("HANDLER_NOT_FOUND") expect(res_body["message"]).to eq("Not Found: No route for GET /vtest/foo") expect(res[:status]).to eq(404) end it "returns a structured error response with a stacktrace when the server encounters an internal error" do handler = TestingHandler.new( Puppet::Network::HTTP::Route.path(/.*/).get(lambda { |_, _| raise Exception.new("the sky is falling!")})) req = a_request("GET", "/vtest/foo") res = {} handler.process(req, res) res_body = JSON(res[:body]) expect(res[:content_type_header]).to eq("application/json") expect(res_body["issue_kind"]).to eq(Puppet::Network::HTTP::Issues::RUNTIME_ERROR.to_s) expect(res_body["message"]).to eq("Server Error: the sky is falling!") expect(res_body["stacktrace"].is_a?(Array) && !res_body["stacktrace"].empty?).to be_true expect(res_body["stacktrace"][0]).to match("spec/unit/network/http/handler_spec.rb") expect(res[:status]).to eq(500) end end describe "when processing a request" do let(:response) do { :status => 200 } end before do handler.stubs(:check_authorization) handler.stubs(:warn_if_near_expiration) end it "should check the client certificate for upcoming expiration" do request = a_request cert = mock 'cert' handler.expects(:client_cert).returns(cert).with(request) handler.expects(:warn_if_near_expiration).with(cert) handler.process(request, response) end it "should setup a profiler when the puppet-profiling header exists" do request = a_request request[:headers][Puppet::Network::HTTP::HEADER_ENABLE_PROFILING.downcase] = "true" - p = TestProfiler.new + p = HandlerTestProfiler.new Puppet::Util::Profiler.expects(:add_profiler).with { |profiler| profiler.is_a? Puppet::Util::Profiler::WallClock }.returns(p) Puppet::Util::Profiler.expects(:remove_profiler).with { |profiler| profiler == p } handler.process(request, response) end it "should not setup profiler when the profile parameter is missing" do request = a_request request[:params] = { } Puppet::Util::Profiler.expects(:add_profiler).never handler.process(request, response) end it "should raise an error if the request is formatted in an unknown format" do handler.stubs(:content_type_header).returns "unknown format" lambda { handler.request_format(request) }.should raise_error end it "should still find the correct format if content type contains charset information" do request = Puppet::Network::HTTP::Request.new({ 'content-type' => "text/plain; charset=UTF-8" }, {}, 'GET', '/', nil) request.format.should == "s" end it "should deserialize YAML parameters" do params = {'my_param' => [1,2,3].to_yaml} decoded_params = handler.send(:decode_params, params) decoded_params.should == {:my_param => [1,2,3]} end it "should ignore tags on YAML parameters" do params = {'my_param' => "--- !ruby/object:Array {}"} decoded_params = handler.send(:decode_params, params) decoded_params[:my_param].should be_a(Hash) end end describe "when resolving node" do it "should use a look-up from the ip address" do Resolv.expects(:getname).with("1.2.3.4").returns("host.domain.com") handler.resolve_node(:ip => "1.2.3.4") end it "should return the look-up result" do Resolv.stubs(:getname).with("1.2.3.4").returns("host.domain.com") handler.resolve_node(:ip => "1.2.3.4").should == "host.domain.com" end it "should return the ip address if resolving fails" do Resolv.stubs(:getname).with("1.2.3.4").raises(RuntimeError, "no such host") handler.resolve_node(:ip => "1.2.3.4").should == "1.2.3.4" end end class TestingHandler include Puppet::Network::HTTP::Handler def initialize(* routes) register(routes) end def set_content_type(response, format) response[:content_type_header] = format end def set_response(response, body, status = 200) response[:body] = body response[:status] = status end def http_method(request) request[:http_method] end def path(request) request[:path] end def params(request) request[:params] end def client_cert(request) request[:client_cert] end def body(request) request[:body] end def headers(request) request[:headers] || {} end end - class TestProfiler - attr_accessor :context, :description + class HandlerTestProfiler + def start(metric, description) + end - def start(description) - description + def finish(context, metric, description) end - def finish(context, description) - @context = context - @description = description + def shutdown() end end end diff --git a/spec/unit/util/profiler/aggregate_spec.rb b/spec/unit/util/profiler/aggregate_spec.rb new file mode 100644 index 000000000..20845d670 --- /dev/null +++ b/spec/unit/util/profiler/aggregate_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' +require 'puppet/util/profiler' +require 'puppet/util/profiler/around_profiler' +require 'puppet/util/profiler/aggregate' + +describe Puppet::Util::Profiler::Aggregate do + let(:logger) { AggregateSimpleLog.new } + let(:profiler) { Puppet::Util::Profiler::Aggregate.new(logger, nil) } + let(:profiler_mgr) do + p = Puppet::Util::Profiler::AroundProfiler.new + p.add_profiler(profiler) + p + end + + it "tracks the aggregate counts and time for the hierarchy of metrics" do + profiler_mgr.profile("Looking up hiera data in production environment", ["function", "hiera_lookup", "production"]) {} + profiler_mgr.profile("Looking up hiera data in test environment", ["function", "hiera_lookup", "test"]) {} + profiler_mgr.profile("looking up stuff for compilation", ["compiler", "lookup"]) {} + profiler_mgr.profile("COMPILING ALL OF THE THINGS!", ["compiler", "compiling"]) {} + + profiler.values["function"].count.should == 2 + profiler.values["function"].time.should be > 0 + profiler.values["function"]["hiera_lookup"].count.should == 2 + profiler.values["function"]["hiera_lookup"]["production"].count.should == 1 + profiler.values["function"]["hiera_lookup"]["test"].count.should == 1 + profiler.values["function"].time.should be >= profiler.values["function"]["hiera_lookup"].time + + profiler.values["compiler"].count.should == 2 + profiler.values["compiler"].time.should be > 0 + profiler.values["compiler"]["lookup"].count.should == 1 + profiler.values["compiler"]["compiling"].count.should == 1 + profiler.values["compiler"].time.should be >= profiler.values["compiler"]["lookup"].time + + profiler.shutdown + + logger.output.should =~ /function -> hiera_lookup: .*\(2 calls\)\nfunction -> hiera_lookup ->.*\(1 calls\)/ + logger.output.should =~ /compiler: .*\(2 calls\)\ncompiler ->.*\(1 calls\)/ + end + + it "tolerates calls to `profile` that don't include a metric id" do + profiler_mgr.profile("yo") {} + end + + it "supports both symbols and strings as components of a metric id" do + profiler_mgr.profile("yo", [:foo, "bar"]) {} + end + + class AggregateSimpleLog + attr_reader :output + + def initialize + @output = "" + end + + def call(msg) + @output << msg << "\n" + end + end +end diff --git a/spec/unit/util/profiler/manager_spec.rb b/spec/unit/util/profiler/around_profiler_spec.rb similarity index 63% rename from spec/unit/util/profiler/manager_spec.rb rename to spec/unit/util/profiler/around_profiler_spec.rb index 9dbc6a79c..0837395b5 100644 --- a/spec/unit/util/profiler/manager_spec.rb +++ b/spec/unit/util/profiler/around_profiler_spec.rb @@ -1,61 +1,61 @@ require 'spec_helper' require 'puppet/util/profiler' describe Puppet::Util::Profiler::AroundProfiler do - let(:child) { TestProfiler.new() } + let(:child) { TestAroundProfiler.new() } let(:profiler) { Puppet::Util::Profiler::AroundProfiler.new } before :each do profiler.add_profiler(child) end it "returns the value of the profiled segment" do - retval = profiler.profile("Testing") { "the return value" } + retval = profiler.profile("Testing", ["testing"]) { "the return value" } retval.should == "the return value" end - it "propogates any errors raised in the profiled segment" do + it "propagates any errors raised in the profiled segment" do expect do - profiler.profile("Testing") { raise "a problem" } + profiler.profile("Testing", ["testing"]) { raise "a problem" } end.to raise_error("a problem") end it "makes the description and the context available to the `start` and `finish` methods" do - profiler.profile("Testing") { } + profiler.profile("Testing", ["testing"]) { } child.context.should == "Testing" child.description.should == "Testing" end it "calls finish even when an error is raised" do begin - profiler.profile("Testing") { raise "a problem" } + profiler.profile("Testing", ["testing"]) { raise "a problem" } rescue child.context.should == "Testing" end end it "supports multiple profilers" do - profiler2 = TestProfiler.new + profiler2 = TestAroundProfiler.new profiler.add_profiler(profiler2) - profiler.profile("Testing") {} + profiler.profile("Testing", ["testing"]) {} child.context.should == "Testing" profiler2.context.should == "Testing" end - class TestProfiler + class TestAroundProfiler attr_accessor :context, :description - def start(description) + def start(description, metric_id) description end - def finish(context, description) + def finish(context, description, metric_id) @context = context @description = description end end end diff --git a/spec/unit/util/profiler/logging_spec.rb b/spec/unit/util/profiler/logging_spec.rb index 08d83cde3..3f6a728dd 100644 --- a/spec/unit/util/profiler/logging_spec.rb +++ b/spec/unit/util/profiler/logging_spec.rb @@ -1,67 +1,70 @@ require 'spec_helper' require 'puppet/util/profiler' describe Puppet::Util::Profiler::Logging do let(:logger) { SimpleLog.new } let(:identifier) { "Profiling ID" } let(:logging_profiler) { TestLoggingProfiler.new(logger, identifier) } let(:profiler) do p = Puppet::Util::Profiler::AroundProfiler.new p.add_profiler(logging_profiler) p end - it "logs the explanation of the profile results" do - profiler.profile("Testing") { } + profiler.profile("Testing", ["test"]) { } logger.messages.first.should =~ /the explanation/ end it "describes the profiled segment" do - profiler.profile("Tested measurement") { } + profiler.profile("Tested measurement", ["test"]) { } logger.messages.first.should =~ /PROFILE \[#{identifier}\] \d Tested measurement/ end it "indicates the order in which segments are profiled" do - profiler.profile("Measurement") { } - profiler.profile("Another measurement") { } + profiler.profile("Measurement", ["measurement"]) { } + profiler.profile("Another measurement", ["measurement"]) { } logger.messages[0].should =~ /1 Measurement/ logger.messages[1].should =~ /2 Another measurement/ end it "indicates the nesting of profiled segments" do - profiler.profile("Measurement") { profiler.profile("Nested measurement") { } } - profiler.profile("Another measurement") { profiler.profile("Another nested measurement") { } } + profiler.profile("Measurement", ["measurement1"]) do + profiler.profile("Nested measurement", ["measurement2"]) { } + end + profiler.profile("Another measurement", ["measurement1"]) do + profiler.profile("Another nested measurement", ["measurement2"]) { } + end logger.messages[0].should =~ /1.1 Nested measurement/ logger.messages[1].should =~ /1 Measurement/ logger.messages[2].should =~ /2.1 Another nested measurement/ logger.messages[3].should =~ /2 Another measurement/ end class TestLoggingProfiler < Puppet::Util::Profiler::Logging - def do_start + def do_start(metric, description) "the start" end - def do_finish(context) - "the explanation of #{context}" + def do_finish(context, metric, description) + {:msg => "the explanation of #{context}"} end end class SimpleLog attr_reader :messages def initialize @messages = [] end def call(msg) @messages << msg end end end diff --git a/spec/unit/util/profiler/wall_clock_spec.rb b/spec/unit/util/profiler/wall_clock_spec.rb index 8d95a3ce9..1adcf0d61 100644 --- a/spec/unit/util/profiler/wall_clock_spec.rb +++ b/spec/unit/util/profiler/wall_clock_spec.rb @@ -1,13 +1,13 @@ require 'spec_helper' require 'puppet/util/profiler' describe Puppet::Util::Profiler::WallClock do it "logs the number of seconds it took to execute the segment" do profiler = Puppet::Util::Profiler::WallClock.new(nil, nil) - message = profiler.do_finish(profiler.start("Testing")) + message = profiler.do_finish(profiler.start(["foo", "bar"], "Testing"), ["foo", "bar"], "Testing")[:msg] message.should =~ /took \d\.\d{4} seconds/ end end diff --git a/spec/unit/util/profiler_spec.rb b/spec/unit/util/profiler_spec.rb index e9ed568aa..c7fc48cb9 100644 --- a/spec/unit/util/profiler_spec.rb +++ b/spec/unit/util/profiler_spec.rb @@ -1,44 +1,55 @@ require 'spec_helper' require 'puppet/util/profiler' describe Puppet::Util::Profiler do let(:profiler) { TestProfiler.new() } it "supports adding profilers" do subject.add_profiler(profiler) subject.current[0].should == profiler end it "supports removing profilers" do subject.add_profiler(profiler) subject.remove_profiler(profiler) subject.current.length.should == 0 end it "supports clearing profiler list" do subject.add_profiler(profiler) subject.clear subject.current.length.should == 0 end it "supports profiling" do + subject.add_profiler(profiler) + subject.profile("hi", ["mymetric"]) {} + profiler.context[:metric_id].should == ["mymetric"] + profiler.context[:description].should == "hi" + profiler.description.should == "hi" + end + + it "supports profiling without a metric id" do subject.add_profiler(profiler) subject.profile("hi") {} - profiler.context = "hi" - profiler.description = "hi" + profiler.context[:metric_id].should == nil + profiler.context[:description].should == "hi" + profiler.description.should == "hi" end class TestProfiler - attr_accessor :context, :description + attr_accessor :context, :metric, :description - def start(description) - description + def start(description, metric_id) + {:metric_id => metric_id, + :description => description} end - def finish(context, description) + def finish(context, description, metric_id) @context = context + @metric_id = metric_id @description = description end end end