diff --git a/lib/puppet/agent.rb b/lib/puppet/agent.rb index 678fdb5d0..14c6c693b 100644 --- a/lib/puppet/agent.rb +++ b/lib/puppet/agent.rb @@ -1,123 +1,118 @@ -require 'sync' require 'puppet/application' # A general class for triggering a run of another # class. class Puppet::Agent require 'puppet/agent/locker' include Puppet::Agent::Locker require 'puppet/agent/disabler' include Puppet::Agent::Disabler attr_reader :client_class, :client, :splayed, :should_fork # Just so we can specify that we are "the" instance. def initialize(client_class, should_fork=true) @splayed = false @should_fork = should_fork @client_class = client_class end def needing_restart? Puppet::Application.restart_requested? end # Perform a run with our client. def run(client_options = {}) if running? Puppet.notice "Run of #{client_class} already in progress; skipping (#{lockfile_path} exists)" return end if disabled? Puppet.notice "Skipping run of #{client_class}; administratively disabled (Reason: '#{disable_message}');\nUse 'puppet agent --enable' to re-enable." return end result = nil block_run = Puppet::Application.controlled_run do splay client_options.fetch :splay, Puppet[:splay] result = run_in_fork(should_fork) do with_client do |client| begin client_args = client_options.merge(:pluginsync => Puppet[:pluginsync]) - sync.synchronize { lock { client.run(client_args) } } + lock { client.run(client_args) } rescue SystemExit,NoMemoryError raise rescue Exception => detail Puppet.log_exception(detail, "Could not run #{client_class}: #{detail}") end end end true end Puppet.notice "Shutdown/restart in progress (#{Puppet::Application.run_status.inspect}); skipping run" unless block_run result end def stopping? Puppet::Application.stop_requested? end # Have we splayed already? def splayed? splayed end # Sleep when splay is enabled; else just return. def splay(do_splay = Puppet[:splay]) return unless do_splay return if splayed? time = rand(Puppet[:splaylimit] + 1) Puppet.info "Sleeping for #{time} seconds (splay is enabled)" sleep(time) @splayed = true end - def sync - @sync ||= Sync.new - end - def run_in_fork(forking = true) return yield unless forking or Puppet.features.windows? child_pid = Kernel.fork do $0 = "puppet agent: applying configuration" begin exit(yield) rescue SystemExit exit(-1) rescue NoMemoryError exit(-2) end end exit_code = Process.waitpid2(child_pid) case exit_code[1].exitstatus when -1 raise SystemExit when -2 raise NoMemoryError end exit_code[1].exitstatus end private # Create and yield a client instance, keeping a reference # to it during the yield. def with_client begin @client = client_class.new rescue SystemExit,NoMemoryError raise rescue Exception => detail Puppet.log_exception(detail, "Could not create instance of #{client_class}: #{detail}") return end yield @client ensure @client = nil end end diff --git a/lib/puppet/configurer.rb b/lib/puppet/configurer.rb index 685a1791e..b961e739b 100644 --- a/lib/puppet/configurer.rb +++ b/lib/puppet/configurer.rb @@ -1,262 +1,262 @@ # The client for interacting with the puppetmaster config server. require 'sync' require 'timeout' require 'puppet/network/http_pool' require 'puppet/util' require 'securerandom' class Puppet::Configurer require 'puppet/configurer/fact_handler' require 'puppet/configurer/plugin_handler' include Puppet::Configurer::FactHandler include Puppet::Configurer::PluginHandler # For benchmarking include Puppet::Util attr_reader :compile_time, :environment # Provide more helpful strings to the logging that the Agent does def self.to_s "Puppet configuration client" end class << self # Puppet agent should only have one instance running, and we need a # way to retrieve it. attr_accessor :instance include Puppet::Util end def execute_postrun_command execute_from_setting(:postrun_command) end def execute_prerun_command execute_from_setting(:prerun_command) end # Initialize and load storage def init_storage Puppet::Util::Storage.load @compile_time ||= Puppet::Util::Storage.cache(:configuration)[:compile_time] rescue => detail Puppet.log_exception(detail, "Removing corrupt state file #{Puppet[:statefile]}: #{detail}") begin ::File.unlink(Puppet[:statefile]) retry rescue => detail raise Puppet::Error.new("Cannot remove #{Puppet[:statefile]}: #{detail}") end end # Just so we can specify that we are "the" instance. def initialize Puppet.settings.use(:main, :ssl, :agent) self.class.instance = self @running = false @splayed = false @environment = Puppet[:environment] @transaction_uuid = SecureRandom.uuid end # Get the remote catalog, yo. Returns nil if no catalog can be found. def retrieve_catalog(query_options) query_options ||= {} # First try it with no cache, then with the cache. unless (Puppet[:use_cached_catalog] and result = retrieve_catalog_from_cache(query_options)) or result = retrieve_new_catalog(query_options) if ! Puppet[:usecacheonfailure] Puppet.warning "Not using cache on failed catalog" return nil end result = retrieve_catalog_from_cache(query_options) end return nil unless result convert_catalog(result, @duration) end # Convert a plain resource catalog into our full host catalog. def convert_catalog(result, duration) catalog = result.to_ral catalog.finalize catalog.retrieval_duration = duration catalog.write_class_file catalog.write_resource_file catalog end def get_facts(options) download_plugins if options[:pluginsync] if Puppet::Resource::Catalog.indirection.terminus_class == :rest # This is a bit complicated. We need the serialized and escaped facts, # and we need to know which format they're encoded in. Thus, we # get a hash with both of these pieces of information. # # facts_for_uploading may set Puppet[:node_name_value] as a side effect return facts_for_uploading end end def prepare_and_retrieve_catalog(options, query_options) # set report host name now that we have the fact options[:report].host = Puppet[:node_name_value] unless catalog = (options.delete(:catalog) || retrieve_catalog(query_options)) Puppet.err "Could not retrieve catalog; skipping run" return end catalog end # Retrieve (optionally) and apply a catalog. If a catalog is passed in # the options, then apply that one, otherwise retrieve it. def apply_catalog(catalog, options) report = options[:report] report.configuration_version = catalog.version report.transaction_uuid = @transaction_uuid report.environment = @environment benchmark(:notice, "Finished catalog run") do catalog.apply(options) end report.finalize_report report end def get_transaction_uuid { :transaction_uuid => @transaction_uuid } end # The code that actually runs the catalog. # This just passes any options on to the catalog, # which accepts :tags and :ignoreschedules. def run(options = {}) options[:report] ||= Puppet::Transaction::Report.new("apply") report = options[:report] init_storage Puppet::Util::Log.newdestination(report) begin unless Puppet[:node_name_fact].empty? query_options = get_facts(options) end begin if node = Puppet::Node.indirection.find(Puppet[:node_name_value], :environment => @environment, :ignore_cache => true) if node.environment.to_s != @environment Puppet.warning "Local environment: \"#{@environment}\" doesn't match server specified node environment \"#{node.environment}\", switching agent to \"#{node.environment}\"." @environment = node.environment.to_s query_options = nil end end rescue Puppet::Error, Net::HTTPError => detail Puppet.warning("Unable to fetch my node definition, but the agent run will continue:") Puppet.warning(detail) end query_options = get_facts(options) unless query_options # add the transaction uuid to the catalog query options hash query_options.merge! get_transaction_uuid if query_options unless catalog = prepare_and_retrieve_catalog(options, query_options) return nil end # Here we set the local environment based on what we get from the # catalog. Since a change in environment means a change in facts, and # facts may be used to determine which catalog we get, we need to # rerun the process if the environment is changed. tries = 0 while catalog.environment and not catalog.environment.empty? and catalog.environment != @environment if tries > 3 raise Puppet::Error, "Catalog environment didn't stabilize after #{tries} fetches, aborting run" end Puppet.warning "Local environment: \"#{@environment}\" doesn't match server specified environment \"#{catalog.environment}\", restarting agent run with environment \"#{catalog.environment}\"" @environment = catalog.environment return nil unless catalog = prepare_and_retrieve_catalog(options, query_options) tries += 1 end execute_prerun_command or return nil apply_catalog(catalog, options) report.exit_status rescue => detail Puppet.log_exception(detail, "Failed to apply catalog: #{detail}") return nil ensure execute_postrun_command or return nil end ensure # Between Puppet runs we need to forget the cached values. This lets us # pick up on new functions installed by gems or new modules being added # without the daemon being restarted. - Thread.current[:env_module_directories] = nil + $env_module_directories = nil Puppet::Util::Log.close(report) send_report(report) end def send_report(report) puts report.summary if Puppet[:summarize] save_last_run_summary(report) Puppet::Transaction::Report.indirection.save(report, nil, :environment => @environment) if Puppet[:report] rescue => detail Puppet.log_exception(detail, "Could not send report: #{detail}") end def save_last_run_summary(report) mode = Puppet.settings.setting(:lastrunfile).mode Puppet::Util.replace_file(Puppet[:lastrunfile], mode) do |fh| fh.print YAML.dump(report.raw_summary) end rescue => detail Puppet.log_exception(detail, "Could not save last run local report: #{detail}") end private def execute_from_setting(setting) return true if (command = Puppet[setting]) == "" begin Puppet::Util::Execution.execute([command]) true rescue => detail Puppet.log_exception(detail, "Could not run command from #{setting}: #{detail}") false end end def retrieve_catalog_from_cache(query_options) result = nil @duration = thinmark do result = Puppet::Resource::Catalog.indirection.find(Puppet[:node_name_value], query_options.merge(:ignore_terminus => true, :environment => @environment)) end Puppet.notice "Using cached catalog" result rescue => detail Puppet.log_exception(detail, "Could not retrieve catalog from cache: #{detail}") return nil end def retrieve_new_catalog(query_options) result = nil @duration = thinmark do result = Puppet::Resource::Catalog.indirection.find(Puppet[:node_name_value], query_options.merge(:ignore_cache => true, :environment => @environment)) end result rescue SystemExit,NoMemoryError raise rescue Exception => detail Puppet.log_exception(detail, "Could not retrieve catalog from remote server: #{detail}") return nil end end diff --git a/lib/puppet/daemon.rb b/lib/puppet/daemon.rb index 562602418..c931184f1 100755 --- a/lib/puppet/daemon.rb +++ b/lib/puppet/daemon.rb @@ -1,186 +1,182 @@ require 'puppet/application' require 'puppet/scheduler' # Run periodic actions and a network server in a daemonized process. # # A Daemon has 3 parts: # * config reparse # * (optional) an agent that responds to #run # * (optional) a server that response to #stop, #start, and #wait_for_shutdown # # The config reparse will occur periodically based on Settings. The server will # be started and is expected to manage its own run loop (and so not block the # start call). The server will, however, still be waited for by using the # #wait_for_shutdown method. The agent is run periodically and a time interval # based on Settings. The config reparse will update this time interval when # needed. # # The Daemon is also responsible for signal handling, starting, stopping, # running the agent on demand, and reloading the entire process. It ensures # that only one Daemon is running by using a lockfile. # # @api private class Puppet::Daemon attr_accessor :agent, :server, :argv def initialize(pidfile, scheduler = Puppet::Scheduler::Scheduler.new()) @scheduler = scheduler @pidfile = pidfile end def daemonname Puppet.run_mode.name end # Put the daemon into the background. def daemonize if pid = fork Process.detach(pid) exit(0) end create_pidfile # Get rid of console logging Puppet::Util::Log.close(:console) Process.setsid Dir.chdir("/") close_streams end # Close stdin/stdout/stderr so that we can finish our transition into 'daemon' mode. # @return nil def self.close_streams() Puppet.debug("Closing streams for daemon mode") begin $stdin.reopen "/dev/null" $stdout.reopen "/dev/null", "a" $stderr.reopen $stdout Puppet::Util::Log.reopen Puppet.debug("Finished closing streams for daemon mode") rescue => detail Puppet.err "Could not start #{Puppet.run_mode.name}: #{detail}" Puppet::Util::replace_file("/tmp/daemonout", 0644) do |f| f.puts "Could not start #{Puppet.run_mode.name}: #{detail}" end exit(12) end end # Convenience signature for calling Puppet::Daemon.close_streams def close_streams() Puppet::Daemon.close_streams end def reexec raise Puppet::DevError, "Cannot reexec unless ARGV arguments are set" unless argv command = $0 + " " + argv.join(" ") Puppet.notice "Restarting with '#{command}'" stop(:exit => false) exec(command) end def reload return unless agent if agent.running? Puppet.notice "Not triggering already-running agent" return end agent.run({:splay => false}) end def restart Puppet::Application.restart! reexec unless agent and agent.running? end def reopen_logs Puppet::Util::Log.reopen end # Trap a couple of the main signals. This should probably be handled # in a way that anyone else can register callbacks for traps, but, eh. def set_signal_traps signals = {:INT => :stop, :TERM => :stop } # extended signals not supported under windows signals.update({:HUP => :restart, :USR1 => :reload, :USR2 => :reopen_logs }) unless Puppet.features.microsoft_windows? signals.each do |signal, method| Signal.trap(signal) do Puppet.notice "Caught #{signal}; calling #{method}" send(method) end end end # Stop everything def stop(args = {:exit => true}) Puppet::Application.stop! server.stop if server remove_pidfile Puppet::Util::Log.close_all exit if args[:exit] end def start set_signal_traps create_pidfile raise Puppet::DevError, "Daemons must have an agent, server, or both" unless agent or server # Start the listening server, if required. server.start if server # Finally, loop forever running events - or, at least, until we exit. run_event_loop server.wait_for_shutdown if server end private # Create a pidfile for our daemon, so we can be stopped and others # don't try to start. def create_pidfile - Puppet::Util.synchronize_on(Puppet.run_mode.name,Sync::EX) do - raise "Could not create PID file: #{@pidfile.file_path}" unless @pidfile.lock - end + raise "Could not create PID file: #{@pidfile.file_path}" unless @pidfile.lock end # Remove the pid file for our daemon. def remove_pidfile - Puppet::Util.synchronize_on(Puppet.run_mode.name,Sync::EX) do - @pidfile.unlock - end + @pidfile.unlock end def run_event_loop agent_run = Puppet::Scheduler.create_job(Puppet[:runinterval], Puppet[:splay], Puppet[:splaylimit]) do # Splay for the daemon is handled in the scheduler agent.run(:splay => false) end reparse_run = Puppet::Scheduler.create_job(Puppet[:filetimeout]) do Puppet.settings.reparse_config_files agent_run.run_interval = Puppet[:runinterval] if Puppet[:filetimeout] == 0 reparse_run.disable else reparse_run.run_interval = Puppet[:filetimeout] end end reparse_run.disable if Puppet[:filetimeout] == 0 agent_run.disable unless agent @scheduler.run_loop([reparse_run, agent_run]) end end diff --git a/lib/puppet/file_serving/configuration.rb b/lib/puppet/file_serving/configuration.rb index d3ed15d61..70dc6a14f 100644 --- a/lib/puppet/file_serving/configuration.rb +++ b/lib/puppet/file_serving/configuration.rb @@ -1,111 +1,106 @@ -require 'monitor' require 'puppet' require 'puppet/file_serving' require 'puppet/file_serving/mount' require 'puppet/file_serving/mount/file' require 'puppet/file_serving/mount/modules' require 'puppet/file_serving/mount/plugins' class Puppet::FileServing::Configuration require 'puppet/file_serving/configuration/parser' - extend MonitorMixin - def self.configuration - synchronize do - @configuration ||= new - end + @configuration ||= new end Mount = Puppet::FileServing::Mount private_class_method :new attr_reader :mounts #private :mounts # Find the right mount. Does some shenanigans to support old-style module # mounts. def find_mount(mount_name, environment) # Reparse the configuration if necessary. readconfig # This can be nil. mounts[mount_name] end def initialize @mounts = {} @config_file = nil # We don't check to see if the file is modified the first time, # because we always want to parse at first. readconfig(false) end # Is a given mount available? def mounted?(name) @mounts.include?(name) end # Split the path into the separate mount point and path. def split_path(request) # Reparse the configuration if necessary. readconfig mount_name, path = request.key.split(File::Separator, 2) raise(ArgumentError, "Cannot find file: Invalid mount '#{mount_name}'") unless mount_name =~ %r{^[-\w]+$} raise(ArgumentError, "Cannot find file: Invalid relative path '#{path}'") if path and path.split('/').include?('..') return nil unless mount = find_mount(mount_name, request.environment) if mount.name == "modules" and mount_name != "modules" # yay backward-compatibility path = "#{mount_name}/#{path}" end if path == "" path = nil elsif path # Remove any double slashes that might have occurred path = path.gsub(/\/+/, "/") end return mount, path end def umount(name) @mounts.delete(name) if @mounts.include? name end private def mk_default_mounts @mounts["modules"] ||= Mount::Modules.new("modules") @mounts["modules"].allow('*') if @mounts["modules"].empty? @mounts["plugins"] ||= Mount::Plugins.new("plugins") @mounts["plugins"].allow('*') if @mounts["plugins"].empty? end # Read the configuration file. def readconfig(check = true) config = Puppet[:fileserverconfig] return unless FileTest.exists?(config) @parser ||= Puppet::FileServing::Configuration::Parser.new(config) return if check and ! @parser.changed? # Don't assign the mounts hash until we're sure the parsing succeeded. begin newmounts = @parser.parse @mounts = newmounts rescue => detail Puppet.log_exception(detail, "Error parsing fileserver configuration: #{detail}; using old configuration") end ensure # Make sure we've got our plugins and modules. mk_default_mounts end end diff --git a/lib/puppet/network/authconfig.rb b/lib/puppet/network/authconfig.rb index 1c3eaede6..527774598 100644 --- a/lib/puppet/network/authconfig.rb +++ b/lib/puppet/network/authconfig.rb @@ -1,75 +1,73 @@ require 'puppet/network/rights' module Puppet class ConfigurationError < Puppet::Error; end class Network::AuthConfig - - extend MonitorMixin attr_accessor :rights DEFAULT_ACL = [ { :acl => "~ ^\/catalog\/([^\/]+)$", :method => :find, :allow => '$1', :authenticated => true }, { :acl => "~ ^\/node\/([^\/]+)$", :method => :find, :allow => '$1', :authenticated => true }, # this one will allow all file access, and thus delegate # to fileserver.conf { :acl => "/file" }, { :acl => "/certificate_revocation_list/ca", :method => :find, :authenticated => true }, { :acl => "~ ^\/report\/([^\/]+)$", :method => :save, :allow => '$1', :authenticated => true }, # These allow `auth any`, because if you can do them anonymously you # should probably also be able to do them when trusted. { :acl => "/certificate/ca", :method => :find, :authenticated => :any }, { :acl => "/certificate/", :method => :find, :authenticated => :any }, { :acl => "/certificate_request", :method => [:find, :save], :authenticated => :any }, { :acl => "/status", :method => [:find], :authenticated => true }, ] # Just proxy the setting methods to our rights stuff [:allow, :deny].each do |method| define_method(method) do |*args| @rights.send(method, *args) end end # force regular ACLs to be present def insert_default_acl DEFAULT_ACL.each do |acl| unless rights[acl[:acl]] Puppet.info "Inserting default '#{acl[:acl]}' (auth #{acl[:authenticated]}) ACL" mk_acl(acl) end end # queue an empty (ie deny all) right for every other path # actually this is not strictly necessary as the rights system # denies not explicitely allowed paths unless rights["/"] rights.newright("/").restrict_authenticated(:any) end end def mk_acl(acl) right = @rights.newright(acl[:acl]) right.allow(acl[:allow] || "*") if method = acl[:method] method = [method] unless method.is_a?(Array) method.each { |m| right.restrict_method(m) } end right.restrict_authenticated(acl[:authenticated]) unless acl[:authenticated].nil? end # check whether this request is allowed in our ACL # raise an Puppet::Network::AuthorizedError if the request # is denied. def check_authorization(indirection, method, key, params) if authorization_failure_exception = @rights.is_request_forbidden_and_why?(indirection, method, key, params) Puppet.warning("Denying access: #{authorization_failure_exception}") raise authorization_failure_exception end end def initialize(rights=nil) @rights = rights || Puppet::Network::Rights.new insert_default_acl end end end diff --git a/lib/puppet/network/authstore.rb b/lib/puppet/network/authstore.rb index 90498002d..ba86ed90b 100755 --- a/lib/puppet/network/authstore.rb +++ b/lib/puppet/network/authstore.rb @@ -1,283 +1,282 @@ # standard module for determining whether a given hostname or IP has access to # the requested resource require 'ipaddr' require 'puppet/util/logging' module Puppet class AuthStoreError < Puppet::Error; end class AuthorizationError < Puppet::Error; end class Network::AuthStore include Puppet::Util::Logging # Is a given combination of name and ip address allowed? If either input # is non-nil, then both inputs must be provided. If neither input # is provided, then the authstore is considered local and defaults to "true". def allowed?(name, ip) if name or ip # This is probably unnecessary, and can cause some weirdnesses in # cases where we're operating over localhost but don't have a real # IP defined. raise Puppet::DevError, "Name and IP must be passed to 'allowed?'" unless name and ip # else, we're networked and such else # we're local return true end # yay insecure overrides return true if globalallow? if decl = declarations.find { |d| d.match?(name, ip) } return decl.result end info "defaulting to no access for #{name}" false end # Mark a given pattern as allowed. def allow(pattern) # a simple way to allow anyone at all to connect if pattern == "*" @globalallow = true else store(:allow, pattern) end nil end def allow_ip(pattern) store(:allow_ip, pattern) end # Deny a given pattern. def deny(pattern) store(:deny, pattern) end def deny_ip(pattern) store(:deny_ip, pattern) end # Is global allow enabled? def globalallow? @globalallow end # does this auth store has any rules? def empty? @globalallow.nil? && @declarations.size == 0 end def initialize @globalallow = nil @declarations = [] end def to_s "authstore" end def interpolate(match) - Thread.current[:declarations] = @declarations.collect { |ace| ace.interpolate(match) }.sort + @modified_declarations = @declarations.collect { |ace| ace.interpolate(match) }.sort end def reset_interpolation - Thread.current[:declarations] = nil + @modified_declarations = nil end private - # returns our ACEs list, but if we have a modification of it - # in our current thread, let's return it - # this is used if we want to override the this purely immutable list - # by a modified version in a multithread safe way. + # Returns our ACEs list, but if we have a modification of it, let's return + # it. This is used if we want to override the this purely immutable list + # by a modified version. def declarations - Thread.current[:declarations] || @declarations + @modified_declarations || @declarations end # Store the results of a pattern into our hash. Basically just # converts the pattern and sticks it into the hash. def store(type, pattern) @declarations << Declaration.new(type, pattern) @declarations.sort! nil end # A single declaration. Stores the info for a given declaration, # provides the methods for determining whether a declaration matches, # and handles sorting the declarations appropriately. class Declaration include Puppet::Util include Comparable # The type of declaration: either :allow or :deny attr_reader :type VALID_TYPES = [ :allow, :deny, :allow_ip, :deny_ip ] attr_accessor :name # The pattern we're matching against. Can be an IPAddr instance, # or an array of strings, resulting from reversing a hostname # or domain name. attr_reader :pattern # The length. Only used for iprange and domain. attr_accessor :length # Sort the declarations most specific first. def <=>(other) compare(exact?, other.exact?) || compare(ip?, other.ip?) || ((length != other.length) && (other.length <=> length)) || compare(deny?, other.deny?) || ( ip? ? pattern.to_s <=> other.pattern.to_s : pattern <=> other.pattern) end def deny? type == :deny end def exact? @exact == :exact end def initialize(type, pattern) self.type = type self.pattern = pattern end # Are we an IP type? def ip? name == :ip end # Does this declaration match the name/ip combo? def match?(name, ip) if ip? pattern.include?(IPAddr.new(ip)) else matchname?(name) end end # Set the pattern appropriately. Also sets the name and length. def pattern=(pattern) if [:allow_ip, :deny_ip].include?(self.type) parse_ip(pattern) else parse(pattern) end @orig = pattern end # Mapping a type of statement into a return value. def result [:allow, :allow_ip].include?(type) end def to_s "#{type}: #{pattern}" end # Set the declaration type. Either :allow or :deny. def type=(type) type = type.intern raise ArgumentError, "Invalid declaration type #{type}" unless VALID_TYPES.include?(type) @type = type end # interpolate a pattern to replace any # backreferences by the given match # for instance if our pattern is $1.reductivelabs.com # and we're called with a MatchData whose capture 1 is puppet # we'll return a pattern of puppet.reductivelabs.com def interpolate(match) clone = dup if @name == :dynamic clone.pattern = clone.pattern.reverse.collect do |p| p.gsub(/\$(\d)/) { |m| match[$1.to_i] } end.join(".") end clone end private # Returns nil if both values are true or both are false, returns # -1 if the first is true, and 1 if the second is true. Used # in the <=> operator. def compare(me, them) (me and them) ? nil : me ? -1 : them ? 1 : nil end # Does the name match our pattern? def matchname?(name) case @name when :domain, :dynamic, :opaque name = munge_name(name) (pattern == name) or (not exact? and pattern.zip(name).all? { |p,n| p == n }) when :regex Regexp.new(pattern.slice(1..-2)).match(name) end end # Convert the name to a common pattern. def munge_name(name) # Change to name.downcase.split(".",-1).reverse for FQDN support name.downcase.split(".").reverse end # Parse our input pattern and figure out what kind of allowable # statement it is. The output of this is used for later matching. Octet = '(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])' IPv4 = "#{Octet}\.#{Octet}\.#{Octet}\.#{Octet}" IPv6_full = "_:_:_:_:_:_:_:_|_:_:_:_:_:_::_?|_:_:_:_:_::((_:)?_)?|_:_:_:_::((_:){0,2}_)?|_:_:_::((_:){0,3}_)?|_:_::((_:){0,4}_)?|_::((_:){0,5}_)?|::((_:){0,6}_)?" IPv6_partial = "_:_:_:_:_:_:|_:_:_:_::(_:)?|_:_::(_:){0,2}|_::(_:){0,3}" # It should be: # IP = "#{IPv4}|#{IPv6_full}|(#{IPv6_partial}#{IPv4})".gsub(/_/,'([0-9a-fA-F]{1,4})').gsub(/\(/,'(?:') # but ruby's ipaddr lib doesn't support the hybrid format IP = "#{IPv4}|#{IPv6_full}".gsub(/_/,'([0-9a-fA-F]{1,4})').gsub(/\(/,'(?:') def parse_ip(value) @name = :ip @exact, @length, @pattern = *case value when /^(?:#{IP})\/(\d+)$/ # 12.34.56.78/24, a001:b002::efff/120, c444:1000:2000::9:192.168.0.1/112 [:inexact, $1.to_i, IPAddr.new(value)] when /^(#{IP})$/ # 10.20.30.40, [:exact, nil, IPAddr.new(value)] when /^(#{Octet}\.){1,3}\*$/ # an ip address with a '*' at the end segments = value.split(".")[0..-2] bits = 8*segments.length [:inexact, bits, IPAddr.new((segments+[0,0,0])[0,4].join(".") + "/#{bits}")] else raise AuthStoreError, "Invalid IP pattern #{value}" end end def parse(value) @name,@exact,@length,@pattern = *case value when /^(\w[-\w]*\.)+[-\w]+$/ # a full hostname # Change to /^(\w[-\w]*\.)+[-\w]+\.?$/ for FQDN support [:domain,:exact,nil,munge_name(value)] when /^\*(\.(\w[-\w]*)){1,}$/ # *.domain.com host_sans_star = munge_name(value)[0..-2] [:domain,:inexact,host_sans_star.length,host_sans_star] when /\$\d+/ # a backreference pattern ala $1.reductivelabs.com or 192.168.0.$1 or $1.$2 [:dynamic,:exact,nil,munge_name(value)] when /^\w[-.@\w]*$/ # ? Just like a host name but allow '@'s and ending '.'s [:opaque,:exact,nil,[value]] when /^\/.*\/$/ # a regular expression [:regex,:inexact,nil,value] else raise AuthStoreError, "Invalid pattern #{value}" end end end end end diff --git a/lib/puppet/network/http/webrick.rb b/lib/puppet/network/http/webrick.rb index 123b493c0..869900dad 100644 --- a/lib/puppet/network/http/webrick.rb +++ b/lib/puppet/network/http/webrick.rb @@ -1,123 +1,116 @@ require 'webrick' require 'webrick/https' require 'puppet/network/http/webrick/rest' require 'thread' require 'puppet/ssl/certificate' require 'puppet/ssl/certificate_revocation_list' require 'puppet/ssl/configuration' class Puppet::Network::HTTP::WEBrick def initialize @listening = false - @mutex = Mutex.new end def listen(address, port) arguments = {:BindAddress => address, :Port => port, :DoNotReverseLookup => true} arguments.merge!(setup_logger) arguments.merge!(setup_ssl) BasicSocket.do_not_reverse_lookup = true @server = WEBrick::HTTPServer.new(arguments) @server.listeners.each { |l| l.start_immediately = false } @server.mount('/', Puppet::Network::HTTP::WEBrickREST, :this_value_is_apparently_necessary_but_unused) - @mutex.synchronize do - raise "WEBrick server is already listening" if @listening - @listening = true - @thread = Thread.new { - @server.start { |sock| - raise "Client disconnected before connection could be established" unless IO.select([sock],nil,nil,6.2) - sock.accept - @server.run(sock) - } - } - sleep 0.1 until @server.status == :Running + raise "WEBrick server is already listening" if @listening + @listening = true + @thread = Thread.new do + @server.start do |sock| + raise "Client disconnected before connection could be established" unless IO.select([sock],nil,nil,6.2) + sock.accept + @server.run(sock) + end end + sleep 0.1 until @server.status == :Running end def unlisten - @mutex.synchronize do - raise "WEBrick server is not listening" unless @listening - @server.shutdown - wait_for_shutdown - @server = nil - @listening = false - end + raise "WEBrick server is not listening" unless @listening + @server.shutdown + wait_for_shutdown + @server = nil + @listening = false end def listening? - @mutex.synchronize do - @listening - end + @listening end def wait_for_shutdown @thread.join end # Configure our http log file. def setup_logger # Make sure the settings are all ready for us. Puppet.settings.use(:main, :ssl, :application) if Puppet.run_mode.master? file = Puppet[:masterhttplog] else file = Puppet[:httplog] end # open the log manually to prevent file descriptor leak file_io = ::File.open(file, "a+") file_io.sync = true if defined?(Fcntl::FD_CLOEXEC) file_io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) end args = [file_io] args << WEBrick::Log::DEBUG if Puppet::Util::Log.level == :debug logger = WEBrick::Log.new(*args) return :Logger => logger, :AccessLog => [ [logger, WEBrick::AccessLog::COMMON_LOG_FORMAT ], [logger, WEBrick::AccessLog::REFERER_LOG_FORMAT ] ] end # Add all of the ssl cert information. def setup_ssl results = {} # Get the cached copy. We know it's been generated, too. host = Puppet::SSL::Host.localhost raise Puppet::Error, "Could not retrieve certificate for #{host.name} and not running on a valid certificate authority" unless host.certificate results[:SSLPrivateKey] = host.key.content results[:SSLCertificate] = host.certificate.content results[:SSLStartImmediately] = true results[:SSLEnable] = true results[:SSLOptions] = OpenSSL::SSL::OP_NO_SSLv2 raise Puppet::Error, "Could not find CA certificate" unless Puppet::SSL::Certificate.indirection.find(Puppet::SSL::CA_NAME) results[:SSLCACertificateFile] = ssl_configuration.ca_auth_file results[:SSLVerifyClient] = OpenSSL::SSL::VERIFY_PEER results[:SSLCertificateStore] = host.ssl_store results end private def ssl_configuration @ssl_configuration ||= Puppet::SSL::Configuration.new( Puppet[:localcacert], :ca_chain_file => Puppet[:ssl_server_ca_chain], :ca_auth_file => Puppet[:ssl_server_ca_auth]) end end diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 25b454638..f79b8b35b 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -1,489 +1,487 @@ require 'puppet/util' require 'puppet/util/cacher' require 'monitor' require 'puppet/parser/parser_factory' # Just define it, so this class has fewer load dependencies. class Puppet::Node end # Puppet::Node::Environment acts as a container for all configuration # that is expected to vary between environments. # -# ## Thread local variables +# ## Global variables # -# The Puppet::Node::Environment uses a number of `Thread.current` variables. -# Since all web servers that Puppet runs on are single threaded these -# variables are effectively global. +# The Puppet::Node::Environment uses a number of global variables. # -# ### `Thread.current[:environment]` +# ### `$environment` # -# The 'environment' thread variable represents the current environment that's +# The 'environment' global variable represents the current environment that's # being used in the compiler. # -# ### `Thread.current[:known_resource_types]` +# ### `$known_resource_types` # -# The 'known_resource_types' thread variable represents a singleton instance +# The 'known_resource_types' global variable represents a singleton instance # of the Puppet::Resource::TypeCollection class. The variable is discarded # and regenerated if it is accessed by an environment that doesn't match the # environment of the 'known_resource_types' # # This behavior of discarding the known_resource_types every time the # environment changes is not ideal. In the best case this can cause valid data # to be discarded and reloaded. If Puppet is being used with numerous # environments then this penalty will be repeatedly incurred. # # In the worst case (#15106) demonstrates that if a different environment is # accessed during catalog compilation, for whatever reason, the # known_resource_types can be discarded which loses information that cannot # be recovered and can cause a catalog compilation to completely fail. # # ## The root environment # # In addition to normal environments that are defined by the user,there is a # special 'root' environment. It is defined as an instance variable on the # Puppet::Node::Environment metaclass. The environment name is `*root*` and can # be accessed by calling {Puppet::Node::Environment.root}. # # The primary purpose of the root environment is to contain parser functions # that are not bound to a specific environment. The main case for this is for # logging functions. Logging functions are attached to the 'root' environment # when {Puppet::Parser::Functions.reset} is called. # # The root environment is also used as a fallback environment when the # current environment has been requested by {Puppet::Node::Environment.current} # requested and no environment was set by {Puppet::Node::Environment.current=} class Puppet::Node::Environment # This defines a mixin for classes that have an environment. It implements # `environment` and `environment=` that respects the semantics of the # Puppet::Node::Environment class # # @api public module Helper def environment Puppet::Node::Environment.new(@environment) end def environment=(env) if env.is_a?(String) or env.is_a?(Symbol) @environment = env else @environment = env.name end end end include Puppet::Util::Cacher # @!attribute seen # @scope class # @api private # @return [Hash] All memoized environments @seen = {} # Create a new environment with the given name, or return an existing one # # The environment class memoizes instances so that attempts to instantiate an # environment with the same name with an existing environment will return the # existing environment. # # @overload self.new(environment) # @param environment [Puppet::Node::Environment] # @return [Puppet::Node::Environment] the environment passed as the param, # this is implemented so that a calling class can use strings or # environments interchangeably. # # @overload self.new(string) # @param string [String, Symbol] # @return [Puppet::Node::Environment] An existing environment if it exists, # else a new environment with that name # # @overload self.new() # @return [Puppet::Node::Environment] The environment as set by # Puppet.settings[:environment] # # @api public def self.new(name = nil) return name if name.is_a?(self) name ||= Puppet.settings.value(:environment) raise ArgumentError, "Environment name must be specified" unless name symbol = name.to_sym return @seen[symbol] if @seen[symbol] obj = self.allocate obj.send :initialize, symbol @seen[symbol] = obj end # Retrieve the environment for the current thread # # @note This should only used when a catalog is being compiled. # # @api private # # @return [Puppet::Node::Environment] the currently set environment if one # has been explicitly set, else it will return the '*root*' environment def self.current - Thread.current[:environment] || root + $environment || root end # Set the environment for the current thread # # @note This should only set when a catalog is being compiled. Under normal # This value is initially set in {Puppet::Parser::Compiler#environment} # # @note Setting this affects global state during catalog compilation, and # changing the current environment during compilation can cause unexpected # and generally very bad behaviors. # # @api private # # @param env [Puppet::Node::Environment] def self.current=(env) - Thread.current[:environment] = new(env) + $environment = new(env) end # @return [Puppet::Node::Environment] The `*root*` environment. # # This is only used for handling functions that are not attached to a # specific environment. # # @api private def self.root @root end # Clear all memoized environments and the 'current' environment # # @api private def self.clear @seen.clear - Thread.current[:environment] = nil + $environment = nil end # @!attribute [r] name # @api public # @return [Symbol] the human readable environment name that serves as the # environment identifier attr_reader :name # Return an environment-specific Puppet setting. # # @api public # # @param param [String, Symbol] The environment setting to look up # @return [Object] The resolved setting value def [](param) Puppet.settings.value(param, self.name) end # Instantiate a new environment # # @note {Puppet::Node::Environment.new} is overridden to return memoized # objects, so this will not be invoked with the normal Ruby initialization # semantics. # # @param name [Symbol] The environment name def initialize(name) @name = name - extend MonitorMixin end # The current global TypeCollection # # @note The environment is loosely coupled with the {Puppet::Resource::TypeCollection} # class. While there is a 1:1 relationship between an environment and a - # TypeCollection instance, there is only one TypeCollection instance available - # at any given time. It is stored in the Thread.current collection as - # 'known_resource_types'. 'known_resource_types' is accessed as an instance - # method, but is global to all environment variables. + # TypeCollection instance, there is only one TypeCollection instance + # available at any given time. It is stored in `$known_resource_types`. + # `$known_resource_types` is accessed as an instance method, but is global + # to all environment variables. # # @api public # @return [Puppet::Resource::TypeCollection] The current global TypeCollection def known_resource_types # This makes use of short circuit evaluation to get the right thread-safe # per environment semantics with an efficient most common cases; we almost # always just return our thread's known-resource types. Only at the start # of a compilation (after our thread var has been set to nil) or when the # environment has changed do we delve deeper. - Thread.current[:known_resource_types] = nil if (krt = Thread.current[:known_resource_types]) && krt.environment != self - Thread.current[:known_resource_types] ||= synchronize { + $known_resource_types = nil if $known_resource_types && $known_resource_types.environment != self + $known_resource_types ||= if @known_resource_types.nil? or @known_resource_types.require_reparse? @known_resource_types = Puppet::Resource::TypeCollection.new(self) @known_resource_types.import_ast(perform_initial_import, '') + @known_resource_types + else + @known_resource_types end - @known_resource_types - } end # Yields each modules' plugin directory if the plugin directory (modulename/lib) # is present on the filesystem. # # @yield [String] Yields the plugin directory from each module to the block. # @api public def each_plugin_directory(&block) modules.map(&:plugin_directory).each do |lib| lib = Puppet::Util::Autoload.cleanpath(lib) yield lib if File.directory?(lib) end end # Locate a module instance by the module name alone. # # @api public # # @param name [String] The module name # @return [Puppet::Module, nil] The module if found, else nil def module(name) modules.find {|mod| mod.name == name} end # Locate a module instance by the full forge name (EG authorname/module) # # @api public # # @param forge_name [String] The module name # @return [Puppet::Module, nil] The module if found, else nil def module_by_forge_name(forge_name) author, modname = forge_name.split('/') found_mod = self.module(modname) found_mod and found_mod.forge_name == forge_name ? found_mod : nil end # @!attribute [r] modulepath # Return all existent directories in the modulepath for this environment # @note This value is cached so that the filesystem doesn't have to be # re-enumerated every time this method is invoked, since that # enumeration could be a costly operation and this method is called # frequently. The cache expiry is determined by `Puppet[:filetimeout]`. # @see Puppet::Util::Cacher.cached_attr # @api public # @return [Array] All directories present in the modulepath cached_attr(:modulepath, Puppet[:filetimeout]) do dirs = self[:modulepath].split(File::PATH_SEPARATOR) dirs = ENV["PUPPETLIB"].split(File::PATH_SEPARATOR) + dirs if ENV["PUPPETLIB"] validate_dirs(dirs) end # @!attribute [r] modules # Return all modules for this environment in the order they appear in the # modulepath. # @note If multiple modules with the same name are present they will # both be added, but methods like {#module} and {#module_by_forge_name} # will return the first matching entry in this list. # @note This value is cached so that the filesystem doesn't have to be # re-enumerated every time this method is invoked, since that # enumeration could be a costly operation and this method is called # frequently. The cache expiry is determined by `Puppet[:filetimeout]`. # @see Puppet::Util::Cacher.cached_attr # @api public # @return [Array] All modules for this environment cached_attr(:modules, Puppet[:filetimeout]) do module_references = [] seen_modules = {} modulepath.each do |path| Dir.entries(path).each do |name| warn_about_mistaken_path(path, name) next if module_references.include?(name) if not seen_modules[name] module_references << {:name => name, :path => File.join(path, name)} seen_modules[name] = true end end end module_references.collect do |reference| begin Puppet::Module.new(reference[:name], reference[:path], self) rescue Puppet::Module::Error nil end end.compact end # Generate a warning if the given directory in a module path entry is named `lib`. # # @api private # # @param path [String] The module directory containing the given directory # @param name [String] The directory name def warn_about_mistaken_path(path, name) if name == "lib" Puppet.debug("Warning: Found directory named 'lib' in module path ('#{path}/lib'); unless " + "you are expecting to load a module named 'lib', your module path may be set " + "incorrectly.") end end # Modules broken out by directory in the modulepath # # @note This method _changes_ the current working directory while enumerating # the modules. This seems rather dangerous. # # @api public # # @return [Hash>] A hash whose keys are file # paths, and whose values is an array of Puppet Modules for that path def modules_by_path modules_by_path = {} modulepath.each do |path| Dir.chdir(path) do module_names = Dir.glob('*').select do |d| FileTest.directory?(d) && (File.basename(d) =~ /\A\w+(-\w+)*\Z/) end modules_by_path[path] = module_names.sort.map do |name| Puppet::Module.new(name, File.join(path, name), self) end end end modules_by_path end # All module requirements for all modules in the environment modulepath # # @api public # # @comment This has nothing to do with an environment. It seems like it was # stuffed into the first convenient class that vaguely involved modules. # # @example # environment.module_requirements # # => { # # 'username/amodule' => [ # # { # # 'name' => 'username/moduledep', # # 'version' => '1.2.3', # # 'version_requirement' => '>= 1.0.0', # # }, # # { # # 'name' => 'username/anotherdep', # # 'version' => '4.5.6', # # 'version_requirement' => '>= 3.0.0', # # } # # ] # # } # # # # @return [Hash>>] See the method example # for an explanation of the return value. def module_requirements deps = {} modules.each do |mod| next unless mod.forge_name deps[mod.forge_name] ||= [] mod.dependencies and mod.dependencies.each do |mod_dep| deps[mod_dep['name']] ||= [] dep_details = { 'name' => mod.forge_name, 'version' => mod.version, 'version_requirement' => mod_dep['version_requirement'] } deps[mod_dep['name']] << dep_details end end deps.each do |mod, mod_deps| deps[mod] = mod_deps.sort_by {|d| d['name']} end deps end # @return [String] The stringified value of the `name` instance variable # @api public def to_s name.to_s end # @return [Symbol] The `name` value, cast to a string, then cast to a symbol. # # @api public # # @note the `name` instance variable is a Symbol, but this casts the value # to a String and then converts it back into a Symbol which will needlessly # create an object that needs to be garbage collected def to_sym to_s.to_sym end # Return only the environment name when serializing. # # The only thing we care about when serializing an environment is its # identity; everything else is ephemeral and should not be stored or # transmitted. # # @api public def to_zaml(z) self.to_s.to_zaml(z) end # Validate a list of file paths and return the paths that are directories on the filesystem # # @api private # # @param dirs [Array] The file paths to validate # @return [Array] All file paths that exist and are directories def validate_dirs(dirs) dirs.collect do |dir| File.expand_path(dir) end.find_all do |p| FileTest.directory?(p) end end private # Reparse the manifests for the given environment # # There are two sources that can be used for the initial parse: # # 1. The value of `Puppet.settings[:code]`: Puppet can take a string from # its settings and parse that as a manifest. This is used by various # Puppet applications to read in a manifest and pass it to the # environment as a side effect. This is attempted first. # 2. The contents of `Puppet.settings[:manifest]`: Puppet will try to load # the environment manifest. By default this is `$manifestdir/site.pp` # # @note This method will return an empty hostclass if # `Puppet.settings[:ignoreimport]` is set to true. # # @return [Puppet::Parser::AST::Hostclass] The AST hostclass object # representing the 'main' hostclass def perform_initial_import return empty_parse_result if Puppet.settings[:ignoreimport] # parser = Puppet::Parser::Parser.new(self) parser = Puppet::Parser::ParserFactory.parser(self) if code = Puppet.settings.uninterpolated_value(:code, name.to_s) and code != "" parser.string = code else file = Puppet.settings.value(:manifest, name.to_s) parser.file = file end parser.parse rescue => detail known_resource_types.parse_failed = true msg = "Could not parse for environment #{self}: #{detail}" error = Puppet::Error.new(msg) error.set_backtrace(detail.backtrace) raise error end # Return an empty toplevel hostclass to indicate that no file was loaded # # This is used as the return value of {#perform_initial_import} when # `Puppet.settings[:ignoreimport]` is true. # # @return [Puppet::Parser::AST::Hostclass] def empty_parse_result return Puppet::Parser::AST::Hostclass.new('') end @root = new(:'*root*') end diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 20f668504..968a37e2e 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -1,542 +1,537 @@ require 'forwardable' require 'puppet/node' require 'puppet/resource/catalog' require 'puppet/util/errors' require 'puppet/resource/type_collection_helper' # Maintain a graph of scopes, along with a bunch of data # about the individual catalog we're compiling. class Puppet::Parser::Compiler extend Forwardable include Puppet::Util include Puppet::Util::Errors include Puppet::Util::MethodHelper include Puppet::Resource::TypeCollectionHelper def self.compile(node) - # We get these from the environment and only cache them in a thread - # variable for the duration of the compilation. If nothing else is using - # the thread, though, we can leave 'em hanging round with no ill effects, - # and this is safer than cleaning them at the end and assuming that will - # stick until the next entry to this function. - Thread.current[:known_resource_types] = nil - Thread.current[:env_module_directories] = nil + $known_resource_types = nil + $env_module_directories = nil # ...and we actually do the compile now we have caching ready. new(node).compile.to_resource rescue => detail message = "#{detail} on node #{node.name}" Puppet.log_exception(detail, message) raise Puppet::Error, message, detail.backtrace end attr_reader :node, :facts, :collections, :catalog, :resources, :relationships, :topscope # The injector that provides lookup services, or nil if accessed before the compiler has started compiling and # bootstrapped. The injector is initialized and available before any manifests are evaluated. # # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services for this compiler/environment # @api public # attr_accessor :injector # The injector that provides lookup services during the creation of the {#injector}. # @return [Puppet::Pops::Binder::Injector, nil] The injector that provides lookup services during injector creation # for this compiler/environment # # @api private # attr_accessor :boot_injector # Add a collection to the global list. def_delegator :@collections, :<<, :add_collection def_delegator :@relationships, :<<, :add_relationship # Store a resource override. def add_override(override) # If possible, merge the override in immediately. if resource = @catalog.resource(override.ref) resource.merge(override) else # Otherwise, store the override for later; these # get evaluated in Resource#finish. @resource_overrides[override.ref] << override end end def add_resource(scope, resource) @resources << resource # Note that this will fail if the resource is not unique. @catalog.add_resource(resource) if not resource.class? and resource[:stage] raise ArgumentError, "Only classes can set 'stage'; normal resources like #{resource} cannot change run stage" end # Stages should not be inside of classes. They are always a # top-level container, regardless of where they appear in the # manifest. return if resource.stage? # This adds a resource to the class it lexically appears in in the # manifest. unless resource.class? return @catalog.add_edge(scope.resource, resource) end end # Do we use nodes found in the code, vs. the external node sources? def_delegator :known_resource_types, :nodes?, :ast_nodes? # Store the fact that we've evaluated a class def add_class(name) @catalog.add_class(name) unless name == "" end # Return a list of all of the defined classes. def_delegator :@catalog, :classes, :classlist # Compiler our catalog. This mostly revolves around finding and evaluating classes. # This is the main entry into our catalog. def compile # Set the client's parameters into the top scope. Puppet::Util::Profiler.profile("Compile: Set node parameters") { set_node_parameters } Puppet::Util::Profiler.profile("Compile: Created settings scope") { create_settings_scope } if is_binder_active? Puppet::Util::Profiler.profile("Compile: Created injector") { create_injector } end Puppet::Util::Profiler.profile("Compile: Evaluated main") { evaluate_main } Puppet::Util::Profiler.profile("Compile: Evaluated AST node") { evaluate_ast_node } Puppet::Util::Profiler.profile("Compile: Evaluated node classes") { evaluate_node_classes } Puppet::Util::Profiler.profile("Compile: Evaluated generators") { evaluate_generators } Puppet::Util::Profiler.profile("Compile: Finished catalog") { finish } fail_on_unevaluated @catalog end def_delegator :@collections, :delete, :delete_collection # Return the node's environment. def environment unless defined?(@environment) unless node.environment.is_a? Puppet::Node::Environment raise Puppet::DevError, "node #{node} has an invalid environment!" end @environment = node.environment end Puppet::Node::Environment.current = @environment @environment end # Evaluate all of the classes specified by the node. def evaluate_node_classes evaluate_classes(@node.classes, @node_scope || topscope) end # Evaluate each specified class in turn. If there are any classes we can't # find, raise an error. This method really just creates resource objects # that point back to the classes, and then the resources are themselves # evaluated later in the process. # # Sometimes we evaluate classes with a fully qualified name already, in which # case, we tell scope.find_hostclass we've pre-qualified the name so it # doesn't need to search its namespaces again. This gets around a weird # edge case of duplicate class names, one at top scope and one nested in our # namespace and the wrong one (or both!) getting selected. See ticket #13349 # for more detail. --jeffweiss 26 apr 2012 def evaluate_classes(classes, scope, lazy_evaluate = true, fqname = false) raise Puppet::DevError, "No source for scope passed to evaluate_classes" unless scope.source class_parameters = nil # if we are a param class, save the classes hash # and transform classes to be the keys if classes.class == Hash class_parameters = classes classes = classes.keys end classes.each do |name| # If we can find the class, then make a resource that will evaluate it. if klass = scope.find_hostclass(name, :assume_fqname => fqname) # If parameters are passed, then attempt to create a duplicate resource # so the appropriate error is thrown. if class_parameters resource = klass.ensure_in_catalog(scope, class_parameters[name] || {}) else next if scope.class_scope(klass) resource = klass.ensure_in_catalog(scope) end # If they've disabled lazy evaluation (which the :include function does), # then evaluate our resource immediately. resource.evaluate unless lazy_evaluate else raise Puppet::Error, "Could not find class #{name} for #{node.name}" end end end def evaluate_relationships @relationships.each { |rel| rel.evaluate(catalog) } end # Return a resource by either its ref or its type and title. def_delegator :@catalog, :resource, :findresource def initialize(node, options = {}) @node = node set_options(options) initvars end # Create a new scope, with either a specified parent scope or # using the top scope. def newscope(parent, options = {}) parent ||= topscope scope = Puppet::Parser::Scope.new(self, options) scope.parent = parent scope end # Return any overrides for the given resource. def resource_overrides(resource) @resource_overrides[resource.ref] end def injector create_injector if @injector.nil? @injector end def boot_injector create_boot_injector(nil) if @boot_injector.nil? @boot_injector end # Creates the boot injector from registered system, default, and injector config. # @return [Puppet::Pops::Binder::Injector] the created boot injector # @api private Cannot be 'private' since it is called from the BindingsComposer. # def create_boot_injector(env_boot_bindings) assert_binder_active() boot_contribution = Puppet::Pops::Binder::SystemBindings.injector_boot_contribution(env_boot_bindings) final_contribution = Puppet::Pops::Binder::SystemBindings.final_contribution binder = Puppet::Pops::Binder::Binder.new() binder.define_categories(boot_contribution.effective_categories) binder.define_layers(Puppet::Pops::Binder::BindingsFactory.layered_bindings(final_contribution, boot_contribution)) @boot_injector = Puppet::Pops::Binder::Injector.new(binder) end # Answers if Puppet Binder should be active or not, and if it should and is not active, then it is activated. # @return [Boolean] true if the Puppet Binder should be activated def is_binder_active? should_be_active = Puppet[:binder] || Puppet[:parser] == 'future' if should_be_active # TODO: this should be in a central place, not just for ParserFactory anymore... Puppet::Parser::ParserFactory.assert_rgen_installed() @@binder_loaded ||= false unless @@binder_loaded require 'puppet/pops' require 'puppetx' @@binder_loaded = true end end should_be_active end private # If ast nodes are enabled, then see if we can find and evaluate one. def evaluate_ast_node return unless ast_nodes? # Now see if we can find the node. astnode = nil @node.names.each do |name| break if astnode = known_resource_types.node(name.to_s.downcase) end unless (astnode ||= known_resource_types.node("default")) raise Puppet::ParseError, "Could not find default node or by name with '#{node.names.join(", ")}'" end # Create a resource to model this node, and then add it to the list # of resources. resource = astnode.ensure_in_catalog(topscope) resource.evaluate @node_scope = topscope.class_scope(astnode) end # Evaluate our collections and return true if anything returned an object. # The 'true' is used to continue a loop, so it's important. def evaluate_collections return false if @collections.empty? exceptwrap do # We have to iterate over a dup of the array because # collections can delete themselves from the list, which # changes its length and causes some collections to get missed. Puppet::Util::Profiler.profile("Evaluated collections") do found_something = false @collections.dup.each do |collection| found_something = true if collection.evaluate end found_something end end end # Make sure all of our resources have been evaluated into native resources. # We return true if any resources have, so that we know to continue the # evaluate_generators loop. def evaluate_definitions exceptwrap do Puppet::Util::Profiler.profile("Evaluated definitions") do !unevaluated_resources.each do |resource| Puppet::Util::Profiler.profile("Evaluated resource #{resource}") do resource.evaluate end end.empty? end end end # Iterate over collections and resources until we're sure that the whole # compile is evaluated. This is necessary because both collections # and defined resources can generate new resources, which themselves could # be defined resources. def evaluate_generators count = 0 loop do done = true Puppet::Util::Profiler.profile("Iterated (#{count + 1}) on generators") do # Call collections first, then definitions. done = false if evaluate_collections done = false if evaluate_definitions end break if done count += 1 if count > 1000 raise Puppet::ParseError, "Somehow looped more than 1000 times while evaluating host catalog" end end end # Find and evaluate our main object, if possible. def evaluate_main @main = known_resource_types.find_hostclass([""], "") || known_resource_types.add(Puppet::Resource::Type.new(:hostclass, "")) @topscope.source = @main @main_resource = Puppet::Parser::Resource.new("class", :main, :scope => @topscope, :source => @main) @topscope.resource = @main_resource add_resource(@topscope, @main_resource) @main_resource.evaluate end # Make sure the entire catalog is evaluated. def fail_on_unevaluated fail_on_unevaluated_overrides fail_on_unevaluated_resource_collections end # If there are any resource overrides remaining, then we could # not find the resource they were supposed to override, so we # want to throw an exception. def fail_on_unevaluated_overrides remaining = @resource_overrides.values.flatten.collect(&:ref) if !remaining.empty? fail Puppet::ParseError, "Could not find resource(s) #{remaining.join(', ')} for overriding" end end # Make sure we don't have any remaining collections that specifically # look for resources, because we want to consider those to be # parse errors. def fail_on_unevaluated_resource_collections remaining = @collections.collect(&:resources).flatten.compact if !remaining.empty? raise Puppet::ParseError, "Failed to realize virtual resources #{remaining.join(', ')}" end end # Make sure all of our resources and such have done any last work # necessary. def finish evaluate_relationships resources.each do |resource| # Add in any resource overrides. if overrides = resource_overrides(resource) overrides.each do |over| resource.merge(over) end # Remove the overrides, so that the configuration knows there # are none left. overrides.clear end resource.finish if resource.respond_to?(:finish) end add_resource_metaparams end def add_resource_metaparams unless main = catalog.resource(:class, :main) raise "Couldn't find main" end names = Puppet::Type.metaparams.select do |name| !Puppet::Parser::Resource.relationship_parameter?(name) end data = {} catalog.walk(main, :out) do |source, target| if source_data = data[source] || metaparams_as_data(source, names) # only store anything in the data hash if we've actually got # data data[source] ||= source_data source_data.each do |param, value| target[param] = value if target[param].nil? end data[target] = source_data.merge(metaparams_as_data(target, names)) end target.tag(*(source.tags)) end end def metaparams_as_data(resource, params) data = nil params.each do |param| unless resource[param].nil? # Because we could be creating a hash for every resource, # and we actually probably don't often have any data here at all, # we're optimizing a bit by only creating a hash if there's # any data to put in it. data ||= {} data[param] = resource[param] end end data end # Set up all of our internal variables. def initvars # The list of overrides. This is used to cache overrides on objects # that don't exist yet. We store an array of each override. @resource_overrides = Hash.new do |overs, ref| overs[ref] = [] end # The list of collections that have been created. This is a global list, # but they each refer back to the scope that created them. @collections = [] # The list of relationships to evaluate. @relationships = [] # For maintaining the relationship between scopes and their resources. @catalog = Puppet::Resource::Catalog.new(@node.name) @catalog.version = known_resource_types.version @catalog.environment = @node.environment.to_s # Create our initial scope and a resource that will evaluate main. @topscope = Puppet::Parser::Scope.new(self) @catalog.add_resource(Puppet::Parser::Resource.new("stage", :main, :scope => @topscope)) # local resource array to maintain resource ordering @resources = [] # Make sure any external node classes are in our class list if @node.classes.class == Hash @catalog.add_class(*@node.classes.keys) else @catalog.add_class(*@node.classes) end end # Set the node's parameters into the top-scope as variables. def set_node_parameters node.parameters.each do |param, value| @topscope[param.to_s] = value end # These might be nil. catalog.client_version = node.parameters["clientversion"] catalog.server_version = node.parameters["serverversion"] end def create_settings_scope unless settings_type = environment.known_resource_types.hostclass("settings") settings_type = Puppet::Resource::Type.new :hostclass, "settings" environment.known_resource_types.add(settings_type) end settings_resource = Puppet::Parser::Resource.new("class", "settings", :scope => @topscope) @catalog.add_resource(settings_resource) settings_type.evaluate_code(settings_resource) scope = @topscope.class_scope(settings_type) Puppet.settings.each do |name, setting| next if name.to_s == "name" scope[name.to_s] = environment[name] end end # Return an array of all of the unevaluated resources. These will be definitions, # which need to get evaluated into native resources. def unevaluated_resources # The order of these is significant for speed due to short-circuting resources.reject { |resource| resource.evaluated? or resource.virtual? or resource.builtin_type? } end # Creates the injector from bindings found in the current environment. # @return [void] # @api private # def create_injector assert_binder_active() composer = Puppet::Pops::Binder::BindingsComposer.new() layered_bindings = composer.compose(topscope) binder = Puppet::Pops::Binder::Binder.new() binder.define_categories(composer.effective_categories(topscope)) binder.define_layers(layered_bindings) @injector = Puppet::Pops::Binder::Injector.new(binder) end def assert_binder_active unless is_binder_active? raise ArgumentError, "The Puppet Binder is only available when either '--binder true' or '--parser future' is used" end end end diff --git a/lib/puppet/parser/functions.rb b/lib/puppet/parser/functions.rb index 7015ec4d9..663c5b795 100644 --- a/lib/puppet/parser/functions.rb +++ b/lib/puppet/parser/functions.rb @@ -1,251 +1,242 @@ require 'puppet/util/autoload' require 'puppet/parser/scope' -require 'monitor' # A module for managing parser functions. Each specified function # is added to a central module that then gets included into the Scope # class. # # @api public module Puppet::Parser::Functions Environment = Puppet::Node::Environment class << self include Puppet::Util end # Reset the list of loaded functions. # # @api private def self.reset - @functions = Hash.new { |h,k| h[k] = {} }.extend(MonitorMixin) - @modules = Hash.new.extend(MonitorMixin) + @functions = Hash.new { |h,k| h[k] = {} } + @modules = Hash.new # Runs a newfunction to create a function for each of the log levels Puppet::Util::Log.levels.each do |level| newfunction(level, :doc => "Log a message on the server at level #{level.to_s}.") do |vals| send(level, vals.join(" ")) end end end # Accessor for singleton autoloader # # @api private def self.autoloader @autoloader ||= Puppet::Util::Autoload.new( self, "puppet/parser/functions", :wrap => false ) end # Get the module that functions are mixed into corresponding to an # environment # # @api private def self.environment_module(env = nil) if env and ! env.is_a?(Puppet::Node::Environment) env = Puppet::Node::Environment.new(env) end - @modules.synchronize { - @modules[ (env || Environment.current || Environment.root).name ] ||= Module.new - } + @modules[ (env || Environment.current || Environment.root).name ] ||= Module.new end # Create a new Puppet DSL function. # # **The {newfunction} method provides a public API.** # # This method is used both internally inside of Puppet to define parser # functions. For example, template() is defined in # {file:lib/puppet/parser/functions/template.rb template.rb} using the # {newfunction} method. Third party Puppet modules such as # [stdlib](https://forge.puppetlabs.com/puppetlabs/stdlib) use this method to # extend the behavior and functionality of Puppet. # # See also [Docs: Custom # Functions](http://docs.puppetlabs.com/guides/custom_functions.html) # # @example Define a new Puppet DSL Function # >> Puppet::Parser::Functions.newfunction(:double, :arity => 1, # :doc => "Doubles an object, typically a number or string.", # :type => :rvalue) {|i| i[0]*2 } # => {:arity=>1, :type=>:rvalue, # :name=>"function_double", # :doc=>"Doubles an object, typically a number or string."} # # @example Invoke the double function from irb as is done in RSpec examples: # >> scope = Puppet::Parser::Scope.new_for_test_harness('example') # => Scope() # >> scope.function_double([2]) # => 4 # >> scope.function_double([4]) # => 8 # >> scope.function_double([]) # ArgumentError: double(): Wrong number of arguments given (0 for 1) # >> scope.function_double([4,8]) # ArgumentError: double(): Wrong number of arguments given (2 for 1) # >> scope.function_double(["hello"]) # => "hellohello" # # @param [Symbol] name the name of the function represented as a ruby Symbol. # The {newfunction} method will define a Ruby method based on this name on # the parser scope instance. # # @param [Proc] block the block provided to the {newfunction} method will be # executed when the Puppet DSL function is evaluated during catalog # compilation. The arguments to the function will be passed as an array to # the first argument of the block. The return value of the block will be # the return value of the Puppet DSL function for `:rvalue` functions. # # @option options [:rvalue, :statement] :type (:statement) the type of function. # Either `:rvalue` for functions that return a value, or `:statement` for # functions that do not return a value. # # @option options [String] :doc ('') the documentation for the function. # This string will be extracted by documentation generation tools. # # @option options [Integer] :arity (-1) the # [arity](http://en.wikipedia.org/wiki/Arity) of the function. When # specified as a positive integer the function is expected to receive # _exactly_ the specified number of arguments. When specified as a # negative number, the function is expected to receive _at least_ the # absolute value of the specified number of arguments incremented by one. # For example, a function with an arity of `-4` is expected to receive at # minimum 3 arguments. A function with the default arity of `-1` accepts # zero or more arguments. A function with an arity of 2 must be provided # with exactly two arguments, no more and no less. Added in Puppet 3.1.0. # # @return [Hash] describing the function. # # @api public def self.newfunction(name, options = {}, &block) name = name.intern Puppet.warning "Overwriting previous definition for function #{name}" if get_function(name) arity = options[:arity] || -1 ftype = options[:type] || :statement unless ftype == :statement or ftype == :rvalue raise Puppet::DevError, "Invalid statement type #{ftype.inspect}" end # the block must be installed as a method because it may use "return", # which is not allowed from procs. real_fname = "real_function_#{name}" environment_module.send(:define_method, real_fname, &block) fname = "function_#{name}" environment_module.send(:define_method, fname) do |*args| Puppet::Util::Profiler.profile("Called #{name}") do if args[0].is_a? Array if arity >= 0 and args[0].size != arity raise ArgumentError, "#{name}(): Wrong number of arguments given (#{args[0].size} for #{arity})" elsif arity < 0 and args[0].size < (arity+1).abs raise ArgumentError, "#{name}(): Wrong number of arguments given (#{args[0].size} for minimum #{(arity+1).abs})" end self.send(real_fname, args[0]) else raise ArgumentError, "custom functions must be called with a single array that contains the arguments. For example, function_example([1]) instead of function_example(1)" end end end func = {:arity => arity, :type => ftype, :name => fname} func[:doc] = options[:doc] if options[:doc] add_function(name, func) func end # Determine if a function is defined # # @param [Symbol] name the function # # @return [Symbol, false] The name of the function if it's defined, # otherwise false. # # @api public def self.function(name) name = name.intern func = nil - @functions.synchronize do - unless func = get_function(name) - autoloader.load(name, Environment.current) - func = get_function(name) - end + unless func = get_function(name) + autoloader.load(name, Environment.current) + func = get_function(name) end if func func[:name] else false end end def self.functiondocs autoloader.loadall ret = "" merged_functions.sort { |a,b| a[0].to_s <=> b[0].to_s }.each do |name, hash| ret += "#{name}\n#{"-" * name.to_s.length}\n" if hash[:doc] ret += Puppet::Util::Docs.scrub(hash[:doc]) else ret += "Undocumented.\n" end ret += "\n\n- *Type*: #{hash[:type]}\n\n" end ret end # Determine whether a given function returns a value. # # @param [Symbol] name the function # # @api public def self.rvalue?(name) func = get_function(name) func ? func[:type] == :rvalue : false end # Return the number of arguments a function expects. # # @param [Symbol] name the function # @return [Integer] The arity of the function. See {newfunction} for # the meaning of negative values. # # @api public def self.arity(name) func = get_function(name) func ? func[:arity] : -1 end class << self private def merged_functions - @functions.synchronize { - @functions[Environment.root].merge(@functions[Environment.current]) - } + @functions[Environment.root].merge(@functions[Environment.current]) end def get_function(name) name = name.intern merged_functions[name] end def add_function(name, func) name = name.intern - @functions.synchronize { - @functions[Environment.current][name] = func - } + @functions[Environment.current][name] = func end end reset # initialize the class instance variables end diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 132242f88..4ace9f90a 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -1,178 +1,122 @@ require 'find' require 'forwardable' require 'puppet/node/environment' require 'puppet/parser/parser_factory' class Puppet::Parser::TypeLoader extend Forwardable include Puppet::Node::Environment::Helper - # Helper class that makes sure we don't try to import the same file - # more than once from either the same thread or different threads. - class Helper - include MonitorMixin - def initialize - super - # These hashes are indexed by filename - @state = {} # :doing or :done - @thread = {} # if :doing, thread that's doing the parsing - @cond_var = {} # if :doing, condition var that will be signaled when done. - end - - # Execute the supplied block exactly once per file, no matter how - # many threads have asked for it to run. If another thread is - # already executing it, wait for it to finish. If this thread is - # already executing it, return immediately without executing the - # block. - # - # Note: the reason for returning immediately if this thread is - # already executing the block is to handle the case of a circular - # import--when this happens, we attempt to recursively re-parse a - # file that we are already in the process of parsing. To prevent - # an infinite regress we need to simply do nothing when the - # recursive import is attempted. - def do_once(file) - need_to_execute = synchronize do - case @state[file] - when :doing - if @thread[file] != Thread.current - @cond_var[file].wait - end - false - when :done - false - else - @state[file] = :doing - @thread[file] = Thread.current - @cond_var[file] = new_cond - true - end - end - if need_to_execute - begin - yield - ensure - synchronize do - @state[file] = :done - @thread.delete(file) - @cond_var.delete(file).broadcast - end - end - end - end - end - # Import manifest files that match a given file glob pattern. # # @param pattern [String] the file glob to apply when determining which files # to load # @param dir [String] base directory to use when the file is not # found in a module # @api private def import(pattern, dir) return if Puppet[:ignoreimport] modname, files = Puppet::Parser::Files.find_manifests_in_modules(pattern, environment) if files.empty? abspat = File.expand_path(pattern, dir) file_pattern = abspat + (File.extname(abspat).empty? ? '{.pp,.rb}' : '' ) files = Dir.glob(file_pattern).uniq.reject { |f| FileTest.directory?(f) } modname = nil if files.empty? raise_no_files_found(pattern) end end load_files(modname, files) end # Load all of the manifest files in all known modules. # @api private def import_all # And then load all files from each module, but (relying on system # behavior) only load files from the first module of a given name. E.g., # given first/foo and second/foo, only files from first/foo will be loaded. environment.modules.each do |mod| load_files(mod.name, mod.all_manifests) end end def_delegator :environment, :known_resource_types def initialize(env) self.environment = env - @loading_helper = Helper.new end # Try to load the object with the given fully qualified name. def try_load_fqname(type, fqname) return nil if fqname == "" # special-case main. files_to_try_for(fqname).each do |filename| begin imported_types = import_from_modules(filename) if result = imported_types.find { |t| t.type == type and t.name == fqname } Puppet.debug "Automatically imported #{fqname} from #{filename} into #{environment}" return result end rescue Puppet::ImportError => detail # I'm not convienced we should just drop these errors, but this # preserves existing behaviours. end end # Nothing found. return nil end def parse_file(file) Puppet.debug("importing '#{file}' in environment #{environment}") parser = Puppet::Parser::ParserFactory.parser(environment) parser.file = file return parser.parse end private def import_from_modules(pattern) modname, files = Puppet::Parser::Files.find_manifests_in_modules(pattern, environment) if files.empty? raise_no_files_found(pattern) end load_files(modname, files) end def raise_no_files_found(pattern) raise Puppet::ImportError, "No file(s) found for import of '#{pattern}'" end def load_files(modname, files) + @loaded ||= {} loaded_asts = [] - files.each do |file| - @loading_helper.do_once(file) do - loaded_asts << parse_file(file) - end + files.reject { |file| @loaded[file] }.each do |file| + loaded_asts << parse_file(file) + @loaded[file] = true end loaded_asts.collect do |ast| known_resource_types.import_ast(ast, modname) end.flatten end # Return a list of all file basenames that should be tried in order # to load the object with the given fully qualified name. def files_to_try_for(qualified_name) qualified_name.split('::').inject([]) do |paths, name| add_path_for_name(paths, name) end end def add_path_for_name(paths, name) if paths.empty? [name] else paths.unshift(File.join(paths.first, name)) end end end diff --git a/lib/puppet/settings.rb b/lib/puppet/settings.rb index 680f5d465..e5df6a24d 100644 --- a/lib/puppet/settings.rb +++ b/lib/puppet/settings.rb @@ -1,1231 +1,1204 @@ require 'puppet' -require 'sync' require 'getoptlong' require 'puppet/util/watched_file' require 'puppet/util/command_line/puppet_option_parser' # The class for handling configuration files. class Puppet::Settings include Enumerable require 'puppet/settings/errors' require 'puppet/settings/base_setting' require 'puppet/settings/string_setting' require 'puppet/settings/enum_setting' require 'puppet/settings/file_setting' require 'puppet/settings/directory_setting' require 'puppet/settings/path_setting' require 'puppet/settings/boolean_setting' require 'puppet/settings/terminus_setting' require 'puppet/settings/duration_setting' require 'puppet/settings/config_file' require 'puppet/settings/value_translator' # local reference for convenience PuppetOptionParser = Puppet::Util::CommandLine::PuppetOptionParser attr_accessor :files attr_reader :timer # These are the settings that every app is required to specify; there are reasonable defaults defined in application.rb. REQUIRED_APP_SETTINGS = [:logdir, :confdir, :vardir] # This method is intended for puppet internal use only; it is a convenience method that # returns reasonable application default settings values for a given run_mode. def self.app_defaults_for_run_mode(run_mode) { :name => run_mode.to_s, :run_mode => run_mode.name, :confdir => run_mode.conf_dir, :vardir => run_mode.var_dir, :rundir => run_mode.run_dir, :logdir => run_mode.log_dir, } end def self.default_certname() hostname = hostname_fact domain = domain_fact if domain and domain != "" fqdn = [hostname, domain].join(".") else fqdn = hostname end fqdn.to_s.gsub(/\.$/, '') end def self.hostname_fact() Facter["hostname"].value end def self.domain_fact() Facter["domain"].value end def self.default_config_file_name "puppet.conf" end # Create a new collection of config settings. def initialize @config = {} @shortnames = {} @created = [] @searchpath = nil - # Mutex-like thing to protect @values - @sync = Sync.new - # Keep track of set values. @values = Hash.new { |hash, key| hash[key] = {} } # Hold parsed metadata until run_mode is known @metas = {} # And keep a per-environment cache @cache = Hash.new { |hash, key| hash[key] = {} } # The list of sections we've used. @used = [] @hooks_to_call_on_application_initialization = [] @translate = Puppet::Settings::ValueTranslator.new @config_file_parser = Puppet::Settings::ConfigFile.new(@translate) end # Retrieve a config value def [](param) value(param) end # Set a config value. This doesn't set the defaults, it sets the value itself. def []=(param, value) set_value(param, value, :memory) end # Generate the list of valid arguments, in a format that GetoptLong can # understand, and add them to the passed option list. def addargs(options) # Add all of the config parameters as valid options. self.each { |name, setting| setting.getopt_args.each { |args| options << args } } options end # Generate the list of valid arguments, in a format that OptionParser can # understand, and add them to the passed option list. def optparse_addargs(options) # Add all of the config parameters as valid options. self.each { |name, setting| options << setting.optparse_args } options end # Is our parameter a boolean parameter? def boolean?(param) param = param.to_sym @config.include?(param) and @config[param].kind_of?(BooleanSetting) end # Remove all set values, potentially skipping cli values. def clear - @sync.synchronize do - unsafe_clear - end + unsafe_clear end # Remove all set values, potentially skipping cli values. def unsafe_clear(clear_cli = true, clear_application_defaults = false) @values.each do |name, values| next if ((name == :application_defaults) and !clear_application_defaults) next if ((name == :cli) and !clear_cli) @values.delete(name) end # Only clear the 'used' values if we were explicitly asked to clear out # :cli values; otherwise, it may be just a config file reparse, # and we want to retain this cli values. @used = [] if clear_cli @app_defaults_initialized = false if clear_application_defaults @cache.clear end private :unsafe_clear # Clear @cache, @used and the Environment. # # Whenever an object is returned by Settings, a copy is stored in @cache. # As long as Setting attributes that determine the content of returned # objects remain unchanged, Settings can keep returning objects from @cache # without re-fetching or re-generating them. # # Whenever a Settings attribute changes, such as @values or @preferred_run_mode, # this method must be called to clear out the caches so that updated # objects will be returned. def flush_cache - @sync.synchronize do - unsafe_flush_cache - end + unsafe_flush_cache end def unsafe_flush_cache clearused # Clear the list of environments, because they cache, at least, the module path. # We *could* preferentially just clear them if the modulepath is changed, # but we don't really know if, say, the vardir is changed and the modulepath # is defined relative to it. We need the defined?(stuff) because of loading # order issues. Puppet::Node::Environment.clear if defined?(Puppet::Node) and defined?(Puppet::Node::Environment) end private :unsafe_flush_cache def clearused @cache.clear @used = [] end def global_defaults_initialized?() @global_defaults_initialized end def initialize_global_settings(args = []) raise Puppet::DevError, "Attempting to initialize global default settings more than once!" if global_defaults_initialized? # The first two phases of the lifecycle of a puppet application are: # 1) Parse the command line options and handle any of them that are # registered, defined "global" puppet settings (mostly from defaults.rb). # 2) Parse the puppet config file(s). parse_global_options(args) parse_config_files @global_defaults_initialized = true end # This method is called during application bootstrapping. It is responsible for parsing all of the # command line options and initializing the settings accordingly. # # It will ignore options that are not defined in the global puppet settings list, because they may # be valid options for the specific application that we are about to launch... however, at this point # in the bootstrapping lifecycle, we don't yet know what that application is. def parse_global_options(args) # Create an option parser option_parser = PuppetOptionParser.new option_parser.ignore_invalid_options = true # Add all global options to it. self.optparse_addargs([]).each do |option| option_parser.on(*option) do |arg| opt, val = Puppet::Settings.clean_opt(option[0], arg) handlearg(opt, val) end end option_parser.on('--run_mode', "The effective 'run mode' of the application: master, agent, or user.", :REQUIRED) do |arg| Puppet.settings.preferred_run_mode = arg end option_parser.parse(args) # remove run_mode options from the arguments so that later parses don't think # it is an unknown option. while option_index = args.index('--run_mode') do args.delete_at option_index args.delete_at option_index end args.reject! { |arg| arg.start_with? '--run_mode=' } end private :parse_global_options # A utility method (public, is used by application.rb and perhaps elsewhere) that munges a command-line # option string into the format that Puppet.settings expects. (This mostly has to deal with handling the # "no-" prefix on flag/boolean options). # # @param [String] opt the command line option that we are munging # @param [String, TrueClass, FalseClass] val the value for the setting (as determined by the OptionParser) def self.clean_opt(opt, val) # rewrite --[no-]option to --no-option if that's what was given if opt =~ /\[no-\]/ and !val opt = opt.gsub(/\[no-\]/,'no-') end # otherwise remove the [no-] prefix to not confuse everybody opt = opt.gsub(/\[no-\]/, '') [opt, val] end def app_defaults_initialized? @app_defaults_initialized end def initialize_app_defaults(app_defaults) REQUIRED_APP_SETTINGS.each do |key| raise SettingsError, "missing required app default setting '#{key}'" unless app_defaults.has_key?(key) end app_defaults.each do |key, value| if key == :run_mode self.preferred_run_mode = value else set_value(key, value, :application_defaults) end end apply_metadata call_hooks_deferred_to_application_initialization @app_defaults_initialized = true end def call_hooks_deferred_to_application_initialization(options = {}) @hooks_to_call_on_application_initialization.each do |setting| begin setting.handle(self.value(setting.name)) rescue InterpolationError => err raise err unless options[:ignore_interpolation_dependency_errors] #swallow. We're not concerned if we can't call hooks because dependencies don't exist yet #we'll get another chance after application defaults are initialized end end end private :call_hooks_deferred_to_application_initialization # Do variable interpolation on the value. def convert(value, environment = nil) return nil if value.nil? return value unless value.is_a? String newval = value.gsub(/\$(\w+)|\$\{(\w+)\}/) do |value| varname = $2 || $1 if varname == "environment" and environment environment elsif varname == "run_mode" preferred_run_mode elsif pval = self.value(varname, environment) pval else raise InterpolationError, "Could not find value for #{value}" end end newval end # Return a value's description. def description(name) if obj = @config[name.to_sym] obj.desc else nil end end def each @config.each { |name, object| yield name, object } end # Iterate over each section name. def eachsection yielded = [] @config.each do |name, object| section = object.section unless yielded.include? section yield section yielded << section end end end # Return an object by name. def setting(param) param = param.to_sym @config[param] end # Handle a command-line argument. def handlearg(opt, value = nil) @cache.clear if value.is_a?(FalseClass) value = "false" elsif value.is_a?(TrueClass) value = "true" end value &&= @translate[value] str = opt.sub(/^--/,'') bool = true newstr = str.sub(/^no-/, '') if newstr != str str = newstr bool = false end str = str.intern if @config[str].is_a?(Puppet::Settings::BooleanSetting) if value == "" or value.nil? value = bool end end set_value(str, value, :cli) end def include?(name) name = name.intern if name.is_a? String @config.include?(name) end # check to see if a short name is already defined def shortinclude?(short) short = short.intern if name.is_a? String @shortnames.include?(short) end # Prints the contents of a config file with the available config settings, or it # prints a single value of a config setting. def print_config_options env = value(:environment) val = value(:configprint) if val == "all" hash = {} each do |name, obj| val = value(name,env) val = val.inspect if val == "" hash[name] = val end hash.sort { |a,b| a[0].to_s <=> b[0].to_s }.each do |name, val| puts "#{name} = #{val}" end else val.split(/\s*,\s*/).sort.each do |v| if include?(v) #if there is only one value, just print it for back compatibility if v == val puts value(val,env) break end puts "#{v} = #{value(v,env)}" else puts "invalid parameter: #{v}" return false end end end true end def generate_config puts to_config true end def generate_manifest puts to_manifest true end def print_configs return print_config_options if value(:configprint) != "" return generate_config if value(:genconfig) generate_manifest if value(:genmanifest) end def print_configs? (value(:configprint) != "" || value(:genconfig) || value(:genmanifest)) && true end # Return a given object's file metadata. def metadata(param) if obj = @config[param.to_sym] and obj.is_a?(FileSetting) return [:owner, :group, :mode].inject({}) do |meta, p| if v = obj.send(p) meta[p] = v end meta end else nil end end # Make a directory with the appropriate user, group, and mode def mkdir(default) obj = get_config_file_default(default) Puppet::Util::SUIDManager.asuser(obj.owner, obj.group) do mode = obj.mode || 0750 Dir.mkdir(obj.value, mode) end end # The currently configured run mode that is preferred for constructing the application configuration. def preferred_run_mode @preferred_run_mode_name || :user end # PRIVATE! This only exists because we need a hook to validate the run mode when it's being set, and # it should never, ever, ever, ever be called from outside of this file. # This method is also called when --run_mode MODE is used on the command line to set the default # # @param mode [String|Symbol] the name of the mode to have in effect # @api private def preferred_run_mode=(mode) mode = mode.to_s.downcase.intern raise ValidationError, "Invalid run mode '#{mode}'" unless [:master, :agent, :user].include?(mode) @preferred_run_mode_name = mode # Changing the run mode has far-reaching consequences. Flush any cached # settings so they will be re-generated. flush_cache mode end # Return all of the parameters associated with a given section. def params(section = nil) if section section = section.intern if section.is_a? String @config.find_all { |name, obj| obj.section == section }.collect { |name, obj| name } else @config.keys end end # Parse the configuration file. Just provides thread safety. def parse_config_files - @sync.synchronize do - unsafe_parse(which_configuration_file) - end + unsafe_parse(which_configuration_file) call_hooks_deferred_to_application_initialization :ignore_interpolation_dependency_errors => true end private :parse_config_files def main_config_file if explicit_config_file? return self[:config] else return File.join(Puppet::Util::RunMode[:master].conf_dir, config_file_name) end end private :main_config_file def user_config_file return File.join(Puppet::Util::RunMode[:user].conf_dir, config_file_name) end private :user_config_file # This method is here to get around some life-cycle issues. We need to be # able to determine the config file name before the settings / defaults are # fully loaded. However, we also need to respect any overrides of this value # that the user may have specified on the command line. # # The easiest way to do this is to attempt to read the setting, and if we # catch an error (meaning that it hasn't been set yet), we'll fall back to # the default value. def config_file_name begin return self[:config_file_name] if self[:config_file_name] rescue SettingsError # This just means that the setting wasn't explicitly set on the command line, so we will ignore it and # fall through to the default name. end return self.class.default_config_file_name end private :config_file_name # Unsafely parse the file -- this isn't thread-safe and causes plenty of problems if used directly. def unsafe_parse(file) # build up a single data structure that contains the values from all of the parsed files. data = {} if FileTest.exist?(file) begin file_data = parse_file(file) # This is a little kludgy; basically we are merging a hash of hashes. We can't use "merge" at the # outermost level or we risking losing data from the hash we're merging into. file_data.keys.each do |key| if data.has_key?(key) data[key].merge!(file_data[key]) else data[key] = file_data[key] end end rescue => detail Puppet.log_exception(detail, "Could not parse #{file}: #{detail}") return end end # If we get here and don't have any data, we just return and don't muck with the current state of the world. return if data.empty? # If we get here then we have some data, so we need to clear out any previous settings that may have come from # config files. unsafe_clear(false, false) # And now we can repopulate with the values from our last parsing of the config files. data.each do |area, values| @metas[area] = values.delete(:_meta) values.each do |key,value| set_value(key, value, area, :dont_trigger_handles => true, :ignore_bad_settings => true ) end end # Determine our environment, if we have one. if @config[:environment] env = self.value(:environment).to_sym else env = "none" end # Call any hooks we should be calling. settings_with_hooks.each do |setting| each_source(env) do |source| if @values[source][setting.name] # We still have to use value to retrieve the value, since # we want the fully interpolated value, not $vardir/lib or whatever. # This results in extra work, but so few of the settings # will have associated hooks that it ends up being less work this # way overall. setting.handle(self.value(setting.name, env)) break end end end # Take a best guess at metadata based on uninitialized run_mode apply_metadata end private :unsafe_parse def apply_metadata # We have to do it in the reverse of the search path, # because multiple sections could set the same value # and I'm too lazy to only set the metadata once. searchpath.reverse.each do |source| source = preferred_run_mode if source == :run_mode source = @name if (@name && source == :name) if meta = @metas[source] set_metadata(meta) end end end private :apply_metadata SETTING_TYPES = { :string => StringSetting, :file => FileSetting, :directory => DirectorySetting, :path => PathSetting, :boolean => BooleanSetting, :terminus => TerminusSetting, :duration => DurationSetting, :enum => EnumSetting, } # Create a new setting. The value is passed in because it's used to determine # what kind of setting we're creating, but the value itself might be either # a default or a value, so we can't actually assign it. # # See #define_settings for documentation on the legal values for the ":type" option. def newsetting(hash) klass = nil hash[:section] = hash[:section].to_sym if hash[:section] if type = hash[:type] unless klass = SETTING_TYPES[type] raise ArgumentError, "Invalid setting type '#{type}'" end hash.delete(:type) else # The only implicit typing we still do for settings is to fall back to "String" type if they didn't explicitly # specify a type. Personally I'd like to get rid of this too, and make the "type" option mandatory... but # there was a little resistance to taking things quite that far for now. --cprice 2012-03-19 klass = StringSetting end hash[:settings] = self setting = klass.new(hash) setting end # This has to be private, because it doesn't add the settings to @config private :newsetting # Iterate across all of the objects in a given section. def persection(section) section = section.to_sym self.each { |name, obj| if obj.section == section yield obj end } end # Reparse our config file, if necessary. def reparse_config_files if files if filename = any_files_changed? Puppet.notice "Config file #{filename} changed; triggering re-parse of all config files." parse_config_files reuse end end end def files return @files if @files @files = [] [main_config_file, user_config_file].each do |path| if FileTest.exist?(path) @files << Puppet::Util::WatchedFile.new(path) end end @files end private :files # Checks to see if any of the config files have been modified # @return the filename of the first file that is found to have changed, or nil if no files have changed def any_files_changed? files.each do |file| return file.file if file.changed? end nil end private :any_files_changed? def reuse return unless defined?(@used) - @sync.synchronize do # yay, thread-safe - new = @used - @used = [] - self.use(*new) - end + new = @used + @used = [] + self.use(*new) end # The order in which to search for values. def searchpath(environment = nil) if environment [:cli, :memory, environment, :run_mode, :main, :application_defaults] else [:cli, :memory, :run_mode, :main, :application_defaults] end end # Get a list of objects per section def sectionlist sectionlist = [] self.each { |name, obj| section = obj.section || "puppet" sections[section] ||= [] sectionlist << section unless sectionlist.include?(section) sections[section] << obj } return sectionlist, sections end def service_user_available? return @service_user_available if defined?(@service_user_available) if self[:user] user = Puppet::Type.type(:user).new :name => self[:user], :audit => :ensure @service_user_available = user.exists? else @service_user_available = false end end def service_group_available? return @service_group_available if defined?(@service_group_available) if self[:group] group = Puppet::Type.type(:group).new :name => self[:group], :audit => :ensure @service_group_available = group.exists? else @service_group_available = false end end # Allow later inspection to determine if the setting was set on the # command line, or through some other code path. Used for the # `dns_alt_names` option during cert generate. --daniel 2011-10-18 def set_by_cli?(param) param = param.to_sym !@values[:cli][param].nil? end def set_value(param, value, type, options = {}) param = param.to_sym if !(setting = @config[param]) if options[:ignore_bad_settings] return else raise ArgumentError, "Attempt to assign a value to unknown configuration parameter #{param.inspect}" end end setting.handle(value) if setting.has_hook? and not options[:dont_trigger_handles] - @sync.synchronize do # yay, thread-safe - - @values[type][param] = value - unsafe_flush_cache - - end + @values[type][param] = value + unsafe_flush_cache value end # Deprecated; use #define_settings instead def setdefaults(section, defs) Puppet.deprecation_warning("'setdefaults' is deprecated and will be removed; please call 'define_settings' instead") define_settings(section, defs) end # Define a group of settings. # # @param [Symbol] section a symbol to use for grouping multiple settings together into a conceptual unit. This value # (and the conceptual separation) is not used very often; the main place where it will have a potential impact # is when code calls Settings#use method. See docs on that method for further details, but basically that method # just attempts to do any preparation that may be necessary before code attempts to leverage the value of a particular # setting. This has the most impact for file/directory settings, where #use will attempt to "ensure" those # files / directories. # @param [Hash[Hash]] defs the settings to be defined. This argument is a hash of hashes; each key should be a symbol, # which is basically the name of the setting that you are defining. The value should be another hash that specifies # the parameters for the particular setting. Legal values include: # [:default] => required; this is a string value that will be used as a default value for a setting if no other # value is specified (via cli, config file, etc.) This string may include "variables", demarcated with $ or ${}, # which will be interpolated with values of other settings. # [:desc] => required; a description of the setting, used in documentation / help generation # [:type] => not required, but highly encouraged! This specifies the data type that the setting represents. If # you do not specify it, it will default to "string". Legal values include: # :string - A generic string setting # :boolean - A boolean setting; values are expected to be "true" or "false" # :file - A (single) file path; puppet may attempt to create this file depending on how the settings are used. This type # also supports additional options such as "mode", "owner", "group" # :directory - A (single) directory path; puppet may attempt to create this file depending on how the settings are used. This type # also supports additional options such as "mode", "owner", "group" # :path - This is intended to be used for settings whose value can contain multiple directory paths, respresented # as strings separated by the system path separator (e.g. system path, module path, etc.). # [:mode] => an (optional) octal value to be used as the permissions/mode for :file and :directory settings # [:owner] => optional owner username/uid for :file and :directory settings # [:group] => optional group name/gid for :file and :directory settings # def define_settings(section, defs) section = section.to_sym call = [] defs.each { |name, hash| raise ArgumentError, "setting definition for '#{name}' is not a hash!" unless hash.is_a? Hash name = name.to_sym hash[:name] = name hash[:section] = section raise ArgumentError, "Parameter #{name} is already defined" if @config.include?(name) tryconfig = newsetting(hash) if short = tryconfig.short if other = @shortnames[short] raise ArgumentError, "Parameter #{other.name} is already using short name '#{short}'" end @shortnames[short] = tryconfig end @config[name] = tryconfig # Collect the settings that need to have their hooks called immediately. # We have to collect them so that we can be sure we're fully initialized before # the hook is called. call << tryconfig if tryconfig.call_hook_on_define? @hooks_to_call_on_application_initialization << tryconfig if tryconfig.call_hook_on_initialize? } call.each { |setting| setting.handle(self.value(setting.name)) } end # Convert the settings we manage into a catalog full of resources that model those settings. def to_catalog(*sections) sections = nil if sections.empty? catalog = Puppet::Resource::Catalog.new("Settings") @config.keys.find_all { |key| @config[key].is_a?(FileSetting) }.each do |key| file = @config[key] next unless (sections.nil? or sections.include?(file.section)) next unless resource = file.to_resource next if catalog.resource(resource.ref) Puppet.debug("Using settings: adding file resource '#{key}': '#{resource.inspect}'") catalog.add_resource(resource) end add_user_resources(catalog, sections) catalog end # Convert our list of config settings into a configuration file. def to_config str = %{The configuration file for #{Puppet.run_mode.name}. Note that this file is likely to have unused configuration parameters in it; any parameter that's valid anywhere in Puppet can be in any config file, even if it's not used. Every section can specify three special parameters: owner, group, and mode. These parameters affect the required permissions of any files specified after their specification. Puppet will sometimes use these parameters to check its own configured state, so they can be used to make Puppet a bit more self-managing. The file format supports octothorpe-commented lines, but not partial-line comments. Generated on #{Time.now}. }.gsub(/^/, "# ") # Add a section heading that matches our name. str += "[#{preferred_run_mode}]\n" eachsection do |section| persection(section) do |obj| str += obj.to_config + "\n" unless obj.name == :genconfig end end return str end # Convert to a parseable manifest def to_manifest catalog = to_catalog catalog.resource_refs.collect do |ref| catalog.resource(ref).to_manifest end.join("\n\n") end # Create the necessary objects to use a section. This is idempotent; # you can 'use' a section as many times as you want. def use(*sections) sections = sections.collect { |s| s.to_sym } - @sync.synchronize do # yay, thread-safe - sections = sections.reject { |s| @used.include?(s) } + sections = sections.reject { |s| @used.include?(s) } - return if sections.empty? + return if sections.empty? - begin - catalog = to_catalog(*sections).to_ral - rescue => detail - Puppet.log_and_raise(detail, "Could not create resources for managing Puppet's files and directories in sections #{sections.inspect}: #{detail}") - end + begin + catalog = to_catalog(*sections).to_ral + rescue => detail + Puppet.log_and_raise(detail, "Could not create resources for managing Puppet's files and directories in sections #{sections.inspect}: #{detail}") + end - catalog.host_config = false - catalog.apply do |transaction| - if transaction.any_failed? - report = transaction.report - failures = report.logs.find_all { |log| log.level == :err } - raise "Got #{failures.length} failure(s) while initializing: #{failures.collect { |l| l.to_s }.join("; ")}" - end + catalog.host_config = false + catalog.apply do |transaction| + if transaction.any_failed? + report = transaction.report + failures = report.logs.find_all { |log| log.level == :err } + raise "Got #{failures.length} failure(s) while initializing: #{failures.collect { |l| l.to_s }.join("; ")}" end - - sections.each { |s| @used << s } - @used.uniq! end + + sections.each { |s| @used << s } + @used.uniq! end def valid?(param) param = param.to_sym @config.has_key?(param) end def uninterpolated_value(param, environment = nil) param = param.to_sym environment &&= environment.to_sym # See if we can find it within our searchable list of values val = find_value(environment, param) # If we didn't get a value, use the default val = @config[param].default if val.nil? val end def find_value(environment, param) each_source(environment) do |source| # Look for the value. We have to test the hash for whether # it exists, because the value might be false. - @sync.synchronize do - return @values[source][param] if @values[source].include?(param) - end + return @values[source][param] if @values[source].include?(param) end return nil end private :find_value # Find the correct value using our search path. # # @param param [String, Symbol] The value to look up # @param environment [String, Symbol] The environment to check for the value # @param bypass_interpolation [true, false] Whether to skip interpolation # # @return [Object] The looked up value # # @raise [InterpolationError] def value(param, environment = nil, bypass_interpolation = false) param = param.to_sym environment &&= environment.to_sym setting = @config[param] # Short circuit to nil for undefined parameters. return nil unless @config.include?(param) # Yay, recursion. #self.reparse unless [:config, :filetimeout].include?(param) # Check the cache first. It needs to be a per-environment # cache so that we don't spread values from one env # to another. if @cache[environment||"none"].has_key?(param) return @cache[environment||"none"][param] end val = uninterpolated_value(param, environment) return val if bypass_interpolation if param == :code # if we interpolate code, all hell breaks loose. return val end # Convert it if necessary begin val = convert(val, environment) rescue InterpolationError => err # This happens because we don't have access to the param name when the # exception is originally raised, but we want it in the message raise InterpolationError, "Error converting value for param '#{param}': #{err}", err.backtrace end val = setting.munge(val) if setting.respond_to?(:munge) # And cache it @cache[environment||"none"][param] = val val end # Open a file with the appropriate user, group, and mode def write(default, *args, &bloc) obj = get_config_file_default(default) writesub(default, value(obj.name), *args, &bloc) end # Open a non-default file under a default dir with the appropriate user, # group, and mode def writesub(default, file, *args, &bloc) obj = get_config_file_default(default) chown = nil if Puppet.features.root? chown = [obj.owner, obj.group] else chown = [nil, nil] end Puppet::Util::SUIDManager.asuser(*chown) do mode = obj.mode ? obj.mode.to_i : 0640 args << "w" if args.empty? args << mode # Update the umask to make non-executable files Puppet::Util.withumask(File.umask ^ 0111) do File.open(file, *args) do |file| yield file end end end end def readwritelock(default, *args, &bloc) file = value(get_config_file_default(default).name) tmpfile = file + ".tmp" - sync = Sync.new raise Puppet::DevError, "Cannot create #{file}; directory #{File.dirname(file)} does not exist" unless FileTest.directory?(File.dirname(tmpfile)) - sync.synchronize(Sync::EX) do - File.open(file, ::File::CREAT|::File::RDWR, 0600) do |rf| - rf.lock_exclusive do - if File.exist?(tmpfile) - raise Puppet::Error, ".tmp file already exists for #{file}; Aborting locked write. Check the .tmp file and delete if appropriate" - end + File.open(file, ::File::CREAT|::File::RDWR, 0600) do |rf| + rf.lock_exclusive do + if File.exist?(tmpfile) + raise Puppet::Error, ".tmp file already exists for #{file}; Aborting locked write. Check the .tmp file and delete if appropriate" + end - # If there's a failure, remove our tmpfile - begin - writesub(default, tmpfile, *args, &bloc) - rescue - File.unlink(tmpfile) if FileTest.exist?(tmpfile) - raise - end + # If there's a failure, remove our tmpfile + begin + writesub(default, tmpfile, *args, &bloc) + rescue + File.unlink(tmpfile) if FileTest.exist?(tmpfile) + raise + end - begin - File.rename(tmpfile, file) - rescue => detail - Puppet.err "Could not rename #{file} to #{tmpfile}: #{detail}" - File.unlink(tmpfile) if FileTest.exist?(tmpfile) - end + begin + File.rename(tmpfile, file) + rescue => detail + Puppet.err "Could not rename #{file} to #{tmpfile}: #{detail}" + File.unlink(tmpfile) if FileTest.exist?(tmpfile) end end end end private def get_config_file_default(default) obj = nil unless obj = @config[default] raise ArgumentError, "Unknown default #{default}" end raise ArgumentError, "Default #{default} is not a file" unless obj.is_a? FileSetting obj end def add_user_resources(catalog, sections) return unless Puppet.features.root? return if Puppet.features.microsoft_windows? return unless self[:mkusers] @config.each do |name, setting| next unless setting.respond_to?(:owner) next unless sections.nil? or sections.include?(setting.section) if user = setting.owner and user != "root" and catalog.resource(:user, user).nil? resource = Puppet::Resource.new(:user, user, :parameters => {:ensure => :present}) resource[:gid] = self[:group] if self[:group] catalog.add_resource resource end if group = setting.group and ! %w{root wheel}.include?(group) and catalog.resource(:group, group).nil? catalog.add_resource Puppet::Resource.new(:group, group, :parameters => {:ensure => :present}) end end end # Yield each search source in turn. def each_source(environment) searchpath(environment).each do |source| # Modify the source as necessary. source = self.preferred_run_mode if source == :run_mode yield source end end # Return all settings that have associated hooks; this is so # we can call them after parsing the configuration file. def settings_with_hooks @config.values.find_all { |setting| setting.has_hook? } end # This method just turns a file in to a hash of hashes. def parse_file(file) @config_file_parser.parse_file(file, read_file(file)) end # Read the file in. def read_file(file) begin return File.read(file) rescue Errno::ENOENT raise ArgumentError, "No such file #{file}" rescue Errno::EACCES raise ArgumentError, "Permission denied to file #{file}" end end # Set file metadata. def set_metadata(meta) meta.each do |var, values| values.each do |param, value| - @sync.synchronize do # yay, thread-safe - @config[var].send(param.to_s + "=", value) - end + @config[var].send(param.to_s + "=", value) end end end # Private method for internal test use only; allows to do a comprehensive clear of all settings between tests. # # @return nil def clear_everything_for_tests() - @sync.synchronize do - unsafe_clear(true, true) - @global_defaults_initialized = false - @app_defaults_initialized = false - end + unsafe_clear(true, true) + @global_defaults_initialized = false + @app_defaults_initialized = false end private :clear_everything_for_tests ## # (#15337) All of the logic to determine the configuration file to use # should be centralized into this method. The simplified approach is: # # 1. If there is an explicit configuration file, use that. (--confdir or # --config) # 2. If we're running as a root process, use the system puppet.conf # (usually /etc/puppet/puppet.conf) # 3. Otherwise, use the user puppet.conf (usually ~/.puppet/puppet.conf) # # @todo this code duplicates {Puppet::Util::RunMode#which_dir} as described # in {http://projects.puppetlabs.com/issues/16637 #16637} def which_configuration_file if explicit_config_file? or Puppet.features.root? then return main_config_file else return user_config_file end end def explicit_config_file? # Figure out if the user has provided an explicit configuration file. If # so, return the path to the file, if not return nil. # # The easiest way to determine whether an explicit one has been specified # is to simply attempt to evaluate the value of ":config". This will # obviously be successful if they've passed an explicit value for :config, # but it will also result in successful interpolation if they've only # passed an explicit value for :confdir. # # If they've specified neither, then the interpolation will fail and we'll # get an exception. # begin return true if self[:config] rescue InterpolationError # This means we failed to interpolate, which means that they didn't # explicitly specify either :config or :confdir... so we'll fall out to # the default value. return false end end private :explicit_config_file? end diff --git a/lib/puppet/ssl/certificate_authority.rb b/lib/puppet/ssl/certificate_authority.rb index 83ff6a114..4bb9d7f1d 100644 --- a/lib/puppet/ssl/certificate_authority.rb +++ b/lib/puppet/ssl/certificate_authority.rb @@ -1,475 +1,470 @@ -require 'monitor' require 'puppet/ssl/host' require 'puppet/ssl/certificate_request' require 'puppet/ssl/certificate_signer' require 'puppet/util' # The class that knows how to sign certificates. It creates # a 'special' SSL::Host whose name is 'ca', thus indicating # that, well, it's the CA. There's some magic in the # indirector/ssl_file terminus base class that does that # for us. # This class mostly just signs certs for us, but # it can also be seen as a general interface into all of the # SSL stuff. class Puppet::SSL::CertificateAuthority # We will only sign extensions on this whitelist, ever. Any CSR with a # requested extension that we don't recognize is rejected, against the risk # that it will introduce some security issue through our ignorance of it. # # Adding an extension to this whitelist simply means we will consider it # further, not that we will always accept a certificate with an extension # requested on this list. RequestExtensionWhitelist = %w{subjectAltName} require 'puppet/ssl/certificate_factory' require 'puppet/ssl/inventory' require 'puppet/ssl/certificate_revocation_list' require 'puppet/ssl/certificate_authority/interface' require 'puppet/network/authstore' - extend MonitorMixin - class CertificateVerificationError < RuntimeError attr_accessor :error_code def initialize(code) @error_code = code end end def self.singleton_instance - synchronize do - @singleton_instance ||= new - end + @singleton_instance ||= new end class CertificateSigningError < RuntimeError attr_accessor :host def initialize(host) @host = host end end def self.ca? return false unless Puppet[:ca] return false unless Puppet.run_mode.master? true end # If this process can function as a CA, then return a singleton # instance. def self.instance return nil unless ca? singleton_instance end attr_reader :name, :host # Create and run an applicator. I wanted to build an interface where you could do # something like 'ca.apply(:generate).to(:all) but I don't think it's really possible. def apply(method, options) raise ArgumentError, "You must specify the hosts to apply to; valid values are an array or the symbol :all" unless options[:to] applier = Interface.new(method, options) applier.apply(self) end # If autosign is configured, then autosign all CSRs that match our configuration. def autosign return unless auto = autosign? store = nil store = autosign_store(auto) if auto != true Puppet::SSL::CertificateRequest.indirection.search("*").each do |csr| if auto == true or store.allowed?(csr.name, "127.1.1.1") Puppet.info "Autosigning #{csr.name}" sign(csr.name) end end end # Do we autosign? This returns true, false, or a filename. def autosign? auto = Puppet[:autosign] return false if ['false', false].include?(auto) return true if ['true', true].include?(auto) raise ArgumentError, "The autosign configuration '#{auto}' must be a fully qualified file" unless Puppet::Util.absolute_path?(auto) FileTest.exist?(auto) && auto end # Create an AuthStore for autosigning. def autosign_store(file) auth = Puppet::Network::AuthStore.new File.readlines(file).each do |line| next if line =~ /^\s*#/ next if line =~ /^\s*$/ auth.allow(line.chomp) end auth end # Retrieve (or create, if necessary) the certificate revocation list. def crl unless defined?(@crl) unless @crl = Puppet::SSL::CertificateRevocationList.indirection.find(Puppet::SSL::CA_NAME) @crl = Puppet::SSL::CertificateRevocationList.new(Puppet::SSL::CA_NAME) @crl.generate(host.certificate.content, host.key.content) Puppet::SSL::CertificateRevocationList.indirection.save(@crl) end end @crl end # Delegate this to our Host class. def destroy(name) Puppet::SSL::Host.destroy(name) end # Generate a new certificate. # @return Puppet::SSL::Certificate def generate(name, options = {}) raise ArgumentError, "A Certificate already exists for #{name}" if Puppet::SSL::Certificate.indirection.find(name) # Pass on any requested subjectAltName field. san = options[:dns_alt_names] host = Puppet::SSL::Host.new(name) host.generate_certificate_request(:dns_alt_names => san) # CSR may have been implicitly autosigned, generating a certificate # Or sign explicitly host.certificate || sign(name, !!san) end # Generate our CA certificate. def generate_ca_certificate generate_password unless password? host.generate_key unless host.key # Create a new cert request. We do this specially, because we don't want # to actually save the request anywhere. request = Puppet::SSL::CertificateRequest.new(host.name) # We deliberately do not put any subjectAltName in here: the CA # certificate absolutely does not need them. --daniel 2011-10-13 request.generate(host.key) # Create a self-signed certificate. @certificate = sign(host.name, false, request) # And make sure we initialize our CRL. crl end def initialize Puppet.settings.use :main, :ssl, :ca @name = Puppet[:certname] @host = Puppet::SSL::Host.new(Puppet::SSL::Host.ca_name) setup end # Retrieve (or create, if necessary) our inventory manager. def inventory @inventory ||= Puppet::SSL::Inventory.new end # Generate a new password for the CA. def generate_password pass = "" 20.times { pass += (rand(74) + 48).chr } begin Puppet.settings.write(:capass) { |f| f.print pass } rescue Errno::EACCES => detail raise Puppet::Error, "Could not write CA password: #{detail}" end @password = pass pass end # Lists the names of all signed certificates. # # @return [Array] def list list_certificates.collect { |c| c.name } end # Return all the certificate objects as found by the indirector # API for PE license checking. # # Created to prevent the case of reading all certs from disk, getting # just their names and verifying the cert for each name, which then # causes the cert to again be read from disk. # # @author Jeff Weiss # @api Puppet Enterprise Licensing # # @return [Array] def list_certificates Puppet::SSL::Certificate.indirection.search("*") end # Read the next serial from the serial file, and increment the # file so this one is considered used. def next_serial serial = nil # This is slightly odd. If the file doesn't exist, our readwritelock creates # it, but with a mode we can't actually read in some cases. So, use # a default before the lock. serial = 0x1 unless FileTest.exist?(Puppet[:serial]) Puppet.settings.readwritelock(:serial) { |f| serial ||= File.read(Puppet.settings[:serial]).chomp.hex if FileTest.exist?(Puppet[:serial]) # We store the next valid serial, not the one we just used. f << "%04X" % (serial + 1) } serial end # Does the password file exist? def password? FileTest.exist? Puppet[:capass] end # Print a given host's certificate as text. def print(name) (cert = Puppet::SSL::Certificate.indirection.find(name)) ? cert.to_text : nil end # Revoke a given certificate. def revoke(name) raise ArgumentError, "Cannot revoke certificates when the CRL is disabled" unless crl if cert = Puppet::SSL::Certificate.indirection.find(name) serial = cert.content.serial elsif name =~ /^0x[0-9A-Fa-f]+$/ serial = name.hex elsif ! serial = inventory.serial(name) raise ArgumentError, "Could not find a serial number for #{name}" end crl.revoke(serial, host.key.content) end # This initializes our CA so it actually works. This should be a private # method, except that you can't any-instance stub private methods, which is # *awesome*. This method only really exists to provide a stub-point during # testing. def setup generate_ca_certificate unless @host.certificate end # Sign a given certificate request. def sign(hostname, allow_dns_alt_names = false, self_signing_csr = nil) # This is a self-signed certificate if self_signing_csr # # This is a self-signed certificate, which is for the CA. Since this # # forces the certificate to be self-signed, anyone who manages to trick # # the system into going through this path gets a certificate they could # # generate anyway. There should be no security risk from that. csr = self_signing_csr cert_type = :ca issuer = csr.content else allow_dns_alt_names = true if hostname == Puppet[:certname].downcase unless csr = Puppet::SSL::CertificateRequest.indirection.find(hostname) raise ArgumentError, "Could not find certificate request for #{hostname}" end cert_type = :server issuer = host.certificate.content # Make sure that the CSR conforms to our internal signing policies. # This will raise if the CSR doesn't conform, but just in case... check_internal_signing_policies(hostname, csr, allow_dns_alt_names) or raise CertificateSigningError.new(hostname), "CSR had an unknown failure checking internal signing policies, will not sign!" end cert = Puppet::SSL::Certificate.new(hostname) cert.content = Puppet::SSL::CertificateFactory. build(cert_type, csr, issuer, next_serial) signer = Puppet::SSL::CertificateSigner.new signer.sign(cert.content, host.key.content) Puppet.notice "Signed certificate request for #{hostname}" # Add the cert to the inventory before we save it, since # otherwise we could end up with it being duplicated, if # this is the first time we build the inventory file. inventory.add(cert) # Save the now-signed cert. This should get routed correctly depending # on the certificate type. Puppet::SSL::Certificate.indirection.save(cert) # And remove the CSR if this wasn't self signed. Puppet::SSL::CertificateRequest.indirection.destroy(csr.name) unless self_signing_csr cert end def check_internal_signing_policies(hostname, csr, allow_dns_alt_names) # Reject unknown request extensions. unknown_req = csr.request_extensions. reject {|x| RequestExtensionWhitelist.include? x["oid"] } if unknown_req and not unknown_req.empty? names = unknown_req.map {|x| x["oid"] }.sort.uniq.join(", ") raise CertificateSigningError.new(hostname), "CSR has request extensions that are not permitted: #{names}" end # Do not sign misleading CSRs cn = csr.content.subject.to_a.assoc("CN")[1] if hostname != cn raise CertificateSigningError.new(hostname), "CSR subject common name #{cn.inspect} does not match expected certname #{hostname.inspect}" end if hostname !~ Puppet::SSL::Base::VALID_CERTNAME raise CertificateSigningError.new(hostname), "CSR #{hostname.inspect} subject contains unprintable or non-ASCII characters" end # Wildcards: we don't allow 'em at any point. # # The stringification here makes the content visible, and saves us having # to scrobble through the content of the CSR subject field to make sure it # is what we expect where we expect it. if csr.content.subject.to_s.include? '*' raise CertificateSigningError.new(hostname), "CSR subject contains a wildcard, which is not allowed: #{csr.content.subject.to_s}" end unless csr.content.verify(csr.content.public_key) raise CertificateSigningError.new(hostname), "CSR contains a public key that does not correspond to the signing key" end unless csr.subject_alt_names.empty? # If you alt names are allowed, they are required. Otherwise they are # disallowed. Self-signed certs are implicitly trusted, however. unless allow_dns_alt_names raise CertificateSigningError.new(hostname), "CSR '#{csr.name}' contains subject alternative names (#{csr.subject_alt_names.join(', ')}), which are disallowed. Use `puppet cert --allow-dns-alt-names sign #{csr.name}` to sign this request." end # If subjectAltNames are present, validate that they are only for DNS # labels, not any other kind. unless csr.subject_alt_names.all? {|x| x =~ /^DNS:/ } raise CertificateSigningError.new(hostname), "CSR '#{csr.name}' contains a subjectAltName outside the DNS label space: #{csr.subject_alt_names.join(', ')}. To continue, this CSR needs to be cleaned." end # Check for wildcards in the subjectAltName fields too. if csr.subject_alt_names.any? {|x| x.include? '*' } raise CertificateSigningError.new(hostname), "CSR '#{csr.name}' subjectAltName contains a wildcard, which is not allowed: #{csr.subject_alt_names.join(', ')} To continue, this CSR needs to be cleaned." end end return true # good enough for us! end # Utility method for optionally caching the X509 Store for verifying a # large number of certificates in a short amount of time--exactly the # case we have during PE license checking. # # @example Use the cached X509 store # x509store(:cache => true) # # @example Use a freshly create X509 store # x509store # x509store(:cache => false) # # @param [Hash] options the options used for retrieving the X509 Store # @option options [Boolean] :cache whether or not to use a cached version # of the X509 Store # # @return [OpenSSL::X509::Store] def x509_store(options = {}) if (options[:cache]) return @x509store unless @x509store.nil? @x509store = create_x509_store else create_x509_store end end private :x509_store # Creates a brand new OpenSSL::X509::Store with the appropriate # Certificate Revocation List and flags # # @return [OpenSSL::X509::Store] def create_x509_store store = OpenSSL::X509::Store.new store.add_file Puppet[:cacert] store.add_crl crl.content if self.crl store.purpose = OpenSSL::X509::PURPOSE_SSL_CLIENT store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL|OpenSSL::X509::V_FLAG_CRL_CHECK if Puppet.settings[:certificate_revocation] store end private :create_x509_store # Utility method which is API for PE license checking. # This is used rather than `verify` because # 1) We have already read the certificate from disk into memory. # To read the certificate from disk again is just wasteful. # 2) Because we're checking a large number of certificates against # a transient CertificateAuthority, we can relatively safely cache # the X509 Store that actually does the verification. # # Long running instances of CertificateAuthority will certainly # want to use `verify` because it will recreate the X509 Store with # the absolutely latest CRL. # # Additionally, this method explicitly returns a boolean whereas # `verify` will raise an error if the certificate has been revoked. # # @author Jeff Weiss # @api Puppet Enterprise Licensing # # @param cert [Puppet::SSL::Certificate] the certificate to check validity of # # @return [Boolean] true if signed, false if unsigned or revoked def certificate_is_alive?(cert) x509_store(:cache => true).verify(cert.content) end # Verify a given host's certificate. The certname is passed in, and # the indirector will be used to locate the actual contents of the # certificate with that name. # # @param name [String] certificate name to verify # # @raise [ArgumentError] if the certificate name cannot be found # (i.e. doesn't exist or is unsigned) # @raise [CertificateVerficationError] if the certificate has been revoked # # @return [Boolean] true if signed, there are no cases where false is returned def verify(name) unless cert = Puppet::SSL::Certificate.indirection.find(name) raise ArgumentError, "Could not find a certificate for #{name}" end store = x509_store raise CertificateVerificationError.new(store.error), store.error_string unless store.verify(cert.content) end def fingerprint(name, md = :SHA256) unless cert = Puppet::SSL::Certificate.indirection.find(name) || Puppet::SSL::CertificateRequest.indirection.find(name) raise ArgumentError, "Could not find a certificate or csr for #{name}" end cert.fingerprint(md) end # List the waiting certificate requests. def waiting? Puppet::SSL::CertificateRequest.indirection.search("*").collect { |r| r.name } end end diff --git a/lib/puppet/util.rb b/lib/puppet/util.rb index c1a82340e..fa1f29cba 100644 --- a/lib/puppet/util.rb +++ b/lib/puppet/util.rb @@ -1,577 +1,551 @@ # A module to collect utility functions. require 'English' require 'puppet/external/lock' require 'puppet/error' require 'puppet/util/execution_stub' require 'uri' -require 'sync' -require 'monitor' require 'tempfile' require 'pathname' require 'ostruct' require 'puppet/util/platform' require 'puppet/util/symbolic_file_mode' require 'securerandom' module Puppet module Util require 'puppet/util/monkey_patches' require 'benchmark' # These are all for backward compatibility -- these are methods that used # to be in Puppet::Util but have been moved into external modules. require 'puppet/util/posix' extend Puppet::Util::POSIX extend Puppet::Util::SymbolicFileMode - @@sync_objects = {}.extend MonitorMixin - - def self.activerecord_version if (defined?(::ActiveRecord) and defined?(::ActiveRecord::VERSION) and defined?(::ActiveRecord::VERSION::MAJOR) and defined?(::ActiveRecord::VERSION::MINOR)) ([::ActiveRecord::VERSION::MAJOR, ::ActiveRecord::VERSION::MINOR].join('.').to_f) else 0 end end # Run some code with a specific environment. Resets the environment back to # what it was at the end of the code. def self.withenv(hash) saved = ENV.to_hash hash.each do |name, val| ENV[name.to_s] = val end yield ensure ENV.clear saved.each do |name, val| ENV[name] = val end end # Execute a given chunk of code with a new umask. def self.withumask(mask) cur = File.umask(mask) begin yield ensure File.umask(cur) end end - def self.synchronize_on(x,type) - sync_object,users = 0,1 - begin - @@sync_objects.synchronize { - (@@sync_objects[x] ||= [Sync.new,0])[users] += 1 - } - @@sync_objects[x][sync_object].synchronize(type) { yield } - ensure - @@sync_objects.synchronize { - @@sync_objects.delete(x) unless (@@sync_objects[x][users] -= 1) > 0 - } - end - end - # Change the process to a different user def self.chuser if group = Puppet[:group] begin Puppet::Util::SUIDManager.change_group(group, true) rescue => detail Puppet.warning "could not change to group #{group.inspect}: #{detail}" $stderr.puts "could not change to group #{group.inspect}" # Don't exit on failed group changes, since it's # not fatal #exit(74) end end if user = Puppet[:user] begin Puppet::Util::SUIDManager.change_user(user, true) rescue => detail $stderr.puts "Could not change to user #{user}: #{detail}" exit(74) end end end # Create instance methods for each of the log levels. This allows # the messages to be a little richer. Most classes will be calling this # method. def self.logmethods(klass, useself = true) Puppet::Util::Log.eachlevel { |level| klass.send(:define_method, level, proc { |args| args = args.join(" ") if args.is_a?(Array) if useself Puppet::Util::Log.create( :level => level, :source => self, :message => args ) else Puppet::Util::Log.create( :level => level, :message => args ) end }) } end # Proxy a bunch of methods to another object. def self.classproxy(klass, objmethod, *methods) classobj = class << klass; self; end methods.each do |method| classobj.send(:define_method, method) do |*args| obj = self.send(objmethod) obj.send(method, *args) end end end # Proxy a bunch of methods to another object. def self.proxy(klass, objmethod, *methods) methods.each do |method| klass.send(:define_method, method) do |*args| obj = self.send(objmethod) obj.send(method, *args) end end end - def benchmark(*args) msg = args.pop level = args.pop object = nil if args.empty? if respond_to?(level) object = self else object = Puppet end else object = args.pop end raise Puppet::DevError, "Failed to provide level to :benchmark" unless level unless level == :none or object.respond_to? level raise Puppet::DevError, "Benchmarked object does not respond to #{level}" end # Only benchmark if our log level is high enough if level != :none and Puppet::Util::Log.sendlevel?(level) seconds = Benchmark.realtime { yield } object.send(level, msg + (" in %0.2f seconds" % seconds)) return seconds else yield end end + module_function :benchmark # Resolve a path for an executable to the absolute path. This tries to behave # in the same manner as the unix `which` command and uses the `PATH` # environment variable. # # @api public # @param bin [String] the name of the executable to find. # @return [String] the absolute path to the found executable. def which(bin) if absolute_path?(bin) return bin if FileTest.file? bin and FileTest.executable? bin else ENV['PATH'].split(File::PATH_SEPARATOR).each do |dir| begin dest = File.expand_path(File.join(dir, bin)) rescue ArgumentError => e # if the user's PATH contains a literal tilde (~) character and HOME is not set, we may get # an ArgumentError here. Let's check to see if that is the case; if not, re-raise whatever error # was thrown. if e.to_s =~ /HOME/ and (ENV['HOME'].nil? || ENV['HOME'] == "") # if we get here they have a tilde in their PATH. We'll issue a single warning about this and then # ignore this path element and carry on with our lives. Puppet::Util::Warnings.warnonce("PATH contains a ~ character, and HOME is not set; ignoring PATH element '#{dir}'.") elsif e.to_s =~ /doesn't exist|can't find user/ # ...otherwise, we just skip the non-existent entry, and do nothing. Puppet::Util::Warnings.warnonce("Couldn't expand PATH containing a ~ character; ignoring PATH element '#{dir}'.") else raise end else if Puppet.features.microsoft_windows? && File.extname(dest).empty? exts = ENV['PATHEXT'] exts = exts ? exts.split(File::PATH_SEPARATOR) : %w[.COM .EXE .BAT .CMD] exts.each do |ext| destext = File.expand_path(dest + ext) return destext if FileTest.file? destext and FileTest.executable? destext end end return dest if FileTest.file? dest and FileTest.executable? dest end end end nil end module_function :which # Determine in a platform-specific way whether a path is absolute. This # defaults to the local platform if none is specified. # # Escape once for the string literal, and once for the regex. slash = '[\\\\/]' label = '[^\\\\/]+' AbsolutePathWindows = %r!^(?:(?:[A-Z]:#{slash})|(?:#{slash}#{slash}#{label}#{slash}#{label})|(?:#{slash}#{slash}\?#{slash}#{label}))!io AbsolutePathPosix = %r!^/! def absolute_path?(path, platform=nil) # Ruby only sets File::ALT_SEPARATOR on Windows and the Ruby standard # library uses that to test what platform it's on. Normally in Puppet we # would use Puppet.features.microsoft_windows?, but this method needs to # be called during the initialization of features so it can't depend on # that. platform ||= Puppet::Util::Platform.windows? ? :windows : :posix regex = case platform when :windows AbsolutePathWindows when :posix AbsolutePathPosix else raise Puppet::DevError, "unknown platform #{platform} in absolute_path" end !! (path =~ regex) end module_function :absolute_path? # Convert a path to a file URI def path_to_uri(path) return unless path params = { :scheme => 'file' } if Puppet.features.microsoft_windows? path = path.gsub(/\\/, '/') if unc = /^\/\/([^\/]+)(\/[^\/]+)/.match(path) params[:host] = unc[1] path = unc[2] elsif path =~ /^[a-z]:\//i path = '/' + path end end params[:path] = URI.escape(path) begin URI::Generic.build(params) rescue => detail raise Puppet::Error, "Failed to convert '#{path}' to URI: #{detail}" end end module_function :path_to_uri # Get the path component of a URI def uri_to_path(uri) return unless uri.is_a?(URI) path = URI.unescape(uri.path) if Puppet.features.microsoft_windows? and uri.scheme == 'file' if uri.host path = "//#{uri.host}" + path # UNC else path.sub!(/^\//, '') end end path end module_function :uri_to_path def safe_posix_fork(stdin=$stdin, stdout=$stdout, stderr=$stderr, &block) child_pid = Kernel.fork do $stdin.reopen(stdin) $stdout.reopen(stdout) $stderr.reopen(stderr) 3.upto(256){|fd| IO::new(fd).close rescue nil} block.call if block end child_pid end module_function :safe_posix_fork - # Create an exclusive lock. - def threadlock(resource, type = Sync::EX) - Puppet::Util.synchronize_on(resource,type) { yield } - end - - module_function :benchmark - def memory unless defined?(@pmap) @pmap = which('pmap') end if @pmap %x{#{@pmap} #{Process.pid}| grep total}.chomp.sub(/^\s*total\s+/, '').sub(/K$/, '').to_i else 0 end end def symbolizehash(hash) newhash = {} hash.each do |name, val| name = name.intern if name.respond_to? :intern newhash[name] = val end newhash end module_function :symbolizehash # Just benchmark, with no logging. def thinmark seconds = Benchmark.realtime { yield } seconds end module_function :memory, :thinmark # Because IO#binread is only available in 1.9 def binread(file) File.open(file, 'rb') { |f| f.read } end module_function :binread # utility method to get the current call stack and format it to a human-readable string (which some IDEs/editors # will recognize as links to the line numbers in the trace) def self.pretty_backtrace(backtrace = caller(1)) backtrace.collect do |line| _, path, rest = /^(.*):(\d+.*)$/.match(line).to_a # If the path doesn't exist - like in one test, and like could happen in # the world - we should just tolerate it and carry on. --daniel 2012-09-05 # Also, if we don't match, just include the whole line. if path path = Pathname(path).realpath rescue path "#{path}:#{rest}" else line end end.join("\n") end # Replace a file, securely. This takes a block, and passes it the file # handle of a file open for writing. Write the replacement content inside # the block and it will safely replace the target file. # # This method will make no changes to the target file until the content is # successfully written and the block returns without raising an error. # # As far as possible the state of the existing file, such as mode, is # preserved. This works hard to avoid loss of any metadata, but will result # in an inode change for the file. # # Arguments: `filename`, `default_mode` # # The filename is the file we are going to replace. # # The default_mode is the mode to use when the target file doesn't already # exist; if the file is present we copy the existing mode/owner/group values # across. The default_mode can be expressed as an octal integer, a numeric string (ie '0664') # or a symbolic file mode. def replace_file(file, default_mode, &block) raise Puppet::DevError, "replace_file requires a block" unless block_given? unless valid_symbolic_mode?(default_mode) raise Puppet::DevError, "replace_file default_mode: #{default_mode} is invalid" end mode = symbolic_mode_to_int(normalize_symbolic_mode(default_mode)) file = Pathname(file) tempfile = Tempfile.new(file.basename.to_s, file.dirname.to_s) # Set properties of the temporary file before we write the content, because # Tempfile doesn't promise to be safe from reading by other people, just # that it avoids races around creating the file. # # Our Windows emulation is pretty limited, and so we have to carefully # and specifically handle the platform, which has all sorts of magic. # So, unlike Unix, we don't pre-prep security; we use the default "quite # secure" tempfile permissions instead. Magic happens later. unless Puppet.features.microsoft_windows? # Grab the current file mode, and fall back to the defaults. stat = file.lstat rescue OpenStruct.new(:mode => mode, :uid => Process.euid, :gid => Process.egid) # We only care about the bottom four slots, which make the real mode, # and not the rest of the platform stat call fluff and stuff. tempfile.chmod(stat.mode & 07777) tempfile.chown(stat.uid, stat.gid) end # OK, now allow the caller to write the content of the file. yield tempfile # Now, make sure the data (which includes the mode) is safe on disk. tempfile.flush begin tempfile.fsync rescue NotImplementedError # fsync may not be implemented by Ruby on all platforms, but # there is absolutely no recovery path if we detect that. So, we just # ignore the return code. # # However, don't be fooled: that is accepting that we are running in # an unsafe fashion. If you are porting to a new platform don't stub # that out. end tempfile.close if Puppet.features.microsoft_windows? # This will appropriately clone the file, but only if the file we are # replacing exists. Which is kind of annoying; thanks Microsoft. # # So, to avoid getting into an infinite loop we will retry once if the # file doesn't exist, but only the once... have_retried = false begin # Yes, the arguments are reversed compared to the rename in the rest # of the world. Puppet::Util::Windows::File.replace_file(file, tempfile.path) rescue Puppet::Util::Windows::Error # This might race, but there are enough possible cases that there # isn't a good, solid "better" way to do this, and the next call # should fail in the same way anyhow. raise if have_retried or File.exist?(file) have_retried = true # OK, so, we can't replace a file that doesn't exist, so let us put # one in place and set the permissions. Then we can retry and the # magic makes this all work. # # This is the least-worst option for handling Windows, as far as we # can determine. File.open(file, 'a') do |fh| # this space deliberately left empty for auto-close behaviour, # append mode, and not actually changing any of the content. end # Set the permissions to what we want. Puppet::Util::Windows::Security.set_mode(mode, file.to_s) # ...and finally retry the operation. retry end else File.rename(tempfile.path, file.to_s) end # Ideally, we would now fsync the directory as well, but Ruby doesn't # have support for that, and it doesn't matter /that/ much... # Return something true, and possibly useful. file end module_function :replace_file # Executes a block of code, wrapped with some special exception handling. Causes the ruby interpreter to # exit if the block throws an exception. # # @api public # @param [String] message a message to log if the block fails # @param [Integer] code the exit code that the ruby interpreter should return if the block fails # @yield def exit_on_fail(message, code = 1) yield # First, we need to check and see if we are catching a SystemExit error. These will be raised # when we daemonize/fork, and they do not necessarily indicate a failure case. rescue SystemExit => err raise err # Now we need to catch *any* other kind of exception, because we may be calling third-party # code (e.g. webrick), and we have no idea what they might throw. rescue Exception => err ## NOTE: when debugging spec failures, these two lines can be very useful #puts err.inspect #puts Puppet::Util.pretty_backtrace(err.backtrace) Puppet.log_exception(err, "Could not #{message}: #{err}") Puppet::Util::Log.force_flushqueue() exit(code) end module_function :exit_on_fail def deterministic_rand(seed,max) if defined?(Random) == 'constant' && Random.class == Class Random.new(seed).rand(max).to_s else srand(seed) result = rand(max).to_s srand() result end end module_function :deterministic_rand ####################################################################################################### # Deprecated methods relating to process execution; these have been moved to Puppet::Util::Execution ####################################################################################################### def execpipe(command, failonfail = true, &block) Puppet.deprecation_warning("Puppet::Util.execpipe is deprecated; please use Puppet::Util::Execution.execpipe") Puppet::Util::Execution.execpipe(command, failonfail, &block) end module_function :execpipe def execfail(command, exception) Puppet.deprecation_warning("Puppet::Util.execfail is deprecated; please use Puppet::Util::Execution.execfail") Puppet::Util::Execution.execfail(command, exception) end module_function :execfail def execute(*args) Puppet.deprecation_warning("Puppet::Util.execute is deprecated; please use Puppet::Util::Execution.execute") Puppet::Util::Execution.execute(*args) end module_function :execute end end require 'puppet/util/errors' require 'puppet/util/methodhelper' require 'puppet/util/metaid' require 'puppet/util/classgen' require 'puppet/util/docs' require 'puppet/util/execution' require 'puppet/util/logging' require 'puppet/util/package' require 'puppet/util/warnings' diff --git a/lib/puppet/util/autoload.rb b/lib/puppet/util/autoload.rb index fc407a971..cab29bf07 100644 --- a/lib/puppet/util/autoload.rb +++ b/lib/puppet/util/autoload.rb @@ -1,224 +1,224 @@ require 'pathname' require 'puppet/util/rubygems' require 'puppet/util/warnings' require 'puppet/util/methodhelper' # Autoload paths, either based on names or all at once. class Puppet::Util::Autoload include Puppet::Util::MethodHelper @autoloaders = {} @loaded = {} class << self attr_reader :autoloaders attr_accessor :loaded private :autoloaders, :loaded def gem_source @gem_source ||= Puppet::Util::RubyGems::Source.new end # Has a given path been loaded? This is used for testing whether a # changed file should be loaded or just ignored. This is only # used in network/client/master, when downloading plugins, to # see if a given plugin is currently loaded and thus should be # reloaded. def loaded?(path) path = cleanpath(path).chomp('.rb') loaded.include?(path) end # Save the fact that a given path has been loaded. This is so # we can load downloaded plugins if they've already been loaded # into memory. def mark_loaded(name, file) name = cleanpath(name).chomp('.rb') ruby_file = name + ".rb" $LOADED_FEATURES << ruby_file unless $LOADED_FEATURES.include?(ruby_file) loaded[name] = [file, File.mtime(file)] end def changed?(name) name = cleanpath(name).chomp('.rb') return true unless loaded.include?(name) file, old_mtime = loaded[name] return true unless file == get_file(name) begin old_mtime != File.mtime(file) rescue Errno::ENOENT true end end # Load a single plugin by name. We use 'load' here so we can reload a # given plugin. def load_file(name, env=nil) file = get_file(name.to_s, env) return false unless file begin mark_loaded(name, file) Kernel.load file, @wrap return true rescue SystemExit,NoMemoryError raise rescue Exception => detail message = "Could not autoload #{name}: #{detail}" Puppet.log_exception(detail, message) raise Puppet::Error, message end end def loadall(path) # Load every instance of everything we can find. files_to_load(path).each do |file| name = file.chomp(".rb") load_file(name) unless loaded?(name) end end def reload_changed loaded.keys.each { |file| load_file(file) if changed?(file) } end # Get the correct file to load for a given path # returns nil if no file is found def get_file(name, env=nil) name = name + '.rb' unless name =~ /\.rb$/ path = search_directories(env).find { |dir| File.exist?(File.join(dir, name)) } path and File.join(path, name) end def files_to_load(path) search_directories.map {|dir| files_in_dir(dir, path) }.flatten.uniq end def files_in_dir(dir, path) dir = Pathname.new(File.expand_path(dir)) Dir.glob(File.join(dir, path, "*.rb")).collect do |file| Pathname.new(file).relative_path_from(dir).to_s end end def module_directories(env=nil) # We have to require this late in the process because otherwise we might # have load order issues. Since require is much slower than defined?, we # can skip that - and save some 2,155 invocations of require in my real # world testing. --daniel 2012-07-10 require 'puppet/node/environment' unless defined?(Puppet::Node::Environment) real_env = Puppet::Node::Environment.new(env) # We're using a per-thread cache of module directories so that we don't # scan the filesystem each time we try to load something. This is reset # at the beginning of compilation and at the end of an agent run. - Thread.current[:env_module_directories] ||= {} + $env_module_directories ||= {} # This is a little bit of a hack. Basically, the autoloader is being # called indirectly during application bootstrapping when we do things # such as check "features". However, during bootstrapping, we haven't # yet parsed all of the command line parameters nor the config files, # and thus we don't yet know with certainty what the module path is. # This should be irrelevant during bootstrapping, because anything that # we are attempting to load during bootstrapping should be something # that we ship with puppet, and thus the module path is irrelevant. # # In the long term, I think the way that we want to handle this is to # have the autoloader ignore the module path in all cases where it is # not specifically requested (e.g., by a constructor param or # something)... because there are very few cases where we should # actually be loading code from the module path. However, until that # happens, we at least need a way to prevent the autoloader from # attempting to access the module path before it is initialized. For # now we are accomplishing that by calling the # "app_defaults_initialized?" method on the main puppet Settings object. # --cprice 2012-03-16 if Puppet.settings.app_defaults_initialized? # if the app defaults have been initialized then it should be safe to access the module path setting. - Thread.current[:env_module_directories][real_env] ||= real_env.modulepath.collect do |dir| + $env_module_directories[real_env] ||= real_env.modulepath.collect do |dir| Dir.entries(dir).reject { |f| f =~ /^\./ }.collect { |f| File.join(dir, f) } end.flatten.collect { |d| File.join(d, "lib") }.find_all do |d| FileTest.directory?(d) end else # if we get here, the app defaults have not been initialized, so we basically use an empty module path. [] end end def libdirs() # See the comments in #module_directories above. Basically, we need to be careful not to try to access the # libdir before we know for sure that all of the settings have been initialized (e.g., during bootstrapping). if (Puppet.settings.app_defaults_initialized?) Puppet[:libdir].split(File::PATH_SEPARATOR) else [] end end def gem_directories gem_source.directories end def search_directories(env=nil) [gem_directories, module_directories(env), libdirs(), $LOAD_PATH].flatten end # Normalize a path. This converts ALT_SEPARATOR to SEPARATOR on Windows # and eliminates unnecessary parts of a path. def cleanpath(path) # There are two cases here because cleanpath does not handle absolute # paths correctly on windows (c:\ and c:/ are treated as distinct) but # we don't want to convert relative paths to absolute if Puppet::Util.absolute_path?(path) File.expand_path(path) else Pathname.new(path).cleanpath.to_s end end end # Send [] and []= to the @autoloaders hash Puppet::Util.classproxy self, :autoloaders, "[]", "[]=" attr_accessor :object, :path, :objwarn, :wrap def initialize(obj, path, options = {}) @path = path.to_s raise ArgumentError, "Autoload paths cannot be fully qualified" if Puppet::Util.absolute_path?(@path) @object = obj self.class[obj] = self set_options(options) @wrap = true unless defined?(@wrap) end def load(name, env=nil) self.class.load_file(expand(name), env) end # Load all instances that we can. This uses require, rather than load, # so that already-loaded files don't get reloaded unnecessarily. def loadall self.class.loadall(@path) end def loaded?(name) self.class.loaded?(expand(name)) end def changed?(name) self.class.changed?(expand(name)) end def files_to_load self.class.files_to_load(@path) end def expand(name) ::File.join(@path, name.to_s) end end diff --git a/lib/puppet/util/cacher.rb b/lib/puppet/util/cacher.rb index 136c9973e..24017de32 100644 --- a/lib/puppet/util/cacher.rb +++ b/lib/puppet/util/cacher.rb @@ -1,80 +1,74 @@ -require 'monitor' - module Puppet::Util::Cacher # Our module has been extended in a class; we can only add the Instance methods, # which become *class* methods in the class. def self.extended(other) class << other extend ClassMethods include InstanceMethods end end # Our module has been included in a class, which means the class gets the class methods # and all of its instances get the instance methods. def self.included(other) other.extend(ClassMethods) other.send(:include, InstanceMethods) end # Methods that can get added to a class. module ClassMethods # Provide a means of defining an attribute whose value will be cached. # Must provide a block capable of defining the value if it's flushed.. def cached_attr(name, ttl, &block) init_method = "init_#{name}" define_method(init_method, &block) set_attr_ttl(name, ttl) define_method(name) do cached_value(name) end define_method(name.to_s + "=") do |value| # Make sure the cache timestamp is set - value_cache.synchronize do - value_cache[name] = value - set_expiration(name) - end + value_cache[name] = value + set_expiration(name) end end def attr_ttl(name) @attr_ttls[name] end def set_attr_ttl(name, value) @attr_ttls ||= {} @attr_ttls[name] = Integer(value) end end # Methods that get added to instances. module InstanceMethods private def cached_value(name) - value_cache.synchronize do - if value_cache[name].nil? or expired_by_ttl?(name) - value_cache[name] = send("init_#{name}") - set_expiration(name) - end - value_cache[name] + if value_cache[name].nil? or expired_by_ttl?(name) + value_cache[name] = send("init_#{name}") + set_expiration(name) end + value_cache[name] end def expired_by_ttl?(name) @attr_expirations[name] < Time.now end def set_expiration(name) @attr_expirations ||= {} @attr_expirations[name] = Time.now + self.class.attr_ttl(name) end def value_cache - @value_cache ||= {}.extend(MonitorMixin) + @value_cache ||= {} end end end diff --git a/lib/puppet/util/instrumentation.rb b/lib/puppet/util/instrumentation.rb index eb522437e..94a75ac03 100644 --- a/lib/puppet/util/instrumentation.rb +++ b/lib/puppet/util/instrumentation.rb @@ -1,171 +1,152 @@ require 'puppet' require 'puppet/util/classgen' require 'puppet/util/instance_loader' class Puppet::Util::Instrumentation extend Puppet::Util::ClassGen extend Puppet::Util::InstanceLoader - extend MonitorMixin # we're using a ruby lazy autoloader to prevent a loop when requiring listeners # since this class sets up an indirection which is also used in Puppet::Indirector::Indirection # which is used to setup indirections... autoload :Listener, 'puppet/util/instrumentation/listener' autoload :Data, 'puppet/util/instrumentation/data' # Set up autoloading and retrieving of instrumentation listeners. instance_load :listener, 'puppet/util/instrumentation/listeners' class << self attr_accessor :listeners, :listeners_of end # instrumentation layer # Triggers an instrumentation # # Call this method around the instrumentation point # Puppet::Util::Instrumentation.instrument(:my_long_computation) do # ... a long computation # end # # This will send an event to all the listeners of "my_long_computation". # Note: this method uses ruby yield directive to call the instrumented code. # It is usually way slower than calling start and stop directly around the instrumented code. # For high traffic code path, it is thus advisable to not use this method. def self.instrument(label, data = {}) id = self.start(label, data) yield ensure self.stop(label, id, data) end # Triggers a "start" instrumentation event # # Important note: # For proper use, the data hash instance used for start should also # be used when calling stop. The idea is to use the current scope # where start is called to retain a reference to 'data' so that it is possible # to send it back to stop. # This way listeners can match start and stop events more easily. def self.start(label, data) data[:started] = Time.now publish(label, :start, data) data[:id] = next_id end # Triggers a "stop" instrumentation event def self.stop(label, id, data) data[:finished] = Time.now publish(label, :stop, data) end def self.publish(label, event, data) each_listener(label) do |k,l| l.notify(label, event, data) end end def self.listeners @listeners.values end def self.each_listener(label) - synchronize { - @listeners_of[label] ||= @listeners.select do |k,l| - l.listen_to?(label) - end - }.each do |l| + @listeners_of[label] ||= @listeners.select do |k,l| + l.listen_to?(label) + end.each do |l| yield l end end # Adds a new listener # # Usage: # Puppet::Util::Instrumentation.new_listener(:my_instrumentation, pattern) do # # def notify(label, data) # ... do something for data... # end # end # # It is possible to use a "pattern". The listener will be notified only # if the pattern match the label of the event. # The pattern can be a symbol, a string or a regex. # If no pattern is provided, then the listener will be called for every events def self.new_listener(name, options = {}, &block) Puppet.debug "new listener called #{name}" name = name.intern listener = genclass(name, :hash => instance_hash(:listener), :block => block) listener.send(:define_method, :name) do name end subscribe(listener.new, options[:label_pattern], options[:event]) end def self.subscribe(listener, label_pattern, event) - synchronize { - raise "Listener #{listener.name} is already subscribed" if @listeners.include?(listener.name) - Puppet.debug "registering instrumentation listener #{listener.name}" - @listeners[listener.name] = Listener.new(listener, label_pattern, event) - listener.subscribed if listener.respond_to?(:subscribed) - rehash - } + raise "Listener #{listener.name} is already subscribed" if @listeners.include?(listener.name) + Puppet.debug "registering instrumentation listener #{listener.name}" + @listeners[listener.name] = Listener.new(listener, label_pattern, event) + listener.subscribed if listener.respond_to?(:subscribed) + rehash end def self.unsubscribe(listener) - synchronize { - Puppet.warning("#{listener.name} hasn't been registered but asked to be unregistered") unless @listeners.include?(listener.name) - Puppet.info "unregistering instrumentation listener #{listener.name}" - @listeners.delete(listener.name) - listener.unsubscribed if listener.respond_to?(:unsubscribed) - rehash - } + Puppet.warning("#{listener.name} hasn't been registered but asked to be unregistered") unless @listeners.include?(listener.name) + Puppet.info "unregistering instrumentation listener #{listener.name}" + @listeners.delete(listener.name) + listener.unsubscribed if listener.respond_to?(:unsubscribed) + rehash end def self.init # let's init our probe indirection require 'puppet/util/instrumentation/indirection_probe' - synchronize { - @listeners ||= {} - @listeners_of ||= {} - instance_loader(:listener).loadall - } + @listeners ||= {} + @listeners_of ||= {} + instance_loader(:listener).loadall end def self.clear - synchronize { - @listeners = {} - @listeners_of = {} - @id = 0 - } + @listeners = {} + @listeners_of = {} + @id = 0 end def self.[](key) - synchronize { - @listeners[key.intern] - } + @listeners[key.intern] end def self.[]=(key, value) - synchronize { - @listeners[key.intern] = value - rehash - } + @listeners[key.intern] = value + rehash end private - # should be called only under the guard - # self.synchronize def self.rehash @listeners_of = {} end def self.next_id - synchronize { - @id = (@id || 0) + 1 - } + @id = (@id || 0) + 1 end end diff --git a/lib/puppet/util/instrumentation/instrumentable.rb b/lib/puppet/util/instrumentation/instrumentable.rb index 4c6d2f97e..01c75ff77 100644 --- a/lib/puppet/util/instrumentation/instrumentable.rb +++ b/lib/puppet/util/instrumentation/instrumentable.rb @@ -1,143 +1,136 @@ -require 'monitor' require 'puppet/util/instrumentation' # This is the central point of all declared probes. # Every class needed to declare probes should include this module # and declare the methods that are subject to instrumentation: # # class MyClass # extend Puppet::Util::Instrumentation::Instrumentable # # probe :mymethod # # def mymethod # ... this is code to be instrumented ... # end # end module Puppet::Util::Instrumentation::Instrumentable - INSTRUMENTED_CLASSES = {}.extend(MonitorMixin) + INSTRUMENTED_CLASSES = {} attr_reader :probes class Probe attr_reader :klass, :method, :label, :data def initialize(method, klass, options = {}) @method = method @klass = klass @label = options[:label] || method @data = options[:data] || {} end def enable raise "Probe already enabled" if enabled? # We're forced to perform this copy because in the class_eval'uated # block below @method would be evaluated in the class context. It's better # to close on locally-scoped variables than to resort to complex namespacing # to get access to the probe instance variables. method = @method; label = @label; data = @data klass.class_eval { alias_method("instrumented_#{method}", method) define_method(method) do |*args| id = nil instrumentation_data = nil begin instrumentation_label = label.respond_to?(:call) ? label.call(self, args) : label instrumentation_data = data.respond_to?(:call) ? data.call(self, args) : data id = Puppet::Util::Instrumentation.start(instrumentation_label, instrumentation_data) send("instrumented_#{method}".to_sym, *args) ensure Puppet::Util::Instrumentation.stop(instrumentation_label, id, instrumentation_data || {}) end end } @enabled = true end def disable raise "Probe is not enabled" unless enabled? # For the same reason as in #enable, we're forced to do a local # copy method = @method klass.class_eval do alias_method(method, "instrumented_#{method}") remove_method("instrumented_#{method}".to_sym) end @enabled = false end def enabled? !!@enabled end end # Declares a new probe # # It is possible to pass several options that will be later on evaluated # and sent to the instrumentation layer. # # label:: # this can either be a static symbol/string or a block. If it's a block # this one will be evaluated on every call of the instrumented method and # should return a string or a symbol # # data:: # this can be a hash or a block. If it's a block this one will be evaluated # on every call of the instrumented method and should return a hash. # #Example: # # class MyClass # extend Instrumentable # # probe :mymethod, :data => Proc.new { |args| { :data => args[1] } }, :label => Proc.new { |args| args[0] } # # def mymethod(name, options) # end # # end # def probe(method, options = {}) - INSTRUMENTED_CLASSES.synchronize { - (@probes ||= []) << Probe.new(method, self, options) - INSTRUMENTED_CLASSES[self] = @probes - } + (@probes ||= []) << Probe.new(method, self, options) + INSTRUMENTED_CLASSES[self] = @probes end def self.probes @probes end def self.probe_names probe_names = [] each_probe { |probe| probe_names << "#{probe.klass}.#{probe.method}" } probe_names end def self.enable_probes each_probe { |probe| probe.enable } end def self.disable_probes each_probe { |probe| probe.disable } end def self.clear_probes - INSTRUMENTED_CLASSES.synchronize { - INSTRUMENTED_CLASSES.clear - } + INSTRUMENTED_CLASSES.clear nil # do not leak our probes to the exterior world end def self.each_probe - INSTRUMENTED_CLASSES.synchronize { - INSTRUMENTED_CLASSES.each_key do |klass| - klass.probes.each { |probe| yield probe } - end - } + INSTRUMENTED_CLASSES.each_key do |klass| + klass.probes.each { |probe| yield probe } + end nil # do not leak our probes to the exterior world end end diff --git a/lib/puppet/util/instrumentation/listeners/log.rb b/lib/puppet/util/instrumentation/listeners/log.rb index fcff14621..72797d628 100644 --- a/lib/puppet/util/instrumentation/listeners/log.rb +++ b/lib/puppet/util/instrumentation/listeners/log.rb @@ -1,29 +1,23 @@ -require 'monitor' - # This is an example instrumentation listener that stores the last # 20 instrumented probe run time. Puppet::Util::Instrumentation.new_listener(:log) do SIZE = 20 attr_accessor :last_logs def initialize - @last_logs = {}.extend(MonitorMixin) + @last_logs = {} end def notify(label, event, data) return if event == :start log_line = "#{label} took #{data[:finished] - data[:started]}" - @last_logs.synchronize { - (@last_logs[label] ||= []) << log_line - @last_logs[label].shift if @last_logs[label].length > SIZE - } + (@last_logs[label] ||= []) << log_line + @last_logs[label].shift if @last_logs[label].length > SIZE end def data - @last_logs.synchronize { - @last_logs.dup - } + @last_logs.dup end end diff --git a/lib/puppet/util/instrumentation/listeners/performance.rb b/lib/puppet/util/instrumentation/listeners/performance.rb index 3ad51b7de..5bf3e2c5b 100644 --- a/lib/puppet/util/instrumentation/listeners/performance.rb +++ b/lib/puppet/util/instrumentation/listeners/performance.rb @@ -1,30 +1,24 @@ -require 'monitor' - Puppet::Util::Instrumentation.new_listener(:performance) do attr_reader :samples def initialize - @samples = {}.extend(MonitorMixin) + @samples = {} end def notify(label, event, data) return if event == :start duration = data[:finished] - data[:started] - samples.synchronize do - @samples[label] ||= { :count => 0, :max => 0, :min => nil, :sum => 0, :average => 0 } - @samples[label][:count] += 1 - @samples[label][:sum] += duration - @samples[label][:max] = [ @samples[label][:max], duration ].max - @samples[label][:min] = [ @samples[label][:min], duration ].reject { |val| val.nil? }.min - @samples[label][:average] = @samples[label][:sum] / @samples[label][:count] - end + @samples[label] ||= { :count => 0, :max => 0, :min => nil, :sum => 0, :average => 0 } + @samples[label][:count] += 1 + @samples[label][:sum] += duration + @samples[label][:max] = [ @samples[label][:max], duration ].max + @samples[label][:min] = [ @samples[label][:min], duration ].reject { |val| val.nil? }.min + @samples[label][:average] = @samples[label][:sum] / @samples[label][:count] end def data - samples.synchronize do - @samples.dup - end + @samples.dup end end diff --git a/lib/puppet/util/log.rb b/lib/puppet/util/log.rb index 3d34bba2c..cf88f89e0 100644 --- a/lib/puppet/util/log.rb +++ b/lib/puppet/util/log.rb @@ -1,330 +1,328 @@ require 'puppet/util/tagging' require 'puppet/util/classgen' # Pass feedback to the user. Log levels are modeled after syslog's, and it is # expected that that will be the most common log destination. Supports # multiple destinations, one of which is a remote server. class Puppet::Util::Log include Puppet::Util extend Puppet::Util::ClassGen include Puppet::Util::Tagging @levels = [:debug,:info,:notice,:warning,:err,:alert,:emerg,:crit] @loglevel = 2 @desttypes = {} # Create a new destination type. def self.newdesttype(name, options = {}, &block) dest = genclass( name, :parent => Puppet::Util::Log::Destination, :prefix => "Dest", :block => block, :hash => @desttypes, :attributes => options ) dest.match(dest.name) dest end require 'puppet/util/log/destination' require 'puppet/util/log/destinations' @destinations = {} @queued = [] class << self include Puppet::Util include Puppet::Util::ClassGen attr_reader :desttypes end # Reset log to basics. Basically just flushes and closes files and # undefs other objects. def Log.close(destination) if @destinations.include?(destination) @destinations[destination].flush if @destinations[destination].respond_to?(:flush) @destinations[destination].close if @destinations[destination].respond_to?(:close) @destinations.delete(destination) end end def self.close_all destinations.keys.each { |dest| close(dest) } raise Puppet::DevError.new("Log.close_all failed to close #{@destinations.keys.inspect}") if !@destinations.empty? end # Flush any log destinations that support such operations. def Log.flush @destinations.each { |type, dest| dest.flush if dest.respond_to?(:flush) } end def Log.autoflush=(v) @destinations.each do |type, dest| dest.autoflush = v if dest.respond_to?(:autoflush=) end end # Create a new log message. The primary role of this method is to # avoid creating log messages below the loglevel. def Log.create(hash) raise Puppet::DevError, "Logs require a level" unless hash.include?(:level) raise Puppet::DevError, "Invalid log level #{hash[:level]}" unless @levels.index(hash[:level]) @levels.index(hash[:level]) >= @loglevel ? Puppet::Util::Log.new(hash) : nil end def Log.destinations @destinations end # Yield each valid level in turn def Log.eachlevel @levels.each { |level| yield level } end # Return the current log level. def Log.level @levels[@loglevel] end # Set the current log level. def Log.level=(level) level = level.intern unless level.is_a?(Symbol) raise Puppet::DevError, "Invalid loglevel #{level}" unless @levels.include?(level) @loglevel = @levels.index(level) end def Log.levels @levels.dup end # Create a new log destination. def Log.newdestination(dest) # Each destination can only occur once. if @destinations.find { |name, obj| obj.name == dest } return end name, type = @desttypes.find do |name, klass| klass.match?(dest) end if type.respond_to?(:suitable?) and not type.suitable?(dest) return end raise Puppet::DevError, "Unknown destination type #{dest}" unless type begin if type.instance_method(:initialize).arity == 1 @destinations[dest] = type.new(dest) else @destinations[dest] = type.new end flushqueue @destinations[dest] rescue => detail Puppet.log_exception(detail) # If this was our only destination, then add the console back in. newdestination(:console) if @destinations.empty? and (dest != :console and dest != "console") end end def Log.with_destination(destination, &block) if @destinations.include?(destination) yield else newdestination(destination) begin yield ensure close(destination) end end end # Route the actual message. FIXME There are lots of things this method # should do, like caching and a bit more. It's worth noting that there's # a potential for a loop here, if the machine somehow gets the destination set as # itself. def Log.newmessage(msg) return if @levels.index(msg.level) < @loglevel queuemessage(msg) if @destinations.length == 0 @destinations.each do |name, dest| - threadlock(dest) do - dest.handle(msg) - end + dest.handle(msg) end end def Log.queuemessage(msg) @queued.push(msg) end def Log.flushqueue return unless @destinations.size >= 1 @queued.each do |msg| Log.newmessage(msg) end @queued.clear end # Flush the logging queue. If there are no destinations available, # adds in a console logger before flushing the queue. # This is mainly intended to be used as a last-resort attempt # to ensure that logging messages are not thrown away before # the program is about to exit--most likely in a horrific # error scenario. # @return nil def Log.force_flushqueue() if (@destinations.empty? and !(@queued.empty?)) newdestination(:console) end flushqueue end def Log.sendlevel?(level) @levels.index(level) >= @loglevel end # Reopen all of our logs. def Log.reopen Puppet.notice "Reopening log files" types = @destinations.keys @destinations.each { |type, dest| dest.close if dest.respond_to?(:close) } @destinations.clear # We need to make sure we always end up with some kind of destination begin types.each { |type| Log.newdestination(type) } rescue => detail if @destinations.empty? Log.setup_default Puppet.err detail.to_s end end end def self.setup_default Log.newdestination( (Puppet.features.syslog? ? :syslog : (Puppet.features.eventlog? ? :eventlog : Puppet[:puppetdlog]))) end # Is the passed level a valid log level? def self.validlevel?(level) @levels.include?(level) end def self.from_pson(data) obj = allocate obj.initialize_from_hash(data) obj end attr_accessor :time, :remote, :file, :line, :source attr_reader :level, :message def initialize(args) self.level = args[:level] self.message = args[:message] self.source = args[:source] || "Puppet" @time = Time.now if tags = args[:tags] tags.each { |t| self.tag(t) } end [:file, :line].each do |attr| next unless value = args[attr] send(attr.to_s + "=", value) end Log.newmessage(self) end def initialize_from_hash(data) @level = data['level'].intern @message = data['message'] @source = data['source'] @tags = data['tags'] @time = data['time'] if @time.is_a? String @time = Time.parse(@time) end @file = data['file'] if data['file'] @line = data['line'] if data['line'] end def to_pson { 'level' => @level, 'message' => @message, 'source' => @source, 'tags' => @tags, 'time' => @time.iso8601(9), 'file' => @file, 'line' => @line, }.to_pson end def message=(msg) raise ArgumentError, "Puppet::Util::Log requires a message" unless msg @message = msg.to_s end def level=(level) raise ArgumentError, "Puppet::Util::Log requires a log level" unless level raise ArgumentError, "Puppet::Util::Log requires a symbol or string" unless level.respond_to? "to_sym" @level = level.to_sym raise ArgumentError, "Invalid log level #{@level}" unless self.class.validlevel?(@level) # Tag myself with my log level tag(level) end # If they pass a source in to us, we make sure it is a string, and # we retrieve any tags we can. def source=(source) if source.respond_to?(:path) @source = source.path source.tags.each { |t| tag(t) } self.file = source.file self.line = source.line else @source = source.to_s end end def to_report "#{time} #{source} (#{level}): #{to_s}" end def to_s message end end # This is for backward compatibility from when we changed the constant to Puppet::Util::Log # because the reports include the constant name. Apparently the alias was created in # March 2007, should could probably be removed soon. Puppet::Log = Puppet::Util::Log diff --git a/lib/puppet/util/profiler.rb b/lib/puppet/util/profiler.rb index 0c3a3768f..49ae17769 100644 --- a/lib/puppet/util/profiler.rb +++ b/lib/puppet/util/profiler.rb @@ -1,28 +1,28 @@ require 'benchmark' # A simple profiling callback system. # # @api private module Puppet::Util::Profiler require 'puppet/util/profiler/wall_clock' require 'puppet/util/profiler/object_counts' require 'puppet/util/profiler/none' NONE = Puppet::Util::Profiler::None.new # @return This thread's configured profiler def self.current - Thread.current[:profiler] || NONE + @profiler || NONE end # @param profiler [#profile] A profiler for the current thread def self.current=(profiler) - Thread.current[:profiler] = profiler + @profiler = profiler end # @param message [String] A description of the profiled event # @param block [Block] The segment of code to profile def self.profile(message, &block) current.profile(message, &block) end end diff --git a/spec/unit/agent_spec.rb b/spec/unit/agent_spec.rb index 83524eae8..2c1e1ebbd 100755 --- a/spec/unit/agent_spec.rb +++ b/spec/unit/agent_spec.rb @@ -1,330 +1,318 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet/agent' class AgentTestClient def run # no-op end def stop # no-op end end def without_warnings flag = $VERBOSE $VERBOSE = nil yield $VERBOSE = flag end describe Puppet::Agent do before do @agent = Puppet::Agent.new(AgentTestClient, false) # So we don't actually try to hit the filesystem. @agent.stubs(:lock).yields # make Puppet::Application safe for stubbing; restore in an :after block; silence warnings for this. without_warnings { Puppet::Application = Class.new(Puppet::Application) } Puppet::Application.stubs(:clear?).returns(true) Puppet::Application.class_eval do class << self def controlled_run(&block) block.call end end end end after do # restore Puppet::Application from stub-safe subclass, and silence warnings without_warnings { Puppet::Application = Puppet::Application.superclass } end it "should set its client class at initialization" do Puppet::Agent.new("foo", false).client_class.should == "foo" end it "should include the Locker module" do Puppet::Agent.ancestors.should be_include(Puppet::Agent::Locker) end it "should create an instance of its client class and run it when asked to run" do client = mock 'client' AgentTestClient.expects(:new).returns client client.expects(:run) @agent.stubs(:running?).returns false @agent.stubs(:disabled?).returns false @agent.run end it "should be considered running if the lock file is locked" do lockfile = mock 'lockfile' @agent.expects(:lockfile).returns(lockfile) lockfile.expects(:locked?).returns true @agent.should be_running end describe "when being run" do before do AgentTestClient.stubs(:lockfile_path).returns "/my/lock" @agent.stubs(:running?).returns false @agent.stubs(:disabled?).returns false end it "should splay" do @agent.expects(:splay) @agent.run end it "should do nothing if already running" do @agent.expects(:running?).returns true AgentTestClient.expects(:new).never @agent.run end it "should do nothing if disabled" do @agent.expects(:disabled?).returns(true) AgentTestClient.expects(:new).never @agent.run end it "(#11057) should notify the user about why a run is skipped" do Puppet::Application.stubs(:controlled_run).returns(false) Puppet::Application.stubs(:run_status).returns('MOCK_RUN_STATUS') # This is the actual test that we inform the user why the run is skipped. # We assume this information is contained in # Puppet::Application.run_status Puppet.expects(:notice).with(regexp_matches(/MOCK_RUN_STATUS/)) @agent.run end it "should display an informative message if the agent is administratively disabled" do @agent.expects(:disabled?).returns true @agent.expects(:disable_message).returns "foo" Puppet.expects(:notice).with(regexp_matches(/Skipping run of .*; administratively disabled.*\(Reason: 'foo'\)/)) @agent.run end it "should use Puppet::Application.controlled_run to manage process state behavior" do calls = sequence('calls') Puppet::Application.expects(:controlled_run).yields.in_sequence(calls) AgentTestClient.expects(:new).once.in_sequence(calls) @agent.run end it "should not fail if a client class instance cannot be created" do AgentTestClient.expects(:new).raises "eh" Puppet.expects(:err) @agent.run end it "should not fail if there is an exception while running its client" do client = AgentTestClient.new AgentTestClient.expects(:new).returns client client.expects(:run).raises "eh" Puppet.expects(:err) @agent.run end - it "should use a mutex to restrict multi-threading" do - client = AgentTestClient.new - AgentTestClient.expects(:new).returns client - - mutex = mock 'mutex' - @agent.expects(:sync).returns mutex - - mutex.expects(:synchronize) - client.expects(:run).never # if it doesn't run, then we know our yield is what triggers it - @agent.run - end - it "should use a filesystem lock to restrict multiple processes running the agent" do client = AgentTestClient.new AgentTestClient.expects(:new).returns client @agent.expects(:lock) client.expects(:run).never # if it doesn't run, then we know our yield is what triggers it @agent.run end it "should make its client instance available while running" do client = AgentTestClient.new AgentTestClient.expects(:new).returns client client.expects(:run).with { @agent.client.should equal(client); true } @agent.run end it "should run the client instance with any arguments passed to it" do client = AgentTestClient.new AgentTestClient.expects(:new).returns client client.expects(:run).with(:pluginsync => true, :other => :options) @agent.run(:other => :options) end it "should return the agent result" do client = AgentTestClient.new AgentTestClient.expects(:new).returns client @agent.expects(:lock).returns(:result) @agent.run.should == :result end describe "when should_fork is true" do before do @agent = Puppet::Agent.new(AgentTestClient, true) # So we don't actually try to hit the filesystem. @agent.stubs(:lock).yields Kernel.stubs(:fork) Process.stubs(:waitpid2).returns [123, (stub 'process::status', :exitstatus => 0)] @agent.stubs(:exit) end it "should run the agent in a forked process" do client = AgentTestClient.new AgentTestClient.expects(:new).returns client client.expects(:run) Kernel.expects(:fork).yields @agent.run end it "should exit child process if child exit" do client = AgentTestClient.new AgentTestClient.expects(:new).returns client client.expects(:run).raises(SystemExit) Kernel.expects(:fork).yields @agent.expects(:exit).with(-1) @agent.run end it "should re-raise exit happening in the child" do Process.stubs(:waitpid2).returns [123, (stub 'process::status', :exitstatus => -1)] lambda { @agent.run }.should raise_error(SystemExit) end it "should re-raise NoMoreMemory happening in the child" do Process.stubs(:waitpid2).returns [123, (stub 'process::status', :exitstatus => -2)] lambda { @agent.run }.should raise_error(NoMemoryError) end it "should return the child exit code" do Process.stubs(:waitpid2).returns [123, (stub 'process::status', :exitstatus => 777)] @agent.run.should == 777 end it "should return the block exit code as the child exit code" do Kernel.expects(:fork).yields @agent.expects(:exit).with(777) @agent.run_in_fork { 777 } end end end describe "when splaying" do before do Puppet[:splay] = true Puppet[:splaylimit] = "10" end it "should do nothing if splay is disabled" do Puppet[:splay] = false @agent.expects(:sleep).never @agent.splay end it "should do nothing if it has already splayed" do @agent.expects(:splayed?).returns true @agent.expects(:sleep).never @agent.splay end it "should log that it is splaying" do @agent.stubs :sleep Puppet.expects :info @agent.splay end it "should sleep for a random portion of the splaylimit plus 1" do Puppet[:splaylimit] = "50" @agent.expects(:rand).with(51).returns 10 @agent.expects(:sleep).with(10) @agent.splay end it "should mark that it has splayed" do @agent.stubs(:sleep) @agent.splay @agent.should be_splayed end end describe "when checking execution state" do describe 'with regular run status' do before :each do Puppet::Application.stubs(:restart_requested?).returns(false) Puppet::Application.stubs(:stop_requested?).returns(false) Puppet::Application.stubs(:interrupted?).returns(false) Puppet::Application.stubs(:clear?).returns(true) end it 'should be false for :stopping?' do @agent.stopping?.should be_false end it 'should be false for :needing_restart?' do @agent.needing_restart?.should be_false end end describe 'with a stop requested' do before :each do Puppet::Application.stubs(:clear?).returns(false) Puppet::Application.stubs(:restart_requested?).returns(false) Puppet::Application.stubs(:stop_requested?).returns(true) Puppet::Application.stubs(:interrupted?).returns(true) end it 'should be true for :stopping?' do @agent.stopping?.should be_true end it 'should be false for :needing_restart?' do @agent.needing_restart?.should be_false end end describe 'with a restart requested' do before :each do Puppet::Application.stubs(:clear?).returns(false) Puppet::Application.stubs(:restart_requested?).returns(true) Puppet::Application.stubs(:stop_requested?).returns(false) Puppet::Application.stubs(:interrupted?).returns(true) end it 'should be false for :stopping?' do @agent.stopping?.should be_false end it 'should be true for :needing_restart?' do @agent.needing_restart?.should be_true end end end end diff --git a/spec/unit/configurer_spec.rb b/spec/unit/configurer_spec.rb index cba1df584..9dea9c438 100755 --- a/spec/unit/configurer_spec.rb +++ b/spec/unit/configurer_spec.rb @@ -1,645 +1,645 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet/configurer' describe Puppet::Configurer do before do Puppet.settings.stubs(:use).returns(true) @agent = Puppet::Configurer.new @agent.stubs(:init_storage) Puppet::Util::Storage.stubs(:store) Puppet[:server] = "puppetmaster" Puppet[:report] = true end it "should include the Plugin Handler module" do Puppet::Configurer.ancestors.should be_include(Puppet::Configurer::PluginHandler) end it "should include the Fact Handler module" do Puppet::Configurer.ancestors.should be_include(Puppet::Configurer::FactHandler) end describe "when executing a pre-run hook" do it "should do nothing if the hook is set to an empty string" do Puppet.settings[:prerun_command] = "" Puppet::Util.expects(:exec).never @agent.execute_prerun_command end it "should execute any pre-run command provided via the 'prerun_command' setting" do Puppet.settings[:prerun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @agent.execute_prerun_command end it "should fail if the command fails" do Puppet.settings[:prerun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @agent.execute_prerun_command.should be_false end end describe "when executing a post-run hook" do it "should do nothing if the hook is set to an empty string" do Puppet.settings[:postrun_command] = "" Puppet::Util.expects(:exec).never @agent.execute_postrun_command end it "should execute any post-run command provided via the 'postrun_command' setting" do Puppet.settings[:postrun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @agent.execute_postrun_command end it "should fail if the command fails" do Puppet.settings[:postrun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @agent.execute_postrun_command.should be_false end end describe "when executing a catalog run" do before do Puppet.settings.stubs(:use).returns(true) @agent.stubs(:download_plugins) Puppet::Node::Facts.indirection.terminus_class = :memory @facts = Puppet::Node::Facts.new(Puppet[:node_name_value]) Puppet::Node::Facts.indirection.save(@facts) @catalog = Puppet::Resource::Catalog.new @catalog.stubs(:to_ral).returns(@catalog) Puppet::Resource::Catalog.indirection.terminus_class = :rest Puppet::Resource::Catalog.indirection.stubs(:find).returns(@catalog) @agent.stubs(:send_report) @agent.stubs(:save_last_run_summary) Puppet::Util::Log.stubs(:close_all) end after :all do Puppet::Node::Facts.indirection.reset_terminus_class Puppet::Resource::Catalog.indirection.reset_terminus_class end it "should initialize storage" do Puppet::Util::Storage.expects(:load) @agent.run end it "downloads plugins when told" do @agent.expects(:download_plugins) @agent.run(:pluginsync => true) end it "does not download plugins when told" do @agent.expects(:download_plugins).never @agent.run(:pluginsync => false) end it "should carry on when it can't fetch its node definition" do error = Net::HTTPError.new(400, 'dummy server communication error') Puppet::Node.indirection.expects(:find).raises(error) @agent.run.should == 0 end it "should initialize a transaction report if one is not provided" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns report @agent.run end it "should respect node_name_fact when setting the host on a report" do Puppet[:node_name_fact] = 'my_name_fact' @facts.values = {'my_name_fact' => 'node_name_from_fact'} report = Puppet::Transaction::Report.new("apply") @agent.run(:report => report) report.host.should == 'node_name_from_fact' end it "should pass the new report to the catalog" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.stubs(:new).returns report @catalog.expects(:apply).with{|options| options[:report] == report} @agent.run end it "should use the provided report if it was passed one" do report = Puppet::Transaction::Report.new("apply") @catalog.expects(:apply).with {|options| options[:report] == report} @agent.run(:report => report) end it "should set the report as a log destination" do report = Puppet::Transaction::Report.new("apply") report.expects(:<<).with(instance_of(Puppet::Util::Log)).at_least_once @agent.run(:report => report) end it "should retrieve the catalog" do @agent.expects(:retrieve_catalog) @agent.run end it "should log a failure and do nothing if no catalog can be retrieved" do @agent.expects(:retrieve_catalog).returns nil Puppet.expects(:err).with "Could not retrieve catalog; skipping run" @agent.run end it "should apply the catalog with all options to :run" do @agent.expects(:retrieve_catalog).returns @catalog @catalog.expects(:apply).with { |args| args[:one] == true } @agent.run :one => true end it "should accept a catalog and use it instead of retrieving a different one" do @agent.expects(:retrieve_catalog).never @catalog.expects(:apply) @agent.run :one => true, :catalog => @catalog end it "should benchmark how long it takes to apply the catalog" do @agent.expects(:benchmark).with(:notice, "Finished catalog run") @agent.expects(:retrieve_catalog).returns @catalog @catalog.expects(:apply).never # because we're not yielding @agent.run end it "should execute post-run hooks after the run" do @agent.expects(:execute_postrun_command) @agent.run end it "should send the report" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) @agent.expects(:send_report).with(report) @agent.run end it "should send the transaction report even if the catalog could not be retrieved" do @agent.expects(:retrieve_catalog).returns nil report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) @agent.expects(:send_report) @agent.run end it "should send the transaction report even if there is a failure" do @agent.expects(:retrieve_catalog).raises "whatever" report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) @agent.expects(:send_report) @agent.run.should be_nil end it "should remove the report as a log destination when the run is finished" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) @agent.run Puppet::Util::Log.destinations.should_not include(report) end it "should return the report exit_status as the result of the run" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) report.expects(:exit_status).returns(1234) @agent.run.should == 1234 end it "should send the transaction report even if the pre-run command fails" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) Puppet.settings[:prerun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @agent.expects(:send_report) @agent.run.should be_nil end it "should include the pre-run command failure in the report" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) Puppet.settings[:prerun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @agent.run.should be_nil report.logs.find { |x| x.message =~ /Could not run command from prerun_command/ }.should be end it "should send the transaction report even if the post-run command fails" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) Puppet.settings[:postrun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @agent.expects(:send_report) @agent.run.should be_nil end it "should include the post-run command failure in the report" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) Puppet.settings[:postrun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") report.expects(:<<).with { |log| log.message.include?("Could not run command from postrun_command") } @agent.run.should be_nil end it "should execute post-run command even if the pre-run command fails" do Puppet.settings[:prerun_command] = "/my/precommand" Puppet.settings[:postrun_command] = "/my/postcommand" Puppet::Util::Execution.expects(:execute).with(["/my/precommand"]).raises(Puppet::ExecutionFailure, "Failed") Puppet::Util::Execution.expects(:execute).with(["/my/postcommand"]) @agent.run.should be_nil end it "should finalize the report" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) report.expects(:finalize_report) @agent.run end it "should not apply the catalog if the pre-run command fails" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) Puppet.settings[:prerun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @catalog.expects(:apply).never() @agent.expects(:send_report) @agent.run.should be_nil end it "should apply the catalog, send the report, and return nil if the post-run command fails" do report = Puppet::Transaction::Report.new("apply") Puppet::Transaction::Report.expects(:new).returns(report) Puppet.settings[:postrun_command] = "/my/command" Puppet::Util::Execution.expects(:execute).with(["/my/command"]).raises(Puppet::ExecutionFailure, "Failed") @catalog.expects(:apply) @agent.expects(:send_report) @agent.run.should be_nil end it "should refetch the catalog if the server specifies a new environment in the catalog" do @catalog.stubs(:environment).returns("second_env") @agent.expects(:retrieve_catalog).returns(@catalog).twice @agent.run end it "should change the environment setting if the server specifies a new environment in the catalog" do @catalog.stubs(:environment).returns("second_env") @agent.run @agent.environment.should == "second_env" end - it "should clear the thread local caches" do - Thread.current[:env_module_directories] = false + it "should clear the global caches" do + $env_module_directories = false @agent.run - Thread.current[:env_module_directories].should == nil + $env_module_directories.should == nil end describe "when not using a REST terminus for catalogs" do it "should not pass any facts when retrieving the catalog" do Puppet::Resource::Catalog.indirection.terminus_class = :compiler @agent.expects(:facts_for_uploading).never Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:facts].nil? }.returns @catalog @agent.run end end describe "when using a REST terminus for catalogs" do it "should pass the prepared facts and the facts format as arguments when retrieving the catalog" do Puppet::Resource::Catalog.indirection.terminus_class = :rest @agent.expects(:facts_for_uploading).returns(:facts => "myfacts", :facts_format => :foo) Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:facts] == "myfacts" and options[:facts_format] == :foo }.returns @catalog @agent.run end end end describe "when sending a report" do include PuppetSpec::Files before do Puppet.settings.stubs(:use).returns(true) @configurer = Puppet::Configurer.new Puppet[:lastrunfile] = tmpfile('last_run_file') @report = Puppet::Transaction::Report.new("apply") Puppet[:reports] = "none" end it "should print a report summary if configured to do so" do Puppet.settings[:summarize] = true @report.expects(:summary).returns "stuff" @configurer.expects(:puts).with("stuff") @configurer.send_report(@report) end it "should not print a report summary if not configured to do so" do Puppet.settings[:summarize] = false @configurer.expects(:puts).never @configurer.send_report(@report) end it "should save the report if reporting is enabled" do Puppet.settings[:report] = true Puppet::Transaction::Report.indirection.expects(:save).with(@report, nil, instance_of(Hash)) @configurer.send_report(@report) end it "should not save the report if reporting is disabled" do Puppet.settings[:report] = false Puppet::Transaction::Report.indirection.expects(:save).with(@report, nil, instance_of(Hash)).never @configurer.send_report(@report) end it "should save the last run summary if reporting is enabled" do Puppet.settings[:report] = true @configurer.expects(:save_last_run_summary).with(@report) @configurer.send_report(@report) end it "should save the last run summary if reporting is disabled" do Puppet.settings[:report] = false @configurer.expects(:save_last_run_summary).with(@report) @configurer.send_report(@report) end it "should log but not fail if saving the report fails" do Puppet.settings[:report] = true Puppet::Transaction::Report.indirection.expects(:save).raises("whatever") Puppet.expects(:err) lambda { @configurer.send_report(@report) }.should_not raise_error end end describe "when saving the summary report file" do include PuppetSpec::Files before do Puppet.settings.stubs(:use).returns(true) @configurer = Puppet::Configurer.new @report = stub 'report', :raw_summary => {} Puppet[:lastrunfile] = tmpfile('last_run_file') end it "should write the last run file" do @configurer.save_last_run_summary(@report) FileTest.exists?(Puppet[:lastrunfile]).should be_true end it "should write the raw summary as yaml" do @report.expects(:raw_summary).returns("summary") @configurer.save_last_run_summary(@report) File.read(Puppet[:lastrunfile]).should == YAML.dump("summary") end it "should log but not fail if saving the last run summary fails" do # The mock will raise an exception on any method used. This should # simulate a nice hard failure from the underlying OS for us. fh = Class.new(Object) do def method_missing(*args) raise "failed to do #{args[0]}" end end.new Puppet::Util.expects(:replace_file).yields(fh) Puppet.expects(:err) expect { @configurer.save_last_run_summary(@report) }.to_not raise_error end it "should create the last run file with the correct mode" do Puppet.settings.setting(:lastrunfile).expects(:mode).returns('664') @configurer.save_last_run_summary(@report) if Puppet::Util::Platform.windows? require 'puppet/util/windows/security' mode = Puppet::Util::Windows::Security.get_mode(Puppet[:lastrunfile]) else mode = File.stat(Puppet[:lastrunfile]).mode end (mode & 0777).should == 0664 end it "should report invalid last run file permissions" do Puppet.settings.setting(:lastrunfile).expects(:mode).returns('892') Puppet.expects(:err).with(regexp_matches(/Could not save last run local report.*892 is invalid/)) @configurer.save_last_run_summary(@report) end end describe "when retrieving a catalog" do before do Puppet.settings.stubs(:use).returns(true) @agent.stubs(:facts_for_uploading).returns({}) @catalog = Puppet::Resource::Catalog.new # this is the default when using a Configurer instance Puppet::Resource::Catalog.indirection.stubs(:terminus_class).returns :rest @agent.stubs(:convert_catalog).returns @catalog end describe "and configured to only retrieve a catalog from the cache" do before do Puppet.settings[:use_cached_catalog] = true end it "should first look in the cache for a catalog" do Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_terminus] == true }.returns @catalog Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.never @agent.retrieve_catalog({}).should == @catalog end it "should compile a new catalog if none is found in the cache" do Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_terminus] == true }.returns nil Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.returns @catalog @agent.retrieve_catalog({}).should == @catalog end end it "should use the Catalog class to get its catalog" do Puppet::Resource::Catalog.indirection.expects(:find).returns @catalog @agent.retrieve_catalog({}) end it "should use its node_name_value to retrieve the catalog" do Facter.stubs(:value).returns "eh" Puppet.settings[:node_name_value] = "myhost.domain.com" Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| name == "myhost.domain.com" }.returns @catalog @agent.retrieve_catalog({}) end it "should default to returning a catalog retrieved directly from the server, skipping the cache" do Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.returns @catalog @agent.retrieve_catalog({}).should == @catalog end it "should log and return the cached catalog when no catalog can be retrieved from the server" do Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.returns nil Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_terminus] == true }.returns @catalog Puppet.expects(:notice) @agent.retrieve_catalog({}).should == @catalog end it "should not look in the cache for a catalog if one is returned from the server" do Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.returns @catalog Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_terminus] == true }.never @agent.retrieve_catalog({}).should == @catalog end it "should return the cached catalog when retrieving the remote catalog throws an exception" do Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.raises "eh" Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_terminus] == true }.returns @catalog @agent.retrieve_catalog({}).should == @catalog end it "should log and return nil if no catalog can be retrieved from the server and :usecacheonfailure is disabled" do Puppet[:usecacheonfailure] = false Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.returns nil Puppet.expects(:warning) @agent.retrieve_catalog({}).should be_nil end it "should return nil if no cached catalog is available and no catalog can be retrieved from the server" do Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_cache] == true }.returns nil Puppet::Resource::Catalog.indirection.expects(:find).with { |name, options| options[:ignore_terminus] == true }.returns nil @agent.retrieve_catalog({}).should be_nil end it "should convert the catalog before returning" do Puppet::Resource::Catalog.indirection.stubs(:find).returns @catalog @agent.expects(:convert_catalog).with { |cat, dur| cat == @catalog }.returns "converted catalog" @agent.retrieve_catalog({}).should == "converted catalog" end it "should return nil if there is an error while retrieving the catalog" do Puppet::Resource::Catalog.indirection.expects(:find).at_least_once.raises "eh" @agent.retrieve_catalog({}).should be_nil end end describe "when converting the catalog" do before do Puppet.settings.stubs(:use).returns(true) @catalog = Puppet::Resource::Catalog.new @oldcatalog = stub 'old_catalog', :to_ral => @catalog end it "should convert the catalog to a RAL-formed catalog" do @oldcatalog.expects(:to_ral).returns @catalog @agent.convert_catalog(@oldcatalog, 10).should equal(@catalog) end it "should finalize the catalog" do @catalog.expects(:finalize) @agent.convert_catalog(@oldcatalog, 10) end it "should record the passed retrieval time with the RAL catalog" do @catalog.expects(:retrieval_duration=).with 10 @agent.convert_catalog(@oldcatalog, 10) end it "should write the RAL catalog's class file" do @catalog.expects(:write_class_file) @agent.convert_catalog(@oldcatalog, 10) end it "should write the RAL catalog's resource file" do @catalog.expects(:write_resource_file) @agent.convert_catalog(@oldcatalog, 10) end end end diff --git a/spec/unit/node/environment_spec.rb b/spec/unit/node/environment_spec.rb index d2d3e8257..27c627baf 100755 --- a/spec/unit/node/environment_spec.rb +++ b/spec/unit/node/environment_spec.rb @@ -1,469 +1,455 @@ #! /usr/bin/env ruby require 'spec_helper' require 'tmpdir' require 'puppet/node/environment' require 'puppet/util/execution' require 'puppet_spec/modules' require 'puppet/parser/parser_factory' describe Puppet::Node::Environment do let(:env) { Puppet::Node::Environment.new("testing") } include PuppetSpec::Files after do Puppet::Node::Environment.clear end shared_examples_for 'the environment' do it "should use the filetimeout for the ttl for the modulepath" do Puppet::Node::Environment.attr_ttl(:modulepath).should == Integer(Puppet[:filetimeout]) end it "should use the filetimeout for the ttl for the module list" do Puppet::Node::Environment.attr_ttl(:modules).should == Integer(Puppet[:filetimeout]) end it "should use the default environment if no name is provided while initializing an environment" do Puppet[:environment] = "one" Puppet::Node::Environment.new.name.should == :one end it "should treat environment instances as singletons" do Puppet::Node::Environment.new("one").should equal(Puppet::Node::Environment.new("one")) end it "should treat an environment specified as names or strings as equivalent" do Puppet::Node::Environment.new(:one).should equal(Puppet::Node::Environment.new("one")) end it "should return its name when converted to a string" do Puppet::Node::Environment.new(:one).to_s.should == "one" end it "should just return any provided environment if an environment is provided as the name" do one = Puppet::Node::Environment.new(:one) Puppet::Node::Environment.new(one).should equal(one) end describe "when managing known resource types" do before do @collection = Puppet::Resource::TypeCollection.new(env) env.stubs(:perform_initial_import).returns(Puppet::Parser::AST::Hostclass.new('')) - Thread.current[:known_resource_types] = nil + $known_resource_types = nil end it "should create a resource type collection if none exists" do Puppet::Resource::TypeCollection.expects(:new).with(env).returns @collection env.known_resource_types.should equal(@collection) end it "should reuse any existing resource type collection" do env.known_resource_types.should equal(env.known_resource_types) end it "should perform the initial import when creating a new collection" do env.expects(:perform_initial_import).returns(Puppet::Parser::AST::Hostclass.new('')) env.known_resource_types end it "should return the same collection even if stale if it's the same thread" do Puppet::Resource::TypeCollection.stubs(:new).returns @collection env.known_resource_types.stubs(:stale?).returns true env.known_resource_types.should equal(@collection) end it "should return the current thread associated collection if there is one" do - Thread.current[:known_resource_types] = @collection + $known_resource_types = @collection env.known_resource_types.should equal(@collection) end - it "should give to all threads using the same environment the same collection if the collection isn't stale" do - @original_thread_type_collection = Puppet::Resource::TypeCollection.new(env) - Puppet::Resource::TypeCollection.expects(:new).with(env).returns @original_thread_type_collection - env.known_resource_types.should equal(@original_thread_type_collection) - - @original_thread_type_collection.expects(:require_reparse?).returns(false) - Puppet::Resource::TypeCollection.stubs(:new).with(env).returns @collection - - t = Thread.new { - env.known_resource_types.should equal(@original_thread_type_collection) - } - t.join - end - it "should generate a new TypeCollection if the current one requires reparsing" do old_type_collection = env.known_resource_types old_type_collection.stubs(:require_reparse?).returns true - Thread.current[:known_resource_types] = nil + $known_resource_types = nil new_type_collection = env.known_resource_types new_type_collection.should be_a Puppet::Resource::TypeCollection new_type_collection.should_not equal(old_type_collection) end end it "should validate the modulepath directories" do real_file = tmpdir('moduledir') path = %W[/one /two #{real_file}].join(File::PATH_SEPARATOR) Puppet[:modulepath] = path env.modulepath.should == [real_file] end it "should prefix the value of the 'PUPPETLIB' environment variable to the module path if present" do Puppet::Util.withenv("PUPPETLIB" => %w{/l1 /l2}.join(File::PATH_SEPARATOR)) do module_path = %w{/one /two}.join(File::PATH_SEPARATOR) env.expects(:validate_dirs).with(%w{/l1 /l2 /one /two}).returns %w{/l1 /l2 /one /two} env.expects(:[]).with(:modulepath).returns module_path env.modulepath.should == %w{/l1 /l2 /one /two} end end describe "when validating modulepath or manifestdir directories" do before :each do @path_one = tmpdir("path_one") @path_two = tmpdir("path_one") sep = File::PATH_SEPARATOR Puppet[:modulepath] = "#{@path_one}#{sep}#{@path_two}" end it "should not return non-directories" do FileTest.expects(:directory?).with(@path_one).returns true FileTest.expects(:directory?).with(@path_two).returns false env.validate_dirs([@path_one, @path_two]).should == [@path_one] end it "should use the current working directory to fully-qualify unqualified paths" do FileTest.stubs(:directory?).returns true two = File.expand_path(File.join(Dir.getwd, "two")) env.validate_dirs([@path_one, 'two']).should == [@path_one, two] end end describe "when modeling a specific environment" do it "should have a method for returning the environment name" do Puppet::Node::Environment.new("testing").name.should == :testing end it "should provide an array-like accessor method for returning any environment-specific setting" do env.should respond_to(:[]) end it "should ask the Puppet settings instance for the setting qualified with the environment name" do Puppet.settings.set_value(:server, "myval", :testing) env[:server].should == "myval" end it "should be able to return an individual module that exists in its module path" do env.stubs(:modules).returns [Puppet::Module.new('one', "/one", mock("env"))] mod = env.module('one') mod.should be_a(Puppet::Module) mod.name.should == 'one' end it "should not return a module if the module doesn't exist" do env.stubs(:modules).returns [Puppet::Module.new('one', "/one", mock("env"))] env.module('two').should be_nil end it "should return nil if asked for a module that does not exist in its path" do modpath = tmpdir('modpath') env.modulepath = [modpath] env.module("one").should be_nil end describe "module data" do before do dir = tmpdir("deep_path") @first = File.join(dir, "first") @second = File.join(dir, "second") Puppet[:modulepath] = "#{@first}#{File::PATH_SEPARATOR}#{@second}" FileUtils.mkdir_p(@first) FileUtils.mkdir_p(@second) end describe "#modules_by_path" do it "should return an empty list if there are no modules" do env.modules_by_path.should == { @first => [], @second => [] } end it "should include modules even if they exist in multiple dirs in the modulepath" do modpath1 = File.join(@first, "foo") FileUtils.mkdir_p(modpath1) modpath2 = File.join(@second, "foo") FileUtils.mkdir_p(modpath2) env.modules_by_path.should == { @first => [Puppet::Module.new('foo', modpath1, env)], @second => [Puppet::Module.new('foo', modpath2, env)] } end it "should ignore modules with invalid names" do FileUtils.mkdir_p(File.join(@first, 'foo')) FileUtils.mkdir_p(File.join(@first, 'foo2')) FileUtils.mkdir_p(File.join(@first, 'foo-bar')) FileUtils.mkdir_p(File.join(@first, 'foo_bar')) FileUtils.mkdir_p(File.join(@first, 'foo=bar')) FileUtils.mkdir_p(File.join(@first, 'foo bar')) FileUtils.mkdir_p(File.join(@first, 'foo.bar')) FileUtils.mkdir_p(File.join(@first, '-foo')) FileUtils.mkdir_p(File.join(@first, 'foo-')) FileUtils.mkdir_p(File.join(@first, 'foo--bar')) env.modules_by_path[@first].collect{|mod| mod.name}.sort.should == %w{foo foo-bar foo2 foo_bar} end end describe "#module_requirements" do it "should return a list of what modules depend on other modules" do PuppetSpec::Modules.create( 'foo', @first, :metadata => { :author => 'puppetlabs', :dependencies => [{ 'name' => 'puppetlabs/bar', "version_requirement" => ">= 1.0.0" }] } ) PuppetSpec::Modules.create( 'bar', @second, :metadata => { :author => 'puppetlabs', :dependencies => [{ 'name' => 'puppetlabs/foo', "version_requirement" => "<= 2.0.0" }] } ) PuppetSpec::Modules.create( 'baz', @first, :metadata => { :author => 'puppetlabs', :dependencies => [{ 'name' => 'puppetlabs/bar', "version_requirement" => "3.0.0" }] } ) PuppetSpec::Modules.create( 'alpha', @first, :metadata => { :author => 'puppetlabs', :dependencies => [{ 'name' => 'puppetlabs/bar', "version_requirement" => "~3.0.0" }] } ) env.module_requirements.should == { 'puppetlabs/alpha' => [], 'puppetlabs/foo' => [ { "name" => "puppetlabs/bar", "version" => "9.9.9", "version_requirement" => "<= 2.0.0" } ], 'puppetlabs/bar' => [ { "name" => "puppetlabs/alpha", "version" => "9.9.9", "version_requirement" => "~3.0.0" }, { "name" => "puppetlabs/baz", "version" => "9.9.9", "version_requirement" => "3.0.0" }, { "name" => "puppetlabs/foo", "version" => "9.9.9", "version_requirement" => ">= 1.0.0" } ], 'puppetlabs/baz' => [] } end end describe ".module_by_forge_name" do it "should find modules by forge_name" do mod = PuppetSpec::Modules.create( 'baz', @first, :metadata => {:author => 'puppetlabs'}, :environment => env ) env.module_by_forge_name('puppetlabs/baz').should == mod end it "should not find modules with same name by the wrong author" do mod = PuppetSpec::Modules.create( 'baz', @first, :metadata => {:author => 'sneakylabs'}, :environment => env ) env.module_by_forge_name('puppetlabs/baz').should == nil end it "should return nil when the module can't be found" do env.module_by_forge_name('ima/nothere').should be_nil end end describe ".modules" do it "should return an empty list if there are no modules" do env.modules.should == [] end it "should return a module named for every directory in each module path" do %w{foo bar}.each do |mod_name| FileUtils.mkdir_p(File.join(@first, mod_name)) end %w{bee baz}.each do |mod_name| FileUtils.mkdir_p(File.join(@second, mod_name)) end env.modules.collect{|mod| mod.name}.sort.should == %w{foo bar bee baz}.sort end it "should remove duplicates" do FileUtils.mkdir_p(File.join(@first, 'foo')) FileUtils.mkdir_p(File.join(@second, 'foo')) env.modules.collect{|mod| mod.name}.sort.should == %w{foo} end it "should ignore modules with invalid names" do FileUtils.mkdir_p(File.join(@first, 'foo')) FileUtils.mkdir_p(File.join(@first, 'foo2')) FileUtils.mkdir_p(File.join(@first, 'foo-bar')) FileUtils.mkdir_p(File.join(@first, 'foo_bar')) FileUtils.mkdir_p(File.join(@first, 'foo=bar')) FileUtils.mkdir_p(File.join(@first, 'foo bar')) env.modules.collect{|mod| mod.name}.sort.should == %w{foo foo-bar foo2 foo_bar} end it "should create modules with the correct environment" do FileUtils.mkdir_p(File.join(@first, 'foo')) env.modules.each {|mod| mod.environment.should == env } end end end it "should cache the module list" do env.modulepath = %w{/a} Dir.expects(:entries).once.with("/a").returns %w{foo} env.modules env.modules end end describe Puppet::Node::Environment::Helper do before do @helper = Object.new @helper.extend(Puppet::Node::Environment::Helper) end it "should be able to set and retrieve the environment as a symbol" do @helper.environment = :foo @helper.environment.name.should == :foo end it "should accept an environment directly" do @helper.environment = Puppet::Node::Environment.new(:foo) @helper.environment.name.should == :foo end it "should accept an environment as a string" do @helper.environment = 'foo' @helper.environment.name.should == :foo end end describe "when performing initial import" do before do @parser = Puppet::Parser::ParserFactory.parser("test") # @parser = Puppet::Parser::EParserAdapter.new(Puppet::Parser::Parser.new("test")) # TODO: FIX PARSER FACTORY Puppet::Parser::ParserFactory.stubs(:parser).returns @parser end it "should set the parser's string to the 'code' setting and parse if code is available" do Puppet.settings[:code] = "my code" @parser.expects(:string=).with "my code" @parser.expects(:parse) env.instance_eval { perform_initial_import } end it "should set the parser's file to the 'manifest' setting and parse if no code is available and the manifest is available" do filename = tmpfile('myfile') File.open(filename, 'w'){|f| } Puppet.settings[:manifest] = filename @parser.expects(:file=).with filename @parser.expects(:parse) env.instance_eval { perform_initial_import } end it "should pass the manifest file to the parser even if it does not exist on disk" do filename = tmpfile('myfile') Puppet.settings[:code] = "" Puppet.settings[:manifest] = filename @parser.expects(:file=).with(filename).once @parser.expects(:parse).once env.instance_eval { perform_initial_import } end it "should fail helpfully if there is an error importing" do File.stubs(:exist?).returns true env.stubs(:known_resource_types).returns Puppet::Resource::TypeCollection.new(env) @parser.expects(:file=).once @parser.expects(:parse).raises ArgumentError lambda { env.instance_eval { perform_initial_import } }.should raise_error(Puppet::Error) end it "should not do anything if the ignore_import settings is set" do Puppet.settings[:ignoreimport] = true @parser.expects(:string=).never @parser.expects(:file=).never @parser.expects(:parse).never env.instance_eval { perform_initial_import } end it "should mark the type collection as needing a reparse when there is an error parsing" do @parser.expects(:parse).raises Puppet::ParseError.new("Syntax error at ...") env.stubs(:known_resource_types).returns Puppet::Resource::TypeCollection.new(env) lambda { env.instance_eval { perform_initial_import } }.should raise_error(Puppet::Error, /Syntax error at .../) env.known_resource_types.require_reparse?.should be_true end end end describe 'with classic parser' do before :each do Puppet[:parser] = 'current' end it_behaves_like 'the environment' end describe 'with future parser' do before :each do Puppet[:parser] = 'future' end it_behaves_like 'the environment' end end diff --git a/spec/unit/parser/compiler_spec.rb b/spec/unit/parser/compiler_spec.rb index 8e71756e1..5eea7ca63 100755 --- a/spec/unit/parser/compiler_spec.rb +++ b/spec/unit/parser/compiler_spec.rb @@ -1,833 +1,831 @@ #! /usr/bin/env ruby require 'spec_helper' class CompilerTestResource attr_accessor :builtin, :virtual, :evaluated, :type, :title def initialize(type, title) @type = type @title = title end def [](attr) return nil if attr == :stage :main end def ref "#{type.to_s.capitalize}[#{title}]" end def evaluated? @evaluated end def builtin_type? @builtin end def virtual? @virtual end def class? false end def stage? false end def evaluate end def file "/fake/file/goes/here" end def line "42" end end describe Puppet::Parser::Compiler do include PuppetSpec::Files def resource(type, title) Puppet::Parser::Resource.new(type, title, :scope => @scope) end before :each do # Push me faster, I wanna go back in time! (Specifically, freeze time # across the test since we have a bunch of version == timestamp code # hidden away in the implementation and we keep losing the race.) # --daniel 2011-04-21 now = Time.now Time.stubs(:now).returns(now) @node = Puppet::Node.new("testnode", :facts => Puppet::Node::Facts.new("facts", {})) @known_resource_types = Puppet::Resource::TypeCollection.new "development" @compiler = Puppet::Parser::Compiler.new(@node) @scope = Puppet::Parser::Scope.new(@compiler, :source => stub('source')) @scope_resource = Puppet::Parser::Resource.new(:file, "/my/file", :scope => @scope) @scope.resource = @scope_resource @compiler.environment.stubs(:known_resource_types).returns @known_resource_types end it "should have a class method that compiles, converts, and returns a catalog" do compiler = stub 'compiler' Puppet::Parser::Compiler.expects(:new).with(@node).returns compiler catalog = stub 'catalog' compiler.expects(:compile).returns catalog converted_catalog = stub 'converted_catalog' catalog.expects(:to_resource).returns converted_catalog Puppet::Parser::Compiler.compile(@node).should equal(converted_catalog) end it "should fail intelligently when a class-level compile fails" do Puppet::Parser::Compiler.expects(:new).raises ArgumentError lambda { Puppet::Parser::Compiler.compile(@node) }.should raise_error(Puppet::Error) end it "should use the node's environment as its environment" do @compiler.environment.should equal(@node.environment) end it "should include the resource type collection helper" do Puppet::Parser::Compiler.ancestors.should be_include(Puppet::Resource::TypeCollectionHelper) end it "should be able to return a class list containing all added classes" do @compiler.add_class "" @compiler.add_class "one" @compiler.add_class "two" @compiler.classlist.sort.should == %w{one two}.sort end - it "should clear the thread local caches before compile" do + it "should clear the global caches before compile" do compiler = stub 'compiler' Puppet::Parser::Compiler.expects(:new).with(@node).returns compiler catalog = stub 'catalog' compiler.expects(:compile).returns catalog catalog.expects(:to_resource) - [:known_resource_types, :env_module_directories].each do |var| - Thread.current[var] = "rspec" - end + $known_resource_types = "rspec" + $env_module_directories = "rspec" Puppet::Parser::Compiler.compile(@node) - [:known_resource_types, :env_module_directories].each do |var| - Thread.current[var].should == nil - end + $known_resource_types = nil + $env_module_directories = nil end describe "when initializing" do it "should set its node attribute" do @compiler.node.should equal(@node) end it "should detect when ast nodes are absent" do @compiler.ast_nodes?.should be_false end it "should detect when ast nodes are present" do @known_resource_types.expects(:nodes?).returns true @compiler.ast_nodes?.should be_true end it "should copy the known_resource_types version to the catalog" do @compiler.catalog.version.should == @known_resource_types.version end it "should copy any node classes into the class list" do node = Puppet::Node.new("mynode") node.classes = %w{foo bar} compiler = Puppet::Parser::Compiler.new(node) compiler.classlist.should =~ ['foo', 'bar'] end it "should transform node class hashes into a class list" do node = Puppet::Node.new("mynode") node.classes = {'foo'=>{'one'=>'p1'}, 'bar'=>{'two'=>'p2'}} compiler = Puppet::Parser::Compiler.new(node) compiler.classlist.should =~ ['foo', 'bar'] end it "should add a 'main' stage to the catalog" do @compiler.catalog.resource(:stage, :main).should be_instance_of(Puppet::Parser::Resource) end end describe "when managing scopes" do it "should create a top scope" do @compiler.topscope.should be_instance_of(Puppet::Parser::Scope) end it "should be able to create new scopes" do @compiler.newscope(@compiler.topscope).should be_instance_of(Puppet::Parser::Scope) end it "should set the parent scope of the new scope to be the passed-in parent" do scope = mock 'scope' newscope = @compiler.newscope(scope) newscope.parent.should equal(scope) end it "should set the parent scope of the new scope to its topscope if the parent passed in is nil" do scope = mock 'scope' newscope = @compiler.newscope(nil) newscope.parent.should equal(@compiler.topscope) end end describe "when compiling" do def compile_methods [:set_node_parameters, :evaluate_main, :evaluate_ast_node, :evaluate_node_classes, :evaluate_generators, :fail_on_unevaluated, :finish, :store, :extract, :evaluate_relationships] end # Stub all of the main compile methods except the ones we're specifically interested in. def compile_stub(*except) (compile_methods - except).each { |m| @compiler.stubs(m) } end it "should set node parameters as variables in the top scope" do params = {"a" => "b", "c" => "d"} @node.stubs(:parameters).returns(params) compile_stub(:set_node_parameters) @compiler.compile @compiler.topscope['a'].should == "b" @compiler.topscope['c'].should == "d" end it "should set the client and server versions on the catalog" do params = {"clientversion" => "2", "serverversion" => "3"} @node.stubs(:parameters).returns(params) compile_stub(:set_node_parameters) @compiler.compile @compiler.catalog.client_version.should == "2" @compiler.catalog.server_version.should == "3" end it "should evaluate any existing classes named in the node" do classes = %w{one two three four} main = stub 'main' one = stub 'one', :name => "one" three = stub 'three', :name => "three" @node.stubs(:name).returns("whatever") @node.stubs(:classes).returns(classes) compile_stub(:evaluate_node_classes) @compiler.expects(:evaluate_classes).with(classes, @compiler.topscope) @compiler.compile end it "should evaluate any parameterized classes named in the node" do classes = {'foo'=>{'p1'=>'one'}, 'bar'=>{'p2'=>'two'}} @node.stubs(:classes).returns(classes) @compiler.expects(:evaluate_classes).with(classes, @compiler.topscope) @compiler.compile end it "should evaluate the main class if it exists" do compile_stub(:evaluate_main) main_class = @known_resource_types.add Puppet::Resource::Type.new(:hostclass, "") main_class.expects(:evaluate_code).with { |r| r.is_a?(Puppet::Parser::Resource) } @compiler.topscope.expects(:source=).with(main_class) @compiler.compile end it "should create a new, empty 'main' if no main class exists" do compile_stub(:evaluate_main) @compiler.compile @known_resource_types.find_hostclass([""], "").should be_instance_of(Puppet::Resource::Type) end it "should add an edge between the main stage and main class" do @compiler.compile (stage = @compiler.catalog.resource(:stage, "main")).should be_instance_of(Puppet::Parser::Resource) (klass = @compiler.catalog.resource(:class, "")).should be_instance_of(Puppet::Parser::Resource) @compiler.catalog.edge?(stage, klass).should be_true end it "should evaluate any node classes" do @node.stubs(:classes).returns(%w{one two three four}) @compiler.expects(:evaluate_classes).with(%w{one two three four}, @compiler.topscope) @compiler.send(:evaluate_node_classes) end it "should evaluate all added collections" do colls = [] # And when the collections fail to evaluate. colls << mock("coll1-false") colls << mock("coll2-false") colls.each { |c| c.expects(:evaluate).returns(false) } @compiler.add_collection(colls[0]) @compiler.add_collection(colls[1]) compile_stub(:evaluate_generators) @compiler.compile end it "should ignore builtin resources" do resource = resource(:file, "testing") @compiler.add_resource(@scope, resource) resource.expects(:evaluate).never @compiler.compile end it "should evaluate unevaluated resources" do resource = CompilerTestResource.new(:file, "testing") @compiler.add_resource(@scope, resource) # We have to now mark the resource as evaluated resource.expects(:evaluate).with { |*whatever| resource.evaluated = true } @compiler.compile end it "should not evaluate already-evaluated resources" do resource = resource(:file, "testing") resource.stubs(:evaluated?).returns true @compiler.add_resource(@scope, resource) resource.expects(:evaluate).never @compiler.compile end it "should evaluate unevaluated resources created by evaluating other resources" do resource = CompilerTestResource.new(:file, "testing") @compiler.add_resource(@scope, resource) resource2 = CompilerTestResource.new(:file, "other") # We have to now mark the resource as evaluated resource.expects(:evaluate).with { |*whatever| resource.evaluated = true; @compiler.add_resource(@scope, resource2) } resource2.expects(:evaluate).with { |*whatever| resource2.evaluated = true } @compiler.compile end describe "when finishing" do before do @compiler.send(:evaluate_main) @catalog = @compiler.catalog end def add_resource(name, parent = nil) resource = Puppet::Parser::Resource.new "file", name, :scope => @scope @compiler.add_resource(@scope, resource) @catalog.add_edge(parent, resource) if parent resource end it "should call finish() on all resources" do # Add a resource that does respond to :finish resource = Puppet::Parser::Resource.new "file", "finish", :scope => @scope resource.expects(:finish) @compiler.add_resource(@scope, resource) # And one that does not dnf_resource = stub_everything "dnf", :ref => "File[dnf]", :type => "file" @compiler.add_resource(@scope, dnf_resource) @compiler.send(:finish) end it "should call finish() in add_resource order" do resources = sequence('resources') resource1 = add_resource("finish1") resource1.expects(:finish).in_sequence(resources) resource2 = add_resource("finish2") resource2.expects(:finish).in_sequence(resources) @compiler.send(:finish) end it "should add each container's metaparams to its contained resources" do main = @catalog.resource(:class, :main) main[:noop] = true resource1 = add_resource("meh", main) @compiler.send(:finish) resource1[:noop].should be_true end it "should add metaparams recursively" do main = @catalog.resource(:class, :main) main[:noop] = true resource1 = add_resource("meh", main) resource2 = add_resource("foo", resource1) @compiler.send(:finish) resource2[:noop].should be_true end it "should prefer metaparams from immediate parents" do main = @catalog.resource(:class, :main) main[:noop] = true resource1 = add_resource("meh", main) resource2 = add_resource("foo", resource1) resource1[:noop] = false @compiler.send(:finish) resource2[:noop].should be_false end it "should merge tags downward" do main = @catalog.resource(:class, :main) main.tag("one") resource1 = add_resource("meh", main) resource1.tag "two" resource2 = add_resource("foo", resource1) @compiler.send(:finish) resource2.tags.should be_include("one") resource2.tags.should be_include("two") end it "should work if only middle resources have metaparams set" do main = @catalog.resource(:class, :main) resource1 = add_resource("meh", main) resource1[:noop] = true resource2 = add_resource("foo", resource1) @compiler.send(:finish) resource2[:noop].should be_true end end it "should return added resources in add order" do resource1 = resource(:file, "yay") @compiler.add_resource(@scope, resource1) resource2 = resource(:file, "youpi") @compiler.add_resource(@scope, resource2) @compiler.resources.should == [resource1, resource2] end it "should add resources that do not conflict with existing resources" do resource = resource(:file, "yay") @compiler.add_resource(@scope, resource) @compiler.catalog.should be_vertex(resource) end it "should fail to add resources that conflict with existing resources" do path = make_absolute("/foo") file1 = resource(:file, path) file2 = resource(:file, path) @compiler.add_resource(@scope, file1) lambda { @compiler.add_resource(@scope, file2) }.should raise_error(Puppet::Resource::Catalog::DuplicateResourceError) end it "should add an edge from the scope resource to the added resource" do resource = resource(:file, "yay") @compiler.add_resource(@scope, resource) @compiler.catalog.should be_edge(@scope.resource, resource) end it "should not add non-class resources that don't specify a stage to the 'main' stage" do main = @compiler.catalog.resource(:stage, :main) resource = resource(:file, "foo") @compiler.add_resource(@scope, resource) @compiler.catalog.should_not be_edge(main, resource) end it "should not add any parent-edges to stages" do stage = resource(:stage, "other") @compiler.add_resource(@scope, stage) @scope.resource = resource(:class, "foo") @compiler.catalog.edge?(@scope.resource, stage).should be_false end it "should not attempt to add stages to other stages" do other_stage = resource(:stage, "other") second_stage = resource(:stage, "second") @compiler.add_resource(@scope, other_stage) @compiler.add_resource(@scope, second_stage) second_stage[:stage] = "other" @compiler.catalog.edge?(other_stage, second_stage).should be_false end it "should have a method for looking up resources" do resource = resource(:yay, "foo") @compiler.add_resource(@scope, resource) @compiler.findresource("Yay[foo]").should equal(resource) end it "should be able to look resources up by type and title" do resource = resource(:yay, "foo") @compiler.add_resource(@scope, resource) @compiler.findresource("Yay", "foo").should equal(resource) end it "should not evaluate virtual defined resources" do resource = resource(:file, "testing") resource.virtual = true @compiler.add_resource(@scope, resource) resource.expects(:evaluate).never @compiler.compile end end describe "when evaluating collections" do it "should evaluate each collection" do 2.times { |i| coll = mock 'coll%s' % i @compiler.add_collection(coll) # This is the hard part -- we have to emulate the fact that # collections delete themselves if they are done evaluating. coll.expects(:evaluate).with do @compiler.delete_collection(coll) end } @compiler.compile end it "should not fail when there are unevaluated resource collections that do not refer to specific resources" do coll = stub 'coll', :evaluate => false coll.expects(:resources).returns(nil) @compiler.add_collection(coll) lambda { @compiler.compile }.should_not raise_error end it "should fail when there are unevaluated resource collections that refer to a specific resource" do coll = stub 'coll', :evaluate => false coll.expects(:resources).returns(:something) @compiler.add_collection(coll) lambda { @compiler.compile }.should raise_error Puppet::ParseError, 'Failed to realize virtual resources something' end it "should fail when there are unevaluated resource collections that refer to multiple specific resources" do coll = stub 'coll', :evaluate => false coll.expects(:resources).returns([:one, :two]) @compiler.add_collection(coll) lambda { @compiler.compile }.should raise_error Puppet::ParseError, 'Failed to realize virtual resources one, two' end end describe "when evaluating relationships" do it "should evaluate each relationship with its catalog" do dep = stub 'dep' dep.expects(:evaluate).with(@compiler.catalog) @compiler.add_relationship dep @compiler.evaluate_relationships end end describe "when told to evaluate missing classes" do it "should fail if there's no source listed for the scope" do scope = stub 'scope', :source => nil proc { @compiler.evaluate_classes(%w{one two}, scope) }.should raise_error(Puppet::DevError) end it "should raise an error if a class is not found" do @scope.expects(:find_hostclass).with("notfound", {:assume_fqname => false}).returns(nil) lambda{ @compiler.evaluate_classes(%w{notfound}, @scope) }.should raise_error(Puppet::Error, /Could not find class/) end it "should raise an error when it can't find class" do klasses = {'foo'=>nil} @node.classes = klasses @compiler.topscope.expects(:find_hostclass).with('foo', {:assume_fqname => false}).returns(nil) lambda{ @compiler.compile }.should raise_error(Puppet::Error, /Could not find class foo for testnode/) end end describe "when evaluating found classes" do before do Puppet.settings[:data_binding_terminus] = "none" @class = stub 'class', :name => "my::class" @scope.stubs(:find_hostclass).with("myclass", {:assume_fqname => false}).returns(@class) @resource = stub 'resource', :ref => "Class[myclass]", :type => "file" end it "should evaluate each class" do @compiler.catalog.stubs(:tag) @class.expects(:ensure_in_catalog).with(@scope) @scope.stubs(:class_scope).with(@class) @compiler.evaluate_classes(%w{myclass}, @scope) end describe "and the classes are specified as a hash with parameters" do before do @node.classes = {} @ast_obj = Puppet::Parser::AST::String.new(:value => 'foo') end # Define the given class with default parameters def define_class(name, parameters) @node.classes[name] = parameters klass = Puppet::Resource::Type.new(:hostclass, name, :arguments => {'p1' => @ast_obj, 'p2' => @ast_obj}) @compiler.topscope.known_resource_types.add klass end def compile @catalog = @compiler.compile end it "should record which classes are evaluated" do classes = {'foo'=>{}, 'bar::foo'=>{}, 'bar'=>{}} classes.each { |c, params| define_class(c, params) } compile() classes.each { |name, p| @catalog.classes.should include(name) } end it "should provide default values for parameters that have no values specified" do define_class('foo', {}) compile() @catalog.resource(:class, 'foo')['p1'].should == "foo" end it "should use any provided values" do define_class('foo', {'p1' => 'real_value'}) compile() @catalog.resource(:class, 'foo')['p1'].should == "real_value" end it "should support providing some but not all values" do define_class('foo', {'p1' => 'real_value'}) compile() @catalog.resource(:class, 'Foo')['p1'].should == "real_value" @catalog.resource(:class, 'Foo')['p2'].should == "foo" end it "should ensure each node class is in catalog and has appropriate tags" do klasses = ['bar::foo'] @node.classes = klasses ast_obj = Puppet::Parser::AST::String.new(:value => 'foo') klasses.each do |name| klass = Puppet::Resource::Type.new(:hostclass, name, :arguments => {'p1' => ast_obj, 'p2' => ast_obj}) @compiler.topscope.known_resource_types.add klass end catalog = @compiler.compile r2 = catalog.resources.detect {|r| r.title == 'Bar::Foo' } r2.tags.should =~ ['bar::foo', 'class', 'bar', 'foo'] end end it "should fail if required parameters are missing" do klass = {'foo'=>{'a'=>'one'}} @node.classes = klass klass = Puppet::Resource::Type.new(:hostclass, 'foo', :arguments => {'a' => nil, 'b' => nil}) @compiler.topscope.known_resource_types.add klass lambda { @compiler.compile }.should raise_error(Puppet::ParseError, "Must pass b to Class[Foo]") end it "should fail if invalid parameters are passed" do klass = {'foo'=>{'3'=>'one'}} @node.classes = klass klass = Puppet::Resource::Type.new(:hostclass, 'foo', :arguments => {}) @compiler.topscope.known_resource_types.add klass lambda { @compiler.compile }.should raise_error(Puppet::ParseError, "Invalid parameter 3") end it "should ensure class is in catalog without params" do @node.classes = klasses = {'foo'=>nil} foo = Puppet::Resource::Type.new(:hostclass, 'foo') @compiler.topscope.known_resource_types.add foo catalog = @compiler.compile catalog.classes.should include 'foo' end it "should not evaluate the resources created for found classes unless asked" do @compiler.catalog.stubs(:tag) @resource.expects(:evaluate).never @class.expects(:ensure_in_catalog).returns(@resource) @scope.stubs(:class_scope).with(@class) @compiler.evaluate_classes(%w{myclass}, @scope) end it "should immediately evaluate the resources created for found classes when asked" do @compiler.catalog.stubs(:tag) @resource.expects(:evaluate) @class.expects(:ensure_in_catalog).returns(@resource) @scope.stubs(:class_scope).with(@class) @compiler.evaluate_classes(%w{myclass}, @scope, false) end it "should skip classes that have already been evaluated" do @compiler.catalog.stubs(:tag) @scope.stubs(:class_scope).with(@class).returns("something") @compiler.expects(:add_resource).never @resource.expects(:evaluate).never Puppet::Parser::Resource.expects(:new).never @compiler.evaluate_classes(%w{myclass}, @scope, false) end it "should skip classes previously evaluated with different capitalization" do @compiler.catalog.stubs(:tag) @scope.stubs(:find_hostclass).with("MyClass",{:assume_fqname => false}).returns(@class) @scope.stubs(:class_scope).with(@class).returns("something") @compiler.expects(:add_resource).never @resource.expects(:evaluate).never Puppet::Parser::Resource.expects(:new).never @compiler.evaluate_classes(%w{MyClass}, @scope, false) end end describe "when evaluating AST nodes with no AST nodes present" do it "should do nothing" do @compiler.expects(:ast_nodes?).returns(false) @compiler.known_resource_types.expects(:nodes).never Puppet::Parser::Resource.expects(:new).never @compiler.send(:evaluate_ast_node) end end describe "when evaluating AST nodes with AST nodes present" do before do @compiler.known_resource_types.stubs(:nodes?).returns true # Set some names for our test @node.stubs(:names).returns(%w{a b c}) @compiler.known_resource_types.stubs(:node).with("a").returns(nil) @compiler.known_resource_types.stubs(:node).with("b").returns(nil) @compiler.known_resource_types.stubs(:node).with("c").returns(nil) # It should check this last, of course. @compiler.known_resource_types.stubs(:node).with("default").returns(nil) end it "should fail if the named node cannot be found" do proc { @compiler.send(:evaluate_ast_node) }.should raise_error(Puppet::ParseError) end it "should evaluate the first node class matching the node name" do node_class = stub 'node', :name => "c", :evaluate_code => nil @compiler.known_resource_types.stubs(:node).with("c").returns(node_class) node_resource = stub 'node resource', :ref => "Node[c]", :evaluate => nil, :type => "node" node_class.expects(:ensure_in_catalog).returns(node_resource) @compiler.compile end it "should match the default node if no matching node can be found" do node_class = stub 'node', :name => "default", :evaluate_code => nil @compiler.known_resource_types.stubs(:node).with("default").returns(node_class) node_resource = stub 'node resource', :ref => "Node[default]", :evaluate => nil, :type => "node" node_class.expects(:ensure_in_catalog).returns(node_resource) @compiler.compile end it "should evaluate the node resource immediately rather than using lazy evaluation" do node_class = stub 'node', :name => "c" @compiler.known_resource_types.stubs(:node).with("c").returns(node_class) node_resource = stub 'node resource', :ref => "Node[c]", :type => "node" node_class.expects(:ensure_in_catalog).returns(node_resource) node_resource.expects(:evaluate) @compiler.send(:evaluate_ast_node) end end describe "when managing resource overrides" do before do @override = stub 'override', :ref => "File[/foo]", :type => "my" @resource = resource(:file, "/foo") end it "should be able to store overrides" do lambda { @compiler.add_override(@override) }.should_not raise_error end it "should apply overrides to the appropriate resources" do @compiler.add_resource(@scope, @resource) @resource.expects(:merge).with(@override) @compiler.add_override(@override) @compiler.compile end it "should accept overrides before the related resource has been created" do @resource.expects(:merge).with(@override) # First store the override @compiler.add_override(@override) # Then the resource @compiler.add_resource(@scope, @resource) # And compile, so they get resolved @compiler.compile end it "should fail if the compile is finished and resource overrides have not been applied" do @compiler.add_override(@override) lambda { @compiler.compile }.should raise_error Puppet::ParseError, 'Could not find resource(s) File[/foo] for overriding' end end end diff --git a/spec/unit/parser/functions_spec.rb b/spec/unit/parser/functions_spec.rb index 81ff655a3..8ad33c874 100755 --- a/spec/unit/parser/functions_spec.rb +++ b/spec/unit/parser/functions_spec.rb @@ -1,201 +1,201 @@ #! /usr/bin/env ruby require 'spec_helper' describe Puppet::Parser::Functions do def callable_functions_from(mod) Class.new { include mod }.new end it "should have a method for returning an environment-specific module" do Puppet::Parser::Functions.environment_module(Puppet::Node::Environment.new("myenv")).should be_instance_of(Module) end it "should use the current default environment if no environment is provided" do Puppet::Parser::Functions.environment_module.should be_instance_of(Module) end it "should be able to retrieve environment modules asked for by name rather than instance" do Puppet::Parser::Functions.environment_module(Puppet::Node::Environment.new("myenv")).should equal(Puppet::Parser::Functions.environment_module("myenv")) end describe "when calling newfunction" do let(:function_module) { Module.new } before do Puppet::Parser::Functions.stubs(:environment_module).returns(function_module) end it "should create the function in the environment module" do Puppet::Parser::Functions.newfunction("name", :type => :rvalue) { |args| } function_module.should be_method_defined :function_name end it "should warn if the function already exists" do Puppet::Parser::Functions.newfunction("name", :type => :rvalue) { |args| } Puppet.expects(:warning) Puppet::Parser::Functions.newfunction("name", :type => :rvalue) { |args| } end it "should raise an error if the function type is not correct" do expect { Puppet::Parser::Functions.newfunction("name", :type => :unknown) { |args| } }.to raise_error Puppet::DevError, "Invalid statement type :unknown" end it "instruments the function to profiles the execution" do messages = [] Puppet::Util::Profiler.current = Puppet::Util::Profiler::WallClock.new(proc { |msg| messages << msg }, "id") Puppet::Parser::Functions.newfunction("name", :type => :rvalue) { |args| } callable_functions_from(function_module).function_name([]) messages.first.should =~ /Called name/ end end describe "when calling function to test function existence" do let(:function_module) { Module.new } before do Puppet::Parser::Functions.stubs(:environment_module).returns(function_module) end it "should return false if the function doesn't exist" do Puppet::Parser::Functions.autoloader.stubs(:load) Puppet::Parser::Functions.function("name").should be_false end it "should return its name if the function exists" do Puppet::Parser::Functions.newfunction("name", :type => :rvalue) { |args| } Puppet::Parser::Functions.function("name").should == "function_name" end it "should try to autoload the function if it doesn't exist yet" do Puppet::Parser::Functions.autoloader.expects(:load) Puppet::Parser::Functions.function("name") end end describe "when calling function to test arity" do let(:function_module) { Module.new } before do Puppet::Parser::Functions.stubs(:environment_module).returns(function_module) end it "should raise an error if the function is called with too many arguments" do Puppet::Parser::Functions.newfunction("name", :arity => 2) { |args| } expect { callable_functions_from(function_module).function_name([1,2,3]) }.to raise_error ArgumentError end it "should raise an error if the function is called with too few arguments" do Puppet::Parser::Functions.newfunction("name", :arity => 2) { |args| } expect { callable_functions_from(function_module).function_name([1]) }.to raise_error ArgumentError end it "should not raise an error if the function is called with correct number of arguments" do Puppet::Parser::Functions.newfunction("name", :arity => 2) { |args| } expect { callable_functions_from(function_module).function_name([1,2]) }.to_not raise_error end it "should raise an error if the variable arg function is called with too few arguments" do Puppet::Parser::Functions.newfunction("name", :arity => -3) { |args| } expect { callable_functions_from(function_module).function_name([1]) }.to raise_error ArgumentError end it "should not raise an error if the variable arg function is called with correct number of arguments" do Puppet::Parser::Functions.newfunction("name", :arity => -3) { |args| } expect { callable_functions_from(function_module).function_name([1,2]) }.to_not raise_error end it "should not raise an error if the variable arg function is called with more number of arguments" do Puppet::Parser::Functions.newfunction("name", :arity => -3) { |args| } expect { callable_functions_from(function_module).function_name([1,2,3]) }.to_not raise_error end end describe "::arity" do it "returns the given arity of a function" do Puppet::Parser::Functions.newfunction("name", :arity => 4) { |args| } Puppet::Parser::Functions.arity(:name).should == 4 end it "returns -1 if no arity is given" do Puppet::Parser::Functions.newfunction("name") { |args| } Puppet::Parser::Functions.arity(:name).should == -1 end end describe "::get_function" do it "can retrieve a function defined on the *root* environment" do - Thread.current[:environment] = nil + $environment = nil function = Puppet::Parser::Functions.newfunction("atest", :type => :rvalue) do nil end Puppet::Node::Environment.current = "test_env" Puppet::Parser::Functions.send(:get_function, "atest").should equal(function) end it "can retrieve a function from the current environment" do Puppet::Node::Environment.current = "test_env" function = Puppet::Parser::Functions.newfunction("atest", :type => :rvalue) do nil end Puppet::Parser::Functions.send(:get_function, "atest").should equal(function) end it "takes a function in the current environment over one in the root" do root = Puppet::Node::Environment.root env = Puppet::Node::Environment.current = "test_env" func1 = {:type => :rvalue, :name => :testfunc, :extra => :func1} func2 = {:type => :rvalue, :name => :testfunc, :extra => :func2} Puppet::Parser::Functions.instance_eval do @functions[Puppet::Node::Environment.root][:atest] = func1 @functions[Puppet::Node::Environment.current][:atest] = func2 end Puppet::Parser::Functions.send(:get_function, "atest").should equal(func2) end end describe "::merged_functions" do it "returns functions in both the current and root environment" do - Thread.current[:environment] = nil + $environment = nil func_a = Puppet::Parser::Functions.newfunction("test_a", :type => :rvalue) do nil end Puppet::Node::Environment.current = "test_env" func_b = Puppet::Parser::Functions.newfunction("test_b", :type => :rvalue) do nil end Puppet::Parser::Functions.send(:merged_functions).should include(:test_a, :test_b) end it "returns functions from the current environment over the root environment" do root = Puppet::Node::Environment.root env = Puppet::Node::Environment.current = "test_env" func1 = {:type => :rvalue, :name => :testfunc, :extra => :func1} func2 = {:type => :rvalue, :name => :testfunc, :extra => :func2} Puppet::Parser::Functions.instance_eval do @functions[Puppet::Node::Environment.root][:atest] = func1 @functions[Puppet::Node::Environment.current][:atest] = func2 end Puppet::Parser::Functions.send(:merged_functions)[:atest].should equal(func2) end end describe "::add_function" do it "adds functions to the current environment" do func = {:type => :rvalue, :name => :testfunc} Puppet::Node::Environment.current = "add_function_test" Puppet::Parser::Functions.send(:add_function, :testfunc, func) Puppet::Parser::Functions.instance_variable_get(:@functions)[Puppet::Node::Environment.root].should_not include(:testfunc) Puppet::Parser::Functions.instance_variable_get(:@functions)[Puppet::Node::Environment.current].should include(:testfunc) end end end