diff --git a/lib/puppet/configurer.rb b/lib/puppet/configurer.rb index ab714141b..364bd43c3 100644 --- a/lib/puppet/configurer.rb +++ b/lib/puppet/configurer.rb @@ -1,252 +1,252 @@ # The client for interacting with the puppetmaster config server. require 'sync' require 'timeout' require 'puppet/network/http_pool' require 'puppet/util' require 'puppet/util/config_timeout' class Puppet::Configurer require 'puppet/configurer/fact_handler' require 'puppet/configurer/plugin_handler' extend Puppet::Util::ConfigTimeout include Puppet::Configurer::FactHandler include Puppet::Configurer::PluginHandler # For benchmarking include Puppet::Util attr_reader :compile_time, :environment # Provide more helpful strings to the logging that the Agent does def self.to_s "Puppet configuration client" end class << self # Puppetd should only have one instance running, and we need a way # to retrieve it. attr_accessor :instance include Puppet::Util end # How to lock instances of this class. def self.lockfile_path Puppet[:puppetdlockfile] end def execute_postrun_command execute_from_setting(:postrun_command) end def execute_prerun_command execute_from_setting(:prerun_command) end # Initialize and load storage def init_storage Puppet::Util::Storage.load @compile_time ||= Puppet::Util::Storage.cache(:configuration)[:compile_time] rescue => detail Puppet.log_exception(detail, "Removing corrupt state file #{Puppet[:statefile]}: #{detail}") begin ::File.unlink(Puppet[:statefile]) retry rescue => detail raise Puppet::Error.new("Cannot remove #{Puppet[:statefile]}: #{detail}") end end # Just so we can specify that we are "the" instance. def initialize Puppet.settings.use(:main, :ssl, :agent) self.class.instance = self @running = false @splayed = false @environment = Puppet[:environment] end # Get the remote catalog, yo. Returns nil if no catalog can be found. def retrieve_catalog(fact_options) fact_options ||= {} # First try it with no cache, then with the cache. unless (Puppet[:use_cached_catalog] and result = retrieve_catalog_from_cache(fact_options)) or result = retrieve_new_catalog(fact_options) if ! Puppet[:usecacheonfailure] Puppet.warning "Not using cache on failed catalog" return nil end result = retrieve_catalog_from_cache(fact_options) end return nil unless result convert_catalog(result, @duration) end # Convert a plain resource catalog into our full host catalog. def convert_catalog(result, duration) catalog = result.to_ral catalog.finalize catalog.retrieval_duration = duration catalog.write_class_file catalog.write_resource_file catalog end def get_facts(options) download_plugins unless options[:skip_plugin_download] if Puppet::Resource::Catalog.indirection.terminus_class == :rest # This is a bit complicated. We need the serialized and escaped facts, # and we need to know which format they're encoded in. Thus, we # get a hash with both of these pieces of information. # # facts_for_uploading may set Puppet[:node_name_value] as a side effect return facts_for_uploading end end def prepare_and_retrieve_catalog(options, fact_options) # set report host name now that we have the fact options[:report].host = Puppet[:node_name_value] unless catalog = (options.delete(:catalog) || retrieve_catalog(fact_options)) Puppet.err "Could not retrieve catalog; skipping run" return end catalog end # Retrieve (optionally) and apply a catalog. If a catalog is passed in # the options, then apply that one, otherwise retrieve it. def apply_catalog(catalog, options) report = options[:report] report.configuration_version = catalog.version report.environment = @environment benchmark(:notice, "Finished catalog run") do catalog.apply(options) end report.finalize_report report end # The code that actually runs the catalog. # This just passes any options on to the catalog, # which accepts :tags and :ignoreschedules. def run(options = {}) options[:report] ||= Puppet::Transaction::Report.new("apply") report = options[:report] init_storage Puppet::Util::Log.newdestination(report) begin unless Puppet[:node_name_fact].empty? fact_options = get_facts(options) end - if node = Puppet::Node.indirection.find(Puppet[:node_name_value]) + if node = Puppet::Node.indirection.find(Puppet[:node_name_value], :environment => @environment, :ignore_cache => true) if node.environment.to_s != @environment Puppet.warning "Local environment: \"#{@environment}\" doesn't match server specified node environment \"#{node.environment}\", changing." @environment = node.environment.to_s fact_options = nil end end fact_options = get_facts(options) unless fact_options unless catalog = prepare_and_retrieve_catalog(options, fact_options) return nil end # Here we set the local environment based on what we get from the # catalog. Since a change in environment means a change in facts, and # facts may be used to determine which catalog we get, we need to # rerun the process if the environment is changed. tries = 0 while catalog.environment and not catalog.environment.empty? and catalog.environment != @environment if tries > 3 raise Puppet::Error, "Catalog environment didn't stabilize after #{tries} fetches, aborting run" end Puppet.warning "Local environment: \"#{@environment}\" doesn't match server specified environment \"#{catalog.environment}\", restarting agent run with new environment" @environment = catalog.environment return nil unless catalog = prepare_and_retrieve_catalog(options, fact_options) tries += 1 end execute_prerun_command or return nil apply_catalog(catalog, options) report.exit_status rescue => detail Puppet.log_exception(detail, "Failed to apply catalog: #{detail}") return nil ensure execute_postrun_command or return nil end ensure # Make sure we forget the retained module_directories of any autoload # we might have used. Thread.current[:env_module_directories] = nil Puppet::Util::Log.close(report) send_report(report) end def send_report(report) puts report.summary if Puppet[:summarize] save_last_run_summary(report) - Puppet::Transaction::Report.indirection.save(report) if Puppet[:report] + Puppet::Transaction::Report.indirection.save(report, nil, :environment => @environment) if Puppet[:report] rescue => detail Puppet.log_exception(detail, "Could not send report: #{detail}") end def save_last_run_summary(report) mode = Puppet.settings.setting(:lastrunfile).mode Puppet::Util.replace_file(Puppet[:lastrunfile], mode) do |fh| fh.print YAML.dump(report.raw_summary) end rescue => detail Puppet.log_exception(detail, "Could not save last run local report: #{detail}") end private def execute_from_setting(setting) return true if (command = Puppet[setting]) == "" begin Puppet::Util::Execution.execute([command]) true rescue => detail Puppet.log_exception(detail, "Could not run command from #{setting}: #{detail}") false end end def retrieve_catalog_from_cache(fact_options) result = nil @duration = thinmark do result = Puppet::Resource::Catalog.indirection.find(Puppet[:node_name_value], fact_options.merge(:ignore_terminus => true, :environment => @environment)) end Puppet.notice "Using cached catalog" result rescue => detail Puppet.log_exception(detail, "Could not retrieve catalog from cache: #{detail}") return nil end def retrieve_new_catalog(fact_options) result = nil @duration = thinmark do result = Puppet::Resource::Catalog.indirection.find(Puppet[:node_name_value], fact_options.merge(:ignore_cache => true, :environment => @environment)) end result rescue SystemExit,NoMemoryError raise rescue Exception => detail Puppet.log_exception(detail, "Could not retrieve catalog from remote server: #{detail}") return nil end end diff --git a/lib/puppet/node.rb b/lib/puppet/node.rb index 08634be79..a45e511f5 100644 --- a/lib/puppet/node.rb +++ b/lib/puppet/node.rb @@ -1,145 +1,145 @@ require 'puppet/indirector' # A class for managing nodes, including their facts and environment. class Puppet::Node require 'puppet/node/facts' require 'puppet/node/environment' # Set up indirection, so that nodes can be looked for in # the node sources. extend Puppet::Indirector # Adds the environment getter and setter, with some instance/string conversion include Puppet::Node::Environment::Helper # Use the node source as the indirection terminus. indirects :node, :terminus_setting => :node_terminus, :doc => "Where to find node information. A node is composed of its name, its facts, and its environment." attr_accessor :name, :classes, :source, :ipaddress, :parameters attr_reader :time # # Load json before trying to register. Puppet.features.pson? and ::PSON.register_document_type('Node',self) def self.from_pson(pson) raise ArgumentError, "No name provided in pson data" unless name = pson['name'] node = new(name) node.classes = pson['classes'] node.parameters = pson['parameters'] node.environment = pson['environment'] node end def to_pson(*args) result = { 'document_type' => "Node", 'data' => {} } result['data']['name'] = name result['data']['classes'] = classes unless classes.empty? result['data']['parameters'] = parameters unless parameters.empty? result['data']['environment'] = environment.name result.to_pson(*args) end def environment return super if @environment if env = parameters["environment"] self.environment = env return super end # Else, return the default Puppet::Node::Environment.new end def initialize(name, options = {}) raise ArgumentError, "Node names cannot be nil" unless name @name = name if classes = options[:classes] if classes.is_a?(String) @classes = [classes] else @classes = classes end else @classes = [] end @parameters = options[:parameters] || {} if env = options[:environment] self.environment = env end @time = Time.now end # Merge the node facts with parameters from the node source. def fact_merge - if facts = Puppet::Node::Facts.indirection.find(name) - merge(facts.values) - end + if facts = Puppet::Node::Facts.indirection.find(name, :environment => environment) + merge(facts.values) + end rescue => detail - error = Puppet::Error.new("Could not retrieve facts for #{name}: #{detail}") - error.set_backtrace(detail.backtrace) - raise error + error = Puppet::Error.new("Could not retrieve facts for #{name}: #{detail}") + error.set_backtrace(detail.backtrace) + raise error end # Merge any random parameters into our parameter list. def merge(params) params.each do |name, value| @parameters[name] = value unless @parameters.include?(name) end @parameters["environment"] ||= self.environment.name.to_s if self.environment end # Calculate the list of names we might use for looking # up our node. This is only used for AST nodes. def names return [name] if Puppet.settings[:strict_hostname_checking] names = [] names += split_name(name) if name.include?(".") # First, get the fqdn unless fqdn = parameters["fqdn"] if parameters["hostname"] and parameters["domain"] fqdn = parameters["hostname"] + "." + parameters["domain"] else Puppet.warning "Host is missing hostname and/or domain: #{name}" end end # Now that we (might) have the fqdn, add each piece to the name # list to search, in order of longest to shortest. names += split_name(fqdn) if fqdn # And make sure the node name is first, since that's the most # likely usage. # The name is usually the Certificate CN, but it can be # set to the 'facter' hostname instead. if Puppet[:node_name] == 'cert' names.unshift name else names.unshift parameters["hostname"] end names.uniq end def split_name(name) list = name.split(".") tmp = [] list.each_with_index do |short, i| tmp << list[0..i].join(".") end tmp.reverse end end diff --git a/lib/puppet/node/facts.rb b/lib/puppet/node/facts.rb index b38c25b06..ef8d793a0 100755 --- a/lib/puppet/node/facts.rb +++ b/lib/puppet/node/facts.rb @@ -1,102 +1,102 @@ require 'time' require 'puppet/node' require 'puppet/indirector' require 'puppet/util/pson' # Manage a given node's facts. This either accepts facts and stores them, or # returns facts for a given node. class Puppet::Node::Facts # Set up indirection, so that nodes can be looked for in # the node sources. extend Puppet::Indirector extend Puppet::Util::Pson # We want to expire any cached nodes if the facts are saved. module NodeExpirer - def save(instance, key = nil) - Puppet::Node.indirection.expire(instance.name) + def save(instance, key = nil, options={}) + Puppet::Node.indirection.expire(instance.name, options) super end end indirects :facts, :terminus_setting => :facts_terminus, :extend => NodeExpirer attr_accessor :name, :values def add_local_facts values["clientcert"] = Puppet.settings[:certname] values["clientversion"] = Puppet.version.to_s values["environment"] ||= Puppet.settings[:environment] end def initialize(name, values = {}) @name = name @values = values add_timestamp end def downcase_if_necessary return unless Puppet.settings[:downcasefacts] Puppet.deprecation_warning "DEPRECATION NOTICE: Fact downcasing is deprecated; please disable (20080122)" values.each do |fact, value| values[fact] = value.downcase if value.is_a?(String) end end # Convert all fact values into strings. def stringify values.each do |fact, value| values[fact] = value.to_s end end def ==(other) return false unless self.name == other.name strip_internal == other.send(:strip_internal) end def self.from_pson(data) result = new(data['name'], data['values']) result.timestamp = Time.parse(data['timestamp']) if data['timestamp'] result.expiration = Time.parse(data['expiration']) if data['expiration'] result end def to_pson(*args) result = { 'name' => name, 'values' => strip_internal, } result['timestamp'] = timestamp if timestamp result['expiration'] = expiration if expiration result.to_pson(*args) end # Add internal data to the facts for storage. def add_timestamp self.timestamp = Time.now end def timestamp=(time) self.values[:_timestamp] = time end def timestamp self.values[:_timestamp] end private # Strip out that internal data. def strip_internal newvals = values.dup newvals.find_all { |name, value| name.to_s =~ /^_/ }.each { |name, value| newvals.delete(name) } newvals end end diff --git a/lib/puppet/transaction.rb b/lib/puppet/transaction.rb index d0c426c25..e0d49fbbe 100644 --- a/lib/puppet/transaction.rb +++ b/lib/puppet/transaction.rb @@ -1,490 +1,490 @@ # the class that actually walks our resource/property tree, collects the changes, # and performs them require 'puppet' require 'puppet/util/tagging' require 'puppet/application' require 'digest/sha1' class Puppet::Transaction require 'puppet/transaction/event' require 'puppet/transaction/event_manager' require 'puppet/transaction/resource_harness' require 'puppet/resource/status' attr_accessor :component, :catalog, :ignoreschedules, :for_network_device attr_accessor :configurator # The report, once generated. attr_reader :report # Routes and stores any events and subscriptions. attr_reader :event_manager # Handles most of the actual interacting with resources attr_reader :resource_harness include Puppet::Util include Puppet::Util::Tagging # Wraps application run state check to flag need to interrupt processing def stop_processing? Puppet::Application.stop_requested? end # Add some additional times for reporting def add_times(hash) hash.each do |name, num| report.add_times(name, num) end end # Are there any failed resources in this transaction? def any_failed? report.resource_statuses.values.detect { |status| status.failed? } end # Apply all changes for a resource def apply(resource, ancestor = nil) status = resource_harness.evaluate(resource) add_resource_status(status) event_manager.queue_events(ancestor || resource, status.events) unless status.failed? rescue => detail resource.err "Could not evaluate: #{detail}" end # Find all of the changed resources. def changed? report.resource_statuses.values.find_all { |status| status.changed }.collect { |status| catalog.resource(status.resource) } end # Find all of the applied resources (including failed attempts). def applied_resources report.resource_statuses.values.collect { |status| catalog.resource(status.resource) } end # Copy an important relationships from the parent to the newly-generated # child resource. def add_conditional_directed_dependency(parent, child, label=nil) relationship_graph.add_vertex(child) edge = parent.depthfirst? ? [child, parent] : [parent, child] if relationship_graph.edge?(*edge.reverse) parent.debug "Skipping automatic relationship to #{child}" else relationship_graph.add_edge(edge[0],edge[1],label) end end # Evaluate a single resource. def eval_resource(resource, ancestor = nil) if skip?(resource) resource_status(resource).skipped = true else resource_status(resource).scheduled = true apply(resource, ancestor) end # Check to see if there are any events queued for this resource event_manager.process_events(resource) end # This method does all the actual work of running a transaction. It # collects all of the changes, executes them, and responds to any # necessary events. def evaluate add_dynamically_generated_resources Puppet.info "Applying configuration version '#{catalog.version}'" if catalog.version relationship_graph.traverse do |resource| if resource.is_a?(Puppet::Type::Component) Puppet.warning "Somehow left a component in the relationship graph" else resource.info "Starting to evaluate the resource" if Puppet[:evaltrace] and @catalog.host_config? seconds = thinmark { eval_resource(resource) } resource.info "Evaluated in %0.2f seconds" % seconds if Puppet[:evaltrace] and @catalog.host_config? end end Puppet.debug "Finishing transaction #{object_id}" end def events event_manager.events end def failed?(resource) s = resource_status(resource) and s.failed? end # Does this resource have any failed dependencies? def failed_dependencies?(resource) # First make sure there are no failed dependencies. To do this, # we check for failures in any of the vertexes above us. It's not # enough to check the immediate dependencies, which is why we use # a tree from the reversed graph. found_failed = false # When we introduced the :whit into the graph, to reduce the combinatorial # explosion of edges, we also ended up reporting failures for containers # like class and stage. This is undesirable; while just skipping the # output isn't perfect, it is RC-safe. --daniel 2011-06-07 suppress_report = (resource.class == Puppet::Type.type(:whit)) relationship_graph.dependencies(resource).each do |dep| next unless failed?(dep) found_failed = true # See above. --daniel 2011-06-06 unless suppress_report then resource.notice "Dependency #{dep} has failures: #{resource_status(dep).failed}" end end found_failed end def eval_generate(resource) return false unless resource.respond_to?(:eval_generate) raise Puppet::DevError,"Depthfirst resources are not supported by eval_generate" if resource.depthfirst? begin made = resource.eval_generate.uniq return false if made.empty? made = made.inject({}) {|a,v| a.merge(v.name => v) } rescue => detail resource.log_exception(detail, "Failed to generate additional resources using 'eval_generate: #{detail}") return false end made.values.each do |res| begin res.tag(*resource.tags) @catalog.add_resource(res) res.finish rescue Puppet::Resource::Catalog::DuplicateResourceError res.info "Duplicate generated resource; skipping" end end sentinel = Puppet::Type.type(:whit).new(:name => "completed_#{resource.title}", :catalog => resource.catalog) # The completed whit is now the thing that represents the resource is done relationship_graph.adjacent(resource,:direction => :out,:type => :edges).each { |e| # But children run as part of the resource, not after it next if made[e.target.name] add_conditional_directed_dependency(sentinel, e.target, e.label) relationship_graph.remove_edge! e } default_label = Puppet::Resource::Catalog::Default_label made.values.each do |res| # Depend on the nearest ancestor we generated, falling back to the # resource if we have none parent_name = res.ancestors.find { |a| made[a] and made[a] != res } parent = made[parent_name] || resource add_conditional_directed_dependency(parent, res) # This resource isn't 'completed' until each child has run add_conditional_directed_dependency(res, sentinel, default_label) end # This edge allows the resource's events to propagate, though it isn't # strictly necessary for ordering purposes add_conditional_directed_dependency(resource, sentinel, default_label) true end # A general method for recursively generating new resources from a # resource. def generate_additional_resources(resource) return unless resource.respond_to?(:generate) begin made = resource.generate rescue => detail resource.log_exception(detail, "Failed to generate additional resources using 'generate': #{detail}") end return unless made made = [made] unless made.is_a?(Array) made.uniq.each do |res| begin res.tag(*resource.tags) @catalog.add_resource(res) res.finish add_conditional_directed_dependency(resource, res) generate_additional_resources(res) rescue Puppet::Resource::Catalog::DuplicateResourceError res.info "Duplicate generated resource; skipping" end end end def add_dynamically_generated_resources @catalog.vertices.each { |resource| generate_additional_resources(resource) } end # Should we ignore tags? def ignore_tags? ! (@catalog.host_config? or Puppet[:name] == "puppet") end # this should only be called by a Puppet::Type::Component resource now # and it should only receive an array def initialize(catalog, report = nil) @catalog = catalog - @report = report || Puppet::Transaction::Report.new("apply", catalog.version, Puppet[:environment]) + @report = report || Puppet::Transaction::Report.new("apply", catalog.version, catalog.environment) @event_manager = Puppet::Transaction::EventManager.new(self) @resource_harness = Puppet::Transaction::ResourceHarness.new(self) @prefetched_providers = Hash.new { |h,k| h[k] = {} } end def resources_by_provider(type_name, provider_name) unless @resources_by_provider @resources_by_provider = Hash.new { |h, k| h[k] = Hash.new { |h, k| h[k] = {} } } @catalog.vertices.each do |resource| if resource.class.attrclass(:provider) prov = resource.provider && resource.provider.class.name @resources_by_provider[resource.type][prov][resource.name] = resource end end end @resources_by_provider[type_name][provider_name] || {} end def prefetch_if_necessary(resource) provider_class = resource.provider.class return unless provider_class.respond_to?(:prefetch) and !prefetched_providers[resource.type][provider_class.name] resources = resources_by_provider(resource.type, provider_class.name) if provider_class == resource.class.defaultprovider providerless_resources = resources_by_provider(resource.type, nil) providerless_resources.values.each {|res| res.provider = provider_class.name} resources.merge! providerless_resources end prefetch(provider_class, resources) end attr_reader :prefetched_providers # Prefetch any providers that support it, yo. We don't support prefetching # types, just providers. def prefetch(provider_class, resources) type_name = provider_class.resource_type.name return if @prefetched_providers[type_name][provider_class.name] Puppet.debug "Prefetching #{provider_class.name} resources for #{type_name}" begin provider_class.prefetch(resources) rescue => detail Puppet.log_exception(detail, "Could not prefetch #{type_name} provider '#{provider_class.name}': #{detail}") end @prefetched_providers[type_name][provider_class.name] = true end # We want to monitor changes in the relationship graph of our # catalog but this is complicated by the fact that the catalog # both is_a graph and has_a graph, by the fact that changes to # the structure of the object can have adverse serialization # effects, by threading issues, by order-of-initialization issues, # etc. # # Since the proper lifetime/scope of the monitoring is a transaction # and the transaction is already commiting a mild law-of-demeter # transgression, we cut the Gordian knot here by simply wrapping the # transaction's view of the resource graph to capture and maintain # the information we need. Nothing outside the transaction needs # this information, and nothing outside the transaction can see it # except via the Transaction#relationship_graph class Relationship_graph_wrapper require 'puppet/rb_tree_map' attr_reader :real_graph,:transaction,:ready,:generated,:done,:blockers,:unguessable_deterministic_key def initialize(real_graph,transaction) @real_graph = real_graph @transaction = transaction @ready = Puppet::RbTreeMap.new @generated = {} @done = {} @blockers = {} @unguessable_deterministic_key = Hash.new { |h,k| h[k] = Digest::SHA1.hexdigest("NaCl, MgSO4 (salts) and then #{k.ref}") } @providerless_types = [] end def method_missing(*args,&block) real_graph.send(*args,&block) end def add_vertex(v) real_graph.add_vertex(v) end def add_edge(f,t,label=nil) key = unguessable_deterministic_key[t] ready.delete(key) real_graph.add_edge(f,t,label) end # Enqueue the initial set of resources, those with no dependencies. def enqueue_roots vertices.each do |v| blockers[v] = direct_dependencies_of(v).length enqueue(v) if blockers[v] == 0 end end # Decrement the blocker count for the resource by 1. If the number of # blockers is unknown, count them and THEN decrement by 1. def unblock(resource) blockers[resource] ||= direct_dependencies_of(resource).select { |r2| !done[r2] }.length if blockers[resource] > 0 blockers[resource] -= 1 else resource.warning "appears to have a negative number of dependencies" end blockers[resource] <= 0 end def enqueue(*resources) resources.each do |resource| key = unguessable_deterministic_key[resource] ready[key] = resource end end def finish(resource) direct_dependents_of(resource).each do |v| enqueue(v) if unblock(v) end done[resource] = true end def next_resource ready.delete_min end def traverse(&block) real_graph.report_cycles_in_graph enqueue_roots deferred_resources = [] while (resource = next_resource) && !transaction.stop_processing? if resource.suitable? made_progress = true transaction.prefetch_if_necessary(resource) # If we generated resources, we don't know what they are now # blocking, so we opt to recompute it, rather than try to track every # change that would affect the number. blockers.clear if transaction.eval_generate(resource) yield resource finish(resource) else deferred_resources << resource end if ready.empty? and deferred_resources.any? if made_progress enqueue(*deferred_resources) else fail_unsuitable_resources(deferred_resources) end made_progress = false deferred_resources = [] end end # Just once per type. No need to punish the user. @providerless_types.uniq.each do |type| Puppet.err "Could not find a suitable provider for #{type}" end end def fail_unsuitable_resources(resources) resources.each do |resource| # We don't automatically assign unsuitable providers, so if there # is one, it must have been selected by the user. if resource.provider resource.err "Provider #{resource.provider.class.name} is not functional on this host" else @providerless_types << resource.type end transaction.resource_status(resource).failed = true finish(resource) end end end def relationship_graph @relationship_graph ||= Relationship_graph_wrapper.new(catalog.relationship_graph,self) end def add_resource_status(status) report.add_resource_status status end def resource_status(resource) report.resource_statuses[resource.to_s] || add_resource_status(Puppet::Resource::Status.new(resource)) end # Is the resource currently scheduled? def scheduled?(resource) self.ignoreschedules or resource_harness.scheduled?(resource_status(resource), resource) end # Should this resource be skipped? def skip?(resource) if missing_tags?(resource) resource.debug "Not tagged with #{tags.join(", ")}" elsif ! scheduled?(resource) resource.debug "Not scheduled" elsif failed_dependencies?(resource) # When we introduced the :whit into the graph, to reduce the combinatorial # explosion of edges, we also ended up reporting failures for containers # like class and stage. This is undesirable; while just skipping the # output isn't perfect, it is RC-safe. --daniel 2011-06-07 unless resource.class == Puppet::Type.type(:whit) then resource.warning "Skipping because of failed dependencies" end elsif resource.virtual? resource.debug "Skipping because virtual" elsif resource.appliable_to_device? ^ for_network_device resource.debug "Skipping #{resource.appliable_to_device? ? 'device' : 'host'} resources because running on a #{for_network_device ? 'device' : 'host'}" else return false end true end # The tags we should be checking. def tags self.tags = Puppet[:tags] unless defined?(@tags) super end def handle_qualified_tags( qualified ) # The default behavior of Puppet::Util::Tagging is # to split qualified tags into parts. That would cause # qualified tags to match too broadly here. return end # Is this resource tagged appropriately? def missing_tags?(resource) return false if ignore_tags? return false if tags.empty? not resource.tagged?(*tags) end end require 'puppet/transaction/report' diff --git a/lib/puppet/type/file/source.rb b/lib/puppet/type/file/source.rb index a7b1ac25a..99308093e 100755 --- a/lib/puppet/type/file/source.rb +++ b/lib/puppet/type/file/source.rb @@ -1,199 +1,199 @@ require 'puppet/file_serving/content' require 'puppet/file_serving/metadata' module Puppet # Copy files from a local or remote source. This state *only* does any work # when the remote file is an actual file; in that case, this state copies # the file down. If the remote file is a dir or a link or whatever, then # this state, during retrieval, modifies the appropriate other states # so that things get taken care of appropriately. Puppet::Type.type(:file).newparam(:source) do include Puppet::Util::Diff attr_accessor :source, :local desc <<-EOT A source file, which will be copied into place on the local system. Values can be URIs pointing to remote files, or fully qualified paths to files available on the local system (including files on NFS shares or Windows mapped drives). This attribute is mutually exclusive with `content` and `target`. The available URI schemes are *puppet* and *file*. *Puppet* URIs will retrieve files from Puppet's built-in file server, and are usually formatted as: `puppet:///modules/name_of_module/filename` This will fetch a file from a module on the puppet master (or from a local module when using puppet apply). Given a `modulepath` of `/etc/puppetlabs/puppet/modules`, the example above would resolve to `/etc/puppetlabs/puppet/modules/name_of_module/files/filename`. Unlike `content`, the `source` attribute can be used to recursively copy directories if the `recurse` attribute is set to `true` or `remote`. If a source directory contains symlinks, use the `links` attribute to specify whether to recreate links or follow them. Multiple `source` values can be specified as an array, and Puppet will use the first source that exists. This can be used to serve different files to different system types: file { "/etc/nfs.conf": source => [ "puppet:///modules/nfs/conf.$host", "puppet:///modules/nfs/conf.$operatingsystem", "puppet:///modules/nfs/conf" ] } Alternately, when serving directories recursively, multiple sources can be combined by setting the `sourceselect` attribute to `all`. EOT validate do |sources| sources = [sources] unless sources.is_a?(Array) sources.each do |source| next if Puppet::Util.absolute_path?(source) begin uri = URI.parse(URI.escape(source)) rescue => detail self.fail "Could not understand source #{source}: #{detail}" end self.fail "Cannot use relative URLs '#{source}'" unless uri.absolute? self.fail "Cannot use opaque URLs '#{source}'" unless uri.hierarchical? self.fail "Cannot use URLs of type '#{uri.scheme}' as source for fileserving" unless %w{file puppet}.include?(uri.scheme) end end SEPARATOR_REGEX = [Regexp.escape(File::SEPARATOR.to_s), Regexp.escape(File::ALT_SEPARATOR.to_s)].join munge do |sources| sources = [sources] unless sources.is_a?(Array) sources.map do |source| source = source.sub(/[#{SEPARATOR_REGEX}]+$/, '') if Puppet::Util.absolute_path?(source) URI.unescape(Puppet::Util.path_to_uri(source).to_s) else source end end end def change_to_s(currentvalue, newvalue) # newvalue = "{md5}#{@metadata.checksum}" if @resource.property(:ensure).retrieve == :absent return "creating from source #{metadata.source} with contents #{metadata.checksum}" else return "replacing from source #{metadata.source} with contents #{metadata.checksum}" end end def checksum metadata && metadata.checksum end # Look up (if necessary) and return remote content. def content return @content if @content raise Puppet::DevError, "No source for content was stored with the metadata" unless metadata.source - unless tmp = Puppet::FileServing::Content.indirection.find(metadata.source) + unless tmp = Puppet::FileServing::Content.indirection.find(metadata.source, :environment => resource.catalog.environment) fail "Could not find any content at %s" % metadata.source end @content = tmp.content end # Copy the values from the source to the resource. Yay. def copy_source_values devfail "Somehow got asked to copy source values without any metadata" unless metadata # Take each of the stats and set them as states on the local file # if a value has not already been provided. [:owner, :mode, :group, :checksum].each do |metadata_method| param_name = (metadata_method == :checksum) ? :content : metadata_method next if metadata_method == :owner and !Puppet.features.root? next if metadata_method == :checksum and metadata.ftype == "directory" next if metadata_method == :checksum and metadata.ftype == "link" and metadata.links == :manage if Puppet.features.microsoft_windows? next if [:owner, :group].include?(metadata_method) and !local? end if resource[param_name].nil? or resource[param_name] == :absent resource[param_name] = metadata.send(metadata_method) end end if resource[:ensure] == :absent # We know all we need to elsif metadata.ftype != "link" resource[:ensure] = metadata.ftype elsif @resource[:links] == :follow resource[:ensure] = :present else resource[:ensure] = "link" resource[:target] = metadata.destination end end def found? ! (metadata.nil? or metadata.ftype.nil?) end attr_writer :metadata # Provide, and retrieve if necessary, the metadata for this file. Fail # if we can't find data about this host, and fail if there are any # problems in our query. def metadata return @metadata if @metadata return nil unless value value.each do |source| begin - if data = Puppet::FileServing::Metadata.indirection.find(source) + if data = Puppet::FileServing::Metadata.indirection.find(source, :environment => resource.catalog.environment) @metadata = data @metadata.source = source break end rescue => detail fail detail, "Could not retrieve file metadata for #{source}: #{detail}" end end - fail "Could not retrieve information from environment #{Puppet[:environment]} source(s) #{value.join(", ")}" unless @metadata + fail "Could not retrieve information from environment #{resource.catalog.environment} source(s) #{value.join(", ")}" unless @metadata @metadata end def local? found? and scheme == "file" end def full_path Puppet::Util.uri_to_path(uri) if found? end def server? uri and uri.host end def server (uri and uri.host) or Puppet.settings[:server] end def port (uri and uri.port) or Puppet.settings[:masterport] end private def scheme (uri and uri.scheme) end def uri @uri ||= URI.parse(URI.escape(metadata.source)) end end end diff --git a/spec/unit/type/file/source_spec.rb b/spec/unit/type/file/source_spec.rb index 50e3679ef..44f0bda29 100755 --- a/spec/unit/type/file/source_spec.rb +++ b/spec/unit/type/file/source_spec.rb @@ -1,358 +1,360 @@ #!/usr/bin/env rspec require 'spec_helper' require 'uri' source = Puppet::Type.type(:file).attrclass(:source) describe Puppet::Type.type(:file).attrclass(:source) do include PuppetSpec::Files before do # Wow that's a messy interface to the resource. - @resource = stub 'resource', :[]= => nil, :property => nil, :catalog => stub("catalog", :dependent_data_expired? => false), :line => 0, :file => '' + @environment = "myenv" + @resource = stub 'resource', :[]= => nil, :property => nil, :catalog => stub("catalog", :dependent_data_expired? => false, :environment => @environment), :line => 0, :file => '' @foobar = make_absolute("/foo/bar baz") @feebooz = make_absolute("/fee/booz baz") @foobar_uri = URI.unescape(Puppet::Util.path_to_uri(@foobar).to_s) @feebooz_uri = URI.unescape(Puppet::Util.path_to_uri(@feebooz).to_s) end it "should be a subclass of Parameter" do source.superclass.must == Puppet::Parameter end describe "#validate" do let(:path) { tmpfile('file_source_validate') } let(:resource) { Puppet::Type.type(:file).new(:path => path) } it "should fail if the set values are not URLs" do URI.expects(:parse).with('foo').raises RuntimeError lambda { resource[:source] = %w{foo} }.must raise_error(Puppet::Error) end it "should fail if the URI is not a local file, file URI, or puppet URI" do lambda { resource[:source] = %w{http://foo/bar} }.must raise_error(Puppet::Error, /Cannot use URLs of type 'http' as source for fileserving/) end it "should strip trailing forward slashes", :unless => Puppet.features.microsoft_windows? do resource[:source] = "/foo/bar\\//" resource[:source].should == %w{file:/foo/bar\\} end it "should strip trailing forward and backslashes", :if => Puppet.features.microsoft_windows? do resource[:source] = "X:/foo/bar\\//" resource[:source].should == %w{file:/X:/foo/bar} end it "should accept an array of sources" do resource[:source] = %w{file:///foo/bar puppet://host:8140/foo/bar} resource[:source].should == %w{file:///foo/bar puppet://host:8140/foo/bar} end it "should accept file path characters that are not valid in URI" do resource[:source] = 'file:///foo bar' end it "should reject relative URI sources" do lambda { resource[:source] = 'foo/bar' }.must raise_error(Puppet::Error) end it "should reject opaque sources" do lambda { resource[:source] = 'mailto:foo@com' }.must raise_error(Puppet::Error) end it "should accept URI authority component" do resource[:source] = 'file://host/foo' resource[:source].should == %w{file://host/foo} end it "should accept when URI authority is absent" do resource[:source] = 'file:///foo/bar' resource[:source].should == %w{file:///foo/bar} end end describe "#munge" do let(:path) { tmpfile('file_source_munge') } let(:resource) { Puppet::Type.type(:file).new(:path => path) } it "should prefix file scheme to absolute paths" do resource[:source] = path resource[:source].should == [URI.unescape(Puppet::Util.path_to_uri(path).to_s)] end %w[file puppet].each do |scheme| it "should not prefix valid #{scheme} URIs" do resource[:source] = "#{scheme}:///foo bar" resource[:source].should == ["#{scheme}:///foo bar"] end end end describe "when returning the metadata" do before do @metadata = stub 'metadata', :source= => nil end it "should return already-available metadata" do @source = source.new(:resource => @resource) @source.metadata = "foo" @source.metadata.should == "foo" end it "should return nil if no @should value is set and no metadata is available" do @source = source.new(:resource => @resource) @source.metadata.should be_nil end it "should collect its metadata using the Metadata class if it is not already set" do @source = source.new(:resource => @resource, :value => @foobar) - Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri).returns @metadata + Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri, :environment => @environment).returns @metadata @source.metadata end it "should use the metadata from the first found source" do metadata = stub 'metadata', :source= => nil @source = source.new(:resource => @resource, :value => [@foobar, @feebooz]) - Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri).returns nil - Puppet::FileServing::Metadata.indirection.expects(:find).with(@feebooz_uri).returns metadata + Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri, :environment => @environment).returns nil + Puppet::FileServing::Metadata.indirection.expects(:find).with(@feebooz_uri, :environment => @environment).returns metadata @source.metadata.should equal(metadata) end it "should store the found source as the metadata's source" do metadata = mock 'metadata' @source = source.new(:resource => @resource, :value => @foobar) - Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri).returns metadata + Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri, :environment => @environment).returns metadata metadata.expects(:source=).with(@foobar_uri) @source.metadata end it "should fail intelligently if an exception is encountered while querying for metadata" do @source = source.new(:resource => @resource, :value => @foobar) - Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri).raises RuntimeError + Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri, :environment => @environment).raises RuntimeError @source.expects(:fail).raises ArgumentError lambda { @source.metadata }.should raise_error(ArgumentError) end it "should fail if no specified sources can be found" do @source = source.new(:resource => @resource, :value => @foobar) - Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri).returns nil + Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri, :environment => @environment).returns nil @source.expects(:fail).raises RuntimeError lambda { @source.metadata }.should raise_error(RuntimeError) end end it "should have a method for setting the desired values on the resource" do source.new(:resource => @resource).must respond_to(:copy_source_values) end describe "when copying the source values" do before do @resource = Puppet::Type.type(:file).new :path => @foobar @source = source.new(:resource => @resource) @metadata = stub 'metadata', :owner => 100, :group => 200, :mode => 123, :checksum => "{md5}asdfasdf", :ftype => "file", :source => @foobar @source.stubs(:metadata).returns @metadata Puppet.features.stubs(:root?).returns true end it "should fail if there is no metadata" do @source.stubs(:metadata).returns nil @source.expects(:devfail).raises ArgumentError lambda { @source.copy_source_values }.should raise_error(ArgumentError) end it "should set :ensure to the file type" do @metadata.stubs(:ftype).returns "file" @source.copy_source_values @resource[:ensure].must == :file end it "should not set 'ensure' if it is already set to 'absent'" do @metadata.stubs(:ftype).returns "file" @resource[:ensure] = :absent @source.copy_source_values @resource[:ensure].must == :absent end describe "and the source is a file" do before do @metadata.stubs(:ftype).returns "file" Puppet.features.stubs(:microsoft_windows?).returns false end it "should copy the metadata's owner, group, checksum, and mode to the resource if they are not set on the resource" do @source.copy_source_values @resource[:owner].must == 100 @resource[:group].must == 200 @resource[:mode].must == "173" # Metadata calls it checksum, we call it content. @resource[:content].must == @metadata.checksum end it "should not copy the metadata's owner to the resource if it is already set" do @resource[:owner] = 1 @resource[:group] = 2 @resource[:mode] = 3 @resource[:content] = "foobar" @source.copy_source_values @resource[:owner].must == 1 @resource[:group].must == 2 @resource[:mode].must == "3" @resource[:content].should_not == @metadata.checksum end describe "and puppet is not running as root" do it "should not try to set the owner" do Puppet.features.expects(:root?).returns false @source.copy_source_values @resource[:owner].should be_nil end end describe "on Windows" do before :each do Puppet.features.stubs(:microsoft_windows?).returns true end it "should not copy owner and group from remote sources" do @source.stubs(:local?).returns false @source.copy_source_values @resource[:owner].must be_nil @resource[:group].must be_nil end it "should copy owner and group from local sources" do @source.stubs(:local?).returns true @source.copy_source_values @resource[:owner].must == 100 @resource[:group].must == 200 end end end describe "and the source is a link" do it "should set the target to the link destination" do @metadata.stubs(:ftype).returns "link" @metadata.stubs(:links).returns "manage" @resource.stubs(:[]) @resource.stubs(:[]=) @metadata.expects(:destination).returns "/path/to/symlink" @resource.expects(:[]=).with(:target, "/path/to/symlink") @source.copy_source_values end end end it "should have a local? method" do source.new(:resource => @resource).must be_respond_to(:local?) end context "when accessing source properties" do + let(:catalog) { Puppet::Resource::Catalog.new } let(:path) { tmpfile('file_resource') } - let(:resource) { Puppet::Type.type(:file).new(:path => path) } + let(:resource) { Puppet::Type.type(:file).new(:path => path, :catalog => catalog) } let(:sourcepath) { tmpfile('file_source') } describe "for local sources" do before :each do FileUtils.touch(sourcepath) end describe "on POSIX systems", :if => Puppet.features.posix? do ['', "file:", "file://"].each do |prefix| it "with prefix '#{prefix}' should be local" do resource[:source] = "#{prefix}#{sourcepath}" resource.parameter(:source).must be_local end it "should be able to return the metadata source full path" do resource[:source] = "#{prefix}#{sourcepath}" resource.parameter(:source).full_path.should == sourcepath end end end describe "on Windows systems", :if => Puppet.features.microsoft_windows? do ['', "file:/", "file:///"].each do |prefix| it "should be local with prefix '#{prefix}'" do resource[:source] = "#{prefix}#{sourcepath}" resource.parameter(:source).must be_local end it "should be able to return the metadata source full path" do resource[:source] = "#{prefix}#{sourcepath}" resource.parameter(:source).full_path.should == sourcepath end it "should convert backslashes to forward slashes" do resource[:source] = "#{prefix}#{sourcepath.gsub(/\\/, '/')}" end end it "should be UNC with two slashes" end end describe "for remote sources" do let(:sourcepath) { "/path/to/source" } let(:uri) { URI::Generic.build(:scheme => 'puppet', :host => 'server', :port => 8192, :path => sourcepath).to_s } before(:each) do metadata = Puppet::FileServing::Metadata.new(path, :source => uri, 'type' => 'file') #metadata = stub('remote', :ftype => "file", :source => uri) - Puppet::FileServing::Metadata.indirection.stubs(:find).with(uri).returns metadata + Puppet::FileServing::Metadata.indirection.stubs(:find).with(uri, has_key(:environment)).returns metadata resource[:source] = uri end it "should not be local" do resource.parameter(:source).should_not be_local end it "should be able to return the metadata source full path" do resource.parameter(:source).full_path.should == "/path/to/source" end it "should be able to return the source server" do resource.parameter(:source).server.should == "server" end it "should be able to return the source port" do resource.parameter(:source).port.should == 8192 end describe "which don't specify server or port" do let(:uri) { "puppet:///path/to/source" } it "should return the default source server" do Puppet.settings.expects(:[]).with(:server).returns("myserver") resource.parameter(:source).server.should == "myserver" end it "should return the default source port" do Puppet.settings.expects(:[]).with(:masterport).returns(1234) resource.parameter(:source).port.should == 1234 end end end end end