diff --git a/lib/puppet/type/file.rb b/lib/puppet/type/file.rb index 2659336b1..b1e65b390 100644 --- a/lib/puppet/type/file.rb +++ b/lib/puppet/type/file.rb @@ -1,804 +1,804 @@ require 'digest/md5' require 'cgi' require 'etc' require 'uri' require 'fileutils' require 'enumerator' require 'pathname' require 'puppet/network/handler' require 'puppet/util/diff' require 'puppet/util/checksums' require 'puppet/util/backups' Puppet::Type.newtype(:file) do include Puppet::Util::MethodHelper include Puppet::Util::Checksums include Puppet::Util::Backups @doc = "Manages local files, including setting ownership and permissions, creation of both files and directories, and retrieving entire files from remote servers. As Puppet matures, it expected that the `file` resource will be used less and less to manage content, and instead native resources will be used to do so. If you find that you are often copying files in from a central location, rather than using native resources, please contact Puppet Labs and we can hopefully work with you to develop a native resource to support what you are doing. **Autorequires:** If Puppet is managing the user or group that owns a file, the file resource will autorequire them. If Puppet is managing any parent directories of a file, the file resource will autorequire them." def self.title_patterns [ [ /^(.*?)\/*\Z/m, [ [ :path, lambda{|x| x} ] ] ] ] end newparam(:path) do desc "The path to the file to manage. Must be fully qualified." isnamevar validate do |value| unless Puppet::Util.absolute_path?(value) fail Puppet::Error, "File paths must be fully qualified, not '#{value}'" end end # convert the current path in an index into the collection and the last # path name. The aim is to use less storage for all common paths in a hierarchy munge do |value| # We know the value is absolute, so expanding it will just standardize it. path, name = ::File.split(::File.expand_path(value)) { :index => Puppet::FileCollection.collection.index(path), :name => name } end # and the reverse unmunge do |value| basedir = Puppet::FileCollection.collection.path(value[:index]) ::File.expand_path ::File.join( basedir, value[:name] ) end end newparam(:backup) do desc "Whether files should be backed up before being replaced. The preferred method of backing files up is via a `filebucket`, which stores files by their MD5 sums and allows easy retrieval without littering directories with backups. You can specify a local filebucket or a network-accessible server-based filebucket by setting `backup => bucket-name`. Alternatively, if you specify any value that begins with a `.` (e.g., `.puppet-bak`), then Puppet will use copy the file in the same directory with that value as the extension of the backup. Setting `backup => false` disables all backups of the file in question. Puppet automatically creates a local filebucket named `puppet` and defaults to backing up there. To use a server-based filebucket, you must specify one in your configuration. filebucket { main: server => puppet, path => false, # The path => false line works around a known issue with the filebucket type. } The `puppet master` daemon creates a filebucket by default, so you can usually back up to your main server with this configuration. Once you've described the bucket in your configuration, you can use it in any file's backup attribute: file { \"/my/file\": source => \"/path/in/nfs/or/something\", backup => main } This will back the file up to the central server. At this point, the benefits of using a central filebucket are that you do not have backup files lying around on each of your machines, a given version of a file is only backed up once, you can restore any given file manually (no matter how old), and you can use Puppet Dashboard to view file contents. Eventually, transactional support will be able to automatically restore filebucketed files. " defaultto "puppet" munge do |value| # I don't really know how this is happening. value = value.shift if value.is_a?(Array) case value when false, "false", :false false when true, "true", ".puppet-bak", :true ".puppet-bak" when String value else self.fail "Invalid backup type #{value.inspect}" end end end newparam(:recurse) do desc "Whether and how deeply to do recursive management. Options are: * `inf,true` --- Regular style recursion on both remote and local directory structure. * `remote` --- Descends recursively into the remote directory but not the local directory. Allows copying of a few files into a directory containing many unmanaged files without scanning all the local files. * `false` --- Default of no recursion. * `[0-9]+` --- Same as true, but limit recursion. Warning: this syntax has been deprecated in favor of the `recurselimit` attribute. " newvalues(:true, :false, :inf, :remote, /^[0-9]+$/) # Replace the validation so that we allow numbers in # addition to string representations of them. validate { |arg| } munge do |value| newval = super(value) case newval when :true, :inf; true when :false; false when :remote; :remote when Integer, Fixnum, Bignum self.warning "Setting recursion depth with the recurse parameter is now deprecated, please use recurselimit" # recurse == 0 means no recursion return false if value == 0 resource[:recurselimit] = value true when /^\d+$/ self.warning "Setting recursion depth with the recurse parameter is now deprecated, please use recurselimit" value = Integer(value) # recurse == 0 means no recursion return false if value == 0 resource[:recurselimit] = value true else self.fail "Invalid recurse value #{value.inspect}" end end end newparam(:recurselimit) do desc "How deeply to do recursive management." newvalues(/^[0-9]+$/) munge do |value| newval = super(value) case newval when Integer, Fixnum, Bignum; value when /^\d+$/; Integer(value) else self.fail "Invalid recurselimit value #{value.inspect}" end end end newparam(:replace, :boolean => true) do desc "Whether or not to replace a file that is sourced but exists. This is useful for using file sources purely for initialization." newvalues(:true, :false) aliasvalue(:yes, :true) aliasvalue(:no, :false) defaultto :true end newparam(:force, :boolean => true) do desc "Force the file operation. Currently only used when replacing directories with links." newvalues(:true, :false) defaultto false end newparam(:ignore) do desc "A parameter which omits action on files matching specified patterns during recursion. Uses Ruby's builtin globbing engine, so shell metacharacters are fully supported, e.g. `[a-z]*`. Matches that would descend into the directory structure are ignored, e.g., `*/*`." validate do |value| unless value.is_a?(Array) or value.is_a?(String) or value == false self.devfail "Ignore must be a string or an Array" end end end newparam(:links) do desc "How to handle links during file actions. During file copying, `follow` will copy the target file instead of the link, `manage` will copy the link itself, and `ignore` will just pass it by. When not copying, `manage` and `ignore` behave equivalently (because you cannot really ignore links entirely during local recursion), and `follow` will manage the file to which the link points." newvalues(:follow, :manage) defaultto :manage end newparam(:purge, :boolean => true) do desc "Whether unmanaged files should be purged. If you have a filebucket configured the purged files will be uploaded, but if you do not, this will destroy data. Only use this option for generated files unless you really know what you are doing. This option only makes sense when recursively managing directories. Note that when using `purge` with `source`, Puppet will purge any files that are not on the remote system." defaultto :false newvalues(:true, :false) end newparam(:sourceselect) do desc "Whether to copy all valid sources, or just the first one. This parameter is only used in recursive copies; by default, the first valid source is the only one used as a recursive source, but if this parameter is set to `all`, then all valid sources will have all of their contents copied to the local host, and for sources that have the same file, the source earlier in the list will be used." defaultto :first newvalues(:first, :all) end # Autorequire the nearest ancestor directory found in the catalog. autorequire(:file) do path = Pathname(self[:path]) if !path.root? # Start at our parent, to avoid autorequiring ourself parents = path.parent.enum_for(:ascend) found = parents.find { |p| catalog.resource(:file, p.to_s) } found and found.to_s end end # Autorequire the owner and group of the file. {:user => :owner, :group => :group}.each do |type, property| autorequire(type) do if @parameters.include?(property) # The user/group property automatically converts to IDs next unless should = @parameters[property].shouldorig val = should[0] if val.is_a?(Integer) or val =~ /^\d+$/ nil else val end end end end CREATORS = [:content, :source, :target] SOURCE_ONLY_CHECKSUMS = [:none, :ctime, :mtime] validate do creator_count = 0 CREATORS.each do |param| creator_count += 1 if self.should(param) end creator_count += 1 if @parameters.include?(:source) self.fail "You cannot specify more than one of #{CREATORS.collect { |p| p.to_s}.join(", ")}" if creator_count > 1 self.fail "You cannot specify a remote recursion without a source" if !self[:source] and self[:recurse] == :remote self.fail "You cannot specify source when using checksum 'none'" if self[:checksum] == :none && !self[:source].nil? SOURCE_ONLY_CHECKSUMS.each do |checksum_type| self.fail "You cannot specify content when using checksum '#{checksum_type}'" if self[:checksum] == checksum_type && !self[:content].nil? end self.warning "Possible error: recurselimit is set but not recurse, no recursion will happen" if !self[:recurse] and self[:recurselimit] end def self.[](path) return nil unless path super(path.gsub(/\/+/, '/').sub(/\/$/, '')) end def self.instances return [] end # Determine the user to write files as. def asuser if self.should(:owner) and ! self.should(:owner).is_a?(Symbol) writeable = Puppet::Util::SUIDManager.asuser(self.should(:owner)) { FileTest.writable?(::File.dirname(self[:path])) } # If the parent directory is writeable, then we execute # as the user in question. Otherwise we'll rely on # the 'owner' property to do things. asuser = self.should(:owner) if writeable end asuser end def bucket return @bucket if @bucket backup = self[:backup] return nil unless backup return nil if backup =~ /^\./ unless catalog or backup == "puppet" fail "Can not find filebucket for backups without a catalog" end unless catalog and filebucket = catalog.resource(:filebucket, backup) or backup == "puppet" fail "Could not find filebucket #{backup} specified in backup" end return default_bucket unless filebucket @bucket = filebucket.bucket @bucket end def default_bucket Puppet::Type.type(:filebucket).mkdefaultbucket.bucket end # Does the file currently exist? Just checks for whether # we have a stat def exist? stat ? true : false end # We have to do some extra finishing, to retrieve our bucket if # there is one. def finish # Look up our bucket, if there is one bucket super end # Create any children via recursion or whatever. def eval_generate return [] unless self.recurse? recurse #recurse.reject do |resource| # catalog.resource(:file, resource[:path]) #end.each do |child| # catalog.add_resource child # catalog.relationship_graph.add_edge self, child #end end def flush # We want to make sure we retrieve metadata anew on each transaction. @parameters.each do |name, param| param.flush if param.respond_to?(:flush) end @stat = :needs_stat end def initialize(hash) # Used for caching clients @clients = {} super # If they've specified a source, we get our 'should' values # from it. unless self[:ensure] if self[:target] self[:ensure] = :symlink elsif self[:content] self[:ensure] = :file end end @stat = :needs_stat end # Configure discovered resources to be purged. def mark_children_for_purging(children) children.each do |name, child| next if child[:source] child[:ensure] = :absent end end # Create a new file or directory object as a child to the current # object. def newchild(path) full_path = ::File.join(self[:path], path) # Add some new values to our original arguments -- these are the ones # set at initialization. We specifically want to exclude any param # values set by the :source property or any default values. # LAK:NOTE This is kind of silly, because the whole point here is that # the values set at initialization should live as long as the resource # but values set by default or by :source should only live for the transaction # or so. Unfortunately, we don't have a straightforward way to manage # the different lifetimes of this data, so we kludge it like this. # The right-side hash wins in the merge. options = @original_parameters.merge(:path => full_path).reject { |param, value| value.nil? } # These should never be passed to our children. [:parent, :ensure, :recurse, :recurselimit, :target, :alias, :source].each do |param| options.delete(param) if options.include?(param) end self.class.new(options) end # Files handle paths specially, because they just lengthen their # path names, rather than including the full parent's title each # time. def pathbuilder # We specifically need to call the method here, so it looks # up our parent in the catalog graph. if parent = parent() # We only need to behave specially when our parent is also # a file if parent.is_a?(self.class) # Remove the parent file name list = parent.pathbuilder list.pop # remove the parent's path info return list << self.ref else return super end else return [self.ref] end end # Should we be purging? def purge? @parameters.include?(:purge) and (self[:purge] == :true or self[:purge] == "true") end # Recursively generate a list of file resources, which will # be used to copy remote files, manage local files, and/or make links # to map to another directory. def recurse children = (self[:recurse] == :remote) ? {} : recurse_local if self[:target] recurse_link(children) elsif self[:source] recurse_remote(children) end # If we're purging resources, then delete any resource that isn't on the # remote system. mark_children_for_purging(children) if self.purge? result = children.values.sort { |a, b| a[:path] <=> b[:path] } remove_less_specific_files(result) end # This is to fix bug #2296, where two files recurse over the same # set of files. It's a rare case, and when it does happen you're # not likely to have many actual conflicts, which is good, because # this is a pretty inefficient implementation. def remove_less_specific_files(files) mypath = self[:path].split(::File::Separator) other_paths = catalog.vertices. select { |r| r.is_a?(self.class) and r[:path] != self[:path] }. collect { |r| r[:path].split(::File::Separator) }. select { |p| p[0,mypath.length] == mypath } return files if other_paths.empty? files.reject { |file| path = file[:path].split(::File::Separator) other_paths.any? { |p| path[0,p.length] == p } } end # A simple method for determining whether we should be recursing. def recurse? self[:recurse] == true or self[:recurse] == :remote end # Recurse the target of the link. def recurse_link(children) perform_recursion(self[:target]).each do |meta| if meta.relative_path == "." self[:ensure] = :directory next end children[meta.relative_path] ||= newchild(meta.relative_path) if meta.ftype == "directory" children[meta.relative_path][:ensure] = :directory else children[meta.relative_path][:ensure] = :link children[meta.relative_path][:target] = meta.full_path end end children end # Recurse the file itself, returning a Metadata instance for every found file. def recurse_local result = perform_recursion(self[:path]) return {} unless result result.inject({}) do |hash, meta| next hash if meta.relative_path == "." hash[meta.relative_path] = newchild(meta.relative_path) hash end end # Recurse against our remote file. def recurse_remote(children) sourceselect = self[:sourceselect] total = self[:source].collect do |source| next unless result = perform_recursion(source) return if top = result.find { |r| r.relative_path == "." } and top.ftype != "directory" result.each { |data| data.source = "#{source}/#{data.relative_path}" } break result if result and ! result.empty? and sourceselect == :first result end.flatten # This only happens if we have sourceselect == :all unless sourceselect == :first found = [] total.reject! do |data| result = found.include?(data.relative_path) found << data.relative_path unless found.include?(data.relative_path) result end end total.each do |meta| if meta.relative_path == "." parameter(:source).metadata = meta next end children[meta.relative_path] ||= newchild(meta.relative_path) children[meta.relative_path][:source] = meta.source children[meta.relative_path][:checksum] = :md5 if meta.ftype == "file" children[meta.relative_path].parameter(:source).metadata = meta end children end def perform_recursion(path) Puppet::FileServing::Metadata.indirection.search( path, :links => self[:links], :recurse => (self[:recurse] == :remote ? true : self[:recurse]), :recurselimit => self[:recurselimit], :ignore => self[:ignore], :checksum_type => (self[:source] || self[:content]) ? self[:checksum] : :none ) end # Remove any existing data. This is only used when dealing with # links or directories. def remove_existing(should) return unless s = stat self.fail "Could not back up; will not replace" unless perform_backup unless should.to_s == "link" return if s.ftype.to_s == should.to_s end case s.ftype when "directory" if self[:force] == :true debug "Removing existing directory for replacement with #{should}" FileUtils.rmtree(self[:path]) else notice "Not removing directory; use 'force' to override" return end when "link", "file" debug "Removing existing #{s.ftype} for replacement with #{should}" ::File.unlink(self[:path]) else self.fail "Could not back up files of type #{s.ftype}" end @stat = :needs_stat true end def retrieve if source = parameter(:source) source.copy_source_values end super end # Set the checksum, from another property. There are multiple # properties that modify the contents of a file, and they need the # ability to make sure that the checksum value is in sync. def setchecksum(sum = nil) if @parameters.include? :checksum if sum @parameters[:checksum].checksum = sum else # If they didn't pass in a sum, then tell checksum to # figure it out. currentvalue = @parameters[:checksum].retrieve @parameters[:checksum].checksum = currentvalue end end end # Should this thing be a normal file? This is a relatively complex # way of determining whether we're trying to create a normal file, # and it's here so that the logic isn't visible in the content property. def should_be_file? return true if self[:ensure] == :file # I.e., it's set to something like "directory" return false if e = self[:ensure] and e != :present # The user doesn't really care, apparently if self[:ensure] == :present return true unless s = stat return(s.ftype == "file" ? true : false) end # If we've gotten here, then :ensure isn't set return true if self[:content] return true if stat and stat.ftype == "file" false end # Stat our file. Depending on the value of the 'links' attribute, we # use either 'stat' or 'lstat', and we expect the properties to use the # resulting stat object accordingly (mostly by testing the 'ftype' # value). # # We use the initial value :needs_stat to ensure we only stat the file once, # but can also keep track of a failed stat (@stat == nil). This also allows # us to re-stat on demand by setting @stat = :needs_stat. def stat return @stat unless @stat == :needs_stat method = :stat # Files are the only types that support links if (self.class.name == :file and self[:links] != :follow) or self.class.name == :tidy method = :lstat end @stat = begin ::File.send(method, self[:path]) rescue Errno::ENOENT => error nil rescue Errno::EACCES => error warning "Could not stat; permission denied" nil end end # We have to hack this just a little bit, because otherwise we'll get # an error when the target and the contents are created as properties on # the far side. def to_trans(retrieve = true) obj = super obj.delete(:target) if obj[:target] == :notlink obj end # Write out the file. Requires the property name for logging. # Write will be done by the content property, along with checksum computation def write(property) remove_existing(:file) use_temporary_file = write_temporary_file? if use_temporary_file path = "#{self[:path]}.puppettmp_#{rand(10000)}" path = "#{self[:path]}.puppettmp_#{rand(10000)}" while ::File.exists?(path) or ::File.symlink?(path) else path = self[:path] end mode = self.should(:mode) # might be nil umask = mode ? 000 : 022 mode_int = mode ? mode.to_i(8) : nil - content_checksum = Puppet::Util.withumask(umask) { ::File.open(path, 'w', mode_int ) { |f| write_content(f) } } + content_checksum = Puppet::Util.withumask(umask) { ::File.open(path, 'wb', mode_int ) { |f| write_content(f) } } # And put our new file in place if use_temporary_file # This is only not true when our file is empty. begin fail_if_checksum_is_wrong(path, content_checksum) if validate_checksum? ::File.rename(path, self[:path]) rescue => detail fail "Could not rename temporary file #{path} to #{self[:path]}: #{detail}" ensure # Make sure the created file gets removed ::File.unlink(path) if FileTest.exists?(path) end end # make sure all of the modes are actually correct property_fix end private # Should we validate the checksum of the file we're writing? def validate_checksum? self[:checksum] !~ /time/ end # Make sure the file we wrote out is what we think it is. def fail_if_checksum_is_wrong(path, content_checksum) newsum = parameter(:checksum).sum_file(path) return if [:absent, nil, content_checksum].include?(newsum) self.fail "File written to disk did not match checksum; discarding changes (#{content_checksum} vs #{newsum})" end # write the current content. Note that if there is no content property # simply opening the file with 'w' as done in write is enough to truncate # or write an empty length file. def write_content(file) (content = property(:content)) && content.write(file) end private def write_temporary_file? # unfortunately we don't know the source file size before fetching it # so let's assume the file won't be empty (c = property(:content) and c.length) || (s = @parameters[:source] and 1) end # There are some cases where all of the work does not get done on # file creation/modification, so we have to do some extra checking. def property_fix properties.each do |thing| next unless [:mode, :owner, :group, :seluser, :selrole, :seltype, :selrange].include?(thing.name) # Make sure we get a new stat objct @stat = :needs_stat currentvalue = thing.retrieve thing.sync unless thing.safe_insync?(currentvalue) end end end # We put all of the properties in separate files, because there are so many # of them. The order these are loaded is important, because it determines # the order they are in the property lit. require 'puppet/type/file/checksum' require 'puppet/type/file/content' # can create the file require 'puppet/type/file/source' # can create the file require 'puppet/type/file/target' # creates a different type of file require 'puppet/type/file/ensure' # can create the file require 'puppet/type/file/owner' require 'puppet/type/file/group' require 'puppet/type/file/mode' require 'puppet/type/file/type' require 'puppet/type/file/selcontext' # SELinux file context require 'puppet/type/file/ctime' require 'puppet/type/file/mtime' diff --git a/lib/puppet/util/checksums.rb b/lib/puppet/util/checksums.rb index e129301e6..dd90caa5f 100644 --- a/lib/puppet/util/checksums.rb +++ b/lib/puppet/util/checksums.rb @@ -1,148 +1,148 @@ # A stand-alone module for calculating checksums # in a generic way. module Puppet::Util::Checksums class FakeChecksum def <<(*args) self end end # Is the provided string a checksum? def checksum?(string) string =~ /^\{(\w{3,5})\}\S+/ end # Strip the checksum type from an existing checksum def sumdata(checksum) checksum =~ /^\{(\w+)\}(.+)/ ? $2 : nil end # Strip the checksum type from an existing checksum def sumtype(checksum) checksum =~ /^\{(\w+)\}/ ? $1 : nil end # Calculate a checksum using Digest::MD5. def md5(content) require 'digest/md5' Digest::MD5.hexdigest(content) end # Calculate a checksum of the first 500 chars of the content using Digest::MD5. def md5lite(content) md5(content[0..511]) end # Calculate a checksum of a file's content using Digest::MD5. def md5_file(filename, lite = false) require 'digest/md5' digest = Digest::MD5.new checksum_file(digest, filename, lite) end # Calculate a checksum of the first 500 chars of a file's content using Digest::MD5. def md5lite_file(filename) md5_file(filename, true) end def md5_stream(&block) require 'digest/md5' digest = Digest::MD5.new yield digest digest.hexdigest end alias :md5lite_stream :md5_stream # Return the :mtime timestamp of a file. def mtime_file(filename) File.stat(filename).send(:mtime) end # by definition this doesn't exist # but we still need to execute the block given def mtime_stream noop_digest = FakeChecksum.new yield noop_digest nil end def mtime(content) "" end # Calculate a checksum using Digest::SHA1. def sha1(content) require 'digest/sha1' Digest::SHA1.hexdigest(content) end # Calculate a checksum of the first 500 chars of the content using Digest::SHA1. def sha1lite(content) sha1(content[0..511]) end # Calculate a checksum of a file's content using Digest::SHA1. def sha1_file(filename, lite = false) require 'digest/sha1' digest = Digest::SHA1.new checksum_file(digest, filename, lite) end # Calculate a checksum of the first 500 chars of a file's content using Digest::SHA1. def sha1lite_file(filename) sha1_file(filename, true) end def sha1_stream require 'digest/sha1' digest = Digest::SHA1.new yield digest digest.hexdigest end alias :sha1lite_stream :sha1_stream # Return the :ctime of a file. def ctime_file(filename) File.stat(filename).send(:ctime) end alias :ctime_stream :mtime_stream def ctime(content) "" end # Return a "no checksum" def none_file(filename) "" end def none_stream noop_digest = FakeChecksum.new yield noop_digest "" end def none(content) "" end private # Perform an incremental checksum on a file. def checksum_file(digest, filename, lite = false) buffer = lite ? 512 : 4096 - File.open(filename, 'r') do |file| + File.open(filename, 'rb') do |file| while content = file.read(buffer) digest << content break if lite end end digest.hexdigest end end diff --git a/spec/unit/application/inspect_spec.rb b/spec/unit/application/inspect_spec.rb index 750f25ab8..58eff023a 100755 --- a/spec/unit/application/inspect_spec.rb +++ b/spec/unit/application/inspect_spec.rb @@ -1,283 +1,284 @@ #!/usr/bin/env rspec require 'spec_helper' require 'puppet/application/inspect' require 'puppet/resource/catalog' require 'puppet/indirector/catalog/yaml' require 'puppet/indirector/report/rest' require 'puppet/indirector/file_bucket_file/rest' describe Puppet::Application::Inspect do include PuppetSpec::Files before :each do @inspect = Puppet::Application[:inspect] @inspect.preinit end it "should operate in agent run_mode" do @inspect.class.run_mode.name.should == :agent end describe "during setup" do it "should print its configuration if asked" do Puppet[:configprint] = "all" Puppet.settings.expects(:print_configs).returns(true) expect { @inspect.setup }.to exit_with 0 end it "should fail if reporting is turned off" do Puppet[:report] = false lambda { @inspect.setup }.should raise_error(/report=true/) end end describe "when executing" do before :each do Puppet[:report] = true @inspect.options[:logset] = true Puppet::Transaction::Report::Rest.any_instance.stubs(:save) @inspect.setup end it "should retrieve the local catalog" do Puppet::Resource::Catalog::Yaml.any_instance.expects(:find).with {|request| request.key == Puppet[:certname] }.returns(Puppet::Resource::Catalog.new) @inspect.run_command end it "should save the report to REST" do Puppet::Resource::Catalog::Yaml.any_instance.stubs(:find).returns(Puppet::Resource::Catalog.new) Puppet::Transaction::Report::Rest.any_instance.expects(:save).with {|request| request.instance.host == Puppet[:certname] } @inspect.run_command end it "should audit the specified properties" do catalog = Puppet::Resource::Catalog.new file = Tempfile.new("foo") + file.binmode file.puts("file contents") file.close resource = Puppet::Resource.new(:file, file.path, :parameters => {:audit => "all"}) catalog.add_resource(resource) Puppet::Resource::Catalog::Yaml.any_instance.stubs(:find).returns(catalog) events = nil Puppet::Transaction::Report::Rest.any_instance.expects(:save).with do |request| events = request.instance.resource_statuses.values.first.events end @inspect.run_command properties = events.inject({}) do |property_values, event| property_values.merge(event.property => event.previous_value) end properties["ensure"].should == :file properties["content"].should == "{md5}#{Digest::MD5.hexdigest("file contents\n")}" properties.has_key?("target").should == false end it "should set audited to true for all events" do catalog = Puppet::Resource::Catalog.new file = Tempfile.new("foo") resource = Puppet::Resource.new(:file, file.path, :parameters => {:audit => "all"}) catalog.add_resource(resource) Puppet::Resource::Catalog::Yaml.any_instance.stubs(:find).returns(catalog) events = nil Puppet::Transaction::Report::Rest.any_instance.expects(:save).with do |request| events = request.instance.resource_statuses.values.first.events end @inspect.run_command events.each do |event| event.audited.should == true end end it "should not report irrelevent attributes if the resource is absent" do catalog = Puppet::Resource::Catalog.new file = Tempfile.new("foo") resource = Puppet::Resource.new(:file, file.path, :parameters => {:audit => "all"}) file.close file.delete catalog.add_resource(resource) Puppet::Resource::Catalog::Yaml.any_instance.stubs(:find).returns(catalog) events = nil Puppet::Transaction::Report::Rest.any_instance.expects(:save).with do |request| events = request.instance.resource_statuses.values.first.events end @inspect.run_command properties = events.inject({}) do |property_values, event| property_values.merge(event.property => event.previous_value) end properties.should == {"ensure" => :absent} end describe "when archiving to a bucket" do before :each do Puppet[:archive_files] = true Puppet[:archive_file_server] = "filebucketserver" @catalog = Puppet::Resource::Catalog.new Puppet::Resource::Catalog::Yaml.any_instance.stubs(:find).returns(@catalog) end describe "when auditing files" do before :each do @file = tmpfile("foo") @resource = Puppet::Resource.new(:file, @file, :parameters => {:audit => "content"}) @catalog.add_resource(@resource) end it "should send an existing file to the file bucket" do File.open(@file, 'w') { |f| f.write('stuff') } Puppet::FileBucketFile::Rest.any_instance.expects(:head).with do |request| request.server == Puppet[:archive_file_server] end.returns(false) Puppet::FileBucketFile::Rest.any_instance.expects(:save).with do |request| request.server == Puppet[:archive_file_server] and request.instance.contents == 'stuff' end @inspect.run_command end it "should not send unreadable files", :unless => Puppet.features.microsoft_windows? do File.open(@file, 'w') { |f| f.write('stuff') } File.chmod(0, @file) Puppet::FileBucketFile::Rest.any_instance.expects(:head).never Puppet::FileBucketFile::Rest.any_instance.expects(:save).never @inspect.run_command end it "should not try to send non-existent files" do Puppet::FileBucketFile::Rest.any_instance.expects(:head).never Puppet::FileBucketFile::Rest.any_instance.expects(:save).never @inspect.run_command end it "should not try to send files whose content we are not auditing" do @resource[:audit] = "group" Puppet::FileBucketFile::Rest.any_instance.expects(:head).never Puppet::FileBucketFile::Rest.any_instance.expects(:save).never @inspect.run_command end it "should continue if bucketing a file fails" do File.open(@file, 'w') { |f| f.write('stuff') } Puppet::FileBucketFile::Rest.any_instance.stubs(:head).returns false Puppet::FileBucketFile::Rest.any_instance.stubs(:save).raises "failure" Puppet::Transaction::Report::Rest.any_instance.expects(:save).with do |request| @report = request.instance end @inspect.run_command @report.logs.first.should_not == nil @report.logs.first.message.should =~ /Could not back up/ end end describe "when auditing non-files" do before :each do Puppet::Type.newtype(:stub_type) do newparam(:name) do desc "The name var" isnamevar end newproperty(:content) do desc "content" def retrieve :whatever end end end @resource = Puppet::Resource.new(:stub_type, 'foo', :parameters => {:audit => "all"}) @catalog.add_resource(@resource) end after :each do Puppet::Type.rmtype(:stub_type) end it "should not try to send non-files" do Puppet::FileBucketFile::Rest.any_instance.expects(:head).never Puppet::FileBucketFile::Rest.any_instance.expects(:save).never @inspect.run_command end end end describe "when there are failures" do before :each do Puppet::Type.newtype(:stub_type) do newparam(:name) do desc "The name var" isnamevar end newproperty(:content) do desc "content" def retrieve raise "failed" end end end @catalog = Puppet::Resource::Catalog.new Puppet::Resource::Catalog::Yaml.any_instance.stubs(:find).returns(@catalog) Puppet::Transaction::Report::Rest.any_instance.expects(:save).with do |request| @report = request.instance end end after :each do Puppet::Type.rmtype(:stub_type) end it "should mark the report failed and create failed events for each property" do @resource = Puppet::Resource.new(:stub_type, 'foo', :parameters => {:audit => "all"}) @catalog.add_resource(@resource) @inspect.run_command @report.status.should == "failed" @report.logs.select{|log| log.message =~ /Could not inspect/}.size.should == 1 @report.resource_statuses.size.should == 1 @report.resource_statuses['Stub_type[foo]'].events.size.should == 1 event = @report.resource_statuses['Stub_type[foo]'].events.first event.property.should == "content" event.status.should == "failure" event.audited.should == true event.instance_variables.should_not include("@previous_value") end it "should continue to the next resource" do @resource = Puppet::Resource.new(:stub_type, 'foo', :parameters => {:audit => "all"}) @other_resource = Puppet::Resource.new(:stub_type, 'bar', :parameters => {:audit => "all"}) @catalog.add_resource(@resource) @catalog.add_resource(@other_resource) @inspect.run_command @report.resource_statuses.size.should == 2 @report.resource_statuses.keys.should =~ ['Stub_type[foo]', 'Stub_type[bar]'] end end end after :all do Puppet::Resource::Catalog.indirection.reset_terminus_class Puppet::Transaction::Report.indirection.terminus_class = :processor end end diff --git a/spec/unit/file_serving/metadata_spec.rb b/spec/unit/file_serving/metadata_spec.rb index 077c34251..222b76de8 100755 --- a/spec/unit/file_serving/metadata_spec.rb +++ b/spec/unit/file_serving/metadata_spec.rb @@ -1,329 +1,329 @@ #!/usr/bin/env rspec require 'spec_helper' require 'puppet/file_serving/metadata' describe Puppet::FileServing::Metadata do it "should should be a subclass of Base" do Puppet::FileServing::Metadata.superclass.should equal(Puppet::FileServing::Base) end it "should indirect file_metadata" do Puppet::FileServing::Metadata.indirection.name.should == :file_metadata end it "should should include the IndirectionHooks module in its indirection" do Puppet::FileServing::Metadata.indirection.singleton_class.included_modules.should include(Puppet::FileServing::IndirectionHooks) end it "should have a method that triggers attribute collection" do Puppet::FileServing::Metadata.new("/foo/bar").should respond_to(:collect) end it "should support pson serialization" do Puppet::FileServing::Metadata.new("/foo/bar").should respond_to(:to_pson) end it "should support to_pson_data_hash" do Puppet::FileServing::Metadata.new("/foo/bar").should respond_to(:to_pson_data_hash) end it "should support pson deserialization" do Puppet::FileServing::Metadata.should respond_to(:from_pson) end describe "when serializing" do before do @metadata = Puppet::FileServing::Metadata.new("/foo/bar") end it "should perform pson serialization by calling to_pson on it's pson_data_hash" do pdh = mock "data hash" pdh_as_pson = mock "data as pson" @metadata.expects(:to_pson_data_hash).returns pdh pdh.expects(:to_pson).returns pdh_as_pson @metadata.to_pson.should == pdh_as_pson end it "should serialize as FileMetadata" do @metadata.to_pson_data_hash['document_type'].should == "FileMetadata" end it "the data should include the path, relative_path, links, owner, group, mode, checksum, type, and destination" do @metadata.to_pson_data_hash['data'].keys.sort.should == %w{ path relative_path links owner group mode checksum type destination }.sort end it "should pass the path in the hash verbatum" do @metadata.to_pson_data_hash['data']['path'] == @metadata.path end it "should pass the relative_path in the hash verbatum" do @metadata.to_pson_data_hash['data']['relative_path'] == @metadata.relative_path end it "should pass the links in the hash verbatum" do @metadata.to_pson_data_hash['data']['links'] == @metadata.links end it "should pass the path owner in the hash verbatum" do @metadata.to_pson_data_hash['data']['owner'] == @metadata.owner end it "should pass the group in the hash verbatum" do @metadata.to_pson_data_hash['data']['group'] == @metadata.group end it "should pass the mode in the hash verbatum" do @metadata.to_pson_data_hash['data']['mode'] == @metadata.mode end it "should pass the ftype in the hash verbatum as the 'type'" do @metadata.to_pson_data_hash['data']['type'] == @metadata.ftype end it "should pass the destination verbatum" do @metadata.to_pson_data_hash['data']['destination'] == @metadata.destination end it "should pass the checksum in the hash as a nested hash" do @metadata.to_pson_data_hash['data']['checksum'].should be_is_a(Hash) end it "should pass the checksum_type in the hash verbatum as the checksum's type" do @metadata.to_pson_data_hash['data']['checksum']['type'] == @metadata.checksum_type end it "should pass the checksum in the hash verbatum as the checksum's value" do @metadata.to_pson_data_hash['data']['checksum']['value'] == @metadata.checksum end end end describe Puppet::FileServing::Metadata do include PuppetSpec::Files shared_examples_for "metadata collector" do let(:metadata) do data = described_class.new(path) data.collect data end describe "when collecting attributes" do describe "when managing files" do let(:path) { tmpfile('file_serving_metadata') } before :each do FileUtils.touch(path) end it "should be able to produce xmlrpc-style attribute information" do metadata.should respond_to(:attributes_with_tabs) end it "should set the owner to the file's current owner" do metadata.owner.should == owner end it "should set the group to the file's current group" do metadata.group.should == group end it "should set the mode to the file's masked mode" do set_mode(33261, path) metadata.mode.should == 0755 end describe "#checksum" do let(:checksum) { Digest::MD5.hexdigest("some content\n") } before :each do - File.open(path, "w") {|f| f.print("some content\n")} + File.open(path, "wb") {|f| f.print("some content\n")} end it "should default to a checksum of type MD5 with the file's current checksum" do metadata.checksum.should == "{md5}#{checksum}" end it "should give a mtime checksum when checksum_type is set" do time = Time.now metadata.checksum_type = "mtime" metadata.expects(:mtime_file).returns(@time) metadata.collect metadata.checksum.should == "{mtime}#{@time}" end it "should produce tab-separated mode, type, owner, group, and checksum for xmlrpc" do set_mode(0755, path) metadata.attributes_with_tabs.should == "#{0755.to_s}\tfile\t#{owner}\t#{group}\t{md5}#{checksum}" end end end describe "when managing directories" do let(:path) { tmpdir('file_serving_metadata_dir') } let(:time) { Time.now } before :each do metadata.expects(:ctime_file).returns(time) end it "should only use checksums of type 'ctime' for directories" do metadata.collect metadata.checksum.should == "{ctime}#{time}" end it "should only use checksums of type 'ctime' for directories even if checksum_type set" do metadata.checksum_type = "mtime" metadata.expects(:mtime_file).never metadata.collect metadata.checksum.should == "{ctime}#{time}" end it "should produce tab-separated mode, type, owner, group, and checksum for xmlrpc" do set_mode(0755, path) metadata.collect metadata.attributes_with_tabs.should == "#{0755.to_s}\tdirectory\t#{owner}\t#{group}\t{ctime}#{time.to_s}" end end describe "when managing links", :unless => Puppet.features.microsoft_windows? do # 'path' is a link that points to 'target' let(:path) { tmpfile('file_serving_metadata_link') } let(:target) { tmpfile('file_serving_metadata_target') } let(:checksum) { Digest::MD5.hexdigest("some content\n") } let(:fmode) { File.lstat(path).mode & 0777 } before :each do - File.open(target, "w") {|f| f.print("some content\n")} + File.open(target, "wb") {|f| f.print("some content\n")} set_mode(0644, target) FileUtils.symlink(target, path) end it "should read links instead of returning their checksums" do metadata.destination.should == target end pending "should produce tab-separated mode, type, owner, group, and destination for xmlrpc" do # "We'd like this to be true, but we need to always collect the checksum because in the server/client/server round trip we lose the distintion between manage and follow." metadata.attributes_with_tabs.should == "#{0755}\tlink\t#{owner}\t#{group}\t#{target}" end it "should produce tab-separated mode, type, owner, group, checksum, and destination for xmlrpc" do metadata.attributes_with_tabs.should == "#{fmode}\tlink\t#{owner}\t#{group}\t{md5}eb9c2bf0eb63f3a7bc0ea37ef18aeba5\t#{target}" end end end describe Puppet::FileServing::Metadata, " when finding the file to use for setting attributes" do let(:path) { tmpfile('file_serving_metadata_find_file') } before :each do - File.open(path, "w") {|f| f.print("some content\n")} + File.open(path, "wb") {|f| f.print("some content\n")} set_mode(0755, path) end it "should accept a base path to which the file should be relative" do dir = tmpdir('metadata_dir') metadata = described_class.new(dir) metadata.relative_path = 'relative_path' FileUtils.touch(metadata.full_path) metadata.collect end it "should use the set base path if one is not provided" do metadata.collect end it "should raise an exception if the file does not exist" do File.delete(path) proc { metadata.collect}.should raise_error(Errno::ENOENT) end end end describe "on POSIX systems", :if => Puppet.features.posix? do let(:owner) {10} let(:group) {20} before :each do File::Stat.any_instance.stubs(:uid).returns owner File::Stat.any_instance.stubs(:gid).returns group end it_should_behave_like "metadata collector" def set_mode(mode, path) File.chmod(mode, path) end end describe "on Windows systems", :if => Puppet.features.microsoft_windows? do let(:owner) {'S-1-1-50'} let(:group) {'S-1-1-51'} before :each do require 'puppet/util/windows/security' Puppet::Util::Windows::Security.stubs(:get_owner).returns owner Puppet::Util::Windows::Security.stubs(:get_group).returns group end it_should_behave_like "metadata collector" def set_mode(mode, path) Puppet::Util::Windows::Security.set_mode(mode, path) end end end describe Puppet::FileServing::Metadata, " when pointing to a link", :unless => Puppet.features.microsoft_windows? do describe "when links are managed" do before do @file = Puppet::FileServing::Metadata.new("/base/path/my/file", :links => :manage) File.expects(:lstat).with("/base/path/my/file").returns stub("stat", :uid => 1, :gid => 2, :ftype => "link", :mode => 0755) File.expects(:readlink).with("/base/path/my/file").returns "/some/other/path" @checksum = Digest::MD5.hexdigest("some content\n") # Remove these when :managed links are no longer checksumed. @file.stubs(:md5_file).returns(@checksum) # end it "should store the destination of the link in :destination if links are :manage" do @file.collect @file.destination.should == "/some/other/path" end pending "should not collect the checksum if links are :manage" do # We'd like this to be true, but we need to always collect the checksum because in the server/client/server round trip we lose the distintion between manage and follow. @file.collect @file.checksum.should be_nil end it "should collect the checksum if links are :manage" do # see pending note above @file.collect @file.checksum.should == "{md5}#{@checksum}" end end describe "when links are followed" do before do @file = Puppet::FileServing::Metadata.new("/base/path/my/file", :links => :follow) File.expects(:stat).with("/base/path/my/file").returns stub("stat", :uid => 1, :gid => 2, :ftype => "file", :mode => 0755) File.expects(:readlink).with("/base/path/my/file").never @checksum = Digest::MD5.hexdigest("some content\n") @file.stubs(:md5_file).returns(@checksum) end it "should not store the destination of the link in :destination if links are :follow" do @file.collect @file.destination.should be_nil end it "should collect the checksum if links are :follow" do @file.collect @file.checksum.should == "{md5}#{@checksum}" end end end diff --git a/spec/unit/util/checksums_spec.rb b/spec/unit/util/checksums_spec.rb index f8800b512..4fbc097c0 100755 --- a/spec/unit/util/checksums_spec.rb +++ b/spec/unit/util/checksums_spec.rb @@ -1,157 +1,175 @@ #!/usr/bin/env rspec require 'spec_helper' require 'puppet/util/checksums' describe Puppet::Util::Checksums do + include PuppetSpec::Files + before do @summer = Object.new @summer.extend(Puppet::Util::Checksums) end content_sums = [:md5, :md5lite, :sha1, :sha1lite] file_only = [:ctime, :mtime, :none] content_sums.each do |sumtype| it "should be able to calculate #{sumtype} sums from strings" do @summer.should be_respond_to(sumtype) end end [content_sums, file_only].flatten.each do |sumtype| it "should be able to calculate #{sumtype} sums from files" do @summer.should be_respond_to(sumtype.to_s + "_file") end end [content_sums, file_only].flatten.each do |sumtype| it "should be able to calculate #{sumtype} sums from stream" do @summer.should be_respond_to(sumtype.to_s + "_stream") end end it "should have a method for determining whether a given string is a checksum" do @summer.should respond_to(:checksum?) end %w{{md5}asdfasdf {sha1}asdfasdf {ctime}asdasdf {mtime}asdfasdf}.each do |sum| it "should consider #{sum} to be a checksum" do @summer.should be_checksum(sum) end end %w{{nosuchsum}asdfasdf {a}asdfasdf {ctime}}.each do |sum| it "should not consider #{sum} to be a checksum" do @summer.should_not be_checksum(sum) end end it "should have a method for stripping a sum type from an existing checksum" do @summer.sumtype("{md5}asdfasdfa").should == "md5" end it "should have a method for stripping the data from a checksum" do @summer.sumdata("{md5}asdfasdfa").should == "asdfasdfa" end it "should return a nil sumtype if the checksum does not mention a checksum type" do @summer.sumtype("asdfasdfa").should be_nil end {:md5 => Digest::MD5, :sha1 => Digest::SHA1}.each do |sum, klass| describe("when using #{sum}") do it "should use #{klass} to calculate string checksums" do klass.expects(:hexdigest).with("mycontent").returns "whatever" @summer.send(sum, "mycontent").should == "whatever" end it "should use incremental #{klass} sums to calculate file checksums" do digest = mock 'digest' klass.expects(:new).returns digest file = "/path/to/my/file" fh = mock 'filehandle' fh.expects(:read).with(4096).times(3).returns("firstline").then.returns("secondline").then.returns(nil) #fh.expects(:read).with(512).returns("secondline") #fh.expects(:read).with(512).returns(nil) - File.expects(:open).with(file, "r").yields(fh) + File.expects(:open).with(file, "rb").yields(fh) digest.expects(:<<).with "firstline" digest.expects(:<<).with "secondline" digest.expects(:hexdigest).returns :mydigest @summer.send(sum.to_s + "_file", file).should == :mydigest end it "should yield #{klass} to the given block to calculate stream checksums" do digest = mock 'digest' klass.expects(:new).returns digest digest.expects(:hexdigest).returns :mydigest @summer.send(sum.to_s + "_stream") do |sum| sum.should == digest end.should == :mydigest end + end end {:md5lite => Digest::MD5, :sha1lite => Digest::SHA1}.each do |sum, klass| describe("when using #{sum}") do it "should use #{klass} to calculate string checksums from the first 512 characters of the string" do content = "this is a test" * 100 klass.expects(:hexdigest).with(content[0..511]).returns "whatever" @summer.send(sum, content).should == "whatever" end it "should use #{klass} to calculate a sum from the first 512 characters in the file" do digest = mock 'digest' klass.expects(:new).returns digest file = "/path/to/my/file" fh = mock 'filehandle' fh.expects(:read).with(512).returns('my content') - File.expects(:open).with(file, "r").yields(fh) + File.expects(:open).with(file, "rb").yields(fh) digest.expects(:<<).with "my content" digest.expects(:hexdigest).returns :mydigest @summer.send(sum.to_s + "_file", file).should == :mydigest end end end [:ctime, :mtime].each do |sum| describe("when using #{sum}") do it "should use the '#{sum}' on the file to determine the ctime" do file = "/my/file" stat = mock 'stat', sum => "mysum" File.expects(:stat).with(file).returns(stat) @summer.send(sum.to_s + "_file", file).should == "mysum" end it "should return nil for streams" do expectation = stub "expectation" expectation.expects(:do_something!).at_least_once @summer.send(sum.to_s + "_stream"){ |checksum| checksum << "anything" ; expectation.do_something! }.should be_nil end end end describe "when using the none checksum" do it "should return an empty string" do @summer.none_file("/my/file").should == "" end it "should return an empty string for streams" do expectation = stub "expectation" expectation.expects(:do_something!).at_least_once @summer.none_stream{ |checksum| checksum << "anything" ; expectation.do_something! }.should == "" end end + + {:md5 => Digest::MD5, :sha1 => Digest::SHA1}.each do |sum, klass| + describe "when using #{sum}" do + let(:content) { "hello\r\nworld" } + let(:path) do + path = tmpfile("checksum_#{sum}") + File.open(path, 'wb') {|f| f.write(content)} + path + end + + it "should preserve nl/cr sequences" do + @summer.send(sum.to_s + "_file", path).should == klass.hexdigest(content) + end + end + end end