diff --git a/lib/puppet/type/file/source.rb b/lib/puppet/type/file/source.rb index bb4ffcec6..71e0b539e 100644 --- a/lib/puppet/type/file/source.rb +++ b/lib/puppet/type/file/source.rb @@ -1,258 +1,259 @@ require 'puppet/file_serving/content' require 'puppet/file_serving/metadata' module Puppet # Copy files from a local or remote source. This state *only* does any work # when the remote file is an actual file; in that case, this state copies # the file down. If the remote file is a dir or a link or whatever, then # this state, during retrieval, modifies the appropriate other states # so that things get taken care of appropriately. Puppet::Type.type(:file).newparam(:source) do include Puppet::Util::Diff attr_accessor :source, :local desc <<-'EOT' A source file, which will be copied into place on the local system. Values can be URIs pointing to remote files, or fully qualified paths to files available on the local system (including files on NFS shares or Windows mapped drives). This attribute is mutually exclusive with `content` and `target`. The available URI schemes are *puppet* and *file*. *Puppet* URIs will retrieve files from Puppet's built-in file server, and are usually formatted as: `puppet:///modules/name_of_module/filename` This will fetch a file from a module on the puppet master (or from a local module when using puppet apply). Given a `modulepath` of `/etc/puppetlabs/puppet/modules`, the example above would resolve to `/etc/puppetlabs/puppet/modules/name_of_module/files/filename`. Unlike `content`, the `source` attribute can be used to recursively copy directories if the `recurse` attribute is set to `true` or `remote`. If a source directory contains symlinks, use the `links` attribute to specify whether to recreate links or follow them. Multiple `source` values can be specified as an array, and Puppet will use the first source that exists. This can be used to serve different files to different system types: file { "/etc/nfs.conf": source => [ "puppet:///modules/nfs/conf.$host", "puppet:///modules/nfs/conf.$operatingsystem", "puppet:///modules/nfs/conf" ] } Alternately, when serving directories recursively, multiple sources can be combined by setting the `sourceselect` attribute to `all`. EOT validate do |sources| sources = [sources] unless sources.is_a?(Array) sources.each do |source| next if Puppet::Util.absolute_path?(source) begin uri = URI.parse(URI.escape(source)) rescue => detail self.fail Puppet::Error, "Could not understand source #{source}: #{detail}", detail end self.fail "Cannot use relative URLs '#{source}'" unless uri.absolute? self.fail "Cannot use opaque URLs '#{source}'" unless uri.hierarchical? self.fail "Cannot use URLs of type '#{uri.scheme}' as source for fileserving" unless %w{file puppet}.include?(uri.scheme) end end SEPARATOR_REGEX = [Regexp.escape(File::SEPARATOR.to_s), Regexp.escape(File::ALT_SEPARATOR.to_s)].join munge do |sources| sources = [sources] unless sources.is_a?(Array) sources.map do |source| source = source.sub(/[#{SEPARATOR_REGEX}]+$/, '') if Puppet::Util.absolute_path?(source) URI.unescape(Puppet::Util.path_to_uri(source).to_s) else source end end end def change_to_s(currentvalue, newvalue) # newvalue = "{md5}#{@metadata.checksum}" if resource.property(:ensure).retrieve == :absent return "creating from source #{metadata.source} with contents #{metadata.checksum}" else return "replacing from source #{metadata.source} with contents #{metadata.checksum}" end end def checksum metadata && metadata.checksum end # Look up (if necessary) and return local content. def content return @content if @content raise Puppet::DevError, "No source for content was stored with the metadata" unless metadata.source unless tmp = Puppet::FileServing::Content.indirection.find(metadata.source, :environment => resource.catalog.environment_instance, :links => resource[:links]) self.fail "Could not find any content at %s" % metadata.source end @content = tmp.content end # Copy the values from the source to the resource. Yay. def copy_source_values devfail "Somehow got asked to copy source values without any metadata" unless metadata # conditionally copy :checksum if metadata.ftype != "directory" && !(metadata.ftype == "link" && metadata.links == :manage) copy_source_value(:checksum) end # Take each of the stats and set them as states on the local file # if a value has not already been provided. [:owner, :mode, :group].each do |metadata_method| next if metadata_method == :owner and !Puppet.features.root? next if metadata_method == :group and !Puppet.features.root? if Puppet.features.microsoft_windows? # Warn on Windows if source permissions are being used and the file resource # does not have mode owner and group all set (which would take precedence). if [:use, :use_when_creating].include?(resource[:source_permissions]) && (resource[:owner] == nil || resource[:group] == nil || resource[:mode] == nil) err_msg = "Copying %s from the source" << " file on Windows is not supported;" << " use source_permissions => ignore." self.fail Puppet::Error, err_msg % 'owner/mode/group' end # But never try to copy remote owner/group on Windows next if [:owner, :group].include?(metadata_method) && !local? end case resource[:source_permissions] when :ignore next when :use_when_creating next if Puppet::FileSystem.exist?(resource[:path]) end copy_source_value(metadata_method) end if resource[:ensure] == :absent # We know all we need to elsif metadata.ftype != "link" resource[:ensure] = metadata.ftype elsif resource[:links] == :follow resource[:ensure] = :present else resource[:ensure] = "link" resource[:target] = metadata.destination end end attr_writer :metadata # Provide, and retrieve if necessary, the metadata for this file. Fail # if we can't find data about this host, and fail if there are any # problems in our query. def metadata return @metadata if @metadata return nil unless value value.each do |source| begin options = { :environment => resource.catalog.environment_instance, :links => resource[:links], + :checksum_type => resource[:checksum], :source_permissions => resource[:source_permissions] } if data = Puppet::FileServing::Metadata.indirection.find(source, options) @metadata = data @metadata.source = source break end rescue => detail self.fail Puppet::Error, "Could not retrieve file metadata for #{source}: #{detail}", detail end end self.fail "Could not retrieve information from environment #{resource.catalog.environment} source(s) #{value.join(", ")}" unless @metadata @metadata end def local? found? and scheme == "file" end def full_path Puppet::Util.uri_to_path(uri) if found? end def server? uri and uri.host end def server (uri and uri.host) or Puppet.settings[:server] end def port (uri and uri.port) or Puppet.settings[:masterport] end def uri @uri ||= URI.parse(URI.escape(metadata.source)) end private def scheme (uri and uri.scheme) end def found? ! (metadata.nil? or metadata.ftype.nil?) end def copy_source_value(metadata_method) param_name = (metadata_method == :checksum) ? :content : metadata_method if resource[param_name].nil? or resource[param_name] == :absent value = metadata.send(metadata_method) # Force the mode value in file resources to be a string containing octal. value = value.to_s(8) if param_name == :mode && value.is_a?(Numeric) resource[param_name] = value end end end Puppet::Type.type(:file).newparam(:source_permissions) do desc <<-'EOT' Whether (and how) Puppet should copy owner, group, and mode permissions from the `source` to `file` resources when the permissions are not explicitly specified. (In all cases, explicit permissions will take precedence.) Valid values are `use`, `use_when_creating`, and `ignore`: * `ignore` (the default) will never apply the owner, group, or mode from the `source` when managing a file. When creating new files without explicit permissions, the permissions they receive will depend on platform-specific behavior. On POSIX, Puppet will use the umask of the user it is running as. On Windows, Puppet will use the default DACL associated with the user it is running as. * `use` will cause Puppet to apply the owner, group, and mode from the `source` to any files it is managing. * `use_when_creating` will only apply the owner, group, and mode from the `source` when creating a file; existing files will not have their permissions overwritten. EOT defaultto :ignore newvalues(:use, :use_when_creating, :ignore) end end diff --git a/lib/puppet/util/checksums.rb b/lib/puppet/util/checksums.rb index 6cb49a8b1..286e12c11 100644 --- a/lib/puppet/util/checksums.rb +++ b/lib/puppet/util/checksums.rb @@ -1,216 +1,239 @@ require 'digest/md5' require 'digest/sha1' # A stand-alone module for calculating checksums # in a generic way. module Puppet::Util::Checksums module_function # It's not a good idea to use some of these in some contexts: for example, I # wouldn't try bucketing a file using the :none checksum type. def known_checksum_types [:sha256, :sha256lite, :md5, :md5lite, :sha1, :sha1lite, :mtime, :ctime, :none] end class FakeChecksum def <<(*args) self end end # Is the provided string a checksum? def checksum?(string) # 'sha256lite'.length == 10 string =~ /^\{(\w{3,10})\}\S+/ end # Strip the checksum type from an existing checksum def sumdata(checksum) checksum =~ /^\{(\w+)\}(.+)/ ? $2 : nil end # Strip the checksum type from an existing checksum def sumtype(checksum) checksum =~ /^\{(\w+)\}/ ? $1 : nil end # Calculate a checksum using Digest::SHA256. def sha256(content) require 'digest/sha2' Digest::SHA256.hexdigest(content) end def sha256lite(content) sha256(content[0..511]) end def sha256_file(filename, lite = false) require 'digest/sha2' digest = Digest::SHA256.new checksum_file(digest, filename, lite) end def sha256lite_file(filename) sha256_file(filename, true) end - def sha256_stream(&block) + def sha256_stream(lite = false, &block) require 'digest/sha2' digest = Digest::SHA256.new - yield digest - digest.hexdigest + checksum_stream(digest, block, lite) end def sha256_hex_length 64 end def sha256lite_stream(&block) - sha256_stream(&block) + sha256_stream(true, &block) end def sha256lite_hex_length sha256_hex_length end # Calculate a checksum using Digest::MD5. def md5(content) Digest::MD5.hexdigest(content) end # Calculate a checksum of the first 500 chars of the content using Digest::MD5. def md5lite(content) md5(content[0..511]) end # Calculate a checksum of a file's content using Digest::MD5. def md5_file(filename, lite = false) digest = Digest::MD5.new checksum_file(digest, filename, lite) end # Calculate a checksum of the first 500 chars of a file's content using Digest::MD5. def md5lite_file(filename) md5_file(filename, true) end - def md5_stream(&block) + def md5_stream(lite = false, &block) digest = Digest::MD5.new - yield digest - digest.hexdigest + checksum_stream(digest, block, lite) end def md5_hex_length 32 end def md5lite_stream(&block) - md5_stream(&block) + md5_stream(true, &block) end def md5lite_hex_length md5_hex_length end # Return the :mtime timestamp of a file. def mtime_file(filename) Puppet::FileSystem.stat(filename).send(:mtime) end # by definition this doesn't exist # but we still need to execute the block given def mtime_stream(&block) noop_digest = FakeChecksum.new yield noop_digest nil end def mtime(content) "" end # Calculate a checksum using Digest::SHA1. def sha1(content) Digest::SHA1.hexdigest(content) end # Calculate a checksum of the first 500 chars of the content using Digest::SHA1. def sha1lite(content) sha1(content[0..511]) end # Calculate a checksum of a file's content using Digest::SHA1. def sha1_file(filename, lite = false) digest = Digest::SHA1.new checksum_file(digest, filename, lite) end # Calculate a checksum of the first 500 chars of a file's content using Digest::SHA1. def sha1lite_file(filename) sha1_file(filename, true) end - def sha1_stream(&block) + def sha1_stream(lite = false, &block) digest = Digest::SHA1.new - yield digest - digest.hexdigest + checksum_stream(digest, block, lite) end def sha1_hex_length 40 end def sha1lite_stream(&block) - sha1_stream(&block) + sha1_stream(true, &block) end def sha1lite_hex_length sha1_hex_length end # Return the :ctime of a file. def ctime_file(filename) Puppet::FileSystem.stat(filename).send(:ctime) end def ctime_stream(&block) mtime_stream(&block) end def ctime(content) "" end # Return a "no checksum" def none_file(filename) "" end def none_stream noop_digest = FakeChecksum.new yield noop_digest "" end def none(content) "" end + class DigestLite + def initialize(digest, lite = false) + @digest = digest + @lite = lite + @bytes = 0 + end + + # Provide an interface for digests. If lite, only digest the first 512 bytes + def <<(str) + if @lite + if @bytes < 512 + buf = str[0, 512 - @bytes] + @digest << buf + @bytes += buf.length + end + else + @digest << str + end + end + end + private_class_method # Perform an incremental checksum on a file. def checksum_file(digest, filename, lite = false) buffer = lite ? 512 : 4096 File.open(filename, 'rb') do |file| while content = file.read(buffer) digest << content break if lite end end digest.hexdigest end + def checksum_stream(digest, block, lite = false) + block.call(DigestLite.new(digest, lite)) + digest.hexdigest + end + end diff --git a/spec/integration/type/file_spec.rb b/spec/integration/type/file_spec.rb index c9a0a43ba..e78c20f95 100755 --- a/spec/integration/type/file_spec.rb +++ b/spec/integration/type/file_spec.rb @@ -1,1300 +1,1319 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet_spec/files' if Puppet.features.microsoft_windows? require 'puppet/util/windows' class WindowsSecurity extend Puppet::Util::Windows::Security end end describe Puppet::Type.type(:file), :uses_checksums => true do include PuppetSpec::Files + include_context 'with supported checksum types' let(:catalog) { Puppet::Resource::Catalog.new } let(:path) do # we create a directory first so backups of :path that are stored in # the same directory will also be removed after the tests parent = tmpdir('file_spec') File.join(parent, 'file_testing') end let(:dir) do # we create a directory first so backups of :path that are stored in # the same directory will also be removed after the tests parent = tmpdir('file_spec') File.join(parent, 'dir_testing') end if Puppet.features.posix? def set_mode(mode, file) File.chmod(mode, file) end def get_mode(file) Puppet::FileSystem.lstat(file).mode end def get_owner(file) Puppet::FileSystem.lstat(file).uid end def get_group(file) Puppet::FileSystem.lstat(file).gid end else class SecurityHelper extend Puppet::Util::Windows::Security end def set_mode(mode, file) SecurityHelper.set_mode(mode, file) end def get_mode(file) SecurityHelper.get_mode(file) end def get_owner(file) SecurityHelper.get_owner(file) end def get_group(file) SecurityHelper.get_group(file) end def get_aces_for_path_by_sid(path, sid) SecurityHelper.get_aces_for_path_by_sid(path, sid) end end around :each do |example| Puppet.override(:environments => Puppet::Environments::Static.new) do example.run end end before do # stub this to not try to create state.yaml Puppet::Util::Storage.stubs(:store) end it "should not attempt to manage files that do not exist if no means of creating the file is specified" do source = tmpfile('source') catalog.add_resource described_class.new :path => source, :mode => '0755' status = catalog.apply.report.resource_statuses["File[#{source}]"] expect(status).not_to be_failed expect(status).not_to be_changed expect(Puppet::FileSystem.exist?(source)).to be_falsey end describe "when ensure is absent" do it "should remove the file if present" do FileUtils.touch(path) catalog.add_resource(described_class.new(:path => path, :ensure => :absent, :backup => :false)) report = catalog.apply.report expect(report.resource_statuses["File[#{path}]"]).not_to be_failed expect(Puppet::FileSystem.exist?(path)).to be_falsey end it "should do nothing if file is not present" do catalog.add_resource(described_class.new(:path => path, :ensure => :absent, :backup => :false)) report = catalog.apply.report expect(report.resource_statuses["File[#{path}]"]).not_to be_failed expect(Puppet::FileSystem.exist?(path)).to be_falsey end # issue #14599 it "should not fail if parts of path aren't directories" do FileUtils.touch(path) catalog.add_resource(described_class.new(:path => File.join(path,'no_such_file'), :ensure => :absent, :backup => :false)) report = catalog.apply.report expect(report.resource_statuses["File[#{File.join(path,'no_such_file')}]"]).not_to be_failed end end describe "when setting permissions" do it "should set the owner" do target = tmpfile_with_contents('target', '') owner = get_owner(target) catalog.add_resource described_class.new( :name => target, :owner => owner ) catalog.apply expect(get_owner(target)).to eq(owner) end it "should set the group" do target = tmpfile_with_contents('target', '') group = get_group(target) catalog.add_resource described_class.new( :name => target, :group => group ) catalog.apply expect(get_group(target)).to eq(group) end describe "when setting mode" do describe "for directories" do let(:target) { tmpdir('dir_mode') } it "should set executable bits for newly created directories" do catalog.add_resource described_class.new(:path => target, :ensure => :directory, :mode => '0600') catalog.apply expect(get_mode(target) & 07777).to eq(0700) end it "should set executable bits for existing readable directories" do set_mode(0600, target) catalog.add_resource described_class.new(:path => target, :ensure => :directory, :mode => '0644') catalog.apply expect(get_mode(target) & 07777).to eq(0755) end it "should not set executable bits for unreadable directories" do begin catalog.add_resource described_class.new(:path => target, :ensure => :directory, :mode => '0300') catalog.apply expect(get_mode(target) & 07777).to eq(0300) ensure # so we can cleanup set_mode(0700, target) end end it "should set user, group, and other executable bits" do catalog.add_resource described_class.new(:path => target, :ensure => :directory, :mode => '0664') catalog.apply expect(get_mode(target) & 07777).to eq(0775) end it "should set executable bits when overwriting a non-executable file" do target_path = tmpfile_with_contents('executable', '') set_mode(0444, target_path) catalog.add_resource described_class.new(:path => target_path, :ensure => :directory, :mode => '0666', :backup => false) catalog.apply expect(get_mode(target_path) & 07777).to eq(0777) expect(File).to be_directory(target_path) end end describe "for files" do it "should not set executable bits" do catalog.add_resource described_class.new(:path => path, :ensure => :file, :mode => '0666') catalog.apply expect(get_mode(path) & 07777).to eq(0666) end it "should not set executable bits when replacing an executable directory (#10365)" do pending("bug #10365") FileUtils.mkdir(path) set_mode(0777, path) catalog.add_resource described_class.new(:path => path, :ensure => :file, :mode => 0666, :backup => false, :force => true) catalog.apply expect(get_mode(path) & 07777).to eq(0666) end end describe "for links", :if => described_class.defaultprovider.feature?(:manages_symlinks) do let(:link) { tmpfile('link_mode') } describe "when managing links" do let(:link_target) { tmpfile('target') } before :each do FileUtils.touch(link_target) File.chmod(0444, link_target) Puppet::FileSystem.symlink(link_target, link) end it "should not set the executable bit on the link nor the target" do catalog.add_resource described_class.new(:path => link, :ensure => :link, :mode => '0666', :target => link_target, :links => :manage) catalog.apply (Puppet::FileSystem.stat(link).mode & 07777) == 0666 (Puppet::FileSystem.lstat(link_target).mode & 07777) == 0444 end it "should ignore dangling symlinks (#6856)" do File.delete(link_target) catalog.add_resource described_class.new(:path => link, :ensure => :link, :mode => '0666', :target => link_target, :links => :manage) catalog.apply expect(Puppet::FileSystem.exist?(link)).to be_falsey end it "should create a link to the target if ensure is omitted" do FileUtils.touch(link_target) catalog.add_resource described_class.new(:path => link, :target => link_target) catalog.apply expect(Puppet::FileSystem.exist?(link)).to be_truthy expect(Puppet::FileSystem.lstat(link).ftype).to eq('link') expect(Puppet::FileSystem.readlink(link)).to eq(link_target) end end describe "when following links" do it "should ignore dangling symlinks (#6856)" do target = tmpfile('dangling') FileUtils.touch(target) Puppet::FileSystem.symlink(target, link) File.delete(target) catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0600', :links => :follow) catalog.apply end describe "to a directory" do let(:link_target) { tmpdir('dir_target') } before :each do File.chmod(0600, link_target) Puppet::FileSystem.symlink(link_target, link) end after :each do File.chmod(0750, link_target) end describe "that is readable" do it "should set the executable bits when creating the destination (#10315)" do catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0666', :links => :follow) catalog.apply expect(File).to be_directory(path) expect(get_mode(path) & 07777).to eq(0777) end it "should set the executable bits when overwriting the destination (#10315)" do FileUtils.touch(path) catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0666', :links => :follow, :backup => false) catalog.apply expect(File).to be_directory(path) expect(get_mode(path) & 07777).to eq(0777) end end describe "that is not readable" do before :each do set_mode(0300, link_target) end # so we can cleanup after :each do set_mode(0700, link_target) end it "should set executable bits when creating the destination (#10315)" do catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0666', :links => :follow) catalog.apply expect(File).to be_directory(path) expect(get_mode(path) & 07777).to eq(0777) end it "should set executable bits when overwriting the destination" do FileUtils.touch(path) catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0666', :links => :follow, :backup => false) catalog.apply expect(File).to be_directory(path) expect(get_mode(path) & 07777).to eq(0777) end end end describe "to a file" do let(:link_target) { tmpfile('file_target') } before :each do FileUtils.touch(link_target) Puppet::FileSystem.symlink(link_target, link) end it "should create the file, not a symlink (#2817, #10315)" do catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0600', :links => :follow) catalog.apply expect(File).to be_file(path) expect(get_mode(path) & 07777).to eq(0600) end it "should overwrite the file" do FileUtils.touch(path) catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0600', :links => :follow) catalog.apply expect(File).to be_file(path) expect(get_mode(path) & 07777).to eq(0600) end end describe "to a link to a directory" do let(:real_target) { tmpdir('real_target') } let(:target) { tmpfile('target') } before :each do File.chmod(0666, real_target) # link -> target -> real_target Puppet::FileSystem.symlink(real_target, target) Puppet::FileSystem.symlink(target, link) end after :each do File.chmod(0750, real_target) end describe "when following all links" do it "should create the destination and apply executable bits (#10315)" do catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0600', :links => :follow) catalog.apply expect(File).to be_directory(path) expect(get_mode(path) & 07777).to eq(0700) end it "should overwrite the destination and apply executable bits" do FileUtils.mkdir(path) catalog.add_resource described_class.new(:path => path, :source => link, :mode => '0600', :links => :follow) catalog.apply expect(File).to be_directory(path) expect(get_mode(path) & 0111).to eq(0100) end end end end end end end describe "when writing files" do - with_digest_algorithms do + shared_examples "files are backed up" do |resource_options| it "should backup files to a filebucket when one is configured" do filebucket = Puppet::Type.type(:filebucket).new :path => tmpfile("filebucket"), :name => "mybucket" - file = described_class.new :path => path, :backup => "mybucket", :content => "foo" + file = described_class.new({:path => path, :backup => "mybucket", :content => "foo"}.merge(resource_options)) catalog.add_resource file catalog.add_resource filebucket File.open(file[:path], "w") { |f| f.write("bar") } - d = digest(IO.binread(file[:path])) + d = filebucket_digest.call(IO.binread(file[:path])) catalog.apply expect(filebucket.bucket.getfile(d)).to eq("bar") end it "should backup files in the local directory when a backup string is provided" do - file = described_class.new :path => path, :backup => ".bak", :content => "foo" + file = described_class.new({:path => path, :backup => ".bak", :content => "foo"}.merge(resource_options)) catalog.add_resource file File.open(file[:path], "w") { |f| f.puts "bar" } catalog.apply backup = file[:path] + ".bak" expect(Puppet::FileSystem.exist?(backup)).to be_truthy expect(File.read(backup)).to eq("bar\n") end it "should fail if no backup can be performed" do dir = tmpdir("backups") - file = described_class.new :path => File.join(dir, "testfile"), :backup => ".bak", :content => "foo" + file = described_class.new({:path => File.join(dir, "testfile"), :backup => ".bak", :content => "foo"}.merge(resource_options)) catalog.add_resource file File.open(file[:path], 'w') { |f| f.puts "bar" } # Create a directory where the backup should be so that writing to it fails Dir.mkdir(File.join(dir, "testfile.bak")) Puppet::Util::Log.stubs(:newmessage) catalog.apply expect(File.read(file[:path])).to eq("bar\n") end it "should not backup symlinks", :if => described_class.defaultprovider.feature?(:manages_symlinks) do link = tmpfile("link") dest1 = tmpfile("dest1") dest2 = tmpfile("dest2") bucket = Puppet::Type.type(:filebucket).new :path => tmpfile("filebucket"), :name => "mybucket" - file = described_class.new :path => link, :target => dest2, :ensure => :link, :backup => "mybucket" + file = described_class.new({:path => link, :target => dest2, :ensure => :link, :backup => "mybucket"}.merge(resource_options)) catalog.add_resource file catalog.add_resource bucket File.open(dest1, "w") { |f| f.puts "whatever" } Puppet::FileSystem.symlink(dest1, link) - d = digest(File.read(file[:path])) + d = filebucket_digest.call(File.read(file[:path])) catalog.apply expect(Puppet::FileSystem.readlink(link)).to eq(dest2) expect(Puppet::FileSystem.exist?(bucket[:path])).to be_falsey end it "should backup directories to the local filesystem by copying the whole directory" do - file = described_class.new :path => path, :backup => ".bak", :content => "foo", :force => true + file = described_class.new({:path => path, :backup => ".bak", :content => "foo", :force => true}.merge(resource_options)) catalog.add_resource file Dir.mkdir(path) otherfile = File.join(path, "foo") File.open(otherfile, "w") { |f| f.print "yay" } catalog.apply backup = "#{path}.bak" expect(FileTest).to be_directory(backup) expect(File.read(File.join(backup, "foo"))).to eq("yay") end it "should backup directories to filebuckets by backing up each file separately" do bucket = Puppet::Type.type(:filebucket).new :path => tmpfile("filebucket"), :name => "mybucket" - file = described_class.new :path => tmpfile("bucket_backs"), :backup => "mybucket", :content => "foo", :force => true + file = described_class.new({:path => tmpfile("bucket_backs"), :backup => "mybucket", :content => "foo", :force => true}.merge(resource_options)) catalog.add_resource file catalog.add_resource bucket Dir.mkdir(file[:path]) foofile = File.join(file[:path], "foo") barfile = File.join(file[:path], "bar") File.open(foofile, "w") { |f| f.print "fooyay" } File.open(barfile, "w") { |f| f.print "baryay" } - food = digest(File.read(foofile)) - bard = digest(File.read(barfile)) + food = filebucket_digest.call(File.read(foofile)) + bard = filebucket_digest.call(File.read(barfile)) catalog.apply expect(bucket.bucket.getfile(food)).to eq("fooyay") expect(bucket.bucket.getfile(bard)).to eq("baryay") end end + + with_digest_algorithms do + it_should_behave_like "files are backed up", {} do + let(:filebucket_digest) { method(:digest) } + end + end + + CHECKSUM_TYPES_TO_TRY.each do |checksum_type, checksum| + describe "when checksum_type is #{checksum_type}" do + # FileBucket uses the globally configured default for lookup by digest, which right now is MD5. + it_should_behave_like "files are backed up", {:checksum => checksum_type} do + let(:filebucket_digest) { Proc.new {|x| Puppet::Util::Checksums.md5(x)} } + end + end + end end describe "when recursing" do def build_path(dir) Dir.mkdir(dir) File.chmod(0750, dir) @dirs = [dir] @files = [] %w{one two}.each do |subdir| fdir = File.join(dir, subdir) Dir.mkdir(fdir) File.chmod(0750, fdir) @dirs << fdir %w{three}.each do |file| ffile = File.join(fdir, file) @files << ffile File.open(ffile, "w") { |f| f.puts "test #{file}" } File.chmod(0640, ffile) end end end it "should be able to recurse over a nonexistent file" do @file = described_class.new( :name => path, :mode => '0644', :recurse => true, :backup => false ) catalog.add_resource @file expect { @file.eval_generate }.not_to raise_error end it "should be able to recursively set properties on existing files" do path = tmpfile("file_integration_tests") build_path(path) file = described_class.new( :name => path, :mode => '0644', :recurse => true, :backup => false ) catalog.add_resource file catalog.apply expect(@dirs).not_to be_empty @dirs.each do |path| expect(get_mode(path) & 007777).to eq(0755) end expect(@files).not_to be_empty @files.each do |path| expect(get_mode(path) & 007777).to eq(0644) end end it "should be able to recursively make links to other files", :if => described_class.defaultprovider.feature?(:manages_symlinks) do source = tmpfile("file_link_integration_source") build_path(source) dest = tmpfile("file_link_integration_dest") @file = described_class.new(:name => dest, :target => source, :recurse => true, :ensure => :link, :backup => false) catalog.add_resource @file catalog.apply @dirs.each do |path| link_path = path.sub(source, dest) expect(Puppet::FileSystem.lstat(link_path)).to be_directory end @files.each do |path| link_path = path.sub(source, dest) expect(Puppet::FileSystem.lstat(link_path).ftype).to eq("link") end end it "should be able to recursively copy files" do source = tmpfile("file_source_integration_source") build_path(source) dest = tmpfile("file_source_integration_dest") @file = described_class.new(:name => dest, :source => source, :recurse => true, :backup => false) catalog.add_resource @file catalog.apply @dirs.each do |path| newpath = path.sub(source, dest) expect(Puppet::FileSystem.lstat(newpath)).to be_directory end @files.each do |path| newpath = path.sub(source, dest) expect(Puppet::FileSystem.lstat(newpath).ftype).to eq("file") end end it "should not recursively manage files managed by a more specific explicit file" do dir = tmpfile("recursion_vs_explicit_1") subdir = File.join(dir, "subdir") file = File.join(subdir, "file") FileUtils.mkdir_p(subdir) File.open(file, "w") { |f| f.puts "" } base = described_class.new(:name => dir, :recurse => true, :backup => false, :mode => "755") sub = described_class.new(:name => subdir, :recurse => true, :backup => false, :mode => "644") catalog.add_resource base catalog.add_resource sub catalog.apply expect(get_mode(file) & 007777).to eq(0644) end it "should recursively manage files even if there is an explicit file whose name is a prefix of the managed file" do managed = File.join(path, "file") generated = File.join(path, "file_with_a_name_starting_with_the_word_file") FileUtils.mkdir_p(path) FileUtils.touch(managed) FileUtils.touch(generated) catalog.add_resource described_class.new(:name => path, :recurse => true, :backup => false, :mode => '0700') catalog.add_resource described_class.new(:name => managed, :recurse => true, :backup => false, :mode => "644") catalog.apply expect(get_mode(generated) & 007777).to eq(0700) end describe "when recursing remote directories" do describe "when sourceselect first" do describe "for a directory" do it "should recursively copy the first directory that exists" do one = File.expand_path('thisdoesnotexist') two = tmpdir('two') FileUtils.mkdir_p(File.join(two, 'three')) FileUtils.touch(File.join(two, 'three', 'four')) catalog.add_resource Puppet::Type.newfile( :path => path, :ensure => :directory, :backup => false, :recurse => true, :sourceselect => :first, :source => [one, two] ) catalog.apply expect(File).to be_directory(path) expect(Puppet::FileSystem.exist?(File.join(path, 'one'))).to be_falsey expect(Puppet::FileSystem.exist?(File.join(path, 'three', 'four'))).to be_truthy end it "should recursively copy an empty directory" do one = File.expand_path('thisdoesnotexist') two = tmpdir('two') three = tmpdir('three') file_in_dir_with_contents(three, 'a', '') catalog.add_resource Puppet::Type.newfile( :path => path, :ensure => :directory, :backup => false, :recurse => true, :sourceselect => :first, :source => [one, two, three] ) catalog.apply expect(File).to be_directory(path) expect(Puppet::FileSystem.exist?(File.join(path, 'a'))).to be_falsey end it "should only recurse one level" do one = tmpdir('one') FileUtils.mkdir_p(File.join(one, 'a', 'b')) FileUtils.touch(File.join(one, 'a', 'b', 'c')) two = tmpdir('two') FileUtils.mkdir_p(File.join(two, 'z')) FileUtils.touch(File.join(two, 'z', 'y')) catalog.add_resource Puppet::Type.newfile( :path => path, :ensure => :directory, :backup => false, :recurse => true, :recurselimit => 1, :sourceselect => :first, :source => [one, two] ) catalog.apply expect(Puppet::FileSystem.exist?(File.join(path, 'a'))).to be_truthy expect(Puppet::FileSystem.exist?(File.join(path, 'a', 'b'))).to be_falsey expect(Puppet::FileSystem.exist?(File.join(path, 'z'))).to be_falsey end end describe "for a file" do it "should copy the first file that exists" do one = File.expand_path('thisdoesnotexist') two = tmpfile_with_contents('two', 'yay') three = tmpfile_with_contents('three', 'no') catalog.add_resource Puppet::Type.newfile( :path => path, :ensure => :file, :backup => false, :sourceselect => :first, :source => [one, two, three] ) catalog.apply expect(File.read(path)).to eq('yay') end it "should copy an empty file" do one = File.expand_path('thisdoesnotexist') two = tmpfile_with_contents('two', '') three = tmpfile_with_contents('three', 'no') catalog.add_resource Puppet::Type.newfile( :path => path, :ensure => :file, :backup => false, :sourceselect => :first, :source => [one, two, three] ) catalog.apply expect(File.read(path)).to eq('') end end end describe "when sourceselect all" do describe "for a directory" do it "should recursively copy all sources from the first valid source" do dest = tmpdir('dest') one = tmpdir('one') two = tmpdir('two') three = tmpdir('three') four = tmpdir('four') file_in_dir_with_contents(one, 'a', one) file_in_dir_with_contents(two, 'a', two) file_in_dir_with_contents(two, 'b', two) file_in_dir_with_contents(three, 'a', three) file_in_dir_with_contents(three, 'c', three) obj = Puppet::Type.newfile( :path => dest, :ensure => :directory, :backup => false, :recurse => true, :sourceselect => :all, :source => [one, two, three, four] ) catalog.add_resource obj catalog.apply expect(File.read(File.join(dest, 'a'))).to eq(one) expect(File.read(File.join(dest, 'b'))).to eq(two) expect(File.read(File.join(dest, 'c'))).to eq(three) end it "should only recurse one level from each valid source" do one = tmpdir('one') FileUtils.mkdir_p(File.join(one, 'a', 'b')) FileUtils.touch(File.join(one, 'a', 'b', 'c')) two = tmpdir('two') FileUtils.mkdir_p(File.join(two, 'z')) FileUtils.touch(File.join(two, 'z', 'y')) obj = Puppet::Type.newfile( :path => path, :ensure => :directory, :backup => false, :recurse => true, :recurselimit => 1, :sourceselect => :all, :source => [one, two] ) catalog.add_resource obj catalog.apply expect(Puppet::FileSystem.exist?(File.join(path, 'a'))).to be_truthy expect(Puppet::FileSystem.exist?(File.join(path, 'a', 'b'))).to be_falsey expect(Puppet::FileSystem.exist?(File.join(path, 'z'))).to be_truthy expect(Puppet::FileSystem.exist?(File.join(path, 'z', 'y'))).to be_falsey end end end end end describe "when generating resources" do before do source = tmpdir("generating_in_catalog_source") s1 = file_in_dir_with_contents(source, "one", "uno") s2 = file_in_dir_with_contents(source, "two", "dos") @file = described_class.new( :name => path, :source => source, :recurse => true, :backup => false ) catalog.add_resource @file end it "should add each generated resource to the catalog" do catalog.apply do |trans| expect(catalog.resource(:file, File.join(path, "one"))).to be_a(described_class) expect(catalog.resource(:file, File.join(path, "two"))).to be_a(described_class) end end it "should have an edge to each resource in the relationship graph" do catalog.apply do |trans| one = catalog.resource(:file, File.join(path, "one")) expect(catalog.relationship_graph).to be_edge(@file, one) two = catalog.resource(:file, File.join(path, "two")) expect(catalog.relationship_graph).to be_edge(@file, two) end end end describe "when copying files" do it "should be able to copy files with pound signs in their names (#285)" do source = tmpfile_with_contents("filewith#signs", "foo") dest = tmpfile("destwith#signs") catalog.add_resource described_class.new(:name => dest, :source => source) catalog.apply expect(File.read(dest)).to eq("foo") end it "should be able to copy files with spaces in their names" do dest = tmpfile("destwith spaces") source = tmpfile_with_contents("filewith spaces", "foo") catalog.add_resource described_class.new(:path => dest, :source => source) catalog.apply expect(File.read(dest)).to eq("foo") end it "should be able to copy individual files even if recurse has been specified" do source = tmpfile_with_contents("source", "foo") dest = tmpfile("dest") catalog.add_resource described_class.new(:name => dest, :source => source, :recurse => true) catalog.apply expect(File.read(dest)).to eq("foo") end end it "should create a file with content if ensure is omitted" do catalog.add_resource described_class.new( :path => path, :content => "this is some content, yo" ) catalog.apply expect(File.read(path)).to eq("this is some content, yo") end it "should create files with content if both content and ensure are set" do file = described_class.new( :path => path, :ensure => "file", :content => "this is some content, yo" ) catalog.add_resource file catalog.apply expect(File.read(path)).to eq("this is some content, yo") end it "should delete files with sources but that are set for deletion" do source = tmpfile_with_contents("source_source_with_ensure", "yay") dest = tmpfile_with_contents("source_source_with_ensure", "boo") file = described_class.new( :path => dest, :ensure => :absent, :source => source, :backup => false ) catalog.add_resource file catalog.apply expect(Puppet::FileSystem.exist?(dest)).to be_falsey end describe "when sourcing" do - let(:source) { tmpfile_with_contents("source_default_values", "yay") } + with_checksum_types "source", "default_values" do + describe "on POSIX systems", :if => Puppet.features.posix? do + it "should apply the source metadata values" do + set_mode(0770, checksum_file) + + file = described_class.new( + :path => path, + :ensure => :file, + :source => checksum_file, + :source_permissions => :use, + :checksum => checksum_type, + :backup => false + ) + + catalog.add_resource file + catalog.apply - describe "on POSIX systems", :if => Puppet.features.posix? do - it "should apply the source metadata values" do - set_mode(0770, source) + expect(get_owner(path)).to eq(get_owner(checksum_file)) + expect(get_group(path)).to eq(get_group(checksum_file)) + expect(get_mode(path) & 07777).to eq(0770) + end + end + + it "should override the default metadata values" do + set_mode(0770, checksum_file) file = described_class.new( :path => path, :ensure => :file, - :source => source, - :source_permissions => :use, - :backup => false + :source => checksum_file, + :checksum => checksum_type, + :backup => false, + :mode => '0440' ) catalog.add_resource file catalog.apply - expect(get_owner(path)).to eq(get_owner(source)) - expect(get_group(path)).to eq(get_group(source)) - expect(get_mode(path) & 07777).to eq(0770) + expect(get_mode(path) & 07777).to eq(0440) end end - it "should override the default metadata values" do - set_mode(0770, source) - - file = described_class.new( - :path => path, - :ensure => :file, - :source => source, - :backup => false, - :mode => '0440' - ) - - catalog.add_resource file - catalog.apply - - expect(get_mode(path) & 07777).to eq(0440) - end - + let(:source) { tmpfile_with_contents("source_default_values", "yay") } describe "on Windows systems", :if => Puppet.features.microsoft_windows? do def expects_sid_granted_full_access_explicitly(path, sid) inherited_ace = Puppet::Util::Windows::AccessControlEntry::INHERITED_ACE aces = get_aces_for_path_by_sid(path, sid) expect(aces).not_to be_empty aces.each do |ace| expect(ace.mask).to eq(Puppet::Util::Windows::File::FILE_ALL_ACCESS) expect(ace.flags & inherited_ace).not_to eq(inherited_ace) end end def expects_system_granted_full_access_explicitly(path) expects_sid_granted_full_access_explicitly(path, @sids[:system]) end def expects_at_least_one_inherited_ace_grants_full_access(path, sid) inherited_ace = Puppet::Util::Windows::AccessControlEntry::INHERITED_ACE aces = get_aces_for_path_by_sid(path, sid) expect(aces).not_to be_empty expect(aces.any? do |ace| ace.mask == Puppet::Util::Windows::File::FILE_ALL_ACCESS && (ace.flags & inherited_ace) == inherited_ace end).to be_truthy end def expects_at_least_one_inherited_system_ace_grants_full_access(path) expects_at_least_one_inherited_ace_grants_full_access(path, @sids[:system]) end describe "when processing SYSTEM ACEs" do before do @sids = { :current_user => Puppet::Util::Windows::SID.name_to_sid(Puppet::Util::Windows::ADSI::User.current_user_name), :system => Win32::Security::SID::LocalSystem, :admin => Puppet::Util::Windows::SID.name_to_sid("Administrator"), :guest => Puppet::Util::Windows::SID.name_to_sid("Guest"), :users => Win32::Security::SID::BuiltinUsers, :power_users => Win32::Security::SID::PowerUsers, :none => Win32::Security::SID::Nobody } end describe "on files" do before :each do @file = described_class.new( :path => path, :ensure => :file, :source => source, :backup => false ) catalog.add_resource @file end describe "when permissions are not insync?" do before :each do @file[:owner] = 'None' @file[:group] = 'None' end it "preserves the inherited SYSTEM ACE for an existing file" do FileUtils.touch(path) expects_at_least_one_inherited_system_ace_grants_full_access(path) catalog.apply expects_at_least_one_inherited_system_ace_grants_full_access(path) end it "applies the inherited SYSTEM ACEs for a new file" do catalog.apply expects_at_least_one_inherited_system_ace_grants_full_access(path) end end describe "created with SYSTEM as the group" do before :each do @file[:owner] = @sids[:users] @file[:group] = @sids[:system] @file[:mode] = '0644' catalog.apply end it "should allow the user to explicitly set the mode to 4" do system_aces = get_aces_for_path_by_sid(path, @sids[:system]) expect(system_aces).not_to be_empty system_aces.each do |ace| expect(ace.mask).to eq(Puppet::Util::Windows::File::FILE_GENERIC_READ) end end it "prepends SYSTEM ace when changing group from system to power users" do @file[:group] = @sids[:power_users] catalog.apply system_aces = get_aces_for_path_by_sid(path, @sids[:system]) expect(system_aces.size).to eq(1) end end describe "with :links set to :follow" do it "should not fail to apply" do # at minimal, we need an owner and/or group @file[:owner] = @sids[:users] @file[:links] = :follow catalog.apply do |transaction| if transaction.any_failed? pretty_transaction_error(transaction) end end end end end describe "on directories" do before :each do @directory = described_class.new( :path => dir, :ensure => :directory ) catalog.add_resource @directory end def grant_everyone_full_access(path) sd = Puppet::Util::Windows::Security.get_security_descriptor(path) sd.dacl.allow( 'S-1-1-0', #everyone Puppet::Util::Windows::File::FILE_ALL_ACCESS, Puppet::Util::Windows::AccessControlEntry::OBJECT_INHERIT_ACE | Puppet::Util::Windows::AccessControlEntry::CONTAINER_INHERIT_ACE) Puppet::Util::Windows::Security.set_security_descriptor(path, sd) end after :each do grant_everyone_full_access(dir) end describe "when permissions are not insync?" do before :each do @directory[:owner] = 'None' @directory[:group] = 'None' end it "preserves the inherited SYSTEM ACEs for an existing directory" do FileUtils.mkdir(dir) expects_at_least_one_inherited_system_ace_grants_full_access(dir) catalog.apply expects_at_least_one_inherited_system_ace_grants_full_access(dir) end it "applies the inherited SYSTEM ACEs for a new directory" do catalog.apply expects_at_least_one_inherited_system_ace_grants_full_access(dir) end describe "created with SYSTEM as the group" do before :each do @directory[:owner] = @sids[:users] @directory[:group] = @sids[:system] @directory[:mode] = '0644' catalog.apply end it "should allow the user to explicitly set the mode to 4" do system_aces = get_aces_for_path_by_sid(dir, @sids[:system]) expect(system_aces).not_to be_empty system_aces.each do |ace| # unlike files, Puppet sets execute bit on directories that are readable expect(ace.mask).to eq(Puppet::Util::Windows::File::FILE_GENERIC_READ | Puppet::Util::Windows::File::FILE_GENERIC_EXECUTE) end end it "prepends SYSTEM ace when changing group from system to power users" do @directory[:group] = @sids[:power_users] catalog.apply system_aces = get_aces_for_path_by_sid(dir, @sids[:system]) expect(system_aces.size).to eq(1) end end describe "with :links set to :follow" do it "should not fail to apply" do # at minimal, we need an owner and/or group @directory[:owner] = @sids[:users] @directory[:links] = :follow catalog.apply do |transaction| if transaction.any_failed? pretty_transaction_error(transaction) end end end end end end end end end describe "when purging files" do before do sourcedir = tmpdir("purge_source") destdir = tmpdir("purge_dest") sourcefile = File.join(sourcedir, "sourcefile") @copiedfile = File.join(destdir, "sourcefile") @localfile = File.join(destdir, "localfile") @purgee = File.join(destdir, "to_be_purged") File.open(@localfile, "w") { |f| f.print "oldtest" } File.open(sourcefile, "w") { |f| f.print "funtest" } # this file should get removed File.open(@purgee, "w") { |f| f.print "footest" } lfobj = Puppet::Type.newfile( :title => "localfile", :path => @localfile, :content => "rahtest", :ensure => :file, :backup => false ) destobj = Puppet::Type.newfile( :title => "destdir", :path => destdir, :source => sourcedir, :backup => false, :purge => true, :recurse => true ) catalog.add_resource lfobj, destobj catalog.apply end it "should still copy remote files" do expect(File.read(@copiedfile)).to eq('funtest') end it "should not purge managed, local files" do expect(File.read(@localfile)).to eq('rahtest') end it "should purge files that are neither remote nor otherwise managed" do expect(Puppet::FileSystem.exist?(@purgee)).to be_falsey end end describe "when using validate_cmd" do it "should fail the file resource if command fails" do catalog.add_resource(described_class.new(:path => path, :content => "foo", :validate_cmd => "/usr/bin/env false")) Puppet::Util::Execution.expects(:execute).with("/usr/bin/env false", {:combine => true, :failonfail => true}).raises(Puppet::ExecutionFailure, "Failed") report = catalog.apply.report expect(report.resource_statuses["File[#{path}]"]).to be_failed expect(Puppet::FileSystem.exist?(path)).to be_falsey end it "should succeed the file resource if command succeeds" do catalog.add_resource(described_class.new(:path => path, :content => "foo", :validate_cmd => "/usr/bin/env true")) Puppet::Util::Execution.expects(:execute).with("/usr/bin/env true", {:combine => true, :failonfail => true}).returns '' report = catalog.apply.report expect(report.resource_statuses["File[#{path}]"]).not_to be_failed expect(Puppet::FileSystem.exist?(path)).to be_truthy end end def tmpfile_with_contents(name, contents) file = tmpfile(name) File.open(file, "w") { |f| f.write contents } file end def file_in_dir_with_contents(dir, name, contents) full_name = File.join(dir, name) File.open(full_name, "w") { |f| f.write contents } full_name end def pretty_transaction_error(transaction) report = transaction.report status_failures = report.resource_statuses.values.select { |r| r.failed? } status_fail_msg = status_failures. collect(&:events). flatten. select { |event| event.status == 'failure' }. collect { |event| "#{event.resource}: #{event.message}" }.join("; ") raise "Got #{status_failures.length} failure(s) while applying: #{status_fail_msg}" end end diff --git a/spec/shared_contexts/checksum.rb b/spec/shared_contexts/checksum.rb index 3310762dd..4d5e0f405 100644 --- a/spec/shared_contexts/checksum.rb +++ b/spec/shared_contexts/checksum.rb @@ -1,48 +1,53 @@ # Shared contexts for testing against all supported checksum types. # # These helpers define nested rspec example groups to test code against all our # supported checksum types. Example groups that need to be run against all # types should use the `with_checksum_types` helper which will # create a new example group for each types and will run the given block # in each example group. CHECKSUM_STAT_TIME = Time.now CHECKSUM_TYPES_TO_TRY = [ ['md5', 'a7a169ac84bb863b30484d0aa03139c1'], ['md5lite', '22b4182363e81b326e98231fde616782'], ['sha256', '47fcae62967db2fb5cba2fc0d9cf3e6767035d763d825ecda535a7b1928b9746'], ['sha256lite', 'fd50217a2b0286ba25121bf2297bbe6c197933992de67e4e568f19861444ecf8'], +] +TIME_TYPES_TO_TRY = [ ['ctime', "#{CHECKSUM_STAT_TIME}"], ['mtime', "#{CHECKSUM_STAT_TIME}"] ] shared_context('with supported checksum types') do - def self.with_checksum_types(path, file, &block) def expect_correct_checksum(meta, checksum_type, checksum, type) expect(meta).to_not be_nil expect(meta).to be_instance_of(type) expect(meta.checksum_type).to eq(checksum_type) expect(meta.checksum).to eq("{#{checksum_type}}#{checksum}") end - CHECKSUM_TYPES_TO_TRY.each do |checksum_type, checksum| + (CHECKSUM_TYPES_TO_TRY + TIME_TYPES_TO_TRY).each do |checksum_type, checksum| describe("when checksum_type is #{checksum_type}") do let(:checksum_type) { checksum_type } let(:plaintext) { "1\r\n"*4000 } let(:checksum) { checksum } let(:env_path) { tmpfile(path) } let(:checksum_file) { File.join(env_path, file) } - before do + def digest(content) + Puppet::Util::Checksums.send(checksum_type, content) + end + + before(:each) do FileUtils.mkdir_p(File.dirname(checksum_file)) File.open(checksum_file, "wb") { |f| f.write plaintext } - Puppet::FileSystem.stubs(:stat).returns(stub('stat', :ctime => CHECKSUM_STAT_TIME, :mtime => CHECKSUM_STAT_TIME)) + File::Stat.any_instance.stubs(:ctime).returns(CHECKSUM_STAT_TIME) + File::Stat.any_instance.stubs(:mtime).returns(CHECKSUM_STAT_TIME) end instance_eval(&block) end end end end - diff --git a/spec/unit/type/file/source_spec.rb b/spec/unit/type/file/source_spec.rb index 451b3d9d1..8e60beed7 100755 --- a/spec/unit/type/file/source_spec.rb +++ b/spec/unit/type/file/source_spec.rb @@ -1,560 +1,566 @@ #! /usr/bin/env ruby require 'spec_helper' require 'uri' source = Puppet::Type.type(:file).attrclass(:source) describe Puppet::Type.type(:file).attrclass(:source) do include PuppetSpec::Files around :each do |example| Puppet.override(:environments => Puppet::Environments::Static.new) do example.run end end before do # Wow that's a messy interface to the resource. @environment = Puppet::Node::Environment.remote("myenv") @resource = stub 'resource', :[]= => nil, :property => nil, :catalog => Puppet::Resource::Catalog.new(nil, @environment), :line => 0, :file => '' @foobar = make_absolute("/foo/bar baz") @feebooz = make_absolute("/fee/booz baz") @foobar_uri = URI.unescape(Puppet::Util.path_to_uri(@foobar).to_s) @feebooz_uri = URI.unescape(Puppet::Util.path_to_uri(@feebooz).to_s) end it "should be a subclass of Parameter" do expect(source.superclass).to eq(Puppet::Parameter) end describe "#validate" do let(:path) { tmpfile('file_source_validate') } let(:resource) { Puppet::Type.type(:file).new(:path => path) } it "should fail if the set values are not URLs" do URI.expects(:parse).with('foo').raises RuntimeError expect(lambda { resource[:source] = %w{foo} }).to raise_error(Puppet::Error) end it "should fail if the URI is not a local file, file URI, or puppet URI" do expect(lambda { resource[:source] = %w{http://foo/bar} }).to raise_error(Puppet::Error, /Cannot use URLs of type 'http' as source for fileserving/) end it "should strip trailing forward slashes", :unless => Puppet.features.microsoft_windows? do resource[:source] = "/foo/bar\\//" expect(resource[:source]).to eq(%w{file:/foo/bar\\}) end it "should strip trailing forward and backslashes", :if => Puppet.features.microsoft_windows? do resource[:source] = "X:/foo/bar\\//" expect(resource[:source]).to eq(%w{file:/X:/foo/bar}) end it "should accept an array of sources" do resource[:source] = %w{file:///foo/bar puppet://host:8140/foo/bar} expect(resource[:source]).to eq(%w{file:///foo/bar puppet://host:8140/foo/bar}) end it "should accept file path characters that are not valid in URI" do resource[:source] = 'file:///foo bar' end it "should reject relative URI sources" do expect(lambda { resource[:source] = 'foo/bar' }).to raise_error(Puppet::Error) end it "should reject opaque sources" do expect(lambda { resource[:source] = 'mailto:foo@com' }).to raise_error(Puppet::Error) end it "should accept URI authority component" do resource[:source] = 'file://host/foo' expect(resource[:source]).to eq(%w{file://host/foo}) end it "should accept when URI authority is absent" do resource[:source] = 'file:///foo/bar' expect(resource[:source]).to eq(%w{file:///foo/bar}) end end describe "#munge" do let(:path) { tmpfile('file_source_munge') } let(:resource) { Puppet::Type.type(:file).new(:path => path) } it "should prefix file scheme to absolute paths" do resource[:source] = path expect(resource[:source]).to eq([URI.unescape(Puppet::Util.path_to_uri(path).to_s)]) end %w[file puppet].each do |scheme| it "should not prefix valid #{scheme} URIs" do resource[:source] = "#{scheme}:///foo bar" expect(resource[:source]).to eq(["#{scheme}:///foo bar"]) end end end describe "when returning the metadata" do before do @metadata = stub 'metadata', :source= => nil @resource.stubs(:[]).with(:links).returns :manage @resource.stubs(:[]).with(:source_permissions).returns :use + @resource.stubs(:[]).with(:checksum).returns :checksum end it "should return already-available metadata" do @source = source.new(:resource => @resource) @source.metadata = "foo" expect(@source.metadata).to eq("foo") end it "should return nil if no @should value is set and no metadata is available" do @source = source.new(:resource => @resource) expect(@source.metadata).to be_nil end it "should collect its metadata using the Metadata class if it is not already set" do @source = source.new(:resource => @resource, :value => @foobar) Puppet::FileServing::Metadata.indirection.expects(:find).with do |uri, options| expect(uri).to eq @foobar_uri expect(options[:environment]).to eq @environment expect(options[:links]).to eq :manage + expect(options[:checksum_type]).to eq :checksum end.returns @metadata @source.metadata end it "should use the metadata from the first found source" do metadata = stub 'metadata', :source= => nil @source = source.new(:resource => @resource, :value => [@foobar, @feebooz]) options = { :environment => @environment, :links => :manage, - :source_permissions => :use + :source_permissions => :use, + :checksum_type => :checksum } Puppet::FileServing::Metadata.indirection.expects(:find).with(@foobar_uri, options).returns nil Puppet::FileServing::Metadata.indirection.expects(:find).with(@feebooz_uri, options).returns metadata expect(@source.metadata).to equal(metadata) end it "should store the found source as the metadata's source" do metadata = mock 'metadata' @source = source.new(:resource => @resource, :value => @foobar) Puppet::FileServing::Metadata.indirection.expects(:find).with do |uri, options| expect(uri).to eq @foobar_uri expect(options[:environment]).to eq @environment expect(options[:links]).to eq :manage + expect(options[:checksum_type]).to eq :checksum end.returns metadata metadata.expects(:source=).with(@foobar_uri) @source.metadata end it "should fail intelligently if an exception is encountered while querying for metadata" do @source = source.new(:resource => @resource, :value => @foobar) Puppet::FileServing::Metadata.indirection.expects(:find).with do |uri, options| expect(uri).to eq @foobar_uri expect(options[:environment]).to eq @environment expect(options[:links]).to eq :manage + expect(options[:checksum_type]).to eq :checksum end.raises RuntimeError @source.expects(:fail).raises ArgumentError expect { @source.metadata }.to raise_error(ArgumentError) end it "should fail if no specified sources can be found" do @source = source.new(:resource => @resource, :value => @foobar) Puppet::FileServing::Metadata.indirection.expects(:find).with do |uri, options| expect(uri).to eq @foobar_uri expect(options[:environment]).to eq @environment expect(options[:links]).to eq :manage + expect(options[:checksum_type]).to eq :checksum end.returns nil @source.expects(:fail).raises RuntimeError expect { @source.metadata }.to raise_error(RuntimeError) end end it "should have a method for setting the desired values on the resource" do expect(source.new(:resource => @resource)).to respond_to(:copy_source_values) end describe "when copying the source values" do before :each do @resource = Puppet::Type.type(:file).new :path => @foobar @source = source.new(:resource => @resource) @metadata = stub 'metadata', :owner => 100, :group => 200, :mode => "173", :checksum => "{md5}asdfasdf", :ftype => "file", :source => @foobar @source.stubs(:metadata).returns @metadata Puppet.features.stubs(:root?).returns true end it "should not issue an error - except on Windows - if the source mode value is a Numeric" do @metadata.stubs(:mode).returns 0173 @resource[:source_permissions] = :use if Puppet::Util::Platform.windows? expect { @source.copy_source_values }.to raise_error("Copying owner/mode/group from the source file on Windows is not supported; use source_permissions => ignore.") else expect { @source.copy_source_values }.not_to raise_error end end it "should not issue an error - except on Windows - if the source mode value is a String" do @metadata.stubs(:mode).returns "173" @resource[:source_permissions] = :use if Puppet::Util::Platform.windows? expect { @source.copy_source_values }.to raise_error("Copying owner/mode/group from the source file on Windows is not supported; use source_permissions => ignore.") else expect { @source.copy_source_values }.not_to raise_error end end it "should fail if there is no metadata" do @source.stubs(:metadata).returns nil @source.expects(:devfail).raises ArgumentError expect { @source.copy_source_values }.to raise_error(ArgumentError) end it "should set :ensure to the file type" do @metadata.stubs(:ftype).returns "file" @source.copy_source_values expect(@resource[:ensure]).to eq(:file) end it "should not set 'ensure' if it is already set to 'absent'" do @metadata.stubs(:ftype).returns "file" @resource[:ensure] = :absent @source.copy_source_values expect(@resource[:ensure]).to eq(:absent) end describe "and the source is a file" do before do @metadata.stubs(:ftype).returns "file" Puppet.features.stubs(:microsoft_windows?).returns false end context "when source_permissions is `use`" do before :each do @resource[:source_permissions] = "use" end it "should copy the metadata's owner, group, checksum, and mode to the resource if they are not set on the resource" do @source.copy_source_values expect(@resource[:owner]).to eq(100) expect(@resource[:group]).to eq(200) expect(@resource[:mode]).to eq("173") # Metadata calls it checksum, we call it content. expect(@resource[:content]).to eq(@metadata.checksum) end it "should not copy the metadata's owner, group, checksum and mode to the resource if they are already set" do @resource[:owner] = 1 @resource[:group] = 2 @resource[:mode] = '173' @resource[:content] = "foobar" @source.copy_source_values expect(@resource[:owner]).to eq(1) expect(@resource[:group]).to eq(2) expect(@resource[:mode]).to eq('173') expect(@resource[:content]).not_to eq(@metadata.checksum) end describe "and puppet is not running as root" do before do Puppet.features.stubs(:root?).returns false end it "should not try to set the owner" do @source.copy_source_values expect(@resource[:owner]).to be_nil end it "should not try to set the group" do @source.copy_source_values expect(@resource[:group]).to be_nil end end end context "when source_permissions is `use_when_creating`" do before :each do @resource[:source_permissions] = "use_when_creating" Puppet.features.expects(:root?).returns true @source.stubs(:local?).returns(false) end context "when managing a new file" do it "should copy owner and group from local sources" do @source.stubs(:local?).returns true @source.copy_source_values expect(@resource[:owner]).to eq(100) expect(@resource[:group]).to eq(200) expect(@resource[:mode]).to eq("173") end it "copies the remote owner" do @source.copy_source_values expect(@resource[:owner]).to eq(100) end it "copies the remote group" do @source.copy_source_values expect(@resource[:group]).to eq(200) end it "copies the remote mode" do @source.copy_source_values expect(@resource[:mode]).to eq("173") end end context "when managing an existing file" do before :each do Puppet::FileSystem.stubs(:exist?).with(@resource[:path]).returns(true) end it "should not copy owner, group or mode from local sources" do @source.stubs(:local?).returns true @source.copy_source_values expect(@resource[:owner]).to be_nil expect(@resource[:group]).to be_nil expect(@resource[:mode]).to be_nil end it "preserves the local owner" do @source.copy_source_values expect(@resource[:owner]).to be_nil end it "preserves the local group" do @source.copy_source_values expect(@resource[:group]).to be_nil end it "preserves the local mode" do @source.copy_source_values expect(@resource[:mode]).to be_nil end end end context "when source_permissions is default" do before :each do @source.stubs(:local?).returns(false) Puppet.features.expects(:root?).returns true end it "should not copy owner, group or mode from local sources" do @source.stubs(:local?).returns true @source.copy_source_values expect(@resource[:owner]).to be_nil expect(@resource[:group]).to be_nil expect(@resource[:mode]).to be_nil end it "preserves the local owner" do @source.copy_source_values expect(@resource[:owner]).to be_nil end it "preserves the local group" do @source.copy_source_values expect(@resource[:group]).to be_nil end it "preserves the local mode" do @source.copy_source_values expect(@resource[:mode]).to be_nil end end describe "on Windows when source_permissions is `use`" do before :each do Puppet.features.stubs(:microsoft_windows?).returns true @resource[:source_permissions] = "use" end let(:err_message) { "Copying owner/mode/group from the" << " source file on Windows is not supported;" << " use source_permissions => ignore." } it "should issue error when copying from remote sources" do @source.stubs(:local?).returns false expect { @source.copy_source_values }.to raise_error(err_message) end it "should issue error when copying from local sources" do @source.stubs(:local?).returns true expect { @source.copy_source_values }.to raise_error(err_message) end it "should issue error when copying metadata from remote sources if only user is unspecified" do @source.stubs(:local?).returns false @resource[:group] = 2 @resource[:mode] = "0003" expect { @source.copy_source_values }.to raise_error(err_message) end it "should issue error when copying metadata from remote sources if only group is unspecified" do @source.stubs(:local?).returns false @resource[:owner] = 1 @resource[:mode] = "0003" expect { @source.copy_source_values }.to raise_error(err_message) end it "should issue error when copying metadata from remote sources if only mode is unspecified" do @source.stubs(:local?).returns false @resource[:owner] = 1 @resource[:group] = 2 expect { @source.copy_source_values }.to raise_error(err_message) end it "should not issue error when copying metadata from remote sources if group, owner, and mode are all specified" do @source.stubs(:local?).returns false @resource[:owner] = 1 @resource[:group] = 2 @resource[:mode] = "0003" expect { @source.copy_source_values }.not_to raise_error end end end describe "and the source is a link" do it "should set the target to the link destination" do @metadata.stubs(:ftype).returns "link" @metadata.stubs(:links).returns "manage" @resource.stubs(:[]) @resource.stubs(:[]=) @metadata.expects(:destination).returns "/path/to/symlink" @resource.expects(:[]=).with(:target, "/path/to/symlink") @source.copy_source_values end end end it "should have a local? method" do expect(source.new(:resource => @resource)).to be_respond_to(:local?) end context "when accessing source properties" do let(:catalog) { Puppet::Resource::Catalog.new } let(:path) { tmpfile('file_resource') } let(:resource) { Puppet::Type.type(:file).new(:path => path, :catalog => catalog) } let(:sourcepath) { tmpfile('file_source') } describe "for local sources" do before :each do FileUtils.touch(sourcepath) end describe "on POSIX systems", :if => Puppet.features.posix? do ['', "file:", "file://"].each do |prefix| it "with prefix '#{prefix}' should be local" do resource[:source] = "#{prefix}#{sourcepath}" expect(resource.parameter(:source)).to be_local end it "should be able to return the metadata source full path" do resource[:source] = "#{prefix}#{sourcepath}" expect(resource.parameter(:source).full_path).to eq(sourcepath) end end end describe "on Windows systems", :if => Puppet.features.microsoft_windows? do ['', "file:/", "file:///"].each do |prefix| it "should be local with prefix '#{prefix}'" do resource[:source] = "#{prefix}#{sourcepath}" expect(resource.parameter(:source)).to be_local end it "should be able to return the metadata source full path" do resource[:source] = "#{prefix}#{sourcepath}" expect(resource.parameter(:source).full_path).to eq(sourcepath) end it "should convert backslashes to forward slashes" do resource[:source] = "#{prefix}#{sourcepath.gsub(/\\/, '/')}" end end it "should be UNC with two slashes" end end describe "for remote sources" do let(:sourcepath) { "/path/to/source" } let(:uri) { URI::Generic.build(:scheme => 'puppet', :host => 'server', :port => 8192, :path => sourcepath).to_s } before(:each) do metadata = Puppet::FileServing::Metadata.new(path, :source => uri, 'type' => 'file') #metadata = stub('remote', :ftype => "file", :source => uri) Puppet::FileServing::Metadata.indirection.stubs(:find). with(uri,all_of(has_key(:environment), has_key(:links))).returns metadata resource[:source] = uri end it "should not be local" do expect(resource.parameter(:source)).not_to be_local end it "should be able to return the metadata source full path" do expect(resource.parameter(:source).full_path).to eq("/path/to/source") end it "should be able to return the source server" do expect(resource.parameter(:source).server).to eq("server") end it "should be able to return the source port" do expect(resource.parameter(:source).port).to eq(8192) end describe "which don't specify server or port" do let(:uri) { "puppet:///path/to/source" } it "should return the default source server" do Puppet[:server] = "myserver" expect(resource.parameter(:source).server).to eq("myserver") end it "should return the default source port" do Puppet[:masterport] = 1234 expect(resource.parameter(:source).port).to eq(1234) end end end end end diff --git a/spec/unit/util/checksums_spec.rb b/spec/unit/util/checksums_spec.rb index ace6e7ab0..59bb1578f 100755 --- a/spec/unit/util/checksums_spec.rb +++ b/spec/unit/util/checksums_spec.rb @@ -1,179 +1,208 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet/util/checksums' describe Puppet::Util::Checksums do include PuppetSpec::Files before do @summer = Puppet::Util::Checksums end content_sums = [:md5, :md5lite, :sha1, :sha1lite, :sha256, :sha256lite] file_only = [:ctime, :mtime, :none] content_sums.each do |sumtype| it "should be able to calculate #{sumtype} sums from strings" do expect(@summer).to be_respond_to(sumtype) end end content_sums.each do |sumtype| it "should know the expected length of #{sumtype} sums" do expect(@summer).to be_respond_to(sumtype.to_s + "_hex_length") end end [content_sums, file_only].flatten.each do |sumtype| it "should be able to calculate #{sumtype} sums from files" do expect(@summer).to be_respond_to(sumtype.to_s + "_file") end end [content_sums, file_only].flatten.each do |sumtype| it "should be able to calculate #{sumtype} sums from stream" do expect(@summer).to be_respond_to(sumtype.to_s + "_stream") end end it "should have a method for determining whether a given string is a checksum" do expect(@summer).to respond_to(:checksum?) end %w{{md5}asdfasdf {sha1}asdfasdf {ctime}asdasdf {mtime}asdfasdf {sha256}asdfasdf {sha256lite}asdfasdf}.each do |sum| it "should consider #{sum} to be a checksum" do expect(@summer).to be_checksum(sum) end end %w{{nosuchsumthislong}asdfasdf {a}asdfasdf {ctime}}.each do |sum| it "should not consider #{sum} to be a checksum" do expect(@summer).not_to be_checksum(sum) end end it "should have a method for stripping a sum type from an existing checksum" do expect(@summer.sumtype("{md5}asdfasdfa")).to eq("md5") end it "should have a method for stripping the data from a checksum" do expect(@summer.sumdata("{md5}asdfasdfa")).to eq("asdfasdfa") end it "should return a nil sumtype if the checksum does not mention a checksum type" do expect(@summer.sumtype("asdfasdfa")).to be_nil end {:md5 => Digest::MD5, :sha1 => Digest::SHA1, :sha256 => Digest::SHA256}.each do |sum, klass| describe("when using #{sum}") do it "should use #{klass} to calculate string checksums" do klass.expects(:hexdigest).with("mycontent").returns "whatever" expect(@summer.send(sum, "mycontent")).to eq("whatever") end it "should use incremental #{klass} sums to calculate file checksums" do digest = mock 'digest' klass.expects(:new).returns digest file = "/path/to/my/file" fh = mock 'filehandle' fh.expects(:read).with(4096).times(3).returns("firstline").then.returns("secondline").then.returns(nil) - #fh.expects(:read).with(512).returns("secondline") - #fh.expects(:read).with(512).returns(nil) File.expects(:open).with(file, "rb").yields(fh) digest.expects(:<<).with "firstline" digest.expects(:<<).with "secondline" digest.expects(:hexdigest).returns :mydigest expect(@summer.send(sum.to_s + "_file", file)).to eq(:mydigest) end - it "should yield #{klass} to the given block to calculate stream checksums" do + it "should behave like #{klass} to calculate stream checksums" do digest = mock 'digest' klass.expects(:new).returns digest + digest.expects(:<<).with "firstline" + digest.expects(:<<).with "secondline" digest.expects(:hexdigest).returns :mydigest expect(@summer.send(sum.to_s + "_stream") do |checksum| - expect(checksum).to eq(digest) + checksum << "firstline" + checksum << "secondline" end).to eq(:mydigest) end end end {:md5lite => Digest::MD5, :sha1lite => Digest::SHA1, :sha256lite => Digest::SHA256}.each do |sum, klass| describe("when using #{sum}") do it "should use #{klass} to calculate string checksums from the first 512 characters of the string" do content = "this is a test" * 100 klass.expects(:hexdigest).with(content[0..511]).returns "whatever" expect(@summer.send(sum, content)).to eq("whatever") end it "should use #{klass} to calculate a sum from the first 512 characters in the file" do digest = mock 'digest' klass.expects(:new).returns digest file = "/path/to/my/file" fh = mock 'filehandle' fh.expects(:read).with(512).returns('my content') File.expects(:open).with(file, "rb").yields(fh) digest.expects(:<<).with "my content" digest.expects(:hexdigest).returns :mydigest expect(@summer.send(sum.to_s + "_file", file)).to eq(:mydigest) end + + it "should use #{klass} to calculate a sum from the first 512 characters in a stream" do + digest = mock 'digest' + content = "this is a test" * 100 + klass.expects(:new).returns digest + digest.expects(:<<).with content[0..511] + digest.expects(:hexdigest).returns :mydigest + + expect(@summer.send(sum.to_s + "_stream") do |checksum| + checksum << content + end).to eq(:mydigest) + end + + it "should use #{klass} to calculate a sum from the first 512 characters in a multi-part stream" do + digest = mock 'digest' + content = "this is a test" * 100 + klass.expects(:new).returns digest + digest.expects(:<<).with content[0..5] + digest.expects(:<<).with content[6..510] + digest.expects(:<<).with content[511..511] + digest.expects(:hexdigest).returns :mydigest + + expect(@summer.send(sum.to_s + "_stream") do |checksum| + checksum << content[0..5] + checksum << content[6..510] + checksum << content[511..-1] + end).to eq(:mydigest) + end end end [:ctime, :mtime].each do |sum| describe("when using #{sum}") do it "should use the '#{sum}' on the file to determine the ctime" do file = "/my/file" stat = mock 'stat', sum => "mysum" Puppet::FileSystem.expects(:stat).with(file).returns(stat) expect(@summer.send(sum.to_s + "_file", file)).to eq("mysum") end it "should return nil for streams" do expectation = stub "expectation" expectation.expects(:do_something!).at_least_once expect(@summer.send(sum.to_s + "_stream"){ |checksum| checksum << "anything" ; expectation.do_something! }).to be_nil end end end describe "when using the none checksum" do it "should return an empty string" do expect(@summer.none_file("/my/file")).to eq("") end it "should return an empty string for streams" do expectation = stub "expectation" expectation.expects(:do_something!).at_least_once expect(@summer.none_stream{ |checksum| checksum << "anything" ; expectation.do_something! }).to eq("") end end {:md5 => Digest::MD5, :sha1 => Digest::SHA1}.each do |sum, klass| describe "when using #{sum}" do let(:content) { "hello\r\nworld" } let(:path) do path = tmpfile("checksum_#{sum}") File.open(path, 'wb') {|f| f.write(content)} path end it "should preserve nl/cr sequences" do expect(@summer.send(sum.to_s + "_file", path)).to eq(klass.hexdigest(content)) end end end end