diff --git a/lib/puppet/type/tidy.rb b/lib/puppet/type/tidy.rb index 3d7190c27..a4b87d5fa 100755 --- a/lib/puppet/type/tidy.rb +++ b/lib/puppet/type/tidy.rb @@ -1,335 +1,341 @@ Puppet::Type.newtype(:tidy) do require 'puppet/file_serving/fileset' @doc = "Remove unwanted files based on specific criteria. Multiple criteria are OR'd together, so a file that is too large but is not old enough will still get tidied. If you don't specify either 'age' or 'size', then all files will be removed. This resource type works by generating a file resource for every file that should be deleted and then letting that resource perform the actual deletion. " newparam(:path) do desc "The path to the file or directory to manage. Must be fully qualified." isnamevar end + newparam(:recurse) do + desc "If target is a directory, recursively descend + into the directory looking for files to tidy." + + newvalues(:true, :false, :inf, /^[0-9]+$/) + + # Replace the validation so that we allow numbers in + # addition to string representations of them. + validate { |arg| } + munge do |value| + newval = super(value) + case newval + when :true, :inf; true + when :false; false + when Integer, Fixnum, Bignum; value + when /^\d+$/; Integer(value) + else + raise ArgumentError, "Invalid recurse value %s" % value.inspect + end + end + end + newparam(:matches) do desc "One or more (shell type) file glob patterns, which restrict the list of files to be tidied to those whose basenames match at least one of the patterns specified. Multiple patterns can be specified using an array. Example:: tidy { \"/tmp\": age => \"1w\", - recurse => false, + recurse => 1, matches => [ \"[0-9]pub*.tmp\", \"*.temp\", \"tmpfile?\" ] } This removes files from \/tmp if they are one week old or older, are not in a subdirectory and match one of the shell globs given. - Note that the patterns are matched against the - basename of each file -- that is, your glob patterns should not - have any '/' characters in them, since you are only specifying - against the last bit of the file." + Note that the patterns are matched against the basename of each + file -- that is, your glob patterns should not have any '/' + characters in them, since you are only specifying against the last + bit of the file. + + Finally, note that you must now specify a non-zero/non-false value + for recurse if matches is used, as matches only apply to files found + by recursion (there's no reason to use static patterns match against + a statically determined path). Requiering explicit recursion clears + up a common source of confusion." # Make sure we convert to an array. munge do |value| - value = [value] unless value.is_a?(Array) - value + fail "Tidy can't use matches with recurse 0, false, or undef" if "#{@resource[:recurse]}" =~ /^(0|false|)$/ + [value].flatten end # Does a given path match our glob patterns, if any? Return true # if no patterns have been provided. def tidy?(path, stat) basename = File.basename(path) flags = File::FNM_DOTMATCH | File::FNM_PATHNAME return true if value.find {|pattern| File.fnmatch(pattern, basename, flags) } return false end end newparam(:backup) do desc "Whether tidied files should be backed up. Any values are passed directly to the file resources used for actual file deletion, so use its backup documentation to determine valid values." end newparam(:age) do desc "Tidy files whose age is equal to or greater than the specified time. You can choose seconds, minutes, hours, days, or weeks by specifying the first letter of any of those words (e.g., '1w'). Specifying 0 will remove all files." @@ageconvertors = { :s => 1, :m => 60 } @@ageconvertors[:h] = @@ageconvertors[:m] * 60 @@ageconvertors[:d] = @@ageconvertors[:h] * 24 @@ageconvertors[:w] = @@ageconvertors[:d] * 7 def convert(unit, multi) if num = @@ageconvertors[unit] return num * multi else self.fail "Invalid age unit '%s'" % unit end end def tidy?(path, stat) # If the file's older than we allow, we should get rid of it. if (Time.now.to_i - stat.send(resource[:type]).to_i) > value return true else return false end end munge do |age| unit = multi = nil case age when /^([0-9]+)(\w)\w*$/ multi = Integer($1) unit = $2.downcase.intern when /^([0-9]+)$/ multi = Integer($1) unit = :d else self.fail "Invalid tidy age %s" % age end convert(unit, multi) end end newparam(:size) do desc "Tidy files whose size is equal to or greater than the specified size. Unqualified values are in kilobytes, but *b*, *k*, and *m* can be appended to specify *bytes*, *kilobytes*, and *megabytes*, respectively. Only the first character is significant, so the full word can also be used." @@sizeconvertors = { :b => 0, :k => 1, :m => 2, :g => 3 } def convert(unit, multi) if num = @@sizeconvertors[unit] result = multi num.times do result *= 1024 end return result else self.fail "Invalid size unit '%s'" % unit end end def tidy?(path, stat) if stat.size >= value return true else return false end end munge do |size| case size when /^([0-9]+)(\w)\w*$/ multi = Integer($1) unit = $2.downcase.intern when /^([0-9]+)$/ multi = Integer($1) unit = :k else self.fail "Invalid tidy size %s" % age end convert(unit, multi) end end newparam(:type) do desc "Set the mechanism for determining age." newvalues(:atime, :mtime, :ctime) defaultto :atime end - newparam(:recurse) do - desc "If target is a directory, recursively descend - into the directory looking for files to tidy." - - newvalues(:true, :false, :inf, /^[0-9]+$/) - - # Replace the validation so that we allow numbers in - # addition to string representations of them. - validate { |arg| } - munge do |value| - newval = super(value) - case newval - when :true, :inf; true - when :false; false - when Integer, Fixnum, Bignum; value - when /^\d+$/; Integer(value) - else - raise ArgumentError, "Invalid recurse value %s" % value.inspect - end - end - end - newparam(:rmdirs, :boolean => true) do desc "Tidy directories in addition to files; that is, remove directories whose age is older than the specified criteria. This will only remove empty directories, so all contained files must also be tidied before a directory gets removed." newvalues :true, :false end # Erase PFile's validate method validate do end def self.instances [] end @depthfirst = true def initialize(hash) super # only allow backing up into filebuckets unless self[:backup].is_a? Puppet::Network::Client.dipper self[:backup] = false end end # Make a file resource to remove a given file. def mkfile(path) # Force deletion, so directories actually get deleted. Puppet::Type.type(:file).new :path => path, :backup => self[:backup], :ensure => :absent, :force => true end def retrieve # Our ensure property knows how to retrieve everything for us. if obj = @parameters[:ensure] return obj.retrieve else return {} end end # Hack things a bit so we only ever check the ensure property. def properties [] end def eval_generate [] end def generate return [] unless stat(self[:path]) case self[:recurse] when Integer, Fixnum, Bignum, /^\d+$/ parameter = { :recurse => true, :recurselimit => self[:recurse] } when true, :true, :inf parameter = { :recurse => true } end if parameter files = Puppet::FileServing::Fileset.new(self[:path], parameter).files.collect do |f| f == "." ? self[:path] : File.join(self[:path], f) end else files = [self[:path]] end result = files.find_all { |path| tidy?(path) }.collect { |path| mkfile(path) }.each { |file| notice "Tidying %s" % file.ref }.sort { |a,b| b[:path] <=> a[:path] } # No need to worry about relationships if we don't have rmdirs; there won't be # any directories. return result unless rmdirs? # Now make sure that all directories require the files they contain, if all are available, # so that a directory is emptied before we try to remove it. files_by_name = result.inject({}) { |hash, file| hash[file[:path]] = file; hash } files_by_name.keys.sort { |a,b| b <=> b }.each do |path| dir = File.dirname(path) next unless resource = files_by_name[dir] if resource[:require] resource[:require] << Puppet::Resource::Reference.new(:file, path) else resource[:require] = [Puppet::Resource::Reference.new(:file, path)] end end return result end # Does a given path match our glob patterns, if any? Return true # if no patterns have been provided. def matches?(path) return true unless self[:matches] basename = File.basename(path) flags = File::FNM_DOTMATCH | File::FNM_PATHNAME if self[:matches].find {|pattern| File.fnmatch(pattern, basename, flags) } return true else debug "No specified patterns match %s, not tidying" % path return false end end # Should we remove the specified file? def tidy?(path) return false unless stat = self.stat(path) return false if stat.ftype == "directory" and ! rmdirs? # The 'matches' parameter isn't OR'ed with the other tests -- # it's just used to reduce the list of files we can match. return false if param = parameter(:matches) and ! param.tidy?(path, stat) tested = false [:age, :size].each do |name| next unless param = parameter(name) tested = true return true if param.tidy?(path, stat) end # If they don't specify either, then the file should always be removed. return true unless tested return false end def stat(path) begin File.lstat(path) rescue Errno::ENOENT => error info "File does not exist" return nil rescue Errno::EACCES => error warning "Could not stat; permission denied" return nil end end end diff --git a/spec/unit/type/tidy.rb b/spec/unit/type/tidy.rb index ccec9ed7c..6eac6a286 100755 --- a/spec/unit/type/tidy.rb +++ b/spec/unit/type/tidy.rb @@ -1,399 +1,422 @@ #!/usr/bin/env ruby require File.dirname(__FILE__) + '/../../spec_helper' tidy = Puppet::Type.type(:tidy) describe tidy do before do Puppet.settings.stubs(:use) # for an unknown reason some of these specs fails when run individually # with a failed expectation on File.lstat in the autoloader. File.stubs(:lstat) end it "should use :lstat when stating a file" do resource = tidy.new :path => "/foo/bar", :age => "1d" stat = mock 'stat' File.expects(:lstat).with("/foo/bar").returns stat resource.stat("/foo/bar").should == stat end [:age, :size, :path, :matches, :type, :recurse, :rmdirs].each do |param| it "should have a %s parameter" % param do Puppet::Type.type(:tidy).attrclass(param).ancestors.should be_include(Puppet::Parameter) end it "should have documentation for its %s param" % param do Puppet::Type.type(:tidy).attrclass(param).doc.should be_instance_of(String) end end describe "when validating parameter values" do describe "for 'recurse'" do before do @tidy = Puppet::Type.type(:tidy).new :path => "/tmp", :age => "100d" end it "should allow 'true'" do lambda { @tidy[:recurse] = true }.should_not raise_error end it "should allow 'false'" do lambda { @tidy[:recurse] = false }.should_not raise_error end it "should allow integers" do lambda { @tidy[:recurse] = 10 }.should_not raise_error end it "should allow string representations of integers" do lambda { @tidy[:recurse] = "10" }.should_not raise_error end it "should allow 'inf'" do lambda { @tidy[:recurse] = "inf" }.should_not raise_error end it "should not allow arbitrary values" do lambda { @tidy[:recurse] = "whatever" }.should raise_error end end + + describe "for 'matches'" do + before do + @tidy = Puppet::Type.type(:tidy).new :path => "/tmp", :age => "100d" + end + + it "should object if matches is given with recurse is not specified" do + lambda { @tidy[:matches] = '*.doh' }.should raise_error + end + it "should object if matches is given and recurse is 0" do + lambda { @tidy[:recurse] = 0; @tidy[:matches] = '*.doh' }.should raise_error + end + it "should object if matches is given and recurse is false" do + lambda { @tidy[:recurse] = false; @tidy[:matches] = '*.doh' }.should raise_error + end + it "should not object if matches is given and recurse is > 0" do + lambda { @tidy[:recurse] = 1; @tidy[:matches] = '*.doh' }.should_not raise_error + end + it "should not object if matches is given and recurse is true" do + lambda { @tidy[:recurse] = true; @tidy[:matches] = '*.doh' }.should_not raise_error + end + end end describe "when matching files by age" do convertors = { :second => 1, :minute => 60 } convertors[:hour] = convertors[:minute] * 60 convertors[:day] = convertors[:hour] * 24 convertors[:week] = convertors[:day] * 7 convertors.each do |unit, multiple| it "should consider a %s to be %s seconds" % [unit, multiple] do tidy = Puppet::Type.type(:tidy).new :path => "/what/ever", :age => "5%s" % unit.to_s[0..0] tidy[:age].should == 5 * multiple end end end describe "when matching files by size" do convertors = { :b => 0, :kb => 1, :mb => 2, :gb => 3 } convertors.each do |unit, multiple| it "should consider a %s to be 1024^%s bytes" % [unit, multiple] do tidy = Puppet::Type.type(:tidy).new :path => "/what/ever", :size => "5%s" % unit total = 5 multiple.times { total *= 1024 } tidy[:size].should == total end end end describe "when tidying" do before do @tidy = Puppet::Type.type(:tidy).new :path => "/what/ever" @stat = stub 'stat', :ftype => "directory" File.stubs(:lstat).with("/what/ever").returns @stat end describe "and generating files" do it "should set the backup on the file if backup is set on the tidy instance" do @tidy[:backup] = "whatever" Puppet::Type.type(:file).expects(:new).with { |args| args[:backup] == "whatever" } @tidy.mkfile("/what/ever") end it "should set the file's path to the tidy's path" do Puppet::Type.type(:file).expects(:new).with { |args| args[:path] == "/what/ever" } @tidy.mkfile("/what/ever") end it "should configure the file for deletion" do Puppet::Type.type(:file).expects(:new).with { |args| args[:ensure] == :absent } @tidy.mkfile("/what/ever") end it "should force deletion on the file" do Puppet::Type.type(:file).expects(:new).with { |args| args[:force] == true } @tidy.mkfile("/what/ever") end it "should do nothing if the targeted file does not exist" do File.expects(:lstat).with("/what/ever").raises Errno::ENOENT @tidy.generate.should == [] end end describe "and recursion is not used" do it "should generate a file resource if the file should be tidied" do @tidy.expects(:tidy?).with("/what/ever").returns true file = Puppet::Type.type(:file).new(:path => "/eh") @tidy.expects(:mkfile).with("/what/ever").returns file @tidy.generate.should == [file] end it "should do nothing if the file should not be tidied" do @tidy.expects(:tidy?).with("/what/ever").returns false @tidy.expects(:mkfile).never @tidy.generate.should == [] end end describe "and recursion is used" do before do @tidy[:recurse] = true Puppet::FileServing::Fileset.any_instance.stubs(:stat).returns mock("stat") @fileset = Puppet::FileServing::Fileset.new("/what/ever") Puppet::FileServing::Fileset.stubs(:new).returns @fileset end it "should use a Fileset for infinite recursion" do Puppet::FileServing::Fileset.expects(:new).with("/what/ever", :recurse => true).returns @fileset @fileset.expects(:files).returns %w{. one two} @tidy.stubs(:tidy?).returns false @tidy.generate end it "should use a Fileset for limited recursion" do @tidy[:recurse] = 42 Puppet::FileServing::Fileset.expects(:new).with("/what/ever", :recurse => true, :recurselimit => 42).returns @fileset @fileset.expects(:files).returns %w{. one two} @tidy.stubs(:tidy?).returns false @tidy.generate end it "should generate a file resource for every file that should be tidied but not for files that should not be tidied" do @fileset.expects(:files).returns %w{. one two} @tidy.expects(:tidy?).with("/what/ever").returns true @tidy.expects(:tidy?).with("/what/ever/one").returns true @tidy.expects(:tidy?).with("/what/ever/two").returns false file = Puppet::Type.type(:file).new(:path => "/eh") @tidy.expects(:mkfile).with("/what/ever").returns file @tidy.expects(:mkfile).with("/what/ever/one").returns file @tidy.generate end end describe "and determining whether a file matches provided glob patterns" do before do - @tidy = Puppet::Type.type(:tidy).new :path => "/what/ever" + @tidy = Puppet::Type.type(:tidy).new :path => "/what/ever", :recurse => 1 @tidy[:matches] = %w{*foo* *bar*} @stat = mock 'stat' @matcher = @tidy.parameter(:matches) end it "should always convert the globs to an array" do @matcher.value = "*foo*" @matcher.value.should == %w{*foo*} end it "should return true if any pattern matches the last part of the file" do @matcher.value = %w{*foo* *bar*} @matcher.must be_tidy("/file/yaybarness", @stat) end it "should return false if no pattern matches the last part of the file" do @matcher.value = %w{*foo* *bar*} @matcher.should_not be_tidy("/file/yayness", @stat) end end describe "and determining whether a file is too old" do before do @tidy = Puppet::Type.type(:tidy).new :path => "/what/ever" @stat = stub 'stat' @tidy[:age] = "1s" @tidy[:type] = "mtime" @ager = @tidy.parameter(:age) end it "should use the age type specified" do @tidy[:type] = :ctime @stat.expects(:ctime).returns(Time.now) @ager.tidy?("/what/ever", @stat) end it "should return false if the file is more recent than the specified age" do @stat.expects(:mtime).returns(Time.now) @ager.should_not be_tidy("/what/ever", @stat) end it "should return true if the file is older than the specified age" do @stat.expects(:mtime).returns(Time.now - 10) @ager.must be_tidy("/what/ever", @stat) end end describe "and determining whether a file is too large" do before do @tidy = Puppet::Type.type(:tidy).new :path => "/what/ever" @stat = stub 'stat', :ftype => "file" @tidy[:size] = "1kb" @sizer = @tidy.parameter(:size) end it "should return false if the file is smaller than the specified size" do @stat.expects(:size).returns(4) # smaller than a kilobyte @sizer.should_not be_tidy("/what/ever", @stat) end it "should return true if the file is larger than the specified size" do @stat.expects(:size).returns(1500) # larger than a kilobyte @sizer.must be_tidy("/what/ever", @stat) end it "should return true if the file is equal to the specified size" do @stat.expects(:size).returns(1024) @sizer.must be_tidy("/what/ever", @stat) end end describe "and determining whether a file should be tidied" do before do @tidy = Puppet::Type.type(:tidy).new :path => "/what/ever" @stat = stub 'stat', :ftype => "file" File.stubs(:lstat).with("/what/ever").returns @stat end it "should not try to recurse if the file does not exist" do @tidy[:recurse] = true File.stubs(:lstat).with("/what/ever").returns nil @tidy.generate.should == [] end it "should not be tidied if the file does not exist" do File.expects(:lstat).with("/what/ever").raises Errno::ENOENT @tidy.should_not be_tidy("/what/ever") end it "should not be tidied if the user has no access to the file" do File.expects(:lstat).with("/what/ever").raises Errno::EACCES @tidy.should_not be_tidy("/what/ever") end it "should not be tidied if it is a directory and rmdirs is set to false" do stat = mock 'stat', :ftype => "directory" File.expects(:lstat).with("/what/ever").returns stat @tidy.should_not be_tidy("/what/ever") end it "should return false if it does not match any provided globs" do + @tidy[:recurse] = 1 @tidy[:matches] = "globs" matches = @tidy.parameter(:matches) matches.expects(:tidy?).with("/what/ever", @stat).returns false @tidy.should_not be_tidy("/what/ever") end it "should return false if it does not match aging requirements" do @tidy[:age] = "1d" ager = @tidy.parameter(:age) ager.expects(:tidy?).with("/what/ever", @stat).returns false @tidy.should_not be_tidy("/what/ever") end it "should return false if it does not match size requirements" do @tidy[:size] = "1b" sizer = @tidy.parameter(:size) sizer.expects(:tidy?).with("/what/ever", @stat).returns false @tidy.should_not be_tidy("/what/ever") end it "should tidy a file if age and size are set but only size matches" do @tidy[:size] = "1b" @tidy[:age] = "1d" @tidy.parameter(:size).stubs(:tidy?).returns true @tidy.parameter(:age).stubs(:tidy?).returns false @tidy.should be_tidy("/what/ever") end it "should tidy a file if age and size are set but only age matches" do @tidy[:size] = "1b" @tidy[:age] = "1d" @tidy.parameter(:size).stubs(:tidy?).returns false @tidy.parameter(:age).stubs(:tidy?).returns true @tidy.should be_tidy("/what/ever") end it "should tidy all files if neither age nor size is set" do @tidy.must be_tidy("/what/ever") end it "should sort the results inversely by path length, so files are added to the catalog before their directories" do @tidy[:recurse] = true @tidy[:rmdirs] = true fileset = Puppet::FileServing::Fileset.new("/what/ever") Puppet::FileServing::Fileset.expects(:new).returns fileset fileset.expects(:files).returns %w{. one one/two} @tidy.stubs(:tidy?).returns true @tidy.generate.collect { |r| r[:path] }.should == %w{/what/ever/one/two /what/ever/one /what/ever} end end it "should configure directories to require their contained files if rmdirs is enabled, so the files will be deleted first" do @tidy[:recurse] = true @tidy[:rmdirs] = true fileset = mock 'fileset' Puppet::FileServing::Fileset.expects(:new).with("/what/ever", :recurse => true).returns fileset fileset.expects(:files).returns %w{. one two one/subone two/subtwo one/subone/ssone} @tidy.stubs(:tidy?).returns true result = @tidy.generate.inject({}) { |hash, res| hash[res[:path]] = res; hash } { "/what/ever" => %w{/what/ever/one /what/ever/two}, "/what/ever/one" => ["/what/ever/one/subone"], "/what/ever/two" => ["/what/ever/two/subtwo"], "/what/ever/one/subone" => ["/what/ever/one/subone/ssone"] }.each do |parent, children| children.each do |child| ref = Puppet::Resource::Reference.new(:file, child) result[parent][:require].find { |req| req.to_s == ref.to_s }.should_not be_nil end end end end end