diff --git a/lib/puppet/file_bucket/file.rb b/lib/puppet/file_bucket/file.rb index 1e95e5ef6..52ee4f711 100644 --- a/lib/puppet/file_bucket/file.rb +++ b/lib/puppet/file_bucket/file.rb @@ -1,123 +1,136 @@ require 'puppet/file_bucket' require 'puppet/indirector' +require 'puppet/util/checksums' class Puppet::FileBucket::File + include Puppet::Util::Checksums + # This class handles the abstract notion of a file in a filebucket. # There are mechanisms to save and load this file locally and remotely in puppet/indirector/filebucketfile/* # There is a compatibility class that emulates pre-indirector filebuckets in Puppet::FileBucket::Dipper extend Puppet::Indirector require 'puppet/file_bucket/file/indirection_hooks' indirects :file_bucket_file, :terminus_class => :file, :extend => Puppet::FileBucket::File::IndirectionHooks attr :path, true attr :paths, true attr :contents, true attr :checksum_type attr :bucket_path, true def self.default_checksum_type - :md5 + "md5" end def initialize( contents, options = {} ) - @contents = contents @bucket_path = options[:bucket_path] @path = options[:path] @paths = options[:paths] || [] + @checksum = options[:checksum] - @checksum_type = options[:checksum_type] || self.class.default_checksum_type + @checksum_type = options[:checksum_type] + + self.contents = contents yield(self) if block_given? validate! end def validate! - digest_class( @checksum_type ) # raises error on bad types - raise ArgumentError, 'contents must be a string' unless @contents.is_a?(String) - validate_checksum(@checksum) if @checksum + validate_checksum_type!(checksum_type) + validate_checksum!(checksum) if checksum end - def contents=(contents) + def contents=(str) raise "You may not change the contents of a FileBucket File" if @contents - @contents = contents - end - - def checksum=(checksum) - validate_checksum(checksum) - self.checksum_type = checksum # this grabs the prefix only - @checksum = checksum - end - - def validate_checksum(new_checksum) - unless new_checksum == checksum_of_type(new_checksum) - raise Puppet::Error, "checksum does not match contents" - end + validate_content!(str) + @contents = str end def checksum - @checksum ||= checksum_of_type(checksum_type) + return @checksum if @checksum + @checksum = calculate_checksum if contents + @checksum end - def checksum_of_type( type ) - type = checksum_type( type ) # strip out data segment if there is one - type.to_s + ":" + digest_class(type).hexdigest(@contents) + def checksum=(checksum) + validate_checksum!(checksum) + @checksum = checksum end def checksum_type=( new_checksum_type ) @checksum = nil - @checksum_type = checksum_type(new_checksum_type) - end - - def checksum_type(checksum = @checksum_type) - checksum.to_s.split(':',2)[0].to_sym + @checksum_type = new_checksum_type end - def checksum_data(new_checksum = self.checksum) - new_checksum.split(':',2)[1] - end - - def checksum_data=(new_data) - self.checksum = "#{checksum_type}:#{new_data}" + def checksum_type + unless @checksum_type + if @checksum + @checksum_type = sumtype(checksum) + else + @checksum_type = self.class.default_checksum_type + end + end + @checksum_type end - def digest_class(type = nil) - case checksum_type(type) - when :md5 : require 'digest/md5' ; Digest::MD5 - when :sha1 : require 'digest/sha1' ; Digest::SHA1 - else - raise ArgumentError, "not a known checksum type: #{checksum_type(type)}" - end + def checksum_data + sumdata(checksum) end def to_s contents end def name [checksum_type, checksum_data, path].compact.join('/') end def name=(name) - self.checksum_type, self.checksum_data, self.path = name.split('/',3) + data = name.split('/',3) + self.path = data.pop + @checksum_type = nil + self.checksum = "{#{data[0]}}#{data[1]}" end def conflict_check? true end def self.from_s( contents ) self.new( contents ) end def to_pson hash = { "contents" => contents } hash["path"] = @path if @path hash.to_pson end def self.from_pson( pson ) self.new( pson["contents"], :path => pson["path"] ) end + private + + def calculate_checksum + "{#{checksum_type}}" + send(checksum_type, contents) + end + + def validate_content!(content) + raise ArgumentError, "Contents must be a string" if content and ! content.is_a?(String) + end + + def validate_checksum!(new_checksum) + newtype = sumtype(new_checksum) + + unless sumdata(new_checksum) == (calc_sum = send(newtype, contents)) + raise Puppet::Error, "Checksum #{new_checksum} does not match contents #{calc_sum}" + end + end + + def validate_checksum_type!(type) + raise ArgumentError, "Invalid checksum type #{type}" unless respond_to?(type) + end end diff --git a/lib/puppet/indirector/file_bucket_file/file.rb b/lib/puppet/indirector/file_bucket_file/file.rb index 4c08b8d6b..9fe40f62d 100644 --- a/lib/puppet/indirector/file_bucket_file/file.rb +++ b/lib/puppet/indirector/file_bucket_file/file.rb @@ -1,143 +1,148 @@ require 'puppet/indirector/code' require 'puppet/file_bucket/file' +require 'puppet/util/checksums' module Puppet::FileBucketFile class File < Puppet::Indirector::Code + include Puppet::Util::Checksums + desc "Store files in a directory set based on their checksums." def initialize Puppet.settings.use(:filebucket) end def find( request ) checksum, path = request_to_checksum_and_path( request ) return find_by_checksum( checksum, request.options ) end def save( request ) checksum, path = request_to_checksum_and_path( request ) instance = request.instance instance.checksum = checksum if checksum instance.path = path if path save_to_disk(instance) instance.to_s end private def find_by_checksum( checksum, options ) model.new( nil, :checksum => checksum ) do |bucket_file| bucket_file.bucket_path = options[:bucket_path] filename = contents_path_for( bucket_file ) if ! ::File.exist? filename return nil end begin contents = ::File.read filename Puppet.info "FileBucket read #{bucket_file.checksum}" rescue RuntimeError => e raise Puppet::Error, "file could not be read: #{e.message}" end if ::File.exist?(paths_path_for( bucket_file) ) ::File.open(paths_path_for( bucket_file) ) do |f| bucket_file.paths = f.readlines.map { |l| l.chomp } end end bucket_file.contents = contents end end def save_to_disk( bucket_file ) # If the file already exists, just return the md5 sum. if ::File.exist?(contents_path_for( bucket_file) ) verify_identical_file!(bucket_file) else # Make the directories if necessary. unless ::File.directory?( path_for( bucket_file) ) Puppet::Util.withumask(0007) do ::FileUtils.mkdir_p( path_for( bucket_file) ) end end Puppet.info "FileBucket adding #{bucket_file.path} (#{bucket_file.checksum_data})" # Write the file to disk. Puppet::Util.withumask(0007) do ::File.open(contents_path_for(bucket_file), ::File::WRONLY|::File::CREAT, 0440) do |of| of.print bucket_file.contents end end end save_path_to_paths_file(bucket_file) return bucket_file.checksum_data end def request_to_checksum_and_path( request ) - checksum_type, checksum, path = request.key.split(/[:\/]/, 3) + return [request.key, nil] if checksum?(request.key) + + checksum_type, checksum, path = request.key.split(/\//, 3) return nil if checksum_type.to_s == "" - return [ checksum_type + ":" + checksum, path ] + return [ "{#{checksum_type}}#{checksum}", path ] end def path_for(bucket_file, subfile = nil) bucket_path = bucket_file.bucket_path || Puppet[:bucketdir] digest = bucket_file.checksum_data dir = ::File.join(digest[0..7].split("")) basedir = ::File.join(bucket_path, dir, digest) return basedir unless subfile return ::File.join(basedir, subfile) end def contents_path_for(bucket_file) path_for(bucket_file, "contents") end def paths_path_for(bucket_file) path_for(bucket_file, "paths") end def content_check? true end # If conflict_check is enabled, verify that the passed text is # the same as the text in our file. def verify_identical_file!(bucket_file) return unless content_check? disk_contents = ::File.read(contents_path_for(bucket_file)) # If the contents don't match, then we've found a conflict. # Unlikely, but quite bad. if disk_contents != bucket_file.contents raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}", caller else Puppet.info "FileBucket got a duplicate file #{bucket_file.path} (#{bucket_file.checksum})" end end def save_path_to_paths_file(bucket_file) return unless bucket_file.path # check for dupes if ::File.exist?(paths_path_for( bucket_file) ) ::File.open(paths_path_for( bucket_file) ) do |f| return if f.readlines.collect { |l| l.chomp }.include?(bucket_file.path) end end # if it's a new file, or if our path isn't in the file yet, add it ::File.open(paths_path_for(bucket_file), ::File::WRONLY|::File::CREAT|::File::APPEND) do |of| of.puts bucket_file.path end end end end diff --git a/lib/puppet/util/checksums.rb b/lib/puppet/util/checksums.rb index a09523740..41f878687 100644 --- a/lib/puppet/util/checksums.rb +++ b/lib/puppet/util/checksums.rb @@ -1,124 +1,133 @@ # A stand-alone module for calculating checksums # in a generic way. module Puppet::Util::Checksums # Is the provided string a checksum? def checksum?(string) string =~ /^\{(\w{3,5})\}\S+/ end + # Strip the checksum type from an existing checksum + def sumdata(checksum) + if checksum =~ /^\{(\w+)\}(.+)/ + return $2 + else + return nil + end + end + # Strip the checksum type from an existing checksum def sumtype(checksum) if checksum =~ /^\{(\w+)\}/ return $1 else return nil end end # Calculate a checksum using Digest::MD5. def md5(content) require 'digest/md5' Digest::MD5.hexdigest(content) end # Calculate a checksum of the first 500 chars of the content using Digest::MD5. def md5lite(content) md5(content[0..511]) end # Calculate a checksum of a file's content using Digest::MD5. def md5_file(filename, lite = false) require 'digest/md5' digest = Digest::MD5.new() return checksum_file(digest, filename, lite) end # Calculate a checksum of the first 500 chars of a file's content using Digest::MD5. def md5lite_file(filename) md5_file(filename, true) end def md5_stream(&block) require 'digest/md5' digest = Digest::MD5.new() yield digest return digest.hexdigest end alias :md5lite_stream :md5_stream # Return the :mtime timestamp of a file. def mtime_file(filename) File.stat(filename).send(:mtime) end # by definition this doesn't exist def mtime_stream nil end alias :ctime_stream :mtime_stream # Calculate a checksum using Digest::SHA1. def sha1(content) require 'digest/sha1' Digest::SHA1.hexdigest(content) end # Calculate a checksum of the first 500 chars of the content using Digest::SHA1. def sha1lite(content) sha1(content[0..511]) end # Calculate a checksum of a file's content using Digest::SHA1. def sha1_file(filename, lite = false) require 'digest/sha1' digest = Digest::SHA1.new() return checksum_file(digest, filename, lite) end # Calculate a checksum of the first 500 chars of a file's content using Digest::SHA1. def sha1lite_file(filename) sha1_file(filename, true) end def sha1_stream require 'digest/sha1' digest = Digest::SHA1.new() yield digest return digest.hexdigest end alias :sha1lite_stream :sha1_stream # Return the :ctime of a file. def ctime_file(filename) File.stat(filename).send(:ctime) end # Return a "no checksum" def none_file(filename) "" end def none_stream "" end private # Perform an incremental checksum on a file. def checksum_file(digest, filename, lite = false) buffer = lite ? 512 : 4096 File.open(filename, 'r') do |file| while content = file.read(buffer) digest << content break if lite end end return digest.hexdigest end end diff --git a/spec/unit/file_bucket/file.rb b/spec/unit/file_bucket/file.rb index 76f8e2599..73587652c 100644 --- a/spec/unit/file_bucket/file.rb +++ b/spec/unit/file_bucket/file.rb @@ -1,239 +1,230 @@ #!/usr/bin/env ruby require ::File.dirname(__FILE__) + '/../../spec_helper' require 'puppet/file_bucket/file' require 'digest/md5' require 'digest/sha1' describe Puppet::FileBucket::File do before do # this is the default from spec_helper, but it keeps getting reset at odd times Puppet[:bucketdir] = "/dev/null/bucket" @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "{md5}4a8ec4fa5f01b4ab1a0ab8cbccb709f0" @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' @contents = "file contents" end - it "should save a file" do - ::File.expects(:exist?).with("#{@dir}/contents").returns false - ::File.expects(:directory?).with(@dir).returns false - ::FileUtils.expects(:mkdir_p).with(@dir) - ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) - - bucketfile = Puppet::FileBucket::File.new(@contents) - bucketfile.save - - end - - describe "using the indirector's find method" do - it "should return nil if a file doesn't exist" do - ::File.expects(:exist?).with("#{@dir}/contents").returns false - - bucketfile = Puppet::FileBucket::File.find("md5:#{@digest}") - bucketfile.should == nil - end - - it "should find a filebucket if the file exists" do - ::File.expects(:exist?).with("#{@dir}/contents").returns true - ::File.expects(:exist?).with("#{@dir}/paths").returns false - ::File.expects(:read).with("#{@dir}/contents").returns @contents - - bucketfile = Puppet::FileBucket::File.find("md5:#{@digest}") - bucketfile.should_not == nil - end - - describe "using RESTish digest notation" do - it "should return nil if a file doesn't exist" do - ::File.expects(:exist?).with("#{@dir}/contents").returns false - - bucketfile = Puppet::FileBucket::File.find("md5/#{@digest}") - bucketfile.should == nil - end - - it "should find a filebucket if the file exists" do - ::File.expects(:exist?).with("#{@dir}/contents").returns true - ::File.expects(:exist?).with("#{@dir}/paths").returns false - ::File.expects(:read).with("#{@dir}/contents").returns @contents - - bucketfile = Puppet::FileBucket::File.find("md5/#{@digest}") - bucketfile.should_not == nil - end - - end - end - it "should have a to_s method to return the contents" do Puppet::FileBucket::File.new(@contents).to_s.should == @contents end - it "should have a method that returns the digest algorithm" do - Puppet::FileBucket::File.new(@contents, :checksum => @checksum).checksum_type.should == :md5 + it "should calculate the checksum type from the passed in checksum" do + Puppet::FileBucket::File.new(@contents, :checksum => @checksum).checksum_type.should == "md5" end it "should allow contents to be specified in a block" do bucket = Puppet::FileBucket::File.new(nil) do |fb| fb.contents = "content" end bucket.contents.should == "content" end it "should raise an error if changing content" do x = Puppet::FileBucket::File.new("first") proc { x.contents = "new" }.should raise_error end it "should require contents to be a string" do proc { Puppet::FileBucket::File.new(5) }.should raise_error(ArgumentError) end it "should raise an error if setting contents to a non-string" do proc do Puppet::FileBucket::File.new(nil) do |x| x.contents = 5 end end.should raise_error(ArgumentError) end it "should set the contents appropriately" do Puppet::FileBucket::File.new(@contents).contents.should == @contents end + it "should default to 'md5' as the checksum algorithm if the algorithm is not in the name" do + Puppet::FileBucket::File.new(@contents).checksum_type.should == "md5" + end + it "should calculate the checksum" do - Digest::MD5.expects(:hexdigest).with(@contents).returns('mychecksum') - Puppet::FileBucket::File.new(@contents).checksum.should == 'md5:mychecksum' + Puppet::FileBucket::File.new(@contents).checksum.should == @checksum end it "should remove the old checksum value if the algorithm is changed" do - Digest::MD5.expects(:hexdigest).with(@contents).returns('oldsum') sum = Puppet::FileBucket::File.new(@contents) - oldsum = sum.checksum + sum.checksum.should_not be_nil + newsum = Digest::SHA1.hexdigest(@contents).to_s sum.checksum_type = :sha1 - Digest::SHA1.expects(:hexdigest).with(@contents).returns('newsum') - sum.checksum.should == 'sha1:newsum' - end - - it "should default to 'md5' as the checksum algorithm if the algorithm is not in the name" do - Puppet::FileBucket::File.new(@contents).checksum_type.should == :md5 + sum.checksum.should == "{sha1}#{newsum}" end it "should support specifying the checksum_type during initialization" do sum = Puppet::FileBucket::File.new(@contents, :checksum_type => :sha1) sum.checksum_type.should == :sha1 end it "should fail when an unsupported checksum_type is used" do proc { Puppet::FileBucket::File.new(@contents, :checksum_type => :nope) }.should raise_error(ArgumentError) end - it "should fail if given an invalid checksum at initialization" do - proc { Puppet::FileBucket::File.new(@contents, :checksum => "md5:00000000000000000000000000000000") }.should raise_error(RuntimeError) + it "should fail if given an checksum at initialization that does not match the contents" do + proc { Puppet::FileBucket::File.new(@contents, :checksum => "{md5}00000000000000000000000000000000") }.should raise_error(RuntimeError) end - it "should fail if assigned an invalid checksum " do + it "should fail if assigned a checksum that does not match the contents" do bucket = Puppet::FileBucket::File.new(@contents) - proc { bucket.checksum = "md5:00000000000000000000000000000000" }.should raise_error(RuntimeError) + proc { bucket.checksum = "{md5}00000000000000000000000000000000" }.should raise_error(RuntimeError) end - it "should accept checksum_data without a prefix" do - bucket = Puppet::FileBucket::File.new(@contents) - bucket.checksum_data = @digest - end - - describe "when using back-ends" do it "should redirect using Puppet::Indirector" do Puppet::Indirector::Indirection.instance(:file_bucket_file).model.should equal(Puppet::FileBucket::File) end it "should have a :save instance method" do Puppet::FileBucket::File.new("mysum").should respond_to(:save) end it "should respond to :find" do Puppet::FileBucket::File.should respond_to(:find) end it "should respond to :destroy" do Puppet::FileBucket::File.should respond_to(:destroy) end end describe "when saving files" do it "should save the contents to the calculated path" do ::File.stubs(:directory?).with(@dir).returns(true) ::File.expects(:exist?).with("#{@dir}/contents").returns false mockfile = mock "file" mockfile.expects(:print).with(@contents) ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440).yields(mockfile) Puppet::FileBucket::File.new(@contents).save end it "should make any directories necessary for storage" do FileUtils.expects(:mkdir_p).with do |arg| ::File.umask == 0007 and arg == @dir end ::File.expects(:directory?).with(@dir).returns(false) ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) ::File.expects(:exist?).with("#{@dir}/contents").returns false Puppet::FileBucket::File.new(@contents).save end - end - it "should accept a path" do - remote_path = '/path/on/the/remote/box' - Puppet::FileBucket::File.new(@contents, :path => remote_path).path.should == remote_path - end + it "should append the path to the paths file" do + remote_path = '/path/on/the/remote/box' - it "should append the path to the paths file" do - remote_path = '/path/on/the/remote/box' + ::File.expects(:directory?).with(@dir).returns(true) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) + ::File.expects(:exist?).with("#{@dir}/contents").returns false - ::File.expects(:directory?).with(@dir).returns(true) - ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) - ::File.expects(:exist?).with("#{@dir}/contents").returns false + mockfile = mock "file" + mockfile.expects(:puts).with('/path/on/the/remote/box') + ::File.expects(:exist?).with("#{@dir}/paths").returns false + ::File.expects(:open).with("#{@dir}/paths", ::File::WRONLY|::File::CREAT|::File::APPEND).yields mockfile + Puppet::FileBucket::File.new(@contents, :path => remote_path).save - mockfile = mock "file" - mockfile.expects(:puts).with('/path/on/the/remote/box') - ::File.expects(:exist?).with("#{@dir}/paths").returns false - ::File.expects(:open).with("#{@dir}/paths", ::File::WRONLY|::File::CREAT|::File::APPEND).yields mockfile - Puppet::FileBucket::File.new(@contents, :path => remote_path).save + end + end + it "should accept a path" do + remote_path = '/path/on/the/remote/box' + Puppet::FileBucket::File.new(@contents, :path => remote_path).path.should == remote_path end it "should return a url-ish name" do Puppet::FileBucket::File.new(@contents).name.should == "md5/4a8ec4fa5f01b4ab1a0ab8cbccb709f0" end it "should reject a url-ish name with an invalid checksum" do bucket = Puppet::FileBucket::File.new(@contents) lambda { bucket.name = "sha1/4a8ec4fa5f01b4ab1a0ab8cbccb709f0/new/path" }.should raise_error end it "should accept a url-ish name" do bucket = Puppet::FileBucket::File.new(@contents) lambda { bucket.name = "sha1/034fa2ed8e211e4d20f20e792d777f4a30af1a93/new/path" }.should_not raise_error - bucket.checksum_type.should == :sha1 + bucket.checksum_type.should == "sha1" bucket.checksum_data.should == '034fa2ed8e211e4d20f20e792d777f4a30af1a93' bucket.path.should == "new/path" end it "should return a url-ish name with a path" do Puppet::FileBucket::File.new(@contents, :path => 'my/path').name.should == "md5/4a8ec4fa5f01b4ab1a0ab8cbccb709f0/my/path" end it "should convert the contents to PSON" do Puppet::FileBucket::File.new(@contents).to_pson.should == '{"contents":"file contents"}' end it "should load from PSON" do Puppet::FileBucket::File.from_pson({"contents"=>"file contents"}).contents.should == "file contents" end + it "should save a file" do + ::File.expects(:exist?).with("#{@dir}/contents").returns false + ::File.expects(:directory?).with(@dir).returns false + ::FileUtils.expects(:mkdir_p).with(@dir) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) + + bucketfile = Puppet::FileBucket::File.new(@contents) + bucketfile.save + + end + + describe "using the indirector's find method" do + it "should return nil if a file doesn't exist" do + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + bucketfile = Puppet::FileBucket::File.find("{md5}#{@digest}") + bucketfile.should == nil + end + + it "should find a filebucket if the file exists" do + ::File.expects(:exist?).with("#{@dir}/contents").returns true + ::File.expects(:exist?).with("#{@dir}/paths").returns false + ::File.expects(:read).with("#{@dir}/contents").returns @contents + + bucketfile = Puppet::FileBucket::File.find("{md5}#{@digest}") + bucketfile.should_not == nil + end + + describe "using RESTish digest notation" do + it "should return nil if a file doesn't exist" do + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + bucketfile = Puppet::FileBucket::File.find("md5/#{@digest}") + bucketfile.should == nil + end + + it "should find a filebucket if the file exists" do + ::File.expects(:exist?).with("#{@dir}/contents").returns true + ::File.expects(:exist?).with("#{@dir}/paths").returns false + ::File.expects(:read).with("#{@dir}/contents").returns @contents + + bucketfile = Puppet::FileBucket::File.find("md5/#{@digest}") + bucketfile.should_not == nil + end + + end + end end diff --git a/spec/unit/indirector/file_bucket_file/file.rb b/spec/unit/indirector/file_bucket_file/file.rb index 6e0d340f3..bad6de380 100755 --- a/spec/unit/indirector/file_bucket_file/file.rb +++ b/spec/unit/indirector/file_bucket_file/file.rb @@ -1,289 +1,289 @@ #!/usr/bin/env ruby require ::File.dirname(__FILE__) + '/../../../spec_helper' require 'puppet/indirector/file_bucket_file/file' describe Puppet::FileBucketFile::File do it "should be a subclass of the Code terminus class" do Puppet::FileBucketFile::File.superclass.should equal(Puppet::Indirector::Code) end it "should have documentation" do Puppet::FileBucketFile::File.doc.should be_instance_of(String) end describe "when initializing" do it "should use the filebucket settings section" do Puppet.settings.expects(:use).with(:filebucket) Puppet::FileBucketFile::File.new end end describe "the find_by_checksum method" do before do # this is the default from spec_helper, but it keeps getting reset at odd times Puppet[:bucketdir] = "/dev/null/bucket" @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "{md5}4a8ec4fa5f01b4ab1a0ab8cbccb709f0" @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' @contents = "file contents" end it "should return nil if a file doesn't exist" do ::File.expects(:exist?).with("#{@dir}/contents").returns false - bucketfile = Puppet::FileBucketFile::File.new.send(:find_by_checksum, "md5:#{@digest}", {}) + bucketfile = Puppet::FileBucketFile::File.new.send(:find_by_checksum, "{md5}#{@digest}", {}) bucketfile.should == nil end it "should find a filebucket if the file exists" do ::File.expects(:exist?).with("#{@dir}/contents").returns true ::File.expects(:exist?).with("#{@dir}/paths").returns false ::File.expects(:read).with("#{@dir}/contents").returns @contents - bucketfile = Puppet::FileBucketFile::File.new.send(:find_by_checksum, "md5:#{@digest}", {}) + bucketfile = Puppet::FileBucketFile::File.new.send(:find_by_checksum, "{md5}#{@digest}", {}) bucketfile.should_not == nil end it "should load the paths" do paths = ["path1", "path2"] ::File.expects(:exist?).with("#{@dir}/contents").returns true ::File.expects(:exist?).with("#{@dir}/paths").returns true ::File.expects(:read).with("#{@dir}/contents").returns @contents mockfile = mock "file" mockfile.expects(:readlines).returns( paths ) ::File.expects(:open).with("#{@dir}/paths").yields mockfile - Puppet::FileBucketFile::File.new.send(:find_by_checksum, "md5:#{@digest}", {}).paths.should == paths + Puppet::FileBucketFile::File.new.send(:find_by_checksum, "{md5}#{@digest}", {}).paths.should == paths end end describe "when retrieving files" do before :each do Puppet.settings.stubs(:use) @store = Puppet::FileBucketFile::File.new @digest = "70924d6fa4b2d745185fa4660703a5c0" @sum = stub 'sum', :name => @digest @dir = "/what/ever" Puppet.stubs(:[]).with(:bucketdir).returns(@dir) @contents_path = '/what/ever/7/0/9/2/4/d/6/f/70924d6fa4b2d745185fa4660703a5c0/contents' @paths_path = '/what/ever/7/0/9/2/4/d/6/f/70924d6fa4b2d745185fa4660703a5c0/paths' @request = stub 'request', :key => "md5/#{@digest}/remote/path", :options => {} end it "should call find_by_checksum" do - @store.expects(:find_by_checksum).with{|x,opts| x == "md5:#{@digest}"}.returns(false) + @store.expects(:find_by_checksum).with{|x,opts| x == "{md5}#{@digest}"}.returns(false) @store.find(@request) end it "should look for the calculated path" do ::File.expects(:exist?).with(@contents_path).returns(false) @store.find(@request) end it "should return an instance of Puppet::FileBucket::File created with the content if the file exists" do content = "my content" bucketfile = stub 'bucketfile' bucketfile.stubs(:bucket_path) bucketfile.stubs(:bucket_path=) bucketfile.stubs(:checksum_data).returns(@digest) bucketfile.stubs(:checksum).returns(@checksum) bucketfile.expects(:contents=).with(content) - Puppet::FileBucket::File.expects(:new).with(nil, {:checksum => "md5:#{@digest}"}).yields(bucketfile).returns(bucketfile) + Puppet::FileBucket::File.expects(:new).with(nil, {:checksum => "{md5}#{@digest}"}).yields(bucketfile).returns(bucketfile) ::File.expects(:exist?).with(@contents_path).returns(true) ::File.expects(:exist?).with(@paths_path).returns(false) ::File.expects(:read).with(@contents_path).returns(content) @store.find(@request).should equal(bucketfile) end it "should return nil if no file is found" do ::File.expects(:exist?).with(@contents_path).returns(false) @store.find(@request).should be_nil end it "should fail intelligently if a found file cannot be read" do ::File.expects(:exist?).with(@contents_path).returns(true) ::File.expects(:read).with(@contents_path).raises(RuntimeError) proc { @store.find(@request) }.should raise_error(Puppet::Error) end end describe "when determining file paths" do before do Puppet[:bucketdir] = '/dev/null/bucketdir' @digest = 'DEADBEEFC0FFEE' @bucket = stub_everything "bucket" @bucket.expects(:checksum_data).returns(@digest) end it "should use the value of the :bucketdir setting as the root directory" do path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) path.should =~ %r{^/dev/null/bucketdir} end it "should choose a path 8 directories deep with each directory name being the respective character in the filebucket" do path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) dirs = @digest[0..7].split("").join(File::SEPARATOR) path.should be_include(dirs) end it "should use the full filebucket as the final directory name" do path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) ::File.basename(::File.dirname(path)).should == @digest end it "should use 'contents' as the actual file name" do path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) ::File.basename(path).should == "contents" end it "should use the bucketdir, the 8 sum character directories, the full filebucket, and 'contents' as the full file name" do path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) path.should == ['/dev/null/bucketdir', @digest[0..7].split(""), @digest, "contents"].flatten.join(::File::SEPARATOR) end end describe "when saving files" do before do # this is the default from spec_helper, but it keeps getting reset at odd times Puppet[:bucketdir] = "/dev/null/bucket" @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "{md5}4a8ec4fa5f01b4ab1a0ab8cbccb709f0" @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' @contents = "file contents" @bucket = stub "bucket file" @bucket.stubs(:bucket_path) @bucket.stubs(:checksum_data).returns(@digest) @bucket.stubs(:path).returns(nil) @bucket.stubs(:contents).returns("file contents") end it "should save the contents to the calculated path" do ::File.stubs(:directory?).with(@dir).returns(true) ::File.expects(:exist?).with("#{@dir}/contents").returns false mockfile = mock "file" mockfile.expects(:print).with(@contents) ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440).yields(mockfile) Puppet::FileBucketFile::File.new.send(:save_to_disk, @bucket) end it "should make any directories necessary for storage" do FileUtils.expects(:mkdir_p).with do |arg| ::File.umask == 0007 and arg == @dir end ::File.expects(:directory?).with(@dir).returns(false) ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) ::File.expects(:exist?).with("#{@dir}/contents").returns false Puppet::FileBucketFile::File.new.send(:save_to_disk, @bucket) end end describe "when verifying identical files" do before do # this is the default from spec_helper, but it keeps getting reset at odd times Puppet[:bucketdir] = "/dev/null/bucket" @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "{md5}4a8ec4fa5f01b4ab1a0ab8cbccb709f0" @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' @contents = "file contents" @bucket = stub "bucket file" @bucket.stubs(:bucket_path) @bucket.stubs(:checksum).returns(@checksum) @bucket.stubs(:checksum_data).returns(@digest) @bucket.stubs(:path).returns(nil) @bucket.stubs(:contents).returns("file contents") end it "should raise an error if the files don't match" do File.expects(:read).with("#{@dir}/contents").returns("corrupt contents") lambda{ Puppet::FileBucketFile::File.new.send(:verify_identical_file!, @bucket) }.should raise_error(Puppet::FileBucket::BucketError) end it "should do nothing if the files match" do File.expects(:read).with("#{@dir}/contents").returns("file contents") Puppet::FileBucketFile::File.new.send(:verify_identical_file!, @bucket) end end describe "when writing to the paths file" do before do Puppet[:bucketdir] = '/dev/null/bucketdir' @digest = '70924d6fa4b2d745185fa4660703a5c0' @bucket = stub_everything "bucket" @paths_path = '/dev/null/bucketdir/7/0/9/2/4/d/6/f/70924d6fa4b2d745185fa4660703a5c0/paths' @paths = [] @bucket.stubs(:paths).returns(@paths) @bucket.stubs(:checksum_data).returns(@digest) end it "should create a file if it doesn't exist" do @bucket.expects(:path).returns('path/to/save').at_least_once File.expects(:exist?).with(@paths_path).returns(false) file = stub "file" file.expects(:puts).with('path/to/save') File.expects(:open).with(@paths_path, ::File::WRONLY|::File::CREAT|::File::APPEND).yields(file) Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) end it "should append to a file if it exists" do @bucket.expects(:path).returns('path/to/save').at_least_once File.expects(:exist?).with(@paths_path).returns(true) old_file = stub "file" old_file.stubs(:readlines).returns [] File.expects(:open).with(@paths_path).yields(old_file) file = stub "file" file.expects(:puts).with('path/to/save') File.expects(:open).with(@paths_path, ::File::WRONLY|::File::CREAT|::File::APPEND).yields(file) Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) end it "should not alter a file if it already contains the path" do @bucket.expects(:path).returns('path/to/save').at_least_once File.expects(:exist?).with(@paths_path).returns(true) old_file = stub "file" old_file.stubs(:readlines).returns ["path/to/save\n"] File.expects(:open).with(@paths_path).yields(old_file) Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) end it "should do nothing if there is no path" do @bucket.expects(:path).returns(nil).at_least_once Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) end end end diff --git a/spec/unit/util/checksums.rb b/spec/unit/util/checksums.rb index eba564352..35d18633a 100755 --- a/spec/unit/util/checksums.rb +++ b/spec/unit/util/checksums.rb @@ -1,149 +1,153 @@ #!/usr/bin/env ruby # # Created by Luke Kanies on 2007-9-22. # Copyright (c) 2007. All rights reserved. require File.dirname(__FILE__) + '/../../spec_helper' require 'puppet/util/checksums' describe Puppet::Util::Checksums do before do @summer = Object.new @summer.extend(Puppet::Util::Checksums) end content_sums = [:md5, :md5lite, :sha1, :sha1lite] file_only = [:ctime, :mtime, :none] content_sums.each do |sumtype| it "should be able to calculate %s sums from strings" % sumtype do @summer.should be_respond_to(sumtype) end end [content_sums, file_only].flatten.each do |sumtype| it "should be able to calculate %s sums from files" % sumtype do @summer.should be_respond_to(sumtype.to_s + "_file") end end [content_sums, file_only].flatten.each do |sumtype| it "should be able to calculate %s sums from stream" % sumtype do @summer.should be_respond_to(sumtype.to_s + "_stream") end end it "should have a method for determining whether a given string is a checksum" do @summer.should respond_to(:checksum?) end %w{{md5}asdfasdf {sha1}asdfasdf {ctime}asdasdf {mtime}asdfasdf}.each do |sum| it "should consider #{sum} to be a checksum" do @summer.should be_checksum(sum) end end %w{{nosuchsum}asdfasdf {a}asdfasdf {ctime}}.each do |sum| it "should not consider #{sum} to be a checksum" do @summer.should_not be_checksum(sum) end end it "should have a method for stripping a sum type from an existing checksum" do @summer.sumtype("{md5}asdfasdfa").should == "md5" end + it "should have a method for stripping the data from a checksum" do + @summer.sumdata("{md5}asdfasdfa").should == "asdfasdfa" + end + it "should return a nil sumtype if the checksum does not mention a checksum type" do @summer.sumtype("asdfasdfa").should be_nil end {:md5 => Digest::MD5, :sha1 => Digest::SHA1}.each do |sum, klass| describe("when using %s" % sum) do it "should use #{klass} to calculate string checksums" do klass.expects(:hexdigest).with("mycontent").returns "whatever" @summer.send(sum, "mycontent").should == "whatever" end it "should use incremental #{klass} sums to calculate file checksums" do digest = mock 'digest' klass.expects(:new).returns digest file = "/path/to/my/file" fh = mock 'filehandle' fh.expects(:read).with(4096).times(3).returns("firstline").then.returns("secondline").then.returns(nil) #fh.expects(:read).with(512).returns("secondline") #fh.expects(:read).with(512).returns(nil) File.expects(:open).with(file, "r").yields(fh) digest.expects(:<<).with "firstline" digest.expects(:<<).with "secondline" digest.expects(:hexdigest).returns :mydigest @summer.send(sum.to_s + "_file", file).should == :mydigest end it "should yield #{klass} to the given block to calculate stream checksums" do digest = mock 'digest' klass.expects(:new).returns digest digest.expects(:hexdigest).returns :mydigest @summer.send(sum.to_s + "_stream") do |sum| sum.should == digest end.should == :mydigest end end end {:md5lite => Digest::MD5, :sha1lite => Digest::SHA1}.each do |sum, klass| describe("when using %s" % sum) do it "should use #{klass} to calculate string checksums from the first 512 characters of the string" do content = "this is a test" * 100 klass.expects(:hexdigest).with(content[0..511]).returns "whatever" @summer.send(sum, content).should == "whatever" end it "should use #{klass} to calculate a sum from the first 512 characters in the file" do digest = mock 'digest' klass.expects(:new).returns digest file = "/path/to/my/file" fh = mock 'filehandle' fh.expects(:read).with(512).returns('my content') File.expects(:open).with(file, "r").yields(fh) digest.expects(:<<).with "my content" digest.expects(:hexdigest).returns :mydigest @summer.send(sum.to_s + "_file", file).should == :mydigest end end end [:ctime, :mtime].each do |sum| describe("when using %s" % sum) do it "should use the '#{sum}' on the file to determine the ctime" do file = "/my/file" stat = mock 'stat', sum => "mysum" File.expects(:stat).with(file).returns(stat) @summer.send(sum.to_s + "_file", file).should == "mysum" end it "should return nil for streams" do @summer.send(sum.to_s + "_stream").should be_nil end end end describe "when using the none checksum" do it "should return an empty string" do @summer.none_file("/my/file").should == "" end end end