diff --git a/lib/puppet/indirector/file_bucket_file/file.rb b/lib/puppet/indirector/file_bucket_file/file.rb index 0fd8a914f..d32788a0c 100644 --- a/lib/puppet/indirector/file_bucket_file/file.rb +++ b/lib/puppet/indirector/file_bucket_file/file.rb @@ -1,135 +1,136 @@ require 'puppet/indirector/code' require 'puppet/file_bucket/file' require 'puppet/util/checksums' require 'fileutils' module Puppet::FileBucketFile class File < Puppet::Indirector::Code include Puppet::Util::Checksums desc "Store files in a directory set based on their checksums." def initialize Puppet.settings.use(:filebucket) end def find( request ) checksum, files_original_path = request_to_checksum_and_path( request ) dir_path = path_for(request.options[:bucket_path], checksum) file_path = ::File.join(dir_path, 'contents') return nil unless ::File.exists?(file_path) return nil unless path_match(dir_path, files_original_path) if request.options[:diff_with] hash_protocol = sumtype(checksum) file2_path = path_for(request.options[:bucket_path], request.options[:diff_with], 'contents') raise "could not find diff_with #{request.options[:diff_with]}" unless ::File.exists?(file2_path) return `diff #{file_path.inspect} #{file2_path.inspect}` else - contents = ::File.read file_path + contents = Puppet::Util.binread(file_path) Puppet.info "FileBucket read #{checksum}" model.new(contents) end end def head(request) checksum, files_original_path = request_to_checksum_and_path(request) dir_path = path_for(request.options[:bucket_path], checksum) ::File.exists?(::File.join(dir_path, 'contents')) and path_match(dir_path, files_original_path) end def save( request ) instance = request.instance checksum, files_original_path = request_to_checksum_and_path(request) save_to_disk(instance, files_original_path) instance.to_s end private def path_match(dir_path, files_original_path) return true unless files_original_path # if no path was provided, it's a match paths_path = ::File.join(dir_path, 'paths') return false unless ::File.exists?(paths_path) ::File.open(paths_path) do |f| f.each do |line| return true if line.chomp == files_original_path end end return false end def save_to_disk( bucket_file, files_original_path ) filename = path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents') dir_path = path_for(bucket_file.bucket_path, bucket_file.checksum_data) paths_path = ::File.join(dir_path, 'paths') # If the file already exists, do nothing. if ::File.exist?(filename) verify_identical_file!(bucket_file) else # Make the directories if necessary. unless ::File.directory?(dir_path) Puppet::Util.withumask(0007) do ::FileUtils.mkdir_p(dir_path) end end Puppet.info "FileBucket adding #{bucket_file.checksum}" # Write the file to disk. Puppet::Util.withumask(0007) do ::File.open(filename, ::File::WRONLY|::File::CREAT, 0440) do |of| + of.binmode of.print bucket_file.contents end ::File.open(paths_path, ::File::WRONLY|::File::CREAT, 0640) do |of| # path will be written below end end end unless path_match(dir_path, files_original_path) ::File.open(paths_path, 'a') do |f| f.puts(files_original_path) end end end def request_to_checksum_and_path( request ) checksum_type, checksum, path = request.key.split(/\//, 3) if path == '' # Treat "md5//" like "md5/" path = nil end raise "Unsupported checksum type #{checksum_type.inspect}" if checksum_type != 'md5' raise "Invalid checksum #{checksum.inspect}" if checksum !~ /^[0-9a-f]{32}$/ [checksum, path] end def path_for(bucket_path, digest, subfile = nil) bucket_path ||= Puppet[:bucketdir] dir = ::File.join(digest[0..7].split("")) basedir = ::File.join(bucket_path, dir, digest) return basedir unless subfile ::File.join(basedir, subfile) end # If conflict_check is enabled, verify that the passed text is # the same as the text in our file. def verify_identical_file!(bucket_file) - disk_contents = ::File.read(path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents')) + disk_contents = Puppet::Util.binread(path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents')) # If the contents don't match, then we've found a conflict. # Unlikely, but quite bad. if disk_contents != bucket_file.contents raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}" else Puppet.info "FileBucket got a duplicate file #{bucket_file.checksum}" end end end end diff --git a/spec/unit/file_bucket/file_spec.rb b/spec/unit/file_bucket/file_spec.rb index ebf02438c..27579647a 100755 --- a/spec/unit/file_bucket/file_spec.rb +++ b/spec/unit/file_bucket/file_spec.rb @@ -1,112 +1,106 @@ #!/usr/bin/env rspec require 'spec_helper' require 'puppet/file_bucket/file' require 'digest/md5' require 'digest/sha1' describe Puppet::FileBucket::File do include PuppetSpec::Files - before do - # this is the default from spec_helper, but it keeps getting reset at odd times - @bucketdir = tmpdir('bucket') - Puppet[:bucketdir] = @bucketdir - - @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @checksum = "{md5}4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @dir = File.join(@bucketdir, '4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0') - - @contents = "file contents" - end + let(:contents) { "file\r\n contents" } + let(:digest) { "8b3702ad1aed1ace7e32bde76ffffb2d" } + let(:checksum) { "{md5}#{digest}" } + # this is the default from spec_helper, but it keeps getting reset at odd times + let(:bucketdir) { Puppet[:bucketdir] = tmpdir('bucket') } + let(:destdir) { File.join(bucketdir, "8/b/3/7/0/2/a/d/#{digest}") } it "should have a to_s method to return the contents" do - Puppet::FileBucket::File.new(@contents).to_s.should == @contents + Puppet::FileBucket::File.new(contents).to_s.should == contents end it "should raise an error if changing content" do x = Puppet::FileBucket::File.new("first") expect { x.contents = "new" }.to raise_error(NoMethodError, /undefined method .contents=/) end it "should require contents to be a string" do expect { Puppet::FileBucket::File.new(5) }.to raise_error(ArgumentError, /contents must be a String, got a Fixnum$/) end it "should complain about options other than :bucket_path" do expect { Puppet::FileBucket::File.new('5', :crazy_option => 'should not be passed') }.to raise_error(ArgumentError, /Unknown option\(s\): crazy_option/) end it "should set the contents appropriately" do - Puppet::FileBucket::File.new(@contents).contents.should == @contents + Puppet::FileBucket::File.new(contents).contents.should == contents end it "should default to 'md5' as the checksum algorithm if the algorithm is not in the name" do - Puppet::FileBucket::File.new(@contents).checksum_type.should == "md5" + Puppet::FileBucket::File.new(contents).checksum_type.should == "md5" end it "should calculate the checksum" do - Puppet::FileBucket::File.new(@contents).checksum.should == @checksum + Puppet::FileBucket::File.new(contents).checksum.should == checksum end describe "when using back-ends" do it "should redirect using Puppet::Indirector" do Puppet::Indirector::Indirection.instance(:file_bucket_file).model.should equal(Puppet::FileBucket::File) end it "should have a :save instance method" do Puppet::FileBucket::File.indirection.should respond_to(:save) end end it "should return a url-ish name" do - Puppet::FileBucket::File.new(@contents).name.should == "md5/4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + Puppet::FileBucket::File.new(contents).name.should == "md5/#{digest}" end it "should reject a url-ish name with an invalid checksum" do - bucket = Puppet::FileBucket::File.new(@contents) - expect { bucket.name = "sha1/4a8ec4fa5f01b4ab1a0ab8cbccb709f0/new/path" }.to raise_error(NoMethodError, /undefined method .name=/) + bucket = Puppet::FileBucket::File.new(contents) + expect { bucket.name = "sha1/ae548c0cd614fb7885aaa0b6cb191c34/new/path" }.to raise_error(NoMethodError, /undefined method .name=/) end it "should convert the contents to PSON" do - Puppet::FileBucket::File.new(@contents).to_pson.should == '{"contents":"file contents"}' + Puppet::FileBucket::File.new("file contents").to_pson.should == '{"contents":"file contents"}' end it "should load from PSON" do Puppet::FileBucket::File.from_pson({"contents"=>"file contents"}).contents.should == "file contents" end def make_bucketed_file - FileUtils.mkdir_p(@dir) - File.open("#{@dir}/contents", 'w') { |f| f.write @contents } + FileUtils.mkdir_p(destdir) + File.open("#{destdir}/contents", 'wb') { |f| f.write contents } end describe "using the indirector's find method" do it "should return nil if a file doesn't exist" do - bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{@digest}") + bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{digest}") bucketfile.should == nil end it "should find a filebucket if the file exists" do make_bucketed_file - bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{@digest}") - bucketfile.should_not == nil + bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{digest}") + bucketfile.checksum.should == checksum end describe "using RESTish digest notation" do it "should return nil if a file doesn't exist" do - bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{@digest}") + bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{digest}") bucketfile.should == nil end it "should find a filebucket if the file exists" do make_bucketed_file - bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{@digest}") - bucketfile.should_not == nil + bucketfile = Puppet::FileBucket::File.indirection.find("md5/#{digest}") + bucketfile.checksum.should == checksum end - end end end diff --git a/spec/unit/indirector/file_bucket_file/file_spec.rb b/spec/unit/indirector/file_bucket_file/file_spec.rb index 808da17d8..9141221bb 100755 --- a/spec/unit/indirector/file_bucket_file/file_spec.rb +++ b/spec/unit/indirector/file_bucket_file/file_spec.rb @@ -1,273 +1,270 @@ #!/usr/bin/env rspec require 'spec_helper' require 'puppet/indirector/file_bucket_file/file' describe Puppet::FileBucketFile::File do include PuppetSpec::Files it "should be a subclass of the Code terminus class" do Puppet::FileBucketFile::File.superclass.should equal(Puppet::Indirector::Code) end it "should have documentation" do Puppet::FileBucketFile::File.doc.should be_instance_of(String) end describe "non-stubbing tests" do include PuppetSpec::Files before do Puppet[:bucketdir] = tmpdir('bucketdir') end def save_bucket_file(contents, path = "/who_cares") bucket_file = Puppet::FileBucket::File.new(contents) Puppet::FileBucket::File.indirection.save(bucket_file, "md5/#{Digest::MD5.hexdigest(contents)}#{path}") bucket_file.checksum_data end describe "when servicing a save request" do describe "when supplying a path" do it "should store the path if not already stored" do - checksum = save_bucket_file("stuff", "/foo/bar") - dir_path = "#{Puppet[:bucketdir]}/c/1/3/d/8/8/c/b/c13d88cb4cb02003daedb8a84e5d272a" - File.read("#{dir_path}/contents").should == "stuff" + checksum = save_bucket_file("stuff\r\n", "/foo/bar") + dir_path = "#{Puppet[:bucketdir]}/f/c/7/7/7/c/0/b/fc777c0bc467e1ab98b4c6915af802ec" + Puppet::Util.binread("#{dir_path}/contents").should == "stuff\r\n" File.read("#{dir_path}/paths").should == "foo/bar\n" end it "should leave the paths file alone if the path is already stored" do checksum = save_bucket_file("stuff", "/foo/bar") checksum = save_bucket_file("stuff", "/foo/bar") dir_path = "#{Puppet[:bucketdir]}/c/1/3/d/8/8/c/b/c13d88cb4cb02003daedb8a84e5d272a" File.read("#{dir_path}/contents").should == "stuff" File.read("#{dir_path}/paths").should == "foo/bar\n" end it "should store an additional path if the new path differs from those already stored" do checksum = save_bucket_file("stuff", "/foo/bar") checksum = save_bucket_file("stuff", "/foo/baz") dir_path = "#{Puppet[:bucketdir]}/c/1/3/d/8/8/c/b/c13d88cb4cb02003daedb8a84e5d272a" File.read("#{dir_path}/contents").should == "stuff" File.read("#{dir_path}/paths").should == "foo/bar\nfoo/baz\n" end end describe "when not supplying a path" do it "should save the file and create an empty paths file" do checksum = save_bucket_file("stuff", "") dir_path = "#{Puppet[:bucketdir]}/c/1/3/d/8/8/c/b/c13d88cb4cb02003daedb8a84e5d272a" File.read("#{dir_path}/contents").should == "stuff" File.read("#{dir_path}/paths").should == "" end end end describe "when servicing a head/find request" do describe "when supplying a path" do it "should return false/nil if the file isn't bucketed" do Puppet::FileBucket::File.indirection.head("md5/0ae2ec1980410229885fe72f7b44fe55/foo/bar").should == false Puppet::FileBucket::File.indirection.find("md5/0ae2ec1980410229885fe72f7b44fe55/foo/bar").should == nil end it "should return false/nil if the file is bucketed but with a different path" do checksum = save_bucket_file("I'm the contents of a file", '/foo/bar') Puppet::FileBucket::File.indirection.head("md5/#{checksum}/foo/baz").should == false Puppet::FileBucket::File.indirection.find("md5/#{checksum}/foo/baz").should == nil end it "should return true/file if the file is already bucketed with the given path" do contents = "I'm the contents of a file" checksum = save_bucket_file(contents, '/foo/bar') Puppet::FileBucket::File.indirection.head("md5/#{checksum}/foo/bar").should == true find_result = Puppet::FileBucket::File.indirection.find("md5/#{checksum}/foo/bar") find_result.should be_a(Puppet::FileBucket::File) find_result.checksum.should == "{md5}#{checksum}" find_result.to_s.should == contents end end describe "when not supplying a path" do [false, true].each do |trailing_slash| describe "#{trailing_slash ? 'with' : 'without'} a trailing slash" do trailing_string = trailing_slash ? '/' : '' it "should return false/nil if the file isn't bucketed" do Puppet::FileBucket::File.indirection.head("md5/0ae2ec1980410229885fe72f7b44fe55#{trailing_string}").should == false Puppet::FileBucket::File.indirection.find("md5/0ae2ec1980410229885fe72f7b44fe55#{trailing_string}").should == nil end it "should return true/file if the file is already bucketed" do contents = "I'm the contents of a file" checksum = save_bucket_file(contents, '/foo/bar') Puppet::FileBucket::File.indirection.head("md5/#{checksum}#{trailing_string}").should == true find_result = Puppet::FileBucket::File.indirection.find("md5/#{checksum}#{trailing_string}") find_result.should be_a(Puppet::FileBucket::File) find_result.checksum.should == "{md5}#{checksum}" find_result.to_s.should == contents end end end end end describe "when diffing files", :unless => Puppet.features.microsoft_windows? do it "should generate an empty string if there is no diff" do checksum = save_bucket_file("I'm the contents of a file") Puppet::FileBucket::File.indirection.find("md5/#{checksum}", :diff_with => checksum).should == '' end it "should generate a proper diff if there is a diff" do checksum1 = save_bucket_file("foo\nbar\nbaz") checksum2 = save_bucket_file("foo\nbiz\nbaz") diff = Puppet::FileBucket::File.indirection.find("md5/#{checksum1}", :diff_with => checksum2) diff.should == < biz HERE end it "should raise an exception if the hash to diff against isn't found" do checksum = save_bucket_file("whatever") bogus_checksum = "d1bf072d0e2c6e20e3fbd23f022089a1" lambda { Puppet::FileBucket::File.indirection.find("md5/#{checksum}", :diff_with => bogus_checksum) }.should raise_error "could not find diff_with #{bogus_checksum}" end it "should return nil if the hash to diff from isn't found" do checksum = save_bucket_file("whatever") bogus_checksum = "d1bf072d0e2c6e20e3fbd23f022089a1" Puppet::FileBucket::File.indirection.find("md5/#{bogus_checksum}", :diff_with => checksum).should == nil end end end describe "when initializing" do it "should use the filebucket settings section" do Puppet.settings.expects(:use).with(:filebucket) Puppet::FileBucketFile::File.new end end [true, false].each do |override_bucket_path| describe "when bucket path #{if override_bucket_path then 'is' else 'is not' end} overridden" do [true, false].each do |supply_path| describe "when #{supply_path ? 'supplying' : 'not supplying'} a path" do before :each do Puppet.settings.stubs(:use) @store = Puppet::FileBucketFile::File.new @contents = "my content" @digest = "f2bfa7fc155c4f42cb91404198dda01f" @digest.should == Digest::MD5.hexdigest(@contents) @bucket_dir = tmpdir("bucket") if override_bucket_path Puppet[:bucketdir] = "/bogus/path" # should not be used else Puppet[:bucketdir] = @bucket_dir end @dir = "#{@bucket_dir}/f/2/b/f/a/7/f/c/f2bfa7fc155c4f42cb91404198dda01f" @contents_path = "#{@dir}/contents" end describe "when retrieving files" do before :each do request_options = {} if override_bucket_path request_options[:bucket_path] = @bucket_dir end key = "md5/#{@digest}" if supply_path key += "/path/to/file" end @request = Puppet::Indirector::Request.new(:indirection_name, :find, key, request_options) end def make_bucketed_file FileUtils.mkdir_p(@dir) File.open(@contents_path, 'w') { |f| f.write @contents } end it "should return an instance of Puppet::FileBucket::File created with the content if the file exists" do make_bucketed_file if supply_path @store.find(@request).should == nil @store.head(@request).should == false # because path didn't match else bucketfile = @store.find(@request) bucketfile.should be_a(Puppet::FileBucket::File) bucketfile.contents.should == @contents @store.head(@request).should == true end end it "should return nil if no file is found" do @store.find(@request).should be_nil @store.head(@request).should == false end end describe "when saving files" do it "should save the contents to the calculated path" do options = {} if override_bucket_path options[:bucket_path] = @bucket_dir end key = "md5/#{@digest}" if supply_path key += "//path/to/file" end file_instance = Puppet::FileBucket::File.new(@contents, options) request = Puppet::Indirector::Request.new(:indirection_name, :save, key, file_instance) @store.save(request) File.read("#{@dir}/contents").should == @contents end end end end end end describe "when verifying identical files" do - before do - # this is the default from spec_helper, but it keeps getting reset at odd times - Puppet[:bucketdir] = make_absolute("/dev/null/bucket") - - @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @checksum = "{md5}4a8ec4fa5f01b4ab1a0ab8cbccb709f0" - @dir = make_absolute('/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0') - - @contents = "file contents" - - @bucket = stub "bucket file" - @bucket.stubs(:bucket_path) - @bucket.stubs(:checksum).returns(@checksum) - @bucket.stubs(:checksum_data).returns(@digest) - @bucket.stubs(:path).returns(nil) - @bucket.stubs(:contents).returns("file contents") + let(:contents) { "file\r\n contents" } + let(:digest) { "8b3702ad1aed1ace7e32bde76ffffb2d" } + let(:checksum) { "{md5}#{@digest}" } + let(:bucketdir) { tmpdir('file_bucket_file') } + let(:destdir) { "#{bucketdir}/8/b/3/7/0/2/a/d/8b3702ad1aed1ace7e32bde76ffffb2d" } + let(:bucket) { Puppet::FileBucket::File.new(contents) } + + before :each do + Puppet[:bucketdir] = bucketdir + FileUtils.mkdir_p(destdir) end it "should raise an error if the files don't match" do - File.expects(:read).with("#{@dir}/contents").returns("corrupt contents") - lambda{ Puppet::FileBucketFile::File.new.send(:verify_identical_file!, @bucket) }.should raise_error(Puppet::FileBucket::BucketError) + File.open(File.join(destdir, 'contents'), 'wb') { |f| f.print "corrupt contents" } + + lambda{ + Puppet::FileBucketFile::File.new.send(:verify_identical_file!, bucket) + }.should raise_error(Puppet::FileBucket::BucketError) end it "should do nothing if the files match" do - File.expects(:read).with("#{@dir}/contents").returns("file contents") - Puppet::FileBucketFile::File.new.send(:verify_identical_file!, @bucket) - end + File.open(File.join(destdir, 'contents'), 'wb') { |f| f.print contents } + Puppet::FileBucketFile::File.new.send(:verify_identical_file!, bucket) + end end end