diff --git a/lib/puppet/file_bucket/dipper.rb b/lib/puppet/file_bucket/dipper.rb index 870c50eec..29ab9f090 100644 --- a/lib/puppet/file_bucket/dipper.rb +++ b/lib/puppet/file_bucket/dipper.rb @@ -1,106 +1,107 @@ require 'puppet/file_bucket' require 'puppet/file_bucket/file' require 'puppet/indirector/request' class Puppet::FileBucket::Dipper # This is a transitional implementation that uses REST # to access remote filebucket files. attr_accessor :name # Create our bucket client def initialize(hash = {}) # Emulate the XMLRPC client server = hash[:Server] port = hash[:Port] || Puppet[:masterport] environment = Puppet[:environment] if hash.include?(:Path) @local_path = hash[:Path] @rest_path = nil else @local_path = nil @rest_path = "https://#{server}:#{port}/#{environment}/file_bucket_file/" end end def local? !! @local_path end # Back up a file to our bucket def backup(file) raise(ArgumentError, "File #{file} does not exist") unless ::File.exist?(file) contents = ::File.read(file) begin file_bucket_file = Puppet::FileBucket::File.new(contents, :bucket_path => @local_path) files_original_path = absolutize_path(file) dest_path = "#{@rest_path}#{file_bucket_file.name}/#{files_original_path}" file_bucket_path = "#{@rest_path}#{file_bucket_file.checksum_type}/#{file_bucket_file.checksum_data}/#{files_original_path}" # Make a HEAD request for the file so that we don't waste time # uploading it if it already exists in the bucket. unless Puppet::FileBucket::File.indirection.head(file_bucket_path) Puppet::FileBucket::File.indirection.save(file_bucket_file, dest_path) end return file_bucket_file.checksum_data rescue => detail puts detail.backtrace if Puppet[:trace] raise Puppet::Error, "Could not back up #{file}: #{detail}" end end # Retrieve a file by sum. def getfile(sum) source_path = "#{@rest_path}md5/#{sum}" file_bucket_file = Puppet::FileBucket::File.indirection.find(source_path, :bucket_path => @local_path) raise Puppet::Error, "File not found" unless file_bucket_file file_bucket_file.to_s end # Restore the file def restore(file,sum) restore = true if FileTest.exists?(file) - cursum = Digest::MD5.hexdigest(::File.read(file)) + cursum = Digest::MD5.hexdigest(Puppet::Util.binread(file)) # if the checksum has changed... # this might be extra effort if cursum == sum restore = false end end if restore if newcontents = getfile(sum) tmp = "" newsum = Digest::MD5.hexdigest(newcontents) changed = nil if FileTest.exists?(file) and ! FileTest.writable?(file) changed = ::File.stat(file).mode ::File.chmod(changed | 0200, file) end ::File.open(file, ::File::WRONLY|::File::TRUNC|::File::CREAT) { |of| + of.binmode of.print(newcontents) } ::File.chmod(changed, file) if changed else Puppet.err "Could not find file with checksum #{sum}" return nil end return newsum else return nil end end private def absolutize_path( path ) require 'pathname' Pathname.new(path).realpath end end diff --git a/spec/unit/file_bucket/dipper_spec.rb b/spec/unit/file_bucket/dipper_spec.rb index e1d2efaff..f058fb244 100755 --- a/spec/unit/file_bucket/dipper_spec.rb +++ b/spec/unit/file_bucket/dipper_spec.rb @@ -1,113 +1,170 @@ #!/usr/bin/env rspec require 'spec_helper' require 'pathname' require 'puppet/file_bucket/dipper' require 'puppet/indirector/file_bucket_file/rest' describe Puppet::FileBucket::Dipper do include PuppetSpec::Files def make_tmp_file(contents) file = tmpfile("file_bucket_file") File.open(file, 'w') { |f| f.write(contents) } file end it "should fail in an informative way when there are failures checking for the file on the server" do @dipper = Puppet::FileBucket::Dipper.new(:Path => make_absolute("/my/bucket")) file = make_tmp_file('contents') Puppet::FileBucket::File.indirection.expects(:head).raises ArgumentError lambda { @dipper.backup(file) }.should raise_error(Puppet::Error) end it "should fail in an informative way when there are failures backing up to the server" do @dipper = Puppet::FileBucket::Dipper.new(:Path => make_absolute("/my/bucket")) file = make_tmp_file('contents') Puppet::FileBucket::File.indirection.expects(:head).returns false Puppet::FileBucket::File.indirection.expects(:save).raises ArgumentError lambda { @dipper.backup(file) }.should raise_error(Puppet::Error) end it "should backup files to a local bucket" do Puppet[:bucketdir] = "/non/existent/directory" file_bucket = tmpdir("bucket") @dipper = Puppet::FileBucket::Dipper.new(:Path => file_bucket) file = make_tmp_file('my contents') checksum = "2975f560750e71c478b8e3b39a956adb" Digest::MD5.hexdigest('my contents').should == checksum @dipper.backup(file).should == checksum File.exists?("#{file_bucket}/2/9/7/5/f/5/6/0/2975f560750e71c478b8e3b39a956adb/contents").should == true end it "should not backup a file that is already in the bucket" do @dipper = Puppet::FileBucket::Dipper.new(:Path => "/my/bucket") file = make_tmp_file('my contents') checksum = Digest::MD5.hexdigest('my contents') Puppet::FileBucket::File.indirection.expects(:head).returns true Puppet::FileBucket::File.indirection.expects(:save).never @dipper.backup(file).should == checksum end it "should retrieve files from a local bucket" do @dipper = Puppet::FileBucket::Dipper.new(:Path => "/my/bucket") checksum = Digest::MD5.hexdigest('my contents') request = nil Puppet::FileBucketFile::File.any_instance.expects(:find).with{ |r| request = r }.once.returns(Puppet::FileBucket::File.new('my contents')) @dipper.getfile(checksum).should == 'my contents' request.key.should == "md5/#{checksum}" end it "should backup files to a remote server" do @dipper = Puppet::FileBucket::Dipper.new(:Server => "puppetmaster", :Port => "31337") file = make_tmp_file('my contents') checksum = Digest::MD5.hexdigest('my contents') real_path = Pathname.new(file).realpath request1 = nil request2 = nil Puppet::FileBucketFile::Rest.any_instance.expects(:head).with { |r| request1 = r }.once.returns(nil) Puppet::FileBucketFile::Rest.any_instance.expects(:save).with { |r| request2 = r }.once @dipper.backup(file).should == checksum [request1, request2].each do |r| r.server.should == 'puppetmaster' r.port.should == 31337 r.key.should == "md5/#{checksum}/#{real_path}" end end it "should retrieve files from a remote server" do @dipper = Puppet::FileBucket::Dipper.new(:Server => "puppetmaster", :Port => "31337") checksum = Digest::MD5.hexdigest('my contents') request = nil Puppet::FileBucketFile::Rest.any_instance.expects(:find).with { |r| request = r }.returns(Puppet::FileBucket::File.new('my contents')) @dipper.getfile(checksum).should == "my contents" request.server.should == 'puppetmaster' request.port.should == 31337 request.key.should == "md5/#{checksum}" end + + describe "#restore" do + shared_examples_for "a restorable file" do + let(:contents) { "my\ncontents" } + let(:md5) { Digest::MD5.hexdigest(contents) } + let(:dest) { tmpfile('file_bucket_dest') } + + it "should restore the file" do + request = nil + + klass.any_instance.expects(:find).with { |r| request = r }.returns(Puppet::FileBucket::File.new(contents)) + + dipper.restore(dest, md5).should == md5 + Digest::MD5.file(dest).hexdigest.should == md5 + + request.key.should == "md5/#{md5}" + request.server.should == server + request.port.should == port + end + + it "should skip restoring if existing file has the same checksum" do + crnl = "my\r\ncontents" + File.open(dest, 'wb') {|f| f.print(crnl) } + + dipper.expects(:getfile).never + dipper.restore(dest, Digest::MD5.hexdigest(crnl)).should be_nil + end + + it "should overwrite existing file if it has different checksum" do + klass.any_instance.expects(:find).returns(Puppet::FileBucket::File.new(contents)) + + File.open(dest, 'wb') {|f| f.print('other contents') } + + dipper.restore(dest, md5).should == md5 + end + end + + describe "when restoring from a remote server" do + let(:klass) { Puppet::FileBucketFile::Rest } + let(:server) { "puppetmaster" } + let(:port) { 31337 } + + it_behaves_like "a restorable file" do + let (:dipper) { Puppet::FileBucket::Dipper.new(:Server => server, :Port => port.to_s) } + end + end + + describe "when restoring from a local server" do + let(:klass) { Puppet::FileBucketFile::File } + let(:server) { nil } + let(:port) { nil } + + it_behaves_like "a restorable file" do + let (:dipper) { Puppet::FileBucket::Dipper.new(:Path => "/my/bucket") } + end + end + end end