diff --git a/lib/puppet/file_bucket/file.rb b/lib/puppet/file_bucket/file.rb index 9a2e71da6..bdf7f12db 100644 --- a/lib/puppet/file_bucket/file.rb +++ b/lib/puppet/file_bucket/file.rb @@ -1,145 +1,148 @@ require 'puppet/file_bucket' require 'puppet/indirector' require 'puppet/util/checksums' require 'digest/md5' require 'stringio' class Puppet::FileBucket::File # This class handles the abstract notion of a file in a filebucket. # There are mechanisms to save and load this file locally and remotely in puppet/indirector/filebucketfile/* # There is a compatibility class that emulates pre-indirector filebuckets in Puppet::FileBucket::Dipper extend Puppet::Indirector indirects :file_bucket_file, :terminus_class => :selector attr :bucket_path def self.supported_formats - [:s] + [:binary] end def self.default_format - # This should really be :raw, like is done for Puppet::FileServing::Content - # but this class hasn't historically supported `from_raw`, so switching - # would break compatibility between newer 3.x agents talking to older 3.x - # masters. However, to/from_s has been supported and achieves the desired - # result without breaking compatibility. - :s + :binary end def initialize(contents, options = {}) case contents when String @contents = StringContents.new(contents) when Pathname @contents = FileContents.new(contents) else raise ArgumentError.new("contents must be a String or Pathname, got a #{contents.class}") end @bucket_path = options.delete(:bucket_path) @checksum_type = Puppet[:digest_algorithm].to_sym raise ArgumentError.new("Unknown option(s): #{options.keys.join(', ')}") unless options.empty? end # @return [Num] The size of the contents def size @contents.size() end # @return [IO] A stream that reads the contents def stream(&block) @contents.stream(&block) end def checksum_type @checksum_type.to_s end def checksum "{#{checksum_type}}#{checksum_data}" end def checksum_data @checksum_data ||= @contents.checksum_data(@checksum_type) end def to_s @contents.to_s end + def to_binary + @contents.to_s + end + def contents to_s end def name "#{checksum_type}/#{checksum_data}" end def self.from_s(contents) self.new(contents) end + def self.from_binary(contents) + self.new(contents) + end + def to_data_hash # Note that this serializes the entire data to a string and places it in a hash. { "contents" => contents.to_s } end def self.from_data_hash(data) self.new(data["contents"]) end private class StringContents def initialize(content) @contents = content; end def stream(&block) s = StringIO.new(@contents) begin block.call(s) ensure s.close end end def size @contents.size end def checksum_data(base_method) Puppet.info("Computing checksum on string") Puppet::Util::Checksums.method(base_method).call(@contents) end def to_s # This is not so horrible as for FileContent, but still possible to mutate the content that the # checksum is based on... so semi horrible... return @contents; end end class FileContents def initialize(path) @path = path end def stream(&block) Puppet::FileSystem.open(@path, nil, 'rb', &block) end def size Puppet::FileSystem.size(@path) end def checksum_data(base_method) Puppet.info("Computing checksum on file #{@path}") Puppet::Util::Checksums.method(:"#{base_method}_file").call(@path) end def to_s Puppet::FileSystem::binread(@path) end end end diff --git a/lib/puppet/network/formats.rb b/lib/puppet/network/formats.rb index a14b87819..f482834ab 100644 --- a/lib/puppet/network/formats.rb +++ b/lib/puppet/network/formats.rb @@ -1,161 +1,163 @@ require 'puppet/network/format_handler' Puppet::Network::FormatHandler.create_serialized_formats(:msgpack, :weight => 20, :mime => "application/x-msgpack", :required_methods => [:render_method, :intern_method], :intern_method => :from_data_hash) do confine :feature => :msgpack def intern(klass, text) data = MessagePack.unpack(text) return data if data.is_a?(klass) klass.from_data_hash(data) end def intern_multiple(klass, text) MessagePack.unpack(text).collect do |data| klass.from_data_hash(data) end end def render_multiple(instances) instances.to_msgpack end end Puppet::Network::FormatHandler.create_serialized_formats(:yaml) do def intern(klass, text) data = YAML.load(text) data_to_instance(klass, data) end def intern_multiple(klass, text) data = YAML.load(text) unless data.respond_to?(:collect) raise Puppet::Network::FormatHandler::FormatError, "Serialized YAML did not contain a collection of instances when calling intern_multiple" end data.collect do |datum| data_to_instance(klass, datum) end end def data_to_instance(klass, data) return data if data.is_a?(klass) unless data.is_a? Hash raise Puppet::Network::FormatHandler::FormatError, "Serialized YAML did not contain a valid instance of #{klass}" end klass.from_data_hash(data) end def render(instance) instance.to_yaml end # Yaml monkey-patches Array, so this works. def render_multiple(instances) instances.to_yaml end def supported?(klass) true end end Puppet::Network::FormatHandler.create(:s, :mime => "text/plain", :extension => "txt") +Puppet::Network::FormatHandler.create(:binary, :mime => "application/octet-stream") + # A very low-weight format so it'll never get chosen automatically. Puppet::Network::FormatHandler.create(:raw, :mime => "application/x-raw", :weight => 1) do def intern_multiple(klass, text) raise NotImplementedError end def render_multiple(instances) raise NotImplementedError end # LAK:NOTE The format system isn't currently flexible enough to handle # what I need to support raw formats just for individual instances (rather # than both individual and collections), but we don't yet have enough data # to make a "correct" design. # So, we hack it so it works for singular but fail if someone tries it # on plurals. def supported?(klass) true end end Puppet::Network::FormatHandler.create_serialized_formats(:pson, :weight => 10, :required_methods => [:render_method, :intern_method], :intern_method => :from_data_hash) do def intern(klass, text) data_to_instance(klass, PSON.parse(text)) end def intern_multiple(klass, text) PSON.parse(text).collect do |data| data_to_instance(klass, data) end end # PSON monkey-patches Array, so this works. def render_multiple(instances) instances.to_pson end # If they pass class information, we want to ignore it. # This is required for compatibility with Puppet 3.x def data_to_instance(klass, data) if data.is_a?(Hash) and d = data['data'] data = d end return data if data.is_a?(klass) klass.from_data_hash(data) end end # This is really only ever going to be used for Catalogs. Puppet::Network::FormatHandler.create_serialized_formats(:dot, :required_methods => [:render_method]) Puppet::Network::FormatHandler.create(:console, :mime => 'text/x-console-text', :weight => 0) do def json @json ||= Puppet::Network::FormatHandler.format(:pson) end def render(datum) # String to String return datum if datum.is_a? String return datum if datum.is_a? Numeric # Simple hash to table if datum.is_a? Hash and datum.keys.all? { |x| x.is_a? String or x.is_a? Numeric } output = '' column_a = datum.empty? ? 2 : datum.map{ |k,v| k.to_s.length }.max + 2 datum.sort_by { |k,v| k.to_s } .each do |key, value| output << key.to_s.ljust(column_a) output << json.render(value). chomp.gsub(/\n */) { |x| x + (' ' * column_a) } output << "\n" end return output end # Print one item per line for arrays if datum.is_a? Array output = '' datum.each do |item| output << item.to_s output << "\n" end return output end # ...or pretty-print the inspect outcome. return json.render(datum) end def render_multiple(data) data.collect(&:render).join("\n") end end diff --git a/spec/unit/file_bucket/file_spec.rb b/spec/unit/file_bucket/file_spec.rb index c3033a505..13aafd050 100755 --- a/spec/unit/file_bucket/file_spec.rb +++ b/spec/unit/file_bucket/file_spec.rb @@ -1,60 +1,60 @@ #! /usr/bin/env ruby require 'spec_helper' require 'puppet/file_bucket/file' describe Puppet::FileBucket::File, :uses_checksums => true do include PuppetSpec::Files # this is the default from spec_helper, but it keeps getting reset at odd times let(:bucketdir) { Puppet[:bucketdir] = tmpdir('bucket') } - it "defaults to serializing to `:s`" do - expect(Puppet::FileBucket::File.default_format).to eq(:s) + it "defaults to serializing to `:binary`" do + expect(Puppet::FileBucket::File.default_format).to eq(:binary) end it "accepts s" do - expect(Puppet::FileBucket::File.supported_formats).to include(:s) + expect(Puppet::FileBucket::File.supported_formats).to include(:binary) end describe "making round trips through network formats" do with_digest_algorithms do it "can make a round trip through `s`" do file = Puppet::FileBucket::File.new(plaintext) - tripped = Puppet::FileBucket::File.convert_from(:s, file.render) + tripped = Puppet::FileBucket::File.convert_from(:binary, file.render) expect(tripped.contents).to eq(plaintext) end end end it "should require contents to be a string" do expect { Puppet::FileBucket::File.new(5) }.to raise_error(ArgumentError, /contents must be a String or Pathname, got a Fixnum$/) end it "should complain about options other than :bucket_path" do expect { Puppet::FileBucket::File.new('5', :crazy_option => 'should not be passed') }.to raise_error(ArgumentError, /Unknown option\(s\): crazy_option/) end with_digest_algorithms do it "it uses #{metadata[:digest_algorithm]} as the configured digest algorithm" do file = Puppet::FileBucket::File.new(plaintext) expect(file.contents).to eq(plaintext) expect(file.checksum_type).to eq(digest_algorithm) expect(file.checksum).to eq("{#{digest_algorithm}}#{checksum}") expect(file.name).to eq("#{digest_algorithm}/#{checksum}") end end describe "when using back-ends" do it "should redirect using Puppet::Indirector" do expect(Puppet::Indirector::Indirection.instance(:file_bucket_file).model).to equal(Puppet::FileBucket::File) end it "should have a :save instance method" do expect(Puppet::FileBucket::File.indirection).to respond_to(:save) end end end