diff --git a/lib/puppet/util/zaml.rb b/lib/puppet/util/zaml.rb
index dfd2236e6..445b10d13 100644
--- a/lib/puppet/util/zaml.rb
+++ b/lib/puppet/util/zaml.rb
@@ -1,384 +1,385 @@
# encoding: UTF-8
#
# The above encoding line is a magic comment to set the default source encoding
# of this file for the Ruby interpreter. It must be on the first or second
# line of the file if an interpreter is in use. In Ruby 1.9 and later, the
# source encoding determines the encoding of String and Regexp objects created
# from this source file. This explicit encoding is important becuase otherwise
# Ruby will pick an encoding based on LANG or LC_CTYPE environment variables.
# These may be different from site to site so it's important for us to
# establish a consistent behavior. For more information on M17n please see:
# http://links.puppetlabs.com/understanding_m17n
# ZAML -- A partial replacement for YAML, writen with speed and code clarity
# in mind. ZAML fixes one YAML bug (loading Exceptions) and provides
# a replacement for YAML.dump unimaginatively called ZAML.dump,
# which is faster on all known cases and an order of magnitude faster
# with complex structures.
#
# http://github.com/hallettj/zaml
#
# ## License (from upstream)
#
# Copyright (c) 2008-2009 ZAML contributers
#
# This program is dual-licensed under the GNU General Public License
# version 3 or later and under the Apache License, version 2.0.
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version; or under the terms of the Apache License,
# Version 2.0.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License and the Apache License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see
# .
#
# You may obtain a copy of the Apache License at
# .
require 'yaml'
class ZAML
VERSION = "0.1.3"
#
# Class Methods
#
def self.dump(stuff, where='')
z = new
stuff.to_zaml(z)
where << z.to_s
end
#
# Instance Methods
#
def initialize
@result = []
@indent = nil
@structured_key_prefix = nil
@previously_emitted_object = {}
@next_free_label_number = 0
emit('--- ')
end
def nested(tail=' ')
old_indent = @indent
@indent = "#{@indent || "\n"}#{tail}"
yield
@indent = old_indent
end
class Label
#
# YAML only wants objects in the datastream once; if the same object
# occurs more than once, we need to emit a label ("&idxxx") on the
# first occurrence and then emit a back reference (*idxxx") on any
# subsequent occurrence(s).
#
# To accomplish this we keeps a hash (by object id) of the labels of
# the things we serialize as we begin to serialize them. The labels
# initially serialize as an empty string (since most objects are only
# going to be be encountered once), but can be changed to a valid
# (by assigning it a number) the first time it is subsequently used,
# if it ever is. Note that we need to do the label setup BEFORE we
# start to serialize the object so that circular structures (in
# which we will encounter a reference to the object as we serialize
# it can be handled).
#
attr_accessor :this_label_number
def initialize(obj,indent)
@indent = indent
@this_label_number = nil
@obj = obj # prevent garbage collection so that object id isn't reused
end
def to_s
@this_label_number ? ('&id%03d%s' % [@this_label_number, @indent]) : ''
end
def reference
@reference ||= '*id%03d' % @this_label_number
end
end
def label_for(obj)
@previously_emitted_object[obj.object_id]
end
def new_label_for(obj)
label = Label.new(obj,(Hash === obj || Array === obj) ? "#{@indent || "\n"} " : ' ')
@previously_emitted_object[obj.object_id] = label
label
end
def first_time_only(obj)
if label = label_for(obj)
label.this_label_number ||= (@next_free_label_number += 1)
emit(label.reference)
else
if @structured_key_prefix and not obj.is_a? String
emit(@structured_key_prefix)
@structured_key_prefix = nil
end
emit(new_label_for(obj))
yield
end
end
def emit(s)
@result << s
@recent_nl = false unless s.kind_of?(Label)
end
def nl(s='')
emit(@indent || "\n") unless @recent_nl
emit(s)
@recent_nl = true
end
def to_s
@result.join
end
def prefix_structured_keys(x)
@structured_key_prefix = x
yield
nl unless @structured_key_prefix
@structured_key_prefix = nil
end
end
################################################################
#
# Behavior for custom classes
#
################################################################
class Object
def to_yaml_properties
instance_variables.sort # Default YAML behavior
end
def yaml_property_munge(x)
x
end
def zamlized_class_name(root)
cls = self.class
"!ruby/#{root.name.downcase}#{cls == root ? '' : ":#{cls.respond_to?(:name) ? cls.name : cls}"}"
end
def to_zaml(z)
z.first_time_only(self) {
z.emit(zamlized_class_name(Object))
z.nested {
instance_variables = to_yaml_properties
if instance_variables.empty?
z.emit(" {}")
else
instance_variables.each { |v|
z.nl
v[1..-1].to_zaml(z) # Remove leading '@'
z.emit(': ')
yaml_property_munge(instance_variable_get(v)).to_zaml(z)
}
end
}
}
end
end
################################################################
#
# Behavior for built-in classes
#
################################################################
class NilClass
def to_zaml(z)
z.emit('') # NOTE: blank turns into nil in YAML.load
end
end
class Symbol
def to_zaml(z)
z.emit(self.inspect)
end
end
class TrueClass
def to_zaml(z)
z.emit('true')
end
end
class FalseClass
def to_zaml(z)
z.emit('false')
end
end
class Numeric
def to_zaml(z)
z.emit(self)
end
end
class Regexp
def to_zaml(z)
z.first_time_only(self) { z.emit("#{zamlized_class_name(Regexp)} #{inspect}") }
end
end
class Exception
def to_zaml(z)
z.emit(zamlized_class_name(Exception))
z.nested {
z.nl("message: ")
message.to_zaml(z)
}
end
#
# Monkey patch for buggy Exception restore in YAML
#
# This makes it work for now but is not very future-proof; if things
# change we'll most likely want to remove this. To mitigate the risks
# as much as possible, we test for the bug before appling the patch.
#
if respond_to? :yaml_new and yaml_new(self, :tag, "message" => "blurp").message != "blurp"
def self.yaml_new( klass, tag, val )
o = YAML.object_maker( klass, {} ).exception(val.delete( 'message'))
val.each_pair do |k,v|
o.instance_variable_set("@#{k}", v)
end
o
end
end
end
class String
- ZAML_ESCAPES = %w{\x00 \x01 \x02 \x03 \x04 \x05 \x06 \a \x08 \t \n \v \f \r \x0e \x0f \x10 \x11 \x12 \x13 \x14 \x15 \x16 \x17 \x18 \x19 \x1a \e \x1c \x1d \x1e \x1f }
- def escaped_for_zaml
- # JJM (Note the trailing dots to construct a multi-line method chain.) This
- # code is meant to escape all bytes which are not ASCII-8BIT printable
- # characters. Multi-byte unicode characters are handled just fine because
- # each byte of the character results in an escaped string emitted to the
- # YAML stream. When the YAML is de-serialized back into a String the bytes
- # will be reconstructed properly into the unicode character.
- self.to_ascii8bit.gsub( /\x5C/n, "\\\\\\" ). # Demi-kludge for Maglev/rubinius; the regexp should be /\\/ but parsetree chokes on that.
- gsub( /"/n, "\\\"" ).
- gsub( /([\x00-\x1F])/n ) { |x| ZAML_ESCAPES[ x.unpack("C")[0] ] }.
- gsub( /([\x80-\xFF])/n ) { |x| "\\x#{x.unpack("C")[0].to_s(16)}" }
- end
+ ZAML_ESCAPES = {
+ "\a" => "\\a", "\e" => "\\e", "\f" => "\\f", "\n" => "\\n",
+ "\r" => "\\r", "\t" => "\\t", "\v" => "\\v"
+ }
+
def to_zaml(z)
- z.first_time_only(self) {
- hex_num = '0x[a-f\d]+'
- float = '\d+\.?\d*'
- num = "[-+]?(?:#{float}|#{hex_num})"
+ z.first_time_only(self) do
case
- when self == ''
- z.emit('""')
- # when self =~ /[\x00-\x08\x0B\x0C\x0E-\x1F\x80-\xFF]/
- # z.emit("!binary |\n")
- # z.emit([self].pack("m*"))
- when (
- (self =~ /\A(true|false|yes|no|on|null|off|#{num}(:#{num})*|!|=|~)$/i) or
- (self =~ /\A\n* /) or
- (self =~ /[\s:]$/) or
- (self =~ /^[>|][-+\d]*\s/i) or
- (self[-1..-1] =~ /\s/) or
- # This regular expression assumes the string is a byte sequence.
- # It does not concern itself with characters so we convert the string
- # to ASCII-8BIT for Ruby 1.9 to match up encodings.
- (self.to_ascii8bit=~ /[\x00-\x08\x0B\x0C\x0E-\x1F\x80-\xFF]/n) or
- (self =~ /[,\[\]\{\}\r\t]|:\s|\s#/) or
- (self =~ /\A([-:?!#&*'"]|<<|%.+:.)/)
- )
- z.emit("\"#{escaped_for_zaml}\"")
- when self =~ /\n/
- if self[-1..-1] == "\n" then z.emit('|+') else z.emit('|-') end
- z.nested { split("\n",-1).each { |line| z.nl; z.emit(line.chomp("\n")) } }
- else
- z.emit(self)
+ when self == ''
+ z.emit('""')
+ when self.to_ascii8bit !~ /\A(?: # ?: non-capturing group (grouping with no back references)
+ [\x09\x0A\x0D\x20-\x7E] # ASCII
+ | [\xC2-\xDF][\x80-\xBF] # non-overlong 2-byte
+ | \xE0[\xA0-\xBF][\x80-\xBF] # excluding overlongs
+ | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2} # straight 3-byte
+ | \xED[\x80-\x9F][\x80-\xBF] # excluding surrogates
+ | \xF0[\x90-\xBF][\x80-\xBF]{2} # planes 1-3
+ | [\xF1-\xF3][\x80-\xBF]{3} # planes 4-15
+ | \xF4[\x80-\x8F][\x80-\xBF]{2} # plane 16
+ )*\z/mnx
+ # Emit the binary tag, then recurse. Ruby splits BASE64 output at the 60
+ # character mark when packing strings, and we can wind up a multi-line
+ # string here. We could reimplement the multi-line string logic,
+ # but why would we - this does just as well for producing solid output.
+ z.emit("!binary ")
+ [self].pack("m*").to_zaml(z)
+
+ # Only legal UTF-8 characters can make it this far, so we are safe
+ # against emitting something dubious. That means we don't need to mess
+ # about, just emit them directly. --daniel 2012-07-14
+ when self =~ /\n/
+ # embedded newline, split line-wise in quoted string block form.
+ if self[-1..-1] == "\n" then z.emit('|+') else z.emit('|-') end
+ z.nested { split("\n",-1).each { |line| z.nl; z.emit(line.chomp("\n")) } }
+ when ((self =~ /^[a-zA-Z\/][-\[\]_\/.:a-zA-Z0-9]*$/) and
+ (self !~ /^(?:true|false|yes|no|on|null|off)$/i))
+ # simple string literal, safe to emit unquoted.
+ z.emit(self)
+ else
+ # ...though we still have to escape unsafe characters.
+ escaped = gsub(/[\\"\x00-\x1F]/) do |c|
+ ZAML_ESCAPES[c] || "\\x#{c[0].ord.to_s(16)}"
+ end
+ z.emit("\"#{escaped}\"")
end
- }
+ end
end
# Return a guranteed ASCII-8BIT encoding for Ruby 1.9 This is a helper
# method for other methods that perform regular expressions against byte
# sequences deliberately rather than dealing with characters.
# The method may or may not return a new instance.
if String.method_defined?(:encoding)
- ASCII = Encoding.find("ASCII-8BIT")
+ ASCII_ENCODING = Encoding.find("ASCII-8BIT")
def to_ascii8bit
- if self.encoding.name == "ASCII-8BIT"
+ if self.encoding == ASCII_ENCODING
self
else
- self.dup.force_encoding("ASCII-8BIT")
+ self.dup.force_encoding(ASCII_ENCODING)
end
end
else
def to_ascii8bit
self
end
end
end
class Hash
def to_zaml(z)
z.first_time_only(self) {
z.nested {
if empty?
z.emit('{}')
else
each_pair { |k, v|
z.nl
z.prefix_structured_keys('? ') { k.to_zaml(z) }
z.emit(': ')
v.to_zaml(z)
}
end
}
}
end
end
class Array
def to_zaml(z)
z.first_time_only(self) {
z.nested {
if empty?
z.emit('[]')
else
each { |v| z.nl('- '); v.to_zaml(z) }
end
}
}
end
end
class Time
def to_zaml(z)
# 2008-12-06 10:06:51.373758 -07:00
ms = ("%0.6f" % (usec * 1e-6)).sub(/^\d+\./,'')
offset = "%+0.2i:%0.2i" % [utc_offset / 3600, (utc_offset / 60) % 60]
z.emit(self.strftime("%Y-%m-%d %H:%M:%S.#{ms} #{offset}"))
end
end
class Date
def to_zaml(z)
z.emit(strftime('%Y-%m-%d'))
end
end
class Range
def to_zaml(z)
z.first_time_only(self) {
z.emit(zamlized_class_name(Range))
z.nested {
z.nl
z.emit('begin: ')
z.emit(first)
z.nl
z.emit('end: ')
z.emit(last)
z.nl
z.emit('excl: ')
z.emit(exclude_end?)
}
}
end
end
diff --git a/spec/unit/util/zaml_spec.rb b/spec/unit/util/zaml_spec.rb
index d5e8e024a..0086fb527 100755
--- a/spec/unit/util/zaml_spec.rb
+++ b/spec/unit/util/zaml_spec.rb
@@ -1,106 +1,196 @@
#!/usr/bin/env rspec
# encoding: UTF-8
#
# The above encoding line is a magic comment to set the default source encoding
# of this file for the Ruby interpreter. It must be on the first or second
# line of the file if an interpreter is in use. In Ruby 1.9 and later, the
# source encoding determines the encoding of String and Regexp objects created
# from this source file. This explicit encoding is important becuase otherwise
# Ruby will pick an encoding based on LANG or LC_CTYPE environment variables.
# These may be different from site to site so it's important for us to
# establish a consistent behavior. For more information on M17n please see:
# http://links.puppetlabs.com/understanding_m17n
require 'spec_helper'
require 'puppet/util/monkey_patches'
describe "Pure ruby yaml implementation" do
{
7 => "--- 7",
3.14159 => "--- 3.14159",
"3.14159" => '--- "3.14159"',
"+3.14159" => '--- "+3.14159"',
"0x123abc" => '--- "0x123abc"',
"-0x123abc" => '--- "-0x123abc"',
"-0x123" => '--- "-0x123"',
"+0x123" => '--- "+0x123"',
- "0x123.456" => "--- 0x123.456",
+ "0x123.456" => '--- "0x123.456"',
'test' => "--- test",
[] => "--- []",
:symbol => "--- !ruby/sym symbol",
{:a => "A"} => "--- \n !ruby/sym a: A",
- {:a => "x\ny"} => "--- \n !ruby/sym a: |-\n x\n y"
+ {:a => "x\ny"} => "--- \n !ruby/sym a: |-\n x\n y"
}.each { |o,y|
it "should convert the #{o.class} #{o.inspect} to yaml" do
o.to_yaml.should == y
end
it "should produce yaml for the #{o.class} #{o.inspect} that can be reconstituted" do
YAML.load(o.to_yaml).should == o
end
}
#
# Can't test for equality on raw objects
{
Object.new => "--- !ruby/object {}",
[Object.new] => "--- \n - !ruby/object {}",
{Object.new => Object.new} => "--- \n ? !ruby/object {}\n : !ruby/object {}"
}.each { |o,y|
it "should convert the #{o.class} #{o.inspect} to yaml" do
o.to_yaml.should == y
end
it "should produce yaml for the #{o.class} #{o.inspect} that can be reconstituted" do
lambda { YAML.load(o.to_yaml) }.should_not raise_error
end
}
it "should emit proper labels and backreferences for common objects" do
# Note: this test makes assumptions about the names ZAML chooses
# for labels.
x = [1, 2]
y = [3, 4]
z = [x, y, x, y]
z.to_yaml.should == "--- \n - &id001\n - 1\n - 2\n - &id002\n - 3\n - 4\n - *id001\n - *id002"
z2 = YAML.load(z.to_yaml)
z2.should == z
z2[0].should equal(z2[2])
z2[1].should equal(z2[3])
end
it "should emit proper labels and backreferences for recursive objects" do
x = [1, 2]
x << x
x.to_yaml.should == "--- &id001\n \n - 1\n - 2\n - *id001"
x2 = YAML.load(x.to_yaml)
x2.should be_a(Array)
x2.length.should == 3
x2[0].should == 1
x2[1].should == 2
x2[2].should equal(x2)
end
end
# Note, many of these tests will pass on Ruby 1.8 but fail on 1.9 if the patch
# fix is not applied to Puppet or there's a regression. These version
# dependant failures are intentional since the string encoding behavior changed
# significantly in 1.9.
describe "UTF-8 encoded String#to_yaml (Bug #11246)" do
# JJM All of these snowmen are different representations of the same
# UTF-8 encoded string.
let(:snowman) { 'Snowman: [☃]' }
let(:snowman_escaped) { "Snowman: [\xE2\x98\x83]" }
describe "UTF-8 String Literal" do
subject { snowman }
it "should serialize to YAML" do
subject.to_yaml
end
it "should serialize and deserialize to the same thing" do
YAML.load(subject.to_yaml).should == subject
end
it "should serialize and deserialize to a String compatible with a UTF-8 encoded Regexp" do
YAML.load(subject.to_yaml).should =~ /☃/u
end
end
end
+
+describe "binary data" do
+ subject { "M\xC0\xDF\xE5tt\xF6" }
+
+ if String.method_defined?(:encoding)
+ def binary(str)
+ str.force_encoding('binary')
+ end
+ else
+ def binary(str)
+ str
+ end
+ end
+
+ it "should not explode encoding binary data" do
+ expect { subject.to_yaml }.not_to raise_error
+ end
+
+ it "should mark the binary data as binary" do
+ subject.to_yaml.should =~ /!binary/
+ end
+
+ it "should round-trip the data" do
+ yaml = subject.to_yaml
+ read = YAML.load(yaml)
+ binary(read).should == binary(subject)
+ end
+
+ [
+ "\xC0\xAE", # over-long UTF-8 '.' character
+ "\xC0\x80", # over-long NULL byte
+ "\xC0\xFF",
+ "\xC1\xAE",
+ "\xC1\x80",
+ "\xC1\xFF",
+ "\x80", # first continuation byte
+ "\xbf", # last continuation byte
+ # all possible continuation bytes in one shot
+ "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F" +
+ "\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F" +
+ "\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF" +
+ "\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF",
+ # lonely start characters - first, all possible two byte sequences
+ "\xC0 \xC1 \xC2 \xC3 \xC4 \xC5 \xC6 \xC7 \xC8 \xC9 \xCA \xCB \xCC \xCD \xCE \xCF " +
+ "\xD0 \xD1 \xD2 \xD3 \xD4 \xD5 \xD6 \xD7 \xD8 \xD9 \xDA \xDB \xDC \xDD \xDE \xDF ",
+ # and so for three byte sequences, four, five, and six, as follow.
+ "\xE0 \xE1 \xE2 \xE3 \xE4 \xE5 \xE6 \xE7 \xE8 \xE9 \xEA \xEB \xEC \xED \xEE \xEF ",
+ "\xF0 \xF1 \xF2 \xF3 \xF4 \xF5 \xF6 \xF7 ",
+ "\xF8 \xF9 \xFA \xFB ",
+ "\xFC \xFD ",
+ # sequences with the last byte missing
+ "\xC0", "\xE0", "\xF0\x80\x80", "\xF8\x80\x80\x80", "\xFC\x80\x80\x80\x80",
+ "\xDF", "\xEF\xBF", "\xF7\xBF\xBF", "\xFB\xBF\xBF\xBF", "\xFD\xBF\xBF\xBF\xBF",
+ # impossible bytes
+ "\xFE", "\xFF", "\xFE\xFE\xFF\xFF",
+ # over-long '/' character
+ "\xC0\xAF",
+ "\xE0\x80\xAF",
+ "\xF0\x80\x80\xAF",
+ "\xF8\x80\x80\x80\xAF",
+ "\xFC\x80\x80\x80\x80\xAF",
+ # maximum overlong sequences
+ "\xc1\xbf",
+ "\xe0\x9f\xbf",
+ "\xf0\x8f\xbf\xbf",
+ "\xf8\x87\xbf\xbf\xbf",
+ "\xfc\x83\xbf\xbf\xbf\xbf",
+ # overlong NUL
+ "\xc0\x80",
+ "\xe0\x80\x80",
+ "\xf0\x80\x80\x80",
+ "\xf8\x80\x80\x80\x80",
+ "\xfc\x80\x80\x80\x80\x80",
+ ].each do |input|
+ # It might seem like we should more correctly reject these sequences in
+ # the encoder, and I would personally agree, but the sad reality is that
+ # we do not distinguish binary and textual data in our language, and so we
+ # wind up with the same thing - a string - containing both.
+ #
+ # That leads to the position where we must treat these invalid sequences,
+ # which are both legitimate binary content, and illegitimate potential
+ # attacks on the system, as something that passes through correctly in
+ # a string. --daniel 2012-07-14
+ it "binary encode highly dubious non-compliant UTF-8 input #{input.inspect}" do
+ encoded = ZAML.dump(binary(input))
+ encoded.should =~ /!binary/
+ YAML.load(encoded).should == input
+ end
+ end
+end