diff --git a/benchmarks/defined_types/benchmarker.rb b/benchmarks/defined_types/benchmarker.rb index 77d46e71e..a54e228b0 100644 --- a/benchmarks/defined_types/benchmarker.rb +++ b/benchmarks/defined_types/benchmarker.rb @@ -1,73 +1,73 @@ require 'erb' require 'ostruct' require 'fileutils' require 'json' class Benchmarker include FileUtils def initialize(target, size) @target = target @size = size end def setup require 'puppet' config = File.join(@target, 'puppet.conf') Puppet.initialize_settings(['--config', config]) end - def run + def run(args=nil) env = Puppet.lookup(:environments).get('benchmarking') node = Puppet::Node.new("testing", :environment => env) Puppet::Resource::Catalog.indirection.find("testing", :use_node => node) end def generate environment = File.join(@target, 'environments', 'benchmarking') templates = File.join('benchmarks', 'defined_types') mkdir_p(File.join(environment, 'modules')) mkdir_p(File.join(environment, 'manifests')) render(File.join(templates, 'site.pp.erb'), File.join(environment, 'manifests', 'site.pp'), :size => @size) @size.times do |i| module_name = "module#{i}" module_base = File.join(environment, 'modules', module_name) manifests = File.join(module_base, 'manifests') mkdir_p(manifests) File.open(File.join(module_base, 'metadata.json'), 'w') do |f| JSON.dump({ "types" => [], "source" => "", "author" => "Defined Types Benchmark", "license" => "Apache 2.0", "version" => "1.0.0", "description" => "Defined Types benchmark module #{i}", "summary" => "Just this benchmark module, you know?", "dependencies" => [], }, f) end render(File.join(templates, 'module', 'testing.pp.erb'), File.join(manifests, 'testing.pp'), :name => module_name) end render(File.join(templates, 'puppet.conf.erb'), File.join(@target, 'puppet.conf'), :location => @target) end def render(erb_file, output_file, bindings) site = ERB.new(File.read(erb_file)) File.open(output_file, 'w') do |fh| fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) end end end diff --git a/benchmarks/defined_types/benchmarker.rb b/benchmarks/defined_types4/benchmarker.rb similarity index 93% copy from benchmarks/defined_types/benchmarker.rb copy to benchmarks/defined_types4/benchmarker.rb index 77d46e71e..5b0669291 100644 --- a/benchmarks/defined_types/benchmarker.rb +++ b/benchmarks/defined_types4/benchmarker.rb @@ -1,73 +1,73 @@ require 'erb' require 'ostruct' require 'fileutils' require 'json' class Benchmarker include FileUtils def initialize(target, size) @target = target @size = size end def setup require 'puppet' config = File.join(@target, 'puppet.conf') Puppet.initialize_settings(['--config', config]) end - def run + def run(args=nil) env = Puppet.lookup(:environments).get('benchmarking') node = Puppet::Node.new("testing", :environment => env) Puppet::Resource::Catalog.indirection.find("testing", :use_node => node) end def generate environment = File.join(@target, 'environments', 'benchmarking') - templates = File.join('benchmarks', 'defined_types') + templates = File.join('benchmarks', 'defined_types4') mkdir_p(File.join(environment, 'modules')) mkdir_p(File.join(environment, 'manifests')) render(File.join(templates, 'site.pp.erb'), File.join(environment, 'manifests', 'site.pp'), :size => @size) @size.times do |i| module_name = "module#{i}" module_base = File.join(environment, 'modules', module_name) manifests = File.join(module_base, 'manifests') mkdir_p(manifests) File.open(File.join(module_base, 'metadata.json'), 'w') do |f| JSON.dump({ "types" => [], "source" => "", - "author" => "Defined Types Benchmark", + "author" => "Defined Types Benchmark Future Parser", "license" => "Apache 2.0", "version" => "1.0.0", "description" => "Defined Types benchmark module #{i}", "summary" => "Just this benchmark module, you know?", "dependencies" => [], }, f) end render(File.join(templates, 'module', 'testing.pp.erb'), File.join(manifests, 'testing.pp'), :name => module_name) end render(File.join(templates, 'puppet.conf.erb'), File.join(@target, 'puppet.conf'), :location => @target) end def render(erb_file, output_file, bindings) site = ERB.new(File.read(erb_file)) File.open(output_file, 'w') do |fh| fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) end end end diff --git a/benchmarks/defined_types4/description b/benchmarks/defined_types4/description new file mode 100644 index 000000000..f24e9bba4 --- /dev/null +++ b/benchmarks/defined_types4/description @@ -0,0 +1,4 @@ +Benchmark scenario: heavy use of defined types future parser +Benchmark target: catalog compilation +Parser: Future + diff --git a/benchmarks/defined_types4/module/testing.pp.erb b/benchmarks/defined_types4/module/testing.pp.erb new file mode 100644 index 000000000..e723561d5 --- /dev/null +++ b/benchmarks/defined_types4/module/testing.pp.erb @@ -0,0 +1,3 @@ +define <%= name %>::testing { + notify { "in <%= name %>: $title": } +} diff --git a/benchmarks/defined_types4/puppet.conf.erb b/benchmarks/defined_types4/puppet.conf.erb new file mode 100644 index 000000000..618ea6e2f --- /dev/null +++ b/benchmarks/defined_types4/puppet.conf.erb @@ -0,0 +1,4 @@ +confdir = <%= location %> +vardir = <%= location %> +environmentpath = <%= File.join(location, 'environments') %> +parser = future diff --git a/benchmarks/defined_types4/site.pp.erb b/benchmarks/defined_types4/site.pp.erb new file mode 100644 index 000000000..338b635b2 --- /dev/null +++ b/benchmarks/defined_types4/site.pp.erb @@ -0,0 +1,4 @@ +<% size.times do |i| %> + module<%= i %>::testing { "first": } + module<%= i %>::testing{ "second": } +<% end %> diff --git a/benchmarks/defined_types/benchmarker.rb b/benchmarks/empty_catalog/benchmarker.rb similarity index 53% copy from benchmarks/defined_types/benchmarker.rb copy to benchmarks/empty_catalog/benchmarker.rb index 77d46e71e..809685cc2 100644 --- a/benchmarks/defined_types/benchmarker.rb +++ b/benchmarks/empty_catalog/benchmarker.rb @@ -1,73 +1,47 @@ require 'erb' require 'ostruct' require 'fileutils' require 'json' class Benchmarker include FileUtils def initialize(target, size) @target = target @size = size end def setup require 'puppet' config = File.join(@target, 'puppet.conf') Puppet.initialize_settings(['--config', config]) end - def run + def run(args=nil) env = Puppet.lookup(:environments).get('benchmarking') node = Puppet::Node.new("testing", :environment => env) Puppet::Resource::Catalog.indirection.find("testing", :use_node => node) end def generate environment = File.join(@target, 'environments', 'benchmarking') - templates = File.join('benchmarks', 'defined_types') + templates = File.join('benchmarks', 'empty_catalog') mkdir_p(File.join(environment, 'modules')) mkdir_p(File.join(environment, 'manifests')) render(File.join(templates, 'site.pp.erb'), - File.join(environment, 'manifests', 'site.pp'), - :size => @size) - - @size.times do |i| - module_name = "module#{i}" - module_base = File.join(environment, 'modules', module_name) - manifests = File.join(module_base, 'manifests') - - mkdir_p(manifests) - - File.open(File.join(module_base, 'metadata.json'), 'w') do |f| - JSON.dump({ - "types" => [], - "source" => "", - "author" => "Defined Types Benchmark", - "license" => "Apache 2.0", - "version" => "1.0.0", - "description" => "Defined Types benchmark module #{i}", - "summary" => "Just this benchmark module, you know?", - "dependencies" => [], - }, f) - end - - render(File.join(templates, 'module', 'testing.pp.erb'), - File.join(manifests, 'testing.pp'), - :name => module_name) - end + File.join(environment, 'manifests', 'site.pp'),{}) render(File.join(templates, 'puppet.conf.erb'), File.join(@target, 'puppet.conf'), :location => @target) end def render(erb_file, output_file, bindings) site = ERB.new(File.read(erb_file)) File.open(output_file, 'w') do |fh| fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) end end end diff --git a/benchmarks/empty_catalog/description b/benchmarks/empty_catalog/description new file mode 100644 index 000000000..d06e41b82 --- /dev/null +++ b/benchmarks/empty_catalog/description @@ -0,0 +1,4 @@ +Benchmark scenario: an empty catalog (only one call to log a message) shows the setup time for env / compiler +Benchmark target: catalog compilation overhead +Parser: Future + diff --git a/benchmarks/empty_catalog/puppet.conf.erb b/benchmarks/empty_catalog/puppet.conf.erb new file mode 100644 index 000000000..00e2986bf --- /dev/null +++ b/benchmarks/empty_catalog/puppet.conf.erb @@ -0,0 +1,5 @@ +confdir = <%= location %> +vardir = <%= location %> +environmentpath = <%= File.join(location, 'environments') %> +environment_timeout = '0' +parser = future diff --git a/benchmarks/empty_catalog/site.pp.erb b/benchmarks/empty_catalog/site.pp.erb new file mode 100644 index 000000000..054628183 --- /dev/null +++ b/benchmarks/empty_catalog/site.pp.erb @@ -0,0 +1 @@ +notice('hello world') \ No newline at end of file diff --git a/benchmarks/evaluations/benchmarker.rb b/benchmarks/evaluations/benchmarker.rb new file mode 100644 index 000000000..3bf94d6dd --- /dev/null +++ b/benchmarks/evaluations/benchmarker.rb @@ -0,0 +1,140 @@ +require 'erb' +require 'ostruct' +require 'fileutils' +require 'json' + +class Benchmarker + include FileUtils + + def initialize(target, size) + @target = target + @size = size + @micro_benchmarks = {} + @parsecount = 100 + @evalcount = 100 + end + + def setup + require 'puppet' + require 'puppet/pops' + config = File.join(@target, 'puppet.conf') + Puppet.initialize_settings(['--config', config]) + manifests = File.join('benchmarks', 'evaluations', 'manifests') + Dir.foreach(manifests) do |f| + if f =~ /^(.*)\.pp$/ + @micro_benchmarks[$1] = File.read(File.join(manifests, f)) + end + end + # Run / Evaluate the common puppet logic + @env = Puppet.lookup(:environments).get('benchmarking') + @node = Puppet::Node.new("testing", :environment => @env) + @parser = Puppet::Pops::Parser::EvaluatingParser::Transitional.new + @compiler = Puppet::Parser::Compiler.new(@node) + @scope = @compiler.topscope + + # Perform a portion of what a compile does (just enough to evaluate the site.pp logic) + @compiler.catalog.environment_instance = @compiler.environment + @compiler.send(:evaluate_main) + + # Then pretend we are running as part of a compilation + Puppet.push_context(@compiler.context_overrides, "Benchmark masquerading as compiler configured context") + end + + def run(args = {}) + details = args[:detail] || 'all' + measurements = [] + @micro_benchmarks.each do |name, source| + # skip if all but the wanted if a single benchmark is wanted + next unless details == 'all' || match = details.match(/#{name}(?:[\._\s](parse|eval))?$/) + # if name ends with .parse or .eval only do that part, else do both parts + ending = match ? match[1] : nil # parse, eval or nil ending + unless ending == 'eval' + measurements << Benchmark.measure("#{name} parse") do + 1..@parsecount.times { @parser.parse_string(source, name) } + end + end + unless ending == 'parse' + model = @parser.parse_string(source, name) + measurements << Benchmark.measure("#{name} eval") do + 1..@evalcount.times do + begin + # Run each in a local scope + scope_memo = @scope.ephemeral_level + @scope.new_ephemeral(true) + @parser.evaluate(@scope, model) + ensure + # Toss the created local scope + @scope.unset_ephemeral_var(scope_memo) + end + end + end + end + end + measurements + end + + def generate + environment = File.join(@target, 'environments', 'benchmarking') + templates = File.join('benchmarks', 'evaluations') + + mkdir_p(File.join(environment, 'modules')) + mkdir_p(File.join(environment, 'manifests')) + + render(File.join(templates, 'site.pp.erb'), + File.join(environment, 'manifests', 'site.pp'),{}) + + render(File.join(templates, 'puppet.conf.erb'), + File.join(@target, 'puppet.conf'), + :location => @target) + + # Generate one module with a 3x function and a 4x function (namespaces) + module_name = "module1" + module_base = File.join(environment, 'modules', module_name) + manifests = File.join(module_base, 'manifests') + mkdir_p(manifests) + functions_3x = File.join(module_base, 'lib', 'puppet', 'parser', 'functions') + functions_4x = File.join(module_base, 'lib', 'puppet', 'functions') + mkdir_p(functions_3x) + mkdir_p(functions_4x) + + File.open(File.join(module_base, 'metadata.json'), 'w') do |f| + JSON.dump({ + "types" => [], + "source" => "", + "author" => "Evaluations Benchmark", + "license" => "Apache 2.0", + "version" => "1.0.0", + "description" => "Evaluations Benchmark module 1", + "summary" => "Module with supporting logic for evaluations benchmark", + "dependencies" => [], + }, f) + end + + render(File.join(templates, 'module', 'init.pp.erb'), + File.join(manifests, 'init.pp'), + :name => module_name) + + render(File.join(templates, 'module', 'func3.rb.erb'), + File.join(functions_3x, 'func3.rb'), + :name => module_name) + + # namespaced function + mkdir_p(File.join(functions_4x, module_name)) + render(File.join(templates, 'module', 'module1_func4.rb.erb'), + File.join(functions_4x, module_name, 'func4.rb'), + :name => module_name) + + # non namespaced + render(File.join(templates, 'module', 'func4.rb.erb'), + File.join(functions_4x, 'func4.rb'), + :name => module_name) + end + + def render(erb_file, output_file, bindings) + site = ERB.new(File.read(erb_file)) + File.open(output_file, 'w') do |fh| + fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) + end + end + +end diff --git a/benchmarks/evaluations/benchmarker_task.rb b/benchmarks/evaluations/benchmarker_task.rb new file mode 100644 index 000000000..d38ed1088 --- /dev/null +++ b/benchmarks/evaluations/benchmarker_task.rb @@ -0,0 +1,11 @@ +# Helper class that is used by the Rake task generator. +# Currently only supports defining arguments that are passed to run +# (The rake task generator always passes :warm_up_runs as an Integer when profiling). +# Other benchmarks, and for regular runs that wants arguments must specified them +# as an Array of symbols. +# +class BenchmarkerTask + def self.run_args + [:detail] + end +end \ No newline at end of file diff --git a/benchmarks/evaluations/description b/benchmarks/evaluations/description new file mode 100644 index 000000000..812037e6d --- /dev/null +++ b/benchmarks/evaluations/description @@ -0,0 +1,13 @@ +Benchmark scenario: evaluates a select set of time critical expressions +Benchmark target: measuring individual use cases of evaluation +Parser: Future + +Evaluations: +* fcall_3x - calls sprintf 20x times +* fcall_4x - calls assert_type 20x times (is heavier than sprintf, have no similar simple 4x function) +* interpolation - does 20x interpolations of variying length +* var_absolute - references a top scope variable 20x times with absolute reference +* var_relative - references a top scope variable 20x times with non absolute reference +* var_class_absolute - references a class variable 20x times with absolute reference +* var_class_relative - references a class variable 20x times with non absolute reference + diff --git a/benchmarks/evaluations/manifests/assert_type.pp b/benchmarks/evaluations/manifests/assert_type.pp new file mode 100644 index 000000000..0be757281 --- /dev/null +++ b/benchmarks/evaluations/manifests/assert_type.pp @@ -0,0 +1,6 @@ +$tmp = [ +assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), +assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), +assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), +assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), +] diff --git a/benchmarks/evaluations/manifests/fcall_3x.pp b/benchmarks/evaluations/manifests/fcall_3x.pp new file mode 100644 index 000000000..9689ca97a --- /dev/null +++ b/benchmarks/evaluations/manifests/fcall_3x.pp @@ -0,0 +1,6 @@ +$tmp = [ +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), +] diff --git a/benchmarks/evaluations/manifests/fcall_4x.pp b/benchmarks/evaluations/manifests/fcall_4x.pp new file mode 100644 index 000000000..a7b7b712a --- /dev/null +++ b/benchmarks/evaluations/manifests/fcall_4x.pp @@ -0,0 +1,6 @@ +$tmp = [ +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), +] diff --git a/benchmarks/evaluations/manifests/fcall_ns4x.pp b/benchmarks/evaluations/manifests/fcall_ns4x.pp new file mode 100644 index 000000000..d59de8539 --- /dev/null +++ b/benchmarks/evaluations/manifests/fcall_ns4x.pp @@ -0,0 +1,6 @@ +$tmp = [ +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +] diff --git a/benchmarks/evaluations/manifests/interpolation.pp b/benchmarks/evaluations/manifests/interpolation.pp new file mode 100644 index 000000000..817770146 --- /dev/null +++ b/benchmarks/evaluations/manifests/interpolation.pp @@ -0,0 +1,11 @@ +$tmp = [ "...$x...", + "...$x...$x", + "...$x...$x...", + "...$x...$x...$x...", + "...$x...$x...$x...$x", + "...$x...$x...$x...$x...", + "...$x...$x...$x...$x...$x", + "...$x...$x...$x...$x...$x...", + "...$x...$x...$x...$x...$x...$x", + "...$x...$x...$x...$x...$x...$x...", +] \ No newline at end of file diff --git a/benchmarks/evaluations/manifests/var_absolute.pp b/benchmarks/evaluations/manifests/var_absolute.pp new file mode 100644 index 000000000..ed4731817 --- /dev/null +++ b/benchmarks/evaluations/manifests/var_absolute.pp @@ -0,0 +1,3 @@ +$tmp = [ $::x, $::x, $::x, $::x, $::x, $::x, $::x, $::x, $::x, $::x, + $::x, $::x, $::x, $::x, $::x, $::x, $::x, $::x, $::x, $::x, +] diff --git a/benchmarks/evaluations/manifests/var_class_absolute.pp b/benchmarks/evaluations/manifests/var_class_absolute.pp new file mode 100644 index 000000000..e86940f6e --- /dev/null +++ b/benchmarks/evaluations/manifests/var_class_absolute.pp @@ -0,0 +1,5 @@ +$tmp = [ $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, + $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, + $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, + $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, $::testing::param_a, +] diff --git a/benchmarks/evaluations/manifests/var_class_relative.pp b/benchmarks/evaluations/manifests/var_class_relative.pp new file mode 100644 index 000000000..3048db6ad --- /dev/null +++ b/benchmarks/evaluations/manifests/var_class_relative.pp @@ -0,0 +1,5 @@ +$tmp = [ $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, + $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, + $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, + $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, $testing::param_a, +] diff --git a/benchmarks/evaluations/manifests/var_relative.pp b/benchmarks/evaluations/manifests/var_relative.pp new file mode 100644 index 000000000..3fba7b6ef --- /dev/null +++ b/benchmarks/evaluations/manifests/var_relative.pp @@ -0,0 +1,3 @@ +$tmp = [$x, $x, $x, $x, $x, $x, $x, $x, $x, $x, + $x, $x, $x, $x, $x, $x, $x, $x, $x, $x, +] diff --git a/benchmarks/evaluations/module/func3.rb.erb b/benchmarks/evaluations/module/func3.rb.erb new file mode 100644 index 000000000..6b814a0d0 --- /dev/null +++ b/benchmarks/evaluations/module/func3.rb.erb @@ -0,0 +1,8 @@ +Puppet::Parser::Functions::newfunction(:func3, + :arity => 2, + :doc => "Blah blah, this is a lot of documentation that the ruby parser must deal with + because documentation is part of what is loaded at runtime. Some functions have + very little documentation, and some have quite a lot. This simulates documentation + that is slightly longer than the shortest ones.") do |vals| + # produces nil +end diff --git a/benchmarks/evaluations/module/func4.rb.erb b/benchmarks/evaluations/module/func4.rb.erb new file mode 100644 index 000000000..b923ec4f5 --- /dev/null +++ b/benchmarks/evaluations/module/func4.rb.erb @@ -0,0 +1,9 @@ +# Blah blah, this is a lot of documentation that the ruby parser must deal with +# because documentation is part of what is loaded at runtime. Some functions have +# very little documentation, and some have quite a lot. This simulates documentation +# that is slightly longer than the shortest ones. +# +Puppet::Functions.create_function(:func4) do + def func4(x,y) + end +end \ No newline at end of file diff --git a/benchmarks/evaluations/module/init.pp.erb b/benchmarks/evaluations/module/init.pp.erb new file mode 100644 index 000000000..755657c8a --- /dev/null +++ b/benchmarks/evaluations/module/init.pp.erb @@ -0,0 +1 @@ +# empty init (for now) \ No newline at end of file diff --git a/benchmarks/evaluations/module/module1_func4.rb.erb b/benchmarks/evaluations/module/module1_func4.rb.erb new file mode 100644 index 000000000..1891156c0 --- /dev/null +++ b/benchmarks/evaluations/module/module1_func4.rb.erb @@ -0,0 +1,9 @@ +# Blah blah, this is a lot of documentation that the ruby parser must deal with +# because documentation is part of what is loaded at runtime. Some functions have +# very little documentation, and some have quite a lot. This simulates documentation +# that is slightly longer than the shortest ones. +# +Puppet::Functions.create_function(:'<%= name %>::func4') do + def func4(x,y) + end +end \ No newline at end of file diff --git a/benchmarks/evaluations/puppet.conf.erb b/benchmarks/evaluations/puppet.conf.erb new file mode 100644 index 000000000..d0bf4b2f1 --- /dev/null +++ b/benchmarks/evaluations/puppet.conf.erb @@ -0,0 +1,6 @@ +confdir = <%= location %> +vardir = <%= location %> +environmentpath = <%= File.join(location, 'environments') %> +environment_timeout = '0' +parser = future +strict_variables = true diff --git a/benchmarks/evaluations/site.pp.erb b/benchmarks/evaluations/site.pp.erb new file mode 100644 index 000000000..05cc8d8c5 --- /dev/null +++ b/benchmarks/evaluations/site.pp.erb @@ -0,0 +1,10 @@ +# Common setup done once for all micro benchmarks +# +class testing { + $param_a = 10 + $param_b = 20 +} +include testing +$x = 'aaaaaaaa' + + diff --git a/benchmarks/many_modules/benchmarker.rb b/benchmarks/many_modules/benchmarker.rb index 8540f85b1..4a6d4f463 100644 --- a/benchmarks/many_modules/benchmarker.rb +++ b/benchmarks/many_modules/benchmarker.rb @@ -1,77 +1,77 @@ require 'erb' require 'ostruct' require 'fileutils' require 'json' class Benchmarker include FileUtils def initialize(target, size) @target = target @size = size end def setup require 'puppet' config = File.join(@target, 'puppet.conf') Puppet.initialize_settings(['--config', config]) end - def run + def run(args=nil) env = Puppet.lookup(:environments).get('benchmarking') node = Puppet::Node.new("testing", :environment => env) Puppet::Resource::Catalog.indirection.find("testing", :use_node => node) end def generate environment = File.join(@target, 'environments', 'benchmarking') templates = File.join('benchmarks', 'many_modules') mkdir_p(File.join(environment, 'modules')) mkdir_p(File.join(environment, 'manifests')) render(File.join(templates, 'site.pp.erb'), File.join(environment, 'manifests', 'site.pp'), :size => @size) @size.times do |i| module_name = "module#{i}" module_base = File.join(environment, 'modules', module_name) manifests = File.join(module_base, 'manifests') mkdir_p(manifests) File.open(File.join(module_base, 'metadata.json'), 'w') do |f| JSON.dump({ "types" => [], "source" => "", "author" => "ManyModules Benchmark", "license" => "Apache 2.0", "version" => "1.0.0", "description" => "Many Modules benchmark module #{i}", "summary" => "Just this benchmark module, you know?", "dependencies" => [], }, f) end render(File.join(templates, 'module', 'init.pp.erb'), File.join(manifests, 'init.pp'), :name => module_name) render(File.join(templates, 'module', 'internal.pp.erb'), File.join(manifests, 'internal.pp'), :name => module_name) end render(File.join(templates, 'puppet.conf.erb'), File.join(@target, 'puppet.conf'), :location => @target) end def render(erb_file, output_file, bindings) site = ERB.new(File.read(erb_file)) File.open(output_file, 'w') do |fh| fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) end end end diff --git a/benchmarks/system_startup/benchmarker.rb b/benchmarks/system_startup/benchmarker.rb index c48a878dd..4a691a35c 100644 --- a/benchmarks/system_startup/benchmarker.rb +++ b/benchmarks/system_startup/benchmarker.rb @@ -1,17 +1,17 @@ class Benchmarker def initialize(target, size) end def setup end def generate end - def run + def run(args=nil) # Just running help is probably a good proxy of a full startup. # Simply asking for the version might also be good, but it would miss all # of the app searching and loading parts `puppet help` end end diff --git a/lib/puppet/pops/loader/base_loader.rb b/lib/puppet/pops/loader/base_loader.rb index f7b34ece7..fd9bc53c5 100644 --- a/lib/puppet/pops/loader/base_loader.rb +++ b/lib/puppet/pops/loader/base_loader.rb @@ -1,102 +1,103 @@ # BaseLoader # === # An abstract implementation of Puppet::Pops::Loader::Loader # # A derived class should implement `find(typed_name)` and set entries, and possible handle "miss caching". # # @api private # class Puppet::Pops::Loader::BaseLoader < Puppet::Pops::Loader::Loader # The parent loader attr_reader :parent # An internal name used for debugging and error message purposes attr_reader :loader_name def initialize(parent_loader, loader_name) @parent = parent_loader # the higher priority loader to consult @named_values = {} # hash name => NamedEntry @last_name = nil # the last name asked for (optimization) @last_result = nil # the value of the last name (optimization) @loader_name = loader_name # the name of the loader (not the name-space it is a loader for) end # @api public # def load_typed(typed_name) # The check for "last queried name" is an optimization when a module searches. First it checks up its parent # chain, then itself, and then delegates to modules it depends on. # These modules are typically parented by the same # loader as the one initiating the search. It is inefficient to again try to search the same loader for # the same name. if typed_name == @last_name @last_result else @last_name = typed_name @last_result = internal_load(typed_name) end end # This method is final (subclasses should not override it) # # @api private # def get_entry(typed_name) @named_values[typed_name] end # @api private # def set_entry(typed_name, value, origin = nil) if entry = @named_values[typed_name] then fail_redefined(entry); end @named_values[typed_name] = Puppet::Pops::Loader::Loader::NamedEntry.new(typed_name, value, origin) end # @api private # def add_entry(type, name, value, origin) set_entry(Puppet::Pops::Loader::Loader::TypedName.new(type, name), value, origin) end # Promotes an already created entry (typically from another loader) to this loader # # @api private # def promote_entry(named_entry) typed_name = named_entry.typed_name - if entry = @named_values[typed_name] then fail_redefined(entry); end + if entry = @named_values[typed_name] then fail_redefine(entry); end @named_values[typed_name] = named_entry end private def fail_redefine(entry) + require 'debugger'; debugger origin_info = entry.origin ? " Originally set at #{origin_label(entry.origin)}." : "unknown location" - raise ArgumentError, "Attempt to redefine entity '#{entry.typed_name}' originally set at #{origin_label(origin)}.#{origin_info}" + raise ArgumentError, "Attempt to redefine entity '#{entry.typed_name}' originally set at #{origin_info}" end # TODO: Should not really be here?? - TODO: A Label provider ? semantics for the URI? # def origin_label(origin) if origin && origin.is_a?(URI) origin.to_s elsif origin.respond_to?(:uri) origin.uri.to_s else - nil + origin end end # loads in priority order: # 1. already loaded here # 2. load from parent # 3. find it here # 4. give up # def internal_load(typed_name) # avoid calling get_entry, by looking it up @named_values[typed_name] || parent.load_typed(typed_name) || find(typed_name) end end diff --git a/lib/puppet/pops/loader/loader.rb b/lib/puppet/pops/loader/loader.rb index 256fc373e..37c912c2d 100644 --- a/lib/puppet/pops/loader/loader.rb +++ b/lib/puppet/pops/loader/loader.rb @@ -1,180 +1,180 @@ # Loader # === # A Loader is responsible for loading "entities" ("instantiable and executable objects in the puppet language" which # are type, hostclass, definition, function, and bindings. # # The main method for users of a Loader is the `load` or `load_typed methods`, which returns a previously loaded entity # of a given type/name, and searches and loads the entity if not already loaded. # # private entities # --- # TODO: handle loading of entities that are private. Suggest that all calls pass an origin_loader (the loader # where request originated (or symbol :public). A module loader has one (or possibly a list) of what is # considered to represent private loader - i.e. the dependency loader for a module. If an entity is private # it should be stored with this status, and an error should be raised if the origin_loader is not on the list # of accepted "private" loaders. # The private loaders can not be given at creation time (they are parented by the loader in question). Another # alternative is to check if the origin_loader is a child loader, but this requires bidirectional links # between loaders or a search if loader with private entity is a parent of the origin_loader). # # @api public # class Puppet::Pops::Loader::Loader # Produces the value associated with the given name if already loaded, or available for loading # by this loader, one of its parents, or other loaders visible to this loader. # This is the method an external party should use to "get" the named element. # # An implementor of this method should first check if the given name is already loaded by self, or a parent # loader, and if so return that result. If not, it should call `find` to perform the loading. # # @param type [:Symbol] the type to load # @param name [String, Symbol] the name of the entity to load # @return [Object, nil] the value or nil if not found # # @api public # def load(type, name) if result = load_typed(TypedName.new(type, name.to_s)) result.value end end # Loads the given typed name, and returns a NamedEntry if found, else returns nil. # This the same a `load`, but returns a NamedEntry with origin/value information. # # @param typed_name [TypedName] - the type, name combination to lookup # @return [NamedEntry, nil] the entry containing the loaded value, or nil if not found # # @api public # def load_typed(typed_name) raise NotImplementedError.new end # Produces the value associated with the given name if defined **in this loader**, or nil if not defined. # This lookup does not trigger any loading, or search of the given name. # An implementor of this method may not search or look up in any other loader, and it may not # define the name. # # @param typed_name [TypedName] - the type, name combination to lookup # # @api private # def [] (typed_name) if found = get_entry(typed_name) found.value else nil end end # Searches for the given name in this loader's context (parents should already have searched their context(s) without # producing a result when this method is called). # An implementation of find typically caches the result. # # @param typed_name [TypedName] the type, name combination to lookup # @return [NamedEntry, nil] the entry for the loaded entry, or nil if not found # # @api private # def find(typed_name) raise NotImplementedError.new end # Returns the parent of the loader, or nil, if this is the top most loader. This implementation returns nil. def parent nil end # Produces the private loader for loaders that have a one (the visibility given to loaded entities). # For loaders that does not provide a private loader, self is returned. # # @api private def private_loader self end # Binds a value to a name. The name should not start with '::', but may contain multiple segments. # # @param type [:Symbol] the type of the entity being set # @param name [String, Symbol] the name of the entity being set # @param origin [URI, #uri, String] the origin of the set entity, a URI, or provider of URI, or URI in string form # @return [NamedEntry, nil] the created entry # # @api private # def set_entry(type, name, value, origin = nil) raise NotImplementedError.new end # Produces a NamedEntry if a value is bound to the given name, or nil if nothing is bound. # # @param typed_name [TypedName] the type, name combination to lookup # @return [NamedEntry, nil] the value bound in an entry # # @api private # def get_entry(typed_name) raise NotImplementedError.new end # An entry for one entity loaded by the loader. # class NamedEntry attr_reader :typed_name attr_reader :value attr_reader :origin def initialize(typed_name, value, origin) - @name = typed_name + @typed_name = typed_name @value = value @origin = origin freeze() end end # A name/type combination that can be used as a compound hash key # class TypedName attr_reader :type attr_reader :name attr_reader :name_parts # True if name is qualified (more than a single segment) attr_reader :qualified def initialize(type, name) @type = type # relativize the name (get rid of leading ::), and make the split string available @name_parts = name.to_s.split(/::/) @name_parts.shift if name_parts[0].empty? @name = name_parts.join('::') @qualified = name_parts.size > 1 # precompute hash - the name is frozen, so this is safe to do @hash = [self.class, type, @name].hash # Not allowed to have numeric names - 0, 010, 0x10, 1.2 etc if Puppet::Pops::Utils.is_numeric?(@name) raise ArgumentError, "Illegal attempt to use a numeric name '#{name}' at #{origin_label(origin)}." end freeze() end def hash @hash end def ==(o) o.class == self.class && type == o.type && name == o.name end alias eql? == def to_s "#{type}/#{name}" end end end diff --git a/spec/unit/pops/loaders/dependency_loader_spec.rb b/spec/unit/pops/loaders/dependency_loader_spec.rb index dbea5b208..cbdefe897 100644 --- a/spec/unit/pops/loaders/dependency_loader_spec.rb +++ b/spec/unit/pops/loaders/dependency_loader_spec.rb @@ -1,44 +1,61 @@ require 'spec_helper' require 'puppet_spec/files' require 'puppet/pops' require 'puppet/loaders' describe 'dependency loader' do include PuppetSpec::Files let(:static_loader) { Puppet::Pops::Loader::StaticLoader.new() } let(:loaders) { Puppet::Pops::Loaders.new(Puppet::Node::Environment.create(:testing, [])) } describe 'FileBased module loader' do it 'load something in global name space raises an error' do module_dir = dir_containing('testmodule', { 'lib' => { 'puppet' => { 'functions' => { 'testmodule' => { 'foo.rb' => 'Puppet::Functions.create_function("foo") { def foo; end; }' }}}}}) module_loader = Puppet::Pops::Loader::ModuleLoaders::FileBased.new(static_loader, loaders, 'testmodule', module_dir, 'test1') dep_loader = Puppet::Pops::Loader::DependencyLoader.new(static_loader, 'test-dep', [module_loader]) expect do dep_loader.load_typed(typed_name(:function, 'testmodule::foo')).value end.to raise_error(ArgumentError, /produced mis-matched name, expected 'testmodule::foo', got foo/) end it 'can load something in a qualified name space' do module_dir = dir_containing('testmodule', { 'lib' => { 'puppet' => { 'functions' => { 'testmodule' => { 'foo.rb' => 'Puppet::Functions.create_function("testmodule::foo") { def foo; end; }' }}}}}) module_loader = Puppet::Pops::Loader::ModuleLoaders::FileBased.new(static_loader, loaders, 'testmodule', module_dir, 'test1') dep_loader = Puppet::Pops::Loader::DependencyLoader.new(static_loader, 'test-dep', [module_loader]) function = dep_loader.load_typed(typed_name(:function, 'testmodule::foo')).value expect(function.class.name).to eq('testmodule::foo') expect(function.is_a?(Puppet::Functions::Function)).to eq(true) end + + it 'can load something in a qualified name space more than once' do + module_dir = dir_containing('testmodule', { + 'lib' => { 'puppet' => { 'functions' => { 'testmodule' => { + 'foo.rb' => 'Puppet::Functions.create_function("testmodule::foo") { def foo; end; }' + }}}}}) + module_loader = Puppet::Pops::Loader::ModuleLoaders::FileBased.new(static_loader, loaders, 'testmodule', module_dir, 'test1') + dep_loader = Puppet::Pops::Loader::DependencyLoader.new(static_loader, 'test-dep', [module_loader]) + + function = dep_loader.load_typed(typed_name(:function, 'testmodule::foo')).value + expect(function.class.name).to eq('testmodule::foo') + expect(function.is_a?(Puppet::Functions::Function)).to eq(true) + + function = dep_loader.load_typed(typed_name(:function, 'testmodule::foo')).value + expect(function.class.name).to eq('testmodule::foo') + expect(function.is_a?(Puppet::Functions::Function)).to eq(true) + end end def typed_name(type, name) Puppet::Pops::Loader::Loader::TypedName.new(type, name) end end diff --git a/spec/unit/pops/loaders/loaders_spec.rb b/spec/unit/pops/loaders/loaders_spec.rb index daa9c716c..a9156ecd9 100644 --- a/spec/unit/pops/loaders/loaders_spec.rb +++ b/spec/unit/pops/loaders/loaders_spec.rb @@ -1,105 +1,151 @@ require 'spec_helper' require 'puppet_spec/files' require 'puppet/pops' require 'puppet/loaders' +describe 'loader helper classes' do + it 'NamedEntry holds values and is frozen' do + ne = Puppet::Pops::Loader::Loader::NamedEntry.new('name', 'value', 'origin') + expect(ne.frozen?).to be_true + expect(ne.typed_name).to eql('name') + expect(ne.origin).to eq('origin') + expect(ne.value).to eq('value') + end + + it 'TypedName holds values and is frozen' do + tn = Puppet::Pops::Loader::Loader::TypedName.new(:function, '::foo::bar') + expect(tn.frozen?).to be_true + expect(tn.type).to eq(:function) + expect(tn.name_parts).to eq(['foo', 'bar']) + expect(tn.name).to eq('foo::bar') + expect(tn.qualified).to be_true + end +end + describe 'loaders' do include PuppetSpec::Files let(:module_without_metadata) { File.join(config_dir('wo_metadata_module'), 'modules') } let(:module_with_metadata) { File.join(config_dir('single_module'), 'modules') } let(:dependent_modules_with_metadata) { config_dir('dependent_modules_with_metadata') } let(:empty_test_env) { environment_for() } # Loaders caches the puppet_system_loader, must reset between tests before(:each) { Puppet::Pops::Loaders.clear() } it 'creates a puppet_system loader' do loaders = Puppet::Pops::Loaders.new(empty_test_env) expect(loaders.puppet_system_loader()).to be_a(Puppet::Pops::Loader::ModuleLoaders::FileBased) end it 'creates an environment loader' do loaders = Puppet::Pops::Loaders.new(empty_test_env) expect(loaders.public_environment_loader()).to be_a(Puppet::Pops::Loader::SimpleEnvironmentLoader) expect(loaders.public_environment_loader().to_s).to eql("(SimpleEnvironmentLoader 'environment:*test*')") expect(loaders.private_environment_loader()).to be_a(Puppet::Pops::Loader::DependencyLoader) expect(loaders.private_environment_loader().to_s).to eql("(DependencyLoader 'environment' [])") end it 'can load 3x system functions' do Puppet[:biff] = true loaders = Puppet::Pops::Loaders.new(empty_test_env) puppet_loader = loaders.puppet_system_loader() function = puppet_loader.load_typed(typed_name(:function, 'sprintf')).value expect(function.class.name).to eq('sprintf') expect(function).to be_a(Puppet::Functions::Function) end + it 'can load 3x system functions more than once' do + Puppet[:biff] = true + loaders = Puppet::Pops::Loaders.new(empty_test_env) + puppet_loader = loaders.puppet_system_loader() + + function = puppet_loader.load_typed(typed_name(:function, 'sprintf')).value + + expect(function.class.name).to eq('sprintf') + expect(function).to be_a(Puppet::Functions::Function) + + function = puppet_loader.load_typed(typed_name(:function, 'sprintf')).value + expect(function.class.name).to eq('sprintf') + expect(function).to be_a(Puppet::Functions::Function) + end + it 'can load a function using a qualified or unqualified name from a module with metadata' do loaders = Puppet::Pops::Loaders.new(environment_for(module_with_metadata)) modulea_loader = loaders.public_loader_for_module('modulea') unqualified_function = modulea_loader.load_typed(typed_name(:function, 'rb_func_a')).value qualified_function = modulea_loader.load_typed(typed_name(:function, 'modulea::rb_func_a')).value expect(unqualified_function).to be_a(Puppet::Functions::Function) expect(qualified_function).to be_a(Puppet::Functions::Function) expect(unqualified_function.class.name).to eq('rb_func_a') expect(qualified_function.class.name).to eq('modulea::rb_func_a') end it 'can load a function with a qualified name from module without metadata' do loaders = Puppet::Pops::Loaders.new(environment_for(module_without_metadata)) moduleb_loader = loaders.public_loader_for_module('moduleb') function = moduleb_loader.load_typed(typed_name(:function, 'moduleb::rb_func_b')).value expect(function).to be_a(Puppet::Functions::Function) expect(function.class.name).to eq('moduleb::rb_func_b') end it 'cannot load an unqualified function from a module without metadata' do loaders = Puppet::Pops::Loaders.new(environment_for(module_without_metadata)) moduleb_loader = loaders.public_loader_for_module('moduleb') expect(moduleb_loader.load_typed(typed_name(:function, 'rb_func_b'))).to be_nil end it 'makes all other modules visible to a module without metadata' do env = environment_for(module_with_metadata, module_without_metadata) loaders = Puppet::Pops::Loaders.new(env) moduleb_loader = loaders.private_loader_for_module('moduleb') function = moduleb_loader.load_typed(typed_name(:function, 'moduleb::rb_func_b')).value expect(function.call({})).to eql("I am modulea::rb_func_a() + I am moduleb::rb_func_b()") end it 'makes dependent modules visible to a module with metadata' do env = environment_for(dependent_modules_with_metadata) loaders = Puppet::Pops::Loaders.new(env) moduleb_loader = loaders.private_loader_for_module('user') function = moduleb_loader.load_typed(typed_name(:function, 'user::caller')).value expect(function.call({})).to eql("usee::callee() was told 'passed value' + I am user::caller()") end + it 'can load a function more than once from modules' do + env = environment_for(dependent_modules_with_metadata) + loaders = Puppet::Pops::Loaders.new(env) + + moduleb_loader = loaders.private_loader_for_module('user') + function = moduleb_loader.load_typed(typed_name(:function, 'user::caller')).value + expect(function.call({})).to eql("usee::callee() was told 'passed value' + I am user::caller()") + + function = moduleb_loader.load_typed(typed_name(:function, 'user::caller')).value + expect(function.call({})).to eql("usee::callee() was told 'passed value' + I am user::caller()") + end + def environment_for(*module_paths) Puppet::Node::Environment.create(:'*test*', module_paths, '') end def typed_name(type, name) Puppet::Pops::Loader::Loader::TypedName.new(type, name) end def config_dir(config_name) my_fixture(config_name) end end diff --git a/tasks/benchmark.rake b/tasks/benchmark.rake index 7456c5c0a..69c0fc27c 100644 --- a/tasks/benchmark.rake +++ b/tasks/benchmark.rake @@ -1,109 +1,144 @@ require 'benchmark' require 'tmpdir' require 'csv' namespace :benchmark do def generate_scenario_tasks(location, name) desc File.read(File.join(location, 'description')) task name => "#{name}:run" + # Load a BenchmarkerTask to handle config of the benchmark + task_handler_file = File.expand_path(File.join(location, 'benchmarker_task.rb')) + if File.exist?(task_handler_file) + require task_handler_file + run_args = BenchmarkerTask.run_args + else + run_args = [] + end namespace name do task :setup do ENV['ITERATIONS'] ||= '10' ENV['SIZE'] ||= '100' ENV['TARGET'] ||= Dir.mktmpdir(name) ENV['TARGET'] = File.expand_path(ENV['TARGET']) mkdir_p(ENV['TARGET']) require File.expand_path(File.join(location, 'benchmarker.rb')) @benchmark = Benchmarker.new(ENV['TARGET'], ENV['SIZE'].to_i) end task :generate => :setup do @benchmark.generate @benchmark.setup end desc "Run the #{name} scenario." - task :run => :generate do + task :run, [*run_args] => :generate do |_, args| format = if RUBY_VERSION =~ /^1\.8/ Benchmark::FMTSTR else Benchmark::FORMAT end - report = [] + details = [] Benchmark.benchmark(Benchmark::CAPTION, 10, format, "> total:", "> avg:") do |b| times = [] ENV['ITERATIONS'].to_i.times do |i| start_time = Time.now.to_i times << b.report("Run #{i + 1}") do - @benchmark.run + details << @benchmark.run(args) end report << [to_millis(start_time), to_millis(times.last.real), 200, true, name] end sum = times.inject(Benchmark::Tms.new, &:+) [sum, sum / times.length] end write_csv("#{name}.samples", %w{timestamp elapsed responsecode success name}, report) + + # report details, if any were produced + if details[0].is_a?(Array) && details[0][0].is_a?(Benchmark::Tms) + # assume all entries are Tms if the first is + # turn each into a hash of label => tms (since labels are lost when doing arithmetic on Tms) + hashed = details.reduce([]) do |memo, measures| + memo << measures.reduce({}) {|memo2, measure| memo2[measure.label] = measure; memo2} + memo + end + # sum across all hashes + result = {} + + hashed_totals = hashed.reduce {|memo, h| memo.merge(h) {|k, old, new| old + new }} + # average the totals + hashed_totals.keys.each {|k| hashed_totals[k] /= details.length } + min_width = 14 + max_width = (hashed_totals.keys.map(&:length) << min_width).max + puts "\n" + puts sprintf("%2$*1$s %3$s", -max_width, 'Details (avg)', " user system total real") + puts "-" * (46 + max_width) + hashed_totals.sort.each {|k,v| puts sprintf("%2$*1$s %3$s", -max_width, k, v.format) } + end end desc "Profile a single run of the #{name} scenario." - task :profile => :generate do + task :profile, [:warm_up_runs, *run_args] => :generate do |_, args| + warm_up_runs = (args[:warm_up_runs] || '0').to_i + warm_up_runs.times do + @benchmark.run(args) + end + require 'ruby-prof' result = RubyProf.profile do - @benchmark.run + @benchmark.run(args) end printer = RubyProf::CallTreePrinter.new(result) File.open(File.join("callgrind.#{name}.#{Time.now.to_i}.trace"), "w") do |f| printer.print(f) end end def to_millis(seconds) (seconds * 1000).round end def write_csv(file, header, data) CSV.open(file, 'w') do |csv| csv << header data.each do |line| csv << line end end end end end scenarios = [] Dir.glob('benchmarks/*') do |location| name = File.basename(location) scenarios << name generate_scenario_tasks(location, File.basename(location)) end namespace :all do desc "Profile all of the scenarios. (#{scenarios.join(', ')})" task :profile do scenarios.each do |name| sh "rake benchmark:#{name}:profile" end end desc "Run all of the scenarios. (#{scenarios.join(', ')})" task :run do scenarios.each do |name| sh "rake benchmark:#{name}:run" end end end end