diff --git a/benchmarks/evaluations/benchmarker.rb b/benchmarks/evaluations/benchmarker.rb index 368744e0d..968786501 100644 --- a/benchmarks/evaluations/benchmarker.rb +++ b/benchmarks/evaluations/benchmarker.rb @@ -1,89 +1,134 @@ require 'erb' require 'ostruct' require 'fileutils' require 'json' class Benchmarker include FileUtils def initialize(target, size) @target = target @size = size @micro_benchmarks = {} @parsecount = 100 @evalcount = 100 end def setup require 'puppet' require 'puppet/pops' config = File.join(@target, 'puppet.conf') Puppet.initialize_settings(['--config', config]) manifests = File.join('benchmarks', 'evaluations', 'manifests') Dir.foreach(manifests) do |f| if f =~ /^(.*)\.pp$/ @micro_benchmarks[$1] = File.read(File.join(manifests, f)) end end # Run / Evaluate the common puppet logic @env = Puppet.lookup(:environments).get('benchmarking') @node = Puppet::Node.new("testing", :environment => @env) @parser = Puppet::Pops::Parser::EvaluatingParser::Transitional.new @compiler = Puppet::Parser::Compiler.new(@node) @scope = @compiler.topscope # Perform a portion of what a compile does (just enough to evaluate the site.pp logic) @compiler.catalog.environment_instance = @compiler.environment @compiler.send(:evaluate_main) + + # Then pretend we are running as part of a compilation + Puppet.push_context(@compiler.context_overrides, "Benchmark masquerading as compiler configured context") end def run(args = {:detail=>'all'}) details = args[:detail] measurements = [] @micro_benchmarks.each do |name, source| # skip if all but the wanted if a single benchmark is wanted next unless details == 'all' || details == name measurements << Benchmark.measure("#{name} parse") do 1..@parsecount.times { @parser.parse_string(source, name) } end model = @parser.parse_string(source, name) measurements << Benchmark.measure("#{name} eval") do 1..@evalcount.times do begin # Run each in a local scope scope_memo = @scope.ephemeral_level @scope.new_ephemeral(true) @parser.evaluate(@scope, model) ensure # Toss the created local scope @scope.unset_ephemeral_var(scope_memo) end end end end measurements end def generate environment = File.join(@target, 'environments', 'benchmarking') templates = File.join('benchmarks', 'evaluations') mkdir_p(File.join(environment, 'modules')) mkdir_p(File.join(environment, 'manifests')) render(File.join(templates, 'site.pp.erb'), File.join(environment, 'manifests', 'site.pp'),{}) render(File.join(templates, 'puppet.conf.erb'), File.join(@target, 'puppet.conf'), :location => @target) + + # Generate one module with a 3x function and a 4x function (namespaces) + module_name = "module1" + module_base = File.join(environment, 'modules', module_name) + manifests = File.join(module_base, 'manifests') + mkdir_p(manifests) + functions_3x = File.join(module_base, 'lib', 'puppet', 'parser', 'functions') + functions_4x = File.join(module_base, 'lib', 'puppet', 'functions') + mkdir_p(functions_3x) + mkdir_p(functions_4x) + + File.open(File.join(module_base, 'metadata.json'), 'w') do |f| + JSON.dump({ + "types" => [], + "source" => "", + "author" => "Evaluations Benchmark", + "license" => "Apache 2.0", + "version" => "1.0.0", + "description" => "Evaluations Benchmark module 1", + "summary" => "Module with supporting logic for evaluations benchmark", + "dependencies" => [], + }, f) + end + + render(File.join(templates, 'module', 'init.pp.erb'), + File.join(manifests, 'init.pp'), + :name => module_name) + + render(File.join(templates, 'module', 'func3.rb.erb'), + File.join(functions_3x, 'func3.rb'), + :name => module_name) + + # namespaced function + mkdir_p(File.join(functions_4x, module_name)) + render(File.join(templates, 'module', 'module1_func4.rb.erb'), + File.join(functions_4x, module_name, 'func4.rb'), + :name => module_name) + + # non namespaced + render(File.join(templates, 'module', 'func4.rb.erb'), + File.join(functions_4x, 'func4.rb'), + :name => module_name) end def render(erb_file, output_file, bindings) site = ERB.new(File.read(erb_file)) File.open(output_file, 'w') do |fh| fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) end end end diff --git a/benchmarks/evaluations/manifests/fcall_4x.pp b/benchmarks/evaluations/manifests/assert_type.pp similarity index 100% copy from benchmarks/evaluations/manifests/fcall_4x.pp copy to benchmarks/evaluations/manifests/assert_type.pp diff --git a/benchmarks/evaluations/manifests/fcall_3x.pp b/benchmarks/evaluations/manifests/fcall_3x.pp index 3b2e079b1..9689ca97a 100644 --- a/benchmarks/evaluations/manifests/fcall_3x.pp +++ b/benchmarks/evaluations/manifests/fcall_3x.pp @@ -1,6 +1,6 @@ $tmp = [ -sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), -sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), -sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), -sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), sprintf('%s',a), +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), +func3(x,y), func3(x,y), func3(x,y), func3(x,y), func3(x,y), ] diff --git a/benchmarks/evaluations/manifests/fcall_4x.pp b/benchmarks/evaluations/manifests/fcall_4x.pp index 0be757281..a7b7b712a 100644 --- a/benchmarks/evaluations/manifests/fcall_4x.pp +++ b/benchmarks/evaluations/manifests/fcall_4x.pp @@ -1,6 +1,6 @@ $tmp = [ -assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), -assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), -assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), -assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), assert_type(Integer,1), +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), +func4(x,y), func4(x,y), func4(x,y), func4(x,y), func4(x,y), ] diff --git a/benchmarks/evaluations/manifests/fcall_ns4x.pp b/benchmarks/evaluations/manifests/fcall_ns4x.pp new file mode 100644 index 000000000..d59de8539 --- /dev/null +++ b/benchmarks/evaluations/manifests/fcall_ns4x.pp @@ -0,0 +1,6 @@ +$tmp = [ +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), module1::func4(x,y), +] diff --git a/benchmarks/evaluations/module/func3.rb.erb b/benchmarks/evaluations/module/func3.rb.erb new file mode 100644 index 000000000..6b814a0d0 --- /dev/null +++ b/benchmarks/evaluations/module/func3.rb.erb @@ -0,0 +1,8 @@ +Puppet::Parser::Functions::newfunction(:func3, + :arity => 2, + :doc => "Blah blah, this is a lot of documentation that the ruby parser must deal with + because documentation is part of what is loaded at runtime. Some functions have + very little documentation, and some have quite a lot. This simulates documentation + that is slightly longer than the shortest ones.") do |vals| + # produces nil +end diff --git a/benchmarks/evaluations/module/func4.rb.erb b/benchmarks/evaluations/module/func4.rb.erb new file mode 100644 index 000000000..b923ec4f5 --- /dev/null +++ b/benchmarks/evaluations/module/func4.rb.erb @@ -0,0 +1,9 @@ +# Blah blah, this is a lot of documentation that the ruby parser must deal with +# because documentation is part of what is loaded at runtime. Some functions have +# very little documentation, and some have quite a lot. This simulates documentation +# that is slightly longer than the shortest ones. +# +Puppet::Functions.create_function(:func4) do + def func4(x,y) + end +end \ No newline at end of file diff --git a/benchmarks/evaluations/module/init.pp.erb b/benchmarks/evaluations/module/init.pp.erb new file mode 100644 index 000000000..755657c8a --- /dev/null +++ b/benchmarks/evaluations/module/init.pp.erb @@ -0,0 +1 @@ +# empty init (for now) \ No newline at end of file diff --git a/benchmarks/evaluations/module/module1_func4.rb.erb b/benchmarks/evaluations/module/module1_func4.rb.erb new file mode 100644 index 000000000..1891156c0 --- /dev/null +++ b/benchmarks/evaluations/module/module1_func4.rb.erb @@ -0,0 +1,9 @@ +# Blah blah, this is a lot of documentation that the ruby parser must deal with +# because documentation is part of what is loaded at runtime. Some functions have +# very little documentation, and some have quite a lot. This simulates documentation +# that is slightly longer than the shortest ones. +# +Puppet::Functions.create_function(:'<%= name %>::func4') do + def func4(x,y) + end +end \ No newline at end of file diff --git a/benchmarks/evaluations/puppet.conf.erb b/benchmarks/evaluations/puppet.conf.erb index d9f1db9df..d0bf4b2f1 100644 --- a/benchmarks/evaluations/puppet.conf.erb +++ b/benchmarks/evaluations/puppet.conf.erb @@ -1,7 +1,6 @@ confdir = <%= location %> vardir = <%= location %> environmentpath = <%= File.join(location, 'environments') %> environment_timeout = '0' parser = future strict_variables = true -