diff --git a/benchmarks/defined_types/benchmarker.rb b/benchmarks/defined_types/benchmarker.rb new file mode 100644 index 000000000..77d46e71e --- /dev/null +++ b/benchmarks/defined_types/benchmarker.rb @@ -0,0 +1,73 @@ +require 'erb' +require 'ostruct' +require 'fileutils' +require 'json' + +class Benchmarker + include FileUtils + + def initialize(target, size) + @target = target + @size = size + end + + def setup + require 'puppet' + config = File.join(@target, 'puppet.conf') + Puppet.initialize_settings(['--config', config]) + end + + def run + env = Puppet.lookup(:environments).get('benchmarking') + node = Puppet::Node.new("testing", :environment => env) + Puppet::Resource::Catalog.indirection.find("testing", :use_node => node) + end + + def generate + environment = File.join(@target, 'environments', 'benchmarking') + templates = File.join('benchmarks', 'defined_types') + + mkdir_p(File.join(environment, 'modules')) + mkdir_p(File.join(environment, 'manifests')) + + render(File.join(templates, 'site.pp.erb'), + File.join(environment, 'manifests', 'site.pp'), + :size => @size) + + @size.times do |i| + module_name = "module#{i}" + module_base = File.join(environment, 'modules', module_name) + manifests = File.join(module_base, 'manifests') + + mkdir_p(manifests) + + File.open(File.join(module_base, 'metadata.json'), 'w') do |f| + JSON.dump({ + "types" => [], + "source" => "", + "author" => "Defined Types Benchmark", + "license" => "Apache 2.0", + "version" => "1.0.0", + "description" => "Defined Types benchmark module #{i}", + "summary" => "Just this benchmark module, you know?", + "dependencies" => [], + }, f) + end + + render(File.join(templates, 'module', 'testing.pp.erb'), + File.join(manifests, 'testing.pp'), + :name => module_name) + end + + render(File.join(templates, 'puppet.conf.erb'), + File.join(@target, 'puppet.conf'), + :location => @target) + end + + def render(erb_file, output_file, bindings) + site = ERB.new(File.read(erb_file)) + File.open(output_file, 'w') do |fh| + fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) + end + end +end diff --git a/benchmarks/defined_types/description b/benchmarks/defined_types/description new file mode 100644 index 000000000..30c8b7fa1 --- /dev/null +++ b/benchmarks/defined_types/description @@ -0,0 +1,3 @@ +Benchmark scenario: heavy use of defined types +Benchmark target: catalog compilation + diff --git a/benchmarks/defined_types/module/testing.pp.erb b/benchmarks/defined_types/module/testing.pp.erb new file mode 100644 index 000000000..e723561d5 --- /dev/null +++ b/benchmarks/defined_types/module/testing.pp.erb @@ -0,0 +1,3 @@ +define <%= name %>::testing { + notify { "in <%= name %>: $title": } +} diff --git a/benchmarks/defined_types/puppet.conf.erb b/benchmarks/defined_types/puppet.conf.erb new file mode 100644 index 000000000..e0c5d8588 --- /dev/null +++ b/benchmarks/defined_types/puppet.conf.erb @@ -0,0 +1,3 @@ +confdir = <%= location %> +vardir = <%= location %> +environmentpath = <%= File.join(location, 'environments') %> diff --git a/benchmarks/defined_types/site.pp.erb b/benchmarks/defined_types/site.pp.erb new file mode 100644 index 000000000..338b635b2 --- /dev/null +++ b/benchmarks/defined_types/site.pp.erb @@ -0,0 +1,4 @@ +<% size.times do |i| %> + module<%= i %>::testing { "first": } + module<%= i %>::testing{ "second": } +<% end %> diff --git a/benchmarks/many_modules/benchmarker.rb b/benchmarks/many_modules/benchmarker.rb new file mode 100644 index 000000000..b682054d1 --- /dev/null +++ b/benchmarks/many_modules/benchmarker.rb @@ -0,0 +1,78 @@ +require 'erb' +require 'ostruct' +require 'fileutils' +require 'json' + +class Benchmarker + include FileUtils + + def initialize(target, size) + @target = target + @size = size + end + + def setup + require 'puppet' + config = File.join(@target, 'puppet.conf') + Puppet.initialize_settings(['--config', config]) + end + + def run + env = Puppet::Node::Environment.new('benchmarking') +# env = Puppet.lookup(:environments).get('benchmarking') + node = Puppet::Node.new("testing", :environment => env) + Puppet::Resource::Catalog.indirection.find("testing", :use_node => node) + end + + def generate + environment = File.join(@target, 'environments', 'benchmarking') + templates = File.join('benchmarks', 'many_modules') + + mkdir_p(File.join(environment, 'modules')) + mkdir_p(File.join(environment, 'manifests')) + + render(File.join(templates, 'site.pp.erb'), + File.join(environment, 'manifests', 'site.pp'), + :size => @size) + + @size.times do |i| + module_name = "module#{i}" + module_base = File.join(environment, 'modules', module_name) + manifests = File.join(module_base, 'manifests') + + mkdir_p(manifests) + + File.open(File.join(module_base, 'metadata.json'), 'w') do |f| + JSON.dump({ + "types" => [], + "source" => "", + "author" => "ManyModules Benchmark", + "license" => "Apache 2.0", + "version" => "1.0.0", + "description" => "Many Modules benchmark module #{i}", + "summary" => "Just this benchmark module, you know?", + "dependencies" => [], + }, f) + end + + render(File.join(templates, 'module', 'init.pp.erb'), + File.join(manifests, 'init.pp'), + :name => module_name) + + render(File.join(templates, 'module', 'internal.pp.erb'), + File.join(manifests, 'internal.pp'), + :name => module_name) + end + + render(File.join(templates, 'puppet.conf.erb'), + File.join(@target, 'puppet.conf'), + :location => @target) + end + + def render(erb_file, output_file, bindings) + site = ERB.new(File.read(erb_file)) + File.open(output_file, 'w') do |fh| + fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding })) + end + end +end diff --git a/benchmarks/many_modules/description b/benchmarks/many_modules/description new file mode 100644 index 000000000..a2b58ebe7 --- /dev/null +++ b/benchmarks/many_modules/description @@ -0,0 +1,3 @@ +Benchmark scenario: many manifests spread across many modules. +Benchmark target: catalog compilation. + diff --git a/benchmarks/many_modules/module/init.pp.erb b/benchmarks/many_modules/module/init.pp.erb new file mode 100644 index 000000000..49fca07ba --- /dev/null +++ b/benchmarks/many_modules/module/init.pp.erb @@ -0,0 +1,3 @@ +class <%= name %> { + class { "<%= name %>::internal": } +} diff --git a/benchmarks/many_modules/module/internal.pp.erb b/benchmarks/many_modules/module/internal.pp.erb new file mode 100644 index 000000000..83a36947d --- /dev/null +++ b/benchmarks/many_modules/module/internal.pp.erb @@ -0,0 +1,3 @@ +class <%= name %>::internal { + notify { "<%= name %>::internal": } +} diff --git a/benchmarks/many_modules/puppet.conf.erb b/benchmarks/many_modules/puppet.conf.erb new file mode 100644 index 000000000..e0c5d8588 --- /dev/null +++ b/benchmarks/many_modules/puppet.conf.erb @@ -0,0 +1,3 @@ +confdir = <%= location %> +vardir = <%= location %> +environmentpath = <%= File.join(location, 'environments') %> diff --git a/benchmarks/many_modules/site.pp.erb b/benchmarks/many_modules/site.pp.erb new file mode 100644 index 000000000..9f6c9668a --- /dev/null +++ b/benchmarks/many_modules/site.pp.erb @@ -0,0 +1,3 @@ +<% size.times do |i| %> + include module<%= i %> +<% end %> diff --git a/benchmarks/system_startup/benchmarker.rb b/benchmarks/system_startup/benchmarker.rb new file mode 100644 index 000000000..c48a878dd --- /dev/null +++ b/benchmarks/system_startup/benchmarker.rb @@ -0,0 +1,17 @@ +class Benchmarker + def initialize(target, size) + end + + def setup + end + + def generate + end + + def run + # Just running help is probably a good proxy of a full startup. + # Simply asking for the version might also be good, but it would miss all + # of the app searching and loading parts + `puppet help` + end +end diff --git a/benchmarks/system_startup/description b/benchmarks/system_startup/description new file mode 100644 index 000000000..85c6dea3a --- /dev/null +++ b/benchmarks/system_startup/description @@ -0,0 +1,2 @@ +Benchmark scenario: running puppet commands from the CLI +Benchmark target: overhead of loading puppet diff --git a/lib/puppet/metatype/manager.rb b/lib/puppet/metatype/manager.rb index 3d640879f..cef899b6e 100644 --- a/lib/puppet/metatype/manager.rb +++ b/lib/puppet/metatype/manager.rb @@ -1,180 +1,179 @@ require 'puppet' require 'puppet/util/classgen' require 'puppet/node/environment' # This module defines methods dealing with Type management. # This module gets included into the Puppet::Type class, it's just split out here for clarity. # @api public # module Puppet::MetaType module Manager include Puppet::Util::ClassGen # An implementation specific method that removes all type instances during testing. # @note Only use this method for testing purposes. # @api private # def allclear @types.each { |name, type| type.clear } end # Iterates over all already loaded Type subclasses. # @yield [t] a block receiving each type # @yieldparam t [Puppet::Type] each defined type # @yieldreturn [Object] the last returned object is also returned from this method # @return [Object] the last returned value from the block. def eachtype @types.each do |name, type| # Only consider types that have names #if ! type.parameters.empty? or ! type.validproperties.empty? yield type #end end end # Loads all types. # @note Should only be used for purposes such as generating documentation as this is potentially a very # expensive operation. # @return [void] # def loadall typeloader.loadall end # Defines a new type or redefines an existing type with the given name. # A convenience method on the form `new` where name is the name of the type is also created. # (If this generated method happens to clash with an existing method, a warning is issued and the original # method is kept). # # @param name [String] the name of the type to create or redefine. # @param options [Hash] options passed on to {Puppet::Util::ClassGen#genclass} as the option `:attributes` after # first having removed any present `:parent` option. # @option options [Puppet::Type] :parent the parent (super type) of this type. If nil, the default is # Puppet::Type. This option is not passed on as an attribute to genclass. # @yield [ ] a block evaluated in the context of the created class, thus allowing further detailing of # that class. # @return [Class] the created subclass # @see Puppet::Util::ClassGen.genclass # # @dsl type # @api public def newtype(name, options = {}, &block) # Handle backward compatibility unless options.is_a?(Hash) Puppet.warning "Puppet::Type.newtype(#{name}) now expects a hash as the second argument, not #{options.inspect}" options = {:parent => options} end # First make sure we don't have a method sitting around name = name.intern newmethod = "new#{name}" # Used for method manipulation. selfobj = singleton_class @types ||= {} if @types.include?(name) if self.respond_to?(newmethod) # Remove the old newmethod selfobj.send(:remove_method,newmethod) end end options = symbolize_options(options) if parent = options[:parent] options.delete(:parent) end # Then create the class. klass = genclass( name, :parent => (parent || Puppet::Type), :overwrite => true, :hash => @types, :attributes => options, &block ) # Now define a "new" method for convenience. if self.respond_to? newmethod # Refuse to overwrite existing methods like 'newparam' or 'newtype'. Puppet.warning "'new#{name.to_s}' method already exists; skipping" else selfobj.send(:define_method, newmethod) do |*args| klass.new(*args) end end # If they've got all the necessary methods defined and they haven't # already added the property, then do so now. klass.ensurable if klass.ensurable? and ! klass.validproperty?(:ensure) # Now set up autoload any providers that might exist for this type. klass.providerloader = Puppet::Util::Autoload.new(klass, "puppet/provider/#{klass.name.to_s}") # We have to load everything so that we can figure out the default provider. klass.providerloader.loadall klass.providify unless klass.providers.empty? klass end # Removes an existing type. # @note Only use this for testing. # @api private def rmtype(name) # Then create the class. rmclass(name, :hash => @types) singleton_class.send(:remove_method, "new#{name}") if respond_to?("new#{name}") end # Returns a Type instance by name. # This will load the type if not already defined. # @param [String, Symbol] name of the wanted Type # @return [Puppet::Type, nil] the type or nil if the type was not defined and could not be loaded # def type(name) @types ||= {} # We are overwhelmingly symbols here, which usually match, so it is worth # having this special-case to return quickly. Like, 25K symbols vs. 300 # strings in this method. --daniel 2012-07-17 return @types[name] if @types[name] # Try mangling the name, if it is a string. if name.is_a? String name = name.downcase.intern return @types[name] if @types[name] end - # Try loading the type. if typeloader.load(name, Puppet::Node::Environment.current) Puppet.warning "Loaded puppet/type/#{name} but no class was created" unless @types.include? name end # ...and I guess that is that, eh. return @types[name] end # Creates a loader for Puppet types. # Defaults to an instance of {Puppet::Util::Autoload} if no other auto loader has been set. # @return [Puppet::Util::Autoload] the loader to use. # @api private def typeloader unless defined?(@typeloader) @typeloader = Puppet::Util::Autoload.new(self, "puppet/type", :wrap => false) end @typeloader end end end diff --git a/tasks/benchmark.rake b/tasks/benchmark.rake new file mode 100644 index 000000000..bd1dcb9d7 --- /dev/null +++ b/tasks/benchmark.rake @@ -0,0 +1,110 @@ +require 'benchmark' +require 'tmpdir' +require 'csv' + +namespace :benchmark do + def generate_scenario_tasks(location, name) + desc File.read(File.join(location, 'description')) + task name => "#{name}:run" + + namespace name do + task :setup do + ENV['ITERATIONS'] ||= '10' + ENV['SIZE'] ||= '100' + ENV['TARGET'] ||= Dir.mktmpdir(name) + ENV['TARGET'] = File.expand_path(ENV['TARGET']) + + mkdir_p(ENV['TARGET']) + + require File.expand_path(File.join(location, 'benchmarker.rb')) + + @benchmark = Benchmarker.new(ENV['TARGET'], ENV['SIZE'].to_i) + end + + desc "Generate the #{name} scenario." + task :generate => :setup do + @benchmark.generate + @benchmark.setup + end + + desc "Run the #{name} scenario." + task :run => :generate do + format = if RUBY_VERSION =~ /^1\.8/ + Benchmark::FMTSTR + else + Benchmark::FORMAT + end + + report = [] + Benchmark.benchmark(Benchmark::CAPTION, 10, format, "> total:", "> avg:") do |b| + times = [] + ENV['ITERATIONS'].to_i.times do |i| + start_time = Time.now.to_i + times << b.report("Run #{i + 1}") do + @benchmark.run + end + report << [to_millis(start_time), to_millis(times.last.real), 200, true, name] + end + + sum = times.inject(Benchmark::Tms.new, &:+) + + [sum, sum / times.length] + end + + write_csv("#{name}.samples", + %w{timestamp elapsed responsecode success name}, + report) + end + + desc "Profile a single run of the #{name} scenario." + task :profile => :generate do + require 'ruby-prof' + + result = RubyProf.profile do + @benchmark.run + end + + printer = RubyProf::CallTreePrinter.new(result) + File.open(File.join("callgrind.#{name}.#{Time.now.to_i}.trace"), "w") do |f| + printer.print(f) + end + end + + def to_millis(seconds) + (seconds * 1000).round + end + + def write_csv(file, header, data) + CSV.open(file, 'w') do |csv| + csv << header + data.each do |line| + csv << line + end + end + end + end + end + + scenarios = [] + Dir.glob('benchmarks/*') do |location| + name = File.basename(location) + scenarios << name + generate_scenario_tasks(location, File.basename(location)) + end + + namespace :all do + desc "Profile all of the scenarios. (#{scenarios.join(', ')})" + task :profile do + scenarios.each do |name| + sh "rake benchmark:#{name}:profile" + end + end + + desc "Run all of the scenarios. (#{scenarios.join(', ')})" + task :run do + scenarios.each do |name| + sh "rake benchmark:#{name}:run" + end + end + end +end