diff --git a/lib/puppet/rails.rb b/lib/puppet/rails.rb index e006f8190..5175acc8f 100644 --- a/lib/puppet/rails.rb +++ b/lib/puppet/rails.rb @@ -1,136 +1,137 @@ # Load the appropriate libraries, or set a class indicating they aren't available require 'facter' require 'puppet' module Puppet::Rails + TIME_DEBUG = true def self.connect # This global init does not work for testing, because we remove # the state dir on every test. return if ActiveRecord::Base.connected? Puppet.settings.use(:main, :rails, :puppetmasterd) ActiveRecord::Base.logger = Logger.new(Puppet[:railslog]) begin loglevel = Logger.const_get(Puppet[:rails_loglevel].upcase) ActiveRecord::Base.logger.level = loglevel rescue => detail Puppet.warning "'%s' is not a valid Rails log level; using debug" % Puppet[:rails_loglevel] ActiveRecord::Base.logger.level = Logger::DEBUG end ActiveRecord::Base.allow_concurrency = true ActiveRecord::Base.verify_active_connections! begin ActiveRecord::Base.establish_connection(database_arguments()) rescue => detail if Puppet[:trace] puts detail.backtrace end raise Puppet::Error, "Could not connect to database: %s" % detail end end # The arguments for initializing the database connection. def self.database_arguments adapter = Puppet[:dbadapter] args = {:adapter => adapter, :log_level => Puppet[:rails_loglevel]} case adapter when "sqlite3" args[:dbfile] = Puppet[:dblocation] when "mysql", "postgresql" args[:host] = Puppet[:dbserver] unless Puppet[:dbserver].empty? args[:username] = Puppet[:dbuser] unless Puppet[:dbuser].empty? args[:password] = Puppet[:dbpassword] unless Puppet[:dbpassword].empty? args[:database] = Puppet[:dbname] socket = Puppet[:dbsocket] args[:socket] = socket unless socket.empty? else raise ArgumentError, "Invalid db adapter %s" % adapter end args end # Set up our database connection. It'd be nice to have a "use" system # that could make callbacks. def self.init unless Puppet.features.rails? raise Puppet::DevError, "No activerecord, cannot init Puppet::Rails" end connect() unless ActiveRecord::Base.connection.tables.include?("resources") require 'puppet/rails/database/schema' Puppet::Rails::Schema.init end if Puppet[:dbmigrate] migrate() end end # Migrate to the latest db schema. def self.migrate dbdir = nil $:.each { |d| tmp = File.join(d, "puppet/rails/database") if FileTest.directory?(tmp) dbdir = tmp break end } unless dbdir raise Puppet::Error, "Could not find Puppet::Rails database dir" end unless ActiveRecord::Base.connection.tables.include?("resources") raise Puppet::Error, "Database has problems, can't migrate." end Puppet.notice "Migrating" begin ActiveRecord::Migrator.migrate(dbdir) rescue => detail if Puppet[:trace] puts detail.backtrace end raise Puppet::Error, "Could not migrate database: %s" % detail end end # Tear down the database. Mostly only used during testing. def self.teardown unless Puppet.features.rails? raise Puppet::DevError, "No activerecord, cannot init Puppet::Rails" end Puppet.settings.use(:puppetmasterd, :rails) begin ActiveRecord::Base.establish_connection(database_arguments()) rescue => detail if Puppet[:trace] puts detail.backtrace end raise Puppet::Error, "Could not connect to database: %s" % detail end ActiveRecord::Base.connection.tables.each do |t| ActiveRecord::Base.connection.drop_table t end end end if Puppet.features.rails? require 'puppet/rails/host' end diff --git a/lib/puppet/rails/benchmark.rb b/lib/puppet/rails/benchmark.rb new file mode 100644 index 000000000..aadacc243 --- /dev/null +++ b/lib/puppet/rails/benchmark.rb @@ -0,0 +1,69 @@ +require 'benchmark' +module Puppet::Rails::Benchmark + $benchmarks = {:accumulated => {}} + + def time_debug? + Puppet::Rails::TIME_DEBUG + end + + def railsmark(message) + result = nil + seconds = Benchmark.realtime { result = yield } + Puppet.debug(message + " in %0.2f seconds" % seconds) + + $benchmarks[message] = seconds if time_debug? + result + end + + def sometimes_benchmark(message) + unless Puppet::Rails::TIME_DEBUG + return yield + end + + railsmark(message) { yield } + end + + # Collect partial benchmarks to be logged when they're + # all done. + # These are always low-level debugging so we only + # print them if time_debug is enabled. + def accumulate_benchmark(message, label) + unless time_debug? + return yield + end + + $benchmarks[:accumulated][message] ||= Hash.new(0) + $benchmarks[:accumulated][message][label] += Benchmark.realtime { yield } + end + + # Log the accumulated marks. + def log_accumulated_marks(message) + return unless time_debug? + + if $benchmarks[:accumulated].empty? or $benchmarks[:accumulated][message].nil? or $benchmarks[:accumulated][message].empty? + return + end + + $benchmarks[:accumulated][message].each do |label, value| + Puppet.debug(message + ("(%s)" % label) + (" in %0.2f seconds" % value)) + end + end + + def write_benchmarks + return unless time_debug? + + branch = %x{git branch}.split("\n").find { |l| l =~ /^\*/ }.sub("* ", '') + + file = "/tmp/time_debugging.yaml" + + require 'yaml' + + if FileTest.exist?(file) + data = YAML.load_file(file) + else + data = {} + end + data[branch] = $benchmarks + File.open(file, "w") { |f| f.print YAML.dump(data) } + end +end diff --git a/lib/puppet/rails/host.rb b/lib/puppet/rails/host.rb index 8f4c7375c..578974555 100644 --- a/lib/puppet/rails/host.rb +++ b/lib/puppet/rails/host.rb @@ -1,311 +1,312 @@ require 'puppet/rails/resource' require 'puppet/rails/fact_name' require 'puppet/rails/source_file' +require 'puppet/rails/benchmark' require 'puppet/util/rails/collection_merger' class Puppet::Rails::Host < ActiveRecord::Base + include Puppet::Rails::Benchmark + extend Puppet::Rails::Benchmark include Puppet::Util include Puppet::Util::CollectionMerger has_many :fact_values, :dependent => :destroy has_many :fact_names, :through => :fact_values belongs_to :source_file has_many :resources, :dependent => :destroy # If the host already exists, get rid of its objects def self.clean(host) if obj = self.find_by_name(host) obj.rails_objects.clear return obj else return nil end end def self.from_puppet(node) host = find_by_name(node.name) || new(:name => node.name) {"ipaddress" => "ip", "environment" => "environment"}.each do |myparam, itsparam| if value = node.send(myparam) host.send(itsparam + "=", value) end end host end # Store our host in the database. def self.store(node, resources) args = {} host = nil - transaction do - #unless host = find_by_name(name) - seconds = Benchmark.realtime { - unless host = find_by_name(node.name) - host = new(:name => node.name) + railsmark "Stored node" do + transaction do + #unless host = find_by_name(name) + + sometimes_benchmark("Searched for host")do + unless host = find_by_name(node.name) + host = new(:name => node.name) + end + end + if ip = node.parameters["ipaddress"] + host.ip = ip end - } - Puppet.debug("Searched for host in %0.2f seconds" % seconds) - if ip = node.parameters["ipaddress"] - host.ip = ip - end - if env = node.environment - host.environment = env - end + if env = node.environment + host.environment = env + end - # Store the facts into the database. - host.merge_facts(node.parameters) + # Store the facts into the database. + host.merge_facts(node.parameters) - seconds = Benchmark.realtime { - host.merge_resources(resources) - } - Puppet.debug("Handled resources in %0.2f seconds" % seconds) + sometimes_benchmark("Handled resources") { + host.merge_resources(resources) + } - host.last_compile = Time.now + host.last_compile = Time.now + + sometimes_benchmark("Saved host") { + host.save + } + end - seconds = Benchmark.realtime { - host.save - } - Puppet.debug("Saved host in %0.2f seconds" % seconds) end + # This only runs if time debugging is enabled. + write_benchmarks + return host end # Return the value of a fact. def fact(name) if fv = self.fact_values.find(:all, :include => :fact_name, :conditions => "fact_names.name = '#{name}'") return fv else return nil end end # returns a hash of fact_names.name => [ fact_values ] for this host. # Note that 'fact_values' is actually a list of the value instances, not # just actual values. def get_facts_hash fact_values = self.fact_values.find(:all, :include => :fact_name) return fact_values.inject({}) do | hash, value | hash[value.fact_name.name] ||= [] hash[value.fact_name.name] << value hash end end # This is *very* similar to the merge_parameters method # of Puppet::Rails::Resource. def merge_facts(facts) db_facts = {} deletions = [] self.fact_values.find(:all, :include => :fact_name).each do |value| deletions << value['id'] and next unless facts.include?(value['name']) # Now store them for later testing. db_facts[value['name']] ||= [] db_facts[value['name']] << value end # Now get rid of any parameters whose value list is different. # This might be extra work in cases where an array has added or lost # a single value, but in the most common case (a single value has changed) # this makes sense. db_facts.each do |name, value_hashes| values = value_hashes.collect { |v| v['value'] } unless values == facts[name] value_hashes.each { |v| deletions << v['id'] } end end # Perform our deletions. Puppet::Rails::FactValue.delete(deletions) unless deletions.empty? # Lastly, add any new parameters. facts.each do |name, value| next if db_facts.include?(name) values = value.is_a?(Array) ? value : [value] values.each do |v| fact_values.build(:value => v, :fact_name => Puppet::Rails::FactName.find_or_create_by_name(name)) end end end # Set our resources. def merge_resources(list) resources_by_id = nil - seconds = Benchmark.realtime { + sometimes_benchmark("Searched for resources") { resources_by_id = find_resources() } - Puppet.debug("Searched for resources in %0.2f seconds" % seconds) - seconds = Benchmark.realtime { + sometimes_benchmark("Searched for resource params and tags") { find_resources_parameters_tags(resources_by_id) } if id - Puppet.debug("Searched for resource params and tags in %0.2f seconds" % seconds) - seconds = Benchmark.realtime { + sometimes_benchmark("Performed resource comparison") { compare_to_catalog(resources_by_id, list) } - Puppet.debug("Resource comparison took %0.2f seconds" % seconds) end def find_resources resources.find(:all, :include => :source_file).inject({}) do | hash, resource | hash[resource.id] = resource hash end end def find_resources_parameters_tags(resources) # initialize all resource parameters resources.each do |key,resource| resource.params_hash = [] end find_resources_parameters(resources) find_resources_tags(resources) end def compare_to_catalog(existing, list) compiled = list.inject({}) do |hash, resource| hash[resource.ref] = resource hash end resources = nil - seconds = Benchmark.realtime { + sometimes_benchmark("Resource removal") { resources = remove_unneeded_resources(compiled, existing) } - Puppet.debug("Resource removal took %0.2f seconds" % seconds) # Now for all resources in the catalog but not in the db, we're pretty easy. additions = nil - seconds = Benchmark.realtime { + sometimes_benchmark("Resource merger") { additions = perform_resource_merger(compiled, resources) } - Puppet.debug("Resource merger took %0.2f seconds" % seconds) - seconds = Benchmark.realtime { + sometimes_benchmark("Resource addition") { additions.each do |resource| build_rails_resource_from_parser_resource(resource) end } - Puppet.debug("Resource addition took %0.2f seconds" % seconds) end def add_new_resources(additions) additions.each do |resource| Puppet::Rails::Resource.from_parser_resource(self, resource) end end # Turn a parser resource into a Rails resource. def build_rails_resource_from_parser_resource(resource) args = Puppet::Rails::Resource.rails_resource_initial_args(resource) db_resource = self.resources.build(args) # Our file= method does the name to id conversion. db_resource.file = resource.file resource.eachparam do |param| Puppet::Rails::ParamValue.from_parser_param(param).each do |value_hash| db_resource.param_values.build(value_hash) end end resource.tags.each { |tag| db_resource.add_resource_tag(tag) } return db_resource end def perform_resource_merger(compiled, resources) return compiled.values if resources.empty? # Now for all resources in the catalog but not in the db, we're pretty easy. times = Hash.new(0) additions = [] compiled.each do |ref, resource| if db_resource = resources[ref] db_resource.merge_parser_resource(resource).each do |name, time| times[name] += time end else additions << resource end end times.each do |name, time| Puppet.debug("Resource merger(%s) took %0.2f seconds" % [name, time]) end return additions end def remove_unneeded_resources(compiled, existing) deletions = [] resources = {} existing.each do |id, resource| # it seems that it can happen (see bug #2010) some resources are duplicated in the # database (ie logically corrupted database), in which case we remove the extraneous # entries. if resources.include?(resource.ref) deletions << id next end # If the resource is in the db but not in the catalog, mark it # for removal. unless compiled.include?(resource.ref) deletions << id next end resources[resource.ref] = resource end # We need to use 'destroy' here, not 'delete', so that all # dependent objects get removed, too. - Puppet::Rails::Resource.destroy(*deletions) unless deletions.empty? + Puppet::Rails::Resource.destroy(deletions) unless deletions.empty? return resources end def find_resources_parameters(resources) params = Puppet::Rails::ParamValue.find_all_params_from_host(self) # assign each loaded parameters/tags to the resource it belongs to params.each do |param| resources[param['resource_id']].add_param_to_hash(param) if resources.include?(param['resource_id']) end end def find_resources_tags(resources) tags = Puppet::Rails::ResourceTag.find_all_tags_from_host(self) tags.each do |tag| resources[tag['resource_id']].add_tag_to_hash(tag) if resources.include?(tag['resource_id']) end end def update_connect_time self.last_connect = Time.now save end def to_puppet node = Puppet::Node.new(self.name) {"ip" => "ipaddress", "environment" => "environment"}.each do |myparam, itsparam| if value = send(myparam) node.send(itsparam + "=", value) end end node end end diff --git a/lib/puppet/rails/resource.rb b/lib/puppet/rails/resource.rb index d91bb8209..ef1317871 100644 --- a/lib/puppet/rails/resource.rb +++ b/lib/puppet/rails/resource.rb @@ -1,245 +1,243 @@ require 'puppet' require 'puppet/rails/param_name' require 'puppet/rails/param_value' require 'puppet/rails/puppet_tag' require 'puppet/util/rails/collection_merger' class Puppet::Rails::Resource < ActiveRecord::Base include Puppet::Util::CollectionMerger include Puppet::Util::ReferenceSerializer has_many :param_values, :dependent => :destroy, :class_name => "Puppet::Rails::ParamValue" has_many :param_names, :through => :param_values, :class_name => "Puppet::Rails::ParamName" has_many :resource_tags, :dependent => :destroy, :class_name => "Puppet::Rails::ResourceTag" has_many :puppet_tags, :through => :resource_tags, :class_name => "Puppet::Rails::PuppetTag" belongs_to :source_file belongs_to :host @tags = {} def self.tags @tags end # Determine the basic details on the resource. def self.rails_resource_initial_args(resource) return [:type, :title, :line, :exported].inject({}) do |hash, param| # 'type' isn't a valid column name, so we have to use another name. to = (param == :type) ? :restype : param if value = resource.send(param) hash[to] = value end hash end end def add_resource_tag(tag) pt = Puppet::Rails::PuppetTag.accumulate_by_name(tag) resource_tags.build(:puppet_tag => pt) end def file if f = self.source_file return f.filename else return nil end end def file=(file) self.source_file = Puppet::Rails::SourceFile.find_or_create_by_filename(file) end def title unserialize_value(self[:title]) end def add_param_to_hash(param) @params_hash ||= [] @params_hash << param end def add_tag_to_hash(tag) @tags_hash ||= [] @tags_hash << tag end def params_hash=(hash) @params_hash = hash end def tags_hash=(hash) @tags_hash = hash end def [](param) return super || parameter(param) end # Make sure this resource is equivalent to the provided Parser resource. def merge_parser_resource(resource) times = {} times[:attributes] = Benchmark.realtime { merge_attributes(resource) } times[:parameters] = Benchmark.realtime { merge_parameters(resource) } times[:tags] = Benchmark.realtime { merge_tags(resource) } times end def merge_attributes(resource) args = self.class.rails_resource_initial_args(resource) args.each do |param, value| self[param] = value unless resource[param] == value end # Handle file specially if (resource.file and (!resource.file or self.file != resource.file)) self.file = resource.file end end def merge_parameters(resource) catalog_params = {} resource.eachparam do |param| catalog_params[param.name.to_s] = param end db_params = {} deletions = [] - #Puppet::Rails::ParamValue.find_all_params_from_resource(self).each do |value| @params_hash.each do |value| # First remove any parameters our catalog resource doesn't have at all. deletions << value['id'] and next unless catalog_params.include?(value['name']) # Now store them for later testing. db_params[value['name']] ||= [] db_params[value['name']] << value end # Now get rid of any parameters whose value list is different. # This might be extra work in cases where an array has added or lost # a single value, but in the most common case (a single value has changed) # this makes sense. db_params.each do |name, value_hashes| values = value_hashes.collect { |v| v['value'] } unless value_compare(catalog_params[name].value, values) value_hashes.each { |v| deletions << v['id'] } end end # Perform our deletions. Puppet::Rails::ParamValue.delete(deletions) unless deletions.empty? # Lastly, add any new parameters. catalog_params.each do |name, param| next if db_params.include?(name) values = param.value.is_a?(Array) ? param.value : [param.value] values.each do |v| param_values.build(:value => serialize_value(v), :line => param.line, :param_name => Puppet::Rails::ParamName.accumulate_by_name(name)) end end end # Make sure the tag list is correct. def merge_tags(resource) in_db = [] deletions = [] resource_tags = resource.tags - #Puppet::Rails::ResourceTag.find_all_tags_from_resource(self).each do |tag| @tags_hash.each do |tag| deletions << tag['id'] and next unless resource_tags.include?(tag['name']) in_db << tag['name'] end Puppet::Rails::ResourceTag.delete(deletions) unless deletions.empty? (resource_tags - in_db).each do |tag| add_resource_tag(tag) end end def value_compare(v,db_value) v = [v] unless v.is_a?(Array) v == db_value end def name ref() end def parameter(param) if pn = param_names.find_by_name(param) if pv = param_values.find(:first, :conditions => [ 'param_name_id = ?', pn]) return pv.value else return nil end end end def parameters result = get_params_hash result.each do |param, value| if value.is_a?(Array) result[param] = value.collect { |v| v['value'] } else result[param] = value.value end end result end def ref "%s[%s]" % [self[:restype].split("::").collect { |s| s.capitalize }.join("::"), self.title.to_s] end # Returns a hash of parameter names and values, no ActiveRecord instances. def to_hash Puppet::Rails::ParamValue.find_all_params_from_resource(self).inject({}) do |hash, value| hash[value['name']] ||= [] hash[value['name']] << value.value hash end end # Convert our object to a resource. Do not retain whether the object # is exported, though, since that would cause it to get stripped # from the configuration. def to_resource(scope) hash = self.attributes hash["type"] = hash["restype"] hash.delete("restype") # FIXME At some point, we're going to want to retain this information # for logging and auditing. hash.delete("host_id") hash.delete("updated_at") hash.delete("source_file_id") hash.delete("created_at") hash.delete("id") hash.each do |p, v| hash.delete(p) if v.nil? end hash[:scope] = scope hash[:source] = scope.source hash[:params] = [] names = [] self.param_names.each do |pname| # We can get the same name multiple times because of how the # db layout works. next if names.include?(pname.name) names << pname.name hash[:params] << pname.to_resourceparam(self, scope.source) end obj = Puppet::Parser::Resource.new(hash) # Store the ID, so we can check if we're re-collecting the same resource. obj.rails_id = self.id return obj end end