diff --git a/lib/puppet/indirector/catalog/active_record.rb b/lib/puppet/indirector/catalog/active_record.rb index f57427e92..a76606535 100644 --- a/lib/puppet/indirector/catalog/active_record.rb +++ b/lib/puppet/indirector/catalog/active_record.rb @@ -1,34 +1,34 @@ require 'puppet/rails/host' require 'puppet/indirector/active_record' require 'puppet/node/catalog' class Puppet::Node::Catalog::ActiveRecord < Puppet::Indirector::ActiveRecord use_ar_model Puppet::Rails::Host # If we can find the host, then return a catalog with the host's resources # as the vertices. def find(request) return nil unless request.options[:cache_integration_hack] return nil unless host = ar_model.find_by_name(request.key) catalog = Puppet::Node::Catalog.new(host.name) host.resources.each do |resource| catalog.add_resource resource.to_transportable end catalog end # Save the values from a Facts instance as the facts on a Rails Host instance. def save(request) catalog = request.instance host = ar_model.find_by_name(catalog.name) || ar_model.create(:name => catalog.name) - host.setresources(catalog.vertices) + host.merge_resources(catalog.vertices) host.last_compile = Time.now host.save end end diff --git a/lib/puppet/indirector/facts/active_record.rb b/lib/puppet/indirector/facts/active_record.rb index 5fb2596d7..f16d81220 100644 --- a/lib/puppet/indirector/facts/active_record.rb +++ b/lib/puppet/indirector/facts/active_record.rb @@ -1,35 +1,35 @@ require 'puppet/rails/fact_name' require 'puppet/rails/fact_value' require 'puppet/indirector/active_record' class Puppet::Node::Facts::ActiveRecord < Puppet::Indirector::ActiveRecord use_ar_model Puppet::Rails::Host # Find the Rails host and pull its facts as a Facts instance. def find(request) return nil unless host = ar_model.find_by_name(request.key, :include => {:fact_values => :fact_name}) facts = Puppet::Node::Facts.new(host.name) facts.values = host.get_facts_hash.inject({}) do |hash, ary| # Convert all single-member arrays into plain values. param = ary[0] values = ary[1].collect { |v| v.value } values = values[0] if values.length == 1 hash[param] = values hash end facts end # Save the values from a Facts instance as the facts on a Rails Host instance. def save(request) facts = request.instance host = ar_model.find_by_name(facts.name) || ar_model.create(:name => facts.name) - host.setfacts(facts.values) + host.merge_facts(facts.values) host.save end end diff --git a/lib/puppet/parser/resource.rb b/lib/puppet/parser/resource.rb index 8d29ea346..23df1c624 100644 --- a/lib/puppet/parser/resource.rb +++ b/lib/puppet/parser/resource.rb @@ -1,447 +1,379 @@ # A resource that we're managing. This handles making sure that only subclasses # can set parameters. class Puppet::Parser::Resource require 'puppet/parser/resource/param' require 'puppet/parser/resource/reference' require 'puppet/util/tagging' require 'puppet/file_collection/lookup' include Puppet::FileCollection::Lookup include Puppet::Util include Puppet::Util::MethodHelper include Puppet::Util::Errors include Puppet::Util::Logging include Puppet::Util::Tagging attr_accessor :source, :scope, :rails_id attr_accessor :virtual, :override, :translated attr_reader :exported, :evaluated, :params # Determine whether the provided parameter name is a relationship parameter. def self.relationship_parameter?(name) unless defined?(@relationship_names) @relationship_names = Puppet::Type.relationship_params.collect { |p| p.name } end @relationship_names.include?(name) end # Proxy a few methods to our @ref object. [:builtin?, :type, :title].each do |method| define_method(method) do @ref.send(method) end end # Set up some boolean test methods [:exported, :translated, :override, :virtual, :evaluated].each do |method| newmeth = (method.to_s + "?").intern define_method(newmeth) do self.send(method) end end def [](param) param = symbolize(param) if param == :title return self.title end if @params.has_key?(param) @params[param].value else nil end end def builtin=(bool) @ref.builtin = bool end + def eachparam + @params.each do |name, param| + yield param + end + end + # Retrieve the associated definition and evaluate it. def evaluate if klass = @ref.definedtype finish() return klass.evaluate_code(self) elsif builtin? devfail "Cannot evaluate a builtin type" else self.fail "Cannot find definition %s" % self.type end ensure @evaluated = true end # Mark this resource as both exported and virtual, # or remove the exported mark. def exported=(value) if value @virtual = true @exported = value else @exported = value end end # Do any finishing work on this object, called before evaluation or # before storage/translation. def finish return if finished? @finished = true add_defaults() add_metaparams() add_scope_tags() validate() end # Has this resource already been finished? def finished? defined?(@finished) and @finished end def initialize(options) # Set all of the options we can. options.each do |option, value| if respond_to?(option.to_s + "=") send(option.to_s + "=", value) options.delete(option) end end unless self.scope raise ArgumentError, "Resources require a scope" end @source ||= scope.source options = symbolize_options(options) # Set up our reference. if type = options[:type] and title = options[:title] options.delete(:type) options.delete(:title) else raise ArgumentError, "Resources require a type and title" end @ref = Reference.new(:type => type, :title => title, :scope => self.scope) @params = {} # Define all of the parameters if params = options[:params] options.delete(:params) params.each do |param| set_parameter(param) end end # Throw an exception if we've got any arguments left to set. unless options.empty? raise ArgumentError, "Resources do not accept %s" % options.keys.collect { |k| k.to_s }.join(", ") end tag(@ref.type) tag(@ref.title) if valid_tag?(@ref.title.to_s) end # Is this resource modeling an isomorphic resource type? def isomorphic? if builtin? return @ref.builtintype.isomorphic? else return true end end # Merge an override resource in. This will throw exceptions if # any overrides aren't allowed. def merge(resource) # Test the resource scope, to make sure the resource is even allowed # to override. unless self.source.object_id == resource.source.object_id || resource.source.child_of?(self.source) raise Puppet::ParseError.new("Only subclasses can override parameters", resource.line, resource.file) end # Some of these might fail, but they'll fail in the way we want. resource.params.each do |name, param| override_parameter(param) end end - # Modify this resource in the Rails database. Poor design, yo. - def modify_rails(db_resource) - args = rails_args - args.each do |param, value| - db_resource[param] = value unless db_resource[param] == value - end - - # Handle file specially - if (self.file and - (!db_resource.file or db_resource.file != self.file)) - db_resource.file = self.file - end - - updated_params = @params.reject { |name, param| param.value == :undef }.inject({}) do |hash, ary| - hash[ary[0].to_s] = ary[1] - hash - end - - db_resource.ar_hash_merge(db_resource.get_params_hash(), updated_params, - :create => Proc.new { |name, parameter| - parameter.to_rails(db_resource) - }, :delete => Proc.new { |values| - values.each { |value| Puppet::Rails::ParamValue.delete(value['id']) } - }, :modify => Proc.new { |db, mem| - mem.modify_rails_values(db) - }) - - updated_tags = tags.inject({}) { |hash, tag| - hash[tag] = tag - hash - } - - db_resource.ar_hash_merge(db_resource.get_tag_hash(), - updated_tags, - :create => Proc.new { |name, tag| - db_resource.add_resource_tag(name) - }, :delete => Proc.new { |tag| - Puppet::Rails::ResourceTag.delete(tag['id']) - }, :modify => Proc.new { |db, mem| - # nothing here - }) - end - # Return the resource name, or the title if no name # was specified. def name unless defined? @name @name = self[:name] || self.title end @name end # This *significantly* reduces the number of calls to Puppet.[]. def paramcheck? unless defined? @@paramcheck @@paramcheck = Puppet[:paramcheck] end @@paramcheck end # A temporary occasion, until I get paths in the scopes figured out. def path to_s end # Return the short version of our name. def ref @ref.to_s end # Define a parameter in our resource. def set_parameter(param, value = nil) if value param = Puppet::Parser::Resource::Param.new( :name => param, :value => value, :source => self.source ) elsif ! param.is_a?(Puppet::Parser::Resource::Param) raise ArgumentError, "Must pass a parameter or all necessary values" end # And store it in our parameter hash. @params[param.name] = param end def to_hash @params.inject({}) do |hash, ary| param = ary[1] # Skip "undef" values. if param.value != :undef hash[param.name] = param.value end hash end end - # Turn our parser resource into a Rails resource. - def to_rails(host) - args = rails_args - - db_resource = host.resources.build(args) - - # Handle file specially - db_resource.file = self.file - - db_resource.save - - @params.each { |name, param| - next if param.value == :undef - param.to_rails(db_resource) - } - - tags.each { |tag| db_resource.add_resource_tag(tag) } - - return db_resource - end # Create a Puppet::Resource instance from this parser resource. # We plan, at some point, on not needing to do this conversion, but # it's sufficient for now. def to_resource result = Puppet::Resource.new(type, title) to_hash.each do |p, v| if v.is_a?(Puppet::Parser::Resource::Reference) v = Puppet::Resource::Reference.new(v.type, v.title) elsif v.is_a?(Array) # flatten resource references arrays if v.flatten.find { |av| av.is_a?(Puppet::Parser::Resource::Reference) } v = v.flatten end v = v.collect do |av| if av.is_a?(Puppet::Parser::Resource::Reference) av = Puppet::Resource::Reference.new(av.type, av.title) end av end end # If the value is an array with only one value, then # convert it to a single value. This is largely so that # the database interaction doesn't have to worry about # whether it returns an array or a string. result[p] = if v.is_a?(Array) and v.length == 1 v[0] else v end end result.file = self.file result.line = self.line result.tag(*self.tags) return result end def to_s self.ref end # Translate our object to a transportable object. def to_trans return nil if virtual? return to_resource.to_trans end # Convert this resource to a RAL resource. We hackishly go via the # transportable stuff. def to_ral to_resource.to_ral end private # Add default values from our definition. def add_defaults scope.lookupdefaults(self.type).each do |name, param| unless @params.include?(name) self.debug "Adding default for %s" % name @params[name] = param.dup end end end # Add any metaparams defined in our scope. This actually adds any metaparams # from any parent scope, and there's currently no way to turn that off. def add_metaparams Puppet::Type.eachmetaparam do |name| next if self.class.relationship_parameter?(name) # Skip metaparams that we already have defined, unless they're relationship metaparams. # LAK:NOTE Relationship metaparams get treated specially -- we stack them, instead of # overriding. next if @params[name] # Skip metaparams for which we get no value. next unless val = scope.lookupvar(name.to_s, false) and val != :undefined set_parameter(name, val) and next unless @params[name] end end def add_scope_tags if scope_resource = scope.resource tag(*scope_resource.tags) end end # Accept a parameter from an override. def override_parameter(param) # This can happen if the override is defining a new parameter, rather # than replacing an existing one. (@params[param.name] = param and return) unless current = @params[param.name] # The parameter is already set. Fail if they're not allowed to override it. unless param.source.child_of?(current.source) puts caller if Puppet[:trace] msg = "Parameter '%s' is already set on %s" % [param.name, self.to_s] if current.source.to_s != "" msg += " by %s" % current.source end if current.file or current.line fields = [] fields << current.file if current.file fields << current.line.to_s if current.line msg += " at %s" % fields.join(":") end msg += "; cannot redefine" raise Puppet::ParseError.new(msg, param.line, param.file) end # If we've gotten this far, we're allowed to override. # Merge with previous value, if the parameter was generated with the +> syntax. # It's important that we use the new param instance here, not the old one, # so that the source is registered correctly for later overrides. param.value = [current.value, param.value].flatten if param.add @params[param.name] = param end # Verify that all passed parameters are valid. This throws an error if # there's a problem, so we don't have to worry about the return value. def paramcheck(param) param = param.to_s # Now make sure it's a valid argument to our class. These checks # are organized in order of commonhood -- most types, it's a valid # argument and paramcheck is enabled. if @ref.typeclass.validattr?(param) true elsif %w{name title}.include?(param) # always allow these true elsif paramcheck? self.fail Puppet::ParseError, "Invalid parameter '%s' for type '%s'" % [param, @ref.type] end end - def rails_args - return [:type, :title, :line, :exported].inject({}) do |hash, param| - # 'type' isn't a valid column name, so we have to use another name. - to = (param == :type) ? :restype : param - if value = self.send(param) - hash[to] = value - end - hash - end - end - # Make sure the resource's parameters are all valid for the type. def validate @params.each do |name, param| # Make sure it's a valid parameter. paramcheck(name) end end end diff --git a/lib/puppet/parser/resource/param.rb b/lib/puppet/parser/resource/param.rb index 7ce58f4c4..6e22d3e17 100644 --- a/lib/puppet/parser/resource/param.rb +++ b/lib/puppet/parser/resource/param.rb @@ -1,101 +1,29 @@ require 'puppet/file_collection/lookup' # The parameters we stick in Resources. class Puppet::Parser::Resource::Param attr_accessor :name, :value, :source, :add include Puppet::Util include Puppet::Util::Errors include Puppet::Util::MethodHelper include Puppet::FileCollection::Lookup def initialize(hash) set_options(hash) requiredopts(:name, :value, :source) @name = symbolize(@name) end def inspect "#<#{self.class} @name => #{name}, @value => #{value}, @source => #{source.name}, @line => #{line}>" end def line_to_i return line ? Integer(line) : nil end - - # Make sure an array (or possibly not an array) of values is correctly - # set up for Rails. The main thing is that Resource::Reference objects - # should stay objects, so they just get serialized. - def munge_for_rails(values) - values = value.is_a?(Array) ? value : [value] - values.map do |v| - if v.is_a?(Puppet::Parser::Resource::Reference) - v - else - v.to_s - end - end - end - - # Store a new parameter in a Rails db. - def to_rails(db_resource) - values = munge_for_rails(value) - - param_name = Puppet::Rails::ParamName.find_or_create_by_name(self.name.to_s) - line_number = line_to_i() - return values.collect do |v| - db_resource.param_values.create(:value => v, - :line => line_number, - :param_name => param_name) - end - end - - def modify_rails_values(db_values) - #dev_warn if db_values.nil? || db_values.empty? - - values_to_remove(db_values).each { |remove_me| - Puppet::Rails::ParamValue.delete(remove_me['id']) - } - line_number = line_to_i() - db_param_name = db_values[0]['param_name_id'] - values_to_add(db_values).each { |add_me| - Puppet::Rails::ParamValue.create(:value => add_me, - :line => line_number, - :param_name_id => db_param_name, - :resource_id => db_values[0]['resource_id'] ) - } - end def to_s "%s => %s" % [self.name, self.value] end - - def compare(v,db_value) - if (v.is_a?(Puppet::Parser::Resource::Reference)) - return v.to_s == db_value.to_s - else - return v == db_value - end - end - - def values_to_remove(db_values) - values = munge_for_rails(value) - line_number = line_to_i() - db_values.collect do |db| - db unless (db['line'] == line_number && - values.find { |v| - compare(v,db['value']) - } ) - end.compact - end - - def values_to_add(db_values) - values = munge_for_rails(value) - line_number = line_to_i() - values.collect do |v| - v unless db_values.find { |db| (compare(v,db['value']) && - line_number == db['line']) } - end.compact - end end - diff --git a/lib/puppet/rails/host.rb b/lib/puppet/rails/host.rb index 23a22553d..b5831671b 100644 --- a/lib/puppet/rails/host.rb +++ b/lib/puppet/rails/host.rb @@ -1,230 +1,319 @@ require 'puppet/rails/resource' require 'puppet/rails/fact_name' require 'puppet/rails/source_file' require 'puppet/util/rails/collection_merger' class Puppet::Rails::Host < ActiveRecord::Base include Puppet::Util include Puppet::Util::CollectionMerger has_many :fact_values, :dependent => :destroy has_many :fact_names, :through => :fact_values belongs_to :source_file has_many :resources, :dependent => :destroy # If the host already exists, get rid of its objects def self.clean(host) if obj = self.find_by_name(host) obj.rails_objects.clear return obj else return nil end end def self.from_puppet(node) host = find_by_name(node.name) || new(:name => node.name) {"ipaddress" => "ip", "environment" => "environment"}.each do |myparam, itsparam| if value = node.send(myparam) host.send(itsparam + "=", value) end end host end # Store our host in the database. def self.store(node, resources) args = {} host = nil transaction do #unless host = find_by_name(name) seconds = Benchmark.realtime { unless host = find_by_name(node.name) host = new(:name => node.name) end } Puppet.debug("Searched for host in %0.2f seconds" % seconds) if ip = node.parameters["ipaddress"] host.ip = ip end if env = node.environment host.environment = env end # Store the facts into the database. - host.setfacts node.parameters + host.merge_facts(node.parameters) seconds = Benchmark.realtime { - host.setresources(resources) + host.merge_resources(resources) } Puppet.debug("Handled resources in %0.2f seconds" % seconds) host.last_compile = Time.now - host.save + seconds = Benchmark.realtime { + host.save + } + Puppet.debug("Saved host in %0.2f seconds" % seconds) end return host end # Return the value of a fact. def fact(name) if fv = self.fact_values.find(:all, :include => :fact_name, :conditions => "fact_names.name = '#{name}'") return fv else return nil end end # returns a hash of fact_names.name => [ fact_values ] for this host. # Note that 'fact_values' is actually a list of the value instances, not # just actual values. def get_facts_hash fact_values = self.fact_values.find(:all, :include => :fact_name) return fact_values.inject({}) do | hash, value | hash[value.fact_name.name] ||= [] hash[value.fact_name.name] << value hash end end - def setfacts(facts) - facts = facts.dup - - ar_hash_merge(get_facts_hash(), facts, - :create => Proc.new { |name, values| - fact_name = Puppet::Rails::FactName.find_or_create_by_name(name) - values = [values] unless values.is_a?(Array) - values.each do |value| - fact_values.build(:value => value, - :fact_name => fact_name) - end - }, :delete => Proc.new { |values| - values.each { |value| self.fact_values.delete(value) } - }, :modify => Proc.new { |db, mem| - mem = [mem].flatten - fact_name = db[0].fact_name - db_values = db.collect { |fact_value| fact_value.value } - (db_values - (db_values & mem)).each do |value| - db.find_all { |fact_value| - fact_value.value == value - }.each { |fact_value| - fact_values.delete(fact_value) - } - end - (mem - (db_values & mem)).each do |value| - fact_values.build(:value => value, - :fact_name => fact_name) - end - }) + # This is *very* similar to the merge_parameters method + # of Puppet::Rails::Resource. + def merge_facts(facts) + db_facts = {} + + deletions = [] + self.fact_values.find(:all, :include => :fact_name).each do |value| + deletions << value['id'] and next unless facts.include?(value['name']) + # Now store them for later testing. + db_facts[value['name']] ||= [] + db_facts[value['name']] << value + end + + # Now get rid of any parameters whose value list is different. + # This might be extra work in cases where an array has added or lost + # a single value, but in the most common case (a single value has changed) + # this makes sense. + db_facts.each do |name, value_hashes| + values = value_hashes.collect { |v| v['value'] } + + unless values == facts[name] + value_hashes.each { |v| deletions << v['id'] } + end + end + + # Perform our deletions. + Puppet::Rails::FactValue.delete(deletions) unless deletions.empty? + + # Lastly, add any new parameters. + facts.each do |name, value| + next if db_facts.include?(name) + values = value.is_a?(Array) ? value : [value] + + values.each do |v| + fact_values.build(:value => v, :fact_name => Puppet::Rails::FactName.find_or_create_by_name(name)) + end + end end # Set our resources. - def setresources(list) - resource_by_id = nil + def merge_resources(list) + resources_by_id = nil seconds = Benchmark.realtime { - resource_by_id = find_resources() + resources_by_id = find_resources() } Puppet.debug("Searched for resources in %0.2f seconds" % seconds) seconds = Benchmark.realtime { - find_resources_parameters_tags(resource_by_id) + find_resources_parameters_tags(resources_by_id) } if id Puppet.debug("Searched for resource params and tags in %0.2f seconds" % seconds) seconds = Benchmark.realtime { - compare_to_catalog(resource_by_id, list) + compare_to_catalog(resources_by_id, list) } Puppet.debug("Resource comparison took %0.2f seconds" % seconds) end def find_resources resources.find(:all, :include => :source_file).inject({}) do | hash, resource | hash[resource.id] = resource hash end end def find_resources_parameters_tags(resources) # initialize all resource parameters resources.each do |key,resource| resource.params_hash = [] end find_resources_parameters(resources) find_resources_tags(resources) end - # it seems that it can happen (see bug #2010) some resources are duplicated in the - # database (ie logically corrupted database), in which case we remove the extraneous - # entries. def compare_to_catalog(existing, list) - extra_db_resources = [] - resources = existing.inject({}) do |hash, res| - resource = res[1] - if hash.include?(resource.ref) - extra_db_resources << hash[resource.ref] - end + compiled = list.inject({}) do |hash, resource| hash[resource.ref] = resource hash end - compiled = list.inject({}) do |hash, resource| - hash[resource.ref] = resource - hash + resources = nil + seconds = Benchmark.realtime { + resources = remove_unneeded_resources(compiled, existing) + } + Puppet.debug("Resource removal took %0.2f seconds" % seconds) + + # Now for all resources in the catalog but not in the db, we're pretty easy. + additions = nil + seconds = Benchmark.realtime { + additions = perform_resource_merger(compiled, resources) + } + Puppet.debug("Resource merger took %0.2f seconds" % seconds) + + seconds = Benchmark.realtime { + additions.each do |resource| + build_rails_resource_from_parser_resource(resource) + end + } + Puppet.debug("Resource addition took %0.2f seconds" % seconds) + end + + def add_new_resources(additions) + additions.each do |resource| + Puppet::Rails::Resource.from_parser_resource(self, resource) + end + end + + # Turn a parser resource into a Rails resource. + def build_rails_resource_from_parser_resource(resource) + args = Puppet::Rails::Resource.rails_resource_initial_args(resource) + + db_resource = self.resources.build(args) + + # Our file= method does the name to id conversion. + db_resource.file = resource.file + + resource.eachparam do |param| + Puppet::Rails::ParamValue.from_parser_param(param).each do |value_hash| + db_resource.param_values.build(value_hash) + end end - ar_hash_merge(resources, compiled, - :create => Proc.new { |ref, resource| - resource.to_rails(self) - }, :delete => Proc.new { |resource| - self.resources.delete(resource) - }, :modify => Proc.new { |db, mem| - mem.modify_rails(db) - }) + resource.tags.each { |tag| db_resource.add_resource_tag(tag) } + + return db_resource + end + - # fix-up extraneous resources - extra_db_resources.each do |resource| - self.resources.delete(resource) + def perform_resource_merger(compiled, resources) + return compiled.values if resources.empty? + + # Now for all resources in the catalog but not in the db, we're pretty easy. + times = Hash.new(0) + additions = [] + compiled.each do |ref, resource| + if db_resource = resources[ref] + db_resource.merge_parser_resource(resource).each do |name, time| + times[name] += time + end + else + additions << resource + end + end + times.each do |name, time| + Puppet.debug("Resource merger(%s) took %0.2f seconds" % [name, time]) + end + + return additions + end + + def remove_unneeded_resources(compiled, existing) + deletions = [] + resources = {} + existing.each do |id, resource| + # it seems that it can happen (see bug #2010) some resources are duplicated in the + # database (ie logically corrupted database), in which case we remove the extraneous + # entries. + if resources.include?(resource.ref) + deletions << id + next + end + + # If the resource is in the db but not in the catalog, mark it + # for removal. + unless compiled.include?(resource.ref) + deletions << id + next + end + + resources[resource.ref] = resource + end + + # We need to use 'destroy' here, not 'delete', so that all + # dependent objects get removed, too. + Puppet::Rails::Resource.destroy(*deletions) unless deletions.empty? + + # Now for all resources in the catalog but not in the db, we're pretty easy. + compiled.each do |ref, resource| + if db_resource = resources[ref] + db_resource.merge_parser_resource(resource) + else + self.resources << Puppet::Rails::Resource.from_parser_resource(resource) + end end end def find_resources_parameters(resources) params = Puppet::Rails::ParamValue.find_all_params_from_host(self) # assign each loaded parameters/tags to the resource it belongs to params.each do |param| resources[param['resource_id']].add_param_to_hash(param) if resources.include?(param['resource_id']) end end def find_resources_tags(resources) tags = Puppet::Rails::ResourceTag.find_all_tags_from_host(self) tags.each do |tag| resources[tag['resource_id']].add_tag_to_hash(tag) if resources.include?(tag['resource_id']) end end def update_connect_time self.last_connect = Time.now save end def to_puppet node = Puppet::Node.new(self.name) {"ip" => "ipaddress", "environment" => "environment"}.each do |myparam, itsparam| if value = send(myparam) node.send(itsparam + "=", value) end end node end end diff --git a/lib/puppet/rails/param_value.rb b/lib/puppet/rails/param_value.rb index 483e4d2e8..a5dbbaed4 100644 --- a/lib/puppet/rails/param_value.rb +++ b/lib/puppet/rails/param_value.rb @@ -1,47 +1,75 @@ require 'puppet/util/rails/reference_serializer' class Puppet::Rails::ParamValue < ActiveRecord::Base include Puppet::Util::ReferenceSerializer extend Puppet::Util::ReferenceSerializer belongs_to :param_name belongs_to :resource + # Store a new parameter in a Rails db. + def self.from_parser_param(param) + values = munge_parser_values(param.value) + + param_name = Puppet::Rails::ParamName.find_or_create_by_name(param.name.to_s) + line_number = param.line_to_i() + return values.collect do |v| + {:value => v, :line => line_number, :param_name => param_name} + end + end + + # Make sure an array (or possibly not an array) of values is correctly + # set up for Rails. The main thing is that Resource::Reference objects + # should stay objects, so they just get serialized. + def self.munge_parser_values(value) + values = value.is_a?(Array) ? value : [value] + values.map do |v| + if v.is_a?(Puppet::Parser::Resource::Reference) + v + else + v.to_s + end + end + end + + def value unserialize_value(self[:value]) end # I could not find a cleaner way to handle making sure that resource references # were consistently serialized and deserialized. def value=(val) self[:value] = serialize_value(val) end def to_label - "#{self.param_name.name}" + "#{self.param_name.name}" end # returns an array of hash containing all the parameters of a given resource def self.find_all_params_from_resource(db_resource) params = db_resource.connection.select_all("SELECT v.id, v.value, v.line, v.resource_id, v.param_name_id, n.name FROM param_values as v INNER JOIN param_names as n ON v.param_name_id=n.id WHERE v.resource_id=%s" % db_resource.id) params.each do |val| val['value'] = unserialize_value(val['value']) val['line'] = val['line'] ? Integer(val['line']) : nil val['resource_id'] = Integer(val['resource_id']) end params end # returns an array of hash containing all the parameters of a given host def self.find_all_params_from_host(db_host) params = db_host.connection.select_all("SELECT v.id, v.value, v.line, v.resource_id, v.param_name_id, n.name FROM param_values as v INNER JOIN resources r ON v.resource_id=r.id INNER JOIN param_names as n ON v.param_name_id=n.id WHERE r.host_id=%s" % db_host.id) params.each do |val| val['value'] = unserialize_value(val['value']) val['line'] = val['line'] ? Integer(val['line']) : nil val['resource_id'] = Integer(val['resource_id']) end params end - + + def to_s + "%s => %s" % [self.name, self.value] + end end - diff --git a/lib/puppet/rails/resource.rb b/lib/puppet/rails/resource.rb index c3d287af2..27bf96783 100644 --- a/lib/puppet/rails/resource.rb +++ b/lib/puppet/rails/resource.rb @@ -1,151 +1,257 @@ require 'puppet' require 'puppet/rails/param_name' require 'puppet/rails/param_value' require 'puppet/rails/puppet_tag' require 'puppet/util/rails/collection_merger' class Puppet::Rails::Resource < ActiveRecord::Base include Puppet::Util::CollectionMerger include Puppet::Util::ReferenceSerializer has_many :param_values, :dependent => :destroy, :class_name => "Puppet::Rails::ParamValue" has_many :param_names, :through => :param_values, :class_name => "Puppet::Rails::ParamName" has_many :resource_tags, :dependent => :destroy, :class_name => "Puppet::Rails::ResourceTag" has_many :puppet_tags, :through => :resource_tags, :class_name => "Puppet::Rails::PuppetTag" belongs_to :source_file belongs_to :host + # Turn a parser resource into a Rails resource. + def self.from_parser_resource(resource) + args = rails_resource_initial_args(resource) + + db_resource = create(args) + + # Our file= method does the name to id conversion. + db_resource.file = resource.file + + resource.eachparam do |param| + Puppet::Rails::ParamValue.from_parser_param(param).each do |value_hash| + db_resource.param_values.build(value_hash) + end + end + + resource.tags.each { |tag| db_resource.add_resource_tag(tag) } + + return db_resource + end + + # Determine the basic details on the resource. + def self.rails_resource_initial_args(resource) + return [:type, :title, :line, :exported].inject({}) do |hash, param| + # 'type' isn't a valid column name, so we have to use another name. + to = (param == :type) ? :restype : param + if value = resource.send(param) + hash[to] = value + end + hash + end + end + def add_resource_tag(tag) - pt = Puppet::Rails::PuppetTag.find_or_create_by_name(tag, :include => :puppet_tag) - resource_tags.create(:puppet_tag => pt) + pt = Puppet::Rails::PuppetTag.find_or_create_by_name(tag) + resource_tags.build(:puppet_tag => pt) end def file if f = self.source_file return f.filename else return nil end end def file=(file) self.source_file = Puppet::Rails::SourceFile.find_or_create_by_filename(file) end def title unserialize_value(self[:title]) end def add_param_to_hash(param) @params_hash ||= [] @params_hash << param end def add_tag_to_hash(tag) @tags_hash ||= [] @tags_hash << tag end def params_hash=(hash) @params_hash = hash end def tags_hash=(hash) @tags_hash = hash end - # returns a hash of param_names.name => [param_values] - def get_params_hash(values = nil) - values ||= @params_hash || Puppet::Rails::ParamValue.find_all_params_from_resource(self) - if values.size == 0 - return {} + def [](param) + return super || parameter(param) + end + + # Make sure this resource is equivalent to the provided Parser resource. + def merge_parser_resource(resource) + merge_attributes(resource) + merge_parameters(resource) + merge_tags(resource) + end + + def merge_attributes(resource) + args = self.class.rails_resource_initial_args(resource) + args.each do |param, value| + self[param] = value unless resource[param] == value end - values.inject({}) do |hash, value| - hash[value['name']] ||= [] - hash[value['name']] << value - hash + + # Handle file specially + if (resource.file and (!resource.file or self.file != resource.file)) + self.file = resource.file end end - def get_tag_hash(tags = nil) - tags ||= @tags_hash || Puppet::Rails::ResourceTag.find_all_tags_from_resource(self) - return tags.inject({}) do |hash, tag| - # We have to store the tag object, not just the tag name. - hash[tag['name']] = tag - hash + def merge_parameters(resource) + catalog_params = {} + resource.eachparam do |param| + catalog_params[param.name.to_s] = param + end + + db_params = {} + + deletions = [] + #Puppet::Rails::ParamValue.find_all_params_from_resource(self).each do |value| + @params_hash.each do |value| + # First remove any parameters our catalog resource doesn't have at all. + deletions << value['id'] and next unless catalog_params.include?(value['name']) + + # Now store them for later testing. + db_params[value['name']] ||= [] + db_params[value['name']] << value + end + + # Now get rid of any parameters whose value list is different. + # This might be extra work in cases where an array has added or lost + # a single value, but in the most common case (a single value has changed) + # this makes sense. + db_params.each do |name, value_hashes| + values = value_hashes.collect { |v| v['value'] } + + unless value_compare(catalog_params[name].value, values) + value_hashes.each { |v| deletions << v['id'] } + end + end + + # Perform our deletions. + Puppet::Rails::ParamValue.delete(deletions) unless deletions.empty? + + # Lastly, add any new parameters. + catalog_params.each do |name, param| + next if db_params.include?(name) + values = param.value.is_a?(Array) ? param.value : [param.value] + + values.each do |v| + param_values.build(:value => serialize_value(v), :line => param.line, :param_name => Puppet::Rails::ParamName.find_or_create_by_name(name)) + end end end + + # Make sure the tag list is correct. + def merge_tags(resource) + in_db = [] + deletions = [] + resource_tags = resource.tags + #Puppet::Rails::ResourceTag.find_all_tags_from_resource(self).each do |tag| + @tags_hash.each do |tag| + deletions << tag['id'] and next unless resource_tags.include?(tag['name']) + in_db << tag['name'] + end + Puppet::Rails::ResourceTag.delete(deletions) unless deletions.empty? - def [](param) - return super || parameter(param) + (resource_tags - in_db).each do |tag| + add_resource_tag(tag) + end + end + + def value_compare(v,db_value) + v = [v] unless v.is_a?(Array) + + v == db_value end def name ref() end def parameter(param) if pn = param_names.find_by_name(param) - if pv = param_values.find(:first, :conditions => [ 'param_name_id = ?', pn] ) + if pv = param_values.find(:first, :conditions => [ 'param_name_id = ?', pn]) return pv.value else return nil end end end def parameters result = get_params_hash result.each do |param, value| if value.is_a?(Array) result[param] = value.collect { |v| v['value'] } else result[param] = value.value end end result end def ref "%s[%s]" % [self[:restype].split("::").collect { |s| s.capitalize }.join("::"), self.title.to_s] end + # Returns a hash of parameter names and values, no ActiveRecord instances. + def to_hash + Puppet::Rails::ParamValue.find_all_params_from_resource(self).inject({}) do |hash, value| + hash[value['name']] ||= [] + hash[value['name']] << value.value + hash + end + end + # Convert our object to a resource. Do not retain whether the object # is exported, though, since that would cause it to get stripped # from the configuration. def to_resource(scope) hash = self.attributes hash["type"] = hash["restype"] hash.delete("restype") # FIXME At some point, we're going to want to retain this information # for logging and auditing. hash.delete("host_id") hash.delete("updated_at") hash.delete("source_file_id") hash.delete("created_at") hash.delete("id") hash.each do |p, v| hash.delete(p) if v.nil? end hash[:scope] = scope hash[:source] = scope.source hash[:params] = [] names = [] self.param_names.each do |pname| # We can get the same name multiple times because of how the # db layout works. next if names.include?(pname.name) names << pname.name hash[:params] << pname.to_resourceparam(self, scope.source) end obj = Puppet::Parser::Resource.new(hash) # Store the ID, so we can check if we're re-collecting the same resource. obj.rails_id = self.id return obj end end diff --git a/lib/puppet/util/rails/collection_merger.rb b/lib/puppet/util/rails/collection_merger.rb index 4a941b881..3a28bb304 100644 --- a/lib/puppet/util/rails/collection_merger.rb +++ b/lib/puppet/util/rails/collection_merger.rb @@ -1,56 +1,17 @@ module Puppet::Util::CollectionMerger - # Merge new values with the old list. This is only necessary - # because deletion seems to mess things up on unsaved objects. - def collection_merge(collection, args) - remove = [] - list = args[:existing] || send(collection) - hash = args[:updates] - list.each do |object| - name = object.name - if existing = hash[name] - hash.delete(name) - if existing.respond_to?(:to_rails) - existing.to_rails(self, object) - elsif args.include?(:modify) - args[:modify].call(object, name, existing) - else - raise ArgumentError, "Must pass :modify or the new objects must respond to :to_rails" - end - else - remove << object - end - end - - # Make a new rails object for the rest of them - hash.each do |name, object| - if object.respond_to?(:to_rails) - object.to_rails(self) - elsif args.include?(:create) - args[:create].call(name, object) - else - raise ArgumentError, "Must pass :create or the new objects must respond to :to_rails" - end - end - - # Now remove anything necessary. - remove.each do |object| - send(collection).delete(object) - end - end - def ar_hash_merge(db_hash, mem_hash, args) (db_hash.keys | mem_hash.keys).each do |key| if (db_hash[key] && mem_hash[key]) # in both, update value args[:modify].call(db_hash[key], mem_hash[key]) elsif (db_hash[key]) # in db, not memory, delete from database args[:delete].call(db_hash[key]) else # in mem, not in db, insert into the database args[:create].call(key, mem_hash[key]) end end end end diff --git a/spec/unit/indirector/catalog/active_record.rb b/spec/unit/indirector/catalog/active_record.rb index b8571e5b8..948b811d3 100755 --- a/spec/unit/indirector/catalog/active_record.rb +++ b/spec/unit/indirector/catalog/active_record.rb @@ -1,119 +1,119 @@ #!/usr/bin/env ruby require File.dirname(__FILE__) + '/../../../spec_helper' require 'puppet/indirector/catalog/active_record' describe Puppet::Node::Catalog::ActiveRecord do confine "Missing Rails" => Puppet.features.rails? before do Puppet.features.stubs(:rails?).returns true @terminus = Puppet::Node::Catalog::ActiveRecord.new end it "should be a subclass of the ActiveRecord terminus class" do Puppet::Node::Catalog::ActiveRecord.ancestors.should be_include(Puppet::Indirector::ActiveRecord) end it "should use Puppet::Rails::Host as its ActiveRecord model" do Puppet::Node::Catalog::ActiveRecord.ar_model.should equal(Puppet::Rails::Host) end describe "when finding an instance" do before do @request = stub 'request', :key => "foo", :options => {:cache_integration_hack => true} end # This hack is here because we don't want to look in the db unless we actually want # to look in the db, but our indirection architecture in 0.24.x isn't flexible # enough to tune that via configuration. it "should return nil unless ':cache_integration_hack' is set to true" do @request.options[:cache_integration_hack] = false Puppet::Rails::Host.expects(:find_by_name).never @terminus.find(@request).should be_nil end it "should use the Hosts ActiveRecord class to find the host" do Puppet::Rails::Host.expects(:find_by_name).with { |key, args| key == "foo" } @terminus.find(@request) end it "should return nil if no host instance can be found" do Puppet::Rails::Host.expects(:find_by_name).returns nil @terminus.find(@request).should be_nil end it "should return a catalog with the same name as the host if the host can be found" do host = stub 'host', :name => "foo", :resources => [] Puppet::Rails::Host.expects(:find_by_name).returns host result = @terminus.find(@request) result.should be_instance_of(Puppet::Node::Catalog) result.name.should == "foo" end it "should set each of the host's resources as a transportable resource within the catalog" do host = stub 'host', :name => "foo" Puppet::Rails::Host.expects(:find_by_name).returns host res1 = mock 'res1', :to_transportable => "trans_res1" res2 = mock 'res2', :to_transportable => "trans_res2" host.expects(:resources).returns [res1, res2] catalog = stub 'catalog' Puppet::Node::Catalog.expects(:new).returns catalog catalog.expects(:add_resource).with "trans_res1" catalog.expects(:add_resource).with "trans_res2" @terminus.find(@request) end end describe "when saving an instance" do before do - @host = stub 'host', :name => "foo", :save => nil, :setresources => nil, :last_compile= => nil + @host = stub 'host', :name => "foo", :save => nil, :merge_resources => nil, :last_compile= => nil Puppet::Rails::Host.stubs(:find_by_name).returns @host @catalog = Puppet::Node::Catalog.new("foo") @request = stub 'request', :key => "foo", :instance => @catalog end it "should find the Rails host with the same name" do Puppet::Rails::Host.expects(:find_by_name).with("foo").returns @host @terminus.save(@request) end it "should create a new Rails host if none can be found" do Puppet::Rails::Host.expects(:find_by_name).with("foo").returns nil Puppet::Rails::Host.expects(:create).with(:name => "foo").returns @host @terminus.save(@request) end it "should set the catalog vertices as resources on the Rails host instance" do @catalog.expects(:vertices).returns "foo" - @host.expects(:setresources).with("foo") + @host.expects(:merge_resources).with("foo") @terminus.save(@request) end it "should set the last compile time on the host" do now = Time.now Time.expects(:now).returns now @host.expects(:last_compile=).with now @terminus.save(@request) end it "should save the Rails host instance" do @host.expects(:save) @terminus.save(@request) end end end diff --git a/spec/unit/indirector/facts/active_record.rb b/spec/unit/indirector/facts/active_record.rb index 340f2cf4c..fc35f1a45 100755 --- a/spec/unit/indirector/facts/active_record.rb +++ b/spec/unit/indirector/facts/active_record.rb @@ -1,103 +1,103 @@ #!/usr/bin/env ruby require File.dirname(__FILE__) + '/../../../spec_helper' require 'puppet/node/facts' require 'puppet/indirector/facts/active_record' describe Puppet::Node::Facts::ActiveRecord do confine "Missing Rails" => Puppet.features.rails? before do Puppet.features.stubs(:rails?).returns true @terminus = Puppet::Node::Facts::ActiveRecord.new end it "should be a subclass of the ActiveRecord terminus class" do Puppet::Node::Facts::ActiveRecord.ancestors.should be_include(Puppet::Indirector::ActiveRecord) end it "should use Puppet::Rails::Host as its ActiveRecord model" do Puppet::Node::Facts::ActiveRecord.ar_model.should equal(Puppet::Rails::Host) end describe "when finding an instance" do before do @request = stub 'request', :key => "foo" end it "should use the Hosts ActiveRecord class to find the host" do Puppet::Rails::Host.expects(:find_by_name).with { |key, args| key == "foo" } @terminus.find(@request) end it "should include the fact names and values when finding the host" do Puppet::Rails::Host.expects(:find_by_name).with { |key, args| args[:include] == {:fact_values => :fact_name} } @terminus.find(@request) end it "should return nil if no host instance can be found" do Puppet::Rails::Host.expects(:find_by_name).returns nil @terminus.find(@request).should be_nil end it "should convert the node's parameters into a Facts instance if a host instance is found" do host = stub 'host', :name => "foo" host.expects(:get_facts_hash).returns("one" => [mock("two_value", :value => "two")], "three" => [mock("three_value", :value => "four")]) Puppet::Rails::Host.expects(:find_by_name).returns host result = @terminus.find(@request) result.should be_instance_of(Puppet::Node::Facts) result.name.should == "foo" result.values.should == {"one" => "two", "three" => "four"} end it "should convert all single-member arrays into non-arrays" do host = stub 'host', :name => "foo" host.expects(:get_facts_hash).returns("one" => [mock("two_value", :value => "two")]) Puppet::Rails::Host.expects(:find_by_name).returns host @terminus.find(@request).values["one"].should == "two" end end describe "when saving an instance" do before do - @host = stub 'host', :name => "foo", :save => nil, :setfacts => nil + @host = stub 'host', :name => "foo", :save => nil, :merge_facts => nil Puppet::Rails::Host.stubs(:find_by_name).returns @host @facts = Puppet::Node::Facts.new("foo", "one" => "two", "three" => "four") @request = stub 'request', :key => "foo", :instance => @facts end it "should find the Rails host with the same name" do Puppet::Rails::Host.expects(:find_by_name).with("foo").returns @host @terminus.save(@request) end it "should create a new Rails host if none can be found" do Puppet::Rails::Host.expects(:find_by_name).with("foo").returns nil Puppet::Rails::Host.expects(:create).with(:name => "foo").returns @host @terminus.save(@request) end it "should set the facts as facts on the Rails host instance" do # There is other stuff added to the hash. - @host.expects(:setfacts).with { |args| args["one"] == "two" and args["three"] == "four" } + @host.expects(:merge_facts).with { |args| args["one"] == "two" and args["three"] == "four" } @terminus.save(@request) end it "should save the Rails host instance" do @host.expects(:save) @terminus.save(@request) end end end