diff --git a/lib/puppet/parser/resource.rb b/lib/puppet/parser/resource.rb index 747338b3b..2fdd78ddf 100644 --- a/lib/puppet/parser/resource.rb +++ b/lib/puppet/parser/resource.rb @@ -1,453 +1,453 @@ # A resource that we're managing. This handles making sure that only subclasses # can set parameters. class Puppet::Parser::Resource require 'puppet/parser/resource/param' require 'puppet/parser/resource/reference' require 'puppet/util/tagging' include Puppet::Util include Puppet::Util::MethodHelper include Puppet::Util::Errors include Puppet::Util::Logging include Puppet::Util::Tagging attr_accessor :source, :line, :file, :scope, :rails_id attr_accessor :virtual, :override, :translated attr_reader :exported, :evaluated, :params # Determine whether the provided parameter name is a relationship parameter. def self.relationship_parameter?(name) unless defined?(@relationship_names) @relationship_names = Puppet::Type.relationship_params.collect { |p| p.name } end @relationship_names.include?(name) end # Proxy a few methods to our @ref object. [:builtin?, :type, :title].each do |method| define_method(method) do @ref.send(method) end end # Set up some boolean test methods [:exported, :translated, :override, :virtual, :evaluated].each do |method| newmeth = (method.to_s + "?").intern define_method(newmeth) do self.send(method) end end def [](param) param = symbolize(param) if param == :title return self.title end if @params.has_key?(param) @params[param].value else nil end end def builtin=(bool) @ref.builtin = bool end # Retrieve the associated definition and evaluate it. def evaluate if klass = @ref.definedtype finish() return klass.evaluate_code(self) elsif builtin? devfail "Cannot evaluate a builtin type" else self.fail "Cannot find definition %s" % self.type end ensure @evaluated = true end # Mark this resource as both exported and virtual, # or remove the exported mark. def exported=(value) if value @virtual = true @exported = value else @exported = value end end # Do any finishing work on this object, called before evaluation or # before storage/translation. def finish return if finished? @finished = true add_defaults() add_metaparams() add_scope_tags() validate() end # Has this resource already been finished? def finished? defined?(@finished) and @finished end def initialize(options) # Set all of the options we can. options.each do |option, value| if respond_to?(option.to_s + "=") send(option.to_s + "=", value) options.delete(option) end end unless self.scope raise ArgumentError, "Resources require a scope" end @source ||= scope.source options = symbolize_options(options) # Set up our reference. if type = options[:type] and title = options[:title] options.delete(:type) options.delete(:title) else raise ArgumentError, "Resources require a type and title" end @ref = Reference.new(:type => type, :title => title, :scope => self.scope) @params = {} # Define all of the parameters if params = options[:params] options.delete(:params) params.each do |param| set_parameter(param) end end # Throw an exception if we've got any arguments left to set. unless options.empty? raise ArgumentError, "Resources do not accept %s" % options.keys.collect { |k| k.to_s }.join(", ") end tag(@ref.type) tag(@ref.title) if valid_tag?(@ref.title.to_s) end # Is this resource modeling an isomorphic resource type? def isomorphic? if builtin? return @ref.builtintype.isomorphic? else return true end end # Merge an override resource in. This will throw exceptions if # any overrides aren't allowed. def merge(resource) # Test the resource scope, to make sure the resource is even allowed # to override. unless self.source.object_id == resource.source.object_id || resource.source.child_of?(self.source) raise Puppet::ParseError.new("Only subclasses can override parameters", resource.line, resource.file) end # Some of these might fail, but they'll fail in the way we want. resource.params.each do |name, param| override_parameter(param) end end # Modify this resource in the Rails database. Poor design, yo. def modify_rails(db_resource) args = rails_args args.each do |param, value| db_resource[param] = value unless db_resource[param] == value end # Handle file specially if (self.file and (!db_resource.file or db_resource.file != self.file)) db_resource.file = self.file end updated_params = @params.inject({}) do |hash, ary| hash[ary[0].to_s] = ary[1] hash end - db_resource.ar_hash_merge(db_resource.get_params_hash(db_resource.param_values), updated_params, + db_resource.ar_hash_merge(db_resource.get_params_hash(), updated_params, :create => Proc.new { |name, parameter| parameter.to_rails(db_resource) }, :delete => Proc.new { |values| - values.each { |value| db_resource.param_values.delete(value) } + values.each { |value| Puppet::Rails::ParamValue.delete(value['id']) } }, :modify => Proc.new { |db, mem| mem.modify_rails_values(db) }) updated_tags = tags.inject({}) { |hash, tag| hash[tag] = tag hash } - + db_resource.ar_hash_merge(db_resource.get_tag_hash(), updated_tags, :create => Proc.new { |name, tag| db_resource.add_resource_tag(name) }, :delete => Proc.new { |tag| - db_resource.resource_tags.delete(tag) + Puppet::Rails::ResourceTag.delete(tag['id']) }, :modify => Proc.new { |db, mem| # nothing here }) end # Return the resource name, or the title if no name # was specified. def name unless defined? @name @name = self[:name] || self.title end @name end # This *significantly* reduces the number of calls to Puppet.[]. def paramcheck? unless defined? @@paramcheck @@paramcheck = Puppet[:paramcheck] end @@paramcheck end # A temporary occasion, until I get paths in the scopes figured out. def path to_s end # Return the short version of our name. def ref @ref.to_s end # Define a parameter in our resource. def set_parameter(param, value = nil) if value param = Puppet::Parser::Resource::Param.new( :name => param, :value => value, :source => self.source ) elsif ! param.is_a?(Puppet::Parser::Resource::Param) raise ArgumentError, "Must pass a parameter or all necessary values" end # And store it in our parameter hash. @params[param.name] = param end def to_hash @params.inject({}) do |hash, ary| param = ary[1] # Skip "undef" values. if param.value != :undef hash[param.name] = param.value end hash end end # Turn our parser resource into a Rails resource. def to_rails(host) args = rails_args db_resource = host.resources.build(args) # Handle file specially db_resource.file = self.file db_resource.save @params.each { |name, param| param.to_rails(db_resource) } tags.each { |tag| db_resource.add_resource_tag(tag) } return db_resource end def to_s self.ref end # Translate our object to a transportable object. def to_trans return nil if virtual? if builtin? to_transobject else to_transbucket end end def to_transbucket bucket = Puppet::TransBucket.new([]) bucket.type = self.type bucket.name = self.title # TransBuckets don't support parameters, which is why they're being deprecated. return bucket end # Convert this resource to a RAL resource. We hackishly go via the # transportable stuff. def to_type to_trans.to_type end def to_transobject # Now convert to a transobject obj = Puppet::TransObject.new(@ref.title, @ref.type) to_hash.each do |p, v| if v.is_a?(Reference) v = v.to_ref elsif v.is_a?(Array) v = v.collect { |av| if av.is_a?(Reference) av = av.to_ref end av } end # If the value is an array with only one value, then # convert it to a single value. This is largely so that # the database interaction doesn't have to worry about # whether it returns an array or a string. obj[p.to_s] = if v.is_a?(Array) and v.length == 1 v[0] else v end end obj.file = self.file obj.line = self.line obj.tags = self.tags return obj end private # Add default values from our definition. def add_defaults scope.lookupdefaults(self.type).each do |name, param| unless @params.include?(name) self.debug "Adding default for %s" % name @params[name] = param.dup end end end # Add any metaparams defined in our scope. This actually adds any metaparams # from any parent scope, and there's currently no way to turn that off. def add_metaparams Puppet::Type.eachmetaparam do |name| # Skip metaparams that we already have defined, unless they're relationship metaparams. # LAK:NOTE Relationship metaparams get treated specially -- we stack them, instead of # overriding. next if @params[name] and not self.class.relationship_parameter?(name) # Skip metaparams for which we get no value. next unless val = scope.lookupvar(name.to_s, false) and val != :undefined # The default case: just set the value set_parameter(name, val) and next unless @params[name] # For relationship params, though, join the values (a la #446). @params[name].value = [@params[name].value, val].flatten end end def add_scope_tags if scope_resource = scope.resource tag(*scope_resource.tags) end end # Accept a parameter from an override. def override_parameter(param) # This can happen if the override is defining a new parameter, rather # than replacing an existing one. (@params[param.name] = param and return) unless current = @params[param.name] # The parameter is already set. Fail if they're not allowed to override it. unless param.source.child_of?(current.source) puts caller if Puppet[:trace] msg = "Parameter '%s' is already set on %s" % [param.name, self.to_s] if current.source.to_s != "" msg += " by %s" % current.source end if current.file or current.line fields = [] fields << current.file if current.file fields << current.line.to_s if current.line msg += " at %s" % fields.join(":") end msg += "; cannot redefine" raise Puppet::ParseError.new(msg, param.line, param.file) end # If we've gotten this far, we're allowed to override. # Merge with previous value, if the parameter was generated with the +> syntax. # It's important that we use the new param instance here, not the old one, # so that the source is registered correctly for later overrides. param.value = [current.value, param.value].flatten if param.add @params[param.name] = param end # Verify that all passed parameters are valid. This throws an error if # there's a problem, so we don't have to worry about the return value. def paramcheck(param) param = param.to_s # Now make sure it's a valid argument to our class. These checks # are organized in order of commonhood -- most types, it's a valid # argument and paramcheck is enabled. if @ref.typeclass.validattr?(param) true elsif %w{name title}.include?(param) # always allow these true elsif paramcheck? self.fail Puppet::ParseError, "Invalid parameter '%s' for type '%s'" % [param, @ref.type] end end def rails_args return [:type, :title, :line, :exported].inject({}) do |hash, param| # 'type' isn't a valid column name, so we have to use another name. to = (param == :type) ? :restype : param if value = self.send(param) - hash[to] = value + hash[to] = value end hash end end # Make sure the resource's parameters are all valid for the type. def validate @params.each do |name, param| # Make sure it's a valid parameter. paramcheck(name) end end end diff --git a/lib/puppet/parser/resource/param.rb b/lib/puppet/parser/resource/param.rb index c8dd78a26..1a5cfe8fb 100644 --- a/lib/puppet/parser/resource/param.rb +++ b/lib/puppet/parser/resource/param.rb @@ -1,98 +1,97 @@ # The parameters we stick in Resources. class Puppet::Parser::Resource::Param attr_accessor :name, :value, :source, :line, :file, :add include Puppet::Util include Puppet::Util::Errors include Puppet::Util::MethodHelper def initialize(hash) set_options(hash) requiredopts(:name, :value, :source) @name = symbolize(@name) end def inspect - "#<#{self.class} @name => #{name}, @value => #{value}, @source => #{source.name}>" + "#<#{self.class} @name => #{name}, @value => #{value}, @source => #{source.name}, @line => #{line}>" end def line_to_i return line ? Integer(line) : nil end # Make sure an array (or possibly not an array) of values is correctly # set up for Rails. The main thing is that Resource::Reference objects # should stay objects, so they just get serialized. def munge_for_rails(values) values = value.is_a?(Array) ? value : [value] values.map do |v| if v.is_a?(Puppet::Parser::Resource::Reference) v else v.to_s end end end # Store a new parameter in a Rails db. def to_rails(db_resource) values = munge_for_rails(value) param_name = Puppet::Rails::ParamName.find_or_create_by_name(self.name.to_s) line_number = line_to_i() - return values.collect do |v| db_resource.param_values.create(:value => v, :line => line_number, :param_name => param_name) end end def modify_rails_values(db_values) - #dev_warn if db_values.nil? || db_values.empty? + #dev_warn if db_values.nil? || db_values.empty? values_to_remove(db_values).each { |remove_me| - Puppet::Rails::ParamValue.delete(remove_me.id) + Puppet::Rails::ParamValue.delete(remove_me['id']) } line_number = line_to_i() - values_to_add(db_values).each { |add_me| - db_resource = db_values[0].resource - db_param_name = db_values[0].param_name - db_resource.param_values.create(:value => add_me, + db_param_name = db_values[0]['param_name_id'] + values_to_add(db_values).each { |add_me| + Puppet::Rails::ParamValue.create(:value => add_me, :line => line_number, - :param_name => db_param_name) + :param_name_id => db_param_name, + :resource_id => db_values[0]['resource_id'] ) } end def to_s "%s => %s" % [self.name, self.value] end def compare(v,db_value) if (v.is_a?(Puppet::Parser::Resource::Reference)) return v.to_s == db_value.to_s else return v == db_value end end def values_to_remove(db_values) values = munge_for_rails(value) line_number = line_to_i() db_values.collect do |db| - db unless (db.line == line_number && + db unless (db['line'] == line_number && values.find { |v| - compare(v,db.value) + compare(v,db['value']) } ) end.compact end def values_to_add(db_values) values = munge_for_rails(value) line_number = line_to_i() values.collect do |v| - v unless db_values.find { |db| (compare(v,db.value) && - line_number == db.line) } + v unless db_values.find { |db| (compare(v,db['value']) && + line_number == db['line']) } end.compact end end diff --git a/lib/puppet/rails/host.rb b/lib/puppet/rails/host.rb index 187dc657a..a429cbfb1 100644 --- a/lib/puppet/rails/host.rb +++ b/lib/puppet/rails/host.rb @@ -1,155 +1,194 @@ require 'puppet/rails/resource' require 'puppet/rails/fact_name' require 'puppet/rails/source_file' require 'puppet/util/rails/collection_merger' -# Puppet::TIME_DEBUG = true +Puppet::TIME_DEBUG = true class Puppet::Rails::Host < ActiveRecord::Base include Puppet::Util include Puppet::Util::CollectionMerger has_many :fact_values, :dependent => :destroy has_many :fact_names, :through => :fact_values belongs_to :source_file - has_many :resources, - :include => :param_values, - :dependent => :destroy + has_many :resources, :dependent => :destroy # If the host already exists, get rid of its objects def self.clean(host) if obj = self.find_by_name(host) obj.rails_objects.clear return obj else return nil end end # Store our host in the database. def self.store(node, resources) args = {} host = nil transaction do #unless host = find_by_name(name) seconds = Benchmark.realtime { unless host = find_by_name(node.name) host = new(:name => node.name) end } Puppet.notice("Searched for host in %0.2f seconds" % seconds) if defined?(Puppet::TIME_DEBUG) if ip = node.parameters["ipaddress"] host.ip = ip end if env = node.environment host.environment = env end # Store the facts into the database. host.setfacts node.parameters seconds = Benchmark.realtime { host.setresources(resources) } Puppet.notice("Handled resources in %0.2f seconds" % seconds) if defined?(Puppet::TIME_DEBUG) host.last_compile = Time.now host.save end return host end # Return the value of a fact. def fact(name) if fv = self.fact_values.find(:all, :include => :fact_name, :conditions => "fact_names.name = '#{name}'") return fv else return nil end end # returns a hash of fact_names.name => [ fact_values ] for this host. def get_facts_hash fact_values = self.fact_values.find(:all, :include => :fact_name) return fact_values.inject({}) do | hash, value | hash[value.fact_name.name] ||= [] hash[value.fact_name.name] << value hash end end def setfacts(facts) facts = facts.dup ar_hash_merge(get_facts_hash(), facts, :create => Proc.new { |name, values| fact_name = Puppet::Rails::FactName.find_or_create_by_name(name) values = [values] unless values.is_a?(Array) values.each do |value| fact_values.build(:value => value, :fact_name => fact_name) end }, :delete => Proc.new { |values| values.each { |value| self.fact_values.delete(value) } }, :modify => Proc.new { |db, mem| mem = [mem].flatten fact_name = db[0].fact_name db_values = db.collect { |fact_value| fact_value.value } (db_values - (db_values & mem)).each do |value| db.find_all { |fact_value| fact_value.value == value }.each { |fact_value| fact_values.delete(fact_value) } end (mem - (db_values & mem)).each do |value| fact_values.build(:value => value, :fact_name => fact_name) end }) end # Set our resources. def setresources(list) existing = nil seconds = Benchmark.realtime { + existing = find_resources() + } + Puppet.notice("Searched for resources in %0.2f seconds" % seconds) if defined?(Puppet::TIME_DEBUG) - # Preload the parameters with the resource query, but not the tags, since doing so makes the query take about 10x longer. - # I've left the other queries in so that it's straightforward to switch between them for testing, if we so desire. - #existing = resources.find(:all, :include => [{:param_values => :param_name, :resource_tags => :puppet_tag}, :source_file]).inject({}) do | hash, resource | - #existing = resources.find(:all, :include => [{:resource_tags => :puppet_tag}, :source_file]).inject({}) do | hash, resource | - existing = resources.find(:all, :include => [{:param_values => :param_name}, :source_file]).inject({}) do | hash, resource | - hash[resource.ref] = resource - hash - end + seconds = Benchmark.realtime { + find_resources_parameters_tags(existing) + } if id + Puppet.notice("Searched for resource params and tags in %0.2f seconds" % seconds) if defined?(Puppet::TIME_DEBUG) + + seconds = Benchmark.realtime { + compare_to_catalog(existing, list) } + Puppet.notice("Resource comparison took %0.2f seconds" % seconds) if defined?(Puppet::TIME_DEBUG) + end - Puppet.notice("Searched for resources in %0.2f seconds" % seconds) if defined?(Puppet::TIME_DEBUG) + def find_resources + resources.find(:all, :include => :source_file).inject({}) do | hash, resource | + hash[resource.ref] = resource + hash + end + end + + def find_resources_parameters_tags(resources) + # initialize all resource parameters + resources.each do |key,resource| + resource.params_hash = [] + end + resources_by_id = resources.inject({}) do |hash, res| + hash[res[1]['id']] = res[1] + hash + end + + find_resources_parameters(resources_by_id) + find_resources_tags(resources_by_id) + end + + def compare_to_catalog(resources, list) compiled = list.inject({}) do |hash, resource| hash[resource.ref] = resource hash end - - ar_hash_merge(existing, compiled, + ar_hash_merge(resources, compiled, :create => Proc.new { |ref, resource| resource.to_rails(self) }, :delete => Proc.new { |resource| self.resources.delete(resource) }, :modify => Proc.new { |db, mem| mem.modify_rails(db) }) end + def find_resources_parameters(resources) + params = Puppet::Rails::ParamValue.find_all_params_from_host(self) + + # assign each loaded parameters/tags to the resource it belongs to + params.each do |param| + resources[param['resource_id']].add_param_to_hash(param) + end + end + + def find_resources_tags(resources) + tags = Puppet::Rails::ResourceTag.find_all_tags_from_host(self) + + tags.each do |tag| + resources[tag['resource_id']].add_tag_to_hash(tag) + end + end + def update_connect_time self.last_connect = Time.now save end end diff --git a/lib/puppet/rails/resource.rb b/lib/puppet/rails/resource.rb index 255b0e788..bd2739d53 100644 --- a/lib/puppet/rails/resource.rb +++ b/lib/puppet/rails/resource.rb @@ -1,124 +1,150 @@ require 'puppet' require 'puppet/rails/param_name' require 'puppet/rails/puppet_tag' require 'puppet/util/rails/collection_merger' class Puppet::Rails::Resource < ActiveRecord::Base include Puppet::Util::CollectionMerger + include Puppet::Util::ReferenceSerializer has_many :param_values, :dependent => :destroy, :class_name => "Puppet::Rails::ParamValue" has_many :param_names, :through => :param_values, :class_name => "Puppet::Rails::ParamName" has_many :resource_tags, :dependent => :destroy, :class_name => "Puppet::Rails::ResourceTag" has_many :puppet_tags, :through => :resource_tags, :class_name => "Puppet::Rails::PuppetTag" belongs_to :source_file belongs_to :host def add_resource_tag(tag) pt = Puppet::Rails::PuppetTag.find_or_create_by_name(tag, :include => :puppet_tag) resource_tags.create(:puppet_tag => pt) end def file if f = self.source_file return f.filename else return nil end end def file=(file) self.source_file = Puppet::Rails::SourceFile.find_or_create_by_filename(file) end + def title + unserialize_value(self[:title]) + end + + def add_param_to_hash(param) + @params_hash ||= [] + @params_hash << param + end + + def add_tag_to_hash(tag) + @tags_hash ||= [] + @tags_hash << tag + end + + def params_hash=(hash) + @params_hash = hash + end + + def tags_hash=(hash) + @tags_hash = hash + end + # returns a hash of param_names.name => [param_values] def get_params_hash(values = nil) - values ||= param_values.find(:all, :include => :param_name) - values.inject({}) do | hash, value | - hash[value.param_name.name] ||= [] - hash[value.param_name.name] << value + values ||= @params_hash || Puppet::Rails::ParamValues.find_all_params_from_resource(id) + if values.size == 0 + return {} + end + values.inject({}) do |hash, value| + hash[value['name']] ||= [] + hash[value['name']] << value hash end end - + def get_tag_hash(tags = nil) - tags ||= resource_tags.find(:all, :include => :puppet_tag) + tags ||= @tags_hash || Puppet::Rails::ResourceTag.find_all_tags_from_resource(id) return tags.inject({}) do |hash, tag| # We have to store the tag object, not just the tag name. - hash[tag.puppet_tag.name] = tag + hash[tag['name']] = tag hash end end def [](param) return super || parameter(param) end def name ref() end def parameter(param) if pn = param_names.find_by_name(param) if pv = param_values.find(:first, :conditions => [ 'param_name_id = ?', pn] ) return pv.value else return nil end end end def parameters result = get_params_hash result.each do |param, value| if value.is_a?(Array) result[param] = value.collect { |v| v.value } else result[param] = value.value end end result end def ref - "%s[%s]" % [self[:restype].split("::").collect { |s| s.capitalize }.join("::"), self[:title]] + "%s[%s]" % [self[:restype].split("::").collect { |s| s.capitalize }.join("::"), self.title.to_s] end # Convert our object to a resource. Do not retain whether the object # is exported, though, since that would cause it to get stripped # from the configuration. def to_resource(scope) hash = self.attributes hash["type"] = hash["restype"] hash.delete("restype") # FIXME At some point, we're going to want to retain this information # for logging and auditing. hash.delete("host_id") hash.delete("updated_at") hash.delete("source_file_id") hash.delete("created_at") hash.delete("id") hash.each do |p, v| hash.delete(p) if v.nil? end hash[:scope] = scope hash[:source] = scope.source hash[:params] = [] names = [] self.param_names.each do |pname| # We can get the same name multiple times because of how the # db layout works. next if names.include?(pname.name) names << pname.name hash[:params] << pname.to_resourceparam(self, scope.source) end obj = Puppet::Parser::Resource.new(hash) # Store the ID, so we can check if we're re-collecting the same resource. obj.rails_id = self.id return obj end end