diff --git a/lib/puppet/indirector/resource_type/parser.rb b/lib/puppet/indirector/resource_type/parser.rb index 24b4b065e..fd5b3938a 100644 --- a/lib/puppet/indirector/resource_type/parser.rb +++ b/lib/puppet/indirector/resource_type/parser.rb @@ -1,34 +1,32 @@ require 'puppet/resource/type' require 'puppet/indirector/code' require 'puppet/indirector/resource_type' class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code desc "Return the data-form of a resource type." def find(request) krt = request.environment.known_resource_types # This is a bit ugly. [:hostclass, :definition, :node].each do |type| - if r = krt.send(type, request.key) + # We have to us 'find_' here because it will + # load any missing types from disk, whereas the plain + # '' method only returns from memory. + if r = krt.send("find_#{type}", [""], request.key) return r end end nil end def search(request) -=begin - @modulepath = set_modulepath(options[:modulepath]) - get_code(@modulepath)[:manifests].collect do |file| - Puppet[:manifest]=file - get_resources_of_type(:hostclass) - end.flatten -=end raise ArgumentError, "Only '*' is acceptable as a search request" unless request.key == "*" krt = request.environment.known_resource_types - result = [krt.hostclasses.values, krt.definitions.values, krt.nodes.values].flatten + # Make sure we've got all of the types loaded. + krt.loader.import_all + result = [krt.hostclasses.values, krt.definitions.values, krt.nodes.values].flatten.reject { |t| t.name == "" } return nil if result.empty? result end end diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 140c9f2ca..1fba73d0b 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -1,145 +1,173 @@ require 'puppet/node/environment' class Puppet::Parser::TypeLoader include Puppet::Node::Environment::Helper # Helper class that makes sure we don't try to import the same file # more than once from either the same thread or different threads. class Helper include MonitorMixin def initialize super # These hashes are indexed by filename @state = {} # :doing or :done @thread = {} # if :doing, thread that's doing the parsing @cond_var = {} # if :doing, condition var that will be signaled when done. end # Execute the supplied block exactly once per file, no matter how # many threads have asked for it to run. If another thread is # already executing it, wait for it to finish. If this thread is # already executing it, return immediately without executing the # block. # # Note: the reason for returning immediately if this thread is # already executing the block is to handle the case of a circular # import--when this happens, we attempt to recursively re-parse a # file that we are already in the process of parsing. To prevent # an infinite regress we need to simply do nothing when the # recursive import is attempted. def do_once(file) need_to_execute = synchronize do case @state[file] when :doing if @thread[file] != Thread.current @cond_var[file].wait end false when :done false else @state[file] = :doing @thread[file] = Thread.current @cond_var[file] = new_cond true end end if need_to_execute begin yield ensure synchronize do @state[file] = :done @thread.delete(file) @cond_var.delete(file).broadcast end end end end end # Import our files. def import(file, current_file = nil) return if Puppet[:ignoreimport] # use a path relative to the file doing the importing if current_file dir = current_file.sub(%r{[^/]+$},'').sub(/\/$/, '') else dir = "." end if dir == "" dir = "." end pat = file modname, files = Puppet::Parser::Files.find_manifests(pat, :cwd => dir, :environment => environment) if files.size == 0 raise Puppet::ImportError.new("No file(s) found for import of '#{pat}'") end loaded_asts = [] files.each do |file| unless file =~ /^#{File::SEPARATOR}/ file = File.join(dir, file) end @loading_helper.do_once(file) do loaded_asts << parse_file(file) end end loaded_asts.inject([]) do |loaded_types, ast| loaded_types + known_resource_types.import_ast(ast, modname) end end + def import_all + require 'find' + + module_names = [] + # Collect the list of all known modules + environment.modulepath.each do |path| + Dir.chdir(path) do + Dir.glob("*").each do |dir| + next unless FileTest.directory?(dir) + module_names << dir + end + end + end + + module_names.uniq! + # And then load all files from each module, but (relying on system + # behavior) only load files from the first module of a given name. E.g., + # given first/foo and second/foo, only files from first/foo will be loaded. + module_names.each do |name| + mod = Puppet::Module.new(name, environment) + Find.find(File.join(mod.path, "manifests")) do |path| + if path =~ /\.pp$/ or path =~ /\.rb$/ + import(path) + end + end + end + end + def known_resource_types environment.known_resource_types end def initialize(env) self.environment = env @loading_helper = Helper.new end # Try to load the object with the given fully qualified name. def try_load_fqname(type, fqname) return nil if fqname == "" # special-case main. name2files(fqname).each do |filename| begin imported_types = import(filename) if result = imported_types.find { |t| t.type == type and t.name == fqname } Puppet.debug "Automatically imported #{fqname} from #{filename} into #{environment}" return result end rescue Puppet::ImportError => detail # We couldn't load the item # I'm not convienced we should just drop these errors, but this # preserves existing behaviours. end end # Nothing found. return nil end def parse_file(file) Puppet.debug("importing '#{file}' in environment #{environment}") parser = Puppet::Parser::Parser.new(environment) parser.file = file return parser.parse end private # Return a list of all file basenames that should be tried in order # to load the object with the given fully qualified name. def name2files(fqname) result = [] ary = fqname.split("::") while ary.length > 0 result << ary.join(File::SEPARATOR) ary.pop end return result end end diff --git a/spec/unit/indirector/resource_type/parser_spec.rb b/spec/unit/indirector/resource_type/parser_spec.rb index 739e58b35..27e61486c 100755 --- a/spec/unit/indirector/resource_type/parser_spec.rb +++ b/spec/unit/indirector/resource_type/parser_spec.rb @@ -1,75 +1,123 @@ #!/usr/bin/env ruby require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper') require 'puppet/indirector/resource_type/parser' +require 'puppet_spec/files' describe Puppet::Indirector::ResourceType::Parser do + include PuppetSpec::Files + before do @terminus = Puppet::Indirector::ResourceType::Parser.new @request = Puppet::Indirector::Request.new(:resource_type, :find, "foo") - @krt = Puppet::Resource::TypeCollection.new(@request.environment) - @request.environment.stubs(:known_resource_types).returns @krt + @krt = @request.environment.known_resource_types end it "should be registered with the resource_type indirection" do Puppet::Indirector::Terminus.terminus_class(:resource_type, :parser).should equal(Puppet::Indirector::ResourceType::Parser) end describe "when finding" do - it "should use the request's environment's list of known resource types" do - @request.environment.known_resource_types.expects(:hostclass).returns nil + it "should return any found type from the request's environment" do + type = Puppet::Resource::Type.new(:hostclass, "foo") + @request.environment.known_resource_types.add(type) - @terminus.find(@request) + @terminus.find(@request).should == type end - it "should return any found type" do - type = @krt.add(Puppet::Resource::Type.new(:hostclass, "foo")) + it "should attempt to load the type if none is found in memory" do + dir = tmpdir("find_a_type") + FileUtils.mkdir_p(dir) + Puppet[:modulepath] = dir - @terminus.find(@request).should == type + # Make a new request, since we've reset the env + @request = Puppet::Indirector::Request.new(:resource_type, :find, "foo::bar") + + manifest_path = File.join(dir, "foo", "manifests") + FileUtils.mkdir_p(manifest_path) + + File.open(File.join(manifest_path, "bar.pp"), "w") { |f| f.puts "class foo::bar {}" } + + result = @terminus.find(@request) + result.should be_instance_of(Puppet::Resource::Type) + result.name.should == "foo::bar" end it "should return nil if no type can be found" do @terminus.find(@request).should be_nil end it "should prefer definitions to nodes" do type = @krt.add(Puppet::Resource::Type.new(:hostclass, "foo")) node = @krt.add(Puppet::Resource::Type.new(:node, "foo")) @terminus.find(@request).should == type end end describe "when searching" do before do @request.key = "*" end it "should use the request's environment's list of known resource types" do @request.environment.known_resource_types.expects(:hostclasses).returns({}) @terminus.search(@request) end it "should fail if anyther other than '*' was provided as the search key" do @request.key = "foo*" lambda { @terminus.search(@request) }.should raise_error(ArgumentError) end it "should return all known types" do type = @krt.add(Puppet::Resource::Type.new(:hostclass, "foo")) node = @krt.add(Puppet::Resource::Type.new(:node, "bar")) define = @krt.add(Puppet::Resource::Type.new(:definition, "baz")) result = @terminus.search(@request) result.should be_include(type) result.should be_include(node) result.should be_include(define) end + it "should not return the 'main' class" do + main = @krt.add(Puppet::Resource::Type.new(:hostclass, "")) + + # So there is a return value + foo = @krt.add(Puppet::Resource::Type.new(:hostclass, "foo")) + + @terminus.search(@request).should_not be_include(main) + end + it "should return nil if no types can be found" do @terminus.search(@request).should be_nil end + + it "should load all resource types from all search paths" do + dir = tmpdir("searching_in_all") + first = File.join(dir, "first") + second = File.join(dir, "second") + FileUtils.mkdir_p(first) + FileUtils.mkdir_p(second) + Puppet[:modulepath] = "#{first}:#{second}" + + # Make a new request, since we've reset the env + @request = Puppet::Indirector::Request.new(:resource_type, :search, "*") + + onepath = File.join(first, "one", "manifests") + FileUtils.mkdir_p(onepath) + twopath = File.join(first, "two", "manifests") + FileUtils.mkdir_p(twopath) + + File.open(File.join(onepath, "oneklass.pp"), "w") { |f| f.puts "class one::oneklass {}" } + File.open(File.join(twopath, "twoklass.pp"), "w") { |f| f.puts "class two::twoklass {}" } + + result = @terminus.search(@request) + result.find { |t| t.name == "one::oneklass" }.should be_instance_of(Puppet::Resource::Type) + result.find { |t| t.name == "two::twoklass" }.should be_instance_of(Puppet::Resource::Type) + end end end diff --git a/spec/unit/parser/type_loader_spec.rb b/spec/unit/parser/type_loader_spec.rb index bd41adfb6..12bc1ccd6 100644 --- a/spec/unit/parser/type_loader_spec.rb +++ b/spec/unit/parser/type_loader_spec.rb @@ -1,134 +1,231 @@ #!/usr/bin/env ruby require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper') require 'puppet/parser/type_loader' require 'puppet_spec/files' describe Puppet::Parser::TypeLoader do include PuppetSpec::Files before do @loader = Puppet::Parser::TypeLoader.new(:myenv) end it "should support an environment" do loader = Puppet::Parser::TypeLoader.new(:myenv) loader.environment.name.should == :myenv end it "should include the Environment Helper" do @loader.class.ancestors.should be_include(Puppet::Node::Environment::Helper) end it "should delegate its known resource types to its environment" do @loader.known_resource_types.should be_instance_of(Puppet::Resource::TypeCollection) end describe "when loading names from namespaces" do it "should do nothing if the name to import is an empty string" do @loader.expects(:name2files).never @loader.try_load_fqname(:hostclass, "") { |filename, modname| raise :should_not_occur }.should be_nil end it "should attempt to import each generated name" do @loader.expects(:import).with("foo/bar",nil).returns([]) @loader.expects(:import).with("foo",nil).returns([]) @loader.try_load_fqname(:hostclass, "foo::bar") { |f| false } end end describe "when importing" do before do Puppet::Parser::Files.stubs(:find_manifests).returns ["modname", %w{file}] Puppet::Parser::Parser.any_instance.stubs(:parse).returns(Puppet::Parser::AST::Hostclass.new('')) Puppet::Parser::Parser.any_instance.stubs(:file=) end it "should return immediately when imports are being ignored" do Puppet::Parser::Files.expects(:find_manifests).never Puppet[:ignoreimport] = true @loader.import("foo").should be_nil end it "should find all manifests matching the file or pattern" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| pat == "myfile" }.returns ["modname", %w{one}] @loader.import("myfile") end it "should use the directory of the current file if one is set" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| opts[:cwd] == "/current" }.returns ["modname", %w{one}] @loader.import("myfile", "/current/file") end it "should pass the environment when looking for files" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| opts[:environment] == @loader.environment }.returns ["modname", %w{one}] @loader.import("myfile") end it "should fail if no files are found" do Puppet::Parser::Files.expects(:find_manifests).returns [nil, []] lambda { @loader.import("myfile") }.should raise_error(Puppet::ImportError) end it "should parse each found file" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] @loader.expects(:parse_file).with("/one").returns(Puppet::Parser::AST::Hostclass.new('')) @loader.import("myfile") end it "should make each file qualified before attempting to parse it" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{one}] @loader.expects(:parse_file).with("/current/one").returns(Puppet::Parser::AST::Hostclass.new('')) @loader.import("myfile", "/current/file") end it "should not attempt to import files that have already been imported" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] Puppet::Parser::Parser.any_instance.expects(:parse).once.returns(Puppet::Parser::AST::Hostclass.new('')) @loader.import("myfile") # This will fail if it tries to reimport the file. @loader.import("myfile") end end + describe "when importing all" do + before do + @base = tmpdir("base") + + # Create two module path directories + @modulebase1 = File.join(@base, "first") + FileUtils.mkdir_p(@modulebase1) + @modulebase2 = File.join(@base, "second") + FileUtils.mkdir_p(@modulebase2) + + Puppet[:modulepath] = "#{@modulebase1}:#{@modulebase2}" + end + + def mk_module(basedir, name) + module_dir = File.join(basedir, name) + + # Go ahead and make our manifest directory + FileUtils.mkdir_p(File.join(module_dir, "manifests")) + + return Puppet::Module.new(name) + end + + # We have to pass the base path so that we can + # write to modules that are in the second search path + def mk_manifests(base, mod, type, files) + exts = {"ruby" => ".rb", "puppet" => ".pp"} + files.collect do |file| + name = mod.name + "::" + file.gsub("/", "::") + path = File.join(base, mod.name, "manifests", file + exts[type]) + FileUtils.mkdir_p(File.split(path)[0]) + + # write out the class + if type == "ruby" + File.open(path, "w") { |f| f.print "hostclass '#{name}' do\nend" } + else + File.open(path, "w") { |f| f.print "class #{name} {}" } + end + name + end + end + + it "should load all puppet manifests from all modules in the specified environment" do + @module1 = mk_module(@modulebase1, "one") + @module2 = mk_module(@modulebase2, "two") + + mk_manifests(@modulebase1, @module1, "puppet", %w{a b}) + mk_manifests(@modulebase2, @module2, "puppet", %w{c d}) + + @loader.import_all + + @loader.environment.known_resource_types.hostclass("one::a").should be_instance_of(Puppet::Resource::Type) + @loader.environment.known_resource_types.hostclass("one::b").should be_instance_of(Puppet::Resource::Type) + @loader.environment.known_resource_types.hostclass("two::c").should be_instance_of(Puppet::Resource::Type) + @loader.environment.known_resource_types.hostclass("two::d").should be_instance_of(Puppet::Resource::Type) + end + + it "should load all ruby manifests from all modules in the specified environment" do + @module1 = mk_module(@modulebase1, "one") + @module2 = mk_module(@modulebase2, "two") + + mk_manifests(@modulebase1, @module1, "ruby", %w{a b}) + mk_manifests(@modulebase2, @module2, "ruby", %w{c d}) + + @loader.import_all + + @loader.environment.known_resource_types.hostclass("one::a").should be_instance_of(Puppet::Resource::Type) + @loader.environment.known_resource_types.hostclass("one::b").should be_instance_of(Puppet::Resource::Type) + @loader.environment.known_resource_types.hostclass("two::c").should be_instance_of(Puppet::Resource::Type) + @loader.environment.known_resource_types.hostclass("two::d").should be_instance_of(Puppet::Resource::Type) + end + + it "should not load manifests from duplicate modules later in the module path" do + @module1 = mk_module(@modulebase1, "one") + + # duplicate + @module2 = mk_module(@modulebase2, "one") + + mk_manifests(@modulebase1, @module1, "puppet", %w{a}) + mk_manifests(@modulebase2, @module2, "puppet", %w{c}) + + @loader.import_all + + @loader.environment.known_resource_types.hostclass("one::c").should be_nil + end + + it "should load manifests from subdirectories" do + @module1 = mk_module(@modulebase1, "one") + + mk_manifests(@modulebase1, @module1, "puppet", %w{a a/b a/b/c}) + + @loader.import_all + + @loader.environment.known_resource_types.hostclass("one::a::b").should be_instance_of(Puppet::Resource::Type) + @loader.environment.known_resource_types.hostclass("one::a::b::c").should be_instance_of(Puppet::Resource::Type) + end + end + describe "when parsing a file" do before do @parser = Puppet::Parser::Parser.new(@loader.environment) @parser.stubs(:parse).returns(Puppet::Parser::AST::Hostclass.new('')) @parser.stubs(:file=) Puppet::Parser::Parser.stubs(:new).with(@loader.environment).returns @parser end it "should create a new parser instance for each file using the current environment" do Puppet::Parser::Parser.expects(:new).with(@loader.environment).returns @parser @loader.parse_file("/my/file") end it "should assign the parser its file and parse" do @parser.expects(:file=).with("/my/file") @parser.expects(:parse).returns(Puppet::Parser::AST::Hostclass.new('')) @loader.parse_file("/my/file") end end it "should be able to add classes to the current resource type collection" do file = tmpfile("simple_file.pp") File.open(file, "w") { |f| f.puts "class foo {}" } @loader.import(file) @loader.known_resource_types.hostclass("foo").should be_instance_of(Puppet::Resource::Type) end describe "when deciding where to look for files" do { 'foo' => ['foo'], 'foo::bar' => ['foo/bar', 'foo'], 'foo::bar::baz' => ['foo/bar/baz', 'foo/bar', 'foo'] }.each do |fqname, expected_paths| it "should look for #{fqname.inspect} in #{expected_paths.inspect}" do @loader.instance_eval { name2files(fqname) }.should == expected_paths end end end end