diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index b1688cd22..41bebe420 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -1,207 +1,207 @@ # I pulled this into a separate file, because I got # tired of rebuilding the parser.rb file all the time. class Puppet::Parser::Parser require 'puppet/parser/functions' require 'puppet/parser/files' require 'puppet/resource/type_collection' require 'puppet/resource/type_collection_helper' require 'puppet/resource/type' require 'monitor' AST = Puppet::Parser::AST include Puppet::Resource::TypeCollectionHelper attr_reader :version, :environment attr_accessor :files attr_accessor :lexer # Add context to a message; useful for error messages and such. def addcontext(message, obj = nil) obj ||= @lexer message += " on line #{obj.line}" if file = obj.file message += " in file #{file}" end message end # Create an AST array containing a single element def aryfy(arg) ast AST::ASTArray, :children => [arg] end # Create an AST object, and automatically add the file and line information if # available. def ast(klass, hash = {}) klass.new ast_context(klass.use_docs).merge(hash) end def ast_context(include_docs = false) result = { :line => lexer.line, :file => lexer.file } result[:doc] = lexer.getcomment(result[:line]) if include_docs result end # The fully qualifed name, with the full namespace. def classname(name) [@lexer.namespace, name].join("::").sub(/^::/, '') end def clear initvars end # Raise a Parse error. def error(message, options = {}) if brace = @lexer.expected message += "; expected '%s'" end except = Puppet::ParseError.new(message) except.line = options[:line] || @lexer.line except.file = options[:file] || @lexer.file raise except end def file @lexer.file end def file=(file) unless FileTest.exist?(file) unless file =~ /\.pp$/ file = file + ".pp" end raise Puppet::Error, "Could not find file #{file}" unless FileTest.exist?(file) end raise Puppet::AlreadyImportedError, "Import loop detected" if known_resource_types.watching_file?(file) watch_file(file) @lexer.file = file end [:hostclass, :definition, :node, :nodes?].each do |method| define_method(method) do |*args| known_resource_types.send(method, *args) end end def find_hostclass(namespace, name) known_resource_types.find_hostclass(namespace, name) end def find_definition(namespace, name) known_resource_types.find_definition(namespace, name) end def import(file) - known_resource_types.loader.import_if_possible(file, @lexer.file) + known_resource_types.loader.import(file, @lexer.file) end def initialize(env) # The environment is needed to know how to find the resource type collection. @environment = env.is_a?(String) ? Puppet::Node::Environment.new(env) : env initvars end # Initialize or reset all of our variables. def initvars @lexer = Puppet::Parser::Lexer.new end # Split an fq name into a namespace and name def namesplit(fullname) ary = fullname.split("::") n = ary.pop || "" ns = ary.join("::") return ns, n end def on_error(token,value,stack) if token == 0 # denotes end of file value = 'end of file' else value = "'#{value[:value]}'" end error = "Syntax error at #{value}" if brace = @lexer.expected error += "; expected '#{brace}'" end except = Puppet::ParseError.new(error) except.line = @lexer.line except.file = @lexer.file if @lexer.file raise except end # how should I do error handling here? def parse(string = nil) if self.file =~ /\.rb$/ parse_ruby_file main = nil else self.string = string if string begin @yydebug = false main = yyparse(@lexer,:scan) rescue Racc::ParseError => except error = Puppet::ParseError.new(except) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error rescue Puppet::ParseError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::Error => except # and this is a framework error except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::DevError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue => except error = Puppet::DevError.new(except.message) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error end end # Store the results as the top-level class. return Puppet::Parser::AST::Hostclass.new('', :code => main) ensure @lexer.clear end def parse_ruby_file require self.file end def string=(string) @lexer.string = string end def version known_resource_types.version end # Add a new file to be checked when we're checking to see if we should be # reparsed. This is basically only used by the TemplateWrapper to let the # parser know about templates that should be parsed. def watch_file(filename) known_resource_types.watch_file(filename) end end diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 8a183f493..140c9f2ca 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -1,145 +1,145 @@ require 'puppet/node/environment' class Puppet::Parser::TypeLoader include Puppet::Node::Environment::Helper - class Helper < Hash + # Helper class that makes sure we don't try to import the same file + # more than once from either the same thread or different threads. + class Helper include MonitorMixin - def done_with(item) - synchronize do - delete(item)[:busy].signal if self.has_key?(item) and self[item][:loader] == Thread.current - end + def initialize + super + # These hashes are indexed by filename + @state = {} # :doing or :done + @thread = {} # if :doing, thread that's doing the parsing + @cond_var = {} # if :doing, condition var that will be signaled when done. end - def owner_of(item) - synchronize do - if !self.has_key? item - self[item] = { :loader => Thread.current, :busy => self.new_cond} - :nobody - elsif self[item][:loader] == Thread.current - :this_thread + + # Execute the supplied block exactly once per file, no matter how + # many threads have asked for it to run. If another thread is + # already executing it, wait for it to finish. If this thread is + # already executing it, return immediately without executing the + # block. + # + # Note: the reason for returning immediately if this thread is + # already executing the block is to handle the case of a circular + # import--when this happens, we attempt to recursively re-parse a + # file that we are already in the process of parsing. To prevent + # an infinite regress we need to simply do nothing when the + # recursive import is attempted. + def do_once(file) + need_to_execute = synchronize do + case @state[file] + when :doing + if @thread[file] != Thread.current + @cond_var[file].wait + end + false + when :done + false else - flag = self[item][:busy] - flag.wait - flag.signal - :another_thread + @state[file] = :doing + @thread[file] = Thread.current + @cond_var[file] = new_cond + true + end + end + if need_to_execute + begin + yield + ensure + synchronize do + @state[file] = :done + @thread.delete(file) + @cond_var.delete(file).broadcast + end end end end end # Import our files. def import(file, current_file = nil) return if Puppet[:ignoreimport] # use a path relative to the file doing the importing if current_file dir = current_file.sub(%r{[^/]+$},'').sub(/\/$/, '') else dir = "." end if dir == "" dir = "." end pat = file modname, files = Puppet::Parser::Files.find_manifests(pat, :cwd => dir, :environment => environment) if files.size == 0 raise Puppet::ImportError.new("No file(s) found for import of '#{pat}'") end loaded_asts = [] files.each do |file| unless file =~ /^#{File::SEPARATOR}/ file = File.join(dir, file) end - unless imported? file - @imported[file] = true + @loading_helper.do_once(file) do loaded_asts << parse_file(file) end end loaded_asts.inject([]) do |loaded_types, ast| loaded_types + known_resource_types.import_ast(ast, modname) end end - def imported?(file) - @imported.has_key?(file) - end - def known_resource_types environment.known_resource_types end def initialize(env) self.environment = env - @loaded = {} - @loading = Helper.new - - @imported = {} + @loading_helper = Helper.new end # Try to load the object with the given fully qualified name. def try_load_fqname(type, fqname) return nil if fqname == "" # special-case main. name2files(fqname).each do |filename| - if not loaded?(filename) - begin - imported_types = import_if_possible(filename) - if result = imported_types.find { |t| t.type == type and t.name == fqname } - Puppet.debug "Automatically imported #{fqname} from #{filename} into #{environment}" - return result - end - rescue Puppet::ImportError => detail - # We couldn't load the item - # I'm not convienced we should just drop these errors, but this - # preserves existing behaviours. + begin + imported_types = import(filename) + if result = imported_types.find { |t| t.type == type and t.name == fqname } + Puppet.debug "Automatically imported #{fqname} from #{filename} into #{environment}" + return result end + rescue Puppet::ImportError => detail + # We couldn't load the item + # I'm not convienced we should just drop these errors, but this + # preserves existing behaviours. end end # Nothing found. return nil end - def loaded?(name) - @loaded.include?(name) - end - def parse_file(file) Puppet.debug("importing '#{file}' in environment #{environment}") parser = Puppet::Parser::Parser.new(environment) parser.file = file return parser.parse end - # Utility method factored out of load for handling thread-safety. - # This isn't tested in the specs, because that's basically impossible. - def import_if_possible(file, current_file = nil) - @loaded[file] || begin - case @loading.owner_of(file) - when :this_thread - nil - when :another_thread - import_if_possible(file,current_file) - when :nobody - @loaded[file] = import(file,current_file) - end - ensure - @loading.done_with(file) - end - end - private # Return a list of all file basenames that should be tried in order # to load the object with the given fully qualified name. def name2files(fqname) result = [] ary = fqname.split("::") while ary.length > 0 result << ary.join(File::SEPARATOR) ary.pop end return result end end diff --git a/spec/lib/puppet_spec/files.rb b/spec/lib/puppet_spec/files.rb index cab4a1e47..52ed903ec 100644 --- a/spec/lib/puppet_spec/files.rb +++ b/spec/lib/puppet_spec/files.rb @@ -1,19 +1,20 @@ require 'fileutils' +require 'tempfile' # A support module for testing files. module PuppetSpec::Files def tmpfile(name) source = Tempfile.new(name) path = source.path source.close! $tmpfiles ||= [] $tmpfiles << path path end def tmpdir(name) file = tmpfile(name) FileUtils.mkdir_p(file) file end end diff --git a/spec/unit/parser/type_loader_spec.rb b/spec/unit/parser/type_loader_spec.rb index 58c386d96..cfa68871d 100644 --- a/spec/unit/parser/type_loader_spec.rb +++ b/spec/unit/parser/type_loader_spec.rb @@ -1,146 +1,134 @@ #!/usr/bin/env ruby require File.dirname(__FILE__) + '/../../spec_helper' require 'puppet/parser/type_loader' require 'puppet_spec/files' describe Puppet::Parser::TypeLoader do include PuppetSpec::Files before do @loader = Puppet::Parser::TypeLoader.new(:myenv) end it "should support an environment" do loader = Puppet::Parser::TypeLoader.new(:myenv) loader.environment.name.should == :myenv end it "should include the Environment Helper" do @loader.class.ancestors.should be_include(Puppet::Node::Environment::Helper) end it "should delegate its known resource types to its environment" do @loader.known_resource_types.should be_instance_of(Puppet::Resource::TypeCollection) end describe "when loading names from namespaces" do it "should do nothing if the name to import is an empty string" do @loader.expects(:name2files).never @loader.try_load_fqname(:hostclass, "") { |filename, modname| raise :should_not_occur }.should be_nil end it "should attempt to import each generated name" do @loader.expects(:import).with("foo/bar",nil).returns([]) @loader.expects(:import).with("foo",nil).returns([]) @loader.try_load_fqname(:hostclass, "foo::bar") { |f| false } end - - it "should know when a given name has been loaded" do - @loader.expects(:import).with("file",nil).returns([]) - @loader.try_load_fqname(:hostclass, "file") { |f| true } - @loader.should be_loaded("file") - end end describe "when importing" do before do Puppet::Parser::Files.stubs(:find_manifests).returns ["modname", %w{file}] - @loader.stubs(:parse_file).returns(Puppet::Parser::AST::Hostclass.new('')) + Puppet::Parser::Parser.any_instance.stubs(:parse).returns(Puppet::Parser::AST::Hostclass.new('')) + Puppet::Parser::Parser.any_instance.stubs(:file=) end it "should return immediately when imports are being ignored" do Puppet::Parser::Files.expects(:find_manifests).never Puppet[:ignoreimport] = true @loader.import("foo").should be_nil end it "should find all manifests matching the file or pattern" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| pat == "myfile" }.returns ["modname", %w{one}] @loader.import("myfile") end it "should use the directory of the current file if one is set" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| opts[:cwd] == "/current" }.returns ["modname", %w{one}] @loader.import("myfile", "/current/file") end it "should pass the environment when looking for files" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| opts[:environment] == @loader.environment }.returns ["modname", %w{one}] @loader.import("myfile") end it "should fail if no files are found" do Puppet::Parser::Files.expects(:find_manifests).returns [nil, []] lambda { @loader.import("myfile") }.should raise_error(Puppet::ImportError) end it "should parse each found file" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] @loader.expects(:parse_file).with("/one").returns(Puppet::Parser::AST::Hostclass.new('')) @loader.import("myfile") end it "should make each file qualified before attempting to parse it" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{one}] @loader.expects(:parse_file).with("/current/one").returns(Puppet::Parser::AST::Hostclass.new('')) @loader.import("myfile", "/current/file") end - it "should know when a given file has been imported" do - Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] - @loader.import("myfile") - - @loader.should be_imported("/one") - end - it "should not attempt to import files that have already been imported" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] - @loader.expects(:parse_file).once.returns(Puppet::Parser::AST::Hostclass.new('')) + Puppet::Parser::Parser.any_instance.expects(:parse).once.returns(Puppet::Parser::AST::Hostclass.new('')) @loader.import("myfile") # This will fail if it tries to reimport the file. @loader.import("myfile") end end describe "when parsing a file" do before do @parser = Puppet::Parser::Parser.new(@loader.environment) @parser.stubs(:parse).returns(Puppet::Parser::AST::Hostclass.new('')) @parser.stubs(:file=) Puppet::Parser::Parser.stubs(:new).with(@loader.environment).returns @parser end it "should create a new parser instance for each file using the current environment" do Puppet::Parser::Parser.expects(:new).with(@loader.environment).returns @parser @loader.parse_file("/my/file") end it "should assign the parser its file and parse" do @parser.expects(:file=).with("/my/file") @parser.expects(:parse).returns(Puppet::Parser::AST::Hostclass.new('')) @loader.parse_file("/my/file") end end it "should be able to add classes to the current resource type collection" do file = tmpfile("simple_file.pp") File.open(file, "w") { |f| f.puts "class foo {}" } @loader.import(file) @loader.known_resource_types.hostclass("foo").should be_instance_of(Puppet::Resource::Type) end describe "when deciding where to look for files" do { 'foo' => ['foo'], 'foo::bar' => ['foo/bar', 'foo'], 'foo::bar::baz' => ['foo/bar/baz', 'foo/bar', 'foo'] }.each do |fqname, expected_paths| it "should look for #{fqname.inspect} in #{expected_paths.inspect}" do @loader.instance_eval { name2files(fqname) }.should == expected_paths end end end end