diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index c0fd37178..4f3a4ddff 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -1,235 +1,235 @@ # I pulled this into a separate file, because I got # tired of rebuilding the parser.rb file all the time. class Puppet::Parser::Parser require 'puppet/parser/functions' require 'puppet/parser/files' require 'puppet/resource/type_collection' require 'puppet/resource/type_collection_helper' require 'puppet/resource/type' require 'monitor' AST = Puppet::Parser::AST include Puppet::Resource::TypeCollectionHelper attr_reader :version, :environment attr_accessor :files attr_accessor :lexer # Add context to a message; useful for error messages and such. def addcontext(message, obj = nil) obj ||= @lexer message += " on line #{obj.line}" if file = obj.file message += " in file #{file}" end message end # Create an AST array out of all of the args def aryfy(*args) if args[0].instance_of?(AST::ASTArray) result = args.shift args.each { |arg| result.push arg } else result = ast AST::ASTArray, :children => args end result end # Create an AST object, and automatically add the file and line information if # available. def ast(klass, hash = {}) klass.new ast_context(klass.use_docs).merge(hash) end def ast_context(include_docs = false) result = { :line => lexer.line, :file => lexer.file } result[:doc] = lexer.getcomment(result[:line]) if include_docs result end # The fully qualifed name, with the full namespace. def classname(name) [@lexer.namespace, name].join("::").sub(/^::/, '') end def clear initvars end # Raise a Parse error. def error(message) if brace = @lexer.expected message += "; expected '%s'" end except = Puppet::ParseError.new(message) except.line = @lexer.line except.file = @lexer.file if @lexer.file raise except end def file @lexer.file end def file=(file) unless FileTest.exist?(file) unless file =~ /\.pp$/ file = file + ".pp" end raise Puppet::Error, "Could not find file #{file}" unless FileTest.exist?(file) end raise Puppet::AlreadyImportedError, "Import loop detected" if known_resource_types.watching_file?(file) watch_file(file) @lexer.file = file end [:hostclass, :definition, :node, :nodes?].each do |method| define_method(method) do |*args| known_resource_types.send(method, *args) end end def find_hostclass(namespace, name) known_resource_types.find_or_load(namespace, name, :hostclass) end def find_definition(namespace, name) known_resource_types.find_or_load(namespace, name, :definition) end def import(file) - known_resource_types.loader.import_if_possible(file, @lexer.file) + known_resource_types.loader.import(file, @lexer.file) end def initialize(env) # The environment is needed to know how to find the resource type collection. @environment = env.is_a?(String) ? Puppet::Node::Environment.new(env) : env initvars end # Initialize or reset all of our variables. def initvars @lexer = Puppet::Parser::Lexer.new end # Split an fq name into a namespace and name def namesplit(fullname) ary = fullname.split("::") n = ary.pop || "" ns = ary.join("::") return ns, n end # Create a new class, or merge with an existing class. def newclass(name, options = {}) known_resource_types.add Puppet::Resource::Type.new(:hostclass, name, ast_context(true).merge(options)) end # Create a new definition. def newdefine(name, options = {}) known_resource_types.add Puppet::Resource::Type.new(:definition, name, ast_context(true).merge(options)) end # Create a new node. Nodes are special, because they're stored in a global # table, not according to namespaces. def newnode(names, options = {}) names = [names] unless names.instance_of?(Array) context = ast_context(true) names.collect do |name| known_resource_types.add(Puppet::Resource::Type.new(:node, name, context.merge(options))) end end def on_error(token,value,stack) if token == 0 # denotes end of file value = 'end of file' else value = "'#{value[:value]}'" end error = "Syntax error at #{value}" if brace = @lexer.expected error += "; expected '#{brace}'" end except = Puppet::ParseError.new(error) except.line = @lexer.line except.file = @lexer.file if @lexer.file raise except end # how should I do error handling here? def parse(string = nil) return parse_ruby_file if self.file =~ /\.rb$/ self.string = string if string begin @yydebug = false main = yyparse(@lexer,:scan) rescue Racc::ParseError => except error = Puppet::ParseError.new(except) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error rescue Puppet::ParseError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::Error => except # and this is a framework error except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::DevError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue => except error = Puppet::DevError.new(except.message) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error end if main # Store the results as the top-level class. newclass("", :code => main) end return known_resource_types ensure @lexer.clear end def parse_ruby_file require self.file end def string=(string) @lexer.string = string end def version known_resource_types.version end # Add a new file to be checked when we're checking to see if we should be # reparsed. This is basically only used by the TemplateWrapper to let the # parser know about templates that should be parsed. def watch_file(filename) known_resource_types.watch_file(filename) end end diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 09aa636e1..bae560381 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -1,146 +1,148 @@ require 'puppet/node/environment' class Puppet::Parser::TypeLoader include Puppet::Node::Environment::Helper - class Helper < Hash + # Helper class that makes sure we don't try to import the same file + # more than once from either the same thread or different threads. + class Helper include MonitorMixin - def done_with(item) - synchronize do - delete(item)[:busy].signal if self.has_key?(item) and self[item][:loader] == Thread.current - end + def initialize + super + # These hashes are indexed by filename + @state = {} # :doing or :done + @thread = {} # if :doing, thread that's doing the parsing + @cond_var = {} # if :doing, condition var that will be signaled when done. end - def owner_of(item) - synchronize do - if !self.has_key? item - self[item] = { :loader => Thread.current, :busy => self.new_cond} - :nobody - elsif self[item][:loader] == Thread.current - :this_thread + + # Execute the supplied block exactly once per file, no matter how + # many threads have asked for it to run. If another thread is + # already executing it, wait for it to finish. If this thread is + # already executing it, return immediately without executing the + # block. + # + # Note: the reason for returning immediately if this thread is + # already executing the block is to handle the case of a circular + # import--when this happens, we attempt to recursively re-parse a + # file that we are already in the process of parsing. To prevent + # an infinite regress we need to simply do nothing when the + # recursive import is attempted. + def do_once(file) + need_to_execute = synchronize do + case @state[file] + when :doing + if @thread[file] != Thread.current + @cond_var[file].wait + end + false + when :done + false else - flag = self[item][:busy] - flag.wait - flag.signal - :another_thread + @state[file] = :doing + @thread[file] = Thread.current + @cond_var[file] = new_cond + true + end + end + if need_to_execute + begin + yield + ensure + synchronize do + @state[file] = :done + @thread.delete(file) + @cond_var.delete(file).broadcast + end end end end end # Import our files. def import(file, current_file = nil) return if Puppet[:ignoreimport] # use a path relative to the file doing the importing if current_file dir = current_file.sub(%r{[^/]+$},'').sub(/\/$/, '') else dir = "." end if dir == "" dir = "." end pat = file modname, files = Puppet::Parser::Files.find_manifests(pat, :cwd => dir, :environment => environment) if files.size == 0 raise Puppet::ImportError.new("No file(s) found for import of '#{pat}'") end files.each do |file| unless file =~ /^#{File::SEPARATOR}/ file = File.join(dir, file) end - unless imported? file - @imported[file] = true + @loading_helper.do_once(file) do parse_file(file) end end modname end - def imported?(file) - @imported.has_key?(file) - end - def known_resource_types environment.known_resource_types end def initialize(env) self.environment = env - @loaded = {} - @loading = Helper.new - - @imported = {} + @loading_helper = Helper.new end def load_until(namespaces, name) return nil if name == "" # special-case main. name2files(namespaces, name).each do |filename| modname = begin - import_if_possible(filename) + import(filename) rescue Puppet::ImportError => detail # We couldn't load the item # I'm not convienced we should just drop these errors, but this # preserves existing behaviours. nil end if result = yield(filename) Puppet.debug "Automatically imported #{name} from #{filename} into #{environment}" result.module_name = modname if modname and result.respond_to?(:module_name=) return result end end nil end - def loaded?(name) - @loaded.include?(name) - end - def name2files(namespaces, name) return [name.sub(/^::/, '').gsub("::", File::SEPARATOR)] if name =~ /^::/ result = namespaces.inject([]) do |names_to_try, namespace| fullname = (namespace + "::#{name}").sub(/^::/, '') # Try to load the module init file if we're a qualified name names_to_try << fullname.split("::")[0] if fullname.include?("::") # Then the fully qualified name names_to_try << fullname end # Otherwise try to load the bare name on its own. This # is appropriate if the class we're looking for is in a # module that's different from our namespace. result << name result.uniq.collect { |f| f.gsub("::", File::SEPARATOR) } end def parse_file(file) Puppet.debug("importing '#{file}' in environment #{environment}") parser = Puppet::Parser::Parser.new(environment) parser.file = file parser.parse end - - # Utility method factored out of load for handling thread-safety. - # This isn't tested in the specs, because that's basically impossible. - def import_if_possible(file, current_file = nil) - @loaded[file] || begin - case @loading.owner_of(file) - when :this_thread - nil - when :another_thread - import_if_possible(file,current_file) - when :nobody - @loaded[file] = import(file,current_file) - end - ensure - @loading.done_with(file) - end - end end diff --git a/spec/lib/puppet_spec/files.rb b/spec/lib/puppet_spec/files.rb index cab4a1e47..52ed903ec 100644 --- a/spec/lib/puppet_spec/files.rb +++ b/spec/lib/puppet_spec/files.rb @@ -1,19 +1,20 @@ require 'fileutils' +require 'tempfile' # A support module for testing files. module PuppetSpec::Files def tmpfile(name) source = Tempfile.new(name) path = source.path source.close! $tmpfiles ||= [] $tmpfiles << path path end def tmpdir(name) file = tmpfile(name) FileUtils.mkdir_p(file) file end end diff --git a/spec/unit/parser/type_loader_spec.rb b/spec/unit/parser/type_loader_spec.rb index 83006b37b..02d543b02 100644 --- a/spec/unit/parser/type_loader_spec.rb +++ b/spec/unit/parser/type_loader_spec.rb @@ -1,201 +1,188 @@ #!/usr/bin/env ruby require File.dirname(__FILE__) + '/../../spec_helper' require 'puppet/parser/type_loader' require 'puppet_spec/files' describe Puppet::Parser::TypeLoader do include PuppetSpec::Files before do @loader = Puppet::Parser::TypeLoader.new(:myenv) end it "should support an environment" do loader = Puppet::Parser::TypeLoader.new(:myenv) loader.environment.name.should == :myenv end it "should include the Environment Helper" do @loader.class.ancestors.should be_include(Puppet::Node::Environment::Helper) end it "should delegate its known resource types to its environment" do @loader.known_resource_types.should be_instance_of(Puppet::Resource::TypeCollection) end describe "when loading names from namespaces" do it "should do nothing if the name to import is an empty string" do @loader.expects(:name2files).never @loader.load_until(["foo"], "") { |f| false }.should be_nil end it "should turn the provided namespaces and name into a list of files" do @loader.expects(:name2files).with(["foo"], "bar").returns [] @loader.load_until(["foo"], "bar") { |f| false } end it "should attempt to import each generated name" do @loader.expects(:name2files).returns %w{foo bar} @loader.expects(:import).with("foo",nil) @loader.expects(:import).with("bar",nil) @loader.load_until(["foo"], "bar") { |f| false } end it "should yield after each import" do yielded = [] @loader.expects(:name2files).returns %w{foo bar} @loader.expects(:import).with("foo",nil) @loader.expects(:import).with("bar",nil) @loader.load_until(["foo"], "bar") { |f| yielded << f; false } yielded.should == %w{foo bar} end it "should stop importing when the yielded block returns true" do yielded = [] @loader.expects(:name2files).returns %w{foo bar baz} @loader.expects(:import).with("foo",nil) @loader.expects(:import).with("bar",nil) @loader.expects(:import).with("baz",nil).never @loader.load_until(["foo"], "bar") { |f| true if f == "bar" } end it "should return the result of the block" do yielded = [] @loader.expects(:name2files).returns %w{foo bar baz} @loader.expects(:import).with("foo",nil) @loader.expects(:import).with("bar",nil) @loader.expects(:import).with("baz",nil).never @loader.load_until(["foo"], "bar") { |f| 10 if f == "bar" }.should == 10 end it "should return nil if the block never returns true" do @loader.expects(:name2files).returns %w{foo bar} @loader.expects(:import).with("foo",nil) @loader.expects(:import).with("bar",nil) @loader.load_until(["foo"], "bar") { |f| false }.should be_nil end - it "should know when a given name has been loaded" do - @loader.expects(:name2files).returns %w{file} - @loader.expects(:import).with("file",nil) - @loader.load_until(["foo"], "bar") { |f| true } - @loader.should be_loaded("file") - end - it "should set the module name on any created resource types" do type = Puppet::Resource::Type.new(:hostclass, "mytype") Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{one}] @loader.stubs(:parse_file) @loader.load_until(["foo"], "one") { |f| type } type.module_name.should == "modname" end end describe "when mapping names to files" do { [["foo"], "::bar::baz"] => %w{bar/baz}, [[""], "foo::bar"] => %w{foo foo/bar}, [%w{foo}, "bar"] => %w{foo foo/bar bar}, [%w{a b}, "bar"] => %w{a a/bar b b/bar bar}, [%w{a::b::c}, "bar"] => %w{a a/b/c/bar bar}, [%w{a::b}, "foo::bar"] => %w{a a/b/foo/bar foo/bar} }.each do |inputs, outputs| it "should produce #{outputs.inspect} from the #{inputs[0].inspect} namespace and #{inputs[1]} name" do @loader.name2files(*inputs).should == outputs end end end describe "when importing" do before do Puppet::Parser::Files.stubs(:find_manifests).returns ["modname", %w{file}] - @loader.stubs(:parse_file) + Puppet::Parser::Parser.any_instance.stubs(:parse) + Puppet::Parser::Parser.any_instance.stubs(:file=) end it "should return immediately when imports are being ignored" do Puppet::Parser::Files.expects(:find_manifests).never Puppet[:ignoreimport] = true @loader.import("foo").should be_nil end it "should find all manifests matching the file or pattern" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| pat == "myfile" }.returns ["modname", %w{one}] @loader.import("myfile") end it "should use the directory of the current file if one is set" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| opts[:cwd] == "/current" }.returns ["modname", %w{one}] @loader.import("myfile", "/current/file") end it "should pass the environment when looking for files" do Puppet::Parser::Files.expects(:find_manifests).with { |pat, opts| opts[:environment] == @loader.environment }.returns ["modname", %w{one}] @loader.import("myfile") end it "should fail if no files are found" do Puppet::Parser::Files.expects(:find_manifests).returns [nil, []] lambda { @loader.import("myfile") }.should raise_error(Puppet::ImportError) end it "should parse each found file" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] @loader.expects(:parse_file).with("/one") @loader.import("myfile") end it "should make each file qualified before attempting to parse it" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{one}] @loader.expects(:parse_file).with("/current/one") @loader.import("myfile", "/current/file") end - it "should know when a given file has been imported" do - Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] - @loader.import("myfile") - - @loader.should be_imported("/one") - end - it "should not attempt to import files that have already been imported" do Puppet::Parser::Files.expects(:find_manifests).returns ["modname", %w{/one}] - @loader.expects(:parse_file).once + Puppet::Parser::Parser.any_instance.expects(:parse).once @loader.import("myfile") # This will fail if it tries to reimport the file. @loader.import("myfile") end end describe "when parsing a file" do before do @parser = Puppet::Parser::Parser.new(@loader.environment) @parser.stubs(:parse) @parser.stubs(:file=) Puppet::Parser::Parser.stubs(:new).with(@loader.environment).returns @parser end it "should create a new parser instance for each file using the current environment" do Puppet::Parser::Parser.expects(:new).with(@loader.environment).returns @parser @loader.parse_file("/my/file") end it "should assign the parser its file and parse" do @parser.expects(:file=).with("/my/file") @parser.expects(:parse) @loader.parse_file("/my/file") end end it "should be able to add classes to the current resource type collection" do file = tmpfile("simple_file.pp") File.open(file, "w") { |f| f.puts "class foo {}" } @loader.import(file) @loader.known_resource_types.hostclass("foo").should be_instance_of(Puppet::Resource::Type) end end