diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index f1c1da0b0..bc3444c2a 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -1,486 +1,497 @@ # I pulled this into a separate file, because I got # tired of rebuilding the parser.rb file all the time. class Puppet::Parser::Parser require 'puppet/parser/functions' require 'puppet/parser/files' require 'puppet/parser/loaded_code' AST = Puppet::Parser::AST attr_reader :version, :environment attr_accessor :files attr_accessor :lexer # Add context to a message; useful for error messages and such. def addcontext(message, obj = nil) obj ||= @lexer message += " on line %s" % obj.line if file = obj.file message += " in file %s" % file end return message end # Create an AST array out of all of the args def aryfy(*args) if args[0].instance_of?(AST::ASTArray) result = args.shift args.each { |arg| result.push arg } else result = ast AST::ASTArray, :children => args end return result end # Create an AST object, and automatically add the file and line information if # available. def ast(klass, hash = {}) hash[:line] = @lexer.line unless hash.include?(:line) unless hash.include?(:file) if file = @lexer.file hash[:file] = file end end k = klass.new(hash) k.doc = lexer.getcomment(hash[:line]) if !k.nil? and k.use_docs and k.doc.empty? return k end # The fully qualifed name, with the full namespace. def classname(name) [@lexer.namespace, name].join("::").sub(/^::/, '') end def clear initvars end # Raise a Parse error. def error(message) if brace = @lexer.expected message += "; expected '%s'" end except = Puppet::ParseError.new(message) except.line = @lexer.line if @lexer.file except.file = @lexer.file end raise except end def file @lexer.file end def file=(file) unless FileTest.exists?(file) unless file =~ /\.pp$/ file = file + ".pp" end unless FileTest.exists?(file) raise Puppet::Error, "Could not find file %s" % file end end if check_and_add_to_watched_files(file) @lexer.file = file else raise Puppet::AlreadyImportedError.new("Import loop detected") end end [:hostclass, :definition, :node, :nodes?].each do |method| define_method(method) do |*args| @loaded_code.send(method, *args) end end def find_hostclass(namespace, name) find_or_load(namespace, name, :hostclass) end def find_definition(namespace, name) find_or_load(namespace, name, :definition) end def find_or_load(namespace, name, type) - method = "find_" + type.to_s - if result = @loaded_code.send(method, namespace, name) - return result - end - + method = "find_#{type}" fullname = (namespace + "::" + name).sub(/^::/, '') - self.load(fullname) - - if result = @loaded_code.send(method, namespace, name) - return result - end - - # Try to load the module init file if we're a qualified - # name - if fullname.include?("::") - module_name = fullname.split("::")[0] + names_to_try = [fullname] - self.load(module_name) + # Try to load the module init file if we're a qualified name + names_to_try << fullname.split("::")[0] if fullname.include?("::") - if result = @loaded_code.send(method, namespace, name) - return result - end - end - - # Now try to load the bare name on its own. This is - # appropriate if the class we're looking for is in a + # Otherwise try to load the bare name on its own. This + # is appropriate if the class we're looking for is in a # module that's different from our namespace. - self.load(name) + names_to_try << name - @loaded_code.send(method, namespace, name) + until (result = @loaded_code.send(method, namespace, name)) or names_to_try.empty? do + self.load(names_to_try.shift) + end + return result end # Import our files. def import(file) if Puppet[:ignoreimport] return AST::ASTArray.new(:children => []) end # use a path relative to the file doing the importing if @lexer.file dir = @lexer.file.sub(%r{[^/]+$},'').sub(/\/$/, '') else dir = "." end if dir == "" dir = "." end result = ast AST::ASTArray # We can't interpolate at this point since we don't have any # scopes set up. Warn the user if they use a variable reference pat = file if pat.index("$") Puppet.warning( "The import of #{pat} contains a variable reference;" + " variables are not interpolated for imports " + "in file #{@lexer.file} at line #{@lexer.line}" ) end files = Puppet::Parser::Files.find_manifests(pat, :cwd => dir, :environment => @environment) if files.size == 0 raise Puppet::ImportError.new("No file(s) found for import " + "of '#{pat}'") end files.collect { |file| parser = Puppet::Parser::Parser.new(:loaded_code => @loaded_code, :environment => @environment) parser.files = self.files Puppet.debug("importing '%s'" % file) unless file =~ /^#{File::SEPARATOR}/ file = File.join(dir, file) end begin parser.file = file rescue Puppet::AlreadyImportedError # This file has already been imported to just move on next end # This will normally add code to the 'main' class. parser.parse } end def initialize(options = {}) @loaded_code = options[:loaded_code] || Puppet::Parser::LoadedCode.new @environment = options[:environment] initvars() end # Initialize or reset all of our variables. def initvars @lexer = Puppet::Parser::Lexer.new() @files = {} @loaded = [] - end - - # Try to load a class, since we could not find it. - def load(classname) - return false if classname == "" - filename = classname.gsub("::", File::SEPARATOR) - - # First try to load the top-level module - mod = filename.scan(/^[\w-]+/).shift - unless @loaded.include?(mod) - @loaded << mod - begin - import(mod) - Puppet.info "Autoloaded module %s" % mod - rescue Puppet::ImportError => detail - # We couldn't load the module + @loading = {} + @loading.extend(MonitorMixin) + class << @loading + def done_with(item) + synchronize do + delete(item)[:busy].signal if self.has_key?(item) and self[item][:loader] == Thread.current + end + end + def owner_of(item) + synchronize do + if !self.has_key? item + self[item] = { :loader => Thread.current, :busy => self.new_cond} + :nobody + elsif self[item][:loader] == Thread.current + :this_thread + else + flag = self[item][:busy] + flag.wait + flag.signal + :another_thread + end + end end end + end - # We don't know whether we're looking for a class or definition, so we have - # to test for both. - return true if @loaded_code.hostclass(classname) || @loaded_code.definition(classname) - - unless @loaded.include?(filename) - @loaded << filename - # Then the individual file + # Utility method factored out of load + def able_to_import?(classname,item,msg) + unless @loaded.include?(item) begin - import(filename) - Puppet.info "Autoloaded file %s from module %s" % [filename, mod] + case @loading.owner_of(item) + when :this_thread + return + when :another_thread + return able_to_import?(classname,item,msg) + when :nobody + import(item) + Puppet.info "Autoloaded #{msg}" + @loaded << item + end rescue Puppet::ImportError => detail - # We couldn't load the file + # We couldn't load the item + ensure + @loading.done_with(item) end end # We don't know whether we're looking for a class or definition, so we have # to test for both. - return true if @loaded_code.hostclass(classname) || @loaded_code.definition(classname) + return @loaded_code.hostclass(classname) || @loaded_code.definition(classname) + end + + # Try to load a class, since we could not find it. + def load(classname) + return false if classname == "" + filename = classname.gsub("::", File::SEPARATOR) + mod = filename.scan(/^[\w-]+/).shift + + # First try to load the top-level module then the individual file + [[mod, "module %s" % mod ], + [filename,"file %s from module %s" % [filename, mod]] + ].any? { |item,description| able_to_import?(classname,item,description) } end # Split an fq name into a namespace and name def namesplit(fullname) ary = fullname.split("::") n = ary.pop || "" ns = ary.join("::") return ns, n end # Create a new class, or merge with an existing class. def newclass(name, options = {}) name = name.downcase if @loaded_code.definition(name) raise Puppet::ParseError, "Cannot redefine class %s as a definition" % name end code = options[:code] parent = options[:parent] doc = options[:doc] # If the class is already defined, then add code to it. if other = @loaded_code.hostclass(name) || @loaded_code.definition(name) # Make sure the parents match if parent and other.parentclass and (parent != other.parentclass) error("Class %s is already defined at %s:%s; cannot redefine" % [name, other.file, other.line]) end # This might be dangerous... if parent and ! other.parentclass other.parentclass = parent end # This might just be an empty, stub class. if code tmp = name if tmp == "" tmp = "main" end Puppet.debug addcontext("Adding code to %s" % tmp) # Else, add our code to it. if other.code and code # promote if neededcodes to ASTArray so that we can append code # ASTArray knows how to evaluate its members. other.code = ast AST::ASTArray, :children => [other.code] unless other.code.is_a?(AST::ASTArray) code = ast AST::ASTArray, :children => [code] unless code.is_a?(AST::ASTArray) other.code.children += code.children else other.code ||= code end end if other.doc and doc other.doc += doc else other.doc ||= doc end else # Define it anew. # Note we're doing something somewhat weird here -- we're setting # the class's namespace to its fully qualified name. This means # anything inside that class starts looking in that namespace first. args = {:namespace => name, :classname => name, :parser => self} args[:code] = code if code args[:parentclass] = parent if parent args[:doc] = doc args[:line] = options[:line] @loaded_code.add_hostclass(name, ast(AST::HostClass, args)) end return @loaded_code.hostclass(name) end # Create a new definition. def newdefine(name, options = {}) name = name.downcase if @loaded_code.hostclass(name) raise Puppet::ParseError, "Cannot redefine class %s as a definition" % name end # Make sure our definition doesn't already exist if other = @loaded_code.definition(name) error("%s is already defined at %s:%s; cannot redefine" % [name, other.file, other.line]) end ns, whatever = namesplit(name) args = { :namespace => ns, :arguments => options[:arguments], :code => options[:code], :parser => self, :classname => name, :doc => options[:doc], :line => options[:line] } [:code, :arguments].each do |param| args[param] = options[param] if options[param] end @loaded_code.add_definition(name, ast(AST::Definition, args)) end # Create a new node. Nodes are special, because they're stored in a global # table, not according to namespaces. def newnode(names, options = {}) names = [names] unless names.instance_of?(Array) doc = lexer.getcomment names.collect do |name| name = AST::HostName.new :value => name unless name.is_a?(AST::HostName) if other = @loaded_code.node(name) error("Node %s is already defined at %s:%s; cannot redefine" % [other.name, other.file, other.line]) end name = name.to_s if name.is_a?(Symbol) args = { :name => name, :parser => self, :doc => doc, :line => options[:line] } if options[:code] args[:code] = options[:code] end if options[:parent] args[:parentclass] = options[:parent] end node = ast(AST::Node, args) node.classname = name.to_classname @loaded_code.add_node(name, node) node end end def on_error(token,value,stack) if token == 0 # denotes end of file value = 'end of file' else value = "'%s'" % value end error = "Syntax error at %s" % [value] if brace = @lexer.expected error += "; expected '%s'" % brace end except = Puppet::ParseError.new(error) except.line = @lexer.line if @lexer.file except.file = @lexer.file end raise except end # how should I do error handling here? def parse(string = nil) if string self.string = string end begin @yydebug = false main = yyparse(@lexer,:scan) rescue Racc::ParseError => except error = Puppet::ParseError.new(except) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error rescue Puppet::ParseError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::Error => except # and this is a framework error except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::DevError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue => except error = Puppet::DevError.new(except.message) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error end if main # Store the results as the top-level class. newclass("", :code => main) end return @loaded_code ensure @lexer.clear end # See if any of the files have changed. def reparse? if file = @files.detect { |name, file| file.changed? } return file[1].stamp else return false end end def string=(string) @lexer.string = string end def version return @version if defined?(@version) if Puppet[:config_version] == "" @version = Time.now.to_i return @version end @version = %x{#{Puppet[:config_version]}}.chomp end # Add a new file to be checked when we're checking to see if we should be # reparsed. This is basically only used by the TemplateWrapper to let the # parser know about templates that should be parsed. def watch_file(filename) check_and_add_to_watched_files(filename) end private def check_and_add_to_watched_files(filename) unless @files.include?(filename) @files[filename] = Puppet::Util::LoadedFile.new(filename) return true else return false end end end diff --git a/spec/unit/parser/parser.rb b/spec/unit/parser/parser.rb index bd12f7155..75d0c05a3 100755 --- a/spec/unit/parser/parser.rb +++ b/spec/unit/parser/parser.rb @@ -1,331 +1,401 @@ #!/usr/bin/env ruby require File.dirname(__FILE__) + '/../../spec_helper' describe Puppet::Parser do ast = Puppet::Parser::AST before :each do @loaded_code = Puppet::Parser::LoadedCode.new @parser = Puppet::Parser::Parser.new :environment => "development", :loaded_code => @loaded_code @true_ast = Puppet::Parser::AST::Boolean.new :value => true end describe "when parsing append operator" do it "should not raise syntax errors" do lambda { @parser.parse("$var += something") }.should_not raise_error end it "shouldraise syntax error on incomplete syntax " do lambda { @parser.parse("$var += ") }.should raise_error end it "should call ast::VarDef with append=true" do ast::VarDef.expects(:new).with { |h| h[:append] == true } @parser.parse("$var += 2") end it "should work with arrays too" do ast::VarDef.expects(:new).with { |h| h[:append] == true } @parser.parse("$var += ['test']") end end describe "when parsing 'if'" do it "not, it should create the correct ast objects" do ast::Not.expects(:new).with { |h| h[:value].is_a?(ast::Boolean) } @parser.parse("if ! true { $var = 1 }") end it "boolean operation, it should create the correct ast objects" do ast::BooleanOperator.expects(:new).with { |h| h[:rval].is_a?(ast::Boolean) and h[:lval].is_a?(ast::Boolean) and h[:operator]=="or" } @parser.parse("if true or true { $var = 1 }") end it "comparison operation, it should create the correct ast objects" do ast::ComparisonOperator.expects(:new).with { |h| h[:lval].is_a?(ast::Name) and h[:rval].is_a?(ast::Name) and h[:operator]=="<" } @parser.parse("if 1 < 2 { $var = 1 }") end end describe "when parsing if complex expressions" do it "should create a correct ast tree" do aststub = stub_everything 'ast' ast::ComparisonOperator.expects(:new).with { |h| h[:rval].is_a?(ast::Name) and h[:lval].is_a?(ast::Name) and h[:operator]==">" }.returns(aststub) ast::ComparisonOperator.expects(:new).with { |h| h[:rval].is_a?(ast::Name) and h[:lval].is_a?(ast::Name) and h[:operator]=="==" }.returns(aststub) ast::BooleanOperator.expects(:new).with { |h| h[:rval]==aststub and h[:lval]==aststub and h[:operator]=="and" } @parser.parse("if (1 > 2) and (1 == 2) { $var = 1 }") end it "should raise an error on incorrect expression" do lambda { @parser.parse("if (1 > 2 > ) or (1 == 2) { $var = 1 }") }.should raise_error end end describe "when parsing resource references" do it "should not raise syntax errors" do lambda { @parser.parse('exec { test: param => File["a"] }') }.should_not raise_error end it "should not raise syntax errors with multiple references" do lambda { @parser.parse('exec { test: param => File["a","b"] }') }.should_not raise_error end it "should create an ast::ResourceReference" do ast::Resource.stubs(:new) ast::ResourceReference.expects(:new).with { |arg| arg[:line]==1 and arg[:type]=="File" and arg[:title].is_a?(ast::ASTArray) } @parser.parse('exec { test: command => File["a","b"] }') end end describe "when parsing resource overrides" do it "should not raise syntax errors" do lambda { @parser.parse('Resource["title"] { param => value }') }.should_not raise_error end it "should not raise syntax errors with multiple overrides" do lambda { @parser.parse('Resource["title1","title2"] { param => value }') }.should_not raise_error end it "should create an ast::ResourceOverride" do ast::ResourceOverride.expects(:new).with { |arg| arg[:line]==1 and arg[:object].is_a?(ast::ResourceReference) and arg[:params].is_a?(ast::ResourceParam) } @parser.parse('Resource["title1","title2"] { param => value }') end end describe "when parsing if statements" do it "should not raise errors with empty if" do lambda { @parser.parse("if true { }") }.should_not raise_error end it "should not raise errors with empty else" do lambda { @parser.parse("if false { notice('if') } else { }") }.should_not raise_error end it "should not raise errors with empty if and else" do lambda { @parser.parse("if false { } else { }") }.should_not raise_error end it "should create a nop node for empty branch" do ast::Nop.expects(:new) @parser.parse("if true { }") end it "should create a nop node for empty else branch" do ast::Nop.expects(:new) @parser.parse("if true { notice('test') } else { }") end end describe "when parsing function calls" do it "should not raise errors with no arguments" do lambda { @parser.parse("tag()") }.should_not raise_error end it "should not raise errors with rvalue function with no args" do lambda { @parser.parse("$a = template()") }.should_not raise_error end it "should not raise errors with arguments" do lambda { @parser.parse("notice(1)") }.should_not raise_error end it "should not raise errors with multiple arguments" do lambda { @parser.parse("notice(1,2)") }.should_not raise_error end it "should not raise errors with multiple arguments and a trailing comma" do lambda { @parser.parse("notice(1,2,)") }.should_not raise_error end end describe "when parsing arrays with trailing comma" do it "should not raise errors with a trailing comma" do lambda { @parser.parse("$a = [1,2,]") }.should_not raise_error end end describe "when instantiating class of same name" do before :each do @one = stub 'one', :is_a? => true @one.stubs(:is_a?).with(ast::ASTArray).returns(false) @one.stubs(:is_a?).with(ast).returns(true) @two = stub 'two' @two.stubs(:is_a?).with(ast::ASTArray).returns(false) @two.stubs(:is_a?).with(ast).returns(true) end it "should return the first class" do klass1 = @parser.newclass("one", { :code => @one }) @parser.newclass("one", { :code => @two }).should == klass1 end it "should concatenate code" do klass1 = @parser.newclass("one", { :code => @one }) @parser.newclass("one", { :code => @two }) klass1.code.children.should == [@one,@two] end end describe "when parsing comments before statement" do it "should associate the documentation to the statement AST node" do ast = @parser.parse(""" # comment class test {} """) ast.hostclass("test").doc.should == "comment\n" end end describe "when building ast nodes" do it "should get lexer comments if ast node declares use_docs" do lexer = stub 'lexer' ast = mock 'ast', :nil? => false, :use_docs => true, :doc => "" @parser.stubs(:lexer).returns(lexer) Puppet::Parser::AST::Definition.expects(:new).returns(ast) lexer.expects(:getcomment).returns("comment") ast.expects(:doc=).with("comment") @parser.ast(Puppet::Parser::AST::Definition) end end describe "when creating a node" do before :each do @lexer = stub 'lexer' @lexer.stubs(:getcomment) @parser.stubs(:lexer).returns(@lexer) @node = stub_everything 'node' @parser.stubs(:ast).returns(@node) @parser.stubs(:node).returns(nil) @nodename = stub 'nodename', :is_a? => false, :to_classname => "node" @nodename.stubs(:is_a?).with(Puppet::Parser::AST::HostName).returns(true) end it "should get the lexer stacked comments" do @lexer.expects(:getcomment) @parser.newnode(@nodename) end it "should create an HostName if needed" do Puppet::Parser::AST::HostName.expects(:new).with(:value => "node").returns(@nodename) @parser.newnode("node") end it "should raise an error if the node already exists" do @loaded_code.stubs(:node).with(@nodename).returns(@node) lambda { @parser.newnode(@nodename) }.should raise_error end it "should store the created node in the loaded code" do @loaded_code.expects(:add_node).with(@nodename, @node) @parser.newnode(@nodename) end it "should create the node with code if provided" do @parser.stubs(:ast).with { |*args| args[1][:code] == :code }.returns(@node) @parser.newnode(@nodename, :code => :code) end it "should create the node with a parentclass if provided" do @parser.stubs(:ast).with { |*args| args[1][:parent] == :parent }.returns(@node) @parser.newnode(@nodename, :parent => :parent) end it "should set the node classname from the HostName" do @nodename.stubs(:to_classname).returns(:classname) @node.expects(:classname=).with(:classname) @parser.newnode(@nodename) end it "should return an array of nodes" do @parser.newnode(@nodename).should == [@node] end end describe "when retrieving a specific node" do it "should delegate to the loaded_code node" do @loaded_code.expects(:node).with("node") @parser.node("node") end end describe "when retrieving a specific class" do it "should delegate to the loaded code" do @loaded_code.expects(:hostclass).with("class") @parser.hostclass("class") end end describe "when retrieving a specific definitions" do it "should delegate to the loaded code" do @loaded_code.expects(:definition).with("define") @parser.definition("define") end end describe "when determining the configuration version" do it "should default to the current time" do time = Time.now Time.stubs(:now).returns time @parser.version.should == time.to_i end it "should use the output of the config_version setting if one is provided" do Puppet.settings.stubs(:[]).with(:config_version).returns("/my/foo") @parser.expects(:`).with("/my/foo").returns "output\n" @parser.version.should == "output" end end - end + + describe Puppet::Parser,"when looking up definitions" do + it "should check for them by name" do + @parser.stubs(:find_or_load).with("namespace","name",:definition).returns(:this_value) + @parser.find_definition("namespace","name").should == :this_value + end + end + + describe Puppet::Parser,"when looking up hostclasses" do + it "should check for them by name" do + @parser.stubs(:find_or_load).with("namespace","name",:hostclass).returns(:this_value) + @parser.find_hostclass("namespace","name").should == :this_value + end + end + + describe Puppet::Parser,"when looking up names" do + before :each do + @loaded_code = mock 'loaded code' + @loaded_code.stubs(:find_my_type).with('Loaded_namespace', 'Loaded_name').returns(true) + @loaded_code.stubs(:find_my_type).with('Bogus_namespace', 'Bogus_name' ).returns(false) + @parser = Puppet::Parser::Parser.new :environment => "development",:loaded_code => @loaded_code + end + + describe "that are already loaded" do + it "should not try to load anything" do + @parser.expects(:load).never + @parser.find_or_load("Loaded_namespace","Loaded_name",:my_type) + end + it "should return true" do + @parser.find_or_load("Loaded_namespace","Loaded_name",:my_type).should == true + end + end + + describe "that aren't already loaded" do + it "should first attempt to load them with the fully qualified name" do + @loaded_code.stubs(:find_my_type).with("Foo_namespace","Foo_name").returns(false,true,true) + @parser.expects(:load).with("Foo_namespace::Foo_name").returns(true).then.raises(Exception) + @parser.find_or_load("Foo_namespace","Foo_name",:my_type).should == true + end + + it "should next attempt to load them with the namespace" do + @loaded_code.stubs(:find_my_type).with("Foo_namespace","Foo_name").returns(false,false,true,true) + @parser.expects(:load).with("Foo_namespace::Foo_name").returns(false).then.raises(Exception) + @parser.expects(:load).with("Foo_namespace").returns(true).then.raises(Exception) + @parser.find_or_load("Foo_namespace","Foo_name",:my_type).should == true + end + + it "should return false if the name isn't found" do + @parser.stubs(:load).returns(false) + @parser.find_or_load("Bogus_namespace","Bogus_name",:my_type).should == false + end + end + end + + describe Puppet::Parser,"when loading classnames" do + before :each do + @loaded_code = mock 'loaded code' + @parser = Puppet::Parser::Parser.new :environment => "development",:loaded_code => @loaded_code + end + + it "should just return false if the classname is empty" do + @parser.expects(:import).never + @parser.load("").should == false + end + + it "should just return true if the item is loaded" do + pending "Need to access internal state (@parser's @loaded) to force this" + @parser.load("").should == false + end + end +end