diff --git a/ext/puppet-test b/ext/puppet-test index 0f33e0cbb..e18dd1c57 100755 --- a/ext/puppet-test +++ b/ext/puppet-test @@ -1,464 +1,464 @@ #!/usr/bin/env ruby # == Synopsis # # Test individual client performance. Can compile configurations, describe # files, or retrieve files. # # = Usage # # puppet-test [-c|--compile] [-D|--describe ] [-d|--debug] # [--fork ] [-h|--help] [-H|--hostname ] [-l|--list] [-r|--repeat ] # [-R|--retrieve ] [-t|--test ] [-V|--version] [-v|--verbose] # # = Description # # This is a simple script meant for doing performance tests with Puppet. By # default it pulls down a compiled configuration, but it supports multiple # other tests. # # = Options # # Note that any configuration parameter that's valid in the configuration file # is also a valid long argument. For example, 'server' is a valid configuration # parameter, so you can specify '--server ' as an argument. # # See the configuration file documentation at # http://reductivelabs.com/projects/puppet/reference/configref.html for # the full list of acceptable parameters. A commented list of all # configuration $options can also be generated by running puppetd with # '--genconfig'. # # compile:: # Compile the client's configuration. The default. # # debug:: # Enable full debugging. # # describe:: # Describe the file being tested. This is a query to get information about # the file from the server, to determine if it should be copied, and is the # first half of every file transfer. # # fork:: # Fork the specified number of times, thus acting as multiple clients. # # fqdn:: # Set the fully-qualified domain name of the client. This is only used for # certificate purposes, but can be used to override the discovered hostname. # If you need to use this flag, it is generally an indication of a setup problem. # # help:: # Print this help message # # list:: # List all available tests. # # pause:: # Pause before starting test (useful for testing with dtrace). # # repeat:: # How many times to perform the test. # # retrieve:: # Test file retrieval performance. Retrieves the specified file from the # remote system. Note that the server should be specified via --server, # so the argument to this option is just the remote module name and path, # e.g., "/dist/apps/myapp/myfile", where "dist" is the module and # "apps/myapp/myfile" is the path to the file relative to the module. # # test:: # Specify the test to run. You can see the list of tests by running this command with --list. # # verbose:: # Turn on verbose reporting. # # version:: # Print the puppet version number and exit. # # = Example # # puppet-test --retrieve /module/path/to/file # # = Author # # Luke Kanies # # = Copyright # # Copyright (c) 2005, 2006 Reductive Labs, LLC # Licensed under the GNU Public License # Do an initial trap, so that cancels don't get a stack trace. trap(:INT) do $stderr.puts "Cancelling startup" exit(1) end require 'puppet' require 'puppet/network/client' require 'getoptlong' class Suite attr_reader :name, :doc @@suites = {} @@tests = {} def self.[](name) @@suites[name] end # Run a test by first finding the suite then running the appropriate test. def self.run(test) unless suite_name = @@tests[test] raise "Unknown test %s" % test end unless suite = @@suites[suite_name] raise "Unknown test suite %s from test %s" % [suite_name, test] end suite.run(test) end # What suites are available? def self.suites @@suites.keys end def forked? defined? @forking end # Create a new test suite. def initialize(name, doc, &block) @name = name @doc = doc @tests = {} @@suites[name] = self raise "You must pass a block to the Test" unless block_given? instance_eval(&block) end def prepare raise "Test %s did not override 'prepare'" % @name end # Define a new type of test on this suite. def newtest(name, doc, &block) @tests[name] = doc if @@tests[name] raise "Test names must be unique; cannot redefine %s" % name end @@tests[name] = @name meta_def(name, &block) end # Run the actual test. def run(test) unless doc = @tests[test] raise "Suite %s only supports tests %s; not %s" % [@name, @tests.keys.collect { |k| k.to_s }.join(","), test] end puts "Running %s %s test" % [@name, test] prepare() if $options[:pause] puts "Hit any key to continue" $stdin.readline puts "Continuing with test" end if $options[:fork] > 0 @forking = true $options[:fork].times { if pid = fork $pids << pid else break end } end $options[:repeat].times do |i| @count = i if forked? msg = doc + " in PID %s" % Process.pid else msg = doc end Puppet::Util.benchmark(:notice, msg) do begin send(test) rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "%s failed: %s" % [@name, detail.to_s] end end end end # What tests are available on this suite? def tests @tests.keys end end Suite.new :parser, "Manifest parsing" do def prepare - @parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment]) - @parser.file = Puppet[:manifest] end newtest :parse, "Parsed files" do + @parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment]) + @parser.file = Puppet[:manifest] @parser.parse end end Suite.new :catalog, "Catalog handling" do def prepare $args[:cache] = false # Create a config client and pull the config down @client = Puppet::Network::Client.master.new($args) unless @client.read_cert fail "Could not read client certificate" end # Use the facts from the cache, to skip the time it takes # to load them. @client.dostorage @facts = Puppet::Util::Storage.cache(:configuration)[:facts] if @facts.empty? @facts = @client.master.getfacts end if host = $options[:fqdn] @facts["fqdn"] = host @facts["hostname"] = host.sub(/\..+/, '') @facts["domain"] = host.sub(/^[^.]+\./, '') end @facts = YAML.dump(@facts) end newtest :compile, "Compiled catalog" do @client.driver.getconfig(@facts, "yaml") end # This test will always force a false answer. newtest :fresh, "Checked freshness" do @client.driver.freshness end end Suite.new :file, "File interactions" do def prepare unless $options[:file] fail "You must specify a file (using --file ) to interact with on the server" end @client = Puppet::Network::Client.file.new($args) unless @client.read_cert fail "Could not read client certificate" end end newtest :describe, "Described file" do @client.describe($options[:file], :ignore) end newtest :retrieve, "Retrieved file" do @client.retrieve($options[:file], :ignore) end end # Note that this uses an env variable to determine how many resources per # host to create (with a default of 10). 'repeat' determines how # many hosts to create. You probably will get mad failures if you # use forking and sqlite. # Here's an example run of this test, using sqlite: # RESOURCE_COUNT=50 ext/puppet-test --suite rails --test storage --confdir /tmp/storagetesting --vardir /tmp/storagetesting --repeat 10 Suite.new :rails, "Rails Interactions" do def prepare Puppet::Rails.init @facts = Facter.to_hash if num = ENV["RESOURCECOUNT"] @resources = Integer(num) else @resources = 10 end end Resource = Puppet::Parser::Resource def execute(test, msg) begin send(test) rescue => detail puts detail.backtrace if Puppet[:trace] Puppet.err "%s failed: %s" % [@name, detail.to_s] end end def mkresource(type, title, parameters) source = "fakesource" res = Resource.new(:type => type, :title => title, :source => source, :scope => "fakescope") parameters.each do |param, value| res.set(param, value, source) end res end def ref(type, title) Resource::Reference.new(:type => type, :title => title) end newtest :storage, "Stored resources" do hostname = "host%s" % @count @facts["hostname"] = hostname args = {:facts => @facts, :name => hostname} # Make all of the resources we want. Make them at least slightly complex, # so we model real resources as close as we can. resources = [] args[:resources] = resources @resources.times do |resnum| exec = mkresource("exec", "exec%s" % resnum, :command => "/bin/echo do something %s" % resnum) exec.tags = %w{exec one} << "exec%s" % resnum user = mkresource("user", "user%s" % resnum, :uid => resnum.to_s, :require => ref("exec", "exec%s" % resnum)) user.tags = %w{user one two} << "user%s" % resnum file = mkresource("file", "/tmp/file%s" % resnum, :owner => resnum.to_s, :require => [ref("exec", "exec%s" % resnum), ref("user", "user%s" % resnum)]) file.tags = %w{file one three} << "file%s" % resnum file.exported = true resources << exec << user << file end Puppet::Rails::Host.store(args) end end $cmdargs = [ [ "--compile", "-c", GetoptLong::NO_ARGUMENT ], [ "--describe", GetoptLong::REQUIRED_ARGUMENT ], [ "--retrieve", "-R", GetoptLong::REQUIRED_ARGUMENT ], [ "--fork", GetoptLong::REQUIRED_ARGUMENT ], [ "--fqdn", "-F", GetoptLong::REQUIRED_ARGUMENT ], [ "--suite", "-s", GetoptLong::REQUIRED_ARGUMENT ], [ "--test", "-t", GetoptLong::REQUIRED_ARGUMENT ], [ "--pause", "-p", GetoptLong::NO_ARGUMENT ], [ "--repeat", "-r", GetoptLong::REQUIRED_ARGUMENT ], [ "--debug", "-d", GetoptLong::NO_ARGUMENT ], [ "--help", "-h", GetoptLong::NO_ARGUMENT ], [ "--list", "-l", GetoptLong::NO_ARGUMENT ], [ "--verbose", "-v", GetoptLong::NO_ARGUMENT ], [ "--version", "-V", GetoptLong::NO_ARGUMENT ], ] # Add all of the config parameters as valid $options. Puppet.settings.addargs($cmdargs) Puppet::Util::Log.newdestination(:console) result = GetoptLong.new(*$cmdargs) $args = {} $options = {:repeat => 1, :fork => 0, :pause => false} begin explicit_waitforcert = false result.each { |opt,arg| case opt # First check to see if the argument is a valid configuration parameter; # if so, set it. when "--compile" $options[:suite] = :configuration $options[:test] = :compile when "--retrieve" $options[:suite] = :file $options[:test] = :retrieve $options[:file] = arg when "--fork" begin $options[:fork] = Integer(arg) rescue => detail $stderr.puts "The argument to 'fork' must be an integer" exit(14) end when "--describe" $options[:suite] = :file $options[:test] = :describe $options[:file] = arg when "--fqdn" $options[:fqdn] = arg when "--repeat" $options[:repeat] = Integer(arg) when "--help" if Puppet.features.usage? RDoc::usage && exit else puts "No help available unless you have RDoc::usage installed" exit end when "--version" puts "%s" % Puppet.version exit when "--verbose" Puppet::Util::Log.level = :info Puppet::Util::Log.newdestination(:console) when "--debug" Puppet::Util::Log.level = :debug Puppet::Util::Log.newdestination(:console) when "--suite" $options[:suite] = arg.intern when "--test" $options[:test] = arg.intern when "--file" $options[:file] = arg when "--pause" $options[:pause] = true when "--list" Suite.suites.sort { |a,b| a.to_s <=> b.to_s }.each do |suite_name| suite = Suite[suite_name] tests = suite.tests.sort { |a,b| a.to_s <=> b.to_s }.join(", ") puts "%20s: %s" % [suite_name, tests] end exit(0) else Puppet.settings.handlearg(opt, arg) end } rescue GetoptLong::InvalidOption => detail $stderr.puts detail $stderr.puts "Try '#{$0} --help'" exit(1) end # Now parse the config Puppet.parse_config $args[:Server] = Puppet[:server] unless $options[:test] $options[:suite] = :configuration $options[:test] = :compile end unless $options[:test] raise "A suite was specified without a test" end $pids = [] Suite.run($options[:test]) if $options[:fork] > 0 Process.waitall end # $Id$ diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb index 086d82c09..6661650ba 100644 --- a/lib/puppet/parser/lexer.rb +++ b/lib/puppet/parser/lexer.rb @@ -1,332 +1,437 @@ - # the scanner/lexer require 'strscan' require 'puppet' module Puppet class LexError < RuntimeError; end - module Parser - #--------------------------------------------------------------- - class Lexer - attr_reader :line, :last, :file - - attr_accessor :indefine - - #%r{\w+} => :WORD, - @@tokens = { - %r{#.*} => :COMMENT, - %r{\[} => :LBRACK, - %r{\]} => :RBRACK, - %r{\{} => :LBRACE, - %r{\}} => :RBRACE, - %r{\(} => :LPAREN, - %r{\)} => :RPAREN, - %r{\"} => :DQUOTE, - %r{\n} => :RETURN, - %r{\'} => :SQUOTE, - %r{=} => :EQUALS, - %r{==} => :ISEQUAL, - %r{>=} => :GREATEREQUAL, - %r{>} => :GREATERTHAN, - %r{<} => :LESSTHAN, - %r{<=} => :LESSEQUAL, - %r{!=} => :NOTEQUAL, - %r{!} => :NOT, - %r{,} => :COMMA, - %r{\.} => :DOT, - %r{:} => :COLON, - %r{@} => :AT, - %r{<<\|} => :LLCOLLECT, - %r{\|>>} => :RRCOLLECT, - %r{<\|} => :LCOLLECT, - %r{\|>} => :RCOLLECT, - %r{;} => :SEMIC, - %r{\?} => :QMARK, - %r{\\} => :BACKSLASH, - %r{=>} => :FARROW, - %r{\+>} => :PARROW, - %r{[a-z][-\w]*} => :NAME, - %r{([a-z][-\w]*::)+[a-z][-\w]*} => :CLASSNAME, - %r{((::){0,1}[A-Z][-\w]*)+} => :CLASSREF, - %r{[0-9]+} => :NUMBER, - %r{\$(\w*::)*\w+} => :VARIABLE - } - - @@pairs = { - "{" => "}", - "(" => ")", - "[" => "]", - "<|" => "|>", - "<<|" => "|>>" - } - - @@reverse_pairs = @@pairs.inject({}) { |hash, pair| hash[pair[1]] = pair[0]; hash } - - @@keywords = { - "case" => :CASE, - "class" => :CLASS, - "default" => :DEFAULT, - "define" => :DEFINE, - "false" => :BOOLEAN, - "import" => :IMPORT, - "if" => :IF, - "elsif" => :ELSIF, - "else" => :ELSE, - "inherits" => :INHERITS, - "node" => :NODE, - "true" => :BOOLEAN, - "and" => :AND, - "or" => :OR, - "undef" => :UNDEF - } - - def clear - initvars - end +end - def expected - if @expected.empty? - nil - else - token = @expected[-1] - @@tokens.each do |value, name| - if token == name - return value - end - end - return token - end - end +module Puppet::Parser; end - # scan the whole file - # basically just used for testing - def fullscan - array = [] - - self.scan { |token,str| - # Ignore any definition nesting problems - @indefine = false - #Puppet.debug("got token '%s' => '%s'" % [token,str]) - if token.nil? - return array - else - array.push([token,str]) - end - } - return array - end +class Puppet::Parser::Lexer + attr_reader :last, :file + + attr_accessor :line, :indefine - # this is probably pretty damned inefficient... - # it'd be nice not to have to load the whole file first... - def file=(file) - @file = file - @line = 1 - File.open(file) { |of| - str = "" - of.each { |line| str += line } - @scanner = StringScanner.new(str) - } + # Our base token class. + class Token + attr_accessor :regex, :name, :string, :skip, :incr_line, :skip_text + + def initialize(regex, name) + if regex.is_a?(String) + @name, @string = name, regex + @regex = Regexp.new(Regexp.escape(@string)) + else + @name, @regex = name, regex end + end - def indefine? - if defined? @indefine - @indefine - else - false - end + def skip? + self.skip + end + + def to_s + "Lexer token %s" % @name.to_s + end + end + + # Maintain a list of tokens. + class TokenList + attr_reader :regex_tokens, :string_tokens + + def [](name) + @tokens[name] + end + + # Create a new token. + def add_token(name, regex, options = {}, &block) + token = Token.new(regex, name) + raise(ArgumentError, "Token %s already exists" % name) if @tokens.include?(name) + @tokens[token.name] = token + if token.string + @string_tokens << token + @tokens_by_string[token.string] = token + else + @regex_tokens << token end - def initialize - initvars() + options.each do |name, option| + token.send(name.to_s + "=", option) end - def initvars - @line = 1 - @last = "" - @lasttoken = nil - @scanner = nil - @file = nil - # AAARRGGGG! okay, regexes in ruby are bloody annoying - # no one else has "\n" =~ /\s/ - @skip = %r{[ \t]+} + token.meta_def(:convert, &block) if block_given? - @namestack = [] - @indefine = false + token + end - @expected = [] - end + def initialize + @tokens = {} + @regex_tokens = [] + @string_tokens = [] + @tokens_by_string = {} + end - # Go up one in the namespace. - def namepop - @namestack.pop + # Look up a token by its value, rather than name. + def lookup(string) + @tokens_by_string[string] + end + + # Define more tokens. + def add_tokens(hash) + hash.each do |regex, name| + add_token(name, regex) end + end - # Collect the current namespace. - def namespace - @namestack.join("::") + # Sort our tokens by length, so we know once we match, we're done. + # This helps us avoid the O(n^2) nature of token matching. + def sort_tokens + @string_tokens.sort! { |a, b| b.string.length <=> a.string.length } + end + end + + TOKENS = TokenList.new + TOKENS.add_tokens( + '[' => :LBRACK, + ']' => :RBRACK, + '{' => :LBRACE, + '}' => :RBRACE, + '(' => :LPAREN, + ')' => :RPAREN, + '=' => :EQUALS, + '==' => :ISEQUAL, + '>=' => :GREATEREQUAL, + '>' => :GREATERTHAN, + '<' => :LESSTHAN, + '<=' => :LESSEQUAL, + '!=' => :NOTEQUAL, + '!' => :NOT, + ',' => :COMMA, + '.' => :DOT, + ':' => :COLON, + '@' => :AT, + '<<|' => :LLCOLLECT, + '|>>' => :RRCOLLECT, + '<|' => :LCOLLECT, + '|>' => :RCOLLECT, + ';' => :SEMIC, + '?' => :QMARK, + '\\' => :BACKSLASH, + '=>' => :FARROW, + '+>' => :PARROW, + %r{([a-z][-\w]*::)+[a-z][-\w]*} => :CLASSNAME, + %r{((::){0,1}[A-Z][-\w]*)+} => :CLASSREF + ) + + TOKENS.add_tokens "Whatever" => :DQTEXT, "Nomatter" => :SQTEXT, "alsonomatter" => :BOOLEAN + + TOKENS.add_token :NAME, %r{[a-z][-\w]*} do |lexer, value| + string_token = self + # we're looking for keywords here + if tmp = KEYWORDS.lookup(value) + string_token = tmp + if [:TRUE, :FALSE].include?(string_token.name) + value = eval(value) + string_token = TOKENS[:BOOLEAN] end + end + [string_token, value] + end + + TOKENS.add_token :NUMBER, %r{[0-9]+} do |lexer, value| + [TOKENS[:NAME], value] + end + + TOKENS.add_token :COMMENT, %r{#.*}, :skip => true + + TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true + + TOKENS.add_token :SQUOTE, "'" do |lexer, value| + value = lexer.slurpstring(value) + [TOKENS[:SQTEXT], value] + end - # This value might have :: in it, but we don't care -- it'll be - # handled normally when joining, and when popping we want to pop - # this full value, however long the namespace is. - def namestack(value) - @namestack << value + TOKENS.add_token :DQUOTE, '"' do |lexer, value| + value = lexer.slurpstring(value) + [TOKENS[:DQTEXT], value] + end + + TOKENS.add_token :VARIABLE, %r{\$(\w*::)*\w+} do |lexer, value| + value = value.sub(/^\$/, '') + [self, value] + end + + TOKENS.sort_tokens + + @@pairs = { + "{" => "}", + "(" => ")", + "[" => "]", + "<|" => "|>", + "<<|" => "|>>" + } + + KEYWORDS = TokenList.new + + KEYWORDS.add_tokens( + "case" => :CASE, + "class" => :CLASS, + "default" => :DEFAULT, + "define" => :DEFINE, + "import" => :IMPORT, + "if" => :IF, + "elsif" => :ELSIF, + "else" => :ELSE, + "inherits" => :INHERITS, + "node" => :NODE, + "and" => :AND, + "or" => :OR, + "undef" => :UNDEF, + "false" => :FALSE, + "true" => :TRUE + ) + + def clear + initvars + end + + def expected + return nil if @expected.empty? + name = @expected[-1] + raise "Could not find expected token %s" % name unless token = TOKENS.lookup(name) + + return token + end + + # scan the whole file + # basically just used for testing + def fullscan + array = [] + + self.scan { |token, str| + # Ignore any definition nesting problems + @indefine = false + array.push([token,str]) + } + return array + end + + # this is probably pretty damned inefficient... + # it'd be nice not to have to load the whole file first... + def file=(file) + @file = file + @line = 1 + File.open(file) { |of| + str = "" + of.each { |line| str += line } + @scanner = StringScanner.new(str) + } + end + + def find_string_token + matched_token = value = nil + + # We know our longest string token is three chars, so try each size in turn + # until we either match or run out of chars. This way our worst-case is three + # tries, where it is otherwise the number of string chars we have. Also, + # the lookups are optimized hash lookups, instead of regex scans. + [3, 2, 1].each do |i| + str = @scanner.peek(i) + if matched_token = TOKENS.lookup(str) + value = @scanner.scan(matched_token.regex) + break end + end + + return matched_token, value + end - def rest - @scanner.rest + # Find the next token that matches a regex. We look for these first. + def find_regex_token + @regex += 1 + matched_token = nil + value = "" + length = 0 + + # I tried optimizing based on the first char, but it had + # a slightly negative affect and was a good bit more complicated. + TOKENS.regex_tokens.each do |token| + next unless match_length = @scanner.match?(token.regex) + + # We've found a longer match + if match_length > length + value = @scanner.scan(token.regex) + length = value.length + matched_token = token end + end - # this is the heart of the lexer - def scan - #Puppet.debug("entering scan") - if @scanner.nil? - raise TypeError.new("Invalid or empty string") - end + return matched_token, value + end - @scanner.skip(@skip) - until @scanner.eos? do - yielded = false - sendbreak = false # gah, this is a nasty hack - stoken = nil - sregex = nil - value = "" - - # first find out which type of token we've got - @@tokens.each { |regex,token| - # we're just checking, which doesn't advance the scan - # pointer - tmp = @scanner.check(regex) - if tmp.nil? - #puppet.debug("did not match %s to '%s'" % - # [regex,@scanner.rest]) - next - end - - # find the longest match - if tmp.length > value.length - value = tmp - stoken = token - sregex = regex - else - # we've already got a longer match - next - end - } - - # error out if we didn't match anything at all - if stoken.nil? - nword = nil - if @scanner.rest =~ /^(\S+)/ - nword = $1 - elsif@scanner.rest =~ /^(\s+)/ - nword = $1 - else - nword = @scanner.rest - end - raise "Could not match '%s'" % nword - end + # Find the next token, returning the string and the token. + def find_token + @find += 1 + matched_token, value = find_regex_token - value = @scanner.scan(sregex) + unless matched_token + matched_token, value = find_string_token + end - if value == "" - raise "Didn't match regex on token %s" % stoken - end + return matched_token, value + end - # token-specific operations - # if this gets much more complicated, it should - # be moved up to where the tokens themselves are defined - # which will get me about 75% of the way to a lexer generator - ptoken = stoken - case stoken - when :NAME then - wtoken = stoken - # we're looking for keywords here - if @@keywords.include?(value) - wtoken = @@keywords[value] - #Puppet.debug("token '%s'" % wtoken) - if wtoken == :BOOLEAN - value = eval(value) - end - end - ptoken = wtoken - when :NUMBER then - ptoken = :NAME - when :COMMENT then - # just throw comments away - next - when :RETURN then - @line += 1 - @scanner.skip(@skip) - next - when :SQUOTE then - #Puppet.debug("searching '%s' after '%s'" % [self.rest,value]) - value = self.slurpstring(value) - ptoken = :SQTEXT - #Puppet.debug("got string '%s' => '%s'" % [:DQTEXT,value]) - when :DQUOTE then - value = self.slurpstring(value) - ptoken = :DQTEXT - when :VARIABLE then - value = value.sub(/^\$/, '') - end + def indefine? + if defined? @indefine + @indefine + else + false + end + end - if match = @@pairs[value] and ptoken != :DQUOTE and ptoken != :SQUOTE - @expected << match - elsif exp = @expected[-1] and exp == value and ptoken != :DQUOTE and ptoken != :SQUOTE - @expected.pop - end + def initialize + @find = 0 + @regex = 0 + initvars() + end - yield [ptoken, value] + def initvars + @line = 1 + @previous_token = nil + @scanner = nil + @file = nil + # AAARRGGGG! okay, regexes in ruby are bloody annoying + # no one else has "\n" =~ /\s/ + @skip = %r{[ \t]+} + + @namestack = [] + @indefine = false + @expected = [] + end - if @lasttoken == :CLASS - namestack(value) - end + # Make any necessary changes to the token and/or value. + def munge_token(token, value) + @line += 1 if token.incr_line - if @lasttoken == :DEFINE - if indefine? - msg = "Cannot nest definition %s inside %s" % [value, @indefine] - self.indefine = false - raise Puppet::ParseError, msg - end + skip() if token.skip_text - @indefine = value - end + return if token.skip - @last = value - @lasttoken = ptoken + token, value = token.convert(self, value) if token.respond_to?(:convert) - @scanner.skip(@skip) - end - @scanner = nil - yield [false,false] - end + return unless token + + return token, value + end + + # Go up one in the namespace. + def namepop + @namestack.pop + end - # we've encountered an opening quote... - # slurp in the rest of the string and return it - def slurpstring(quote) - # we search for the next quote that isn't preceded by a - # backslash; the caret is there to match empty strings - str = @scanner.scan_until(/([^\\]|^)#{quote}/) - if str.nil? - raise Puppet::LexError.new("Unclosed quote after '%s' in '%s'" % - [self.last,self.rest]) + # Collect the current namespace. + def namespace + @namestack.join("::") + end + + # This value might have :: in it, but we don't care -- it'll be + # handled normally when joining, and when popping we want to pop + # this full value, however long the namespace is. + def namestack(value) + @namestack << value + end + + def rest + @scanner.rest + end + + # this is the heart of the lexer + def scan + #Puppet.debug("entering scan") + raise Puppet::LexError.new("Invalid or empty string") unless @scanner + + # Skip any initial whitespace. + skip() + + until @scanner.eos? do + yielded = false + matched_token, value = find_token + + # error out if we didn't match anything at all + if matched_token.nil? + nword = nil + # Try to pull a 'word' out of the remaining string. + if @scanner.rest =~ /^(\S+)/ + nword = $1 + elsif @scanner.rest =~ /^(\s+)/ + nword = $1 else - str.sub!(/#{quote}\Z/,"") - str.gsub!(/\\#{quote}/,quote) + nword = @scanner.rest end + raise "Could not match '%s'" % nword + end - return str + final_token, value = munge_token(matched_token, value) + + next unless final_token + + if match = @@pairs[value] and final_token.name != :DQUOTE and final_token.name != :SQUOTE + @expected << match + elsif exp = @expected[-1] and exp == value and final_token.name != :DQUOTE and final_token.name != :SQUOTE + @expected.pop end - # just parse a string, not a whole file - def string=(string) - @scanner = StringScanner.new(string) + yield [final_token.name, value] + + if @previous_token + namestack(value) if @previous_token.name == :CLASS + + if @previous_token.name == :DEFINE + if indefine? + msg = "Cannot nest definition %s inside %s" % [value, @indefine] + self.indefine = false + raise Puppet::ParseError, msg + end + + @indefine = value + end end + + @previous_token = final_token + skip() end - #--------------------------------------------------------------- + @scanner = nil + + # This indicates that we're done parsing. + yield [false,false] + end + + # Skip any skipchars in our remaining string. + def skip + @scanner.skip(@skip) end -end + # we've encountered an opening quote... + # slurp in the rest of the string and return it + def slurpstring(quote) + # we search for the next quote that isn't preceded by a + # backslash; the caret is there to match empty strings + str = @scanner.scan_until(/([^\\]|^)#{quote}/) + if str.nil? + raise Puppet::LexError.new("Unclosed quote after '%s' in '%s'" % + [self.last,self.rest]) + else + str.sub!(/#{quote}\Z/,"") + str.gsub!(/\\#{quote}/,quote) + end + + return str + end + + # just parse a string, not a whole file + def string=(string) + @scanner = StringScanner.new(string) + end +end diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index acf3c9f7c..b543cd3ec 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -1,456 +1,453 @@ # I pulled this into a separate file, because I got # tired of rebuilding the parser.rb file all the time. class Puppet::Parser::Parser require 'puppet/parser/functions' ASTSet = Struct.new(:classes, :definitions, :nodes) # Define an accessor method for each table. We hide the existence of # the struct. [:classes, :definitions, :nodes].each do |name| define_method(name) do @astset.send(name) end end AST = Puppet::Parser::AST attr_reader :version, :environment attr_accessor :files # Add context to a message; useful for error messages and such. def addcontext(message, obj = nil) obj ||= @lexer message += " on line %s" % obj.line if file = obj.file message += " in file %s" % file end return message end # Create an AST array out of all of the args def aryfy(*args) if args[0].instance_of?(AST::ASTArray) result = args.shift args.each { |arg| result.push arg } else result = ast AST::ASTArray, :children => args end return result end # Create an AST object, and automatically add the file and line information if # available. - def ast(klass, hash = nil) - hash ||= {} - unless hash.include?(:line) - hash[:line] = @lexer.line - end + def ast(klass, hash = {}) + hash[:line] = @lexer.line unless hash.include?(:line) unless hash.include?(:file) if file = @lexer.file hash[:file] = file end end return klass.new(hash) end # The fully qualifed name, with the full namespace. def classname(name) [@lexer.namespace, name].join("::").sub(/^::/, '') end def clear initvars end # Raise a Parse error. def error(message) if brace = @lexer.expected message += "; expected '%s'" end except = Puppet::ParseError.new(message) except.line = @lexer.line if @lexer.file except.file = @lexer.file end raise except end def file @lexer.file end def file=(file) unless FileTest.exists?(file) unless file =~ /\.pp$/ file = file + ".pp" end unless FileTest.exists?(file) raise Puppet::Error, "Could not find file %s" % file end end if @files.detect { |f| f.file == file } raise Puppet::AlreadyImportedError.new("Import loop detected") else @files << Puppet::Util::LoadedFile.new(file) @lexer.file = file end end # Find a class definition, relative to the current namespace. def findclass(namespace, name) fqfind namespace, name, classes end # Find a component definition, relative to the current namespace. def finddefine(namespace, name) fqfind namespace, name, definitions end # This is only used when nodes are looking up the code for their # parent nodes. def findnode(name) fqfind "", name, nodes end # The recursive method used to actually look these objects up. def fqfind(namespace, name, table) namespace = namespace.downcase name = name.to_s.downcase if name =~ /^::/ or namespace == "" classname = name.sub(/^::/, '') unless table[classname] self.load(classname) end return table[classname] end ary = namespace.split("::") while ary.length > 0 newname = (ary + [name]).join("::").sub(/^::/, '') if obj = table[newname] or (self.load(newname) and obj = table[newname]) return obj end # Delete the second to last object, which reduces our namespace by one. ary.pop end # If we've gotten to this point without finding it, see if the name # exists at the top namespace if obj = table[name] or (self.load(name) and obj = table[name]) return obj end return nil end # Import our files. def import(file) if Puppet[:ignoreimport] return AST::ASTArray.new(:children => []) end # use a path relative to the file doing the importing if @lexer.file dir = @lexer.file.sub(%r{[^/]+$},'').sub(/\/$/, '') else dir = "." end if dir == "" dir = "." end result = ast AST::ASTArray # We can't interpolate at this point since we don't have any # scopes set up. Warn the user if they use a variable reference pat = file if pat.index("$") Puppet.warning( "The import of #{pat} contains a variable reference;" + " variables are not interpolated for imports " + "in file #{@lexer.file} at line #{@lexer.line}" ) end files = Puppet::Module::find_manifests(pat, :cwd => dir) if files.size == 0 raise Puppet::ImportError.new("No file(s) found for import " + "of '#{pat}'") end files.collect { |file| parser = Puppet::Parser::Parser.new(:astset => @astset, :environment => @environment) parser.files = self.files Puppet.debug("importing '%s'" % file) unless file =~ /^#{File::SEPARATOR}/ file = File.join(dir, file) end begin parser.file = file rescue Puppet::AlreadyImportedError # This file has already been imported to just move on next end # This will normally add code to the 'main' class. parser.parse } end def initialize(options = {}) @astset = options[:astset] || ASTSet.new({}, {}, {}) @environment = options[:environment] initvars() end # Initialize or reset all of our variables. def initvars @lexer = Puppet::Parser::Lexer.new() @files = [] @loaded = [] end # Try to load a class, since we could not find it. def load(classname) return false if classname == "" filename = classname.gsub("::", File::SEPARATOR) loaded = false # First try to load the top-level module mod = filename.scan(/^[\w-]+/).shift unless @loaded.include?(mod) @loaded << mod begin import(mod) Puppet.info "Autoloaded module %s" % mod loaded = true rescue Puppet::ImportError => detail # We couldn't load the module end end unless filename == mod and ! @loaded.include?(mod) @loaded << mod # Then the individual file begin import(filename) Puppet.info "Autoloaded file %s from module %s" % [filename, mod] loaded = true rescue Puppet::ImportError => detail # We couldn't load the file end end return loaded end # Split an fq name into a namespace and name def namesplit(fullname) ary = fullname.split("::") n = ary.pop || "" ns = ary.join("::") return ns, n end # Create a new class, or merge with an existing class. def newclass(name, options = {}) name = name.downcase if definitions.include?(name) raise Puppet::ParseError, "Cannot redefine class %s as a definition" % name end code = options[:code] parent = options[:parent] # If the class is already defined, then add code to it. if other = @astset.classes[name] # Make sure the parents match if parent and other.parentclass and (parent != other.parentclass) error("Class %s is already defined at %s:%s; cannot redefine" % [name, other.file, other.line]) end # This might be dangerous... if parent and ! other.parentclass other.parentclass = parent end # This might just be an empty, stub class. if code tmp = name if tmp == "" tmp = "main" end Puppet.debug addcontext("Adding code to %s" % tmp) # Else, add our code to it. if other.code and code other.code.children += code.children else other.code ||= code end end else # Define it anew. # Note we're doing something somewhat weird here -- we're setting # the class's namespace to its fully qualified name. This means # anything inside that class starts looking in that namespace first. args = {:namespace => name, :classname => name, :parser => self} args[:code] = code if code args[:parentclass] = parent if parent @astset.classes[name] = ast AST::HostClass, args end return @astset.classes[name] end # Create a new definition. def newdefine(name, options = {}) name = name.downcase if @astset.classes.include?(name) raise Puppet::ParseError, "Cannot redefine class %s as a definition" % name end # Make sure our definition doesn't already exist if other = @astset.definitions[name] error("%s is already defined at %s:%s; cannot redefine" % [name, other.file, other.line]) end ns, whatever = namesplit(name) args = { :namespace => ns, :arguments => options[:arguments], :code => options[:code], :parser => self, :classname => name } [:code, :arguments].each do |param| args[param] = options[param] if options[param] end @astset.definitions[name] = ast AST::Definition, args end # Create a new node. Nodes are special, because they're stored in a global # table, not according to namespaces. def newnode(names, options = {}) names = [names] unless names.instance_of?(Array) names.collect do |name| name = name.to_s.downcase if other = @astset.nodes[name] error("Node %s is already defined at %s:%s; cannot redefine" % [other.name, other.file, other.line]) end name = name.to_s if name.is_a?(Symbol) args = { :name => name, :parser => self } if options[:code] args[:code] = options[:code] end if options[:parent] args[:parentclass] = options[:parent] end @astset.nodes[name] = ast(AST::Node, args) @astset.nodes[name].classname = name @astset.nodes[name] end end def on_error(token,value,stack) if token == 0 # denotes end of file value = 'end of file' else value = "'%s'" % value end error = "Syntax error at %s" % [value] if brace = @lexer.expected error += "; expected '%s'" % brace end except = Puppet::ParseError.new(error) except.line = @lexer.line if @lexer.file except.file = @lexer.file end raise except end # how should I do error handling here? def parse(string = nil) if string self.string = string end begin main = yyparse(@lexer,:scan) rescue Racc::ParseError => except error = Puppet::ParseError.new(except) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error rescue Puppet::ParseError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::Error => except # and this is a framework error except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue Puppet::DevError => except except.line ||= @lexer.line except.file ||= @lexer.file raise except rescue => except error = Puppet::DevError.new(except.message) error.line = @lexer.line error.file = @lexer.file error.set_backtrace except.backtrace raise error end if main # Store the results as the top-level class. newclass("", :code => main) end @version = Time.now.to_i return @astset ensure @lexer.clear end # See if any of the files have changed. def reparse? if file = @files.detect { |file| file.changed? } return file.stamp else return false end end def string=(string) @lexer.string = string end # Add a new file to be checked when we're checking to see if we should be # reparsed. This is basically only used by the TemplateWrapper to let the # parser know about templates that should be parsed. def watch_file(*files) files.each do |file| unless file.is_a? Puppet::Util::LoadedFile file = Puppet::Util::LoadedFile.new(file) end @files << file end end end diff --git a/spec/unit/parser/lexer.rb b/spec/unit/parser/lexer.rb new file mode 100755 index 000000000..cddbef1ed --- /dev/null +++ b/spec/unit/parser/lexer.rb @@ -0,0 +1,465 @@ +#!/usr/bin/env ruby + +require File.dirname(__FILE__) + '/../../spec_helper' + +require 'puppet/parser/lexer' + +describe Puppet::Parser::Lexer::Token do + before do + @token = Puppet::Parser::Lexer::Token.new(%r{something}, :NAME) + end + + [:regex, :name, :string, :skip, :incr_line, :skip_text].each do |param| + it "should have a #{param.to_s} reader" do + @token.should respond_to?(param) + end + + it "should have a #{param.to_s} writer" do + @token.should respond_to?(param.to_s + "=") + end + end +end + +describe Puppet::Parser::Lexer::Token, "when initializing" do + it "should create a regex if the first argument is a string" do + Puppet::Parser::Lexer::Token.new("something", :NAME).regex.should == %r{something} + end + + it "should set the string if the first argument is one" do + Puppet::Parser::Lexer::Token.new("something", :NAME).string.should == "something" + end + + it "should set the regex if the first argument is one" do + Puppet::Parser::Lexer::Token.new(%r{something}, :NAME).regex.should == %r{something} + end +end + +describe Puppet::Parser::Lexer::TokenList do + before do + @list = Puppet::Parser::Lexer::TokenList.new + end + + it "should have a method for retrieving tokens by the name" do + token = @list.add_token :name, "whatever" + @list[:name].should equal(token) + end + + it "should have a method for retrieving string tokens by the string" do + token = @list.add_token :name, "whatever" + @list.lookup("whatever").should equal(token) + end + + it "should add tokens to the list when directed" do + token = @list.add_token :name, "whatever" + @list[:name].should equal(token) + end + + it "should have a method for adding multiple tokens at once" do + @list.add_tokens "whatever" => :name, "foo" => :bar + @list[:name].should_not be_nil + @list[:bar].should_not be_nil + end + + it "should fail to add tokens sharing a name with an existing token" do + @list.add_token :name, "whatever" + lambda { @list.add_token :name, "whatever" }.should raise_error(ArgumentError) + end + + it "should set provided options on tokens being added" do + token = @list.add_token :name, "whatever", :skip_text => true + token.skip_text.should == true + end + + it "should define any provided blocks as a :convert method" do + token = @list.add_token(:name, "whatever") do "foo" end + token.convert.should == "foo" + end + + it "should store all string tokens in the :string_tokens list" do + one = @list.add_token(:name, "1") + @list.string_tokens.should be_include(one) + end + + it "should store all regex tokens in the :regex_tokens list" do + one = @list.add_token(:name, %r{one}) + @list.regex_tokens.should be_include(one) + end + + it "should not store string tokens in the :regex_tokens list" do + one = @list.add_token(:name, "1") + @list.regex_tokens.should_not be_include(one) + end + + it "should not store regex tokens in the :string_tokens list" do + one = @list.add_token(:name, %r{one}) + @list.string_tokens.should_not be_include(one) + end + + it "should sort the string tokens inversely by length when asked" do + one = @list.add_token(:name, "1") + two = @list.add_token(:other, "12") + @list.sort_tokens + @list.string_tokens.should == [two, one] + end +end + +describe Puppet::Parser::Lexer::TOKENS do + before do + @lexer = Puppet::Parser::Lexer.new() + end + + { + :LBRACK => '[', + :RBRACK => ']', + :LBRACE => '{', + :RBRACE => '}', + :LPAREN => '(', + :RPAREN => ')', + :EQUALS => '=', + :ISEQUAL => '==', + :GREATEREQUAL => '>=', + :GREATERTHAN => '>', + :LESSTHAN => '<', + :LESSEQUAL => '<=', + :NOTEQUAL => '!=', + :NOT => '!', + :COMMA => ',', + :DOT => '.', + :COLON => ':', + :AT => '@', + :LLCOLLECT => '<<|', + :RRCOLLECT => '|>>', + :LCOLLECT => '<|', + :RCOLLECT => '|>', + :SEMIC => ';', + :QMARK => '?', + :BACKSLASH => '\\', + :FARROW => '=>', + :PARROW => '+>' + }.each do |name, string| + it "should have a token named #{name.to_s}" do + Puppet::Parser::Lexer::TOKENS[name].should_not be_nil + end + + it "should match '#{string}' for the token #{name.to_s}" do + Puppet::Parser::Lexer::TOKENS[name].string.should == string + end + end + + { + "case" => :CASE, + "class" => :CLASS, + "default" => :DEFAULT, + "define" => :DEFINE, + "import" => :IMPORT, + "if" => :IF, + "elsif" => :ELSIF, + "else" => :ELSE, + "inherits" => :INHERITS, + "node" => :NODE, + "and" => :AND, + "or" => :OR, + "undef" => :UNDEF, + "false" => :FALSE, + "true" => :TRUE + }.each do |string, name| + it "should have a keyword named #{name.to_s}" do + Puppet::Parser::Lexer::KEYWORDS[name].should_not be_nil + end + + it "should have the keyword for #{name.to_s} set to #{string}" do + Puppet::Parser::Lexer::KEYWORDS[name].string.should == string + end + end + + # These tokens' strings don't matter, just that the tokens exist. + [:DQTEXT, :SQTEXT, :BOOLEAN, :NAME, :NUMBER, :COMMENT, :RETURN, :SQUOTE, :DQUOTE, :VARIABLE].each do |name| + it "should have a token named #{name.to_s}" do + Puppet::Parser::Lexer::TOKENS[name].should_not be_nil + end + end +end + +describe Puppet::Parser::Lexer::TOKENS[:NAME] do + before { @token = Puppet::Parser::Lexer::TOKENS[:NAME] } + + it "should match against lower-case alpha-numeric terms" do + @token.regex.should =~ "one-two" + end + + it "should return itself and the value if the matched term is not a keyword" do + Puppet::Parser::Lexer::KEYWORDS.expects(:lookup).returns(nil) + @token.convert(stub("lexer"), "myval").should == [Puppet::Parser::Lexer::TOKENS[:NAME], "myval"] + end + + it "should return the keyword token and the value if the matched term is a keyword" do + keyword = stub 'keyword', :name => :testing + Puppet::Parser::Lexer::KEYWORDS.expects(:lookup).returns(keyword) + @token.convert(stub("lexer"), "myval").should == [keyword, "myval"] + end + + it "should return the BOOLEAN token and 'true' if the matched term is the string 'true'" do + keyword = stub 'keyword', :name => :TRUE + Puppet::Parser::Lexer::KEYWORDS.expects(:lookup).returns(keyword) + @token.convert(stub('lexer'), "true").should == [Puppet::Parser::Lexer::TOKENS[:BOOLEAN], true] + end + + it "should return the BOOLEAN token and 'false' if the matched term is the string 'false'" do + keyword = stub 'keyword', :name => :FALSE + Puppet::Parser::Lexer::KEYWORDS.expects(:lookup).returns(keyword) + @token.convert(stub('lexer'), "false").should == [Puppet::Parser::Lexer::TOKENS[:BOOLEAN], false] + end +end + +describe Puppet::Parser::Lexer::TOKENS[:NUMBER] do + before { @token = Puppet::Parser::Lexer::TOKENS[:NUMBER] } + + it "should match against numeric terms" do + @token.regex.should =~ "2982383139" + end + + it "should return the NAME token and the value" do + @token.convert(stub("lexer"), "myval").should == [Puppet::Parser::Lexer::TOKENS[:NAME], "myval"] + end +end + +describe Puppet::Parser::Lexer::TOKENS[:COMMENT] do + before { @token = Puppet::Parser::Lexer::TOKENS[:COMMENT] } + + it "should match against lines starting with '#'" do + @token.regex.should =~ "# this is a comment" + end + + it "should be marked to get skipped" do + @token.skip?.should be_true + end +end + +describe Puppet::Parser::Lexer::TOKENS[:RETURN] do + before { @token = Puppet::Parser::Lexer::TOKENS[:RETURN] } + + it "should match against carriage returns" do + @token.regex.should =~ "\n" + end + + it "should be marked to initiate text skipping" do + @token.skip_text.should be_true + end + + it "should be marked to increment the line" do + @token.incr_line.should be_true + end +end + +describe Puppet::Parser::Lexer::TOKENS[:SQUOTE] do + before { @token = Puppet::Parser::Lexer::TOKENS[:SQUOTE] } + + it "should match against single quotes" do + @token.regex.should =~ "'" + end + + it "should slurp the rest of the quoted string" do + lexer = stub("lexer") + lexer.expects(:slurpstring).with("myval").returns("otherval") + @token.convert(lexer, "myval") + end + + it "should return the SQTEXT token with the slurped string" do + lexer = stub("lexer") + lexer.stubs(:slurpstring).with("myval").returns("otherval") + @token.convert(lexer, "myval").should == [Puppet::Parser::Lexer::TOKENS[:SQTEXT], "otherval"] + end +end + +describe Puppet::Parser::Lexer::TOKENS[:DQUOTE] do + before { @token = Puppet::Parser::Lexer::TOKENS[:DQUOTE] } + + it "should match against single quotes" do + @token.regex.should =~ '"' + end + + it "should slurp the rest of the quoted string" do + lexer = stub("lexer") + lexer.expects(:slurpstring).with("myval").returns("otherval") + @token.convert(lexer, "myval") + end + + it "should return the DQTEXT token with the slurped string" do + lexer = stub("lexer") + lexer.stubs(:slurpstring).with("myval").returns("otherval") + @token.convert(lexer, "myval").should == [Puppet::Parser::Lexer::TOKENS[:DQTEXT], "otherval"] + end +end + +describe Puppet::Parser::Lexer::TOKENS[:VARIABLE] do + before { @token = Puppet::Parser::Lexer::TOKENS[:VARIABLE] } + + it "should match against alpha words prefixed with '$'" do + @token.regex.should =~ '$this_var' + end + + it "should return the VARIABLE token and the variable name stripped of the '$'" do + @token.convert(stub("lexer"), "$myval").should == [Puppet::Parser::Lexer::TOKENS[:VARIABLE], "myval"] + end +end + +# FIXME: We need to rewrite all of these tests, but I just don't want to take the time right now. +describe "Puppet::Parser::Lexer in the old tests" do + before { @lexer = Puppet::Parser::Lexer.new } + + it "should do simple lexing" do + strings = { +%q{\\} => [[:BACKSLASH,"\\"],[false,false]], +%q{simplest scanner test} => [[:NAME,"simplest"],[:NAME,"scanner"],[:NAME,"test"],[false,false]], +%q{returned scanner test +} => [[:NAME,"returned"],[:NAME,"scanner"],[:NAME,"test"],[false,false]] + } + strings.each { |str,ary| + @lexer.string = str + @lexer.fullscan().should == ary + } + end + + it "should correctly lex quoted strings" do + strings = { +%q{a simple "scanner" test +} => [[:NAME,"a"],[:NAME,"simple"],[:DQTEXT,"scanner"],[:NAME,"test"],[false,false]], +%q{a simple 'single quote scanner' test +} => [[:NAME,"a"],[:NAME,"simple"],[:SQTEXT,"single quote scanner"],[:NAME,"test"],[false,false]], +%q{a harder 'a $b \c"' +} => [[:NAME,"a"],[:NAME,"harder"],[:SQTEXT,'a $b \c"'],[false,false]], +%q{a harder "scanner test" +} => [[:NAME,"a"],[:NAME,"harder"],[:DQTEXT,"scanner test"],[false,false]], +%q{a hardest "scanner \"test\"" +} => [[:NAME,"a"],[:NAME,"hardest"],[:DQTEXT,'scanner "test"'],[false,false]], +%q{a hardestest "scanner \"test\" +" +} => [[:NAME,"a"],[:NAME,"hardestest"],[:DQTEXT,'scanner "test" +'],[false,false]], +%q{function("call")} => [[:NAME,"function"],[:LPAREN,"("],[:DQTEXT,'call'],[:RPAREN,")"],[false,false]] +} + strings.each { |str,array| + @lexer.string = str + @lexer.fullscan().should == array + } + end + + it "should fail usefully" do + strings = %w{ + ^ + } + strings.each { |str| + @lexer.string = str + lambda { @lexer.fullscan() }.should raise_error(RuntimeError) + } + end + + it "should fail if the string is not set" do + lambda { @lexer.fullscan() }.should raise_error(Puppet::LexError) + end + + it "should correctly identify keywords" do + @lexer.string = "case" + @lexer.fullscan.should == [[:CASE, "case"], [false, false]] + end + + it "should correctly match strings" do + names = %w{this is a bunch of names} + types = %w{Many Different Words A Word} + words = %w{differently Cased words A a} + + names.each { |t| + @lexer.string = t + @lexer.fullscan.should == [[:NAME,t],[false,false]] + } + types.each { |t| + @lexer.string = t + @lexer.fullscan.should == [[:CLASSREF,t],[false,false]] + } + end + + it "should correctly parse empty strings" do + bit = '$var = ""' + + @lexer.string = bit + + lambda { @lexer.fullscan }.should_not raise_error + end + + it "should correctly parse virtual resources" do + string = "@type {" + + @lexer.string = string + + @lexer.fullscan.should == [[:AT, "@"], [:NAME, "type"], [:LBRACE, "{"], [false,false]] + end + + it "should correctly deal with namespaces" do + @lexer.string = %{class myclass} + + @lexer.fullscan + + @lexer.namespace.should == "myclass" + + @lexer.namepop + + @lexer.namespace.should == "" + + @lexer.string = "class base { class sub { class more" + + @lexer.fullscan + + @lexer.namespace.should == "base::sub::more" + + @lexer.namepop + + @lexer.namespace.should == "base::sub" + end + + it "should correctly handle fully qualified names" do + @lexer.string = "class base { class sub::more {" + + @lexer.fullscan + + @lexer.namespace.should == "base::sub::more" + + @lexer.namepop + + @lexer.namespace.should == "base" + end + + it "should correctly lex variables" do + ["$variable", "$::variable", "$qualified::variable", "$further::qualified::variable"].each do |string| + @lexer.string = string + + @lexer.scan do |t, s| + t.should == :VARIABLE + string.sub(/^\$/, '').should == s + break + end + end + end + + # #774 + it "should correctly parse the CLASSREF token" do + string = ["Foo", "::Foo","Foo::Bar","::Foo::Bar"] + + string.each do |foo| + @lexer.string = foo + @lexer.fullscan[0].should == [:CLASSREF, foo] + end + end +end + +require 'puppettest/support/utils' +describe "Puppet::Parser::Lexer in the old tests when lexing example files" do + extend PuppetTest + extend PuppetTest::Support::Utils + textfiles() do |file| + it "should correctly lex #{file}" do + lexer = Puppet::Parser::Lexer.new() + lexer.file = file + lambda { lexer.fullscan() }.should_not raise_error + end + end +end diff --git a/test/language/lexer.rb b/test/language/lexer.rb deleted file mode 100755 index e09828d51..000000000 --- a/test/language/lexer.rb +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env ruby - -require File.dirname(__FILE__) + '/../lib/puppettest' - -require 'puppet' -require 'puppet/parser/lexer' -require 'puppettest' - -#%q{service("telnet") = \{ -# port => "23", -# protocol => "tcp", -# name => "telnet", -#\} -#} => [[:NAME, "service"], [:LPAREN, "("], [:DQUOTE, "\""], [:NAME, "telnet"], [:DQUOTE, "\""], [:RPAREN, ")"], [:EQUALS, "="], [:lbrace, "{"], [:NAME, "port"], [:FARROW, "=>"], [:DQUOTE, "\""], [:NAME, "23"], [:DQUOTE, "\""], [:COMMA, ","], [:NAME, "protocol"], [:FARROW, "=>"], [:DQUOTE, "\""], [:NAME, "tcp"], [:DQUOTE, "\""], [:COMMA, ","], [:NAME, "name"], [:FARROW, "=>"], [:DQUOTE, "\""], [:NAME, "telnet"], [:DQUOTE, "\""], [:COMMA, ","], [:RBRACE, "}"]] - -class TestLexer < Test::Unit::TestCase - include PuppetTest - def setup - super - mklexer - end - - def mklexer - @lexer = Puppet::Parser::Lexer.new() - end - - def test_simple_lex - strings = { -%q{\\} => [[:BACKSLASH,"\\"],[false,false]], -%q{simplest scanner test} => [[:NAME,"simplest"],[:NAME,"scanner"],[:NAME,"test"],[false,false]], -%q{returned scanner test -} => [[:NAME,"returned"],[:NAME,"scanner"],[:NAME,"test"],[false,false]] - } - strings.each { |str,ary| - @lexer.string = str - assert_equal( - ary, - @lexer.fullscan() - ) - } - end - - def test_quoted_strings - strings = { -%q{a simple "scanner" test -} => [[:NAME,"a"],[:NAME,"simple"],[:DQTEXT,"scanner"],[:NAME,"test"],[false,false]], -%q{a simple 'single quote scanner' test -} => [[:NAME,"a"],[:NAME,"simple"],[:SQTEXT,"single quote scanner"],[:NAME,"test"],[false,false]], -%q{a harder 'a $b \c"' -} => [[:NAME,"a"],[:NAME,"harder"],[:SQTEXT,'a $b \c"'],[false,false]], -%q{a harder "scanner test" -} => [[:NAME,"a"],[:NAME,"harder"],[:DQTEXT,"scanner test"],[false,false]], -%q{a hardest "scanner \"test\"" -} => [[:NAME,"a"],[:NAME,"hardest"],[:DQTEXT,'scanner "test"'],[false,false]], -%q{a hardestest "scanner \"test\" -" -} => [[:NAME,"a"],[:NAME,"hardestest"],[:DQTEXT,'scanner "test" -'],[false,false]], -%q{function("call")} => [[:NAME,"function"],[:LPAREN,"("],[:DQTEXT,'call'],[:RPAREN,")"],[false,false]] -} - strings.each { |str,array| - @lexer.string = str - assert_equal( - array, - @lexer.fullscan() - ) - } - end - - def test_errors - strings = %w{ - ^ - } - strings.each { |str| - @lexer.string = str - assert_raise(RuntimeError) { - @lexer.fullscan() - } - } - end - - def test_more_error - assert_raise(TypeError) { - @lexer.fullscan() - } - end - - def test_files - textfiles() { |file| - lexer = Puppet::Parser::Lexer.new() - lexer.file = file - assert_nothing_raised("Failed to lex %s" % file) { - lexer.fullscan() - } - Puppet::Type.allclear - } - end - - def test_strings - names = %w{this is a bunch of names} - types = %w{Many Different Words A Word} - words = %w{differently Cased words A a} - - names.each { |t| - @lexer.string = t - assert_equal( - [[:NAME,t],[false,false]], - @lexer.fullscan - ) - } - types.each { |t| - @lexer.string = t - assert_equal( - [[:CLASSREF,t],[false,false]], - @lexer.fullscan - ) - } - end - - def test_emptystring - bit = '$var = ""' - - assert_nothing_raised { - @lexer.string = bit - } - - assert_nothing_raised { - @lexer.fullscan - } - end - - def test_collectlexing - {"@" => :AT, "<|" => :LCOLLECT, "|>" => :RCOLLECT}.each do |string, token| - assert_nothing_raised { - @lexer.string = string - } - - ret = nil - assert_nothing_raised { - ret = @lexer.fullscan - } - - assert_equal([[token, string],[false, false]], ret) - end - end - - def test_collectabletype - string = "@type {" - - assert_nothing_raised { - @lexer.string = string - } - - ret = nil - assert_nothing_raised { - ret = @lexer.fullscan - } - - assert_equal([[:AT, "@"], [:NAME, "type"], [:LBRACE, "{"], [false,false]],ret) - end - - def test_namespace - @lexer.string = %{class myclass} - - assert_nothing_raised { - @lexer.fullscan - } - - assert_equal("myclass", @lexer.namespace) - - assert_nothing_raised do - @lexer.namepop - end - - assert_equal("", @lexer.namespace) - - @lexer.string = "class base { class sub { class more" - - assert_nothing_raised { - @lexer.fullscan - } - - assert_equal("base::sub::more", @lexer.namespace) - - assert_nothing_raised do - @lexer.namepop - end - - assert_equal("base::sub", @lexer.namespace) - - # Now try it with some fq names - mklexer - - @lexer.string = "class base { class sub::more {" - - assert_nothing_raised { - @lexer.fullscan - } - - assert_equal("base::sub::more", @lexer.namespace) - - assert_nothing_raised do - @lexer.namepop - end - - assert_equal("base", @lexer.namespace) - end - - def test_indefine - @lexer.string = %{define me} - - assert_nothing_raised { - @lexer.scan { |t,s| } - } - - assert(@lexer.indefine?, "Lexer not considered in define") - - # Now make sure we throw an error when trying to nest defines. - assert_raise(Puppet::ParseError) do - @lexer.string = %{define another} - @lexer.scan { |t,s| } - end - - assert_nothing_raised do - @lexer.indefine = false - end - - assert(! @lexer.indefine?, "Lexer still considered in define") - end - - # Make sure the different qualified variables work. - def test_variable - ["$variable", "$::variable", "$qualified::variable", "$further::qualified::variable"].each do |string| - @lexer.string = string - - assert_nothing_raised("Could not lex %s" % string) do - @lexer.scan do |t, s| - assert_equal(:VARIABLE, t, "did not get variable as token") - assert_equal(string.sub(/^\$/, ''), s, "did not get correct string back") - break - end - end - end - end - - # Make sure the expected stack works as it should - def test_expected - @lexer.string = "[a{" - expected = @lexer.instance_variable_get("@expected") - @lexer.scan {} - assert_equal("}", @lexer.expected, "expected value is wrong") - - @lexer.string = "}" - @lexer.scan {} - assert_equal("]", @lexer.expected, "expected value is wrong after pop") - end - - # #774 - def test_classref_token - string = ["Foo", "::Foo","Foo::Bar","::Foo::Bar"] - - string.each do |foo| - assert_nothing_raised { - @lexer.string = foo - } - - ret = nil - assert_nothing_raised { - ret = @lexer.fullscan - } - - assert_equal([:CLASSREF, foo],ret[0], "Did not correctly tokenize '%s'" % foo) - end - end -end -