diff --git a/lib/puppet/pops/parser/lexer2.rb b/lib/puppet/pops/parser/lexer2.rb index 2193ce8fb..6f352931a 100644 --- a/lib/puppet/pops/parser/lexer2.rb +++ b/lib/puppet/pops/parser/lexer2.rb @@ -1,694 +1,694 @@ # The Lexer is responsbile for turning source text into tokens. # This version is a performance enhanced lexer (in comparison to the 3.x and earlier "future parser" lexer. # # Old returns tokens [:KEY, value, { locator = } # Could return [[token], locator] # or Token.new([token], locator) with the same API x[0] = token_symbol, x[1] = self, x[:key] = (:value, :file, :line, :pos) etc require 'strscan' require 'puppet/pops/parser/lexer_support' require 'puppet/pops/parser/heredoc_support' require 'puppet/pops/parser/interpolation_support' require 'puppet/pops/parser/epp_support' require 'puppet/pops/parser/slurp_support' class Puppet::Pops::Parser::Lexer2 include Puppet::Pops::Parser::LexerSupport include Puppet::Pops::Parser::HeredocSupport include Puppet::Pops::Parser::InterpolationSupport include Puppet::Pops::Parser::SlurpSupport include Puppet::Pops::Parser::EppSupport # ALl tokens have three slots, the token name (a Symbol), the token text (String), and a token text length. # All operator and punctuation tokens reuse singleton arrays Tokens that require unique values create # a unique array per token. # # PEFORMANCE NOTES: # This construct reduces the amount of object that needs to be created for operators and punctuation. # The length is pre-calculated for all singleton tokens. The length is used both to signal the length of # the token, and to advance the scanner position (without having to advance it with a scan(regexp)). # TOKEN_LBRACK = [:LBRACK, '['.freeze, 1].freeze TOKEN_LISTSTART = [:LISTSTART, '['.freeze, 1].freeze TOKEN_RBRACK = [:RBRACK, ']'.freeze, 1].freeze TOKEN_LBRACE = [:LBRACE, '{'.freeze, 1].freeze TOKEN_RBRACE = [:RBRACE, '}'.freeze, 1].freeze TOKEN_SELBRACE = [:SELBRACE, '{'.freeze, 1].freeze TOKEN_LPAREN = [:LPAREN, '('.freeze, 1].freeze TOKEN_RPAREN = [:RPAREN, ')'.freeze, 1].freeze TOKEN_EQUALS = [:EQUALS, '='.freeze, 1].freeze TOKEN_APPENDS = [:APPENDS, '+='.freeze, 2].freeze TOKEN_DELETES = [:DELETES, '-='.freeze, 2].freeze TOKEN_ISEQUAL = [:ISEQUAL, '=='.freeze, 2].freeze TOKEN_NOTEQUAL = [:NOTEQUAL, '!='.freeze, 2].freeze TOKEN_MATCH = [:MATCH, '=~'.freeze, 2].freeze TOKEN_NOMATCH = [:NOMATCH, '!~'.freeze, 2].freeze TOKEN_GREATEREQUAL = [:GREATEREQUAL, '>='.freeze, 2].freeze TOKEN_GREATERTHAN = [:GREATERTHAN, '>'.freeze, 1].freeze TOKEN_LESSEQUAL = [:LESSEQUAL, '<='.freeze, 2].freeze TOKEN_LESSTHAN = [:LESSTHAN, '<'.freeze, 1].freeze TOKEN_FARROW = [:FARROW, '=>'.freeze, 2].freeze TOKEN_PARROW = [:PARROW, '+>'.freeze, 2].freeze TOKEN_LSHIFT = [:LSHIFT, '<<'.freeze, 2].freeze TOKEN_LLCOLLECT = [:LLCOLLECT, '<<|'.freeze, 3].freeze TOKEN_LCOLLECT = [:LCOLLECT, '<|'.freeze, 2].freeze TOKEN_RSHIFT = [:RSHIFT, '>>'.freeze, 2].freeze TOKEN_RRCOLLECT = [:RRCOLLECT, '|>>'.freeze, 3].freeze TOKEN_RCOLLECT = [:RCOLLECT, '|>'.freeze, 2].freeze TOKEN_PLUS = [:PLUS, '+'.freeze, 1].freeze TOKEN_MINUS = [:MINUS, '-'.freeze, 1].freeze TOKEN_DIV = [:DIV, '/'.freeze, 1].freeze TOKEN_TIMES = [:TIMES, '*'.freeze, 1].freeze TOKEN_MODULO = [:MODULO, '%'.freeze, 1].freeze TOKEN_NOT = [:NOT, '!'.freeze, 1].freeze TOKEN_DOT = [:DOT, '.'.freeze, 1].freeze TOKEN_PIPE = [:PIPE, '|'.freeze, 1].freeze TOKEN_AT = [:AT , '@'.freeze, 1].freeze TOKEN_ATAT = [:ATAT , '@@'.freeze, 2].freeze TOKEN_COLON = [:COLON, ':'.freeze, 1].freeze TOKEN_COMMA = [:COMMA, ','.freeze, 1].freeze TOKEN_SEMIC = [:SEMIC, ';'.freeze, 1].freeze TOKEN_QMARK = [:QMARK, '?'.freeze, 1].freeze TOKEN_TILDE = [:TILDE, '~'.freeze, 1].freeze # lexed but not an operator in Puppet TOKEN_REGEXP = [:REGEXP, nil, 0].freeze TOKEN_IN_EDGE = [:IN_EDGE, '->'.freeze, 2].freeze TOKEN_IN_EDGE_SUB = [:IN_EDGE_SUB, '~>'.freeze, 2].freeze TOKEN_OUT_EDGE = [:OUT_EDGE, '<-'.freeze, 2].freeze TOKEN_OUT_EDGE_SUB = [:OUT_EDGE_SUB, '<~'.freeze, 2].freeze # Tokens that are always unique to what has been lexed TOKEN_STRING = [:STRING, nil, 0].freeze TOKEN_WORD = [:WORD, nil, 0].freeze TOKEN_DQPRE = [:DQPRE, nil, 0].freeze TOKEN_DQMID = [:DQPRE, nil, 0].freeze TOKEN_DQPOS = [:DQPRE, nil, 0].freeze TOKEN_NUMBER = [:NUMBER, nil, 0].freeze TOKEN_VARIABLE = [:VARIABLE, nil, 1].freeze TOKEN_VARIABLE_EMPTY = [:VARIABLE, ''.freeze, 1].freeze # HEREDOC has syntax as an argument. TOKEN_HEREDOC = [:HEREDOC, nil, 0].freeze # EPP_START is currently a marker token, may later get syntax TOKEN_EPPSTART = [:EPP_START, nil, 0].freeze TOKEN_EPPEND = [:EPP_END, '%>', 2].freeze TOKEN_EPPEND_TRIM = [:EPP_END_TRIM, '-%>', 3].freeze # This is used for unrecognized tokens, will always be a single character. This particular instance # is not used, but is kept here for documentation purposes. TOKEN_OTHER = [:OTHER, nil, 0] # Keywords are all singleton tokens with pre calculated lengths. # Booleans are pre-calculated (rather than evaluating the strings "false" "true" repeatedly. # KEYWORDS = { "case" => [:CASE, 'case', 4], "class" => [:CLASS, 'class', 5], "default" => [:DEFAULT, 'default', 7], "define" => [:DEFINE, 'define', 6], "if" => [:IF, 'if', 2], "elsif" => [:ELSIF, 'elsif', 5], "else" => [:ELSE, 'else', 4], "inherits" => [:INHERITS, 'inherits', 8], "node" => [:NODE, 'node', 4], "and" => [:AND, 'and', 3], "or" => [:OR, 'or', 2], "undef" => [:UNDEF, 'undef', 5], "false" => [:BOOLEAN, false, 5], "true" => [:BOOLEAN, true, 4], "in" => [:IN, 'in', 2], "unless" => [:UNLESS, 'unless', 6], "function" => [:FUNCTION, 'function', 8], "type" => [:TYPE, 'type', 4], "attr" => [:ATTR, 'attr', 4], "private" => [:PRIVATE, 'private', 7], } KEYWORDS.each {|k,v| v[1].freeze; v.freeze } KEYWORDS.freeze # Reverse lookup of keyword name to string KEYWORD_NAMES = {} KEYWORDS.each {|k, v| KEYWORD_NAMES[v[0]] = k } KEYWORD_NAMES.freeze PATTERN_WS = %r{[[:blank:]\r]+} # The single line comment includes the line ending. PATTERN_COMMENT = %r{#.*\r?} PATTERN_MLCOMMENT = %r{/\*(.*?)\*/}m PATTERN_REGEX = %r{/[^/\n]*/} PATTERN_REGEX_END = %r{/} PATTERN_REGEX_A = %r{\A/} # for replacement to "" PATTERN_REGEX_Z = %r{/\Z} # for replacement to "" PATTERN_REGEX_ESC = %r{\\/} # for replacement to "/" # The 3x patterns: # PATTERN_CLASSREF = %r{((::){0,1}[A-Z][-\w]*)+} # PATTERN_NAME = %r{((::)?[a-z0-9][-\w]*)(::[a-z0-9][-\w]*)*} # The NAME and CLASSREF in 4x are strict. Each segment must start with # a letter a-z and may not contain dashes (\w includes letters, digits and _). # PATTERN_CLASSREF = %r{((::){0,1}[A-Z][\w]*)+} PATTERN_NAME = %r{^((::)?[a-z][\w]*)(::[a-z][\w]*)*$} PATTERN_BARE_WORD = %r{((?:::){0,1}(?:[a-z_](?:[\w-]*[\w])?))+} PATTERN_DOLLAR_VAR = %r{\$(::)?(\w+::)*\w+} PATTERN_NUMBER = %r{\b(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b} # PERFORMANCE NOTE: # Comparison against a frozen string is faster (than unfrozen). # STRING_BSLASH_BSLASH = '\\'.freeze attr_reader :locator def initialize() end # Clears the lexer state (it is not required to call this as it will be garbage collected # and the next lex call (lex_string, lex_file) will reset the internal state. # def clear() # not really needed, but if someone wants to ensure garbage is collected as early as possible @scanner = nil @locator = nil @lexing_context = nil end # Convenience method, and for compatibility with older lexer. Use the lex_string instead which allows # passing the path to use without first having to call file= (which reads the file if it exists). # (Bad form to use overloading of assignment operator for something that is not really an assignment. Also, # overloading of = does not allow passing more than one argument). # def string=(string) lex_string(string, '') end def lex_string(string, path='') initvars @scanner = StringScanner.new(string) @locator = Puppet::Pops::Parser::Locator.locator(string, path) end # Lexes an unquoted string. # @param string [String] the string to lex # @param locator [Puppet::Pops::Parser::Locator] the locator to use (a default is used if nil is given) # @param escapes [Array] array of character strings representing the escape sequences to transform # @param interpolate [Boolean] whether interpolation of expressions should be made or not. # def lex_unquoted_string(string, locator, escapes, interpolate) initvars @scanner = StringScanner.new(string) @locator = locator || Puppet::Pops::Parser::Locator.locator(string, '') @lexing_context[:escapes] = escapes || UQ_ESCAPES - @lexing_context[:uq_slurp_pattern] = (interpolate || !escapes.empty?) ? SLURP_UQ_PATTERN : SLURP_ALL_PATTERN + @lexing_context[:uq_slurp_pattern] = interpolate ? (escapes.include?('$') ? SLURP_UQ_PATTERN : SLURP_UQNE_PATTERN) : SLURP_ALL_PATTERN end # Convenience method, and for compatibility with older lexer. Use the lex_file instead. # (Bad form to use overloading of assignment operator for something that is not really an assignment). # def file=(file) lex_file(file) end # TODO: This method should not be used, callers should get the locator since it is most likely required to # compute line, position etc given offsets. # def file @locator ? @locator.file : nil end # Initializes lexing of the content of the given file. An empty string is used if the file does not exist. # def lex_file(file) initvars contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file) : "" @scanner = StringScanner.new(contents.freeze) @locator = Puppet::Pops::Parser::Locator.locator(contents, file) end def initvars @token_queue = [] # NOTE: additional keys are used; :escapes, :uq_slurp_pattern, :newline_jump, :epp_* @lexing_context = { :brace_count => 0, :after => nil, } end # Scans all of the content and returns it in an array # Note that the terminating [false, false] token is included in the result. # def fullscan result = [] scan {|token, value| result.push([token, value]) } result end # A block must be passed to scan. It will be called with two arguments, a symbol for the token, # and an instance of LexerSupport::TokenValue # PERFORMANCE NOTE: The TokenValue is designed to reduce the amount of garbage / temporary data # and to only convert the lexer's internal tokens on demand. It is slightly more costly to create an # instance of a class defined in Ruby than an Array or Hash, but the gain is much bigger since transformation # logic is avoided for many of its members (most are never used (e.g. line/pos information which is only of # value in general for error messages, and for some expressions (which the lexer does not know about). # def scan # PERFORMANCE note: it is faster to access local variables than instance variables. # This makes a small but notable difference since instance member access is avoided for # every token in the lexed content. # scn = @scanner ctx = @lexing_context queue = @token_queue lex_error_without_pos(Puppet::Pops::Issues::NO_INPUT_TO_LEXER) unless scn scn.skip(PATTERN_WS) # This is the lexer's main loop until queue.empty? && scn.eos? do if token = queue.shift || lex_token ctx[:after] = token[0] yield token end end # Signals end of input yield [false, false] end # This lexes one token at the current position of the scanner. # PERFORMANCE NOTE: Any change to this logic should be performance measured. # def lex_token # Using three char look ahead (may be faster to do 2 char look ahead since only 2 tokens require a third scn = @scanner ctx = @lexing_context before = @scanner.pos # A look ahead of 3 characters is used since the longest operator ambiguity is resolved at that point. # PERFORMANCE NOTE: It is faster to peek once and use three separate variables for lookahead 0, 1 and 2. # la = scn.peek(3) return nil if la.empty? # Ruby 1.8.7 requires using offset and length (or integers are returned. # PERFORMANCE NOTE. # It is slightly faster to use these local variables than accessing la[0], la[1] etc. in ruby 1.9.3 # But not big enough to warrant two completely different implementations. # la0 = la[0,1] la1 = la[1,1] la2 = la[2,1] # PERFORMANCE NOTE: # A case when, where all the cases are literal values is the fastest way to map from data to code. # It is much faster than using a hash with lambdas, hash with symbol used to then invoke send etc. # This case statement is evaluated for most character positions in puppet source, and great care must # be taken to not introduce performance regressions. # case la0 when '.' emit(TOKEN_DOT, before) when ',' emit(TOKEN_COMMA, before) when '[' if (before == 0 || scn.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/) emit(TOKEN_LISTSTART, before) else emit(TOKEN_LBRACK, before) end when ']' emit(TOKEN_RBRACK, before) when '(' emit(TOKEN_LPAREN, before) when ')' emit(TOKEN_RPAREN, before) when ';' emit(TOKEN_SEMIC, before) when '?' emit(TOKEN_QMARK, before) when '*' emit(TOKEN_TIMES, before) when '%' if la1 == '>' && ctx[:epp_mode] scn.pos += 2 if ctx[:epp_mode] == :expr enqueue_completed(TOKEN_EPPEND, before) end ctx[:epp_mode] = :text interpolate_epp else emit(TOKEN_MODULO, before) end when '{' # The lexer needs to help the parser since the technology used cannot deal with # lookahead of same token with different precedence. This is solved by making left brace # after ? into a separate token. # ctx[:brace_count] += 1 emit(if ctx[:after] == :QMARK TOKEN_SELBRACE else TOKEN_LBRACE end, before) when '}' ctx[:brace_count] -= 1 emit(TOKEN_RBRACE, before) # TOKENS @, @@, @( when '@' case la1 when '@' emit(TOKEN_ATAT, before) # TODO; Check if this is good for the grammar when '(' heredoc else emit(TOKEN_AT, before) end # TOKENS |, |>, |>> when '|' emit(case la1 when '>' la2 == '>' ? TOKEN_RRCOLLECT : TOKEN_RCOLLECT else TOKEN_PIPE end, before) # TOKENS =, =>, ==, =~ when '=' emit(case la1 when '=' TOKEN_ISEQUAL when '>' TOKEN_FARROW when '~' TOKEN_MATCH else TOKEN_EQUALS end, before) # TOKENS '+', '+=', and '+>' when '+' emit(case la1 when '=' TOKEN_APPENDS when '>' TOKEN_PARROW else TOKEN_PLUS end, before) # TOKENS '-', '->', and epp '-%>' (end of interpolation with trim) when '-' if ctx[:epp_mode] && la1 == '%' && la2 == '>' scn.pos += 3 if ctx[:epp_mode] == :expr enqueue_completed(TOKEN_EPPEND_TRIM, before) end interpolate_epp(:with_trim) else emit(case la1 when '>' TOKEN_IN_EDGE when '=' TOKEN_DELETES else TOKEN_MINUS end, before) end # TOKENS !, !=, !~ when '!' emit(case la1 when '=' TOKEN_NOTEQUAL when '~' TOKEN_NOMATCH else TOKEN_NOT end, before) # TOKENS ~>, ~ when '~' emit(la1 == '>' ? TOKEN_IN_EDGE_SUB : TOKEN_TILDE, before) when '#' scn.skip(PATTERN_COMMENT) nil # TOKENS '/', '/*' and '/ regexp /' when '/' case la1 when '*' scn.skip(PATTERN_MLCOMMENT) nil else # regexp position is a regexp, else a div if regexp_acceptable? && value = scn.scan(PATTERN_REGEX) # Ensure an escaped / was not matched while value[-2..-2] == STRING_BSLASH_BSLASH # i.e. \\ value += scn.scan_until(PATTERN_REGEX_END) end regex = value.sub(PATTERN_REGEX_A, '').sub(PATTERN_REGEX_Z, '').gsub(PATTERN_REGEX_ESC, '/') emit_completed([:REGEX, Regexp.new(regex), scn.pos-before], before) else emit(TOKEN_DIV, before) end end # TOKENS <, <=, <|, <<|, <<, <-, <~ when '<' emit(case la1 when '<' if la2 == '|' TOKEN_LLCOLLECT else TOKEN_LSHIFT end when '=' TOKEN_LESSEQUAL when '|' TOKEN_LCOLLECT when '-' TOKEN_OUT_EDGE when '~' TOKEN_OUT_EDGE_SUB else TOKEN_LESSTHAN end, before) # TOKENS >, >=, >> when '>' emit(case la1 when '>' TOKEN_RSHIFT when '=' TOKEN_GREATEREQUAL else TOKEN_GREATERTHAN end, before) # TOKENS :, ::CLASSREF, ::NAME when ':' if la1 == ':' before = scn.pos # PERFORMANCE NOTE: This could potentially be speeded up by using a case/when listing all # upper case letters. Alternatively, the 'A', and 'Z' comparisons may be faster if they are # frozen. # if la2 >= 'A' && la2 <= 'Z' # CLASSREF or error value = scn.scan(PATTERN_CLASSREF) if value after = scn.pos emit_completed([:CLASSREF, value.freeze, after-before], before) else # move to faulty position ('::' was ok) scn.pos = scn.pos + 3 lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_CLASS_REFERENCE) end else value = scn.scan(PATTERN_BARE_WORD) if value if value =~ PATTERN_NAME emit_completed([:NAME, value.freeze, scn.pos-before], before) else emit_completed([:WORD, value.freeze, scn.pos - before], before) end else # move to faulty position ('::' was ok) scn.pos = scn.pos + 2 lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME) end end else emit(TOKEN_COLON, before) end when '$' if value = scn.scan(PATTERN_DOLLAR_VAR) emit_completed([:VARIABLE, value[1..-1].freeze, scn.pos - before], before) else # consume the $ and let higher layer complain about the error instead of getting a syntax error emit(TOKEN_VARIABLE_EMPTY, before) end when '"' # Recursive string interpolation, 'interpolate' either returns a STRING token, or # a DQPRE with the rest of the string's tokens placed in the @token_queue interpolate_dq when "'" emit_completed([:STRING, slurp_sqstring.freeze, scn.pos - before], before) when '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' value = scn.scan(PATTERN_NUMBER) if value length = scn.pos - before assert_numeric(value, length) emit_completed([:NUMBER, value.freeze, length], before) else # move to faulty position ([0-9] was ok) scn.pos = scn.pos + 1 lex_error(Puppet::Pops::Issues::ILLEGAL_NUMBER) end when 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_' value = scn.scan(PATTERN_BARE_WORD) if value && value =~ PATTERN_NAME emit_completed(KEYWORDS[value] || [:NAME, value.freeze, scn.pos - before], before) elsif value emit_completed([:WORD, value.freeze, scn.pos - before], before) else # move to faulty position ([a-z_] was ok) scn.pos = scn.pos + 1 fully_qualified = scn.match?(/::/) if fully_qualified lex_error(Puppet::Pops::Issues::ILLEGAL_FULLY_QUALIFIED_NAME) else lex_error(Puppet::Pops::Issues::ILLEGAL_NAME_OR_BARE_WORD) end end when 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' value = scn.scan(PATTERN_CLASSREF) if value emit_completed([:CLASSREF, value.freeze, scn.pos - before], before) else # move to faulty position ([A-Z] was ok) scn.pos = scn.pos + 1 lex_error(Puppet::Pops::Issues::ILLEGAL_CLASS_REFERENCE) end when "\n" # If heredoc_cont is in effect there are heredoc text lines to skip over # otherwise just skip the newline. # if ctx[:newline_jump] scn.pos = ctx[:newline_jump] ctx[:newline_jump] = nil else scn.pos += 1 end return nil when ' ', "\t", "\r" scn.skip(PATTERN_WS) return nil else # In case of unicode spaces of various kinds that are captured by a regexp, but not by the # simpler case expression above (not worth handling those special cases with better performance). if scn.skip(PATTERN_WS) nil else # "unrecognized char" emit([:OTHER, la0, 1], before) end end end # Emits (produces) a token [:tokensymbol, TokenValue] and moves the scanner's position past the token # def emit(token, byte_offset) @scanner.pos = byte_offset + token[2] [token[0], TokenValue.new(token, byte_offset, @locator)] end # Emits the completed token on the form [:tokensymbol, TokenValue. This method does not alter # the scanner's position. # def emit_completed(token, byte_offset) [token[0], TokenValue.new(token, byte_offset, @locator)] end # Enqueues a completed token at the given offset def enqueue_completed(token, byte_offset) @token_queue << emit_completed(token, byte_offset) end # Allows subprocessors for heredoc etc to enqueue tokens that are tokenized by a different lexer instance # def enqueue(emitted_token) @token_queue << emitted_token end # Answers after which tokens it is acceptable to lex a regular expression. # PERFORMANCE NOTE: # It may be beneficial to turn this into a hash with default value of true for missing entries. # A case expression with literal values will however create a hash internally. Since a reference is # always needed to the hash, this access is almost as costly as a method call. # def regexp_acceptable? case @lexing_context[:after] # Ends of (potential) R-value generating expressions when :RPAREN, :RBRACK, :RRCOLLECT, :RCOLLECT false # End of (potential) R-value - but must be allowed because of case expressions # Called out here to not be mistaken for a bug. when :RBRACE true # Operands (that can be followed by DIV (even if illegal in grammar) when :NAME, :CLASSREF, :NUMBER, :STRING, :BOOLEAN, :DQPRE, :DQMID, :DQPOST, :HEREDOC, :REGEX, :VARIABLE, :WORD false else true end end end diff --git a/lib/puppet/pops/parser/slurp_support.rb b/lib/puppet/pops/parser/slurp_support.rb index 64e79af24..c16441e90 100644 --- a/lib/puppet/pops/parser/slurp_support.rb +++ b/lib/puppet/pops/parser/slurp_support.rb @@ -1,108 +1,111 @@ # This module is an integral part of the Lexer. # It defines the string slurping behavior - finding the string and non string parts in interpolated # strings, translating escape sequences in strings to their single character equivalence. # # PERFORMANCE NOTE: The various kinds of slurping could be made even more generic, but requires # additional parameter passing and evaluation of conditional logic. # TODO: More detailed performance analysis of excessive character escaping and interpolation. # module Puppet::Pops::Parser::SlurpSupport include Puppet::Pops::Parser::LexerSupport SLURP_SQ_PATTERN = /(?:[^\\]|^|[^\\])(?:[\\]{2})*[']/ SLURP_DQ_PATTERN = /(?:[^\\]|^|[^\\])(?:[\\]{2})*(["]|[$]\{?)/ SLURP_UQ_PATTERN = /(?:[^\\]|^|[^\\])(?:[\\]{2})*([$]\{?|\z)/ + # unquoted, no escapes + SLURP_UQNE_PATTERN = /(\$\{?|\z)/m SLURP_ALL_PATTERN = /.*(\z)/m SQ_ESCAPES = %w{ \\ ' } DQ_ESCAPES = %w{ \\ $ ' " r n t s u}+["\r\n", "\n"] UQ_ESCAPES = %w{ \\ $ r n t s u}+["\r\n", "\n"] def slurp_sqstring # skip the leading ' @scanner.pos += 1 str = slurp(@scanner, SLURP_SQ_PATTERN, SQ_ESCAPES, :ignore_invalid_escapes) lex_error(Puppet::Pops::Issues::UNCLOSED_QUOTE, :after => "\"'\"", :followed_by => followed_by) unless str str[0..-2] # strip closing "'" from result end def slurp_dqstring scn = @scanner last = scn.matched str = slurp(scn, SLURP_DQ_PATTERN, DQ_ESCAPES, false) unless str lex_error(Puppet::Pops::Issues::UNCLOSED_QUOTE, :after => format_quote(last), :followed_by => followed_by) end # Terminator may be a single char '"', '$', or two characters '${' group match 1 (scn[1]) from the last slurp holds this terminator = scn[1] [str[0..(-1 - terminator.length)], terminator] end # Copy from old lexer - can do much better def slurp_uqstring scn = @scanner last = scn.matched ignore = true + str = slurp(scn, @lexing_context[:uq_slurp_pattern], @lexing_context[:escapes], :ignore_invalid_escapes) # Terminator may be a single char '$', two characters '${', or empty string '' at the end of intput. # Group match 1 holds this. # The exceptional case is found by looking at the subgroup 1 of the most recent match made by the scanner (i.e. @scanner[1]). # This is the last match made by the slurp method (having called scan_until on the scanner). # If there is a terminating character is must be stripped and returned separately. # terminator = scn[1] [str[0..(-1 - terminator.length)], terminator] end # Slurps a string from the given scanner until the given pattern and then replaces any escaped # characters given by escapes into their control-character equivalent or in case of line breaks, replaces the # pattern \r?\n with an empty string. # The returned string contains the terminating character. Returns nil if the scanner can not scan until the given # pattern. # def slurp(scanner, pattern, escapes, ignore_invalid_escapes) str = scanner.scan_until(pattern) || return # Process unicode escapes first as they require getting 4 hex digits # If later a \u is found it is warned not to be a unicode escape if escapes.include?('u') str.gsub!(/\\u(?:([\da-fA-F]{4})|\{([\da-fA-F]{1,6})\})/m) { [($1 || $2).hex].pack("U") } end begin str.gsub!(/\\([^\r\n]|(?:\r?\n))/m) { ch = $1 if escapes.include? ch case ch when 'r' ; "\r" when 'n' ; "\n" when 't' ; "\t" when 's' ; " " when 'u' lex_warning(Puppet::Pops::Issues::ILLEGAL_UNICODE_ESCAPE) "\\u" when "\n" ; '' when "\r\n"; '' else ch end else lex_warning(Puppet::Pops::Issues::UNRECOGNIZED_ESCAPE, :ch => ch) unless ignore_invalid_escapes "\\#{ch}" end } rescue ArgumentError => e # A invalid byte sequence may be the result of faulty input as well, but that could not possibly # have reached this far... Unfortunately there is no more specific error and a match on message is # required to differentiate from other internal problems. if e.message =~ /invalid byte sequence/ lex_error(Puppet::Pops::Issues::ILLEGAL_UNICODE_ESCAPE) else raise e end end str end end diff --git a/spec/unit/pops/evaluator/evaluating_parser_spec.rb b/spec/unit/pops/evaluator/evaluating_parser_spec.rb index ca3ba3a34..bfa3343bd 100644 --- a/spec/unit/pops/evaluator/evaluating_parser_spec.rb +++ b/spec/unit/pops/evaluator/evaluating_parser_spec.rb @@ -1,1400 +1,1410 @@ require 'spec_helper' require 'puppet/pops' require 'puppet/pops/evaluator/evaluator_impl' require 'puppet/loaders' require 'puppet_spec/pops' require 'puppet_spec/scope' require 'puppet/parser/e4_parser_adapter' # relative to this spec file (./) does not work as this file is loaded by rspec #require File.join(File.dirname(__FILE__), '/evaluator_rspec_helper') describe 'Puppet::Pops::Evaluator::EvaluatorImpl' do include PuppetSpec::Pops include PuppetSpec::Scope before(:each) do Puppet[:strict_variables] = true # These must be set since the 3x logic switches some behaviors on these even if the tests explicitly # use the 4x parser and evaluator. # Puppet[:parser] = 'future' # Puppetx cannot be loaded until the correct parser has been set (injector is turned off otherwise) require 'puppetx' # Tests needs a known configuration of node/scope/compiler since it parses and evaluates # snippets as the compiler will evaluate them, butwithout the overhead of compiling a complete # catalog for each tested expression. # @parser = Puppet::Pops::Parser::EvaluatingParser.new @node = Puppet::Node.new('node.example.com') @node.environment = Puppet::Node::Environment.create(:testing, []) @compiler = Puppet::Parser::Compiler.new(@node) @scope = Puppet::Parser::Scope.new(@compiler) @scope.source = Puppet::Resource::Type.new(:node, 'node.example.com') @scope.parent = @compiler.topscope end let(:parser) { @parser } let(:scope) { @scope } types = Puppet::Pops::Types::TypeFactory context "When evaluator evaluates literals" do { "1" => 1, "010" => 8, "0x10" => 16, "3.14" => 3.14, "0.314e1" => 3.14, "31.4e-1" => 3.14, "'1'" => '1', "'banana'" => 'banana', '"banana"' => 'banana', "banana" => 'banana', "banana::split" => 'banana::split', "false" => false, "true" => true, "Array" => types.array_of_data(), "/.*/" => /.*/ }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end end context "When the evaluator evaluates Lists and Hashes" do { "[]" => [], "[1,2,3]" => [1,2,3], "[1,[2.0, 2.1, [2.2]],[3.0, 3.1]]" => [1,[2.0, 2.1, [2.2]],[3.0, 3.1]], "[2 + 2]" => [4], "[1,2,3] == [1,2,3]" => true, "[1,2,3] != [2,3,4]" => true, "[1,2,3] == [2,2,3]" => false, "[1,2,3] != [1,2,3]" => false, "[1,2,3][2]" => 3, "[1,2,3] + [4,5]" => [1,2,3,4,5], "[1,2,3, *[4,5]]" => [1,2,3,4,5], "[1,2,3, (*[4,5])]" => [1,2,3,4,5], "[1,2,3, ((*[4,5]))]" => [1,2,3,4,5], "[1,2,3] + [[4,5]]" => [1,2,3,[4,5]], "[1,2,3] + 4" => [1,2,3,4], "[1,2,3] << [4,5]" => [1,2,3,[4,5]], "[1,2,3] << {'a' => 1, 'b'=>2}" => [1,2,3,{'a' => 1, 'b'=>2}], "[1,2,3] << 4" => [1,2,3,4], "[1,2,3,4] - [2,3]" => [1,4], "[1,2,3,4] - [2,5]" => [1,3,4], "[1,2,3,4] - 2" => [1,3,4], "[1,2,3,[2],4] - 2" => [1,3,[2],4], "[1,2,3,[2,3],4] - [[2,3]]" => [1,2,3,4], "[1,2,3,3,2,4,2,3] - [2,3]" => [1,4], "[1,2,3,['a',1],['b',2]] - {'a' => 1, 'b'=>2}" => [1,2,3], "[1,2,3,{'a'=>1,'b'=>2}] - [{'a' => 1, 'b'=>2}]" => [1,2,3], }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "[1,2,3] + {'a' => 1, 'b'=>2}" => [1,2,3,['a',1],['b',2]], }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do # This test must be done with match_array since the order of the hash # is undefined and Ruby 1.8.7 and 1.9.3 produce different results. expect(parser.evaluate_string(scope, source, __FILE__)).to match_array(result) end end { "[1,2,3][a]" => :error, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end { "{}" => {}, "{'a'=>1,'b'=>2}" => {'a'=>1,'b'=>2}, "{'a'=>1,'b'=>{'x'=>2.1,'y'=>2.2}}" => {'a'=>1,'b'=>{'x'=>2.1,'y'=>2.2}}, "{'a'=> 2 + 2}" => {'a'=> 4}, "{'a'=> 1, 'b'=>2} == {'a'=> 1, 'b'=>2}" => true, "{'a'=> 1, 'b'=>2} != {'x'=> 1, 'b'=>2}" => true, "{'a'=> 1, 'b'=>2} == {'a'=> 2, 'b'=>3}" => false, "{'a'=> 1, 'b'=>2} != {'a'=> 1, 'b'=>2}" => false, "{a => 1, b => 2}[b]" => 2, "{2+2 => sum, b => 2}[4]" => 'sum', "{'a'=>1, 'b'=>2} + {'c'=>3}" => {'a'=>1,'b'=>2,'c'=>3}, "{'a'=>1, 'b'=>2} + {'b'=>3}" => {'a'=>1,'b'=>3}, "{'a'=>1, 'b'=>2} + ['c', 3, 'b', 3]" => {'a'=>1,'b'=>3, 'c'=>3}, "{'a'=>1, 'b'=>2} + [['c', 3], ['b', 3]]" => {'a'=>1,'b'=>3, 'c'=>3}, "{'a'=>1, 'b'=>2} - {'b' => 3}" => {'a'=>1}, "{'a'=>1, 'b'=>2, 'c'=>3} - ['b', 'c']" => {'a'=>1}, "{'a'=>1, 'b'=>2, 'c'=>3} - 'c'" => {'a'=>1, 'b'=>2}, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "{'a' => 1, 'b'=>2} << 1" => :error, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end end context "When the evaluator perform comparisons" do { "'a' == 'a'" => true, "'a' == 'b'" => false, "'a' != 'a'" => false, "'a' != 'b'" => true, "'a' < 'b' " => true, "'a' < 'a' " => false, "'b' < 'a' " => false, "'a' <= 'b'" => true, "'a' <= 'a'" => true, "'b' <= 'a'" => false, "'a' > 'b' " => false, "'a' > 'a' " => false, "'b' > 'a' " => true, "'a' >= 'b'" => false, "'a' >= 'a'" => true, "'b' >= 'a'" => true, "'a' == 'A'" => true, "'a' != 'A'" => false, "'a' > 'A'" => false, "'a' >= 'A'" => true, "'A' < 'a'" => false, "'A' <= 'a'" => true, "1 == 1" => true, "1 == 2" => false, "1 != 1" => false, "1 != 2" => true, "1 < 2 " => true, "1 < 1 " => false, "2 < 1 " => false, "1 <= 2" => true, "1 <= 1" => true, "2 <= 1" => false, "1 > 2 " => false, "1 > 1 " => false, "2 > 1 " => true, "1 >= 2" => false, "1 >= 1" => true, "2 >= 1" => true, "1 == 1.0 " => true, "1 < 1.1 " => true, "1.0 == 1 " => true, "1.0 < 2 " => true, "'1.0' < 'a'" => true, "'1.0' < '' " => false, "'1.0' < ' '" => false, "'a' > '1.0'" => true, "/.*/ == /.*/ " => true, "/.*/ != /a.*/" => true, "true == true " => true, "false == false" => true, "true == false" => false, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "a > 1" => /String > Integer/, "a >= 1" => /String >= Integer/, "a < 1" => /String < Integer/, "a <= 1" => /String <= Integer/, "1 > a" => /Integer > String/, "1 >= a" => /Integer >= String/, "1 < a" => /Integer < String/, "1 <= a" => /Integer <= String/, }.each do | source, error| it "should not allow comparison of String and Number '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__)}.to raise_error(error) end end { "'a' =~ /.*/" => true, "'a' =~ '.*'" => true, "/.*/ != /a.*/" => true, "'a' !~ /b.*/" => true, "'a' !~ 'b.*'" => true, '$x = a; a =~ "$x.*"' => true, "a =~ Pattern['a.*']" => true, "a =~ Regexp['a.*']" => false, # String is not subtype of Regexp. PUP-957 "$x = /a.*/ a =~ $x" => true, "$x = Pattern['a.*'] a =~ $x" => true, "1 =~ Integer" => true, "1 !~ Integer" => false, "[1,2,3] =~ Array[Integer[1,10]]" => true, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "666 =~ /6/" => :error, "[a] =~ /a/" => :error, "{a=>1} =~ /a/" => :error, "/a/ =~ /a/" => :error, "Array =~ /A/" => :error, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end { "1 in [1,2,3]" => true, "4 in [1,2,3]" => false, "a in {x=>1, a=>2}" => true, "z in {x=>1, a=>2}" => false, "ana in bananas" => true, "xxx in bananas" => false, "/ana/ in bananas" => true, "/xxx/ in bananas" => false, "ANA in bananas" => false, # ANA is a type, not a String "String[1] in bananas" => false, # Philosophically true though :-) "'ANA' in bananas" => true, "ana in 'BANANAS'" => true, "/ana/ in 'BANANAS'" => false, "/ANA/ in 'BANANAS'" => true, "xxx in 'BANANAS'" => false, "[2,3] in [1,[2,3],4]" => true, "[2,4] in [1,[2,3],4]" => false, "[a,b] in ['A',['A','B'],'C']" => true, "[x,y] in ['A',['A','B'],'C']" => false, "a in {a=>1}" => true, "x in {a=>1}" => false, "'A' in {a=>1}" => true, "'X' in {a=>1}" => false, "a in {'A'=>1}" => true, "x in {'A'=>1}" => false, "/xxx/ in {'aaaxxxbbb'=>1}" => true, "/yyy/ in {'aaaxxxbbb'=>1}" => false, "15 in [1, 0xf]" => true, "15 in [1, '0xf']" => false, "'15' in [1, 0xf]" => false, "15 in [1, 115]" => false, "1 in [11, '111']" => false, "'1' in [11, '111']" => false, "Array[Integer] in [2, 3]" => false, "Array[Integer] in [2, [3, 4]]" => true, "Array[Integer] in [2, [a, 4]]" => false, "Integer in { 2 =>'a'}" => true, "Integer[5,10] in [1,5,3]" => true, "Integer[5,10] in [1,2,3]" => false, "Integer in {'a'=>'a'}" => false, "Integer in {'a'=>1}" => false, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "if /(ana)/ in bananas {$1}" => 'ana', "if /(xyz)/ in bananas {$1} else {$1}" => nil, "$a = bananas =~ /(ana)/; $b = /(xyz)/ in bananas; $1" => 'ana', "$a = xyz =~ /(xyz)/; $b = /(ana)/ in bananas; $1" => 'ana', "if /p/ in [pineapple, bananas] {$0}" => 'p', "if /b/ in [pineapple, bananas] {$0}" => 'b', }.each do |source, result| it "sets match variables for a regexp search using in such that '#{source}' produces '#{result}'" do parser.evaluate_string(scope, source, __FILE__).should == result end end { 'Any' => ['Data', 'Scalar', 'Numeric', 'Integer', 'Float', 'Boolean', 'String', 'Pattern', 'Collection', 'Array', 'Hash', 'CatalogEntry', 'Resource', 'Class', 'Undef', 'File', 'NotYetKnownResourceType'], # Note, Data > Collection is false (so not included) 'Data' => ['Scalar', 'Numeric', 'Integer', 'Float', 'Boolean', 'String', 'Pattern', 'Array', 'Hash',], 'Scalar' => ['Numeric', 'Integer', 'Float', 'Boolean', 'String', 'Pattern'], 'Numeric' => ['Integer', 'Float'], 'CatalogEntry' => ['Class', 'Resource', 'File', 'NotYetKnownResourceType'], 'Integer[1,10]' => ['Integer[2,3]'], }.each do |general, specials| specials.each do |special | it "should compute that #{general} > #{special}" do parser.evaluate_string(scope, "#{general} > #{special}", __FILE__).should == true end it "should compute that #{special} < #{general}" do parser.evaluate_string(scope, "#{special} < #{general}", __FILE__).should == true end it "should compute that #{general} != #{special}" do parser.evaluate_string(scope, "#{special} != #{general}", __FILE__).should == true end end end { 'Integer[1,10] > Integer[2,3]' => true, 'Integer[1,10] == Integer[2,3]' => false, 'Integer[1,10] > Integer[0,5]' => false, 'Integer[1,10] > Integer[1,10]' => false, 'Integer[1,10] >= Integer[1,10]' => true, 'Integer[1,10] == Integer[1,10]' => true, }.each do |source, result| it "should parse and evaluate the integer range comparison expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end end context "When the evaluator performs arithmetic" do context "on Integers" do { "2+2" => 4, "2 + 2" => 4, "7 - 3" => 4, "6 * 3" => 18, "6 / 3" => 2, "6 % 3" => 0, "10 % 3" => 1, "-(6/3)" => -2, "-6/3 " => -2, "8 >> 1" => 4, "8 << 1" => 16, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end context "on Floats" do { "2.2 + 2.2" => 4.4, "7.7 - 3.3" => 4.4, "6.1 * 3.1" => 18.91, "6.6 / 3.3" => 2.0, "-(6.0/3.0)" => -2.0, "-6.0/3.0 " => -2.0, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "3.14 << 2" => :error, "3.14 >> 2" => :error, "6.6 % 3.3" => 0.0, "10.0 % 3.0" => 1.0, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end end context "on strings requiring boxing to Numeric" do { "'2' + '2'" => 4, "'-2' + '2'" => 0, "'- 2' + '2'" => 0, '"-\t 2" + "2"' => 0, "'+2' + '2'" => 4, "'+ 2' + '2'" => 4, "'2.2' + '2.2'" => 4.4, "'-2.2' + '2.2'" => 0.0, "'0xF7' + '010'" => 0xFF, "'0xF7' + '0x8'" => 0xFF, "'0367' + '010'" => 0xFF, "'012.3' + '010'" => 20.3, "'-0x2' + '0x4'" => 2, "'+0x2' + '0x4'" => 6, "'-02' + '04'" => 2, "'+02' + '04'" => 6, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "'0888' + '010'" => :error, "'0xWTF' + '010'" => :error, "'0x12.3' + '010'" => :error, "'0x12.3' + '010'" => :error, '"-\n 2" + "2"' => :error, '"-\v 2" + "2"' => :error, '"-2\n" + "2"' => :error, '"-2\n " + "2"' => :error, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end end end end # arithmetic context "When the evaluator evaluates assignment" do { "$a = 5" => 5, "$a = 5; $a" => 5, "$a = 5; $b = 6; $a" => 5, "$a = $b = 5; $a == $b" => true, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "[a,b,c] = [1,2,3]" => /attempt to assign to 'an Array Expression'/, "[a,b,c] = {b=>2,c=>3,a=>1}" => /attempt to assign to 'an Array Expression'/, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to error with #{result}" do expect { parser.evaluate_string(scope, source, __FILE__)}.to raise_error(Puppet::ParseError, result) end end end context "When the evaluator evaluates conditionals" do { "if true {5}" => 5, "if false {5}" => nil, "if false {2} else {5}" => 5, "if false {2} elsif true {5}" => 5, "if false {2} elsif false {5}" => nil, "unless false {5}" => 5, "unless true {5}" => nil, "unless true {2} else {5}" => 5, "unless true {} else {5}" => 5, "$a = if true {5} $a" => 5, "$a = if false {5} $a" => nil, "$a = if false {2} else {5} $a" => 5, "$a = if false {2} elsif true {5} $a" => 5, "$a = if false {2} elsif false {5} $a" => nil, "$a = unless false {5} $a" => 5, "$a = unless true {5} $a" => nil, "$a = unless true {2} else {5} $a" => 5, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "case 1 { 1 : { yes } }" => 'yes', "case 2 { 1,2,3 : { yes} }" => 'yes', "case 2 { 1,3 : { no } 2: { yes} }" => 'yes', "case 2 { 1,3 : { no } 5: { no } default: { yes }}" => 'yes', "case 2 { 1,3 : { no } 5: { no } }" => nil, "case 'banana' { 1,3 : { no } /.*ana.*/: { yes } }" => 'yes', "case 'banana' { /.*(ana).*/: { $1 } }" => 'ana', "case [1] { Array : { yes } }" => 'yes', "case [1] { Array[String] : { no } Array[Integer]: { yes } }" => 'yes', "case 1 { Integer : { yes } Type[Integer] : { no } }" => 'yes', "case Integer { Integer : { no } Type[Integer] : { yes } }" => 'yes', # supports unfold "case ringo { *[paul, john, ringo, george] : { 'beatle' } }" => 'beatle', "case ringo { (*[paul, john, ringo, george]) : { 'beatle' } }" => 'beatle', "case undef { undef : { 'yes' } }" => 'yes', "case undef { *undef : { 'no' } default :{ 'yes' }}" => 'yes', "case [green, 2, whatever] { [/ee/, Integer[0,10], default] : { 'yes' } default :{ 'no' }}" => 'yes', "case {a=>1, b=>2, whatever=>3, extra => ignored} { { a => Integer[0,5], b => Integer[0,5], whatever => default } : { 'yes' } default : { 'no' }}" => 'yes', }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "2 ? { 1 => no, 2 => yes}" => 'yes', "3 ? { 1 => no, 2 => no, default => yes }" => 'yes', "3 ? { 1 => no, default => yes, 3 => no }" => 'no', "3 ? { 1 => no, 3 => no, default => yes }" => 'no', "4 ? { 1 => no, default => yes, 3 => no }" => 'yes', "4 ? { 1 => no, 3 => no, default => yes }" => 'yes', "'banana' ? { /.*(ana).*/ => $1 }" => 'ana', "[2] ? { Array[String] => yes, Array => yes}" => 'yes', "ringo ? *[paul, john, ringo, george] => 'beatle'" => 'beatle', "ringo ? (*[paul, john, ringo, george]) => 'beatle'"=> 'beatle', "undef ? undef => 'yes'" => 'yes', "undef ? {*undef => 'no', default => 'yes'}" => 'yes', "[green, 2, whatever] ? { [/ee/, Integer[0,10], default ] => 'yes', default => 'no'}" => 'yes', "{a=>1, b=>2, whatever=>3, extra => ignored} ? {{ a => Integer[0,5], b => Integer[0,5], whatever => default } => 'yes', default => 'no' }" => 'yes', }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end it 'fails if a selector does not match' do expect{parser.evaluate_string(scope, "2 ? 3 => 4")}.to raise_error(/No matching entry for selector parameter with value '2'/) end end context "When evaluator evaluated unfold" do { "*[1,2,3]" => [1,2,3], "*1" => [1], "*'a'" => ['a'] }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end it "should parse and evaluate the expression '*{a=>10, b=>20} to [['a',10],['b',20]]" do result = parser.evaluate_string(scope, '*{a=>10, b=>20}', __FILE__) expect(result).to include(['a', 10]) expect(result).to include(['b', 20]) end end context "When evaluator performs [] operations" do { "[1,2,3][0]" => 1, "[1,2,3][2]" => 3, "[1,2,3][3]" => nil, "[1,2,3][-1]" => 3, "[1,2,3][-2]" => 2, "[1,2,3][-4]" => nil, "[1,2,3,4][0,2]" => [1,2], "[1,2,3,4][1,3]" => [2,3,4], "[1,2,3,4][-2,2]" => [3,4], "[1,2,3,4][-3,2]" => [2,3], "[1,2,3,4][3,5]" => [4], "[1,2,3,4][5,2]" => [], "[1,2,3,4][0,-1]" => [1,2,3,4], "[1,2,3,4][0,-2]" => [1,2,3], "[1,2,3,4][0,-4]" => [1], "[1,2,3,4][0,-5]" => [], "[1,2,3,4][-5,2]" => [1], "[1,2,3,4][-5,-3]" => [1,2], "[1,2,3,4][-6,-3]" => [1,2], "[1,2,3,4][2,-3]" => [], "[1,*[2,3],4]" => [1,2,3,4], "[1,*[2,3],4][1]" => 2, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "{a=>1, b=>2, c=>3}[a]" => 1, "{a=>1, b=>2, c=>3}[c]" => 3, "{a=>1, b=>2, c=>3}[x]" => nil, "{a=>1, b=>2, c=>3}[c,b]" => [3,2], "{a=>1, b=>2, c=>3}[a,b,c]" => [1,2,3], "{a=>{b=>{c=>'it works'}}}[a][b][c]" => 'it works', "$a = {undef => 10} $a[free_lunch]" => nil, "$a = {undef => 10} $a[undef]" => 10, "$a = {undef => 10} $a[$a[free_lunch]]" => 10, "$a = {} $a[free_lunch] == undef" => true, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "'abc'[0]" => 'a', "'abc'[2]" => 'c', "'abc'[-1]" => 'c', "'abc'[-2]" => 'b', "'abc'[-3]" => 'a', "'abc'[-4]" => '', "'abc'[3]" => '', "abc[0]" => 'a', "abc[2]" => 'c', "abc[-1]" => 'c', "abc[-2]" => 'b', "abc[-3]" => 'a', "abc[-4]" => '', "abc[3]" => '', "'abcd'[0,2]" => 'ab', "'abcd'[1,3]" => 'bcd', "'abcd'[-2,2]" => 'cd', "'abcd'[-3,2]" => 'bc', "'abcd'[3,5]" => 'd', "'abcd'[5,2]" => '', "'abcd'[0,-1]" => 'abcd', "'abcd'[0,-2]" => 'abc', "'abcd'[0,-4]" => 'a', "'abcd'[0,-5]" => '', "'abcd'[-5,2]" => 'a', "'abcd'[-5,-3]" => 'ab', "'abcd'[-6,-3]" => 'ab', "'abcd'[2,-3]" => '', }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end # Type operations (full set tested by tests covering type calculator) { "Array[Integer]" => types.array_of(types.integer), "Array[Integer,1]" => types.constrain_size(types.array_of(types.integer),1, :default), "Array[Integer,1,2]" => types.constrain_size(types.array_of(types.integer),1, 2), "Array[Integer,Integer[1,2]]" => types.constrain_size(types.array_of(types.integer),1, 2), "Array[Integer,Integer[1]]" => types.constrain_size(types.array_of(types.integer),1, :default), "Hash[Integer,Integer]" => types.hash_of(types.integer, types.integer), "Hash[Integer,Integer,1]" => types.constrain_size(types.hash_of(types.integer, types.integer),1, :default), "Hash[Integer,Integer,1,2]" => types.constrain_size(types.hash_of(types.integer, types.integer),1, 2), "Hash[Integer,Integer,Integer[1,2]]" => types.constrain_size(types.hash_of(types.integer, types.integer),1, 2), "Hash[Integer,Integer,Integer[1]]" => types.constrain_size(types.hash_of(types.integer, types.integer),1, :default), "Resource[File]" => types.resource('File'), "Resource['File']" => types.resource(types.resource('File')), "File[foo]" => types.resource('file', 'foo'), "File[foo, bar]" => [types.resource('file', 'foo'), types.resource('file', 'bar')], "Pattern[a, /b/, Pattern[c], Regexp[d]]" => types.pattern('a', 'b', 'c', 'd'), "String[1,2]" => types.constrain_size(types.string,1, 2), "String[Integer[1,2]]" => types.constrain_size(types.string,1, 2), "String[Integer[1]]" => types.constrain_size(types.string,1, :default), }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end # LHS where [] not supported, and missing key(s) { "Array[]" => :error, "'abc'[]" => :error, "Resource[]" => :error, "File[]" => :error, "String[]" => :error, "1[]" => :error, "3.14[]" => :error, "/.*/[]" => :error, "$a=[1] $a[]" => :error, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do expect { parser.evaluate_string(scope, source, __FILE__)}.to raise_error(/Syntax error/) end end # Errors when wrong number/type of keys are used { "Array[0]" => 'Array-Type[] arguments must be types. Got Fixnum', "Hash[0]" => 'Hash-Type[] arguments must be types. Got Fixnum', "Hash[Integer, 0]" => 'Hash-Type[] arguments must be types. Got Fixnum', "Array[Integer,1,2,3]" => 'Array-Type[] accepts 1 to 3 arguments. Got 4', "Array[Integer,String]" => "A Type's size constraint arguments must be a single Integer type, or 1-2 integers (or default). Got a String-Type", "Hash[Integer,String, 1,2,3]" => 'Hash-Type[] accepts 2 to 4 arguments. Got 5', "'abc'[x]" => "The value 'x' cannot be converted to Numeric", "'abc'[1.0]" => "A String[] cannot use Float where Integer is expected", "'abc'[1,2,3]" => "String supports [] with one or two arguments. Got 3", "Resource[0]" => 'First argument to Resource[] must be a resource type or a String. Got Integer', "Resource[a, 0]" => 'Error creating type specialization of a Resource-Type, Cannot use Integer where a resource title String is expected', "File[0]" => 'Error creating type specialization of a File-Type, Cannot use Integer where a resource title String is expected', "String[a]" => "A Type's size constraint arguments must be a single Integer type, or 1-2 integers (or default). Got a String", "Pattern[0]" => 'Error creating type specialization of a Pattern-Type, Cannot use Integer where String or Regexp or Pattern-Type or Regexp-Type is expected', "Regexp[0]" => 'Error creating type specialization of a Regexp-Type, Cannot use Integer where String or Regexp is expected', "Regexp[a,b]" => 'A Regexp-Type[] accepts 1 argument. Got 2', "true[0]" => "Operator '[]' is not applicable to a Boolean", "1[0]" => "Operator '[]' is not applicable to an Integer", "3.14[0]" => "Operator '[]' is not applicable to a Float", "/.*/[0]" => "Operator '[]' is not applicable to a Regexp", "[1][a]" => "The value 'a' cannot be converted to Numeric", "[1][0.0]" => "An Array[] cannot use Float where Integer is expected", "[1]['0.0']" => "An Array[] cannot use Float where Integer is expected", "[1,2][1, 0.0]" => "An Array[] cannot use Float where Integer is expected", "[1,2][1.0, -1]" => "An Array[] cannot use Float where Integer is expected", "[1,2][1, -1.0]" => "An Array[] cannot use Float where Integer is expected", }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do expect { parser.evaluate_string(scope, source, __FILE__)}.to raise_error(Regexp.new(Regexp.quote(result))) end end context "on catalog types" do it "[n] gets resource parameter [n]" do source = "notify { 'hello': message=>'yo'} Notify[hello][message]" parser.evaluate_string(scope, source, __FILE__).should == 'yo' end it "[n] gets class parameter [n]" do source = "class wonka($produces='chocolate'){ } include wonka Class[wonka][produces]" # This is more complicated since it needs to run like 3.x and do an import_ast adapted_parser = Puppet::Parser::E4ParserAdapter.new adapted_parser.file = __FILE__ ast = adapted_parser.parse(source) Puppet.override({:global_scope => scope}, "test") do scope.known_resource_types.import_ast(ast, '') ast.code.safeevaluate(scope).should == 'chocolate' end end # Resource default and override expressions and resource parameter access with [] { # Properties "notify { id: message=>explicit} Notify[id][message]" => "explicit", "Notify { message=>by_default} notify {foo:} Notify[foo][message]" => "by_default", "notify {foo:} Notify[foo]{message =>by_override} Notify[foo][message]" => "by_override", # Parameters "notify { id: withpath=>explicit} Notify[id][withpath]" => "explicit", "Notify { withpath=>by_default } notify { foo: } Notify[foo][withpath]" => "by_default", "notify {foo:} Notify[foo]{withpath=>by_override} Notify[foo][withpath]" => "by_override", # Metaparameters "notify { foo: tag => evoe} Notify[foo][tag]" => "evoe", # Does not produce the defaults for tag parameter (title, type or names of scopes) "notify { foo: } Notify[foo][tag]" => nil, # But a default may be specified on the type "Notify { tag=>by_default } notify { foo: } Notify[foo][tag]" => "by_default", "Notify { tag=>by_default } notify { foo: } Notify[foo]{ tag=>by_override } Notify[foo][tag]" => "by_override", }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end # Virtual and realized resource default and overridden resource parameter access with [] { # Properties "@notify { id: message=>explicit } Notify[id][message]" => "explicit", "@notify { id: message=>explicit } realize Notify[id] Notify[id][message]" => "explicit", "Notify { message=>by_default } @notify { id: } Notify[id][message]" => "by_default", "Notify { message=>by_default } @notify { id: tag=>thisone } Notify <| tag == thisone |>; Notify[id][message]" => "by_default", "@notify { id: } Notify[id]{message=>by_override} Notify[id][message]" => "by_override", # Parameters "@notify { id: withpath=>explicit } Notify[id][withpath]" => "explicit", "Notify { withpath=>by_default } @notify { id: } Notify[id][withpath]" => "by_default", "@notify { id: } realize Notify[id] Notify[id]{withpath=>by_override} Notify[id][withpath]" => "by_override", # Metaparameters "@notify { id: tag=>explicit } Notify[id][tag]" => "explicit", }.each do |source, result| it "parses and evaluates virtual and realized resources in the expression '#{source}' to #{result}" do expect(parser.evaluate_string(scope, source, __FILE__)).to eq(result) end end # Exported resource attributes { "@@notify { id: message=>explicit } Notify[id][message]" => "explicit", "@@notify { id: message=>explicit, tag=>thisone } Notify <<| tag == thisone |>> Notify[id][message]" => "explicit", }.each do |source, result| it "parses and evaluates exported resources in the expression '#{source}' to #{result}" do expect(parser.evaluate_string(scope, source, __FILE__)).to eq(result) end end # Resource default and override expressions and resource parameter access error conditions { "notify { xid: message=>explicit} Notify[id][message]" => /Resource not found/, "notify { id: message=>explicit} Notify[id][mustard]" => /does not have a parameter called 'mustard'/, # NOTE: these meta-esque parameters are not recognized as such "notify { id: message=>explicit} Notify[id][title]" => /does not have a parameter called 'title'/, "notify { id: message=>explicit} Notify[id]['type']" => /does not have a parameter called 'type'/, "notify { id: message=>explicit } Notify[id]{message=>override}" => /'message' is already set on Notify\[id\]/ }.each do |source, result| it "should parse '#{source}' and raise error matching #{result}" do expect { parser.evaluate_string(scope, source, __FILE__)}.to raise_error(result) end end context 'with errors' do { "Class['fail-whale']" => /Illegal name/, "Class[0]" => /An Integer cannot be used where a String is expected/, "Class[/.*/]" => /A Regexp cannot be used where a String is expected/, "Class[4.1415]" => /A Float cannot be used where a String is expected/, "Class[Integer]" => /An Integer-Type cannot be used where a String is expected/, "Class[File['tmp']]" => /A File\['tmp'\] Resource-Reference cannot be used where a String is expected/, }.each do | source, error_pattern| it "an error is flagged for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__)}.to raise_error(error_pattern) end end end end # end [] operations end context "When the evaluator performs boolean operations" do { "true and true" => true, "false and true" => false, "true and false" => false, "false and false" => false, "true or true" => true, "false or true" => true, "true or false" => true, "false or false" => false, "! true" => false, "!! true" => true, "!! false" => false, "! 'x'" => false, "! ''" => false, "! undef" => true, "! [a]" => false, "! []" => false, "! {a=>1}" => false, "! {}" => false, "true and false and '0xwtf' + 1" => false, "false or true or '0xwtf' + 1" => true, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do parser.evaluate_string(scope, source, __FILE__).should == result end end { "false || false || '0xwtf' + 1" => :error, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end end context "When evaluator performs operations on literal undef" do it "computes non existing hash lookup as undef" do parser.evaluate_string(scope, "{a => 1}[b] == undef", __FILE__).should == true parser.evaluate_string(scope, "undef == {a => 1}[b]", __FILE__).should == true end end context "When evaluator performs calls" do let(:populate) do parser.evaluate_string(scope, "$a = 10 $b = [1,2,3]") end { 'sprintf( "x%iy", $a )' => "x10y", # unfolds 'sprintf( *["x%iy", $a] )' => "x10y", '( *["x%iy", $a] ).sprintf' => "x10y", '((*["x%iy", $a])).sprintf' => "x10y", '"x%iy".sprintf( $a )' => "x10y", '$b.reduce |$memo,$x| { $memo + $x }' => 6, 'reduce($b) |$memo,$x| { $memo + $x }' => 6, }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do populate parser.evaluate_string(scope, source, __FILE__).should == result end end { '"value is ${a*2} yo"' => :error, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end it "provides location information on error in unparenthesized call logic" do expect{parser.evaluate_string(scope, "include non_existing_class", __FILE__)}.to raise_error(Puppet::ParseError, /line 1\:1/) end it 'defaults can be given in a lambda and used only when arg is missing' do env_loader = @compiler.loaders.public_environment_loader fc = Puppet::Functions.create_function(:test) do dispatch :test do param 'Integer', 'count' required_block_param end def test(count) yield(*[].fill(10, 0, count)) end end the_func = fc.new({}, env_loader) env_loader.add_entry(:function, 'test', the_func, __FILE__) expect(parser.evaluate_string(scope, "test(1) |$x, $y=20| { $x + $y}")).to eql(30) expect(parser.evaluate_string(scope, "test(2) |$x, $y=20| { $x + $y}")).to eql(20) end it 'a given undef does not select the default value' do env_loader = @compiler.loaders.public_environment_loader fc = Puppet::Functions.create_function(:test) do dispatch :test do param 'Any', 'lambda_arg' required_block_param end def test(lambda_arg) yield(lambda_arg) end end the_func = fc.new({}, env_loader) env_loader.add_entry(:function, 'test', the_func, __FILE__) expect(parser.evaluate_string(scope, "test(undef) |$x=20| { $x == undef}")).to eql(true) end it 'a given undef is given as nil' do env_loader = @compiler.loaders.public_environment_loader fc = Puppet::Functions.create_function(:assert_no_undef) do dispatch :assert_no_undef do param 'Any', 'x' end def assert_no_undef(x) case x when Array return unless x.include?(:undef) when Hash return unless x.keys.include?(:undef) || x.values.include?(:undef) else return unless x == :undef end raise "contains :undef" end end the_func = fc.new({}, env_loader) env_loader.add_entry(:function, 'assert_no_undef', the_func, __FILE__) expect{parser.evaluate_string(scope, "assert_no_undef(undef)")}.to_not raise_error() expect{parser.evaluate_string(scope, "assert_no_undef([undef])")}.to_not raise_error() expect{parser.evaluate_string(scope, "assert_no_undef({undef => 1})")}.to_not raise_error() expect{parser.evaluate_string(scope, "assert_no_undef({1 => undef})")}.to_not raise_error() end context 'using the 3x function api' do it 'can call a 3x function' do Puppet::Parser::Functions.newfunction("bazinga", :type => :rvalue) { |args| args[0] } parser.evaluate_string(scope, "bazinga(42)", __FILE__).should == 42 end it 'maps :undef to empty string' do Puppet::Parser::Functions.newfunction("bazinga", :type => :rvalue) { |args| args[0] } parser.evaluate_string(scope, "$a = {} bazinga($a[nope])", __FILE__).should == '' parser.evaluate_string(scope, "bazinga(undef)", __FILE__).should == '' end it 'does not map :undef to empty string in arrays' do Puppet::Parser::Functions.newfunction("bazinga", :type => :rvalue) { |args| args[0][0] } parser.evaluate_string(scope, "$a = {} $b = [$a[nope]] bazinga($b)", __FILE__).should == :undef parser.evaluate_string(scope, "bazinga([undef])", __FILE__).should == :undef end it 'does not map :undef to empty string in hashes' do Puppet::Parser::Functions.newfunction("bazinga", :type => :rvalue) { |args| args[0]['a'] } parser.evaluate_string(scope, "$a = {} $b = {a => $a[nope]} bazinga($b)", __FILE__).should == :undef parser.evaluate_string(scope, "bazinga({a => undef})", __FILE__).should == :undef end end end context "When evaluator performs string interpolation" do let(:populate) do parser.evaluate_string(scope, "$a = 10 $b = [1,2,3]") end { '"value is $a yo"' => "value is 10 yo", '"value is \$a yo"' => "value is $a yo", '"value is ${a} yo"' => "value is 10 yo", '"value is \${a} yo"' => "value is ${a} yo", '"value is ${$a} yo"' => "value is 10 yo", '"value is ${$a*2} yo"' => "value is 20 yo", '"value is ${sprintf("x%iy",$a)} yo"' => "value is x10y yo", '"value is ${"x%iy".sprintf($a)} yo"' => "value is x10y yo", '"value is ${[1,2,3]} yo"' => "value is [1, 2, 3] yo", '"value is ${/.*/} yo"' => "value is /.*/ yo", '$x = undef "value is $x yo"' => "value is yo", '$x = default "value is $x yo"' => "value is default yo", '$x = Array[Integer] "value is $x yo"' => "value is Array[Integer] yo", '"value is ${Array[Integer]} yo"' => "value is Array[Integer] yo", }.each do |source, result| it "should parse and evaluate the expression '#{source}' to #{result}" do populate parser.evaluate_string(scope, source, __FILE__).should == result end end it "should parse and evaluate an interpolation of a hash" do source = '"value is ${{a=>1,b=>2}} yo"' # This test requires testing against two options because a hash to string # produces a result that is unordered hashstr = {'a' => 1, 'b' => 2}.to_s alt_results = ["value is {a => 1, b => 2} yo", "value is {b => 2, a => 1} yo" ] populate parse_result = parser.evaluate_string(scope, source, __FILE__) alt_results.include?(parse_result).should == true end it 'should accept a variable with leading underscore when used directly' do source = '$_x = 10; "$_x"' expect(parser.evaluate_string(scope, source, __FILE__)).to eql('10') end it 'should accept a variable with leading underscore when used as an expression' do source = '$_x = 10; "${_x}"' expect(parser.evaluate_string(scope, source, __FILE__)).to eql('10') end it 'should accept a numeric variable expressed as $n' do source = '$x = "abc123def" =~ /(abc)(123)(def)/; "${$2}"' expect(parser.evaluate_string(scope, source, __FILE__)).to eql('123') end it 'should accept a numeric variable expressed as just an integer' do source = '$x = "abc123def" =~ /(abc)(123)(def)/; "${2}"' expect(parser.evaluate_string(scope, source, __FILE__)).to eql('123') end it 'should accept a numeric variable expressed as $n in an access operation' do source = '$x = "abc123def" =~ /(abc)(123)(def)/; "${$0[4,3]}"' expect(parser.evaluate_string(scope, source, __FILE__)).to eql('23d') end it 'should accept a numeric variable expressed as just an integer in an access operation' do source = '$x = "abc123def" =~ /(abc)(123)(def)/; "${0[4,3]}"' expect(parser.evaluate_string(scope, source, __FILE__)).to eql('23d') end { '"value is ${a*2} yo"' => :error, }.each do |source, result| it "should parse and raise error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(Puppet::ParseError) end end end context "When evaluating variables" do context "that are non existing an error is raised for" do it "unqualified variable" do expect { parser.evaluate_string(scope, "$quantum_gravity", __FILE__) }.to raise_error(/Unknown variable/) end it "qualified variable" do expect { parser.evaluate_string(scope, "$quantum_gravity::graviton", __FILE__) }.to raise_error(/Unknown variable/) end end it "a lex error should be raised for '$foo::::bar'" do expect { parser.evaluate_string(scope, "$foo::::bar") }.to raise_error(Puppet::ParseErrorWithIssue, /Illegal fully qualified name at line 1:7/) end { '$a = $0' => nil, '$a = $1' => nil, }.each do |source, value| it "it is ok to reference numeric unassigned variables '#{source}'" do parser.evaluate_string(scope, source, __FILE__).should == value end end { '$00 = 0' => /must be a decimal value/, '$0xf = 0' => /must be a decimal value/, '$0777 = 0' => /must be a decimal value/, '$123a = 0' => /must be a decimal value/, }.each do |source, error_pattern| it "should raise an error for '#{source}'" do expect { parser.evaluate_string(scope, source, __FILE__) }.to raise_error(error_pattern) end end context "an initial underscore in the last segment of a var name is allowed" do { '$_a = 1' => 1, '$__a = 1' => 1, }.each do |source, value| it "as in this example '#{source}'" do parser.evaluate_string(scope, source, __FILE__).should == value end end end end context "When evaluating relationships" do it 'should form a relation with File[a] -> File[b]' do source = "File[a] -> File[b]" parser.evaluate_string(scope, source, __FILE__) scope.compiler.should have_relationship(['File', 'a', '->', 'File', 'b']) end it 'should form a relation with resource -> resource' do source = "notify{a:} -> notify{b:}" parser.evaluate_string(scope, source, __FILE__) scope.compiler.should have_relationship(['Notify', 'a', '->', 'Notify', 'b']) end it 'should form a relation with [File[a], File[b]] -> [File[x], File[y]]' do source = "[File[a], File[b]] -> [File[x], File[y]]" parser.evaluate_string(scope, source, __FILE__) scope.compiler.should have_relationship(['File', 'a', '->', 'File', 'x']) scope.compiler.should have_relationship(['File', 'b', '->', 'File', 'x']) scope.compiler.should have_relationship(['File', 'a', '->', 'File', 'y']) scope.compiler.should have_relationship(['File', 'b', '->', 'File', 'y']) end it 'should tolerate (eliminate) duplicates in operands' do source = "[File[a], File[a]] -> File[x]" parser.evaluate_string(scope, source, __FILE__) scope.compiler.should have_relationship(['File', 'a', '->', 'File', 'x']) scope.compiler.relationships.size.should == 1 end it 'should form a relation with <-' do source = "File[a] <- File[b]" parser.evaluate_string(scope, source, __FILE__) scope.compiler.should have_relationship(['File', 'b', '->', 'File', 'a']) end it 'should form a relation with <-' do source = "File[a] <~ File[b]" parser.evaluate_string(scope, source, __FILE__) scope.compiler.should have_relationship(['File', 'b', '~>', 'File', 'a']) end end context "When evaluating heredoc" do it "evaluates plain heredoc" do src = "@(END)\nThis is\nheredoc text\nEND\n" parser.evaluate_string(scope, src).should == "This is\nheredoc text\n" end it "parses heredoc with margin" do src = [ "@(END)", " This is", " heredoc text", " | END", "" ].join("\n") parser.evaluate_string(scope, src).should == "This is\nheredoc text\n" end it "parses heredoc with margin and right newline trim" do src = [ "@(END)", " This is", " heredoc text", " |- END", "" ].join("\n") parser.evaluate_string(scope, src).should == "This is\nheredoc text" end it "parses escape specification" do src = <<-CODE @(END/t) Tex\\tt\\n |- END CODE parser.evaluate_string(scope, src).should == "Tex\tt\\n" end it "parses syntax checked specification" do src = <<-CODE @(END:json) ["foo", "bar"] |- END CODE parser.evaluate_string(scope, src).should == '["foo", "bar"]' end it "parses syntax checked specification with error and reports it" do src = <<-CODE @(END:json) ['foo', "bar"] |- END CODE expect { parser.evaluate_string(scope, src)}.to raise_error(/Cannot parse invalid JSON string/) end it "parses interpolated heredoc expression" do src = <<-CODE $name = 'Fjodor' @("END") Hello $name |- END CODE parser.evaluate_string(scope, src).should == "Hello Fjodor" end + it "parses interpolated heredoc expression with escapes" do + src = <<-CODE + $name = 'Fjodor' + @("END") + Hello\\ \\$name + |- END + CODE + parser.evaluate_string(scope, src).should == "Hello\\ \\Fjodor" + end + end context "Handles Deprecations and Discontinuations" do it 'of import statements' do source = "\nimport foo" # Error references position 5 at the opening '{' # Set file to nil to make it easier to match with line number (no file name in output) expect { parser.evaluate_string(scope, source) }.to raise_error(/'import' has been discontinued.*line 2:1/) end end context "Detailed Error messages are reported" do it 'for illegal type references' do source = '1+1 { "title": }' # Error references position 5 at the opening '{' # Set file to nil to make it easier to match with line number (no file name in output) expect { parser.evaluate_string(scope, source) }.to raise_error( /Illegal Resource Type expression, expected result to be a type name, or untitled Resource.*line 1:2/) end it 'for non r-value producing <| |>' do expect { parser.parse_string("$a = File <| |>", nil) }.to raise_error(/A Virtual Query does not produce a value at line 1:6/) end it 'for non r-value producing <<| |>>' do expect { parser.parse_string("$a = File <<| |>>", nil) }.to raise_error(/An Exported Query does not produce a value at line 1:6/) end it 'for non r-value producing define' do Puppet::Util::Log.expects(:create).with(has_entries(:level => :err, :message => "Invalid use of expression. A 'define' expression does not produce a value", :line => 1, :pos => 6)) Puppet::Util::Log.expects(:create).with(has_entries(:level => :err, :message => 'Classes, definitions, and nodes may only appear at toplevel or inside other classes', :line => 1, :pos => 6)) expect { parser.parse_string("$a = define foo { }", nil) }.to raise_error(/2 errors/) end it 'for non r-value producing class' do Puppet::Util::Log.expects(:create).with(has_entries(:level => :err, :message => 'Invalid use of expression. A Host Class Definition does not produce a value', :line => 1, :pos => 6)) Puppet::Util::Log.expects(:create).with(has_entries(:level => :err, :message => 'Classes, definitions, and nodes may only appear at toplevel or inside other classes', :line => 1, :pos => 6)) expect { parser.parse_string("$a = class foo { }", nil) }.to raise_error(/2 errors/) end it 'for unclosed quote with indication of start position of string' do source = <<-SOURCE.gsub(/^ {6}/,'') $a = "xx yyy SOURCE # first char after opening " reported as being in error. expect { parser.parse_string(source) }.to raise_error(/Unclosed quote after '"' followed by 'xx\\nyy\.\.\.' at line 1:7/) end it 'for multiple errors with a summary exception' do Puppet::Util::Log.expects(:create).with(has_entries(:level => :err, :message => 'Invalid use of expression. A Node Definition does not produce a value', :line => 1, :pos => 6)) Puppet::Util::Log.expects(:create).with(has_entries(:level => :err, :message => 'Classes, definitions, and nodes may only appear at toplevel or inside other classes', :line => 1, :pos => 6)) expect { parser.parse_string("$a = node x { }",nil) }.to raise_error(/2 errors/) end it 'for a bad hostname' do expect { parser.parse_string("node 'macbook+owned+by+name' { }", nil) }.to raise_error(/The hostname 'macbook\+owned\+by\+name' contains illegal characters.*at line 1:6/) end it 'for a hostname with interpolation' do source = <<-SOURCE.gsub(/^ {6}/,'') $name = 'fred' node "macbook-owned-by$name" { } SOURCE expect { parser.parse_string(source, nil) }.to raise_error(/An interpolated expression is not allowed in a hostname of a node at line 2:23/) end end context 'does not leak variables' do it 'local variables are gone when lambda ends' do source = <<-SOURCE [1,2,3].each |$x| { $y = $x} $a = $y SOURCE expect do parser.evaluate_string(scope, source) end.to raise_error(/Unknown variable: 'y'/) end it 'lambda parameters are gone when lambda ends' do source = <<-SOURCE [1,2,3].each |$x| { $y = $x} $a = $x SOURCE expect do parser.evaluate_string(scope, source) end.to raise_error(/Unknown variable: 'x'/) end it 'does not leak match variables' do source = <<-SOURCE if 'xyz' =~ /(x)(y)(z)/ { notice $2 } case 'abc' { /(a)(b)(c)/ : { $x = $2 } } "-$x-$2-" SOURCE expect(parser.evaluate_string(scope, source)).to eq('-b--') end end matcher :have_relationship do |expected| calc = Puppet::Pops::Types::TypeCalculator.new match do |compiler| op_name = {'->' => :relationship, '~>' => :subscription} compiler.relationships.any? do | relation | relation.source.type == expected[0] && relation.source.title == expected[1] && relation.type == op_name[expected[2]] && relation.target.type == expected[3] && relation.target.title == expected[4] end end failure_message_for_should do |actual| "Relationship #{expected[0]}[#{expected[1]}] #{expected[2]} #{expected[3]}[#{expected[4]}] but was unknown to compiler" end end end diff --git a/spec/unit/pops/parser/parse_heredoc_spec.rb b/spec/unit/pops/parser/parse_heredoc_spec.rb index 9ad9de04c..96709d2b9 100644 --- a/spec/unit/pops/parser/parse_heredoc_spec.rb +++ b/spec/unit/pops/parser/parse_heredoc_spec.rb @@ -1,138 +1,164 @@ require 'spec_helper' require 'puppet/pops' # relative to this spec file (./) does not work as this file is loaded by rspec require File.join(File.dirname(__FILE__), '/parser_rspec_helper') describe "egrammar parsing heredoc" do include ParserRspecHelper it "parses plain heredoc" do dump(parse("@(END)\nThis is\nheredoc text\nEND\n")).should == [ "(@()", " (sublocated 'This is\nheredoc text\n')", ")" ].join("\n") end it "parses heredoc with margin" do src = [ "@(END)", " This is", " heredoc text", " | END", "" ].join("\n") dump(parse(src)).should == [ "(@()", " (sublocated 'This is\nheredoc text\n')", ")" ].join("\n") end it "parses heredoc with margin and right newline trim" do src = [ "@(END)", " This is", " heredoc text", " |- END", "" ].join("\n") dump(parse(src)).should == [ "(@()", " (sublocated 'This is\nheredoc text')", ")" ].join("\n") end it "parses syntax and escape specification" do src = <<-CODE @(END:syntax/t) Tex\\tt\\n |- END CODE dump(parse(src)).should == [ "(@(syntax)", " (sublocated 'Tex\tt\\n')", ")" ].join("\n") end it "parses interpolated heredoc expression" do src = <<-CODE @("END") Hello $name |- END CODE dump(parse(src)).should == [ "(@()", " (sublocated (cat 'Hello ' (str $name) ''))", ")" ].join("\n") end + it "parses interpolated heredoc expression containing escapes" do + src = <<-CODE + @("END") + Hello \\$name + |- END + CODE + dump(parse(src)).should == [ + "(@()", + " (sublocated (cat 'Hello \\' (str $name) ''))", + ")" + ].join("\n") + end + + it "parses interpolated heredoc expression containing escapes when escaping other things than $" do + src = <<-CODE + @("END"/t) + Hello \\$name + |- END + CODE + dump(parse(src)).should == [ + "(@()", + " (sublocated (cat 'Hello \\' (str $name) ''))", + ")" + ].join("\n") + end + it "parses with escaped newlines without preceding whitespace" do src = <<-CODE @(END/L) First Line\\ Second Line |- END CODE parse(src) dump(parse(src)).should == [ "(@()", " (sublocated 'First Line Second Line')", ")" ].join("\n") end it "parses with escaped newlines with proper margin" do src = <<-CODE @(END/L) First Line\\ Second Line |- END CODE parse(src) dump(parse(src)).should == [ "(@()", " (sublocated ' First Line Second Line')", ")" ].join("\n") end it "parses interpolated heredoc expression with false start on $" do src = <<-CODE @("END") Hello $name$%a |- END CODE dump(parse(src)).should == [ "(@()", " (sublocated (cat 'Hello ' (str $name) '$%a'))", ")" ].join("\n") end it 'parses multiple heredocs on the same line' do src = <<-CODE notice({ @(foo) => @(bar) }) hello -foo world -bar notice '!' CODE expect(dump(parse(src))).to eq([ '(block', ' (invoke notice ({} ((@()', ' (sublocated \' hello\')', ' ) (@()', ' (sublocated \' world\')', ' ))))', ' (invoke notice \'!\')', ')' ].join("\n")) end end