diff --git a/lib/puppet/functions.rb b/lib/puppet/functions.rb index b31583784..fbab0e787 100644 --- a/lib/puppet/functions.rb +++ b/lib/puppet/functions.rb @@ -1,542 +1,556 @@ # Functions in the puppet language can be written in Ruby and distributed in # puppet modules. The function is written by creating a file in the module's # `lib/puppet/functions/` directory, where `` is # replaced with the module's name. The file should have the name of the function. # For example, to create a function named `min` in a module named `math` create # a file named `lib/puppet/functions/math/min.rb` in the module. # # A function is implemented by calling {Puppet::Functions.create_function}, and # passing it a block that defines the implementation of the function. # # Functions are namespaced inside the module that contains them. The name of # the function is prefixed with the name of the module. For example, # `math::min`. # # @example A simple function # Puppet::Functions.create_function('math::min') do # def min(a, b) # a <= b ? a : b # end # end # # Anatomy of a function # --- # # Functions are composed of four parts: the name, the implementation methods, # the signatures, and the dispatches. # # The name is the string given to the {Puppet::Functions.create_function} # method. It specifies the name to use when calling the function in the puppet # language, or from other functions. # # The implementation methods are ruby methods (there can be one or more) that # provide that actual implementation of the function's behavior. In the # simplest case the name of the function (excluding any namespace) and the name # of the method are the same. When that is done no other parts (signatures and # dispatches) need to be used. # # Signatures are a way of specifying the types of the function's parameters. # The types of any arguments will be checked against the types declared in the # signature and an error will be produced if they don't match. The types are # defined by using the same syntax for types as in the puppet language. # # Dispatches are how signatures and implementation methods are tied together. # When the function is called, puppet searches the signatures for one that # matches the supplied arguments. Each signature is part of a dispatch, which # specifies the method that should be called for that signature. When a # matching signature is found, the corrosponding method is called. # # Documentation for the function should be placed as comments to the # implementation method(s). # # @todo Documentation for individual instances of these new functions is not # yet tied into the puppet doc system. # # @example Dispatching to different methods by type # Puppet::Functions.create_function('math::min') do # dispatch :numeric_min do # param 'Numeric', 'a' # param 'Numeric', 'b' # end # # dispatch :string_min do # param 'String', 'a' # param 'String', 'b' # end # # def numeric_min(a, b) # a <= b ? a : b # end # # def string_min(a, b) # a.downcase <= b.downcase ? a : b # end # end # # Specifying Signatures # --- # # If nothing is specified, the number of arguments given to the function must # be the same as the number of parameters, and all of the parameters are of # type 'Any'. # # To express that the last parameter captures the rest, the method # `last_captures_rest` can be called. This indicates that the last parameter is # a varargs parameter and will be passed to the implementing method as an array # of the given type. # # When defining a dispatch for a function, the resulting dispatch matches # against the specified argument types and min/max occurrence of optional # entries. When the dispatch makes the call to the implementation method the # arguments are simply passed and it is the responsibility of the method's # implementor to ensure it can handle those arguments (i.e. there is no check # that what was declared as optional actually has a default value, and that # a "captures rest" is declared using a `*`). # # @example Varargs # Puppet::Functions.create_function('foo') do # dispatch :foo do # param 'Numeric', 'first' # param 'Numeric', 'values' # last_captures_rest # end # # def foo(first, *values) # # do something # end # end # # Access to Scope # --- # In general, functions should not need access to scope; they should be # written to act on their given input only. If they absolutely must look up # variable values, they should do so via the closure scope (the scope where # they are defined) - this is done by calling `closure_scope()`. # # Calling other Functions # --- # Calling other functions by name is directly supported via # {Puppet::Pops::Functions::Function#call_function}. This allows a function to # call other functions visible from its loader. # # @api public module Puppet::Functions # @param func_name [String, Symbol] a simple or qualified function name # @param block [Proc] the block that defines the methods and dispatch of the # Function to create # @return [Class] the newly created Function class # # @api public def self.create_function(func_name, function_base = Function, &block) if function_base.ancestors.none? { |s| s == Puppet::Pops::Functions::Function } raise ArgumentError, "Functions must be based on Puppet::Pops::Functions::Function. Got #{function_base}" end func_name = func_name.to_s # Creates an anonymous class to represent the function # The idea being that it is garbage collected when there are no more # references to it. # the_class = Class.new(function_base, &block) # Make the anonymous class appear to have the class-name # Even if this class is not bound to such a symbol in a global ruby scope and # must be resolved via the loader. # This also overrides any attempt to define a name method in the given block # (Since it redefines it) # # TODO, enforce name in lower case (to further make it stand out since Ruby # class names are upper case) # the_class.instance_eval do @func_name = func_name def name @func_name end end # Automatically create an object dispatcher based on introspection if the # loaded user code did not define any dispatchers. Fail if function name # does not match a given method name in user code. # if the_class.dispatcher.empty? simple_name = func_name.split(/::/)[-1] type, names = default_dispatcher(the_class, simple_name) last_captures_rest = (type.size_range[1] == Float::INFINITY) the_class.dispatcher.add_dispatch(type, simple_name, names, nil, nil, nil, last_captures_rest) end # The function class is returned as the result of the create function method the_class end # Creates a default dispatcher configured from a method with the same name as the function # # @api private def self.default_dispatcher(the_class, func_name) unless the_class.method_defined?(func_name) raise ArgumentError, "Function Creation Error, cannot create a default dispatcher for function '#{func_name}', no method with this name found" end any_signature(*min_max_param(the_class.instance_method(func_name))) end # @api private def self.min_max_param(method) result = {:req => 0, :opt => 0, :rest => 0 } # count per parameter kind, and get array of names names = method.parameters.map { |p| result[p[0]] += 1 ; p[1].to_s } from = result[:req] to = result[:rest] > 0 ? :default : from + result[:opt] [from, to, names] end # Construct a signature consisting of Object type, with min, and max, and given names. # (there is only one type entry). # # @api private def self.any_signature(from, to, names) # Construct the type for the signature # Tuple[Object, from, to] factory = Puppet::Pops::Types::TypeFactory [factory.callable(factory.any, from, to), names] end # Function # === # This class is the base class for all Puppet 4x Function API functions. A # specialized class is created for each puppet function. # # @api public class Function < Puppet::Pops::Functions::Function # @api private def self.builder @type_parser ||= Puppet::Pops::Types::TypeParser.new @all_callables ||= Puppet::Pops::Types::TypeFactory.all_callables DispatcherBuilder.new(dispatcher, @type_parser, @all_callables) end # Dispatch any calls that match the signature to the provided method name. # # @param meth_name [Symbol] The name of the implementation method to call # when the signature defined in the block matches the arguments to a call # to the function. # @return [Void] # # @api public def self.dispatch(meth_name, &block) builder().instance_eval do dispatch(meth_name, &block) end end end # Public api methods of the DispatcherBuilder are available within dispatch() # blocks declared in a Puppet::Function.create_function() call. # # @api public class DispatcherBuilder # @api private def initialize(dispatcher, type_parser, all_callables) @type_parser = type_parser @all_callables = all_callables @dispatcher = dispatcher end # Defines a positional parameter with type and name # # @param type [String] The type specification for the parameter. # @param name [Symbol] The name of the parameter. This is primarily used # for error message output and does not have to match the name of the # parameter on the implementation method. # @return [Void] # # @api public def param(type, name) unless type.is_a?(String) raise ArgumentError, "Type signature argument must be a String reference to a Puppet Data Type. Got #{type.class}" end unless name.is_a?(Symbol) raise ArgumentError, "Parameter name argument must be a Symbol. Got #{type.class}" end @types << type @names << name # mark what should be picked for this position when dispatching @weaving << @names.size()-1 end # Defines one required block parameter that may appear last. If type and name is missing the # default type is "Callable", and the name is "block". If only one # parameter is given, then that is the name and the type is "Callable". # # @api public def required_block_param(*type_and_name) case type_and_name.size when 0 # the type must be an independent instance since it will be contained in another type type = @all_callables.copy name = :block when 1 # the type must be an independent instance since it will be contained in another type type = @all_callables.copy name = type_and_name[0] when 2 type_string, name = type_and_name type = @type_parser.parse(type_string) else raise ArgumentError, "block_param accepts max 2 arguments (type, name), got #{type_and_name.size}." end unless Puppet::Pops::Types::TypeCalculator.is_kind_of_callable?(type, false) raise ArgumentError, "Expected PCallableType or PVariantType thereof, got #{type.class}" end unless name.is_a?(Symbol) raise ArgumentError, "Expected block_param name to be a Symbol, got #{name.class}" end if @block_type.nil? @block_type = type @block_name = name else raise ArgumentError, "Attempt to redefine block" end end # Defines one optional block parameter that may appear last. If type or name is missing the # defaults are "any callable", and the name is "block". The implementor of the dispatch target # must use block = nil when it is optional (or an error is raised when the call is made). # # @api public def optional_block_param(*type_and_name) # same as required, only wrap the result in an optional type required_block_param(*type_and_name) @block_type = Puppet::Pops::Types::TypeFactory.optional(@block_type) end # Specifies the min and max occurrence of arguments (of the specified types) # if something other than the exact count from the number of specified # types). The max value may be specified as :default if an infinite number of # arguments are supported. When max is > than the number of specified # types, the last specified type repeats. # # @api public def arg_count(min_occurs, max_occurs) @min = min_occurs @max = max_occurs unless min_occurs.is_a?(Integer) && min_occurs >= 0 raise ArgumentError, "min arg_count of function parameter must be an Integer >=0, got #{min_occurs.class} '#{min_occurs}'" end unless max_occurs == :default || (max_occurs.is_a?(Integer) && max_occurs >= 0) raise ArgumentError, "max arg_count of function parameter must be an Integer >= 0, or :default, got #{max_occurs.class} '#{max_occurs}'" end unless max_occurs == :default || (max_occurs.is_a?(Integer) && max_occurs >= min_occurs) raise ArgumentError, "max arg_count must be :default (infinite) or >= min arg_count, got min: '#{min_occurs}, max: '#{max_occurs}'" end end # Specifies that the last argument captures the rest. # # @api public def last_captures_rest @last_captures = true end private # @api private def dispatch(meth_name, &block) # an array of either an index into names/types, or an array with # injection information [type, name, injection_name] used when the call # is being made to weave injections into the given arguments. # @types = [] @names = [] @weaving = [] @injections = [] @min = nil @max = nil @last_captures = false @block_type = nil @block_name = nil self.instance_eval &block callable_t = create_callable(@types, @block_type, @min, @max) @dispatcher.add_dispatch(callable_t, meth_name, @names, @block_name, @injections, @weaving, @last_captures) end # Handles creation of a callable type from strings specifications of puppet # types and allows the min/max occurs of the given types to be given as one # or two integer values at the end. The given block_type should be # Optional[Callable], Callable, or nil. # # @api private def create_callable(types, block_type, from, to) mapped_types = types.map do |t| @type_parser.parse(t) end if !(from.nil? && to.nil?) mapped_types << from mapped_types << to end if block_type mapped_types << block_type end Puppet::Pops::Types::TypeFactory.callable(*mapped_types) end end private # @note WARNING: This style of creating functions is not public. It is a system # under development that will be used for creating "system" functions. # # This is a private, internal, system for creating functions. It supports # everything that the public function definition system supports as well as a # few extra features. # # Injection Support # === # The Function API supports injection of data and services. It is possible to # make injection that takes effect when the function is loaded (for services # and runtime configuration that does not change depending on how/from where # in what context the function is called. It is also possible to inject and # weave argument values into a call. # # Injection of attributes # --- # Injection of attributes is performed by one of the methods `attr_injected`, # and `attr_injected_producer`. The injected attributes are available via # accessor method calls. # # @example using injected attributes # Puppet::Functions.create_function('test') do # attr_injected String, :larger, 'message_larger' # attr_injected String, :smaller, 'message_smaller' # def test(a, b) # a > b ? larger() : smaller() # end # end # # @api private class InternalFunction < Function # @api private def self.builder @type_parser ||= Puppet::Pops::Types::TypeParser.new @all_callables ||= Puppet::Pops::Types::TypeFactory.all_callables InternalDispatchBuilder.new(dispatcher, @type_parser, @all_callables) end # Defines class level injected attribute with reader method # # @api private def self.attr_injected(type, attribute_name, injection_name = nil) define_method(attribute_name) do ivar = :"@#{attribute_name.to_s}" unless instance_variable_defined?(ivar) injector = Puppet.lookup(:injector) instance_variable_set(ivar, injector.lookup(closure_scope, type, injection_name)) end instance_variable_get(ivar) end end # Defines class level injected producer attribute with reader method # # @api private def self.attr_injected_producer(type, attribute_name, injection_name = nil) define_method(attribute_name) do ivar = :"@#{attribute_name.to_s}" unless instance_variable_defined?(ivar) injector = Puppet.lookup(:injector) instance_variable_set(ivar, injector.lookup_producer(closure_scope, type, injection_name)) end instance_variable_get(ivar) end end + + # Allows the implementation of a function to call other functions by name and pass the caller + # scope. The callable functions are those visible to the same loader that loaded this function + # (the calling function). + # + # @param scope [Puppet::Parser::Scope] The caller scope + # @param function_name [String] The name of the function + # @param *args [Object] splat of arguments + # @return [Object] The result returned by the called function + # + # @api public + def call_function_with_scope(scope, function_name, *args) + internal_call_function(scope, function_name, args) + end end # @note WARNING: This style of creating functions is not public. It is a system # under development that will be used for creating "system" functions. # # Injection and Weaving of parameters # --- # It is possible to inject and weave parameters into a call. These extra # parameters are not part of the parameters passed from the Puppet logic, and # they can not be overridden by parameters given as arguments in the call. # They are invisible to the Puppet Language. # # @example using injected parameters # Puppet::Functions.create_function('test') do # dispatch :test do # param 'Scalar', 'a' # param 'Scalar', 'b' # injected_param 'String', 'larger', 'message_larger' # injected_param 'String', 'smaller', 'message_smaller' # end # def test(a, b, larger, smaller) # a > b ? larger : smaller # end # end # # The function in the example above is called like this: # # test(10, 20) # # Using injected value as default # --- # Default value assignment is handled by using the regular Ruby mechanism (a # value is assigned to the variable). The dispatch simply indicates that the # value is optional. If the default value should be injected, it can be # handled different ways depending on what is desired: # # * by calling the accessor method for an injected Function class attribute. # This is suitable if the value is constant across all instantiations of the # function, and across all calls. # * by injecting a parameter into the call # to the left of the parameter, and then assigning that as the default value. # * One of the above forms, but using an injected producer instead of a # directly injected value. # # @example method with injected default values # Puppet::Functions.create_function('test') do # dispatch :test do # injected_param String, 'b_default', 'b_default_value_key' # param 'Scalar', 'a' # param 'Scalar', 'b' # end # def test(b_default, a, b = b_default) # # ... # end # end # # @api private class InternalDispatchBuilder < DispatcherBuilder def scope_param() @injections << [:scope, 'scope', '', :dispatcher_internal] # mark what should be picked for this position when dispatching @weaving << [@injections.size()-1] end # TODO: is param name really needed? Perhaps for error messages? (it is unused now) # # @api private def injected_param(type, name, injection_name = '') @injections << [type, name, injection_name] # mark what should be picked for this position when dispatching @weaving << [@injections.size() -1] end # TODO: is param name really needed? Perhaps for error messages? (it is unused now) # # @api private def injected_producer_param(type, name, injection_name = '') @injections << [type, name, injection_name, :producer] # mark what should be picked for this position when dispatching @weaving << [@injections.size()-1] end end end diff --git a/lib/puppet/pops.rb b/lib/puppet/pops.rb index 6755715b4..a200ec218 100644 --- a/lib/puppet/pops.rb +++ b/lib/puppet/pops.rb @@ -1,126 +1,127 @@ module Puppet # The Pops language system. This includes the parser, evaluator, AST model, and # Binder. # # @todo Explain how a user should use this to parse and evaluate the puppet # language. # # @note Warning: Pops is still considered experimental, as such the API may # change at any time. # # @api public module Pops require 'puppet/pops/patterns' require 'puppet/pops/utils' require 'puppet/pops/adaptable' require 'puppet/pops/adapters' require 'puppet/pops/visitable' require 'puppet/pops/visitor' require 'puppet/pops/containment' require 'puppet/pops/issues' require 'puppet/pops/semantic_error' require 'puppet/pops/label_provider' require 'puppet/pops/validation' require 'puppet/pops/issue_reporter' require 'puppet/pops/model/model' # (the Types module initializes itself) require 'puppet/pops/types/types' require 'puppet/pops/types/type_calculator' require 'puppet/pops/types/type_factory' require 'puppet/pops/types/type_parser' require 'puppet/pops/types/class_loader' require 'puppet/pops/types/enumeration' module Model require 'puppet/pops/model/tree_dumper' require 'puppet/pops/model/ast_transformer' require 'puppet/pops/model/factory' require 'puppet/pops/model/model_tree_dumper' require 'puppet/pops/model/model_label_provider' end module Binder module SchemeHandler # the handlers are auto loaded via bindings end module Producers require 'puppet/pops/binder/producers' end require 'puppet/pops/binder/binder' require 'puppet/pops/binder/bindings_model' require 'puppet/pops/binder/binder_issues' require 'puppet/pops/binder/bindings_checker' require 'puppet/pops/binder/bindings_factory' require 'puppet/pops/binder/bindings_label_provider' require 'puppet/pops/binder/bindings_validator_factory' require 'puppet/pops/binder/injector_entry' require 'puppet/pops/binder/key_factory' require 'puppet/pops/binder/injector' require 'puppet/pops/binder/bindings_composer' require 'puppet/pops/binder/bindings_model_dumper' require 'puppet/pops/binder/system_bindings' require 'puppet/pops/binder/bindings_loader' require 'puppet/pops/binder/lookup' module Config require 'puppet/pops/binder/config/binder_config' require 'puppet/pops/binder/config/binder_config_checker' require 'puppet/pops/binder/config/issues' require 'puppet/pops/binder/config/diagnostic_producer' end end module Parser require 'puppet/pops/parser/eparser' require 'puppet/pops/parser/parser_support' require 'puppet/pops/parser/locator' require 'puppet/pops/parser/locatable' require 'puppet/pops/parser/lexer2' require 'puppet/pops/parser/evaluating_parser' require 'puppet/pops/parser/epp_parser' require 'puppet/pops/parser/code_merger' end module Validation require 'puppet/pops/validation/checker4_0' require 'puppet/pops/validation/validator_factory_4_0' end module Evaluator require 'puppet/pops/evaluator/callable_signature' + require 'puppet/pops/evaluator/runtime3_converter' require 'puppet/pops/evaluator/runtime3_support' require 'puppet/pops/evaluator/evaluator_impl' require 'puppet/pops/evaluator/epp_evaluator' require 'puppet/pops/evaluator/callable_mismatch_describer' require 'puppet/pops/evaluator/collector_transformer' module Collectors require 'puppet/pops/evaluator/collectors/abstract_collector' require 'puppet/pops/evaluator/collectors/fixed_set_collector' require 'puppet/pops/evaluator/collectors/catalog_collector' require 'puppet/pops/evaluator/collectors/exported_collector' end end # Subsystem for puppet functions defined in ruby. # # @api public module Functions require 'puppet/pops/functions/function' require 'puppet/pops/functions/dispatch' require 'puppet/pops/functions/dispatcher' end end require 'puppet/parser/ast/pops_bridge' require 'puppet/bindings' require 'puppet/functions' end require 'puppet/plugins/data_providers' diff --git a/lib/puppet/pops/evaluator/evaluator_impl.rb b/lib/puppet/pops/evaluator/evaluator_impl.rb index 3a9c9cda5..1fe4d759b 100644 --- a/lib/puppet/pops/evaluator/evaluator_impl.rb +++ b/lib/puppet/pops/evaluator/evaluator_impl.rb @@ -1,1097 +1,1094 @@ require 'rgen/ecore/ecore' require 'puppet/pops/evaluator/compare_operator' require 'puppet/pops/evaluator/relationship_operator' require 'puppet/pops/evaluator/access_operator' require 'puppet/pops/evaluator/closure' require 'puppet/pops/evaluator/external_syntax_support' # This implementation of {Puppet::Pops::Evaluator} performs evaluation using the puppet 3.x runtime system # in a manner largely compatible with Puppet 3.x, but adds new features and introduces constraints. # # The evaluation uses _polymorphic dispatch_ which works by dispatching to the first found method named after # the class or one of its super-classes. The EvaluatorImpl itself mainly deals with evaluation (it currently # also handles assignment), and it uses a delegation pattern to more specialized handlers of some operators # that in turn use polymorphic dispatch; this to not clutter EvaluatorImpl with too much responsibility). # # Since a pattern is used, only the main entry points are fully documented. The parameters _o_ and _scope_ are # the same in all the polymorphic methods, (the type of the parameter _o_ is reflected in the method's name; # either the actual class, or one of its super classes). The _scope_ parameter is always the scope in which # the evaluation takes place. If nothing else is mentioned, the return is always the result of evaluation. # # See {Puppet::Pops::Visitable} and {Puppet::Pops::Visitor} for more information about # polymorphic calling. # class Puppet::Pops::Evaluator::EvaluatorImpl include Puppet::Pops::Utils # Provides access to the Puppet 3.x runtime (scope, etc.) # This separation has been made to make it easier to later migrate the evaluator to an improved runtime. # include Puppet::Pops::Evaluator::Runtime3Support include Puppet::Pops::Evaluator::ExternalSyntaxSupport EMPTY_STRING = ''.freeze COMMA_SEPARATOR = ', '.freeze # Reference to Issues name space makes it easier to refer to issues # (Issues are shared with the validator). # Issues = Puppet::Pops::Issues def initialize @@eval_visitor ||= Puppet::Pops::Visitor.new(self, "eval", 1, 1) @@lvalue_visitor ||= Puppet::Pops::Visitor.new(self, "lvalue", 1, 1) @@assign_visitor ||= Puppet::Pops::Visitor.new(self, "assign", 3, 3) @@string_visitor ||= Puppet::Pops::Visitor.new(self, "string", 1, 1) @@type_calculator ||= Puppet::Pops::Types::TypeCalculator.new() @@type_parser ||= Puppet::Pops::Types::TypeParser.new() @@compare_operator ||= Puppet::Pops::Evaluator::CompareOperator.new() @@relationship_operator ||= Puppet::Pops::Evaluator::RelationshipOperator.new() - - # Initialize the runtime module - Puppet::Pops::Evaluator::Runtime3Support.instance_method(:initialize).bind(self).call() end # @api private def type_calculator @@type_calculator end # Evaluates the given _target_ object in the given scope. # # @overload evaluate(target, scope) # @param target [Object] evaluation target - see methods on the pattern assign_TYPE for actual supported types. # @param scope [Object] the runtime specific scope class where evaluation should take place # @return [Object] the result of the evaluation # # @api public # def evaluate(target, scope) begin @@eval_visitor.visit_this_1(self, target, scope) rescue Puppet::Pops::SemanticError => e # a raised issue may not know the semantic target fail(e.issue, e.semantic || target, e.options, e) rescue StandardError => e if e.is_a? Puppet::ParseError # ParseError's are supposed to be fully configured with location information raise e end fail(Issues::RUNTIME_ERROR, target, {:detail => e.message}, e) end end # Assigns the given _value_ to the given _target_. The additional argument _o_ is the instruction that # produced the target/value tuple and it is used to set the origin of the result. # # @param target [Object] assignment target - see methods on the pattern assign_TYPE for actual supported types. # @param value [Object] the value to assign to `target` # @param o [Puppet::Pops::Model::PopsObject] originating instruction # @param scope [Object] the runtime specific scope where evaluation should take place # # @api private # def assign(target, value, o, scope) @@assign_visitor.visit_this_3(self, target, value, o, scope) end # Computes a value that can be used as the LHS in an assignment. # @param o [Object] the expression to evaluate as a left (assignable) entity # @param scope [Object] the runtime specific scope where evaluation should take place # # @api private # def lvalue(o, scope) @@lvalue_visitor.visit_this_1(self, o, scope) end # Produces a String representation of the given object _o_ as used in interpolation. # @param o [Object] the expression of which a string representation is wanted # @param scope [Object] the runtime specific scope where evaluation should take place # # @api public # def string(o, scope) @@string_visitor.visit_this_1(self, o, scope) end # Evaluate a BlockExpression in a new scope with variables bound to the # given values. # # @param scope [Puppet::Parser::Scope] the parent scope # @param variable_bindings [Hash{String => Object}] the variable names and values to bind (names are keys, bound values are values) # @param block [Puppet::Pops::Model::BlockExpression] the sequence of expressions to evaluate in the new scope # # @api private # def evaluate_block_with_bindings(scope, variable_bindings, block_expr) with_guarded_scope(scope) do # change to create local scope_from - cannot give it file and line - # that is the place of the call, not "here" create_local_scope_from(variable_bindings, scope) evaluate(block_expr, scope) end end protected def lvalue_VariableExpression(o, scope) # evaluate the name evaluate(o.expr, scope) end # Catches all illegal lvalues # def lvalue_Object(o, scope) fail(Issues::ILLEGAL_ASSIGNMENT, o) end # Assign value to named variable. # The '$' sign is never part of the name. # @example In Puppet DSL # $name = value # @param name [String] name of variable without $ # @param value [Object] value to assign to the variable # @param o [Puppet::Pops::Model::PopsObject] originating instruction # @param scope [Object] the runtime specific scope where evaluation should take place # @return [value] # def assign_String(name, value, o, scope) if name =~ /::/ fail(Issues::CROSS_SCOPE_ASSIGNMENT, o.left_expr, {:name => name}) end set_variable(name, value, o, scope) value end def assign_Numeric(n, value, o, scope) fail(Issues::ILLEGAL_NUMERIC_ASSIGNMENT, o.left_expr, {:varname => n.to_s}) end # Catches all illegal assignment (e.g. 1 = 2, {'a'=>1} = 2, etc) # def assign_Object(name, value, o, scope) fail(Issues::ILLEGAL_ASSIGNMENT, o) end def eval_Factory(o, scope) evaluate(o.current, scope) end # Evaluates any object not evaluated to something else to itself. def eval_Object o, scope o end # Allows nil to be used as a Nop, Evaluates to nil def eval_NilClass(o, scope) nil end # Evaluates Nop to nil. def eval_Nop(o, scope) nil end # Captures all LiteralValues not handled elsewhere. # def eval_LiteralValue(o, scope) o.value end # Reserved Words fail to evaluate # def eval_ReservedWord(o, scope) fail(Puppet::Pops::Issues::RESERVED_WORD, o, {:word => o.word}) end def eval_LiteralDefault(o, scope) :default end def eval_LiteralUndef(o, scope) nil end # A QualifiedReference (i.e. a capitalized qualified name such as Foo, or Foo::Bar) evaluates to a PType # def eval_QualifiedReference(o, scope) @@type_parser.interpret(o) end def eval_NotExpression(o, scope) ! is_true?(evaluate(o.expr, scope)) end def eval_UnaryMinusExpression(o, scope) - coerce_numeric(evaluate(o.expr, scope), o, scope) end def eval_UnfoldExpression(o, scope) candidate = evaluate(o.expr, scope) case candidate when Array candidate when Hash candidate.to_a else # turns anything else into an array (so result can be unfolded) [candidate] end end # Abstract evaluation, returns array [left, right] with the evaluated result of left_expr and # right_expr # @return > array with result of evaluating left and right expressions # def eval_BinaryExpression o, scope [ evaluate(o.left_expr, scope), evaluate(o.right_expr, scope) ] end # Evaluates assignment with operators =, +=, -= and # # @example Puppet DSL # $a = 1 # $a += 1 # $a -= 1 # def eval_AssignmentExpression(o, scope) name = lvalue(o.left_expr, scope) value = evaluate(o.right_expr, scope) if o.operator == :'=' assign(name, value, o, scope) else fail(Issues::UNSUPPORTED_OPERATOR, o, {:operator => o.operator}) end value end ARITHMETIC_OPERATORS = [:'+', :'-', :'*', :'/', :'%', :'<<', :'>>'] COLLECTION_OPERATORS = [:'+', :'-', :'<<'] # Handles binary expression where lhs and rhs are array/hash or numeric and operator is +, - , *, % / << >> # def eval_ArithmeticExpression(o, scope) left = evaluate(o.left_expr, scope) right = evaluate(o.right_expr, scope) begin result = calculate(left, right, o.operator, o.left_expr, o.right_expr, scope) rescue ArgumentError => e fail(Issues::RUNTIME_ERROR, o, {:detail => e.message}, e) end result end # Handles binary expression where lhs and rhs are array/hash or numeric and operator is +, - , *, % / << >> # def calculate(left, right, operator, left_o, right_o, scope) unless ARITHMETIC_OPERATORS.include?(operator) fail(Issues::UNSUPPORTED_OPERATOR, left_o.eContainer, {:operator => o.operator}) end if (left.is_a?(Array) || left.is_a?(Hash)) && COLLECTION_OPERATORS.include?(operator) # Handle operation on collections case operator when :'+' concatenate(left, right) when :'-' delete(left, right) when :'<<' unless left.is_a?(Array) fail(Issues::OPERATOR_NOT_APPLICABLE, left_o, {:operator => operator, :left_value => left}) end left + [right] end else # Handle operation on numeric left = coerce_numeric(left, left_o, scope) right = coerce_numeric(right, right_o, scope) begin if operator == :'%' && (left.is_a?(Float) || right.is_a?(Float)) # Deny users the fun of seeing severe rounding errors and confusing results fail(Issues::OPERATOR_NOT_APPLICABLE, left_o, {:operator => operator, :left_value => left}) end result = left.send(operator, right) rescue NoMethodError => e fail(Issues::OPERATOR_NOT_APPLICABLE, left_o, {:operator => operator, :left_value => left}) rescue ZeroDivisionError => e fail(Issues::DIV_BY_ZERO, right_o) end if result == Float::INFINITY || result == -Float::INFINITY fail(Issues::RESULT_IS_INFINITY, left_o, {:operator => operator}) end result end end def eval_EppExpression(o, scope) scope["@epp"] = [] evaluate(o.body, scope) result = scope["@epp"].join result end def eval_RenderStringExpression(o, scope) scope["@epp"] << o.value.dup nil end def eval_RenderExpression(o, scope) scope["@epp"] << string(evaluate(o.expr, scope), scope) nil end # Evaluates Puppet DSL ->, ~>, <-, and <~ def eval_RelationshipExpression(o, scope) # First level evaluation, reduction to basic data types or puppet types, the relationship operator then translates this # to the final set of references (turning strings into references, which can not naturally be done by the main evaluator since # all strings should not be turned into references. # real = eval_BinaryExpression(o, scope) @@relationship_operator.evaluate(real, o, scope) end # Evaluates x[key, key, ...] # def eval_AccessExpression(o, scope) left = evaluate(o.left_expr, scope) keys = o.keys.nil? ? [] : o.keys.collect {|key| evaluate(key, scope) } Puppet::Pops::Evaluator::AccessOperator.new(o).access(left, scope, *keys) end # Evaluates <, <=, >, >=, and == # def eval_ComparisonExpression o, scope left = evaluate(o.left_expr, scope) right = evaluate(o.right_expr, scope) begin # Left is a type if left.is_a?(Puppet::Pops::Types::PAnyType) case o.operator when :'==' @@type_calculator.equals(left,right) when :'!=' !@@type_calculator.equals(left,right) when :'<' # left can be assigned to right, but they are not equal @@type_calculator.assignable?(right, left) && ! @@type_calculator.equals(left,right) when :'<=' # left can be assigned to right @@type_calculator.assignable?(right, left) when :'>' # right can be assigned to left, but they are not equal @@type_calculator.assignable?(left,right) && ! @@type_calculator.equals(left,right) when :'>=' # right can be assigned to left @@type_calculator.assignable?(left, right) else fail(Issues::UNSUPPORTED_OPERATOR, o, {:operator => o.operator}) end else case o.operator when :'==' @@compare_operator.equals(left,right) when :'!=' ! @@compare_operator.equals(left,right) when :'<' @@compare_operator.compare(left,right) < 0 when :'<=' @@compare_operator.compare(left,right) <= 0 when :'>' @@compare_operator.compare(left,right) > 0 when :'>=' @@compare_operator.compare(left,right) >= 0 else fail(Issues::UNSUPPORTED_OPERATOR, o, {:operator => o.operator}) end end rescue ArgumentError => e fail(Issues::COMPARISON_NOT_POSSIBLE, o, { :operator => o.operator, :left_value => left, :right_value => right, :detail => e.message}, e) end end # Evaluates matching expressions with type, string or regexp rhs expression. # If RHS is a type, the =~ matches compatible (instance? of) type. # # @example # x =~ /abc.*/ # @example # x =~ "abc.*/" # @example # y = "abc" # x =~ "${y}.*" # @example # [1,2,3] =~ Array[Integer[1,10]] # # Note that a string is not instance? of Regexp, only Regular expressions are. # The Pattern type should instead be used as it is specified as subtype of String. # # @return [Boolean] if a match was made or not. Also sets $0..$n to matchdata in current scope. # def eval_MatchExpression o, scope left = evaluate(o.left_expr, scope) pattern = evaluate(o.right_expr, scope) # matches RHS types as instance of for all types except a parameterized Regexp[R] if pattern.is_a?(Puppet::Pops::Types::PAnyType) # evaluate as instance? of type check matched = @@type_calculator.instance?(pattern, left) # convert match result to Boolean true, or false return o.operator == :'=~' ? !!matched : !matched end begin pattern = Regexp.new(pattern) unless pattern.is_a?(Regexp) rescue StandardError => e fail(Issues::MATCH_NOT_REGEXP, o.right_expr, {:detail => e.message}, e) end unless left.is_a?(String) fail(Issues::MATCH_NOT_STRING, o.left_expr, {:left_value => left}) end matched = pattern.match(left) # nil, or MatchData set_match_data(matched,scope) # creates ephemeral # convert match result to Boolean true, or false o.operator == :'=~' ? !!matched : !matched end # Evaluates Puppet DSL `in` expression # def eval_InExpression o, scope left = evaluate(o.left_expr, scope) right = evaluate(o.right_expr, scope) @@compare_operator.include?(right, left, scope) end # @example # $a and $b # b is only evaluated if a is true # def eval_AndExpression o, scope is_true?(evaluate(o.left_expr, scope)) ? is_true?(evaluate(o.right_expr, scope)) : false end # @example # a or b # b is only evaluated if a is false # def eval_OrExpression o, scope is_true?(evaluate(o.left_expr, scope)) ? true : is_true?(evaluate(o.right_expr, scope)) end # Evaluates each entry of the literal list and creates a new Array # Supports unfolding of entries # @return [Array] with the evaluated content # def eval_LiteralList o, scope unfold([], o.values, scope) end # Evaluates each entry of the literal hash and creates a new Hash. # @return [Hash] with the evaluated content # def eval_LiteralHash o, scope # optimized o.entries.reduce({}) {|h,entry| h[evaluate(entry.key, scope)] = evaluate(entry.value, scope); h } end # Evaluates all statements and produces the last evaluated value # def eval_BlockExpression o, scope r = nil o.statements.each {|s| r = evaluate(s, scope)} r end # Performs optimized search over case option values, lazily evaluating each # until there is a match. If no match is found, the case expression's default expression # is evaluated (it may be nil or Nop if there is no default, thus producing nil). # If an option matches, the result of evaluating that option is returned. # @return [Object, nil] what a matched option returns, or nil if nothing matched. # def eval_CaseExpression(o, scope) # memo scope level before evaluating test - don't want a match in the case test to leak $n match vars # to expressions after the case expression. # with_guarded_scope(scope) do test = evaluate(o.test, scope) result = nil the_default = nil if o.options.find do |co| # the first case option that matches if co.values.find do |c| case c when Puppet::Pops::Model::LiteralDefault the_default = co.then_expr is_match?(test, evaluate(c, scope), c, scope) when Puppet::Pops::Model::UnfoldExpression # not ideal for error reporting, since it is not known which unfolded result # that caused an error - the entire unfold expression is blamed (i.e. the var c, passed to is_match?) evaluate(c, scope).any? {|v| is_match?(test, v, c, scope) } else is_match?(test, evaluate(c, scope), c, scope) end end result = evaluate(co.then_expr, scope) true # the option was picked end end result # an option was picked, and produced a result else evaluate(the_default, scope) # evaluate the default (should be a nop/nil) if there is no default). end end end # Evaluates a CollectExpression by creating a collector transformer. The transformer # will evaulate the collection, create the appropriate collector, and hand it off # to the compiler to collect the resources specified by the query. # def eval_CollectExpression o, scope Puppet::Pops::Evaluator::CollectorTransformer.new().transform(o,scope) end def eval_ParenthesizedExpression(o, scope) evaluate(o.expr, scope) end # This evaluates classes, nodes and resource type definitions to nil, since 3x: # instantiates them, and evaluates their parameters and body. This is achieved by # providing bridge AST classes in Puppet::Parser::AST::PopsBridge that bridges a # Pops Program and a Pops Expression. # # Since all Definitions are handled "out of band", they are treated as a no-op when # evaluated. # def eval_Definition(o, scope) nil end def eval_Program(o, scope) evaluate(o.body, scope) end # Produces Array[PAnyType], an array of resource references # def eval_ResourceExpression(o, scope) exported = o.exported virtual = o.virtual # Get the type name type_name = if (tmp_name = o.type_name).is_a?(Puppet::Pops::Model::QualifiedName) tmp_name.value # already validated as a name else type_name_acceptable = case o.type_name when Puppet::Pops::Model::QualifiedReference true when Puppet::Pops::Model::AccessExpression o.type_name.left_expr.is_a?(Puppet::Pops::Model::QualifiedReference) end evaluated_name = evaluate(tmp_name, scope) unless type_name_acceptable actual = type_calculator.generalize!(type_calculator.infer(evaluated_name)).to_s fail(Puppet::Pops::Issues::ILLEGAL_RESOURCE_TYPE, o.type_name, {:actual => actual}) end # must be a CatalogEntry subtype case evaluated_name when Puppet::Pops::Types::PHostClassType unless evaluated_name.class_name.nil? fail(Puppet::Pops::Issues::ILLEGAL_RESOURCE_TYPE, o.type_name, {:actual=> evaluated_name.to_s}) end 'class' when Puppet::Pops::Types::PResourceType unless evaluated_name.title().nil? fail(Puppet::Pops::Issues::ILLEGAL_RESOURCE_TYPE, o.type_name, {:actual=> evaluated_name.to_s}) end evaluated_name.type_name # assume validated else actual = type_calculator.generalize!(type_calculator.infer(evaluated_name)).to_s fail(Puppet::Pops::Issues::ILLEGAL_RESOURCE_TYPE, o.type_name, {:actual=>actual}) end end # This is a runtime check - the model is valid, but will have runtime issues when evaluated # and storeconfigs is not set. if(o.exported) optionally_fail(Puppet::Pops::Issues::RT_NO_STORECONFIGS_EXPORT, o); end titles_to_body = {} body_to_titles = {} body_to_params = {} # titles are evaluated before attribute operations o.bodies.map do | body | titles = evaluate(body.title, scope) # Title may not be nil # Titles may be given as an array, it is ok if it is empty, but not if it contains nil entries # Titles may not be an empty String # Titles must be unique in the same resource expression # There may be a :default entry, its entries apply with lower precedence # if titles.nil? fail(Puppet::Pops::Issues::MISSING_TITLE, body.title) end titles = [titles].flatten # Check types of evaluated titles and duplicate entries titles.each_with_index do |title, index| if title.nil? fail(Puppet::Pops::Issues::MISSING_TITLE_AT, body.title, {:index => index}) elsif !title.is_a?(String) && title != :default actual = type_calculator.generalize!(type_calculator.infer(title)).to_s fail(Puppet::Pops::Issues::ILLEGAL_TITLE_TYPE_AT, body.title, {:index => index, :actual => actual}) elsif title == EMPTY_STRING fail(Puppet::Pops::Issues::EMPTY_STRING_TITLE_AT, body.title, {:index => index}) elsif titles_to_body[title] fail(Puppet::Pops::Issues::DUPLICATE_TITLE, o, {:title => title}) end titles_to_body[title] = body end # Do not create a real instance from the :default case titles.delete(:default) body_to_titles[body] = titles # Store evaluated parameters in a hash associated with the body, but do not yet create resource # since the entry containing :defaults may appear later body_to_params[body] = body.operations.reduce({}) do |param_memo, op| params = evaluate(op, scope) params = [params] unless params.is_a?(Array) params.each do |p| if param_memo.include? p.name fail(Puppet::Pops::Issues::DUPLICATE_ATTRIBUTE, o, {:attribute => p.name}) end param_memo[p.name] = p end param_memo end end # Titles and Operations have now been evaluated and resources can be created # Each production is a PResource, and an array of all is produced as the result of # evaluating the ResourceExpression. # defaults_hash = body_to_params[titles_to_body[:default]] || {} o.bodies.map do | body | titles = body_to_titles[body] params = defaults_hash.merge(body_to_params[body] || {}) create_resources(o, scope, virtual, exported, type_name, titles, params.values) end.flatten.compact end def eval_ResourceOverrideExpression(o, scope) evaluated_resources = evaluate(o.resources, scope) evaluated_parameters = o.operations.map { |op| evaluate(op, scope) } create_resource_overrides(o, scope, [evaluated_resources].flatten, evaluated_parameters) evaluated_resources end # Produces 3x parameter def eval_AttributeOperation(o, scope) create_resource_parameter(o, scope, o.attribute_name, evaluate(o.value_expr, scope), o.operator) end def eval_AttributesOperation(o, scope) hashed_params = evaluate(o.expr, scope) unless hashed_params.is_a?(Hash) actual = type_calculator.generalize!(type_calculator.infer(hashed_params)).to_s fail(Puppet::Pops::Issues::TYPE_MISMATCH, o.expr, {:expected => 'Hash', :actual => actual}) end hashed_params.map { |k,v| create_resource_parameter(o, scope, k, v, :'=>') } end # Sets default parameter values for a type, produces the type # def eval_ResourceDefaultsExpression(o, scope) type = evaluate(o.type_ref, scope) type_name = if type.is_a?(Puppet::Pops::Types::PResourceType) && !type.type_name.nil? && type.title.nil? type.type_name # assume it is a valid name else actual = type_calculator.generalize!(type_calculator.infer(type)) fail(Issues::ILLEGAL_RESOURCE_TYPE, o.type_ref, {:actual => actual}) end evaluated_parameters = o.operations.map {|op| evaluate(op, scope) } create_resource_defaults(o, scope, type_name, evaluated_parameters) # Produce the type type end # Evaluates function call by name. # def eval_CallNamedFunctionExpression(o, scope) # The functor expression is not evaluated, it is not possible to select the function to call # via an expression like $a() case o.functor_expr when Puppet::Pops::Model::QualifiedName # ok when Puppet::Pops::Model::RenderStringExpression # helpful to point out this easy to make Epp error fail(Issues::ILLEGAL_EPP_PARAMETERS, o) else fail(Issues::ILLEGAL_EXPRESSION, o.functor_expr, {:feature=>'function name', :container => o}) end name = o.functor_expr.value evaluated_arguments = unfold([], o.arguments, scope) # wrap lambda in a callable block if it is present evaluated_arguments << Puppet::Pops::Evaluator::Closure.new(self, o.lambda, scope) if o.lambda call_function(name, evaluated_arguments, o, scope) end # Evaluation of CallMethodExpression handles a NamedAccessExpression functor (receiver.function_name) # def eval_CallMethodExpression(o, scope) unless o.functor_expr.is_a? Puppet::Pops::Model::NamedAccessExpression fail(Issues::ILLEGAL_EXPRESSION, o.functor_expr, {:feature=>'function accessor', :container => o}) end receiver = evaluate(o.functor_expr.left_expr, scope) name = o.functor_expr.right_expr unless name.is_a? Puppet::Pops::Model::QualifiedName fail(Issues::ILLEGAL_EXPRESSION, o.functor_expr, {:feature=>'function name', :container => o}) end name = name.value # the string function name evaluated_arguments = unfold([receiver], o.arguments || [], scope) # wrap lambda in a callable block if it is present evaluated_arguments << Puppet::Pops::Evaluator::Closure.new(self, o.lambda, scope) if o.lambda call_function(name, evaluated_arguments, o, scope) end # @example # $x ? { 10 => true, 20 => false, default => 0 } # def eval_SelectorExpression o, scope # memo scope level before evaluating test - don't want a match in the case test to leak $n match vars # to expressions after the selector expression. # with_guarded_scope(scope) do test = evaluate(o.left_expr, scope) the_default = nil selected = o.selectors.find do |s| me = s.matching_expr case me when Puppet::Pops::Model::LiteralDefault the_default = s.value_expr false when Puppet::Pops::Model::UnfoldExpression # not ideal for error reporting, since it is not known which unfolded result # that caused an error - the entire unfold expression is blamed (i.e. the var c, passed to is_match?) evaluate(me, scope).any? {|v| is_match?(test, v, me, scope) } else is_match?(test, evaluate(me, scope), me, scope) end end if selected evaluate(selected.value_expr, scope) elsif the_default evaluate(the_default, scope) else fail(Issues::UNMATCHED_SELECTOR, o.left_expr, :param_value => test) end end end # SubLocatable is simply an expression that holds location information def eval_SubLocatedExpression o, scope evaluate(o.expr, scope) end # Evaluates Puppet DSL Heredoc def eval_HeredocExpression o, scope result = evaluate(o.text_expr, scope) assert_external_syntax(scope, result, o.syntax, o.text_expr) result end # Evaluates Puppet DSL `if` def eval_IfExpression o, scope with_guarded_scope(scope) do if is_true?(evaluate(o.test, scope)) evaluate(o.then_expr, scope) else evaluate(o.else_expr, scope) end end end # Evaluates Puppet DSL `unless` def eval_UnlessExpression o, scope with_guarded_scope(scope) do unless is_true?(evaluate(o.test, scope)) evaluate(o.then_expr, scope) else evaluate(o.else_expr, scope) end end end # Evaluates a variable (getting its value) # The evaluator is lenient; any expression producing a String is used as a name # of a variable. # def eval_VariableExpression o, scope # Evaluator is not too fussy about what constitutes a name as long as the result # is a String and a valid variable name # name = evaluate(o.expr, scope) # Should be caught by validation, but make this explicit here as well, or mysterious evaluation issues # may occur for some evaluation use cases. case name when String when Numeric else fail(Issues::ILLEGAL_VARIABLE_EXPRESSION, o.expr) end get_variable_value(name, o, scope) end # Evaluates double quoted strings that may contain interpolation # def eval_ConcatenatedString o, scope o.segments.collect {|expr| string(evaluate(expr, scope), scope)}.join end # If the wrapped expression is a QualifiedName, it is taken as the name of a variable in scope. # Note that this is different from the 3.x implementation, where an initial qualified name # is accepted. (e.g. `"---${var + 1}---"` is legal. This implementation requires such concrete # syntax to be expressed in a model as `(TextExpression (+ (Variable var) 1)` - i.e. moving the decision to # the parser. # # Semantics; the result of an expression is turned into a string, nil is silently transformed to empty # string. # @return [String] the interpolated result # def eval_TextExpression o, scope if o.expr.is_a?(Puppet::Pops::Model::QualifiedName) string(get_variable_value(o.expr.value, o, scope), scope) else string(evaluate(o.expr, scope), scope) end end def string_Object(o, scope) o.to_s end def string_Symbol(o, scope) if :undef == o # optimized comparison 1.44 vs 1.95 EMPTY_STRING else o.to_s end end def string_Array(o, scope) "[#{o.map {|e| string(e, scope)}.join(COMMA_SEPARATOR)}]" end def string_Hash(o, scope) "{#{o.map {|k,v| "#{string(k, scope)} => #{string(v, scope)}"}.join(COMMA_SEPARATOR)}}" end def string_Regexp(o, scope) "/#{o.source}/" end def string_PAnyType(o, scope) @@type_calculator.string(o) end # Produces concatenation / merge of x and y. # # When x is an Array, y of type produces: # # * Array => concatenation `[1,2], [3,4] => [1,2,3,4]` # * Hash => concatenation of hash as array `[key, value, key, value, ...]` # * any other => concatenation of single value # # When x is a Hash, y of type produces: # # * Array => merge of array interpreted as `[key, value, key, value,...]` # * Hash => a merge, where entries in `y` overrides # * any other => error # # When x is something else, wrap it in an array first. # # When x is nil, an empty array is used instead. # # @note to concatenate an Array, nest the array - i.e. `[1,2], [[2,3]]` # # @overload concatenate(obj_x, obj_y) # @param obj_x [Object] object to wrap in an array and concatenate to; see other overloaded methods for return type # @param ary_y [Object] array to concatenate at end of `ary_x` # @return [Object] wraps obj_x in array before using other overloaded option based on type of obj_y # @overload concatenate(ary_x, ary_y) # @param ary_x [Array] array to concatenate to # @param ary_y [Array] array to concatenate at end of `ary_x` # @return [Array] new array with `ary_x` + `ary_y` # @overload concatenate(ary_x, hsh_y) # @param ary_x [Array] array to concatenate to # @param hsh_y [Hash] converted to array form, and concatenated to array # @return [Array] new array with `ary_x` + `hsh_y` converted to array # @overload concatenate (ary_x, obj_y) # @param ary_x [Array] array to concatenate to # @param obj_y [Object] non array or hash object to add to array # @return [Array] new array with `ary_x` + `obj_y` added as last entry # @overload concatenate(hsh_x, ary_y) # @param hsh_x [Hash] the hash to merge with # @param ary_y [Array] array interpreted as even numbered sequence of key, value merged with `hsh_x` # @return [Hash] new hash with `hsh_x` merged with `ary_y` interpreted as hash in array form # @overload concatenate(hsh_x, hsh_y) # @param hsh_x [Hash] the hash to merge to # @param hsh_y [Hash] hash merged with `hsh_x` # @return [Hash] new hash with `hsh_x` merged with `hsh_y` # @raise [ArgumentError] when `xxx_x` is neither an Array nor a Hash # @raise [ArgumentError] when `xxx_x` is a Hash, and `xxx_y` is neither Array nor Hash. # def concatenate(x, y) x = [x] unless x.is_a?(Array) || x.is_a?(Hash) case x when Array y = case y when Array then y when Hash then y.to_a else [y] end x + y # new array with concatenation when Hash y = case y when Hash then y when Array # Hash[[a, 1, b, 2]] => {} # Hash[a,1,b,2] => {a => 1, b => 2} # Hash[[a,1], [b,2]] => {[a,1] => [b,2]} # Hash[[[a,1], [b,2]]] => {a => 1, b => 2} # Use type calcultor to determine if array is Array[Array[?]], and if so use second form # of call t = @@type_calculator.infer(y) if t.element_type.is_a? Puppet::Pops::Types::PArrayType Hash[y] else Hash[*y] end else raise ArgumentError.new("Can only append Array or Hash to a Hash") end x.merge y # new hash with overwrite else raise ArgumentError.new("Can only append to an Array or a Hash.") end end # Produces the result x \ y (set difference) # When `x` is an Array, `y` is transformed to an array and then all matching elements removed from x. # When `x` is a Hash, all contained keys are removed from x as listed in `y` if it is an Array, or all its keys if it is a Hash. # The difference is returned. The given `x` and `y` are not modified by this operation. # @raise [ArgumentError] when `x` is neither an Array nor a Hash # def delete(x, y) result = x.dup case x when Array y = case y when Array then y when Hash then y.to_a else [y] end y.each {|e| result.delete(e) } when Hash y = case y when Array then y when Hash then y.keys else [y] end y.each {|e| result.delete(e) } else raise ArgumentError.new("Can only delete from an Array or Hash.") end result end # Implementation of case option matching. # # This is the type of matching performed in a case option, using == for every type # of value except regular expression where a match is performed. # def is_match? left, right, o, scope if right.is_a?(Regexp) return false unless left.is_a? String matched = right.match(left) set_match_data(matched, scope) # creates or clears ephemeral !!matched # convert to boolean elsif right.is_a?(Puppet::Pops::Types::PAnyType) # right is a type and left is not - check if left is an instance of the given type # (The reverse is not terribly meaningful - computing which of the case options that first produces # an instance of a given type). # @@type_calculator.instance?(right, left) else # Handle equality the same way as the language '==' operator (case insensitive etc.) @@compare_operator.equals(left,right) end end def with_guarded_scope(scope) scope_memo = get_scope_nesting_level(scope) begin yield ensure set_scope_nesting_level(scope, scope_memo) end end # Maps the expression in the given array to their product except for UnfoldExpressions which are first unfolded. # The result is added to the given result Array. # @param result [Array] Where to add the result (may contain information to add to) # @param array [Array[Puppet::Pops::Model::Expression] the expressions to map # @param scope [Puppet::Parser::Scope] the scope to evaluate in # @return [Array] the given result array with content added from the operation # def unfold(result, array, scope) array.each do |x| if x.is_a?(Puppet::Pops::Model::UnfoldExpression) result.concat(evaluate(x, scope)) else result << evaluate(x, scope) end end result end private :unfold end diff --git a/lib/puppet/pops/evaluator/runtime3_converter.rb b/lib/puppet/pops/evaluator/runtime3_converter.rb new file mode 100644 index 000000000..ce779a9b3 --- /dev/null +++ b/lib/puppet/pops/evaluator/runtime3_converter.rb @@ -0,0 +1,175 @@ +module Puppet::Pops::Evaluator +# Converts nested 4x supported values to 3x values. This is required because +# resources and other objects do not know about the new type system, and does not support +# regular expressions. Unfortunately this has to be done for array and hash as well. +# A complication is that catalog types needs to be resolved against the scope. +# +# Users should not create instances of this class. Instead the class methods {Runtime3Converter.convert}, +# {Runtime3Converter.map_args}, or {Runtime3Converter.instance} should be used +class Runtime3Converter + # Converts 4x supported values to a 3x values. Same as calling Runtime3Converter.instance.map_args(...) + # + # @param args [Array] Array of values to convert + # @param scope [Puppet::Parser::Scope] The scope to use when converting + # @param undef_value [Object] The value that nil is converted to + # @return [Array] The converted values + # + def self.map_args(args, scope, undef_value) + @@instance.map_args(args, scope, undef_value) + end + + # Converts 4x supported values to a 3x values. Same as calling Runtime3Converter.instance.convert(...) + # + # @param o [Object]The value to convert + # @param scope [Puppet::Parser::Scope] The scope to use when converting + # @param undef_value [Object] The value that nil is converted to + # @return [Object] The converted value + # + def self.convert(o, scope, undef_value) + @@instance.convert(o, scope, undef_value) + end + + # Returns the singleton instance of this class. + # @return [Runtime3Converter] The singleton instance + def self.instance + @@instance + end + + # Converts 4x supported values to a 3x values. + # + # @param args [Array] Array of values to convert + # @param scope [Puppet::Parser::Scope] The scope to use when converting + # @param undef_value [Object] The value that nil is converted to + # @return [Array] The converted values + # + def map_args(args, scope, undef_value) + args.map {|a| convert(a, scope, undef_value) } + end + + # Converts a 4x supported value to a 3x value. + # + # @param o [Object]The value to convert + # @param scope [Puppet::Parser::Scope] The scope to use when converting + # @param undef_value [Object] The value that nil is converted to + # @return [Object] The converted value + # + def convert(o, scope, undef_value) + @convert_visitor.visit_this_2(self, o, scope, undef_value) + end + + def convert_NilClass(o, scope, undef_value) + undef_value + end + + def convert2_NilClass(o, scope, undef_value) + :undef + end + + def convert_String(o, scope, undef_value) + # although wasteful, needed because user code may mutate these strings in Resources + o.frozen? ? o.dup : o + end + alias convert2_String :convert_String + + def convert_Object(o, scope, undef_value) + o + end + alias :convert2_Object :convert_Object + + def convert_Array(o, scope, undef_value) + o.map {|x| convert2(x, scope, undef_value) } + end + alias :convert2_Array :convert_Array + + def convert_Hash(o, scope, undef_value) + result = {} + o.each {|k,v| result[convert2(k, scope, undef_value)] = convert2(v, scope, undef_value) } + result + end + alias :convert2_Hash :convert_Hash + + def convert_Regexp(o, scope, undef_value) + # Puppet 3x cannot handle parameter values that are reqular expressions. Turn into regexp string in + # source form + o.inspect + end + alias :convert2_Regexp :convert_Regexp + + def convert_Symbol(o, scope, undef_value) + case o + # Support :undef since it may come from a 3x structure + when :undef + undef_value # 3x wants undef as either empty string or :undef + else + o # :default, and all others are verbatim since they are new in future evaluator + end + end + + # The :undef symbol should not be converted when nested in arrays or hashes + def convert2_Symbol(o, scope, undef_value) + o + end + + def convert_PAnyType(o, scope, undef_value) + o + end + alias :convert2_PAnyType :convert_PAnyType + + def convert_PCatalogEntryType(o, scope, undef_value) + # Since 4x does not support dynamic scoping, all names are absolute and can be + # used as is (with some check/transformation/mangling between absolute/relative form + # due to Puppet::Resource's idiosyncratic behavior where some references must be + # absolute and others cannot be. + # Thus there is no need to call scope.resolve_type_and_titles to do dynamic lookup. + + Puppet::Resource.new(*catalog_type_to_split_type_title(o)) + end + alias :convert2_PCatalogEntryType :convert_PCatalogEntryType + + # Produces an array with [type, title] from a PCatalogEntryType + # This method is used to produce the arguments for creation of reference resource instances + # (used when 3x is operating on a resource). + # Ensures that resources are *not* absolute. + # + def catalog_type_to_split_type_title(catalog_type) + split_type = catalog_type.is_a?(Puppet::Pops::Types::PType) ? catalog_type.type : catalog_type + case split_type + when Puppet::Pops::Types::PHostClassType + class_name = split_type.class_name + ['class', class_name.nil? ? nil : class_name.sub(/^::/, '')] + when Puppet::Pops::Types::PResourceType + type_name = split_type.type_name + title = split_type.title + if type_name =~ /^(::)?[Cc]lass/ + ['class', title.nil? ? nil : title.sub(/^::/, '')] + else + # Ensure that title is '' if nil + # Resources with absolute name always results in error because tagging does not support leading :: + [type_name.nil? ? nil : type_name.sub(/^::/, ''), title.nil? ? '' : title] + end + else + raise ArgumentError, "Cannot split the type #{catalog_type.class}, it represents neither a PHostClassType, nor a PResourceType." + end + end + + private + + def initialize + @convert_visitor = Puppet::Pops::Visitor.new(self, 'convert', 2, 2) + @convert2_visitor = Puppet::Pops::Visitor.new(self, 'convert2', 2, 2) + end + + @@instance = self.new + + # Converts a nested 4x supported value to a 3x value. + # + # @param o [Object]The value to convert + # @param scope [Puppet::Parser::Scope] The scope to use when converting + # @param undef_value [Object] The value that nil is converted to + # @return [Object] The converted value + # + def convert2(o, scope, undef_value) + @convert2_visitor.visit_this_2(self, o, scope, undef_value) + end +end +end diff --git a/lib/puppet/pops/evaluator/runtime3_support.rb b/lib/puppet/pops/evaluator/runtime3_support.rb index 9595c6ec1..c20ee14c5 100644 --- a/lib/puppet/pops/evaluator/runtime3_support.rb +++ b/lib/puppet/pops/evaluator/runtime3_support.rb @@ -1,601 +1,484 @@ # A module with bindings between the new evaluator and the 3x runtime. # The intention is to separate all calls into scope, compiler, resource, etc. in this module # to make it easier to later refactor the evaluator for better implementations of the 3x classes. # # @api private module Puppet::Pops::Evaluator::Runtime3Support NAME_SPACE_SEPARATOR = '::'.freeze # Fails the evaluation of _semantic_ with a given issue. # # @param issue [Puppet::Pops::Issue] the issue to report # @param semantic [Puppet::Pops::ModelPopsObject] the object for which evaluation failed in some way. Used to determine origin. # @param options [Hash] hash of optional named data elements for the given issue # @return [!] this method does not return # @raise [Puppet::ParseError] an evaluation error initialized from the arguments (TODO: Change to EvaluationError?) # def fail(issue, semantic, options={}, except=nil) optionally_fail(issue, semantic, options, except) # an error should have been raised since fail always fails raise ArgumentError, "Internal Error: Configuration of runtime error handling wrong: should have raised exception" end # Optionally (based on severity) Fails the evaluation of _semantic_ with a given issue # If the given issue is configured to be of severity < :error it is only reported, and the function returns. # # @param issue [Puppet::Pops::Issue] the issue to report # @param semantic [Puppet::Pops::ModelPopsObject] the object for which evaluation failed in some way. Used to determine origin. # @param options [Hash] hash of optional named data elements for the given issue # @return [!] this method does not return # @raise [Puppet::ParseError] an evaluation error initialized from the arguments (TODO: Change to EvaluationError?) # def optionally_fail(issue, semantic, options={}, except=nil) if except.nil? # Want a stacktrace, and it must be passed as an exception begin raise EvaluationError.new() rescue EvaluationError => e except = e end end diagnostic_producer.accept(issue, semantic, options, except) end # Binds the given variable name to the given value in the given scope. # The reference object `o` is intended to be used for origin information - the 3x scope implementation # only makes use of location when there is an error. This is now handled by other mechanisms; first a check # is made if a variable exists and an error is raised if attempting to change an immutable value. Errors # in name, numeric variable assignment etc. have also been validated prior to this call. In the event the # scope.setvar still raises an error, the general exception handling for evaluation of the assignment # expression knows about its location. Because of this, there is no need to extract the location for each # setting (extraction is somewhat expensive since 3x requires line instead of offset). # def set_variable(name, value, o, scope) # Scope also checks this but requires that location information are passed as options. # Those are expensive to calculate and a test is instead made here to enable failing with better information. # The error is not specific enough to allow catching it - need to check the actual message text. # TODO: Improve the messy implementation in Scope. # if scope.bound?(name) if Puppet::Parser::Scope::RESERVED_VARIABLE_NAMES.include?(name) fail(Puppet::Pops::Issues::ILLEGAL_RESERVED_ASSIGNMENT, o, {:name => name} ) else fail(Puppet::Pops::Issues::ILLEGAL_REASSIGNMENT, o, {:name => name} ) end end scope.setvar(name, value) end # Returns the value of the variable (nil is returned if variable has no value, or if variable does not exist) # def get_variable_value(name, o, scope) # Puppet 3x stores all variables as strings (then converts them back to numeric with a regexp... to see if it is a match variable) # Not ideal, scope should support numeric lookup directly instead. # TODO: consider fixing scope catch(:undefined_variable) { x = scope.lookupvar(name.to_s) # Must convert :undef back to nil - this can happen when an undefined variable is used in a # parameter's default value expression - there nil must be :undef to work with the rest of 3x. # Now that the value comes back to 4x it is changed to nil. return (x == :undef) ? nil : x } # It is always ok to reference numeric variables even if they are not assigned. They are always undef # if not set by a match expression. # unless name =~ Puppet::Pops::Patterns::NUMERIC_VAR_NAME fail(Puppet::Pops::Issues::UNKNOWN_VARIABLE, o, {:name => name}) end end # Returns true if the variable of the given name is set in the given most nested scope. True is returned even if # variable is bound to nil. # def variable_bound?(name, scope) scope.bound?(name.to_s) end # Returns true if the variable is bound to a value or nil, in the scope or it's parent scopes. # def variable_exists?(name, scope) scope.exist?(name.to_s) end def set_match_data(match_data, scope) # See set_variable for rationale for not passing file and line to ephemeral_from. # NOTE: The 3x scope adds one ephemeral(match) to its internal stack per match that succeeds ! It never # clears anything. Thus a context that performs many matches will get very deep (there simply is no way to # clear the match variables without rolling back the ephemeral stack.) # This implementation does not attempt to fix this, it behaves the same bad way. unless match_data.nil? scope.ephemeral_from(match_data) end end # Creates a local scope with vairalbes set from a hash of variable name to value # def create_local_scope_from(hash, scope) # two dummy values are needed since the scope tries to give an error message (can not happen in this # case - it is just wrong, the error should be reported by the caller who knows in more detail where it # is in the source. # raise ArgumentError, "Internal error - attempt to create a local scope without a hash" unless hash.is_a?(Hash) scope.ephemeral_from(hash) end # Creates a nested match scope def create_match_scope_from(scope) # Create a transparent match scope (for future matches) scope.new_match_scope(nil) end def get_scope_nesting_level(scope) scope.ephemeral_level end def set_scope_nesting_level(scope, level) # Yup, 3x uses this method to reset the level, it also supports passing :all to destroy all # ephemeral/local scopes - which is a sure way to create havoc. # scope.unset_ephemeral_var(level) end # Adds a relationship between the given `source` and `target` of the given `relationship_type` # @param source [Puppet:Pops::Types::PCatalogEntryType] the source end of the relationship (from) # @param target [Puppet:Pops::Types::PCatalogEntryType] the target end of the relationship (to) # @param relationship_type [:relationship, :subscription] the type of the relationship # def add_relationship(source, target, relationship_type, scope) # The 3x way is to record a Puppet::Parser::Relationship that is evaluated at the end of the compilation. # This means it is not possible to detect any duplicates at this point (and signal where an attempt is made to # add a duplicate. There is also no location information to signal the original place in the logic. The user will have # to go fish. # The 3.x implementation is based on Strings :-o, so the source and target must be transformed. The resolution is # done by Catalog#resource(type, title). To do that, it creates a Puppet::Resource since it is responsible for # translating the name/type/title and create index-keys used by the catalog. The Puppet::Resource has bizarre parsing of # the type and title (scan for [] that is interpreted as type/title (but it gets it wrong). # Moreover if the type is "" or "component", the type is Class, and if the type is :main, it is :main, all other cases # undergo capitalization of name-segments (foo::bar becomes Foo::Bar). (This was earlier done in the reverse by the parser). # Further, the title undergoes the same munging !!! # # That bug infested nest of messy logic needs serious Exorcism! # # Unfortunately it is not easy to simply call more intelligent methods at a lower level as the compiler evaluates the recorded # Relationship object at a much later point, and it is responsible for invoking all the messy logic. # # TODO: Revisit the below logic when there is a sane implementation of the catalog, compiler and resource. For now # concentrate on transforming the type references to what is expected by the wacky logic. # # HOWEVER, the Compiler only records the Relationships, and the only method it calls is @relationships.each{|x| x.evaluate(catalog) } # Which means a smarter Relationship class could do this right. Instead of obtaining the resource from the catalog using # the borked resource(type, title) which creates a resource for the purpose of looking it up, it needs to instead # scan the catalog's resources # # GAAAH, it is even worse! # It starts in the parser, which parses "File['foo']" into an AST::ResourceReference with type = File, and title = foo # This AST is evaluated by looking up the type/title in the scope - causing it to be loaded if it exists, and if not, the given # type name/title is used. It does not search for resource instances, only classes and types. It returns symbolic information # [type, [title, title]]. From this, instances of Puppet::Resource are created and returned. These only have type/title information # filled out. One or an array of resources are returned. # This set of evaluated (empty reference) Resource instances are then passed to the relationship operator. It creates a # Puppet::Parser::Relationship giving it a source and a target that are (empty reference) Resource instances. These are then remembered # until the relationship is evaluated by the compiler (at the end). When evaluation takes place, the (empty reference) Resource instances # are converted to String (!?! WTF) on the simple format "#{type}[#{title}]", and the catalog is told to find a resource, by giving # it this string. If it cannot find the resource it fails, else the before/notify parameter is appended with the target. # The search for the resource begin with (you guessed it) again creating an (empty reference) resource from type and title (WTF?!?!). # The catalog now uses the reference resource to compute a key [r.type, r.title.to_s] and also gets a uniqueness key from the # resource (This is only a reference type created from title and type). If it cannot find it with the first key, it uses the # uniqueness key to lookup. # # This is probably done to allow a resource type to munge/translate the title in some way (but it is quite unclear from the long # and convoluted path of evaluation. # In order to do this in a way that is similar to 3.x two resources are created to be used as keys. # # And if that is not enough, a source/target may be a Collector (a baked query that will be evaluated by the # compiler - it is simply passed through here for processing by the compiler at the right time). # if source.is_a?(Puppet::Pops::Evaluator::Collectors::AbstractCollector) # use verbatim - behavior defined by 3x source_resource = source else # transform into the wonderful String representation in 3x - type, title = catalog_type_to_split_type_title(source) + type, title = Puppet::Pops::Evaluator::Runtime3Converter.instance.catalog_type_to_split_type_title(source) source_resource = Puppet::Resource.new(type, title) end if target.is_a?(Puppet::Pops::Evaluator::Collectors::AbstractCollector) # use verbatim - behavior defined by 3x target_resource = target else # transform into the wonderful String representation in 3x - type, title = catalog_type_to_split_type_title(target) + type, title = Puppet::Pops::Evaluator::Runtime3Converter.instance.catalog_type_to_split_type_title(target) target_resource = Puppet::Resource.new(type, title) end # Add the relationship to the compiler for later evaluation. scope.compiler.add_relationship(Puppet::Parser::Relationship.new(source_resource, target_resource, relationship_type)) end # Coerce value `v` to numeric or fails. # The given value `v` is coerced to Numeric, and if that fails the operation # calls {#fail}. # @param v [Object] the value to convert # @param o [Object] originating instruction # @param scope [Object] the (runtime specific) scope where evaluation of o takes place # @return [Numeric] value `v` converted to Numeric. # def coerce_numeric(v, o, scope) unless n = Puppet::Pops::Utils.to_n(v) fail(Puppet::Pops::Issues::NOT_NUMERIC, o, {:value => v}) end n end def call_function(name, args, o, scope) Puppet::Util::Profiler.profile("Called #{name}", [:functions, name]) do # Call via 4x API if the function exists there loaders = scope.compiler.loaders # find the loader that loaded the code, or use the private_environment_loader (sees env + all modules) adapter = Puppet::Pops::Utils.find_adapter(o, Puppet::Pops::Adapters::LoaderAdapter) loader = adapter.nil? ? loaders.private_environment_loader : adapter.loader if loader && func = loader.load(:function, name) return func.call(scope, *args) end # Call via 3x API if function exists there fail(Puppet::Pops::Issues::UNKNOWN_FUNCTION, o, {:name => name}) unless Puppet::Parser::Functions.function(name) # Arguments must be mapped since functions are unaware of the new and magical creatures in 4x. # NOTE: Passing an empty string last converts nil/:undef to empty string - mapped_args = args.map {|a| convert(a, scope, '') } + mapped_args = Puppet::Pops::Evaluator::Runtime3Converter.map_args(args, scope, '') result = scope.send("function_#{name}", mapped_args) # Prevent non r-value functions from leaking their result (they are not written to care about this) Puppet::Parser::Functions.rvalue?(name) ? result : nil end end # The o is used for source reference def create_resource_parameter(o, scope, name, value, operator) file, line = extract_file_line(o) Puppet::Parser::Resource::Param.new( :name => name, :value => convert(value, scope, nil), # converted to 3x since 4x supports additional objects / types :source => scope.source, :line => line, :file => file, :add => operator == :'+>' ) end + def convert(value, scope, undef_value) + Puppet::Pops::Evaluator::Runtime3Converter.convert(value, scope, undef_value) + end + CLASS_STRING = 'class'.freeze def create_resources(o, scope, virtual, exported, type_name, resource_titles, evaluated_parameters) # TODO: Unknown resource causes creation of Resource to fail with ArgumentError, should give # a proper Issue. Now the result is "Error while evaluating a Resource Statement" with the message # from the raised exception. (It may be good enough). # resolve in scope. fully_qualified_type, resource_titles = scope.resolve_type_and_titles(type_name, resource_titles) # Not 100% accurate as this is the resource expression location and each title is processed separately # The titles are however the result of evaluation and they have no location at this point (an array # of positions for the source expressions are required for this to work). # TODO: Revisit and possible improve the accuracy. # file, line = extract_file_line(o) # Build a resource for each title resource_titles.map do |resource_title| resource = Puppet::Parser::Resource.new( fully_qualified_type, resource_title, :parameters => evaluated_parameters, :file => file, :line => line, :exported => exported, :virtual => virtual, # WTF is this? Which source is this? The file? The name of the context ? :source => scope.source, :scope => scope, :strict => true ) if resource.resource_type.is_a? Puppet::Resource::Type resource.resource_type.instantiate_resource(scope, resource) end scope.compiler.add_resource(scope, resource) scope.compiler.evaluate_classes([resource_title], scope, false) if fully_qualified_type == CLASS_STRING # Turn the resource into a PType (a reference to a resource type) # weed out nil's resource_to_ptype(resource) end end # Defines default parameters for a type with the given name. # def create_resource_defaults(o, scope, type_name, evaluated_parameters) # Note that name must be capitalized in this 3x call # The 3x impl creates a Resource instance with a bogus title and then asks the created resource # for the type of the name. # Note, locations are available per parameter. # scope.define_settings(capitalize_qualified_name(type_name), evaluated_parameters) end # Capitalizes each segment of a qualified name # def capitalize_qualified_name(name) name.split(/::/).map(&:capitalize).join(NAME_SPACE_SEPARATOR) end # Creates resource overrides for all resource type objects in evaluated_resources. The same set of # evaluated parameters are applied to all. # def create_resource_overrides(o, scope, evaluated_resources, evaluated_parameters) # Not 100% accurate as this is the resource expression location and each title is processed separately # The titles are however the result of evaluation and they have no location at this point (an array # of positions for the source expressions are required for this to work. # TODO: Revisit and possible improve the accuracy. # file, line = extract_file_line(o) evaluated_resources.each do |r| unless r.is_a?(Puppet::Pops::Types::PResourceType) && r.type_name != 'class' fail(Puppet::Pops::Issues::ILLEGAL_OVERRIDEN_TYPE, o, {:actual => r} ) end resource = Puppet::Parser::Resource.new( r.type_name, r.title, :parameters => evaluated_parameters, :file => file, :line => line, # WTF is this? Which source is this? The file? The name of the context ? :source => scope.source, :scope => scope ) scope.compiler.add_override(resource) end end # Finds a resource given a type and a title. # def find_resource(scope, type_name, title) scope.compiler.findresource(type_name, title) end # Returns the value of a resource's parameter by first looking up the parameter in the resource # and then in the defaults for the resource. Since the resource exists (it must in order to look up its # parameters, any overrides have already been applied). Defaults are not applied to a resource until it # has been finished (which typically has not taken place when this is evaluated; hence the dual lookup). # def get_resource_parameter_value(scope, resource, parameter_name) # This gets the parameter value, or nil (for both valid parameters and parameters that do not exist). val = resource[parameter_name] # Sometimes the resource is a Puppet::Parser::Resource and sometimes it is # a Puppet::Resource. The Puppet::Resource case occurs when puppet language # is evaluated against an already completed catalog (where all instances of # Puppet::Parser::Resource are converted to Puppet::Resource instances). # Evaluating against an already completed catalog is really only found in # the language specification tests, where the puppet language is used to # test itself. if resource.is_a?(Puppet::Parser::Resource) # The defaults must be looked up in the scope where the resource was created (not in the given # scope where the lookup takes place. resource_scope = resource.scope if val.nil? && resource_scope && defaults = resource_scope.lookupdefaults(resource.type) # NOTE: 3x resource keeps defaults as hash using symbol for name as key to Parameter which (again) holds # name and value. # NOTE: meta parameters that are unset ends up here, and there are no defaults for those encoded # in the defaults, they may receive hardcoded defaults later (e.g. 'tag'). param = defaults[parameter_name.to_sym] # Some parameters (meta parameters like 'tag') does not return a param from which the value can be obtained # at all times. Instead, they return a nil param until a value has been set. val = param.nil? ? nil : param.value end end val end # Returns true, if the given name is the name of a resource parameter. # def is_parameter_of_resource?(scope, resource, name) return false unless name.is_a?(String) resource.valid_parameter?(name) end def resource_to_ptype(resource) nil if resource.nil? # inference returns the meta type since the 3x Resource is an alternate way to describe a type type_calculator.infer(resource).type end # This is the same type of "truth" as used in the current Puppet DSL. # def is_true? o # Is the value true? This allows us to control the definition of truth # in one place. case o # Support :undef since it may come from a 3x structure when :undef false else !!o end end # Utility method for TrueClass || FalseClass # @param x [Object] the object to test if it is instance of TrueClass or FalseClass def is_boolean? x x.is_a?(TrueClass) || x.is_a?(FalseClass) end - def initialize - @@convert_visitor ||= Puppet::Pops::Visitor.new(self, "convert", 2, 2) - @@convert2_visitor ||= Puppet::Pops::Visitor.new(self, "convert2", 2, 2) - end - - # Converts 4x supported values to 3x values. This is required because - # resources and other objects do not know about the new type system, and does not support - # regular expressions. Unfortunately this has to be done for array and hash as well. - # A complication is that catalog types needs to be resolved against the scope. - # - def convert(o, scope, undef_value) - @@convert_visitor.visit_this_2(self, o, scope, undef_value) - end - - # Converts nested 4x supported values to 3x values. This is required because - # resources and other objects do not know about the new type system, and does not support - # regular expressions. Unfortunately this has to be done for array and hash as well. - # A complication is that catalog types needs to be resolved against the scope. - # - def convert2(o, scope, undef_value) - @@convert2_visitor.visit_this_2(self, o, scope, undef_value) - end - - - def convert_NilClass(o, scope, undef_value) - undef_value - end - - def convert2_NilClass(o, scope, undef_value) - :undef - end - - def convert_String(o, scope, undef_value) - # although wasteful, needed because user code may mutate these strings in Resources - o.frozen? ? o.dup : o - end - alias convert2_String :convert_String - - def convert_Object(o, scope, undef_value) - o - end - alias :convert2_Object :convert_Object - - def convert_Array(o, scope, undef_value) - o.map {|x| convert2(x, scope, undef_value) } - end - alias :convert2_Array :convert_Array - - def convert_Hash(o, scope, undef_value) - result = {} - o.each {|k,v| result[convert2(k, scope, undef_value)] = convert2(v, scope, undef_value) } - result - end - alias :convert2_Hash :convert_Hash - - def convert_Regexp(o, scope, undef_value) - # Puppet 3x cannot handle parameter values that are reqular expressions. Turn into regexp string in - # source form - o.inspect - end - alias :convert2_Regexp :convert_Regexp - - def convert_Symbol(o, scope, undef_value) - case o - # Support :undef since it may come from a 3x structure - when :undef - undef_value # 3x wants undef as either empty string or :undef - else - o # :default, and all others are verbatim since they are new in future evaluator - end - end - - # The :undef symbol should not be converted when nested in arrays or hashes - def convert2_Symbol(o, scope, undef_value) - o - end - - def convert_PAnyType(o, scope, undef_value) - o - end - alias :convert2_PAnyType :convert_PAnyType - - def convert_PCatalogEntryType(o, scope, undef_value) - # Since 4x does not support dynamic scoping, all names are absolute and can be - # used as is (with some check/transformation/mangling between absolute/relative form - # due to Puppet::Resource's idiosyncratic behavior where some references must be - # absolute and others cannot be. - # Thus there is no need to call scope.resolve_type_and_titles to do dynamic lookup. - - Puppet::Resource.new(*catalog_type_to_split_type_title(o)) - end - alias :convert2_PCatalogEntryType :convert_PCatalogEntryType - - private - - # Produces an array with [type, title] from a PCatalogEntryType - # This method is used to produce the arguments for creation of reference resource instances - # (used when 3x is operating on a resource). - # Ensures that resources are *not* absolute. - # - def catalog_type_to_split_type_title(catalog_type) - split_type = catalog_type.is_a?(Puppet::Pops::Types::PType) ? catalog_type.type : catalog_type - case split_type - when Puppet::Pops::Types::PHostClassType - class_name = split_type.class_name - ['class', class_name.nil? ? nil : class_name.sub(/^::/, '')] - when Puppet::Pops::Types::PResourceType - type_name = split_type.type_name - title = split_type.title - if type_name =~ /^(::)?[Cc]lass/ - ['class', title.nil? ? nil : title.sub(/^::/, '')] - else - # Ensure that title is '' if nil - # Resources with absolute name always results in error because tagging does not support leading :: - [type_name.nil? ? nil : type_name.sub(/^::/, ''), title.nil? ? '' : title] - end - else - raise ArgumentError, "Cannot split the type #{catalog_type.class}, it represents neither a PHostClassType, nor a PResourceType." - end - end - def extract_file_line(o) source_pos = Puppet::Pops::Utils.find_closest_positioned(o) return [nil, -1] unless source_pos [source_pos.locator.file, source_pos.line] end def find_closest_positioned(o) return nil if o.nil? || o.is_a?(Puppet::Pops::Model::Program) o.offset.nil? ? find_closest_positioned(o.eContainer) : Puppet::Pops::Adapters::SourcePosAdapter.adapt(o) end # Creates a diagnostic producer def diagnostic_producer Puppet::Pops::Validation::DiagnosticProducer.new( ExceptionRaisingAcceptor.new(), # Raises exception on all issues SeverityProducer.new(), # All issues are errors Puppet::Pops::Model::ModelLabelProvider.new()) end # Configure the severity of failures class SeverityProducer < Puppet::Pops::Validation::SeverityProducer Issues = Puppet::Pops::Issues def initialize super p = self # Issues triggering warning only if --debug is on if Puppet[:debug] p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :warning else p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :ignore end # Store config issues, ignore or warning p[Issues::RT_NO_STORECONFIGS_EXPORT] = Puppet[:storeconfigs] ? :ignore : :warning p[Issues::RT_NO_STORECONFIGS] = Puppet[:storeconfigs] ? :ignore : :warning end end # An acceptor of diagnostics that immediately raises an exception. class ExceptionRaisingAcceptor < Puppet::Pops::Validation::Acceptor def accept(diagnostic) super Puppet::Pops::IssueReporter.assert_and_report(self, {:message => "Evaluation Error:", :emit_warnings => true }) if errors? raise ArgumentError, "Internal Error: Configuration of runtime error handling wrong: should have raised exception" end end end class EvaluationError < StandardError end end diff --git a/lib/puppet/pops/functions/function.rb b/lib/puppet/pops/functions/function.rb index 6211bb801..e30cce030 100644 --- a/lib/puppet/pops/functions/function.rb +++ b/lib/puppet/pops/functions/function.rb @@ -1,77 +1,109 @@ # @note WARNING: This new function API is still under development and may change at # any time # # A function in the puppet evaluator. # # Functions are normally defined by another system, which produces subclasses # of this class as well as constructing delegations to call the appropriate methods. # # This class should rarely be used directly. Instead functions should be # constructed using {Puppet::Functions.create_function}. # # @api public class Puppet::Pops::Functions::Function # The scope where the function was defined attr_reader :closure_scope # The loader that loaded this function. # Should be used if function wants to load other things. # attr_reader :loader def initialize(closure_scope, loader) @closure_scope = closure_scope @loader = loader end # Invokes the function via the dispatching logic that performs type check and weaving. # A specialized function may override this method to do its own dispatching and checking of # the raw arguments. A specialized implementation can rearrange arguments, add or remove # arguments and then delegate to the dispatching logic by calling: # # @example Delegating to the dispatcher # def call(scope, *args) # manipulated_args = args + ['easter_egg'] # self.class.dispatcher.dispatch(self, scope, manipulated_args) # end # # System functions that must have access to the calling scope can use this technique. Functions # in general should not need the calling scope. (The closure scope; what is visible where the function # is defined) is available via the method `closure_scope`). # # @api public def call(scope, *args) self.class.dispatcher.dispatch(self, scope, args) end # Allows the implementation of a function to call other functions by name. The callable functions - # are those visible to the same loader that loaded this function (the calling function). + # are those visible to the same loader that loaded this function (the calling function). The + # referenced function is called with the calling functions closure scope as the caller's scope. + # + # @param function_name [String] The name of the function + # @param *args [Object] splat of arguments + # @return [Object] The result returned by the called function # # @api public def call_function(function_name, *args) - if the_loader = loader - func = the_loader.load(:function, function_name) - if func - return func.call(closure_scope, *args) - end - end - # Raise a generic error to allow upper layers to fill in the details about where in a puppet manifest this - # error originates. (Such information is not available here). - # - raise ArgumentError, "Function #{self.class.name}(): cannot call function '#{function_name}' - not found" + internal_call_function(closure_scope, function_name, args) end # The dispatcher for the function # # @api private def self.dispatcher @dispatcher ||= Puppet::Pops::Functions::Dispatcher.new end # Produces information about parameters in a way that is compatible with Closure # # @api private def self.signatures @dispatcher.signatures end + + protected + + # Allows the implementation of a function to call other functions by name and pass the caller + # scope. The callable functions are those visible to the same loader that loaded this function + # (the calling function). + # + # @param scope [Puppet::Parser::Scope] The caller scope + # @param function_name [String] The name of the function + # @param args [Array] array of arguments + # @return [Object] The result returned by the called function + # + # @api public + def internal_call_function(scope, function_name, args) + + the_loader = loader + raise ArgumentError, "Function #{self.class.name}(): cannot call function '#{function_name}' - no loader specified" unless the_loader + + func = the_loader.load(:function, function_name) + return func.call(scope, *args) if func + + # Check if a 3x function is present. Raise a generic error if it's not to allow upper layers to fill in the details + # about where in a puppet manifest this error originates. (Such information is not available here). + loader_scope = closure_scope + func_3x = Puppet::Parser::Functions.function(function_name, loader_scope.environment) if loader_scope.is_a?(Puppet::Parser::Scope) + raise ArgumentError, "Function #{self.class.name}(): cannot call function '#{function_name}' - not found" unless func_3x + + # Call via 3x API + # Arguments must be mapped since functions are unaware of the new and magical creatures in 4x. + # NOTE: Passing an empty string last converts nil/:undef to empty string + result = scope.send(func_3x, Puppet::Pops::Evaluator::Runtime3Converter.map_args(args, loader_scope, '')) + + # Prevent non r-value functions from leaking their result (they are not written to care about this) + Puppet::Parser::Functions.rvalue?(function_name) ? result : nil + end + end diff --git a/lib/puppet/pops/parser/lexer2.rb b/lib/puppet/pops/parser/lexer2.rb index 7756c0e9a..128ee8d0b 100644 --- a/lib/puppet/pops/parser/lexer2.rb +++ b/lib/puppet/pops/parser/lexer2.rb @@ -1,697 +1,697 @@ # The Lexer is responsbile for turning source text into tokens. # This version is a performance enhanced lexer (in comparison to the 3.x and earlier "future parser" lexer. # # Old returns tokens [:KEY, value, { locator = } # Could return [[token], locator] # or Token.new([token], locator) with the same API x[0] = token_symbol, x[1] = self, x[:key] = (:value, :file, :line, :pos) etc require 'strscan' require 'puppet/pops/parser/lexer_support' require 'puppet/pops/parser/heredoc_support' require 'puppet/pops/parser/interpolation_support' require 'puppet/pops/parser/epp_support' require 'puppet/pops/parser/slurp_support' class Puppet::Pops::Parser::Lexer2 include Puppet::Pops::Parser::LexerSupport include Puppet::Pops::Parser::HeredocSupport include Puppet::Pops::Parser::InterpolationSupport include Puppet::Pops::Parser::SlurpSupport include Puppet::Pops::Parser::EppSupport # ALl tokens have three slots, the token name (a Symbol), the token text (String), and a token text length. # All operator and punctuation tokens reuse singleton arrays Tokens that require unique values create # a unique array per token. # # PEFORMANCE NOTES: # This construct reduces the amount of object that needs to be created for operators and punctuation. # The length is pre-calculated for all singleton tokens. The length is used both to signal the length of # the token, and to advance the scanner position (without having to advance it with a scan(regexp)). # TOKEN_LBRACK = [:LBRACK, '['.freeze, 1].freeze TOKEN_LISTSTART = [:LISTSTART, '['.freeze, 1].freeze TOKEN_RBRACK = [:RBRACK, ']'.freeze, 1].freeze TOKEN_LBRACE = [:LBRACE, '{'.freeze, 1].freeze TOKEN_RBRACE = [:RBRACE, '}'.freeze, 1].freeze TOKEN_SELBRACE = [:SELBRACE, '{'.freeze, 1].freeze TOKEN_LPAREN = [:LPAREN, '('.freeze, 1].freeze TOKEN_RPAREN = [:RPAREN, ')'.freeze, 1].freeze TOKEN_EQUALS = [:EQUALS, '='.freeze, 1].freeze TOKEN_APPENDS = [:APPENDS, '+='.freeze, 2].freeze TOKEN_DELETES = [:DELETES, '-='.freeze, 2].freeze TOKEN_ISEQUAL = [:ISEQUAL, '=='.freeze, 2].freeze TOKEN_NOTEQUAL = [:NOTEQUAL, '!='.freeze, 2].freeze TOKEN_MATCH = [:MATCH, '=~'.freeze, 2].freeze TOKEN_NOMATCH = [:NOMATCH, '!~'.freeze, 2].freeze TOKEN_GREATEREQUAL = [:GREATEREQUAL, '>='.freeze, 2].freeze TOKEN_GREATERTHAN = [:GREATERTHAN, '>'.freeze, 1].freeze TOKEN_LESSEQUAL = [:LESSEQUAL, '<='.freeze, 2].freeze TOKEN_LESSTHAN = [:LESSTHAN, '<'.freeze, 1].freeze TOKEN_FARROW = [:FARROW, '=>'.freeze, 2].freeze TOKEN_PARROW = [:PARROW, '+>'.freeze, 2].freeze TOKEN_LSHIFT = [:LSHIFT, '<<'.freeze, 2].freeze TOKEN_LLCOLLECT = [:LLCOLLECT, '<<|'.freeze, 3].freeze TOKEN_LCOLLECT = [:LCOLLECT, '<|'.freeze, 2].freeze TOKEN_RSHIFT = [:RSHIFT, '>>'.freeze, 2].freeze TOKEN_RRCOLLECT = [:RRCOLLECT, '|>>'.freeze, 3].freeze TOKEN_RCOLLECT = [:RCOLLECT, '|>'.freeze, 2].freeze TOKEN_PLUS = [:PLUS, '+'.freeze, 1].freeze TOKEN_MINUS = [:MINUS, '-'.freeze, 1].freeze TOKEN_DIV = [:DIV, '/'.freeze, 1].freeze TOKEN_TIMES = [:TIMES, '*'.freeze, 1].freeze TOKEN_MODULO = [:MODULO, '%'.freeze, 1].freeze TOKEN_NOT = [:NOT, '!'.freeze, 1].freeze TOKEN_DOT = [:DOT, '.'.freeze, 1].freeze TOKEN_PIPE = [:PIPE, '|'.freeze, 1].freeze TOKEN_AT = [:AT , '@'.freeze, 1].freeze TOKEN_ATAT = [:ATAT , '@@'.freeze, 2].freeze TOKEN_COLON = [:COLON, ':'.freeze, 1].freeze TOKEN_COMMA = [:COMMA, ','.freeze, 1].freeze TOKEN_SEMIC = [:SEMIC, ';'.freeze, 1].freeze TOKEN_QMARK = [:QMARK, '?'.freeze, 1].freeze TOKEN_TILDE = [:TILDE, '~'.freeze, 1].freeze # lexed but not an operator in Puppet TOKEN_REGEXP = [:REGEXP, nil, 0].freeze TOKEN_IN_EDGE = [:IN_EDGE, '->'.freeze, 2].freeze TOKEN_IN_EDGE_SUB = [:IN_EDGE_SUB, '~>'.freeze, 2].freeze TOKEN_OUT_EDGE = [:OUT_EDGE, '<-'.freeze, 2].freeze TOKEN_OUT_EDGE_SUB = [:OUT_EDGE_SUB, '<~'.freeze, 2].freeze # Tokens that are always unique to what has been lexed TOKEN_STRING = [:STRING, nil, 0].freeze TOKEN_WORD = [:WORD, nil, 0].freeze TOKEN_DQPRE = [:DQPRE, nil, 0].freeze TOKEN_DQMID = [:DQPRE, nil, 0].freeze TOKEN_DQPOS = [:DQPRE, nil, 0].freeze TOKEN_NUMBER = [:NUMBER, nil, 0].freeze TOKEN_VARIABLE = [:VARIABLE, nil, 1].freeze TOKEN_VARIABLE_EMPTY = [:VARIABLE, ''.freeze, 1].freeze # HEREDOC has syntax as an argument. TOKEN_HEREDOC = [:HEREDOC, nil, 0].freeze # EPP_START is currently a marker token, may later get syntax TOKEN_EPPSTART = [:EPP_START, nil, 0].freeze TOKEN_EPPEND = [:EPP_END, '%>', 2].freeze TOKEN_EPPEND_TRIM = [:EPP_END_TRIM, '-%>', 3].freeze # This is used for unrecognized tokens, will always be a single character. This particular instance # is not used, but is kept here for documentation purposes. TOKEN_OTHER = [:OTHER, nil, 0] # Keywords are all singleton tokens with pre calculated lengths. # Booleans are pre-calculated (rather than evaluating the strings "false" "true" repeatedly. # KEYWORDS = { "case" => [:CASE, 'case', 4], "class" => [:CLASS, 'class', 5], "default" => [:DEFAULT, 'default', 7], "define" => [:DEFINE, 'define', 6], "if" => [:IF, 'if', 2], "elsif" => [:ELSIF, 'elsif', 5], "else" => [:ELSE, 'else', 4], "inherits" => [:INHERITS, 'inherits', 8], "node" => [:NODE, 'node', 4], "and" => [:AND, 'and', 3], "or" => [:OR, 'or', 2], "undef" => [:UNDEF, 'undef', 5], "false" => [:BOOLEAN, false, 5], "true" => [:BOOLEAN, true, 4], "in" => [:IN, 'in', 2], "unless" => [:UNLESS, 'unless', 6], "function" => [:FUNCTION, 'function', 8], "type" => [:TYPE, 'type', 4], "attr" => [:ATTR, 'attr', 4], "private" => [:PRIVATE, 'private', 7], } KEYWORDS.each {|k,v| v[1].freeze; v.freeze } KEYWORDS.freeze # Reverse lookup of keyword name to string KEYWORD_NAMES = {} KEYWORDS.each {|k, v| KEYWORD_NAMES[v[0]] = k } KEYWORD_NAMES.freeze PATTERN_WS = %r{[[:blank:]\r]+} PATTERN_NON_WS = %r{\w+\b?} # The single line comment includes the line ending. PATTERN_COMMENT = %r{#.*\r?} PATTERN_MLCOMMENT = %r{/\*(.*?)\*/}m PATTERN_REGEX = %r{/[^/\n]*/} PATTERN_REGEX_END = %r{/} PATTERN_REGEX_A = %r{\A/} # for replacement to "" PATTERN_REGEX_Z = %r{/\Z} # for replacement to "" PATTERN_REGEX_ESC = %r{\\/} # for replacement to "/" # The 3x patterns: # PATTERN_CLASSREF = %r{((::){0,1}[A-Z][-\w]*)+} # PATTERN_NAME = %r{((::)?[a-z0-9][-\w]*)(::[a-z0-9][-\w]*)*} # The NAME and CLASSREF in 4x are strict. Each segment must start with # a letter a-z and may not contain dashes (\w includes letters, digits and _). # PATTERN_CLASSREF = %r{((::){0,1}[A-Z][\w]*)+} PATTERN_NAME = %r{((::)?[a-z][\w]*)(::[a-z][\w]*)*} PATTERN_BARE_WORD = %r{[a-z_](?:[\w-]*[\w])?} PATTERN_DOLLAR_VAR = %r{\$(::)?(\w+::)*\w+} PATTERN_NUMBER = %r{\b(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b} # PERFORMANCE NOTE: # Comparison against a frozen string is faster (than unfrozen). # STRING_BSLASH_BSLASH = '\\'.freeze attr_reader :locator def initialize() end # Clears the lexer state (it is not required to call this as it will be garbage collected # and the next lex call (lex_string, lex_file) will reset the internal state. # def clear() # not really needed, but if someone wants to ensure garbage is collected as early as possible @scanner = nil @locator = nil @lexing_context = nil end # Convenience method, and for compatibility with older lexer. Use the lex_string instead which allows # passing the path to use without first having to call file= (which reads the file if it exists). # (Bad form to use overloading of assignment operator for something that is not really an assignment. Also, # overloading of = does not allow passing more than one argument). # def string=(string) lex_string(string, '') end def lex_string(string, path='') initvars @scanner = StringScanner.new(string) @locator = Puppet::Pops::Parser::Locator.locator(string, path) end # Lexes an unquoted string. # @param string [String] the string to lex # @param locator [Puppet::Pops::Parser::Locator] the locator to use (a default is used if nil is given) # @param escapes [Array] array of character strings representing the escape sequences to transform # @param interpolate [Boolean] whether interpolation of expressions should be made or not. # def lex_unquoted_string(string, locator, escapes, interpolate) initvars @scanner = StringScanner.new(string) @locator = locator || Puppet::Pops::Parser::Locator.locator(string, '') @lexing_context[:escapes] = escapes || UQ_ESCAPES @lexing_context[:uq_slurp_pattern] = (interpolate || !escapes.empty?) ? SLURP_UQ_PATTERN : SLURP_ALL_PATTERN end # Convenience method, and for compatibility with older lexer. Use the lex_file instead. # (Bad form to use overloading of assignment operator for something that is not really an assignment). # def file=(file) lex_file(file) end # TODO: This method should not be used, callers should get the locator since it is most likely required to # compute line, position etc given offsets. # def file @locator ? @locator.file : nil end # Initializes lexing of the content of the given file. An empty string is used if the file does not exist. # def lex_file(file) initvars contents = Puppet::FileSystem.exist?(file) ? Puppet::FileSystem.read(file) : "" @scanner = StringScanner.new(contents.freeze) @locator = Puppet::Pops::Parser::Locator.locator(contents, file) end def initvars @token_queue = [] # NOTE: additional keys are used; :escapes, :uq_slurp_pattern, :newline_jump, :epp_* @lexing_context = { :brace_count => 0, :after => nil, } end # Scans all of the content and returns it in an array # Note that the terminating [false, false] token is included in the result. # def fullscan result = [] scan {|token, value| result.push([token, value]) } result end # A block must be passed to scan. It will be called with two arguments, a symbol for the token, # and an instance of LexerSupport::TokenValue # PERFORMANCE NOTE: The TokenValue is designed to reduce the amount of garbage / temporary data # and to only convert the lexer's internal tokens on demand. It is slightly more costly to create an # instance of a class defined in Ruby than an Array or Hash, but the gain is much bigger since transformation # logic is avoided for many of its members (most are never used (e.g. line/pos information which is only of # value in general for error messages, and for some expressions (which the lexer does not know about). # def scan # PERFORMANCE note: it is faster to access local variables than instance variables. # This makes a small but notable difference since instance member access is avoided for # every token in the lexed content. # scn = @scanner ctx = @lexing_context queue = @token_queue lex_error_without_pos("Internal Error: No string or file given to lexer to process.") unless scn scn.skip(PATTERN_WS) # This is the lexer's main loop until queue.empty? && scn.eos? do if token = queue.shift || lex_token ctx[:after] = token[0] yield token end end # Signals end of input yield [false, false] end # This lexes one token at the current position of the scanner. # PERFORMANCE NOTE: Any change to this logic should be performance measured. # def lex_token # Using three char look ahead (may be faster to do 2 char look ahead since only 2 tokens require a third scn = @scanner ctx = @lexing_context before = @scanner.pos # A look ahead of 3 characters is used since the longest operator ambiguity is resolved at that point. # PERFORMANCE NOTE: It is faster to peek once and use three separate variables for lookahead 0, 1 and 2. # la = scn.peek(3) return nil if la.empty? # PERFORMANCE NOTE. # It is slightly faster to use these local variables than accessing la[0], la[1] etc. in ruby 1.9.3 # But not big enough to warrant two completely different implementations. # la0 = la[0] la1 = la[1] la2 = la[2] # PERFORMANCE NOTE: # A case when, where all the cases are literal values is the fastest way to map from data to code. # It is much faster than using a hash with lambdas, hash with symbol used to then invoke send etc. # This case statement is evaluated for most character positions in puppet source, and great care must # be taken to not introduce performance regressions. # case la0 when '.' emit(TOKEN_DOT, before) when ',' emit(TOKEN_COMMA, before) when '[' if (before == 0 || scn.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/) emit(TOKEN_LISTSTART, before) else emit(TOKEN_LBRACK, before) end when ']' emit(TOKEN_RBRACK, before) when '(' emit(TOKEN_LPAREN, before) when ')' emit(TOKEN_RPAREN, before) when ';' emit(TOKEN_SEMIC, before) when '?' emit(TOKEN_QMARK, before) when '*' emit(TOKEN_TIMES, before) when '%' if la1 == '>' && ctx[:epp_mode] scn.pos += 2 if ctx[:epp_mode] == :expr enqueue_completed(TOKEN_EPPEND, before) end ctx[:epp_mode] = :text interpolate_epp else emit(TOKEN_MODULO, before) end when '{' # The lexer needs to help the parser since the technology used cannot deal with # lookahead of same token with different precedence. This is solved by making left brace # after ? into a separate token. # ctx[:brace_count] += 1 emit(if ctx[:after] == :QMARK TOKEN_SELBRACE else TOKEN_LBRACE end, before) when '}' ctx[:brace_count] -= 1 emit(TOKEN_RBRACE, before) # TOKENS @, @@, @( when '@' case la1 when '@' emit(TOKEN_ATAT, before) # TODO; Check if this is good for the grammar when '(' heredoc else emit(TOKEN_AT, before) end # TOKENS |, |>, |>> when '|' emit(case la1 when '>' la2 == '>' ? TOKEN_RRCOLLECT : TOKEN_RCOLLECT else TOKEN_PIPE end, before) # TOKENS =, =>, ==, =~ when '=' emit(case la1 when '=' TOKEN_ISEQUAL when '>' TOKEN_FARROW when '~' TOKEN_MATCH else TOKEN_EQUALS end, before) # TOKENS '+', '+=', and '+>' when '+' emit(case la1 when '=' TOKEN_APPENDS when '>' TOKEN_PARROW else TOKEN_PLUS end, before) # TOKENS '-', '->', and epp '-%>' (end of interpolation with trim) when '-' if ctx[:epp_mode] && la1 == '%' && la2 == '>' scn.pos += 3 if ctx[:epp_mode] == :expr enqueue_completed(TOKEN_EPPEND_TRIM, before) end interpolate_epp(:with_trim) else emit(case la1 when '>' TOKEN_IN_EDGE when '=' TOKEN_DELETES else TOKEN_MINUS end, before) end # TOKENS !, !=, !~ when '!' emit(case la1 when '=' TOKEN_NOTEQUAL when '~' TOKEN_NOMATCH else TOKEN_NOT end, before) # TOKENS ~>, ~ when '~' emit(la1 == '>' ? TOKEN_IN_EDGE_SUB : TOKEN_TILDE, before) when '#' scn.skip(PATTERN_COMMENT) nil # TOKENS '/', '/*' and '/ regexp /' when '/' case la1 when '*' scn.skip(PATTERN_MLCOMMENT) nil else # regexp position is a regexp, else a div if regexp_acceptable? && value = scn.scan(PATTERN_REGEX) # Ensure an escaped / was not matched while value[-2..-2] == STRING_BSLASH_BSLASH # i.e. \\ value += scn.scan_until(PATTERN_REGEX_END) end regex = value.sub(PATTERN_REGEX_A, '').sub(PATTERN_REGEX_Z, '').gsub(PATTERN_REGEX_ESC, '/') emit_completed([:REGEX, Regexp.new(regex), scn.pos-before], before) else emit(TOKEN_DIV, before) end end # TOKENS <, <=, <|, <<|, <<, <-, <~ when '<' emit(case la1 when '<' if la2 == '|' TOKEN_LLCOLLECT else TOKEN_LSHIFT end when '=' TOKEN_LESSEQUAL when '|' TOKEN_LCOLLECT when '-' TOKEN_OUT_EDGE when '~' TOKEN_OUT_EDGE_SUB else TOKEN_LESSTHAN end, before) # TOKENS >, >=, >> when '>' emit(case la1 when '>' TOKEN_RSHIFT when '=' TOKEN_GREATEREQUAL else TOKEN_GREATERTHAN end, before) # TOKENS :, ::CLASSREF, ::NAME when ':' if la1 == ':' before = scn.pos # PERFORMANCE NOTE: This could potentially be speeded up by using a case/when listing all # upper case letters. Alternatively, the 'A', and 'Z' comparisons may be faster if they are # frozen. # if la2 >= 'A' && la2 <= 'Z' # CLASSREF or error value = scn.scan(PATTERN_CLASSREF) if value after = scn.pos emit_completed([:CLASSREF, value.freeze, after-before], before) else # move to faulty position ('::' was ok) scn.pos = scn.pos + 3 lex_error("Illegal fully qualified class reference") end else # NAME or error value = scn.scan(PATTERN_NAME) if value emit_completed([:NAME, value.freeze, scn.pos-before], before) else # move to faulty position ('::' was ok) scn.pos = scn.pos + 2 lex_error("Illegal fully qualified name") end end else emit(TOKEN_COLON, before) end when '$' if value = scn.scan(PATTERN_DOLLAR_VAR) emit_completed([:VARIABLE, value[1..-1].freeze, scn.pos - before], before) else # consume the $ and let higher layer complain about the error instead of getting a syntax error emit(TOKEN_VARIABLE_EMPTY, before) end when '"' # Recursive string interpolation, 'interpolate' either returns a STRING token, or # a DQPRE with the rest of the string's tokens placed in the @token_queue interpolate_dq when "'" emit_completed([:STRING, slurp_sqstring.freeze, scn.pos - before], before) when '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' value = scn.scan(PATTERN_NUMBER) if value length = scn.pos - before assert_numeric(value, length) emit_completed([:NUMBER, value.freeze, length], before) else # move to faulty position ([0-9] was ok) invalid_number = scn.scan_until(PATTERN_NON_WS) scn.pos = before + 1 lex_error("Illegal number '#{invalid_number}'") end when 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_' value = scn.scan(PATTERN_NAME) # NAME or false start because followed by hyphen(s), underscore or word if value && !scn.match?(/^-+\w/) emit_completed(KEYWORDS[value] || [:NAME, value.freeze, scn.pos - before], before) else # Restart and check entire pattern (for ease of detecting non allowed trailing hyphen) scn.pos = before value = scn.scan(PATTERN_BARE_WORD) # If the WORD continues with :: it must be a correct fully qualified name if value && !(fully_qualified = scn.match?(/::/)) emit_completed([:WORD, value.freeze, scn.pos - before], before) else # move to faulty position ([a-z_] was ok) scn.pos = scn.pos + 1 if fully_qualified lex_error("Illegal fully qualified name") else lex_error("Illegal name or bare word") end end end when 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' value = scn.scan(PATTERN_CLASSREF) if value emit_completed([:CLASSREF, value.freeze, scn.pos - before], before) else # move to faulty position ([A-Z] was ok) scn.pos = scn.pos + 1 lex_error("Illegal class reference") end when "\n" # If heredoc_cont is in effect there are heredoc text lines to skip over # otherwise just skip the newline. # if ctx[:newline_jump] scn.pos = ctx[:newline_jump] ctx[:newline_jump] = nil else scn.pos += 1 end return nil when ' ', "\t", "\r" scn.skip(PATTERN_WS) return nil else # In case of unicode spaces of various kinds that are captured by a regexp, but not by the # simpler case expression above (not worth handling those special cases with better performance). if scn.skip(PATTERN_WS) nil else # "unrecognized char" emit([:OTHER, la0, 1], before) end end end # Emits (produces) a token [:tokensymbol, TokenValue] and moves the scanner's position past the token # def emit(token, byte_offset) @scanner.pos = byte_offset + token[2] [token[0], TokenValue.new(token, byte_offset, @locator)] end # Emits the completed token on the form [:tokensymbol, TokenValue. This method does not alter # the scanner's position. # def emit_completed(token, byte_offset) [token[0], TokenValue.new(token, byte_offset, @locator)] end # Enqueues a completed token at the given offset def enqueue_completed(token, byte_offset) @token_queue << emit_completed(token, byte_offset) end # Allows subprocessors for heredoc etc to enqueue tokens that are tokenized by a different lexer instance # def enqueue(emitted_token) @token_queue << emitted_token end # Answers after which tokens it is acceptable to lex a regular expression. # PERFORMANCE NOTE: # It may be beneficial to turn this into a hash with default value of true for missing entries. # A case expression with literal values will however create a hash internally. Since a reference is # always needed to the hash, this access is almost as costly as a method call. # def regexp_acceptable? case @lexing_context[:after] # Ends of (potential) R-value generating expressions when :RPAREN, :RBRACK, :RRCOLLECT, :RCOLLECT false # End of (potential) R-value - but must be allowed because of case expressions # Called out here to not be mistaken for a bug. when :RBRACE true # Operands (that can be followed by DIV (even if illegal in grammar) - when :NAME, :CLASSREF, :NUMBER, :STRING, :BOOLEAN, :DQPRE, :DQMID, :DQPOST, :HEREDOC, :REGEX + when :NAME, :CLASSREF, :NUMBER, :STRING, :BOOLEAN, :DQPRE, :DQMID, :DQPOST, :HEREDOC, :REGEX, :VARIABLE, :WORD false else true end end end diff --git a/lib/puppet/type/package.rb b/lib/puppet/type/package.rb index 1058fcd9f..cabc99a22 100644 --- a/lib/puppet/type/package.rb +++ b/lib/puppet/type/package.rb @@ -1,535 +1,548 @@ # Define the different packaging systems. Each package system is implemented # in a module, which then gets used to individually extend each package object. # This allows packages to exist on the same machine using different packaging # systems. require 'puppet/parameter/package_options' require 'puppet/parameter/boolean' module Puppet Type.newtype(:package) do @doc = "Manage packages. There is a basic dichotomy in package support right now: Some package types (e.g., yum and apt) can retrieve their own package files, while others (e.g., rpm and sun) cannot. For those package formats that cannot retrieve their own files, you can use the `source` parameter to point to the correct file. Puppet will automatically guess the packaging format that you are using based on the platform you are on, but you can override it using the `provider` parameter; each provider defines what it requires in order to function, and you must meet those requirements to use a given provider. **Autorequires:** If Puppet is managing the files specified as a package's `adminfile`, `responsefile`, or `source`, the package resource will autorequire those files." feature :reinstallable, "The provider can reinstall packages.", :methods => [:reinstall] feature :installable, "The provider can install packages.", :methods => [:install] feature :uninstallable, "The provider can uninstall packages.", :methods => [:uninstall] feature :upgradeable, "The provider can upgrade to the latest version of a package. This feature is used by specifying `latest` as the desired value for the package.", :methods => [:update, :latest] feature :purgeable, "The provider can purge packages. This generally means that all traces of the package are removed, including existing configuration files. This feature is thus destructive and should be used with the utmost care.", :methods => [:purge] feature :versionable, "The provider is capable of interrogating the package database for installed version(s), and can select which out of a set of available versions of a package to install if asked." feature :holdable, "The provider is capable of placing packages on hold such that they are not automatically upgraded as a result of other package dependencies unless explicit action is taken by a user or another package. Held is considered a superset of installed.", :methods => [:hold] feature :install_options, "The provider accepts options to be passed to the installer command." feature :uninstall_options, "The provider accepts options to be passed to the uninstaller command." feature :package_settings, "The provider accepts package_settings to be ensured for the given package. The meaning and format of these settings is provider-specific.", :methods => [:package_settings_insync?, :package_settings, :package_settings=] feature :virtual_packages, "The provider accepts virtual package names for install and uninstall." ensurable do desc <<-EOT What state the package should be in. On packaging systems that can retrieve new packages on their own, you can choose which package to retrieve by specifying a version number or `latest` as the ensure value. On packaging systems that manage configuration files separately from "normal" system files, you can uninstall config files by specifying `purged` as the ensure value. This defaults to `installed`. EOT attr_accessor :latest newvalue(:present, :event => :package_installed) do provider.install end newvalue(:absent, :event => :package_removed) do provider.uninstall end newvalue(:purged, :event => :package_purged, :required_features => :purgeable) do provider.purge end newvalue(:held, :event => :package_held, :required_features => :holdable) do provider.hold end # Alias the 'present' value. aliasvalue(:installed, :present) newvalue(:latest, :required_features => :upgradeable) do # Because yum always exits with a 0 exit code, there's a retrieve # in the "install" method. So, check the current state now, # to compare against later. current = self.retrieve begin provider.update rescue => detail self.fail Puppet::Error, "Could not update: #{detail}", detail end if current == :absent :package_installed else :package_changed end end newvalue(/./, :required_features => :versionable) do begin provider.install rescue => detail self.fail Puppet::Error, "Could not update: #{detail}", detail end if self.retrieve == :absent :package_installed else :package_changed end end defaultto :installed # Override the parent method, because we've got all kinds of # funky definitions of 'in sync'. def insync?(is) @lateststamp ||= (Time.now.to_i - 1000) # Iterate across all of the should values, and see how they # turn out. @should.each { |should| case should when :present return true unless [:absent, :purged, :held].include?(is) when :latest # Short-circuit packages that are not present return false if is == :absent or is == :purged # Don't run 'latest' more than about every 5 minutes if @latest and ((Time.now.to_i - @lateststamp) / 60) < 5 #self.debug "Skipping latest check" else begin @latest = provider.latest @lateststamp = Time.now.to_i rescue => detail error = Puppet::Error.new("Could not get latest version: #{detail}") error.set_backtrace(detail.backtrace) raise error end end case when is.is_a?(Array) && is.include?(@latest) return true when is == @latest return true when is == :present # This will only happen on retarded packaging systems # that can't query versions. return true else self.debug "#{@resource.name} #{is.inspect} is installed, latest is #{@latest.inspect}" end when :absent return true if is == :absent or is == :purged when :purged return true if is == :purged # this handles version number matches and # supports providers that can have multiple versions installed when *Array(is) return true else # We have version numbers, and no match. If the provider has # additional logic, run it here. return provider.insync?(is) if provider.respond_to?(:insync?) end } false end # This retrieves the current state. LAK: I think this method is unused. def retrieve provider.properties[:ensure] end # Provide a bit more information when logging upgrades. def should_to_s(newvalue = @should) if @latest @latest.to_s else super(newvalue) end end end newparam(:name) do desc "The package name. This is the name that the packaging system uses internally, which is sometimes (especially on Solaris) a name that is basically useless to humans. If you want to abstract package installation, then you can use aliases to provide a common name to packages: # In the 'openssl' class $ssl = $operatingsystem ? { solaris => SMCossl, default => openssl } # It is not an error to set an alias to the same value as the # object name. package { $ssl: ensure => installed, alias => openssl } . etc. . $ssh = $operatingsystem ? { solaris => SMCossh, default => openssh } # Use the alias to specify a dependency, rather than # having another selector to figure it out again. package { $ssh: ensure => installed, alias => openssh, require => Package[openssl] } " isnamevar validate do |value| if !value.is_a?(String) raise ArgumentError, "Name must be a String not #{value.class}" end end end # We call providify here so that we can set provider as a namevar. # Normally this method is called after newtype finishes constructing this # Type class. providify paramclass(:provider).isnamevar # We have more than one namevar, so we need title_patterns. However, we # cheat and set the patterns to map to name only and completely ignore # provider. So far, the logic that determines uniqueness appears to just # "Do The Right Thing™" when the provider is explicitly set by the user. # # The following resources will be seen as uniqe by puppet: # # # Uniqueness Key: ['mysql', nil] # package{'mysql': } # # # Uniqueness Key: ['mysql', 'gem'] # package{'gem-mysql': # name => 'mysql, # provider => gem # } # # This does not handle the case where providers like 'yum' and 'rpm' should # clash. Also, declarations that implicitly use the default provider will # clash with those that explicitly use the default. def self.title_patterns # This is the default title pattern for all types, except hard-wired to # set only name. [ [ /(.*)/m, [ [:name] ] ] ] end newproperty(:package_settings, :required_features=>:package_settings) do desc "Settings that can change the contents or configuration of a package. The formatting and effects of package_settings are provider-specific; any provider that implements them must explain how to use them in its documentation. (Our general expectation is that if a package is installed but its settings are out of sync, the provider should re-install that package with the desired settings.) An example of how package_settings could be used is FreeBSD's port build options --- a future version of the provider could accept a hash of options, and would reinstall the port if the installed version lacked the correct settings. package { 'www/apache22': package_settings => { 'SUEXEC' => false } } Again, check the documentation of your platform's package provider to see the actual usage." validate do |value| if provider.respond_to?(:package_settings_validate) provider.package_settings_validate(value) else super(value) end end munge do |value| if provider.respond_to?(:package_settings_munge) provider.package_settings_munge(value) else super(value) end end def insync?(is) provider.package_settings_insync?(should, is) end def should_to_s(newvalue) if provider.respond_to?(:package_settings_should_to_s) provider.package_settings_should_to_s(should, newvalue) else super(newvalue) end end def is_to_s(currentvalue) if provider.respond_to?(:package_settings_is_to_s) provider.package_settings_is_to_s(should, currentvalue) else super(currentvalue) end end def change_to_s(currentvalue, newvalue) if provider.respond_to?(:package_settings_change_to_s) provider.package_settings_change_to_s(currentvalue, newvalue) else super(currentvalue,newvalue) end end end newparam(:source) do - desc "Where to find the actual package. This must be a local file - (or on a network file system) or a URL that your specific - packaging type understands; Puppet will not retrieve files for you, - although you can manage packages as `file` resources." + desc "Where to find the package file. This is only used by providers that don't + automatically download packages from a central repository. (For example: + the `yum` and `apt` providers ignore this attribute, but the `rpm` and + `dpkg` providers require it.) + + Different providers accept different values for `source`. Most providers + accept paths to local files stored on the target system. Some providers + may also accept URLs or network drive paths. Puppet will not + automatically retrieve source files for you, and usually just passes the + value of `source` to the package installation command. + + You can use a `file` resource if you need to manually copy package files + to the target system." validate do |value| provider.validate_source(value) end end newparam(:instance) do desc "A read-only parameter set by the package." end newparam(:status) do desc "A read-only parameter set by the package." end newparam(:adminfile) do desc "A file containing package defaults for installing packages. - This is currently only used on Solaris. The value will be - validated according to system rules, which in the case of - Solaris means that it should either be a fully qualified path - or it should be in `/var/sadm/install/admin`." + + This attribute is only used on Solaris. Its value should be a path to a + local file stored on the target system. Solaris's package tools expect + either an absolute file path or a relative path to a file in + `/var/sadm/install/admin`. + + The value of `adminfile` will be passed directly to the `pkgadd` or + `pkgrm` command with the `-a ` option." end newparam(:responsefile) do desc "A file containing any necessary answers to questions asked by the package. This is currently used on Solaris and Debian. The value will be validated according to system rules, but it should generally be a fully qualified path." end newparam(:configfiles) do desc "Whether configfiles should be kept or replaced. Most packages types do not support this parameter. Defaults to `keep`." defaultto :keep newvalues(:keep, :replace) end newparam(:category) do desc "A read-only parameter set by the package." end newparam(:platform) do desc "A read-only parameter set by the package." end newparam(:root) do desc "A read-only parameter set by the package." end newparam(:vendor) do desc "A read-only parameter set by the package." end newparam(:description) do desc "A read-only parameter set by the package." end newparam(:allowcdrom) do desc "Tells apt to allow cdrom sources in the sources.list file. Normally apt will bail if you try this." newvalues(:true, :false) end newparam(:flavor) do desc "OpenBSD supports 'flavors', which are further specifications for which type of package you want." end newparam(:install_options, :parent => Puppet::Parameter::PackageOptions, :required_features => :install_options) do desc <<-EOT An array of additional options to pass when installing a package. These options are package-specific, and should be documented by the software vendor. One commonly implemented option is `INSTALLDIR`: package { 'mysql': ensure => installed, source => 'N:/packages/mysql-5.5.16-winx64.msi', install_options => [ '/S', { 'INSTALLDIR' => 'C:\\mysql-5.5' } ], } Each option in the array can either be a string or a hash, where each key and value pair are interpreted in a provider specific way. Each option will automatically be quoted when passed to the install command. On Windows, this is the **only** place in Puppet where backslash separators should be used. Note that backslashes in double-quoted strings _must_ be double-escaped and backslashes in single-quoted strings _may_ be double-escaped. EOT end newparam(:uninstall_options, :parent => Puppet::Parameter::PackageOptions, :required_features => :uninstall_options) do desc <<-EOT An array of additional options to pass when uninstalling a package. These options are package-specific, and should be documented by the software vendor. For example: package { 'VMware Tools': ensure => absent, uninstall_options => [ { 'REMOVE' => 'Sync,VSS' } ], } Each option in the array can either be a string or a hash, where each key and value pair are interpreted in a provider specific way. Each option will automatically be quoted when passed to the uninstall command. On Windows, this is the **only** place in Puppet where backslash separators should be used. Note that backslashes in double-quoted strings _must_ be double-escaped and backslashes in single-quoted strings _may_ be double-escaped. EOT end newparam(:allow_virtual, :boolean => true, :parent => Puppet::Parameter::Boolean, :required_features => :virtual_packages) do desc 'Specifies if virtual package names are allowed for install and uninstall.' defaultto true end autorequire(:file) do autos = [] [:responsefile, :adminfile].each { |param| if val = self[param] autos << val end } if source = self[:source] and absolute_path?(source) autos << source end autos end # This only exists for testing. def clear if obj = @parameters[:ensure] obj.latest = nil end end # The 'query' method returns a hash of info if the package # exists and returns nil if it does not. def exists? @provider.get(:ensure) != :absent end def present?(current_values) super && current_values[:ensure] != :purged end # This parameter exists to ensure backwards compatibility is preserved. # See https://github.com/puppetlabs/puppet/pull/2614 for discussion. # If/when a metaparameter for controlling how arbitrary resources respond # to refreshing is created, that will supersede this, and this will be # deprecated. newparam(:reinstall_on_refresh) do desc "Whether this resource should respond to refresh events (via `subscribe`, `notify`, or the `~>` arrow) by reinstalling the package. Only works for providers that support the `reinstallable` feature. This is useful for source-based distributions, where you may want to recompile a package if the build options change. If you use this, be careful of notifying classes when you want to restart services. If the class also contains a refreshable package, doing so could cause unnecessary re-installs. Defaults to `false`." newvalues(:true, :false) defaultto :false end # When a refresh event is triggered, calls reinstall on providers # that support the reinstall_on_refresh parameter. def refresh if provider.reinstallable? && @parameters[:reinstall_on_refresh].value == :true && @parameters[:ensure].value != :purged && @parameters[:ensure].value != :absent && @parameters[:ensure].value != :held provider.reinstall end end end end diff --git a/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/lib/puppet/parser/functions/callee.rb b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/lib/puppet/parser/functions/callee.rb new file mode 100644 index 000000000..a95604576 --- /dev/null +++ b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/lib/puppet/parser/functions/callee.rb @@ -0,0 +1,8 @@ +module Puppet::Parser::Functions + newfunction(:callee, :type => :rvalue, :doc => <<-EOS + A function using the 3x API + EOS + ) do |arguments| + "usee::callee() got '#{arguments[0]}'" + end +end diff --git a/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/lib/puppet/parser/functions/callee_ws.rb b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/lib/puppet/parser/functions/callee_ws.rb new file mode 100644 index 000000000..c7f087f4b --- /dev/null +++ b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/lib/puppet/parser/functions/callee_ws.rb @@ -0,0 +1,8 @@ +module Puppet::Parser::Functions + newfunction(:callee_ws, :type => :rvalue, :doc => <<-EOS + A function using the 3x API + EOS + ) do |arguments| + "usee::callee_ws() got '#{self['passed_in_scope']}'" + end +end \ No newline at end of file diff --git a/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/metadata.json b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/metadata.json new file mode 100644 index 000000000..35daeaf58 --- /dev/null +++ b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/usee/metadata.json @@ -0,0 +1,9 @@ +{ + "name": "test-usee", + "author": "test", + "description": "", + "license": "", + "source": "", + "version": "1.0.0", + "dependencies": [] +} diff --git a/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/lib/puppet/functions/user/caller.rb b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/lib/puppet/functions/user/caller.rb new file mode 100644 index 000000000..d0bdc1692 --- /dev/null +++ b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/lib/puppet/functions/user/caller.rb @@ -0,0 +1,5 @@ +Puppet::Functions.create_function(:'user::caller') do + def caller() + call_function('callee', 'first') + ' - ' + call_function('callee', 'second') + end +end diff --git a/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/lib/puppet/functions/user/caller_ws.rb b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/lib/puppet/functions/user/caller_ws.rb new file mode 100644 index 000000000..8241c7e3c --- /dev/null +++ b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/lib/puppet/functions/user/caller_ws.rb @@ -0,0 +1,12 @@ +Puppet::Functions.create_function(:'user::caller_ws', Puppet::Functions::InternalFunction) do + dispatch :caller_ws do + scope_param + param 'String', :value + end + + def caller_ws(scope, value) + scope = scope.compiler.newscope(scope) + scope['passed_in_scope'] = value + call_function_with_scope(scope, 'callee_ws') + end +end diff --git a/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/metadata.json b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/metadata.json new file mode 100644 index 000000000..833e45932 --- /dev/null +++ b/spec/fixtures/unit/pops/loaders/loaders/mix_4x_and_3x_functions/user/metadata.json @@ -0,0 +1,9 @@ +{ + "name": "test-user", + "author": "test", + "description": "", + "license": "", + "source": "", + "version": "1.0.0", + "dependencies": [{ "name": "test/usee" }] +} diff --git a/spec/unit/pops/loaders/loaders_spec.rb b/spec/unit/pops/loaders/loaders_spec.rb index 76958c714..73917760d 100644 --- a/spec/unit/pops/loaders/loaders_spec.rb +++ b/spec/unit/pops/loaders/loaders_spec.rb @@ -1,125 +1,146 @@ require 'spec_helper' require 'puppet_spec/files' require 'puppet/pops' require 'puppet/loaders' describe 'loader helper classes' do it 'NamedEntry holds values and is frozen' do ne = Puppet::Pops::Loader::Loader::NamedEntry.new('name', 'value', 'origin') expect(ne.frozen?).to be_truthy expect(ne.typed_name).to eql('name') expect(ne.origin).to eq('origin') expect(ne.value).to eq('value') end it 'TypedName holds values and is frozen' do tn = Puppet::Pops::Loader::Loader::TypedName.new(:function, '::foo::bar') expect(tn.frozen?).to be_truthy expect(tn.type).to eq(:function) expect(tn.name_parts).to eq(['foo', 'bar']) expect(tn.name).to eq('foo::bar') expect(tn.qualified).to be_truthy end end describe 'loaders' do include PuppetSpec::Files let(:module_without_metadata) { File.join(config_dir('wo_metadata_module'), 'modules') } + let(:mix_4x_and_3x_functions) { config_dir('mix_4x_and_3x_functions') } let(:module_with_metadata) { File.join(config_dir('single_module'), 'modules') } let(:dependent_modules_with_metadata) { config_dir('dependent_modules_with_metadata') } let(:empty_test_env) { environment_for() } # Loaders caches the puppet_system_loader, must reset between tests before(:each) { Puppet::Pops::Loaders.clear() } it 'creates a puppet_system loader' do loaders = Puppet::Pops::Loaders.new(empty_test_env) expect(loaders.puppet_system_loader()).to be_a(Puppet::Pops::Loader::ModuleLoaders::FileBased) end it 'creates an environment loader' do loaders = Puppet::Pops::Loaders.new(empty_test_env) expect(loaders.public_environment_loader()).to be_a(Puppet::Pops::Loader::SimpleEnvironmentLoader) expect(loaders.public_environment_loader().to_s).to eql("(SimpleEnvironmentLoader 'environment:*test*')") expect(loaders.private_environment_loader()).to be_a(Puppet::Pops::Loader::DependencyLoader) expect(loaders.private_environment_loader().to_s).to eql("(DependencyLoader 'environment' [])") end it 'can load a function using a qualified or unqualified name from a module with metadata' do loaders = Puppet::Pops::Loaders.new(environment_for(module_with_metadata)) modulea_loader = loaders.public_loader_for_module('modulea') unqualified_function = modulea_loader.load_typed(typed_name(:function, 'rb_func_a')).value qualified_function = modulea_loader.load_typed(typed_name(:function, 'modulea::rb_func_a')).value expect(unqualified_function).to be_a(Puppet::Functions::Function) expect(qualified_function).to be_a(Puppet::Functions::Function) expect(unqualified_function.class.name).to eq('rb_func_a') expect(qualified_function.class.name).to eq('modulea::rb_func_a') end it 'can load a function with a qualified name from module without metadata' do loaders = Puppet::Pops::Loaders.new(environment_for(module_without_metadata)) moduleb_loader = loaders.public_loader_for_module('moduleb') function = moduleb_loader.load_typed(typed_name(:function, 'moduleb::rb_func_b')).value expect(function).to be_a(Puppet::Functions::Function) expect(function.class.name).to eq('moduleb::rb_func_b') end it 'cannot load an unqualified function from a module without metadata' do loaders = Puppet::Pops::Loaders.new(environment_for(module_without_metadata)) moduleb_loader = loaders.public_loader_for_module('moduleb') expect(moduleb_loader.load_typed(typed_name(:function, 'rb_func_b'))).to be_nil end it 'makes all other modules visible to a module without metadata' do env = environment_for(module_with_metadata, module_without_metadata) loaders = Puppet::Pops::Loaders.new(env) moduleb_loader = loaders.private_loader_for_module('moduleb') function = moduleb_loader.load_typed(typed_name(:function, 'moduleb::rb_func_b')).value expect(function.call({})).to eql("I am modulea::rb_func_a() + I am moduleb::rb_func_b()") end it 'makes dependent modules visible to a module with metadata' do env = environment_for(dependent_modules_with_metadata) loaders = Puppet::Pops::Loaders.new(env) moduleb_loader = loaders.private_loader_for_module('user') function = moduleb_loader.load_typed(typed_name(:function, 'user::caller')).value expect(function.call({})).to eql("usee::callee() was told 'passed value' + I am user::caller()") end + context 'with scope' do + let(:env) { environment_for(mix_4x_and_3x_functions) } + let(:scope) { Puppet::Parser::Compiler.new(Puppet::Node.new("test", :environment => env)).newscope(nil) } + let(:loader) { Puppet::Pops::Loaders.new(env).private_loader_for_module('user') } + + it 'can call 3x function in dependent module from a 4x function' do + Puppet.override({ :current_environment => scope.environment, :global_scope => scope }) do + function = loader.load_typed(typed_name(:function, 'user::caller')).value + expect(function.call(scope)).to eql("usee::callee() got 'first' - usee::callee() got 'second'") + end + end + + it 'can call 3x function and propagate caller scope from a 4x function' do + Puppet.override({ :current_environment => scope.environment, :global_scope => scope }) do + function = loader.load_typed(typed_name(:function, 'user::caller_ws')).value + expect(function.call(scope, 'passed in scope')).to eql("usee::callee_ws() got 'passed in scope'") + end + end + end + it 'can load a function more than once from modules' do env = environment_for(dependent_modules_with_metadata) loaders = Puppet::Pops::Loaders.new(env) moduleb_loader = loaders.private_loader_for_module('user') function = moduleb_loader.load_typed(typed_name(:function, 'user::caller')).value expect(function.call({})).to eql("usee::callee() was told 'passed value' + I am user::caller()") function = moduleb_loader.load_typed(typed_name(:function, 'user::caller')).value expect(function.call({})).to eql("usee::callee() was told 'passed value' + I am user::caller()") end def environment_for(*module_paths) - Puppet::Node::Environment.create(:'*test*', module_paths, '') + Puppet::Node::Environment.create(:'*test*', module_paths) end def typed_name(type, name) Puppet::Pops::Loader::Loader::TypedName.new(type, name) end def config_dir(config_name) my_fixture(config_name) end end diff --git a/spec/unit/pops/parser/lexer2_spec.rb b/spec/unit/pops/parser/lexer2_spec.rb index 11ad574ae..805b62ba0 100644 --- a/spec/unit/pops/parser/lexer2_spec.rb +++ b/spec/unit/pops/parser/lexer2_spec.rb @@ -1,475 +1,477 @@ require 'spec_helper' require 'matchers/match_tokens2' require 'puppet/pops' require 'puppet/pops/parser/lexer2' module EgrammarLexer2Spec def tokens_scanned_from(s) lexer = Puppet::Pops::Parser::Lexer2.new lexer.string = s tokens = lexer.fullscan[0..-2] end def epp_tokens_scanned_from(s) lexer = Puppet::Pops::Parser::Lexer2.new lexer.string = s tokens = lexer.fullscan_epp[0..-2] end end describe 'Lexer2' do include EgrammarLexer2Spec { :LISTSTART => '[', :RBRACK => ']', :LBRACE => '{', :RBRACE => '}', :LPAREN => '(', :RPAREN => ')', :EQUALS => '=', :ISEQUAL => '==', :GREATEREQUAL => '>=', :GREATERTHAN => '>', :LESSTHAN => '<', :LESSEQUAL => '<=', :NOTEQUAL => '!=', :NOT => '!', :COMMA => ',', :DOT => '.', :COLON => ':', :AT => '@', :LLCOLLECT => '<<|', :RRCOLLECT => '|>>', :LCOLLECT => '<|', :RCOLLECT => '|>', :SEMIC => ';', :QMARK => '?', :OTHER => '\\', :FARROW => '=>', :PARROW => '+>', :APPENDS => '+=', :DELETES => '-=', :PLUS => '+', :MINUS => '-', :DIV => '/', :TIMES => '*', :LSHIFT => '<<', :RSHIFT => '>>', :MATCH => '=~', :NOMATCH => '!~', :IN_EDGE => '->', :OUT_EDGE => '<-', :IN_EDGE_SUB => '~>', :OUT_EDGE_SUB => '<~', :PIPE => '|', }.each do |name, string| it "should lex a token named #{name.to_s}" do expect(tokens_scanned_from(string)).to match_tokens2(name) end end it "should lex [ in position after non whitespace as LBRACK" do expect(tokens_scanned_from("a[")).to match_tokens2(:NAME, :LBRACK) end { "case" => :CASE, "class" => :CLASS, "default" => :DEFAULT, "define" => :DEFINE, # "import" => :IMPORT, # done as a function in egrammar "if" => :IF, "elsif" => :ELSIF, "else" => :ELSE, "inherits" => :INHERITS, "node" => :NODE, "and" => :AND, "or" => :OR, "undef" => :UNDEF, "false" => :BOOLEAN, "true" => :BOOLEAN, "in" => :IN, "unless" => :UNLESS, }.each do |string, name| it "should lex a keyword from '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2(name) end end # TODO: Complete with all edge cases [ 'A', 'A::B', '::A', '::A::B',].each do |string| it "should lex a CLASSREF on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:CLASSREF, string]) end end # TODO: Complete with all edge cases [ 'a', 'a::b', '::a', '::a::b',].each do |string| it "should lex a NAME on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:NAME, string]) end end [ 'a-b', 'a--b', 'a-b-c', '_x'].each do |string| it "should lex a BARE WORD STRING on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:WORD, string]) end end [ '_x::y', 'x::_y'].each do |string| it "should consider the bare word '#{string}' to be a bad NAME" do expect { tokens_scanned_from(string) }.to raise_error(/Illegal fully qualified name/) end end { '-a' => [:MINUS, :NAME], '--a' => [:MINUS, :MINUS, :NAME], 'a-' => [:NAME, :MINUS], 'a- b' => [:NAME, :MINUS, :NAME], 'a--' => [:NAME, :MINUS, :MINUS], 'a-$3' => [:NAME, :MINUS, :VARIABLE], }.each do |source, expected| it "should lex leading and trailing hyphens from #{source}" do expect(tokens_scanned_from(source)).to match_tokens2(*expected) end end { 'false'=>false, 'true'=>true}.each do |string, value| it "should lex a BOOLEAN on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:BOOLEAN, value]) end end [ '0', '1', '2982383139'].each do |string| it "should lex a decimal integer NUMBER on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:NUMBER, string]) end end { ' 1' => '1', '1 ' => '1', ' 1 ' => '1'}.each do |string, value| it "should lex a NUMBER with surrounding space '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:NUMBER, value]) end end [ '0.0', '0.1', '0.2982383139', '29823.235', '10e23', '10e-23', '1.234e23'].each do |string| it "should lex a decimal floating point NUMBER on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:NUMBER, string]) end end [ '00', '01', '0123', '0777'].each do |string| it "should lex an octal integer NUMBER on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:NUMBER, string]) end end [ '0x0', '0x1', '0xa', '0xA', '0xabcdef', '0xABCDEF'].each do |string| it "should lex an hex integer NUMBER on the form '#{string}'" do expect(tokens_scanned_from(string)).to match_tokens2([:NUMBER, string]) end end { "''" => '', "'a'" => 'a', "'a\\'b'" =>"a'b", "'a\\rb'" =>"a\\rb", "'a\\nb'" =>"a\\nb", "'a\\tb'" =>"a\\tb", "'a\\sb'" =>"a\\sb", "'a\\$b'" =>"a\\$b", "'a\\\"b'" =>"a\\\"b", "'a\\\\b'" =>"a\\b", "'a\\\\'" =>"a\\", }.each do |source, expected| it "should lex a single quoted STRING on the form #{source}" do expect(tokens_scanned_from(source)).to match_tokens2([:STRING, expected]) end end { "''" => [2, ""], "'a'" => [3, "a"], "'a\\'b'" => [6, "a'b"], }.each do |source, expected| it "should lex a single quoted STRING on the form #{source} as having length #{expected[0]}" do length, value = expected expect(tokens_scanned_from(source)).to match_tokens2([:STRING, value, {:line => 1, :pos=>1, :length=> length}]) end end { '""' => '', '"a"' => 'a', '"a\'b"' => "a'b", }.each do |source, expected| it "should lex a double quoted STRING on the form #{source}" do expect(tokens_scanned_from(source)).to match_tokens2([:STRING, expected]) end end { '"a$x b"' => [[:DQPRE, 'a', {:line => 1, :pos=>1, :length=>2 }], [:VARIABLE, 'x', {:line => 1, :pos=>3, :length=>2 }], [:DQPOST, ' b', {:line => 1, :pos=>5, :length=>3 }]], '"a$x.b"' => [[:DQPRE, 'a', {:line => 1, :pos=>1, :length=>2 }], [:VARIABLE, 'x', {:line => 1, :pos=>3, :length=>2 }], [:DQPOST, '.b', {:line => 1, :pos=>5, :length=>3 }]], '"$x.b"' => [[:DQPRE, '', {:line => 1, :pos=>1, :length=>1 }], [:VARIABLE, 'x', {:line => 1, :pos=>2, :length=>2 }], [:DQPOST, '.b', {:line => 1, :pos=>4, :length=>3 }]], '"a$x"' => [[:DQPRE, 'a', {:line => 1, :pos=>1, :length=>2 }], [:VARIABLE, 'x', {:line => 1, :pos=>3, :length=>2 }], [:DQPOST, '', {:line => 1, :pos=>5, :length=>1 }]], }.each do |source, expected| it "should lex an interpolated variable 'x' from #{source}" do expect(tokens_scanned_from(source)).to match_tokens2(*expected) end end { '"$"' => '$', '"a$"' => 'a$', '"a$%b"' => "a$%b", '"a$$"' => "a$$", '"a$$%"' => "a$$%", }.each do |source, expected| it "should lex interpolation including false starts #{source}" do expect(tokens_scanned_from(source)).to match_tokens2([:STRING, expected]) end end it "differentiates between foo[x] and foo [x] (whitespace)" do expect(tokens_scanned_from("$a[1]")).to match_tokens2(:VARIABLE, :LBRACK, :NUMBER, :RBRACK) expect(tokens_scanned_from("$a [1]")).to match_tokens2(:VARIABLE, :LISTSTART, :NUMBER, :RBRACK) expect(tokens_scanned_from("a[1]")).to match_tokens2(:NAME, :LBRACK, :NUMBER, :RBRACK) expect(tokens_scanned_from("a [1]")).to match_tokens2(:NAME, :LISTSTART, :NUMBER, :RBRACK) expect(tokens_scanned_from(" if \n\r\t\nif if ")).to match_tokens2(:IF, :IF, :IF) end it "skips whitepsace" do expect(tokens_scanned_from(" if if if ")).to match_tokens2(:IF, :IF, :IF) expect(tokens_scanned_from(" if \n\r\t\nif if ")).to match_tokens2(:IF, :IF, :IF) end it "skips single line comments" do expect(tokens_scanned_from("if # comment\nif")).to match_tokens2(:IF, :IF) end ["if /* comment */\nif", "if /* comment\n */\nif", "if /*\n comment\n */\nif", ].each do |source| it "skips multi line comments" do expect(tokens_scanned_from(source)).to match_tokens2(:IF, :IF) end end { "=~" => [:MATCH, "=~ /./"], "!~" => [:NOMATCH, "!~ /./"], "," => [:COMMA, ", /./"], "(" => [:LPAREN, "( /./"], "[" => [:LISTSTART, "[ /./"], "[" => [[:NAME, :LBRACK], "a[ /./"], "[" => [[:NAME, :LISTSTART], "a [ /./"], "{" => [:LBRACE, "{ /./"], "+" => [:PLUS, "+ /./"], "-" => [:MINUS, "- /./"], "*" => [:TIMES, "* /./"], ";" => [:SEMIC, "; /./"], }.each do |token, entry| it "should lex regexp after '#{token}'" do expected = [entry[0], :REGEX].flatten expect(tokens_scanned_from(entry[1])).to match_tokens2(*expected) end end it "should lex a simple expression" do expect(tokens_scanned_from('1 + 1')).to match_tokens2([:NUMBER, '1'], :PLUS, [:NUMBER, '1']) end { "1" => ["1 /./", [:NUMBER, :DIV, :DOT, :DIV]], "'a'" => ["'a' /./", [:STRING, :DIV, :DOT, :DIV]], "true" => ["true /./", [:BOOLEAN, :DIV, :DOT, :DIV]], "false" => ["false /./", [:BOOLEAN, :DIV, :DOT, :DIV]], "/./" => ["/./ /./", [:REGEX, :DIV, :DOT, :DIV]], "a" => ["a /./", [:NAME, :DIV, :DOT, :DIV]], "A" => ["A /./", [:CLASSREF, :DIV, :DOT, :DIV]], ")" => [") /./", [:RPAREN, :DIV, :DOT, :DIV]], "]" => ["] /./", [:RBRACK, :DIV, :DOT, :DIV]], "|>" => ["|> /./", [:RCOLLECT, :DIV, :DOT, :DIV]], "|>>" => ["|>> /./", [:RRCOLLECT, :DIV, :DOT, :DIV]], + "$x" => ["$x /1/", [:VARIABLE, :DIV, :NUMBER, :DIV]], + "a-b" => ["a-b /1/", [:WORD, :DIV, :NUMBER, :DIV]], '"a$a"' => ['"a$a" /./', [:DQPRE, :VARIABLE, :DQPOST, :DIV, :DOT, :DIV]], }.each do |token, entry| it "should not lex regexp after '#{token}'" do expect(tokens_scanned_from(entry[ 0 ])).to match_tokens2(*entry[ 1 ]) end end it 'should lex assignment' do expect(tokens_scanned_from("$a = 10")).to match_tokens2([:VARIABLE, "a"], :EQUALS, [:NUMBER, '10']) end # TODO: Tricky, and heredoc not supported yet # it "should not lex regexp after heredoc" do # tokens_scanned_from("1 / /./").should match_tokens2(:NUMBER, :DIV, :REGEX) # end it "should lex regexp at beginning of input" do expect(tokens_scanned_from(" /./")).to match_tokens2(:REGEX) end it "should lex regexp right of div" do expect(tokens_scanned_from("1 / /./")).to match_tokens2(:NUMBER, :DIV, :REGEX) end context 'when lexer lexes heredoc' do it 'lexes tag, syntax and escapes, margin and right trim' do code = <<-CODE @(END:syntax/t) Tex\\tt\\n |- END CODE expect(tokens_scanned_from(code)).to match_tokens2([:HEREDOC, 'syntax'], :SUBLOCATE, [:STRING, "Tex\tt\\n"]) end it 'lexes "tag", syntax and escapes, margin, right trim and interpolation' do code = <<-CODE @("END":syntax/t) Tex\\tt\\n$var After |- END CODE expect(tokens_scanned_from(code)).to match_tokens2( [:HEREDOC, 'syntax'], :SUBLOCATE, [:DQPRE, "Tex\tt\\n"], [:VARIABLE, "var"], [:DQPOST, " After"] ) end end context 'when dealing with multi byte characters' do it 'should support unicode characters' do code = <<-CODE "x\\u2713y" CODE # >= Ruby 1.9.3 reports \u expect(tokens_scanned_from(code)).to match_tokens2([:STRING, "x\u2713y"]) end it 'should not select LISTSTART token when preceded by multibyte chars' do # This test is sensitive to the number of multibyte characters and position of the expressions # within the string - it is designed to fail if the position is calculated on the byte offset of the '[' # instead of the char offset. # code = "$a = '\u00f6\u00fc\u00fc\u00fc\u00fc\u00e4\u00e4\u00f6\u00e4'\nnotify {'x': message => B['dkda'] }\n" expect(tokens_scanned_from(code)).to match_tokens2( :VARIABLE, :EQUALS, :STRING, [:NAME, 'notify'], :LBRACE, [:STRING, 'x'], :COLON, :NAME, :FARROW, :CLASSREF, :LBRACK, :STRING, :RBRACK, :RBRACE) end end context 'when lexing epp' do it 'epp can contain just text' do code = <<-CODE This is just text CODE expect(epp_tokens_scanned_from(code)).to match_tokens2(:EPP_START, [:RENDER_STRING, " This is just text\n"]) end it 'epp can contain text with interpolated rendered expressions' do code = <<-CODE This is <%= $x %> just text CODE expect(epp_tokens_scanned_from(code)).to match_tokens2( :EPP_START, [:RENDER_STRING, " This is "], [:RENDER_EXPR, nil], [:VARIABLE, "x"], [:EPP_END, "%>"], [:RENDER_STRING, " just text\n"] ) end it 'epp can contain text with trimmed interpolated rendered expressions' do code = <<-CODE This is <%= $x -%> just text CODE expect(epp_tokens_scanned_from(code)).to match_tokens2( :EPP_START, [:RENDER_STRING, " This is "], [:RENDER_EXPR, nil], [:VARIABLE, "x"], [:EPP_END_TRIM, "-%>"], [:RENDER_STRING, "just text\n"] ) end it 'epp can contain text with expressions that are not rendered' do code = <<-CODE This is <% $x=10 %> just text CODE expect(epp_tokens_scanned_from(code)).to match_tokens2( :EPP_START, [:RENDER_STRING, " This is "], [:VARIABLE, "x"], :EQUALS, [:NUMBER, "10"], [:RENDER_STRING, " just text\n"] ) end it 'epp can skip leading space in tail text' do code = <<-CODE This is <% $x=10 -%> just text CODE expect(epp_tokens_scanned_from(code)).to match_tokens2( :EPP_START, [:RENDER_STRING, " This is "], [:VARIABLE, "x"], :EQUALS, [:NUMBER, "10"], [:RENDER_STRING, "just text\n"] ) end it 'epp can skip comments' do code = <<-CODE This is <% $x=10 -%> <%# This is an epp comment -%> just text CODE expect(epp_tokens_scanned_from(code)).to match_tokens2( :EPP_START, [:RENDER_STRING, " This is "], [:VARIABLE, "x"], :EQUALS, [:NUMBER, "10"], [:RENDER_STRING, "just text\n"] ) end it 'epp can escape epp tags' do code = <<-CODE This is <% $x=10 -%> <%% this is escaped epp %%> CODE expect(epp_tokens_scanned_from(code)).to match_tokens2( :EPP_START, [:RENDER_STRING, " This is "], [:VARIABLE, "x"], :EQUALS, [:NUMBER, "10"], [:RENDER_STRING, "<% this is escaped epp %>\n"] ) end end end