diff --git a/lib/puppet/application.rb b/lib/puppet/application.rb index 1d9611093..0e98d8adb 100644 --- a/lib/puppet/application.rb +++ b/lib/puppet/application.rb @@ -1,516 +1,513 @@ require 'optparse' require 'puppet/util/command_line' require 'puppet/util/plugins' require 'puppet/util/constant_inflector' require 'puppet/error' module Puppet # This class handles all the aspects of a Puppet application/executable # * setting up options # * setting up logs # * choosing what to run # * representing execution status # # === Usage # An application is a subclass of Puppet::Application. # # For legacy compatibility, # Puppet::Application[:example].run # is equivalent to # Puppet::Application::Example.new.run # # # class Puppet::Application::Example < Puppet::Application # # def preinit # # perform some pre initialization # @all = false # end # # # run_command is called to actually run the specified command # def run_command # send Puppet::Util::CommandLine.new.args.shift # end # # # option uses metaprogramming to create a method # # and also tells the option parser how to invoke that method # option("--arg ARGUMENT") do |v| # @args << v # end # # option("--debug", "-d") do |v| # @debug = v # end # # option("--all", "-a:) do |v| # @all = v # end # # def handle_unknown(opt,arg) # # last chance to manage an option # ... # # let's say to the framework we finally handle this option # true # end # # def read # # read action # end # # def write # # writeaction # end # # end # # === Preinit # The preinit block is the first code to be called in your application, before option parsing, # setup or command execution. # # === Options # Puppet::Application uses +OptionParser+ to manage the application options. # Options are defined with the +option+ method to which are passed various # arguments, including the long option, the short option, a description... # Refer to +OptionParser+ documentation for the exact format. # * If the option method is given a block, this one will be called whenever # the option is encountered in the command-line argument. # * If the option method has no block, a default functionnality will be used, that # stores the argument (or true/false if the option doesn't require an argument) in # the global (to the application) options array. # * If a given option was not defined by a the +option+ method, but it exists as a Puppet settings: # * if +unknown+ was used with a block, it will be called with the option name and argument # * if +unknown+ wasn't used, then the option/argument is handed to Puppet.settings.handlearg for # a default behavior # # --help is managed directly by the Puppet::Application class, but can be overriden. # # === Setup # Applications can use the setup block to perform any initialization. # The default +setup+ behaviour is to: read Puppet configuration and manage log level and destination # # === What and how to run # If the +dispatch+ block is defined it is called. This block should return the name of the registered command # to be run. # If it doesn't exist, it defaults to execute the +main+ command if defined. # # === Execution state # The class attributes/methods of Puppet::Application serve as a global place to set and query the execution # status of the application: stopping, restarting, etc. The setting of the application status does not directly # affect its running status; it's assumed that the various components within the application will consult these # settings appropriately and affect their own processing accordingly. Control operations (signal handlers and # the like) should set the status appropriately to indicate to the overall system that it's the process of # stopping or restarting (or just running as usual). # # So, if something in your application needs to stop the process, for some reason, you might consider: # # def stop_me! # # indicate that we're stopping # Puppet::Application.stop! # # ...do stuff... # end # # And, if you have some component that involves a long-running process, you might want to consider: # # def my_long_process(giant_list_to_munge) # giant_list_to_munge.collect do |member| # # bail if we're stopping # return if Puppet::Application.stop_requested? # process_member(member) # end # end class Application require 'puppet/util' include Puppet::Util DOCPATTERN = ::File.expand_path(::File.dirname(__FILE__) + "/util/command_line/*" ) CommandLineArgs = Struct.new(:subcommand_name, :args) @loader = Puppet::Util::Autoload.new(self, 'puppet/application') class << self include Puppet::Util attr_accessor :run_status def clear! self.run_status = nil end def stop! self.run_status = :stop_requested end def restart! self.run_status = :restart_requested end # Indicates that Puppet::Application.restart! has been invoked and components should # do what is necessary to facilitate a restart. def restart_requested? :restart_requested == run_status end # Indicates that Puppet::Application.stop! has been invoked and components should do what is necessary # for a clean stop. def stop_requested? :stop_requested == run_status end # Indicates that one of stop! or start! was invoked on Puppet::Application, and some kind of process # shutdown/short-circuit may be necessary. def interrupted? [:restart_requested, :stop_requested].include? run_status end # Indicates that Puppet::Application believes that it's in usual running run_mode (no stop/restart request # currently active). def clear? run_status.nil? end # Only executes the given block if the run status of Puppet::Application is clear (no restarts, stops, # etc. requested). # Upon block execution, checks the run status again; if a restart has been requested during the block's # execution, then controlled_run will send a new HUP signal to the current process. # Thus, long-running background processes can potentially finish their work before a restart. def controlled_run(&block) return unless clear? result = block.call Process.kill(:HUP, $PID) if restart_requested? result end SHOULD_PARSE_CONFIG_DEPRECATION_MSG = "is no longer supported; config file parsing " + "is now controlled by the puppet engine, rather than by individual applications. This " + "method will be removed in a future version of puppet." def should_parse_config Puppet.deprecation_warning("should_parse_config " + SHOULD_PARSE_CONFIG_DEPRECATION_MSG) end def should_not_parse_config Puppet.deprecation_warning("should_not_parse_config " + SHOULD_PARSE_CONFIG_DEPRECATION_MSG) end def should_parse_config? Puppet.deprecation_warning("should_parse_config? " + SHOULD_PARSE_CONFIG_DEPRECATION_MSG) true end # used to declare code that handle an option def option(*options, &block) long = options.find { |opt| opt =~ /^--/ }.gsub(/^--(?:\[no-\])?([^ =]+).*$/, '\1' ).gsub('-','_') fname = "handle_#{long}".intern if (block_given?) define_method(fname, &block) else define_method(fname) do |value| self.options["#{long}".to_sym] = value end end self.option_parser_commands << [options, fname] end def banner(banner = nil) @banner ||= banner end def option_parser_commands @option_parser_commands ||= ( superclass.respond_to?(:option_parser_commands) ? superclass.option_parser_commands.dup : [] ) @option_parser_commands end # @return [Array] the names of available applications # @api public def available_application_names @loader.files_to_load.map do |fn| ::File.basename(fn, '.rb') end.uniq end # Finds the class for a given application and loads the class. This does # not create an instance of the application, it only gets a handle to the # class. The code for the application is expected to live in a ruby file # `puppet/application/#{name}.rb` that is available on the `$LOAD_PATH`. # # @param application_name [String] the name of the application to find (eg. "apply"). # @return [Class] the Class instance of the application that was found. # @raise [Puppet::Error] if the application class was not found. # @raise [LoadError] if there was a problem loading the application file. # @api public def find(application_name) begin require @loader.expand(application_name.to_s.downcase) rescue LoadError => e Puppet.log_and_raise(e, "Unable to find application '#{application_name}'. #{e}") end class_name = Puppet::Util::ConstantInflector.file2constant(application_name.to_s) clazz = try_load_class(class_name) ################################################################ #### Begin 2.7.x backward compatibility hack; #### eventually we need to issue a deprecation warning here, #### and then get rid of this stanza in a subsequent release. ################################################################ if (clazz.nil?) class_name = application_name.capitalize clazz = try_load_class(class_name) end ################################################################ #### End 2.7.x backward compatibility hack ################################################################ if clazz.nil? raise Puppet::Error.new("Unable to load application class '#{class_name}' from file 'puppet/application/#{application_name}.rb'") end return clazz end # Given the fully qualified name of a class, attempt to get the class instance. # @param [String] class_name the fully qualified name of the class to try to load # @return [Class] the Class instance, or nil? if it could not be loaded. def try_load_class(class_name) return self.const_defined?(class_name) ? const_get(class_name) : nil end private :try_load_class def [](name) find(name).new end # Sets or gets the run_mode name. Sets the run_mode name if a mode_name is # passed. Otherwise, gets the run_mode or a default run_mode # def run_mode( mode_name = nil) if mode_name Puppet.settings.preferred_run_mode = mode_name end return @run_mode if @run_mode and not mode_name require 'puppet/util/run_mode' @run_mode = Puppet::Util::RunMode[ mode_name || Puppet.settings.preferred_run_mode ] end # This is for testing only def clear_everything_for_tests @run_mode = @banner = @run_status = @option_parser_commands = nil end end attr_reader :options, :command_line # Every app responds to --version # See also `lib/puppet/util/command_line.rb` for some special case early # handling of this. option("--version", "-V") do |arg| puts "#{Puppet.version}" exit end # Every app responds to --help option("--help", "-h") do |v| puts help exit end def app_defaults() Puppet::Settings.app_defaults_for_run_mode(self.class.run_mode).merge( :name => name ) end def initialize_app_defaults() Puppet.settings.initialize_app_defaults(app_defaults) end # override to execute code before running anything else def preinit end def initialize(command_line = Puppet::Util::CommandLine.new) @command_line = CommandLineArgs.new(command_line.subcommand_name, command_line.args.dup) @options = {} end # Execute the application. # @api public # @return [void] def run # I don't really like the names of these lifecycle phases. It would be nice to change them to some more meaningful # names, and make deprecated aliases. Also, Daniel suggests that we can probably get rid of this "plugin_hook" # pattern, but we need to check with PE and the community first. --cprice 2012-03-16 # exit_on_fail("get application-specific default settings") do plugin_hook('initialize_app_defaults') { initialize_app_defaults } end Puppet.push_context(Puppet.base_context(Puppet.settings), "Update for application settings (#{self.class.run_mode})") # This use of configured environment is correct, this is used to establish # the defaults for an application that does not override, or where an override # has not been made from the command line. # configured_environment_name = Puppet[:environment] if self.class.run_mode.name != :agent - configured_environment = Puppet.lookup(:environments).get(configured_environment_name) - if configured_environment.nil? - fail(Puppet::Environments::EnvironmentNotFound, configured_environment_name) - end + configured_environment = Puppet.lookup(:environments).get!(configured_environment_name) else configured_environment = Puppet::Node::Environment.remote(configured_environment_name) end configured_environment = configured_environment.override_from_commandline(Puppet.settings) # Setup a new context using the app's configuration Puppet.push_context({ :current_environment => configured_environment }, "Update current environment from application's configuration") require 'puppet/util/instrumentation' Puppet::Util::Instrumentation.init exit_on_fail("initialize") { plugin_hook('preinit') { preinit } } exit_on_fail("parse application options") { plugin_hook('parse_options') { parse_options } } exit_on_fail("prepare for execution") { plugin_hook('setup') { setup } } exit_on_fail("configure routes from #{Puppet[:route_file]}") { configure_indirector_routes } exit_on_fail("log runtime debug info") { log_runtime_environment } exit_on_fail("run") { plugin_hook('run_command') { run_command } } end def main raise NotImplementedError, "No valid command or main" end def run_command main end def setup setup_logs end def setup_logs if options[:debug] || options[:verbose] Puppet::Util::Log.newdestination(:console) end set_log_level Puppet::Util::Log.setup_default unless options[:setdest] end def set_log_level if options[:debug] Puppet::Util::Log.level = :debug elsif options[:verbose] Puppet::Util::Log.level = :info end end def handle_logdest_arg(arg) begin Puppet::Util::Log.newdestination(arg) options[:setdest] = true rescue => detail Puppet.log_exception(detail) end end def configure_indirector_routes route_file = Puppet[:route_file] if Puppet::FileSystem.exist?(route_file) routes = YAML.load_file(route_file) application_routes = routes[name.to_s] Puppet::Indirector.configure_routes(application_routes) if application_routes end end # Output basic information about the runtime environment for debugging # purposes. # # @api public # # @param extra_info [Hash{String => #to_s}] a flat hash of extra information # to log. Intended to be passed to super by subclasses. # @return [void] def log_runtime_environment(extra_info=nil) runtime_info = { 'puppet_version' => Puppet.version, 'ruby_version' => RUBY_VERSION, 'run_mode' => self.class.run_mode.name, } runtime_info['default_encoding'] = Encoding.default_external if RUBY_VERSION >= '1.9.3' runtime_info.merge!(extra_info) unless extra_info.nil? Puppet.debug 'Runtime environment: ' + runtime_info.map{|k,v| k + '=' + v.to_s}.join(', ') end def parse_options # Create an option parser option_parser = OptionParser.new(self.class.banner) # Here we're building up all of the options that the application may need to handle. The main # puppet settings defined in "defaults.rb" have already been parsed once (in command_line.rb) by # the time we get here; however, our app may wish to handle some of them specially, so we need to # make the parser aware of them again. We might be able to make this a bit more efficient by # re-using the parser object that gets built up in command_line.rb. --cprice 2012-03-16 # Add all global options to it. Puppet.settings.optparse_addargs([]).each do |option| option_parser.on(*option) do |arg| handlearg(option[0], arg) end end # Add options that are local to this application, which were # created using the "option()" metaprogramming method. If there # are any conflicts, this application's options will be favored. self.class.option_parser_commands.each do |options, fname| option_parser.on(*options) do |value| # Call the method that "option()" created. self.send(fname, value) end end # Scan command line. We just hand any exceptions to our upper levels, # rather than printing help and exiting, so that we can meaningfully # respond with context-sensitive help if we want to. --daniel 2011-04-12 option_parser.parse!(self.command_line.args) end def handlearg(opt, val) opt, val = Puppet::Settings.clean_opt(opt, val) send(:handle_unknown, opt, val) if respond_to?(:handle_unknown) end # this is used for testing def self.exit(code) exit(code) end def name self.class.to_s.sub(/.*::/,"").downcase.to_sym end def help "No help available for puppet #{name}" end def plugin_hook(step,&block) Puppet::Plugins.send("before_application_#{step}",:application_object => self) x = yield Puppet::Plugins.send("after_application_#{step}",:application_object => self, :return_value => x) x end private :plugin_hook end end diff --git a/lib/puppet/indirector/request.rb b/lib/puppet/indirector/request.rb index 04c55f9a3..a156f9de5 100644 --- a/lib/puppet/indirector/request.rb +++ b/lib/puppet/indirector/request.rb @@ -1,318 +1,317 @@ require 'cgi' require 'uri' require 'puppet/indirector' require 'puppet/util/pson' require 'puppet/network/resolver' # This class encapsulates all of the information you need to make an # Indirection call, and as a result also handles REST calls. It's somewhat # analogous to an HTTP Request object, except tuned for our Indirector. class Puppet::Indirector::Request attr_accessor :key, :method, :options, :instance, :node, :ip, :authenticated, :ignore_cache, :ignore_terminus attr_accessor :server, :port, :uri, :protocol attr_reader :indirection_name # trusted_information is specifically left out because we can't serialize it # and keep it "trusted" OPTION_ATTRIBUTES = [:ip, :node, :authenticated, :ignore_terminus, :ignore_cache, :instance, :environment] ::PSON.register_document_type('IndirectorRequest',self) def self.from_data_hash(data) raise ArgumentError, "No indirection name provided in data" unless indirection_name = data['type'] raise ArgumentError, "No method name provided in data" unless method = data['method'] raise ArgumentError, "No key provided in data" unless key = data['key'] request = new(indirection_name, method, key, nil, data['attributes']) if instance = data['instance'] klass = Puppet::Indirector::Indirection.instance(request.indirection_name).model if instance.is_a?(klass) request.instance = instance else request.instance = klass.from_data_hash(instance) end end request end def self.from_pson(json) Puppet.deprecation_warning("from_pson is being removed in favour of from_data_hash.") self.from_data_hash(json) end def to_data_hash result = { 'type' => indirection_name, 'method' => method, 'key' => key } attributes = {} OPTION_ATTRIBUTES.each do |key| next unless value = send(key) attributes[key] = value end options.each do |opt, value| attributes[opt] = value end result['attributes'] = attributes unless attributes.empty? result['instance'] = instance if instance result end def to_pson_data_hash { 'document_type' => 'IndirectorRequest', 'data' => to_data_hash, } end def to_pson(*args) to_pson_data_hash.to_pson(*args) end # Is this an authenticated request? def authenticated? # Double negative, so we just get true or false ! ! authenticated end def environment # If environment has not been set directly, we should use the application's # current environment @environment ||= Puppet.lookup(:current_environment) end def environment=(env) @environment = if env.is_a?(Puppet::Node::Environment) env elsif (current_environment = Puppet.lookup(:current_environment)).name == env current_environment else - Puppet.lookup(:environments).get(env) || - raise(Puppet::Environments::EnvironmentNotFound, env) + Puppet.lookup(:environments).get!(env) end end def escaped_key URI.escape(key) end # LAK:NOTE This is a messy interface to the cache, and it's only # used by the Configurer class. I decided it was better to implement # it now and refactor later, when we have a better design, than # to spend another month coming up with a design now that might # not be any better. def ignore_cache? ignore_cache end def ignore_terminus? ignore_terminus end def initialize(indirection_name, method, key, instance, options = {}) @instance = instance options ||= {} self.indirection_name = indirection_name self.method = method options = options.inject({}) { |hash, ary| hash[ary[0].to_sym] = ary[1]; hash } set_attributes(options) @options = options if key # If the request key is a URI, then we need to treat it specially, # because it rewrites the key. We could otherwise strip server/port/etc # info out in the REST class, but it seemed bad design for the REST # class to rewrite the key. if key.to_s =~ /^\w+:\// and not Puppet::Util.absolute_path?(key.to_s) # it's a URI set_uri_key(key) else @key = key end end @key = @instance.name if ! @key and @instance end # Look up the indirection based on the name provided. def indirection Puppet::Indirector::Indirection.instance(indirection_name) end def indirection_name=(name) @indirection_name = name.to_sym end def model raise ArgumentError, "Could not find indirection '#{indirection_name}'" unless i = indirection i.model end # Are we trying to interact with multiple resources, or just one? def plural? method == :search end # Create the query string, if options are present. def query_string return "" if options.nil? || options.empty? # For backward compatibility with older (pre-3.3) masters, # this puppet option allows serialization of query parameter # arrays as yaml. This can be removed when we remove yaml # support entirely. if Puppet.settings[:legacy_query_parameter_serialization] replace_arrays_with_yaml end "?" + encode_params(expand_into_parameters(options.to_a)) end def replace_arrays_with_yaml options.each do |key, value| case value when Array options[key] = YAML.dump(value) end end end def expand_into_parameters(data) data.inject([]) do |params, key_value| key, value = key_value expanded_value = case value when Array value.collect { |val| [key, val] } else [key_value] end params.concat(expand_primitive_types_into_parameters(expanded_value)) end end def expand_primitive_types_into_parameters(data) data.inject([]) do |params, key_value| key, value = key_value case value when nil params when true, false, String, Symbol, Fixnum, Bignum, Float params << [key, value] else raise ArgumentError, "HTTP REST queries cannot handle values of type '#{value.class}'" end end end def encode_params(params) params.collect do |key, value| "#{key}=#{CGI.escape(value.to_s)}" end.join("&") end def to_hash result = options.dup OPTION_ATTRIBUTES.each do |attribute| if value = send(attribute) result[attribute] = value end end result end def to_s return(uri ? uri : "/#{indirection_name}/#{key}") end def do_request(srv_service=:puppet, default_server=Puppet.settings[:server], default_port=Puppet.settings[:masterport], &block) # We were given a specific server to use, so just use that one. # This happens if someone does something like specifying a file # source using a puppet:// URI with a specific server. return yield(self) if !self.server.nil? if Puppet.settings[:use_srv_records] Puppet::Network::Resolver.each_srv_record(Puppet.settings[:srv_domain], srv_service) do |srv_server, srv_port| begin self.server = srv_server self.port = srv_port return yield(self) rescue SystemCallError => e Puppet.warning "Error connecting to #{srv_server}:#{srv_port}: #{e.message}" end end end # ... Fall back onto the default server. Puppet.debug "No more servers left, falling back to #{default_server}:#{default_port}" if Puppet.settings[:use_srv_records] self.server = default_server self.port = default_port return yield(self) end def remote? self.node or self.ip end private def set_attributes(options) OPTION_ATTRIBUTES.each do |attribute| if options.include?(attribute.to_sym) send(attribute.to_s + "=", options[attribute]) options.delete(attribute) end end end # Parse the key as a URI, setting attributes appropriately. def set_uri_key(key) @uri = key begin uri = URI.parse(URI.escape(key)) rescue => detail raise ArgumentError, "Could not understand URL #{key}: #{detail}", detail.backtrace end # Just short-circuit these to full paths if uri.scheme == "file" @key = Puppet::Util.uri_to_path(uri) return end @server = uri.host if uri.host # If the URI class can look up the scheme, it will provide a port, # otherwise it will default to '0'. if uri.port.to_i == 0 and uri.scheme == "puppet" @port = Puppet.settings[:masterport].to_i else @port = uri.port.to_i end @protocol = uri.scheme if uri.scheme == 'puppet' @key = URI.unescape(uri.path.sub(/^\//, '')) return end env, indirector, @key = URI.unescape(uri.path.sub(/^\//, '')).split('/',3) @key ||= '' self.environment = env unless env == '' end end diff --git a/lib/puppet/module.rb b/lib/puppet/module.rb index 3a3435c35..deecfd45b 100644 --- a/lib/puppet/module.rb +++ b/lib/puppet/module.rb @@ -1,339 +1,339 @@ require 'puppet/util/logging' require 'semver' require 'json' # Support for modules class Puppet::Module class Error < Puppet::Error; end class MissingModule < Error; end class IncompatibleModule < Error; end class UnsupportedPlatform < Error; end class IncompatiblePlatform < Error; end class MissingMetadata < Error; end class InvalidName < Error; end class InvalidFilePattern < Error; end include Puppet::Util::Logging FILETYPES = { "manifests" => "manifests", "files" => "files", "templates" => "templates", "plugins" => "lib", "pluginfacts" => "facts.d", } # Find and return the +module+ that +path+ belongs to. If +path+ is # absolute, or if there is no module whose name is the first component # of +path+, return +nil+ def self.find(modname, environment = nil) return nil unless modname # Unless a specific environment is given, use the current environment - env = environment ? Puppet.lookup(:environments).get(environment) : Puppet.lookup(:current_environment) + env = environment ? Puppet.lookup(:environments).get!(environment) : Puppet.lookup(:current_environment) env.module(modname) end attr_reader :name, :environment, :path, :metadata attr_writer :environment attr_accessor :dependencies, :forge_name attr_accessor :source, :author, :version, :license, :puppetversion, :summary, :description, :project_page def initialize(name, path, environment) @name = name @path = path @environment = environment assert_validity load_metadata if has_metadata? validate_puppet_version @absolute_path_to_manifests = Puppet::FileSystem::PathPattern.absolute(manifests) end def has_metadata? return false unless metadata_file return false unless Puppet::FileSystem.exist?(metadata_file) begin metadata = JSON.parse(File.read(metadata_file)) rescue JSON::JSONError => e Puppet.debug("#{name} has an invalid and unparsable metadata.json file. The parse error: #{e.message}") return false end return metadata.is_a?(Hash) && !metadata.keys.empty? end FILETYPES.each do |type, location| # A boolean method to let external callers determine if # we have files of a given type. define_method(type +'?') do type_subpath = subpath(location) unless Puppet::FileSystem.exist?(type_subpath) Puppet.debug("No #{type} found in subpath '#{type_subpath}' " + "(file / directory does not exist)") return false end return true end # A method for returning a given file of a given type. # e.g., file = mod.manifest("my/manifest.pp") # # If the file name is nil, then the base directory for the # file type is passed; this is used for fileserving. define_method(type.sub(/s$/, '')) do |file| # If 'file' is nil then they're asking for the base path. # This is used for things like fileserving. if file full_path = File.join(subpath(location), file) else full_path = subpath(location) end return nil unless Puppet::FileSystem.exist?(full_path) return full_path end # Return the base directory for the given type define_method(type) do subpath(location) end end def license_file return @license_file if defined?(@license_file) return @license_file = nil unless path @license_file = File.join(path, "License") end def load_metadata @metadata = data = JSON.parse(File.read(metadata_file)) @forge_name = data['name'].gsub('-', '/') if data['name'] [:source, :author, :version, :license, :puppetversion, :dependencies].each do |attr| unless value = data[attr.to_s] unless attr == :puppetversion raise MissingMetadata, "No #{attr} module metadata provided for #{self.name}" end end # NOTICE: The fallback to `versionRequirement` is something we'd like to # not have to support, but we have a reasonable number of releases that # don't use `version_requirement`. When we can deprecate this, we should. if attr == :dependencies value.tap do |dependencies| dependencies.each do |dep| dep['version_requirement'] ||= dep['versionRequirement'] || '>= 0.0.0' end end end send(attr.to_s + "=", value) end end # Return the list of manifests matching the given glob pattern, # defaulting to 'init.{pp,rb}' for empty modules. def match_manifests(rest) if rest wanted_manifests = wanted_manifests_from(rest) searched_manifests = wanted_manifests.glob.reject { |f| FileTest.directory?(f) } else searched_manifests = [] end # (#4220) Always ensure init.pp in case class is defined there. init_manifests = [manifest("init.pp"), manifest("init.rb")].compact init_manifests + searched_manifests end def all_manifests return [] unless Puppet::FileSystem.exist?(manifests) Dir.glob(File.join(manifests, '**', '*.{rb,pp}')) end def metadata_file return @metadata_file if defined?(@metadata_file) return @metadata_file = nil unless path @metadata_file = File.join(path, "metadata.json") end def modulepath File.dirname(path) if path end # Find all plugin directories. This is used by the Plugins fileserving mount. def plugin_directory subpath("lib") end def plugin_fact_directory subpath("facts.d") end def has_external_facts? File.directory?(plugin_fact_directory) end def supports(name, version = nil) @supports ||= [] @supports << [name, version] end def to_s result = "Module #{name}" result += "(#{path})" if path result end def dependencies_as_modules dependent_modules = [] dependencies and dependencies.each do |dep| author, dep_name = dep["name"].split('/') found_module = environment.module(dep_name) dependent_modules << found_module if found_module end dependent_modules end def required_by environment.module_requirements[self.forge_name] || {} end def has_local_changes? Puppet.deprecation_warning("This method is being removed.") require 'puppet/module_tool/applications' changes = Puppet::ModuleTool::Applications::Checksummer.run(path) !changes.empty? end def local_changes Puppet.deprecation_warning("This method is being removed.") require 'puppet/module_tool/applications' Puppet::ModuleTool::Applications::Checksummer.run(path) end # Identify and mark unmet dependencies. A dependency will be marked unmet # for the following reasons: # # * not installed and is thus considered missing # * installed and does not meet the version requirements for this module # * installed and doesn't use semantic versioning # # Returns a list of hashes representing the details of an unmet dependency. # # Example: # # [ # { # :reason => :missing, # :name => 'puppetlabs-mysql', # :version_constraint => 'v0.0.1', # :mod_details => { # :installed_version => '0.0.1' # } # :parent => { # :name => 'puppetlabs-bacula', # :version => 'v1.0.0' # } # } # ] # def unmet_dependencies unmet_dependencies = [] return unmet_dependencies unless dependencies dependencies.each do |dependency| forge_name = dependency['name'] version_string = dependency['version_requirement'] || '>= 0.0.0' dep_mod = begin environment.module_by_forge_name(forge_name) rescue nil end error_details = { :name => forge_name, :version_constraint => version_string.gsub(/^(?=\d)/, "v"), :parent => { :name => self.forge_name, :version => self.version.gsub(/^(?=\d)/, "v") }, :mod_details => { :installed_version => dep_mod.nil? ? nil : dep_mod.version } } unless dep_mod error_details[:reason] = :missing unmet_dependencies << error_details next end if version_string begin required_version_semver_range = SemVer[version_string] actual_version_semver = SemVer.new(dep_mod.version) rescue ArgumentError error_details[:reason] = :non_semantic_version unmet_dependencies << error_details next end unless required_version_semver_range.include? actual_version_semver error_details[:reason] = :version_mismatch unmet_dependencies << error_details next end end end unmet_dependencies end def validate_puppet_version return unless puppetversion and puppetversion != Puppet.version raise IncompatibleModule, "Module #{self.name} is only compatible with Puppet version #{puppetversion}, not #{Puppet.version}" end private def wanted_manifests_from(pattern) begin extended = File.extname(pattern).empty? ? "#{pattern}.{pp,rb}" : pattern relative_pattern = Puppet::FileSystem::PathPattern.relative(extended) rescue Puppet::FileSystem::PathPattern::InvalidPattern => error raise Puppet::Module::InvalidFilePattern.new( "The pattern \"#{pattern}\" to find manifests in the module \"#{name}\" " + "is invalid and potentially unsafe.", error) end relative_pattern.prefix_with(@absolute_path_to_manifests) end def subpath(type) File.join(path, type) end def assert_validity raise InvalidName, "Invalid module name #{name}; module names must be alphanumeric (plus '-'), not '#{name}'" unless name =~ /^[-\w]+$/ end def ==(other) self.name == other.name && self.version == other.version && self.path == other.path && self.environment == other.environment end end diff --git a/lib/puppet/module_tool.rb b/lib/puppet/module_tool.rb index 8f462f6d8..984947d33 100644 --- a/lib/puppet/module_tool.rb +++ b/lib/puppet/module_tool.rb @@ -1,194 +1,194 @@ # encoding: UTF-8 # Load standard libraries require 'pathname' require 'fileutils' require 'puppet/util/colors' module Puppet module ModuleTool require 'puppet/module_tool/tar' extend Puppet::Util::Colors # Directory and names that should not be checksummed. ARTIFACTS = ['pkg', /^\./, /^~/, /^#/, 'coverage', 'checksums.json', 'REVISION'] FULL_MODULE_NAME_PATTERN = /\A([^-\/|.]+)[-|\/](.+)\z/ REPOSITORY_URL = Puppet.settings[:module_repository] # Is this a directory that shouldn't be checksummed? # # TODO: Should this be part of Checksums? # TODO: Rename this method to reflect its purpose? # TODO: Shouldn't this be used when building packages too? def self.artifact?(path) case File.basename(path) when *ARTIFACTS true else false end end # Return the +username+ and +modname+ for a given +full_module_name+, or raise an # ArgumentError if the argument isn't parseable. def self.username_and_modname_from(full_module_name) if matcher = full_module_name.match(FULL_MODULE_NAME_PATTERN) return matcher.captures else raise ArgumentError, "Not a valid full name: #{full_module_name}" end end # Find the module root when given a path by checking each directory up from # its current location until it finds one that contains a file called # 'Modulefile'. # # @param path [Pathname, String] path to start from # @return [Pathname, nil] the root path of the module directory or nil if # we cannot find one def self.find_module_root(path) path = Pathname.new(path) if path.class == String path.expand_path.ascend do |p| return p if is_module_root?(p) end nil end # Analyse path to see if it is a module root directory by detecting a # file named 'metadata.json' or 'Modulefile' in the directory. # # @param path [Pathname, String] path to analyse # @return [Boolean] true if the path is a module root, false otherwise def self.is_module_root?(path) path = Pathname.new(path) if path.class == String FileTest.file?(path + 'metadata.json') || FileTest.file?(path + 'Modulefile') end # Builds a formatted tree from a list of node hashes containing +:text+ # and +:dependencies+ keys. def self.format_tree(nodes, level = 0) str = '' nodes.each_with_index do |node, i| last_node = nodes.length - 1 == i deps = node[:dependencies] || [] str << (indent = " " * level) str << (last_node ? "└" : "├") str << "─" str << (deps.empty? ? "─" : "┬") str << " #{node[:text]}\n" branch = format_tree(deps, level + 1) branch.gsub!(/^#{indent} /, indent + '│') unless last_node str << branch end return str end def self.build_tree(mods, dir) mods.each do |mod| version_string = mod[:version].to_s.sub(/^(?!v)/, 'v') if mod[:action] == :upgrade previous_version = mod[:previous_version].to_s.sub(/^(?!v)/, 'v') version_string = "#{previous_version} -> #{version_string}" end mod[:text] = "#{mod[:name]} (#{colorize(:cyan, version_string)})" mod[:text] += " [#{mod[:path]}]" unless mod[:path].to_s == dir.to_s deps = (mod[:dependencies] || []) deps.sort! { |a, b| a[:name] <=> b[:name] } build_tree(deps, dir) end end # @param options [Hash] This hash will contain any # command-line arguments that are not Settings, as those will have already # been extracted by the underlying application code. # # @note Unfortunately the whole point of this method is the side effect of # modifying the options parameter. This same hash is referenced both # when_invoked and when_rendering. For this reason, we are not returning # a duplicate. # @todo Validate the above note... # # An :environment_instance and a :target_dir are added/updated in the # options parameter. # # @api private def self.set_option_defaults(options) current_environment = environment_from_options(options) modulepath = [options[:target_dir]] + current_environment.full_modulepath face_environment = current_environment.override_with(:modulepath => modulepath.compact) options[:environment_instance] = face_environment # Note: environment will have expanded the path options[:target_dir] = face_environment.full_modulepath.first end # Given a hash of options, we should discover or create a # {Puppet::Node::Environment} instance that reflects the provided options. # # Generally speaking, the `:modulepath` parameter should supercede all # others, the `:environment` parameter should follow after that, and we # should default to Puppet's current environment. # # @param options [{Symbol => Object}] the options to derive environment from # @return [Puppet::Node::Environment] the environment described by the options def self.environment_from_options(options) if options[:modulepath] path = options[:modulepath].split(File::PATH_SEPARATOR) Puppet::Node::Environment.create(:anonymous, path, '') elsif options[:environment].is_a?(Puppet::Node::Environment) options[:environment] elsif options[:environment] # This use of looking up an environment is correct since it honours # a reguest to get a particular environment via environment name. - Puppet.lookup(:environments).get(options[:environment]) + Puppet.lookup(:environments).get!(options[:environment]) else Puppet.lookup(:current_environment) end end # Handles parsing of module dependency expressions into proper # {Semantic::VersionRange}s, including reasonable error handling. # # @param where [String] a description of the thing we're parsing the # dependency expression for # @param dep [Hash] the dependency description to parse # @return [Array(String, Semantic::VersionRange, String)] an tuple of the # dependent module's name, the version range dependency, and the # unparsed range expression. def self.parse_module_dependency(where, dep) dep_name = dep['name'].tr('/', '-') range = dep['version_requirement'] || dep['versionRequirement'] || '>= 0.0.0' begin parsed_range = Semantic::VersionRange.parse(range) rescue ArgumentError => e Puppet.debug "Error in #{where} parsing dependency #{dep_name} (#{e.message}); using empty range." parsed_range = Semantic::VersionRange::EMPTY_RANGE end [ dep_name, parsed_range, range ] end end end # Load remaining libraries require 'puppet/module_tool/errors' require 'puppet/module_tool/applications' require 'puppet/module_tool/checksums' require 'puppet/module_tool/contents_description' require 'puppet/module_tool/dependency' require 'puppet/module_tool/metadata' require 'puppet/module_tool/modulefile' require 'puppet/forge/cache' require 'puppet/forge' diff --git a/lib/puppet/node.rb b/lib/puppet/node.rb index 6f0d89dba..32f11168c 100644 --- a/lib/puppet/node.rb +++ b/lib/puppet/node.rb @@ -1,184 +1,184 @@ require 'puppet/indirector' # A class for managing nodes, including their facts and environment. class Puppet::Node require 'puppet/node/facts' require 'puppet/node/environment' # Set up indirection, so that nodes can be looked for in # the node sources. extend Puppet::Indirector # Use the node source as the indirection terminus. indirects :node, :terminus_setting => :node_terminus, :doc => "Where to find node information. A node is composed of its name, its facts, and its environment." attr_accessor :name, :classes, :source, :ipaddress, :parameters, :trusted_data, :environment_name attr_reader :time, :facts ::PSON.register_document_type('Node',self) def self.from_data_hash(data) raise ArgumentError, "No name provided in serialized data" unless name = data['name'] node = new(name) node.classes = data['classes'] node.parameters = data['parameters'] node.environment_name = data['environment'] node end def self.from_pson(pson) Puppet.deprecation_warning("from_pson is being removed in favour of from_data_hash.") self.from_data_hash(pson) end def to_data_hash result = { 'name' => name, 'environment' => environment.name, } result['classes'] = classes unless classes.empty? result['parameters'] = parameters unless parameters.empty? result end def to_pson_data_hash(*args) { 'document_type' => "Node", 'data' => to_data_hash, } end def to_pson(*args) to_pson_data_hash.to_pson(*args) end def environment if @environment @environment else if env = parameters["environment"] self.environment = env elsif environment_name self.environment = environment_name else # This should not be :current_environment, this is the default # for a node when it has not specified its environment # Tt will be used to establish what the current environment is. # - self.environment = Puppet.lookup(:environments).get(Puppet[:environment]) + self.environment = Puppet.lookup(:environments).get!(Puppet[:environment]) end @environment end end def environment=(env) if env.is_a?(String) or env.is_a?(Symbol) @environment = Puppet.lookup(:environments).get!(env) else @environment = env end end def has_environment_instance? !@environment.nil? end def initialize(name, options = {}) raise ArgumentError, "Node names cannot be nil" unless name @name = name if classes = options[:classes] if classes.is_a?(String) @classes = [classes] else @classes = classes end else @classes = [] end @parameters = options[:parameters] || {} @facts = options[:facts] if env = options[:environment] self.environment = env end @time = Time.now end # Merge the node facts with parameters from the node source. def fact_merge if @facts = Puppet::Node::Facts.indirection.find(name, :environment => environment) @facts.sanitize merge(@facts.values) end rescue => detail error = Puppet::Error.new("Could not retrieve facts for #{name}: #{detail}") error.set_backtrace(detail.backtrace) raise error end # Merge any random parameters into our parameter list. def merge(params) params.each do |name, value| @parameters[name] = value unless @parameters.include?(name) end @parameters["environment"] ||= self.environment.name.to_s end # Calculate the list of names we might use for looking # up our node. This is only used for AST nodes. def names return [name] if Puppet.settings[:strict_hostname_checking] names = [] names += split_name(name) if name.include?(".") # First, get the fqdn unless fqdn = parameters["fqdn"] if parameters["hostname"] and parameters["domain"] fqdn = parameters["hostname"] + "." + parameters["domain"] else Puppet.warning "Host is missing hostname and/or domain: #{name}" end end # Now that we (might) have the fqdn, add each piece to the name # list to search, in order of longest to shortest. names += split_name(fqdn) if fqdn # And make sure the node name is first, since that's the most # likely usage. # The name is usually the Certificate CN, but it can be # set to the 'facter' hostname instead. if Puppet[:node_name] == 'cert' names.unshift name else names.unshift parameters["hostname"] end names.uniq end def split_name(name) list = name.split(".") tmp = [] list.each_with_index do |short, i| tmp << list[0..i].join(".") end tmp.reverse end # Ensures the data is frozen # def trusted_data=(data) Puppet.warning("Trusted node data modified for node #{name}") unless @trusted_data.nil? @trusted_data = data.freeze end end diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 1357f948d..2f6c77257 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -1,152 +1,152 @@ require 'find' require 'forwardable' require 'puppet/node/environment' require 'puppet/parser/parser_factory' class Puppet::Parser::TypeLoader extend Forwardable # Import manifest files that match a given file glob pattern. # # @param pattern [String] the file glob to apply when determining which files # to load # @param dir [String] base directory to use when the file is not # found in a module # @api private def import(pattern, dir) return if Puppet[:ignoreimport] modname, files = Puppet::Parser::Files.find_manifests_in_modules(pattern, environment) if files.empty? abspat = File.expand_path(pattern, dir) file_pattern = abspat + (File.extname(abspat).empty? ? '{.pp,.rb}' : '' ) files = Dir.glob(file_pattern).uniq.reject { |f| FileTest.directory?(f) } modname = nil if files.empty? raise_no_files_found(pattern) end end load_files(modname, files) end # Load all of the manifest files in all known modules. # @api private def import_all # And then load all files from each module, but (relying on system # behavior) only load files from the first module of a given name. E.g., # given first/foo and second/foo, only files from first/foo will be loaded. environment.modules.each do |mod| load_files(mod.name, mod.all_manifests) end end def_delegator :environment, :known_resource_types def initialize(env) self.environment = env end def environment @environment end def environment=(env) if env.is_a?(String) or env.is_a?(Symbol) - @environment = Puppet.lookup(:environments).get(env) + @environment = Puppet.lookup(:environments).get!(env) else @environment = env end end # Try to load the object with the given fully qualified name. def try_load_fqname(type, fqname) return nil if fqname == "" # special-case main. files_to_try_for(fqname).each do |filename| begin imported_types = import_from_modules(filename) if result = imported_types.find { |t| t.type == type and t.name == fqname } Puppet.debug "Automatically imported #{fqname} from #{filename} into #{environment}" return result end rescue Puppet::ImportError => detail # I'm not convienced we should just drop these errors, but this # preserves existing behaviours. end end # Nothing found. return nil end def parse_file(file) Puppet.debug("importing '#{file}' in environment #{environment}") parser = Puppet::Parser::ParserFactory.parser(environment) parser.file = file return parser.parse end private def import_from_modules(pattern) modname, files = Puppet::Parser::Files.find_manifests_in_modules(pattern, environment) if files.empty? raise_no_files_found(pattern) end load_files(modname, files) end def raise_no_files_found(pattern) raise Puppet::ImportError, "No file(s) found for import of '#{pattern}'" end def load_files(modname, files) @loaded ||= {} loaded_asts = [] files.reject { |file| @loaded[file] }.each do |file| # NOTE: This ugly implementation will be replaced in Puppet 3.5. # The implementation now makes use of a global variable because the context support is # not available until Puppet 3.5. # The use case is that parsing for the purpose of searching for information # should not abort. There is currently one such use case in indirector/resourcetype/parser # if Puppet.lookup(:squelch_parse_errors) {|| false } begin loaded_asts << parse_file(file) rescue => e # Resume from errors so that all parseable files would # still be parsed. Mark this file as loaded so that # it would not be parsed next time (handle it as if # it was successfully parsed). Puppet.debug("Unable to parse '#{file}': #{e.message}") end else loaded_asts << parse_file(file) end @loaded[file] = true end loaded_asts.collect do |ast| known_resource_types.import_ast(ast, modname) end.flatten end # Return a list of all file basenames that should be tried in order # to load the object with the given fully qualified name. def files_to_try_for(qualified_name) qualified_name.split('::').inject([]) do |paths, name| add_path_for_name(paths, name) end end def add_path_for_name(paths, name) if paths.empty? [name] else paths.unshift(File.join(paths.first, name)) end end end diff --git a/lib/puppet/util/autoload.rb b/lib/puppet/util/autoload.rb index 51ff94e1c..ddc62734c 100644 --- a/lib/puppet/util/autoload.rb +++ b/lib/puppet/util/autoload.rb @@ -1,227 +1,227 @@ require 'pathname' require 'puppet/util/rubygems' require 'puppet/util/warnings' require 'puppet/util/methodhelper' # Autoload paths, either based on names or all at once. class Puppet::Util::Autoload include Puppet::Util::MethodHelper @autoloaders = {} @loaded = {} class << self attr_reader :autoloaders attr_accessor :loaded private :autoloaders, :loaded def gem_source @gem_source ||= Puppet::Util::RubyGems::Source.new end # Has a given path been loaded? This is used for testing whether a # changed file should be loaded or just ignored. This is only # used in network/client/master, when downloading plugins, to # see if a given plugin is currently loaded and thus should be # reloaded. def loaded?(path) path = cleanpath(path).chomp('.rb') loaded.include?(path) end # Save the fact that a given path has been loaded. This is so # we can load downloaded plugins if they've already been loaded # into memory. def mark_loaded(name, file) name = cleanpath(name).chomp('.rb') ruby_file = name + ".rb" $LOADED_FEATURES << ruby_file unless $LOADED_FEATURES.include?(ruby_file) loaded[name] = [file, File.mtime(file)] end def changed?(name) name = cleanpath(name).chomp('.rb') return true unless loaded.include?(name) file, old_mtime = loaded[name] environment = Puppet.lookup(:current_environment) return true unless file == get_file(name, environment) begin old_mtime.to_i != File.mtime(file).to_i rescue Errno::ENOENT true end end # Load a single plugin by name. We use 'load' here so we can reload a # given plugin. def load_file(name, env) file = get_file(name.to_s, env) return false unless file begin mark_loaded(name, file) Kernel.load file, @wrap return true rescue SystemExit,NoMemoryError raise rescue Exception => detail message = "Could not autoload #{name}: #{detail}" Puppet.log_exception(detail, message) raise Puppet::Error, message, detail.backtrace end end def loadall(path) # Load every instance of everything we can find. files_to_load(path).each do |file| name = file.chomp(".rb") load_file(name, nil) unless loaded?(name) end end def reload_changed loaded.keys.each { |file| load_file(file, nil) if changed?(file) } end # Get the correct file to load for a given path # returns nil if no file is found def get_file(name, env) name = name + '.rb' unless name =~ /\.rb$/ path = search_directories(env).find { |dir| Puppet::FileSystem.exist?(File.join(dir, name)) } path and File.join(path, name) end def files_to_load(path) search_directories(nil).map {|dir| files_in_dir(dir, path) }.flatten.uniq end def files_in_dir(dir, path) dir = Pathname.new(File.expand_path(dir)) Dir.glob(File.join(dir, path, "*.rb")).collect do |file| Pathname.new(file).relative_path_from(dir).to_s end end def module_directories(env) # We're using a per-thread cache of module directories so that we don't # scan the filesystem each time we try to load something. This is reset # at the beginning of compilation and at the end of an agent run. $env_module_directories ||= {} # This is a little bit of a hack. Basically, the autoloader is being # called indirectly during application bootstrapping when we do things # such as check "features". However, during bootstrapping, we haven't # yet parsed all of the command line parameters nor the config files, # and thus we don't yet know with certainty what the module path is. # This should be irrelevant during bootstrapping, because anything that # we are attempting to load during bootstrapping should be something # that we ship with puppet, and thus the module path is irrelevant. # # In the long term, I think the way that we want to handle this is to # have the autoloader ignore the module path in all cases where it is # not specifically requested (e.g., by a constructor param or # something)... because there are very few cases where we should # actually be loading code from the module path. However, until that # happens, we at least need a way to prevent the autoloader from # attempting to access the module path before it is initialized. For # now we are accomplishing that by calling the # "app_defaults_initialized?" method on the main puppet Settings object. # --cprice 2012-03-16 if Puppet.settings.app_defaults_initialized? && - env ||= Puppet.lookup(:environments).get(Puppet[:environment]) + env ||= Puppet.lookup(:environments).get!(Puppet[:environment]) # if the app defaults have been initialized then it should be safe to access the module path setting. $env_module_directories[env] ||= env.modulepath.collect do |dir| Dir.entries(dir).reject { |f| f =~ /^\./ }.collect { |f| File.join(dir, f, "lib") } end.flatten.find_all do |d| FileTest.directory?(d) end else # if we get here, the app defaults have not been initialized, so we basically use an empty module path. [] end end def libdirs() # See the comments in #module_directories above. Basically, we need to be careful not to try to access the # libdir before we know for sure that all of the settings have been initialized (e.g., during bootstrapping). if (Puppet.settings.app_defaults_initialized?) Puppet[:libdir].split(File::PATH_SEPARATOR) else [] end end def gem_directories gem_source.directories end def search_directories(env) [gem_directories, module_directories(env), libdirs(), $LOAD_PATH].flatten end # Normalize a path. This converts ALT_SEPARATOR to SEPARATOR on Windows # and eliminates unnecessary parts of a path. def cleanpath(path) # There are two cases here because cleanpath does not handle absolute # paths correctly on windows (c:\ and c:/ are treated as distinct) but # we don't want to convert relative paths to absolute if Puppet::Util.absolute_path?(path) File.expand_path(path) else Pathname.new(path).cleanpath.to_s end end end # Send [] and []= to the @autoloaders hash Puppet::Util.classproxy self, :autoloaders, "[]", "[]=" attr_accessor :object, :path, :objwarn, :wrap def initialize(obj, path, options = {}) @path = path.to_s raise ArgumentError, "Autoload paths cannot be fully qualified" if Puppet::Util.absolute_path?(@path) @object = obj self.class[obj] = self set_options(options) @wrap = true unless defined?(@wrap) end def load(name, env = nil) self.class.load_file(expand(name), env) end # Load all instances from a path of Autoload.search_directories matching the # relative path this Autoloader was initialized with. For example, if we # have created a Puppet::Util::Autoload for Puppet::Type::User with a path of # 'puppet/provider/user', the search_directories path will be searched for # all ruby files matching puppet/provider/user/*.rb and they will then be # loaded from the first directory in the search path providing them. So # earlier entries in the search path may shadow later entries. # # This uses require, rather than load, so that already-loaded files don't get # reloaded unnecessarily. def loadall self.class.loadall(@path) end def loaded?(name) self.class.loaded?(expand(name)) end def changed?(name) self.class.changed?(expand(name)) end def files_to_load self.class.files_to_load(@path) end def expand(name) ::File.join(@path, name.to_s) end end