diff --git a/lib/puppet/server/fileserver.rb b/lib/puppet/server/fileserver.rb index cb32212c9..85f0f5016 100755 --- a/lib/puppet/server/fileserver.rb +++ b/lib/puppet/server/fileserver.rb @@ -1,595 +1,597 @@ require 'puppet' require 'webrick/httpstatus' require 'cgi' require 'delegate' module Puppet class FileServerError < Puppet::Error; end class Server class FileServer < Handler attr_accessor :local Puppet.setdefaults("fileserver", :fileserverconfig => ["$confdir/fileserver.conf", "Where the fileserver configuration is stored."]) CHECKPARAMS = [:mode, :type, :owner, :group, :checksum] @interface = XMLRPC::Service::Interface.new("fileserver") { |iface| iface.add_method("string describe(string, string)") iface.add_method("string list(string, string, boolean, array)") iface.add_method("string retrieve(string, string)") } # Describe a given file. This returns all of the manageable aspects # of that file. def describe(url, links = :ignore, client = nil, clientip = nil) links = links.intern if links.is_a? String if links == :manage raise Puppet::FileServerError, "Cannot currently copy links" end mount, path = convert(url, client, clientip) if client mount.debug "Describing %s for %s" % [url, client] end obj = nil unless obj = mount.check(path, links) return "" end desc = [] CHECKPARAMS.each { |check| if state = obj.state(check) unless state.is mount.debug "Manually retrieving info for %s" % check state.retrieve end desc << state.is else if check == "checksum" and obj.state(:type).is == "file" mount.notice "File %s does not have data for %s" % [obj.name, check] end desc << nil end } return desc.join("\t") end # Create a new fileserving module. def initialize(hash = {}) @mounts = {} @files = {} if hash[:Local] @local = hash[:Local] else @local = false end if hash[:Config] == false @noreadconfig = true else @config = Puppet::LoadedFile.new( hash[:Config] || Puppet[:fileserverconfig] ) @noreadconfig = false end if hash.include?(:Mount) @passedconfig = true unless hash[:Mount].is_a?(Hash) raise Puppet::DevError, "Invalid mount hash %s" % hash[:Mount].inspect end hash[:Mount].each { |dir, name| if FileTest.exists?(dir) self.mount(dir, name) end } else @passedconfig = false readconfig(false) # don't check the file the first time. end end # List a specific directory's contents. def list(url, links = :ignore, recurse = false, ignore = false, client = nil, clientip = nil) mount, path = convert(url, client, clientip) if client mount.debug "Listing %s for %s" % [url, client] end obj = nil unless FileTest.exists?(path) return "" end # We pass two paths here, but reclist internally changes one # of the arguments when called internally. desc = reclist(mount, path, path, recurse, ignore) if desc.length == 0 mount.notice "Got no information on //%s/%s" % [mount, path] return "" end desc.collect { |sub| sub.join("\t") }.join("\n") end # Mount a new directory with a name. def mount(path, name) if @mounts.include?(name) if @mounts[name] != path raise FileServerError, "%s is already mounted at %s" % [@mounts[name].path, name] else # it's already mounted; no problem return end end # Let the mounts do their own error-checking. @mounts[name] = Mount.new(name, path) @mounts[name].info "Mounted %s" % path return @mounts[name] end # Retrieve a file from the local disk and pass it to the remote # client. def retrieve(url, links = :ignore, client = nil, clientip = nil) links = links.intern if links.is_a? String mount, path = convert(url, client, clientip) if client mount.info "Sending %s to %s" % [url, client] end unless FileTest.exists?(path) return "" end links = links.intern if links.is_a? String if links == :ignore and FileTest.symlink?(path) return "" end str = nil if links == :manage raise Puppet::Error, "Cannot copy links yet." else str = File.read(path) end if @local return str else return CGI.escape(str) end end def umount(name) @mounts.delete(name) if @mounts.include? name end private def authcheck(file, mount, client, clientip) unless mount.allowed?(client, clientip) mount.warning "%s cannot access %s" % [client, file] raise Puppet::Server::AuthorizationError, "Cannot access %s" % mount end end def convert(url, client, clientip) readconfig + url = URI.unescape(url) + mount, stub = splitpath(url, client) authcheck(url, mount, client, clientip) path = nil unless path = mount.subdir(stub, client) mount.notice "Could not find subdirectory %s" % "//%s/%s" % [mount, stub] return "" end return mount, path end # Deal with ignore parameters. def handleignore(children, path, ignore) ignore.each { |ignore| Dir.glob(File.join(path,ignore), File::FNM_DOTMATCH) { |match| children.delete(File.basename(match)) } } return children end # Read the configuration file. def readconfig(check = true) return if @noreadconfig if check and ! @config.changed? return end newmounts = {} begin File.open(@config.file) { |f| mount = nil count = 1 f.each { |line| case line when /^\s*#/: next # skip comments when /^\s*$/: next # skip blank lines when /\[(\w+)\]/: name = $1 if newmounts.include?(name) raise FileServerError, "%s is already mounted at %s" % [newmounts[name], name] end mount = Mount.new(name) newmounts[name] = mount when /^\s*(\w+)\s+(.+)$/: var = $1 value = $2 case var when "path": begin mount.path = value rescue FileServerError => detail Puppet.err "Removing mount %s: %s" % [mount.name, detail] newmounts.delete(mount.name) end when "allow": value.split(/\s*,\s*/).each { |val| begin mount.info "allowing %s access" % val mount.allow(val) rescue AuthStoreError => detail raise FileServerError, "%s at line %s of %s" % [detail.to_s, count, @config] end } when "deny": value.split(/\s*,\s*/).each { |val| begin mount.info "denying %s access" % val mount.deny(val) rescue AuthStoreError => detail raise FileServerError, "%s at line %s of %s" % [detail.to_s, count, @config] end } else raise FileServerError, "Invalid argument '%s' at line %s" % [var, count] end else raise FileServerError, "Invalid line %s: %s" % [count, line] end count += 1 } } rescue Errno::EACCES => detail Puppet.err "FileServer error: Cannot read %s; cannot serve" % @config #raise Puppet::Error, "Cannot read %s" % @config rescue Errno::ENOENT => detail Puppet.err "FileServer error: '%s' does not exist; cannot serve" % @config #raise Puppet::Error, "%s does not exit" % @config #rescue FileServerError => detail # Puppet.err "FileServer error: %s" % detail end # Verify each of the mounts are valid. # We let the check raise an error, so that it can raise an error # pointing to the specific problem. newmounts.each { |name, mount| unless mount.valid? raise FileServerError, "No path specified for mount %s" % name end } @mounts = newmounts end # Recursively list the directory. FIXME This should be using # puppet objects, not directly listing. def reclist(mount, root, path, recurse, ignore) # Take out the root of the path. name = path.sub(root, '') if name == "" name = "/" end if name == path raise FileServerError, "Could not match %s in %s" % [root, path] end desc = [name] ftype = File.stat(path).ftype desc << ftype if recurse.is_a?(Integer) recurse -= 1 end ary = [desc] if recurse == true or (recurse.is_a?(Integer) and recurse > -1) if ftype == "directory" children = Dir.entries(path) if ignore children = handleignore(children, path, ignore) end children.each { |child| next if child =~ /^\.\.?$/ reclist(mount, root, File.join(path, child), recurse, ignore).each { |cobj| ary << cobj } } end end return ary.reject { |c| c.nil? } end # Split the path into the separate mount point and path. def splitpath(dir, client) # the dir is based on one of the mounts # so first retrieve the mount path mount = nil path = nil if dir =~ %r{/(\w+)/?} mount = $1 path = dir.sub(%r{/#{mount}/?}, '') unless @mounts.include?(mount) raise FileServerError, "Fileserver module '%s' not mounted" % mount end unless @mounts[mount].valid? raise FileServerError, "Fileserver error: Mount '%s' does not have a path set" % mount end # And now replace the name with the actual object. mount = @mounts[mount] else raise FileServerError, "Fileserver error: Invalid path '%s'" % dir end if path == "" path = nil else # Remove any double slashes that might have occurred path = URI.unescape(path.gsub(/\/\//, "/")) end return mount, path end def to_s "fileserver" end # A simple class for wrapping mount points. Instances of this class # don't know about the enclosing object; they're mainly just used for # authorization. class Mount < AuthStore attr_reader :name Puppet::Util.logmethods(self, true) # Run 'retrieve' on a file. This gets the actual parameters, so # we can pass them to the client. def check(dir, links) unless FileTest.exists?(dir) self.notice "File source %s does not exist" % dir return nil end obj = fileobj(dir, links) # FIXME we should really have a timeout here -- we don't # want to actually check on every connection, maybe no more # than every 60 seconds or something. It'd be nice if we # could use the builtin scheduling to do this. # Retrieval is enough here, because we don't want to cache # any information in the state file, and we don't want to generate # any state changes or anything. We don't even need to sync # the checksum, because we're always going to hit the disk # directly. obj.retrieve return obj end # Create a map for a specific client. def clientmap(client) { "h" => client.sub(/\..*$/, ""), "H" => client, "d" => client.sub(/[^.]+\./, "") # domain name } end # Replace % patterns as appropriate. def expand(path, client = nil) # This map should probably be moved into a method. map = nil if client map = clientmap(client) else Puppet.notice "No client; expanding '%s' with local host" % path # Else, use the local information map = localmap() end path.gsub(/%(.)/) do |v| key = $1 if key == "%" "%" else map[key] || v end end end # Do we have any patterns in our path, yo? def expandable? if defined? @expandable @expandable else false end end # Create out object. It must have a name. def initialize(name, path = nil) unless name =~ %r{^\w+$} raise FileServerError, "Invalid name format '%s'" % name end @name = name if path self.path = path else @path = nil end @comp = Puppet.type(:component).create( :name => "mount[#{name}]" ) #@comp.type = "mount" #@comp.name = name super() end def fileobj(path, links) obj = nil if obj = Puppet.type(:file)[path] # This can only happen in local fileserving, but it's an # important one. It'd be nice if we didn't just set # the check params every time, but I'm not sure it's worth # the effort. obj[:check] = CHECKPARAMS else obj = Puppet.type(:file).create( :name => path, :check => CHECKPARAMS ) @comp.push(obj) end if links == :manage links = :follow end # This, ah, might be completely redundant unless obj[:links] == links obj[:links] = links end return obj end # Cache this manufactured map, since if it's used it's likely # to get used a lot. def localmap unless defined? @@localmap @@localmap = { "h" => Facter.value("hostname"), "H" => [Facter.value("hostname"), Facter.value("domain")].join("."), "d" => Facter.value("domain") } end @@localmap end # Return the path as appropriate, expanding as necessary. def path(client = nil) if expandable? return expand(@path, client) else return @path end end # Set the path. def path=(path) # FIXME: For now, just don't validate paths with replacement # patterns in them. if path =~ /%./ # Mark that we're expandable. @expandable = true else unless FileTest.exists?(path) raise FileServerError, "%s does not exist" % path end unless FileTest.directory?(path) raise FileServerError, "%s is not a directory" % path end unless FileTest.readable?(path) raise FileServerError, "%s is not readable" % path end @expandable = false end @path = path end # Retrieve a specific directory relative to a mount point. # If they pass in a client, then expand as necessary. def subdir(dir = nil, client = nil) basedir = self.path(client) dirname = if dir File.join(basedir, dir.split("/").join(File::SEPARATOR)) else basedir end dirname end def to_s "mount[#{@name}]" end # Verify our configuration is valid. This should really check to # make sure at least someone will be allowed, but, eh. def valid? return false unless @path return true end end end end end # $Id$ diff --git a/lib/puppet/type/pfile.rb b/lib/puppet/type/pfile.rb index 86b608c07..7ce384077 100644 --- a/lib/puppet/type/pfile.rb +++ b/lib/puppet/type/pfile.rb @@ -1,997 +1,993 @@ require 'digest/md5' require 'cgi' require 'etc' require 'uri' require 'fileutils' require 'puppet/type/state' require 'puppet/server/fileserver' module Puppet newtype(:file) do @doc = "Manages local files, including setting ownership and permissions, creation of both files and directories, and retrieving entire files from remote servers. As Puppet matures, it expected that the ``file`` element will be used less and less to manage content, and instead native elements will be used to do so. If you find that you are often copying files in from a central location, rather than using native elements, please contact Reductive Labs and we can hopefully work with you to develop a native element to support what you are doing." newparam(:path) do desc "The path to the file to manage. Must be fully qualified." isnamevar validate do |value| unless value =~ /^#{File::SEPARATOR}/ raise Puppet::Error, "File paths must be fully qualified" end end end newparam(:backup) do desc "Whether files should be backed up before being replaced. If a filebucket is specified, files will be backed up there; else, they will be backed up in the same directory with a ``.puppet-bak`` extension,, and no backups will be made if backup is ``false``. To use filebuckets, you must first create a filebucket in your configuration: filebucket { main: server => puppet } The ``puppetmasterd`` daemon creates a filebucket by default, so you can usually back up to your main server with this configuration. Once you've described the bucket in your configuration, you can use it in any file: file { \"/my/file\": source => \"/path/in/nfs/or/something\", backup => main } This will back the file up to the central server. At this point, the only benefits to doing so are that you do not have backup files lying around on each of your machines, a given version of a file is only backed up once, and you can restore any given file manually, no matter how old. Eventually, transactional support will be able to automatically restore filebucketed files. " attr_reader :bucket defaultto ".puppet-bak" munge do |value| case value when false, "false", :false: false when true, "true", ".puppet-bak", :true: ".puppet-bak" when String: # We can't depend on looking this up right now, # we have to do it after all of the objects # have been instantiated. @bucket = value value else self.fail "Invalid backup type %s" % value.inspect end end # Provide a straight-through hook for setting the bucket. def bucket=(bucket) @value = bucket @bucket = bucket end end newparam(:linkmaker) do desc "An internal parameter used by the *symlink* type to do recursive link creation." end newparam(:recurse) do desc "Whether and how deeply to do recursive management." newvalues(:true, :false, :inf, /^[0-9]+$/) munge do |value| newval = super(value) case newval when :true, :inf: true when :false: false else newval end end end newparam(:replace) do desc "Whether or not to replace a file that is sourced but exists. This is useful for using file sources purely for initialization." newvalues(:true, :false) defaultto :true end newparam(:force) do desc "Force the file operation. Currently only used when replacing directories with links." newvalues(:true, :false) defaultto false end newparam(:ignore) do desc "A parameter which omits action on files matching specified patterns during recursion. Uses Ruby's builtin globbing engine, so shell metacharacters are fully supported, e.g. ``[a-z]*``. Matches that would descend into the directory structure are ignored, e.g., ``*/*``." defaultto false validate do |value| unless value.is_a?(Array) or value.is_a?(String) or value == false self.devfail "Ignore must be a string or an Array" end end end newparam(:links) do desc "How to handle links during file actions. During file copying, ``follow`` will copy the target file instead of the link, ``manage`` will copy the link itself, and ``ignore`` will just pass it by. When not copying, ``manage`` and ``ignore`` behave equivalently (because you cannot really ignore links entirely during local recursion), and ``follow`` will manage the file to which the link points." newvalues(:follow, :manage, :ignore) # :ignore and :manage behave equivalently on local files, # but don't copy remote links defaultto :ignore end newparam(:purge) do desc "Whether unmanaged files should be purged. If you have a filebucket configured the purged files will be uploaded, but if you do not, this will destroy data. Only use this option for generated files unless you really know what you are doing. This option only makes sense when recursively managing directories." defaultto :false newvalues(:true, :false) end autorequire(:file) do cur = [] pary = self[:path].split(File::SEPARATOR) pary.shift # remove the initial nil pary.pop # remove us pary.inject([""]) do |ary, dir| ary << dir cur << ary.join(File::SEPARATOR) ary end cur end validate do if self[:content] and self[:source] self.fail "You cannot specify both content and a source" end end # List files, but only one level deep. def self.list(base = "/") unless FileTest.directory?(base) return [] end files = [] Dir.entries(base).reject { |e| e == "." or e == ".." }.each do |name| path = File.join(base, name) if obj = self[path] obj[:check] = :all files << obj else files << self.create( :name => path, :check => :all ) end end files end @depthfirst = false def argument?(arg) @arghash.include?(arg) end # Determine the user to write files as. def asuser if self.should(:owner) and ! self.should(:owner).is_a?(Symbol) writeable = Puppet::Util.asuser(self.should(:owner)) { FileTest.writable?(File.dirname(self[:path])) } # If the parent directory is writeable, then we execute # as the user in question. Otherwise we'll rely on # the 'owner' state to do things. if writeable asuser = self.should(:owner) end end return asuser end # We have to do some extra finishing, to retrieve our bucket if # there is one def finish # Let's cache these values, since there should really only be # a couple of these buckets @@filebuckets ||= {} # Look up our bucket, if there is one if @parameters.include?(:backup) and bucket = @parameters[:backup].bucket case bucket when String: if obj = @@filebuckets[bucket] # This sets the @value on :backup, too @parameters[:backup].bucket = obj elsif obj = Puppet.type(:filebucket).bucket(bucket) @@filebuckets[bucket] = obj @parameters[:backup].bucket = obj else self.fail "Could not find filebucket %s" % bucket end when Puppet::Client::Dipper: # things are hunky-dorey else self.fail "Invalid bucket type %s" % bucket.class end end super end # Deal with backups. def handlebackup(file = nil) # let the path be specified file ||= self[:path] # if they specifically don't want a backup, then just say # we're good unless FileTest.exists?(file) return true end unless self[:backup] return true end case File.stat(file).ftype when "directory": if self[:recurse] # we don't need to backup directories when recurse is on return true else backup = self[:backup] case backup when Puppet::Client::Dipper: notice "Recursively backing up to filebucket" require 'find' Find.find(self[:path]) do |f| if File.file?(f) sum = backup.backup(f) self.info "Filebucketed %s to %s with sum %s" % [f, backup.name, sum] end end require 'fileutils' FileUtils.rmtree(self[:path]) return true when String: newfile = file + backup # Just move it, since it's a directory. if FileTest.directory?(newfile) raise Puppet::Error, "Will not replace directory backup; use a filebucket" elsif FileTest.exists?(newfile) begin File.unlink(newfile) rescue => detail if Puppet[:trace] puts detail.backtrace end self.err "Could not remove old backup: %s" % detail return false end end begin bfile = file + backup # Ruby 1.8.1 requires the 'preserve' addition, but # later versions do not appear to require it. FileUtils.cp_r(file, bfile, :preserve => true) return true rescue => detail # since they said they want a backup, let's error out # if we couldn't make one self.fail "Could not back %s up: %s" % [file, detail.message] end else self.err "Invalid backup type %s" % backup.inspect return false end end when "file": backup = self[:backup] case backup when Puppet::Client::Dipper: sum = backup.backup(file) self.info "Filebucketed to %s with sum %s" % [backup.name, sum] return true when String: newfile = file + backup if FileTest.exists?(newfile) begin File.unlink(newfile) rescue => detail self.err "Could not remove old backup: %s" % detail return false end end begin # FIXME Shouldn't this just use a Puppet object with # 'source' specified? bfile = file + backup # Ruby 1.8.1 requires the 'preserve' addition, but # later versions do not appear to require it. FileUtils.cp(file, bfile, :preserve => true) return true rescue => detail # since they said they want a backup, let's error out # if we couldn't make one self.fail "Could not back %s up: %s" % [file, detail.message] end else self.err "Invalid backup type %s" % backup.inspect return false end else self.notice "Cannot backup files of type %s" % File.stat(file).ftype return false end end def handleignore(children) return children unless self[:ignore] self[:ignore].each { |ignore| ignored = [] Dir.glob(File.join(self[:path],ignore), File::FNM_DOTMATCH) { |match| ignored.push(File.basename(match)) } children = children - ignored } return children end def initialize(hash) # Store a copy of the arguments for later. tmphash = hash.to_hash # Used for caching clients @clients = {} super # Clean out as many references to any file paths as possible. # This was the source of many, many bugs. @arghash = tmphash @arghash.delete(self.class.namevar) if @arghash.include?(:source) @arghash.delete(:source) end if @arghash.include?(:parent) @arghash.delete(:parent) end @stat = nil end # Create a new file or directory object as a child to the current # object. def newchild(path, local, hash = {}) # make local copy of arguments args = @arghash.dup if path =~ %r{^#{File::SEPARATOR}} self.devfail( "Must pass relative paths to PFile#newchild()" ) else path = File.join(self[:path], path) end args[:path] = path unless hash.include?(:recurse) if args.include?(:recurse) if args[:recurse].is_a?(Integer) args[:recurse] -= 1 # reduce the level of recursion end end end hash.each { |key,value| args[key] = value } child = nil klass = nil # We specifically look in @parameters here, because 'linkmaker' isn't # a valid attribute for subclasses, so using 'self[:linkmaker]' throws # an error. if @parameters.include?(:linkmaker) and args.include?(:source) and ! FileTest.directory?(args[:source]) klass = Puppet.type(:symlink) # clean up the args a lot for links old = args.dup args = { :ensure => old[:source], :path => path } else klass = self.class end # The child might already exist because 'localrecurse' runs # before 'sourcerecurse'. I could push the override stuff into # a separate method or something, but the work is the same other # than this last bit, so it doesn't really make sense. if child = klass[path] unless @children.include?(child) self.debug "Not managing more explicit file %s" % path return nil end # This is only necessary for sourcerecurse, because we might have # created the object with different 'should' values than are # set remotely. unless local args.each { |var,value| next if var == :path next if var == :name # behave idempotently unless child.should(var) == value child[var] = value end } end else # create it anew #notice "Creating new file with args %s" % args.inspect args[:parent] = self begin child = klass.implicitcreate(args) # implicit creation can return nil if child.nil? return nil end @children << child rescue Puppet::Error => detail self.notice( "Cannot manage: %s" % [detail.message] ) self.debug args.inspect child = nil rescue => detail self.notice( "Cannot manage: %s" % [detail] ) self.debug args.inspect child = nil end end return child end # Paths are special for files, because we don't actually want to show # the parent's full path. def path unless defined? @path if defined? @parent # We only need to behave specially when our parent is also # a file if @parent.is_a?(self.class) # Remove the parent file name ppath = @parent.path.sub(/\/?file=.+/, '') @path = [] if ppath != "/" and ppath != "" @path << ppath end @path << self.class.name.to_s + "=" + self.name else super end else # The top-level name is always puppet[top], so we don't # bother with that. And we don't add the hostname # here, it gets added in the log server thingy. if self.name == "puppet[top]" @path = ["/"] else # We assume that if we don't have a parent that we # should not cache the path @path = [self.class.name.to_s + "=" + self.name] end end end return @path.join("/") end # Recurse into the directory. This basically just calls 'localrecurse' # and maybe 'sourcerecurse'. def recurse recurse = self[:recurse] # we might have a string, rather than a number if recurse.is_a?(String) if recurse =~ /^[0-9]+$/ recurse = Integer(recurse) #elsif recurse =~ /^inf/ # infinite recursion else # anything else is infinite recursion recurse = true end end # are we at the end of the recursion? #if recurse == 0 unless self.recurse? return end if recurse.is_a?(Integer) recurse -= 1 end self.localrecurse(recurse) if @states.include? :target self.linkrecurse(recurse) end if @states.include?(:source) self.sourcerecurse(recurse) end end def recurse? return false unless @parameters.include?(:recurse) val = @parameters[:recurse].value if val and (val == true or val > 0) return true else return false end end # Build a recursive map of a link source def linkrecurse(recurse) target = @states[:target].should method = :lstat if self[:links] == :follow method = :stat end targetstat = nil unless FileTest.exist?(target) #self.info "%s does not exist; not recursing" % # target return end # Now stat our target targetstat = File.send(method, target) unless targetstat.ftype == "directory" #self.info "%s is not a directory; not recursing" % # target return end # Now that we know our corresponding target is a directory, # change our type self[:ensure] = :directory unless FileTest.readable? target self.notice "Cannot manage %s: permission denied" % self.name return end children = Dir.entries(target).reject { |d| d =~ /^\.+$/ } #Get rid of ignored children if @parameters.include?(:ignore) children = handleignore(children) end added = [] children.each do |file| Dir.chdir(target) do longname = File.join(target, file) # Files know to create directories when recursion # is enabled and we're making links args = { :recurse => recurse, :ensure => longname } if child = self.newchild(file, true, args) unless @children.include?(child) self.push child added.push file end end end end end # Build up a recursive map of what's around right now def localrecurse(recurse) unless FileTest.exist?(self[:path]) and self.stat.directory? #self.info "%s is not a directory; not recursing" % # self[:path] return end unless FileTest.readable? self[:path] self.notice "Cannot manage %s: permission denied" % self.name return end children = Dir.entries(self[:path]) #Get rid of ignored children if @parameters.include?(:ignore) children = handleignore(children) end added = [] children.each { |file| file = File.basename(file) next if file =~ /^\.\.?$/ # skip . and .. options = {:recurse => recurse} if child = self.newchild(file, true, options) # Mark any unmanaged files for removal if purge is set. # Use the array rather than [] because tidy uses this method, too. if @parameters.include?(:purge) and self[:purge] == :true and child.implicit? child[:ensure] = :absent end unless @children.include?(child) self.push child added.push file end end } end # This recurses against the remote source and makes sure the local # and remote structures match. It's run after 'localrecurse'. def sourcerecurse(recurse) # FIXME sourcerecurse should support purging non-remote files source = @states[:source].source unless ! source.nil? and source !~ /^\s*$/ self.notice "source %s does not exist" % @states[:source].should return nil end sourceobj, path = uri2obj(source) # we'll set this manually as necessary if @arghash.include?(:ensure) @arghash.delete(:ensure) end # okay, we've got our source object; now we need to # build up a local file structure to match the remote # one server = sourceobj.server sum = "md5" if state = self.state(:checksum) sum = state.checktype end r = false if recurse unless recurse == 0 r = 1 end end - #ignore = self[:ignore] || false ignore = self[:ignore] - #self.warning "Listing path %s with ignore %s" % - # [path.inspect, ignore.inspect] desc = server.list(path, self[:links], r, ignore) - + + # Now create a new child for every file returned in the list. desc.split("\n").each { |line| file, type = line.split("\t") - next if file == "/" + next if file == "/" # skip the listing object name = file.sub(/^\//, '') - #self.warning "child name is %s" % name args = {:source => source + file} if type == file args[:recurse] = nil end + self.newchild(name, false, args) - #self.newchild(hash, source, recurse) - #hash2child(hash, source, recurse) } end # a wrapper method to make sure the file exists before doing anything def retrieve if @states.include?(:source) # This probably isn't the best place for it, but we need # to make sure that we have a corresponding checksum state. unless @states.include?(:checksum) self[:checksum] = "md5" end # We have to retrieve the source info before the recursion happens, # although I'm not exactly clear on why. @states[:source].retrieve end if @parameters.include?(:recurse) self.recurse end unless stat = self.stat(true) self.debug "File does not exist" @states.each { |name,state| # We've already retrieved the source, and we don't # want to overwrite whatever it did. This is a bit # of a hack, but oh well, source is definitely special. next if name == :source state.is = :absent } return end states().each { |state| # We don't want to call 'describe()' twice, so only do a local # retrieve on the source. if state.name == :source state.retrieve(false) else state.retrieve end } end # Set the checksum, from another state. There are multiple states that # modify the contents of a file, and they need the ability to make sure # that the checksum value is in sync. def setchecksum(sum = nil) if @states.include? :checksum if sum @states[:checksum].checksum = sum else # If they didn't pass in a sum, then tell checksum to # figure it out. @states[:checksum].retrieve @states[:checksum].checksum = @states[:checksum].is end end end # Stat our file. Depending on the value of the 'links' attribute, we use # either 'stat' or 'lstat', and we expect the states to use the resulting # stat object accordingly (mostly by testing the 'ftype' value). def stat(refresh = false) method = :stat # Files are the only types that support links if self.class.name == :file and self[:links] != :follow method = :lstat end path = self[:path] # Just skip them when they don't exist at all. unless FileTest.exists?(path) or FileTest.symlink?(path) @stat = nil return @stat end if @stat.nil? or refresh == true begin @stat = File.send(method, self[:path]) rescue Errno::ENOENT => error @stat = nil rescue Errno::EACCES => error self.warning "Could not stat; permission denied" @stat = nil end end return @stat end def uri2obj(source) sourceobj = FileSource.new path = nil unless source devfail "Got a nil source" end if source =~ /^\// source = "file://localhost/%s" % URI.escape(source) sourceobj.mount = "localhost" sourceobj.local = true end begin - uri = URI.parse(source) + uri = URI.parse(URI.escape(source)) rescue => detail self.fail "Could not understand source %s: %s" % [source, detail.to_s] end case uri.scheme when "file": unless defined? @@localfileserver @@localfileserver = Puppet::Server::FileServer.new( :Local => true, :Mount => { "/" => "localhost" }, :Config => false ) #@@localfileserver.mount("/", "localhost") end sourceobj.server = @@localfileserver path = "/localhost" + uri.path when "puppet": args = { :Server => uri.host } if uri.port args[:Port] = uri.port end # FIXME We should cache a copy of this server #sourceobj.server = Puppet::NetworkClient.new(args) unless @clients.include?(source) @clients[source] = Puppet::Client::FileClient.new(args) end sourceobj.server = @clients[source] tmp = uri.path if tmp =~ %r{^/(\w+)} sourceobj.mount = $1 path = tmp #path = tmp.sub(%r{^/\w+},'') || "/" else self.fail "Invalid source path %s" % tmp end else self.fail "Got other recursive file proto %s from %s" % [uri.scheme, source] end return [sourceobj, path.sub(/\/\//, '/')] end # Write out the file. We open the file correctly, with all of the # uid and mode and such, and then yield the file handle for actual # writing. def write(usetmp = true) mode = self.should(:mode) #if FileTest.exists?(self[:path]) if s = stat(false) # this makes sure we have a copy for posterity @backed = self.handlebackup if s.ftype == "link" and self[:links] != :follow # Remove existing links, since we're writing out a file File.unlink(self[:path]) end end # The temporary file path = nil if usetmp path = self[:path] + ".puppettmp" else path = self[:path] end # As the correct user and group Puppet::Util.asuser(asuser(), self.should(:group)) do f = nil # Open our file with the correct modes if mode Puppet::Util.withumask(000) do f = File.open(path, File::CREAT|File::WRONLY|File::TRUNC, mode) end else f = File.open(path, File::CREAT|File::WRONLY|File::TRUNC) end # Yield it yield f f.flush f.close end # And put our new file in place if usetmp begin File.rename(path, self[:path]) rescue => detail self.err "Could not rename tmp %s for replacing: %s" % [self[:path], detail] ensure # Make sure the created file gets removed if FileTest.exists?(path) File.unlink(path) end end end # And then update our checksum, so the next run doesn't find it. # FIXME This is extra work, because it's going to read the whole # file back in again. self.setchecksum end end # Puppet.type(:pfile) # the filesource class can't include the path, because the path # changes for every file instance class FileSource attr_accessor :mount, :root, :server, :local end # We put all of the states in separate files, because there are so many # of them. The order these are loaded is important, because it determines # the order they are in the state list. require 'puppet/type/pfile/checksum' require 'puppet/type/pfile/content' # can create the file require 'puppet/type/pfile/source' # can create the file require 'puppet/type/pfile/target' require 'puppet/type/pfile/ensure' # can create the file require 'puppet/type/pfile/uid' require 'puppet/type/pfile/group' require 'puppet/type/pfile/mode' require 'puppet/type/pfile/type' end # $Id$ diff --git a/test/types/filesources.rb b/test/types/filesources.rb index 3cc546b40..f121586c3 100755 --- a/test/types/filesources.rb +++ b/test/types/filesources.rb @@ -1,644 +1,677 @@ require 'puppet' require 'cgi' require 'fileutils' require 'puppettest' class TestFileSources < Test::Unit::TestCase include PuppetTest::FileTesting def setup super begin initstorage rescue system("rm -rf %s" % Puppet[:statefile]) end if defined? @port @port += 1 else @port = 8800 end end def initstorage Puppet::Storage.init Puppet::Storage.load end def clearstorage Puppet::Storage.store Puppet::Storage.clear end def test_newchild path = tempfile() @@tmpfiles.push path FileUtils.mkdir_p path File.open(File.join(path,"childtest"), "w") { |of| of.puts "yayness" } file = nil comp = nil trans = nil assert_nothing_raised { file = Puppet.type(:file).create( :name => path ) } child = nil assert_nothing_raised { child = file.newchild("childtest", true) } assert(child) assert_raise(Puppet::DevError) { file.newchild(File.join(path,"childtest"), true) } end def test_simplelocalsource path = tempfile() @@tmpfiles.push path FileUtils.mkdir_p path frompath = File.join(path,"source") topath = File.join(path,"dest") fromfile = nil tofile = nil trans = nil File.open(frompath, File::WRONLY|File::CREAT|File::APPEND) { |of| of.puts "yayness" } assert_nothing_raised { tofile = Puppet.type(:file).create( :name => topath, :source => frompath ) } assert_apply(tofile) assert(FileTest.exists?(topath), "File #{topath} is missing") from = File.open(frompath) { |o| o.read } to = File.open(topath) { |o| o.read } assert_equal(from,to) @@tmpfiles.push path end def recursive_source_test(fromdir, todir) Puppet::Type.allclear initstorage tofile = nil trans = nil assert_nothing_raised { tofile = Puppet.type(:file).create( :path => todir, :recurse => true, :backup => false, :source => fromdir ) } assert_apply(tofile) assert(FileTest.exists?(todir), "Created dir %s does not exist" % todir) Puppet::Type.allclear end def run_complex_sources(networked = false) path = tempfile() @@tmpfiles.push path # first create the source directory FileUtils.mkdir_p path # okay, let's create a directory structure fromdir = File.join(path,"fromdir") Dir.mkdir(fromdir) FileUtils.cd(fromdir) { mkranddirsandfiles() } todir = File.join(path, "todir") source = fromdir if networked source = "puppet://localhost/%s%s" % [networked, fromdir] end recursive_source_test(source, todir) return [fromdir,todir] end def test_complex_sources_twice fromdir, todir = run_complex_sources assert_trees_equal(fromdir,todir) recursive_source_test(fromdir, todir) assert_trees_equal(fromdir,todir) end def test_sources_with_deleted_destfiles fromdir, todir = run_complex_sources # then delete some files assert(FileTest.exists?(todir)) missing_files = delete_random_files(todir) # and run recursive_source_test(fromdir, todir) missing_files.each { |file| assert(FileTest.exists?(file), "Deleted file %s is still missing" % file) } # and make sure they're still equal assert_trees_equal(fromdir,todir) end def test_sources_with_readonly_destfiles fromdir, todir = run_complex_sources assert(FileTest.exists?(todir)) readonly_random_files(todir) recursive_source_test(fromdir, todir) # and make sure they're still equal assert_trees_equal(fromdir,todir) end def test_sources_with_modified_dest_files fromdir, todir = run_complex_sources assert(FileTest.exists?(todir)) # then modify some files modify_random_files(todir) recursive_source_test(fromdir, todir) # and make sure they're still equal assert_trees_equal(fromdir,todir) end def test_sources_with_added_destfiles fromdir, todir = run_complex_sources assert(FileTest.exists?(todir)) # and finally, add some new files add_random_files(todir) recursive_source_test(fromdir, todir) fromtree = file_list(fromdir) totree = file_list(todir) assert(fromtree != totree, "Trees are incorrectly equal") # then remove our new files FileUtils.cd(todir) { %x{find . 2>/dev/null}.chomp.split(/\n/).each { |file| if file =~ /file[0-9]+/ File.unlink(file) end } } # and make sure they're still equal assert_trees_equal(fromdir,todir) end def test_RecursionWithAddedFiles basedir = tempfile() Dir.mkdir(basedir) @@tmpfiles << basedir file1 = File.join(basedir, "file1") file2 = File.join(basedir, "file2") subdir1 = File.join(basedir, "subdir1") file3 = File.join(subdir1, "file") File.open(file1, "w") { |f| 3.times { f.print rand(100) } } rootobj = nil assert_nothing_raised { rootobj = Puppet.type(:file).create( :name => basedir, :recurse => true, :check => %w{type owner} ) rootobj.evaluate } klass = Puppet.type(:file) assert(klass[basedir]) assert(klass[file1]) assert_nil(klass[file2]) File.open(file2, "w") { |f| 3.times { f.print rand(100) } } assert_nothing_raised { rootobj.evaluate } assert(klass[file2]) Dir.mkdir(subdir1) File.open(file3, "w") { |f| 3.times { f.print rand(100) } } assert_nothing_raised { rootobj.evaluate } assert(klass[file3]) end def mkfileserverconf(mounts) file = tempfile() File.open(file, "w") { |f| mounts.each { |path, name| f.puts "[#{name}]\n\tpath #{path}\n\tallow *\n" } } @@tmpfiles << file return file end def test_NetworkSources server = nil basedir = tempfile() @@tmpfiles << basedir Dir.mkdir(basedir) mounts = { "/" => "root" } fileserverconf = mkfileserverconf(mounts) Puppet[:confdir] = basedir Puppet[:vardir] = basedir Puppet[:autosign] = true Puppet[:masterport] = 8762 serverpid = nil assert_nothing_raised() { server = Puppet::Server.new( :Handlers => { :CA => {}, # so that certs autogenerate :FileServer => { :Config => fileserverconf } } ) } serverpid = fork { assert_nothing_raised() { #trap(:INT) { server.shutdown; Kernel.exit! } trap(:INT) { server.shutdown } server.start } } @@tmppids << serverpid sleep(1) fromdir, todir = run_complex_sources("root") assert_trees_equal(fromdir,todir) recursive_source_test(fromdir, todir) assert_trees_equal(fromdir,todir) assert_nothing_raised { system("kill -INT %s" % serverpid) } end def test_networkSourcesWithoutService server = nil Puppet[:autosign] = true Puppet[:masterport] = 8765 serverpid = nil assert_nothing_raised() { server = Puppet::Server.new( :Handlers => { :CA => {}, # so that certs autogenerate } ) } serverpid = fork { assert_nothing_raised() { #trap(:INT) { server.shutdown; Kernel.exit! } trap(:INT) { server.shutdown } server.start } } @@tmppids << serverpid sleep(1) name = File.join(tmpdir(), "nosourcefile") file = Puppet.type(:file).create( :source => "puppet://localhost/dist/file", :name => name ) assert_nothing_raised { file.retrieve } comp = newcomp("nosource", file) assert_nothing_raised { comp.evaluate } assert(!FileTest.exists?(name), "File with no source exists anyway") end def test_unmountedNetworkSources server = nil mounts = { "/" => "root", "/noexistokay" => "noexist" } fileserverconf = mkfileserverconf(mounts) Puppet[:autosign] = true Puppet[:masterport] = @port serverpid = nil assert_nothing_raised() { server = Puppet::Server.new( :Port => @port, :Handlers => { :CA => {}, # so that certs autogenerate :FileServer => { :Config => fileserverconf } } ) } serverpid = fork { assert_nothing_raised() { #trap(:INT) { server.shutdown; Kernel.exit! } trap(:INT) { server.shutdown } server.start } } @@tmppids << serverpid sleep(1) name = File.join(tmpdir(), "nosourcefile") file = Puppet.type(:file).create( :source => "puppet://localhost/noexist/file", :name => name ) assert_nothing_raised { file.retrieve } comp = newcomp("nosource", file) assert_nothing_raised { comp.evaluate } assert(!FileTest.exists?(name), "File with no source exists anyway") end def test_alwayschecksum from = tempfile() to = tempfile() File.open(from, "w") { |f| f.puts "yayness" } File.open(to, "w") { |f| f.puts "yayness" } file = nil # Now the files should be exactly the same, so we should not see attempts # at copying assert_nothing_raised { file = Puppet.type(:file).create( :path => to, :source => from ) } file.retrieve assert(file.is(:checksum), "File does not have a checksum state") assert_equal(0, file.evaluate.length, "File produced changes") end def test_sourcepaths files = [] 3.times { files << tempfile() } to = tempfile() File.open(files[-1], "w") { |f| f.puts "yee-haw" } file = nil assert_nothing_raised { file = Puppet.type(:file).create( :name => to, :source => files ) } comp = newcomp(file) assert_events([:file_created], comp) assert(File.exists?(to), "File does not exist") txt = nil File.open(to) { |f| txt = f.read.chomp } assert_equal("yee-haw", txt, "Contents do not match") end # Make sure that source-copying updates the checksum on the same run def test_checksumchange source = tempfile() dest = tempfile() File.open(dest, "w") { |f| f.puts "boo" } File.open(source, "w") { |f| f.puts "yay" } file = nil assert_nothing_raised { file = Puppet.type(:file).create( :name => dest, :source => source ) } file.retrieve assert_events([:file_changed], file) file.retrieve assert_events([], file) end # Make sure that source-copying updates the checksum on the same run def test_sourcebeatsensure source = tempfile() dest = tempfile() File.open(source, "w") { |f| f.puts "yay" } file = nil assert_nothing_raised { file = Puppet.type(:file).create( :name => dest, :ensure => "file", :source => source ) } file.retrieve assert_events([:file_created], file) file.retrieve assert_events([], file) assert_events([], file) end def test_sourcewithlinks source = tempfile() link = tempfile() dest = tempfile() File.open(source, "w") { |f| f.puts "yay" } File.symlink(source, link) file = nil assert_nothing_raised { file = Puppet.type(:file).create( :name => dest, :source => link ) } # Default to skipping links assert_events([], file) assert(! FileTest.exists?(dest), "Created link") # Now follow the links file[:links] = :follow assert_events([:file_created], file) assert(FileTest.file?(dest), "Destination is not a file") # Now copy the links #assert_raise(Puppet::FileServerError) { trans = nil assert_nothing_raised { file[:links] = :manage comp = newcomp(file) trans = comp.evaluate trans.evaluate } assert(trans.failed?(file), "Object did not fail to copy links") end def test_changes source = tempfile() dest = tempfile() File.open(source, "w") { |f| f.puts "yay" } obj = nil assert_nothing_raised { obj = Puppet.type(:file).create( :name => dest, :source => source ) } assert_events([:file_created], obj) assert_equal(File.read(source), File.read(dest), "Files are not equal") assert_events([], obj) File.open(source, "w") { |f| f.puts "boo" } assert_events([:file_changed], obj) assert_equal(File.read(source), File.read(dest), "Files are not equal") assert_events([], obj) File.open(dest, "w") { |f| f.puts "kaboom" } # There are two changes, because first the checksum is noticed, and # then the source causes a change assert_events([:file_changed, :file_changed], obj) assert_equal(File.read(source), File.read(dest), "Files are not equal") assert_events([], obj) end def test_file_source_with_space dir = tempfile() source = File.join(dir, "file with spaces") Dir.mkdir(dir) File.open(source, "w") { |f| f.puts "yayness" } newdir = tempfile() newpath = File.join(newdir, "file with spaces") file = Puppet::Type.newfile( :path => newdir, :source => dir, :recurse => true ) assert_apply(file) assert(FileTest.exists?(newpath), "Did not create file") assert_equal("yayness\n", File.read(newpath)) end # Make sure files aren't replaced when replace is false, but otherwise # are. def test_replace source = tempfile() File.open(source, "w") { |f| f.puts "yayness" } dest = tempfile() file = Puppet::Type.newfile( :path => dest, :source => source, :recurse => true ) assert_apply(file) assert(FileTest.exists?(dest), "Did not create file") assert_equal("yayness\n", File.read(dest)) # Now set :replace assert_nothing_raised { file[:replace] = false } File.open(source, "w") { |f| f.puts "funtest" } assert_apply(file) # Make sure it doesn't change. assert_equal("yayness\n", File.read(dest)) # Now set it to true and make sure it does change. assert_nothing_raised { file[:replace] = true } assert_apply(file) # Make sure it doesn't change. assert_equal("funtest\n", File.read(dest)) end + + # Testing #285. This just makes sure that URI parsing works correctly. + def test_fileswithpoundsigns + dir = tstdir() + subdir = File.join(dir, "#dir") + Dir.mkdir(subdir) + file = File.join(subdir, "file") + File.open(file, "w") { |f| f.puts "yayness" } + + dest = tempfile() + source = "file://localhost#{dir}" + obj = Puppet::Type.newfile( + :path => dest, + :source => source, + :recurse => true + ) + + newfile = File.join(dest, "#dir", "file") + + poundsource = "file://localhost#{subdir}" + + sourceobj = path = nil + assert_nothing_raised { + sourceobj, path = obj.uri2obj(poundsource) + } + + assert_equal("/localhost" + URI.escape(subdir), path) + + assert_apply(obj) + + assert(FileTest.exists?(newfile), "File did not get created") + assert_equal("yayness\n", File.read(newfile)) + end end # $Id$