diff --git a/lib/puppet/parser/functions/extlookup.rb b/lib/puppet/parser/functions/extlookup.rb index bc55410b9..b5688d644 100644 --- a/lib/puppet/parser/functions/extlookup.rb +++ b/lib/puppet/parser/functions/extlookup.rb @@ -1,157 +1,152 @@ require 'csv' module Puppet::Parser::Functions newfunction(:extlookup, :type => :rvalue, :doc => "This is a parser function to read data from external files, this version uses CSV files but the concept can easily be adjust for databases, yaml or any other queryable data source. The object of this is to make it obvious when it's being used, rather than magically loading data in when an module is loaded I prefer to look at the code and see statements like: $snmp_contact = extlookup(\"snmp_contact\") The above snippet will load the snmp_contact value from CSV files, this in its own is useful but a common construct in puppet manifests is something like this: case $domain { \"myclient.com\": { $snmp_contact = \"John Doe \" } default: { $snmp_contact = \"My Support \" } } Over time there will be a lot of this kind of thing spread all over your manifests and adding an additional client involves grepping through manifests to find all the places where you have constructs like this. This is a data problem and shouldn't be handled in code, a using this function you can do just that. First you configure it in site.pp: $extlookup_datadir = \"/etc/puppet/manifests/extdata\" $extlookup_precedence = [\"%{fqdn}\", \"domain_%{domain}\", \"common\"] The array tells the code how to resolve values, first it will try to find it in web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv Now create the following data files in /etc/puppet/manifests/extdata: domain_myclient.com.csv: snmp_contact,John Doe root_contact,support@%{domain} client_trusted_ips,192.168.1.130,192.168.10.0/24 common.csv: snmp_contact,My Support root_contact,support@my.com Now you can replace the case statement with the simple single line to achieve the exact same outcome: $snmp_contact = extlookup(\"snmp_contact\") The above code shows some other features, you can use any fact or variable that is in scope by simply using %{varname} in your data files, you can return arrays by just having multiple values in the csv after the initial variable name. In the event that a variable is nowhere to be found a critical error will be raised that will prevent your manifest from compiling, this is to avoid accidentally putting in empty values etc. You can however specify a default value: $ntp_servers = extlookup(\"ntp_servers\", \"1.${country}.pool.ntp.org\") In this case it will default to \"1.${country}.pool.ntp.org\" if nothing is defined in any data file. You can also specify an additional data file to search first before any others at use time, for example: $version = extlookup(\"rsyslog_version\", \"present\", \"packages\") package{\"rsyslog\": ensure => $version } This will look for a version configured in packages.csv and then in the rest as configured by $extlookup_precedence if it's not found anywhere it will default to `present`, this kind of use case makes puppet a lot nicer for managing large amounts of packages since you do not need to edit a load of manifests to do simple things like adjust a desired version number. Precedence values can have variables embedded in them in the form %{fqdn}, you could for example do: $extlookup_precedence = [\"hosts/%{fqdn}\", \"common\"] This will result in /path/to/extdata/hosts/your.box.com.csv being searched. This is for back compatibility to interpolate variables with %. % interpolation is a workaround for a problem that has been fixed: Puppet variable interpolation at top scope used to only happen on each run.") do |args| key = args[0] default = args[1] datafile = args[2] raise Puppet::ParseError, ("extlookup(): wrong number of arguments (#{args.length}; must be <= 3)") if args.length > 3 - extlookup_datadir = lookupvar('extlookup_datadir') - extlookup_precedence = Array.new + extlookup_datadir = lookupvar('::extlookup_datadir') - extlookup_precedence = lookupvar('extlookup_precedence').collect do |var| - var.gsub(/%\{(.+?)\}/) do |capture| - lookupvar($1) - end - end + extlookup_precedence = lookupvar('::extlookup_precedence').collect { |var| var.gsub(/%\{(.+?)\}/) { lookupvar("::#{$1}") } } datafiles = Array.new # if we got a custom data file, put it first in the array of search files if datafile != "" datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv") end extlookup_precedence.each do |d| datafiles << extlookup_datadir + "/#{d}.csv" end desired = nil datafiles.each do |file| if desired.nil? if File.exists?(file) result = CSV.read(file).find_all do |r| r[0] == key end # return just the single result if theres just one, # else take all the fields in the csv and build an array if result.length > 0 if result[0].length == 2 val = result[0][1].to_s # parse %{}'s in the CSV into local variables using lookupvar() while val =~ /%\{(.+?)\}/ val.gsub!(/%\{#{$1}\}/, lookupvar($1)) end desired = val elsif result[0].length > 1 length = result[0].length cells = result[0][1,length] # Individual cells in a CSV result are a weird data type and throws # puppets yaml parsing, so just map it all to plain old strings desired = cells.map do |c| # parse %{}'s in the CSV into local variables using lookupvar() while c =~ /%\{(.+?)\}/ c.gsub!(/%\{#{$1}\}/, lookupvar($1)) end c.to_s end end end end end end desired || default or raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" end end diff --git a/lib/puppet/parser/functions/fqdn_rand.rb b/lib/puppet/parser/functions/fqdn_rand.rb index 91157a148..93ab98bcd 100644 --- a/lib/puppet/parser/functions/fqdn_rand.rb +++ b/lib/puppet/parser/functions/fqdn_rand.rb @@ -1,12 +1,12 @@ Puppet::Parser::Functions::newfunction(:fqdn_rand, :type => :rvalue, :doc => "Generates random numbers based on the node's fqdn. Generated random values will be a range from 0 up to and excluding n, where n is the first parameter. The second argument specifies a number to add to the seed and is optional, for example: $random_number = fqdn_rand(30) $random_number_seed = fqdn_rand(30,30)") do |args| require 'digest/md5' max = args.shift - srand(Digest::MD5.hexdigest([lookupvar('fqdn'),args].join(':')).hex) + srand(Digest::MD5.hexdigest([lookupvar('::fqdn'),args].join(':')).hex) rand(max).to_s end diff --git a/spec/unit/parser/functions/extlookup_spec.rb b/spec/unit/parser/functions/extlookup_spec.rb index f2560d3d9..ad89f6575 100755 --- a/spec/unit/parser/functions/extlookup_spec.rb +++ b/spec/unit/parser/functions/extlookup_spec.rb @@ -1,96 +1,96 @@ #! /usr/bin/env ruby require 'spec_helper' require 'tempfile' describe "the extlookup function" do before :all do Puppet::Parser::Functions.autoloader.loadall end before :each do @scope = Puppet::Parser::Scope.new @scope.stubs(:environment).returns(Puppet::Node::Environment.new('production')) end it "should exist" do Puppet::Parser::Functions.function("extlookup").should == "function_extlookup" end it "should raise a ParseError if there is less than 1 arguments" do lambda { @scope.function_extlookup([]) }.should( raise_error(Puppet::ParseError)) end it "should raise a ParseError if there is more than 3 arguments" do lambda { @scope.function_extlookup(["foo", "bar", "baz", "gazonk"]) }.should( raise_error(Puppet::ParseError)) end it "should return the default" do result = @scope.function_extlookup([ "key", "default"]) result.should == "default" end it "should lookup the key in a supplied datafile" do t = Tempfile.new('extlookup.csv') do t.puts 'key,value' t.puts 'nonkey,nonvalue' t.close result = @scope.function_extlookup([ "key", "default", t.path]) result.should == "value" end end it "should return an array if the datafile contains more than two columns" do t = Tempfile.new('extlookup.csv') do t.puts 'key,value1,value2' t.puts 'nonkey,nonvalue,nonvalue' t.close result = @scope.function_extlookup([ "key", "default", t.path]) result.should == ["value1", "value2"] end end it "should raise an error if there's no matching key and no default" do t = Tempfile.new('extlookup.csv') do t.puts 'key,value' t.puts 'nonkey,nonvalue' t.close result = @scope.function_extlookup([ "key", nil, t.path]) result.should == "value" end end describe "should look in $extlookup_datadir for data files listed by $extlookup_precedence" do before do - @scope.stubs(:lookupvar).with('extlookup_datadir').returns("/tmp") + @scope.stubs(:lookupvar).with('::extlookup_datadir').returns("/tmp") File.open("/tmp/one.csv","w"){|one| one.puts "key,value1" } File.open("/tmp/two.csv","w") do |two| two.puts "key,value2" two.puts "key2,value_two" end end it "when the key is in the first file" do - @scope.stubs(:lookupvar).with('extlookup_precedence').returns(["one","two"]) + @scope.stubs(:lookupvar).with('::extlookup_precedence').returns(["one","two"]) result = @scope.function_extlookup([ "key" ]) result.should == "value1" end it "when the key is in the second file" do - @scope.stubs(:lookupvar).with('extlookup_precedence').returns(["one","two"]) + @scope.stubs(:lookupvar).with('::extlookup_precedence').returns(["one","two"]) result = @scope.function_extlookup([ "key2" ]) result.should == "value_two" end it "should not modify extlookup_precedence data" do variable = '%{fqdn}' - @scope.stubs(:lookupvar).with('extlookup_precedence').returns([variable,"one"]) - @scope.stubs(:lookupvar).with('fqdn').returns('myfqdn') + @scope.stubs(:lookupvar).with('::extlookup_precedence').returns([variable,"one"]) + @scope.stubs(:lookupvar).with('::fqdn').returns('myfqdn') result = @scope.function_extlookup([ "key" ]) variable.should == '%{fqdn}' end end end diff --git a/spec/unit/parser/functions/fqdn_rand_spec.rb b/spec/unit/parser/functions/fqdn_rand_spec.rb index d810741c5..90792e182 100755 --- a/spec/unit/parser/functions/fqdn_rand_spec.rb +++ b/spec/unit/parser/functions/fqdn_rand_spec.rb @@ -1,65 +1,65 @@ #! /usr/bin/env ruby require 'spec_helper' describe "the fqdn_rand function" do before :all do Puppet::Parser::Functions.autoloader.loadall end before :each do @scope = Puppet::Parser::Scope.new end it "should exist" do Puppet::Parser::Functions.function("fqdn_rand").should == "function_fqdn_rand" end it "should handle 0 arguments" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") lambda { @scope.function_fqdn_rand([]) }.should_not raise_error(Puppet::ParseError) end it "should handle 1 argument'}" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") lambda { @scope.function_fqdn_rand([3]) }.should_not raise_error(Puppet::ParseError) end (1..10).each { |n| it "should handle #{n} additional arguments" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") lambda { @scope.function_fqdn_rand([3,1,2,3,4,5,6,7,8,9,10][0..n]) }.should_not raise_error(Puppet::ParseError) end it "should handle #{n} additional string arguments" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") lambda { @scope.function_fqdn_rand([3,%w{ 1 2 3 4 5 6 7 8 9 10}].flatten[0..n]) }.should_not raise_error(Puppet::ParseError) end } it "should return a value less than max" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") @scope.function_fqdn_rand([3]).should satisfy {|n| n.to_i < 3 } end it "should return the same values on subsequent invocations for the same host" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1").twice + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1").twice @scope.function_fqdn_rand([3,4]).should eql(@scope.function_fqdn_rand([3, 4])) end it "should return different sequences of value for different hosts" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") val1 = @scope.function_fqdn_rand([10000000,4]) - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.2") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.2") val2 = @scope.function_fqdn_rand([10000000,4]) val1.should_not eql(val2) end it "should return different values for the same hosts with different seeds" do - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") val1 = @scope.function_fqdn_rand([10000000,4]) - @scope.expects(:lookupvar).with("fqdn").returns("127.0.0.1") + @scope.expects(:lookupvar).with("::fqdn").returns("127.0.0.1") val2 = @scope.function_fqdn_rand([10000000,42]) val1.should_not eql(val2) end end