diff --git a/ext/python/Tools/freeze/parsesetup.py b/ext/python/Tools/freeze/parsesetup.py index ae0bc43..dbab50c 100644 --- a/ext/python/Tools/freeze/parsesetup.py +++ b/ext/python/Tools/freeze/parsesetup.py @@ -1,112 +1,110 @@ # Parse Makefiles and Python Setup(.in) files. import re # Extract variable definitions from a Makefile. # Return a dictionary mapping names to values. # May raise IOError. makevardef = re.compile('^([a-zA-Z0-9_]+)[ \t]*=(.*)') def getmakevars(filename): variables = {} fp = open(filename) pendingline = "" try: while 1: line = fp.readline() if pendingline: line = pendingline + line pendingline = "" if not line: break if line.endswith('\\\n'): pendingline = line[:-2] matchobj = makevardef.match(line) if not matchobj: continue (name, value) = matchobj.group(1, 2) # Strip trailing comment i = value.find('#') if i >= 0: value = value[:i] value = value.strip() variables[name] = value finally: fp.close() return variables # Parse a Python Setup(.in) file. # Return two dictionaries, the first mapping modules to their # definitions, the second mapping variable names to their values. # May raise IOError. setupvardef = re.compile('^([a-zA-Z0-9_]+)=(.*)') def getsetupinfo(filename): modules = {} variables = {} fp = open(filename) pendingline = "" try: while 1: line = fp.readline() if pendingline: line = pendingline + line pendingline = "" if not line: break # Strip comments i = line.find('#') if i >= 0: line = line[:i] if line.endswith('\\\n'): pendingline = line[:-2] continue matchobj = setupvardef.match(line) if matchobj: (name, value) = matchobj.group(1, 2) variables[name] = value.strip() else: words = line.split() if words: modules[words[0]] = words[1:] finally: fp.close() return modules, variables # Test the above functions. def test(): import sys import os if not sys.argv[1:]: print('usage: python parsesetup.py Makefile*|Setup* ...') sys.exit(2) for arg in sys.argv[1:]: base = os.path.basename(arg) if base[:8] == 'Makefile': print('Make style parsing:', arg) v = getmakevars(arg) prdict(v) elif base[:5] == 'Setup': print('Setup style parsing:', arg) m, v = getsetupinfo(arg) prdict(m) prdict(v) else: print(arg, 'is neither a Makefile nor a Setup file') print('(name must begin with "Makefile" or "Setup")') def prdict(d): - keys = d.keys() - keys.sort() - for key in keys: + for key in sorted(d): value = d[key] print("%-15s" % key, str(value)) if __name__ == '__main__': test() diff --git a/pykolab/cli/commands.py b/pykolab/cli/commands.py index 072c346..fecc9bb 100644 --- a/pykolab/cli/commands.py +++ b/pykolab/cli/commands.py @@ -1,202 +1,197 @@ # -*- coding: utf-8 -*- # Copyright 2010-2013 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from six import string_types import os import sys import pykolab from pykolab.translate import _ log = pykolab.getLogger('pykolab.cli') conf = pykolab.getConf() commands = {} command_groups = {} def __init__(): # We only want the base path commands_base_path = os.path.dirname(__file__) for commands_path, dirnames, filenames in os.walk(commands_base_path): if not commands_path == commands_base_path: continue for filename in filenames: if filename.startswith('cmd_') and filename.endswith('.py'): module_name = filename.replace('.py','') cmd_name = module_name.replace('cmd_', '') #print "exec(\"from %s import __init__ as %s_register\"" % (module_name,cmd_name) try: exec("from .%s import __init__ as %s_register" % (module_name,cmd_name)) except ImportError: pass exec("%s_register()" % (cmd_name)) for dirname in dirnames: register_group(commands_path, dirname) register('help', list_commands) register('delete_user', not_yet_implemented, description="Not yet implemented") register('list_groups', not_yet_implemented, description="Not yet implemented") register('add_group', not_yet_implemented, description="Not yet implemented") register('delete_group', not_yet_implemented, description="Not yet implemented") def list_commands(*args, **kw): """ List commands """ __commands = {} for command in commands: if isinstance(command, tuple): command_group, command = command __commands[command_group] = { command: commands[(command_group,command)] } else: __commands[command] = commands[command] - _commands = __commands.keys() - _commands.sort() - - for _command in _commands: + for _command in sorted(__commands): if 'group' in __commands[_command]: continue if 'function' in __commands[_command]: # This is a top-level command if not __commands[_command]['description'] == None: print("%-25s - %s" % (_command.replace('_','-'),__commands[_command]['description'])) else: print("%-25s" % (_command.replace('_','-'))) - for _command in _commands: + for _command in sorted(__commands): if 'function' not in __commands[_command]: # This is a nested command print("\n" + _("Command Group: %s") % (_command) + "\n") - ___commands = __commands[_command].keys() - ___commands.sort() - for __command in ___commands: + for __command in sorted(__commands[_command]): if not __commands[_command][__command]['description'] == None: print("%-4s%-21s - %s" % ('',__command.replace('_','-'),__commands[_command][__command]['description'])) else: print("%-4s%-21s" % ('',__command.replace('_','-'))) def execute(cmd_name, *args, **kw): if cmd_name == "": execute("help") sys.exit(0) if cmd_name not in commands: log.error(_("No such command.")) sys.exit(1) if 'function' not in commands[cmd_name] and \ 'group' not in commands[cmd_name]: log.error(_("No such command.")) sys.exit(1) if 'group' in commands[cmd_name]: group = commands[cmd_name]['group'] command_name = commands[cmd_name]['cmd_name'] try: exec("from .%s.cmd_%s import cli_options as %s_%s_cli_options" % (group,command_name,group,command_name)) exec("%s_%s_cli_options()" % (group,command_name)) except ImportError: pass else: command_name = commands[cmd_name]['cmd_name'] try: exec("from .cmd_%s import cli_options as %s_cli_options" % (command_name,command_name)) exec("%s_cli_options()" % (command_name)) except ImportError: pass conf.finalize_conf() commands[cmd_name]['function'](conf.cli_args, kw) def register_group(dirname, module): commands_base_path = os.path.join(os.path.dirname(__file__), module) commands[module] = {} for commands_path, dirnames, filenames in os.walk(commands_base_path): if not commands_path == commands_base_path: continue for filename in filenames: if filename.startswith('cmd_') and filename.endswith('.py'): module_name = filename.replace('.py','') cmd_name = module_name.replace('cmd_', '') #print "exec(\"from %s.%s import __init__ as %s_%s_register\"" % (module,module_name,module,cmd_name) exec("from .%s.%s import __init__ as %s_%s_register" % (module,module_name,module,cmd_name)) exec("%s_%s_register()" % (module,cmd_name)) def register(cmd_name, func, group=None, description=None, aliases=[]): if not group == None: command = "%s_%s" % (group,cmd_name) else: command = cmd_name if isinstance(aliases, string_types): aliases = [aliases] if command in commands: log.fatal(_("Command '%s' already registered") % (command)) sys.exit(1) if callable(func): if group == None: commands[cmd_name] = { 'cmd_name': cmd_name, 'function': func, 'description': description } else: commands[group][cmd_name] = { 'cmd_name': cmd_name, 'function': func, 'description': description } commands[command] = commands[group][cmd_name] commands[command]['group'] = group commands[command]['cmd_name'] = cmd_name for alias in aliases: commands[alias] = { 'cmd_name': cmd_name, 'function': func, 'description': _("Alias for %s") % (cmd_name.replace('_','-')) } ## ## Commands not yet implemented ## def not_yet_implemented(*args, **kw): print(_("Not yet implemented")) sys.exit(1) diff --git a/pykolab/conf/__init__.py b/pykolab/conf/__init__.py index eb56bf9..94799ce 100644 --- a/pykolab/conf/__init__.py +++ b/pykolab/conf/__init__.py @@ -1,799 +1,796 @@ # -*- coding: utf-8 -*- # Copyright 2010-2013 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from __future__ import print_function import logging import os import sys from optparse import OptionParser try: from ConfigParser import SafeConfigParser except ImportError: from configparser import SafeConfigParser import pykolab from pykolab.conf.defaults import Defaults from pykolab.constants import * from pykolab.translate import _ log = pykolab.getLogger('pykolab.conf') class Conf(object): def __init__(self): """ self.cli_args == Arguments passed on the CLI self.cli_keywords == Parser results (again, CLI) self.cli_parser == The actual Parser (from OptionParser) self.plugins == Our Kolab Plugins """ self.cli_parser = None self.cli_args = None self.cli_keywords = None self.entitlement = None self.changelog = {} try: from pykolab.conf.entitlement import Entitlement entitlements = True except Exception: entitlements = False pass if entitlements: self.entitlement = Entitlement().get() self.plugins = None # The location where our configuration parser is going to end up self.cfg_parser = None # Create the options self.create_options() def finalize_conf(self, fatal=True): self.create_options_from_plugins() self.parse_options(fatal=fatal) # The defaults can some from; # - a file we ship with the packages # - a customly supplied file (by customer) # - a file we write out # - this python class # # Look, we want defaults self.defaults = Defaults(self.plugins) # But, they should be available in our class as well for option in self.defaults.__dict__: log.debug( _("Setting %s to %r (from defaults)") % ( option, self.defaults.__dict__[option] ), level=8 ) setattr(self, option, self.defaults.__dict__[option]) # This is where we check our parser for the defaults being set there. self.set_defaults_from_cli_options() self.options_set_from_config() # Also set the cli options if hasattr(self, 'cli_keywords') and self.cli_keywords is not None: for option in self.cli_keywords.__dict__: retval = False if hasattr(self, "check_setting_%s" % (option)): retval = eval( "self.check_setting_%s(%r)" % ( option, self.cli_keywords.__dict__[option] ) ) # The warning, error or confirmation dialog is in the check_setting_%s() # function if not retval: continue log.debug( _("Setting %s to %r (from CLI, verified)") % ( option, self.cli_keywords.__dict__[option] ), level=8 ) setattr(self, option, self.cli_keywords.__dict__[option]) else: log.debug( _("Setting %s to %r (from CLI, not checked)") % ( option, self.cli_keywords.__dict__[option] ), level=8 ) setattr(self, option, self.cli_keywords.__dict__[option]) def load_config(self, config): """ Given a SafeConfigParser instance, loads a configuration file and checks, then sets everything it can find. """ for section in self.defaults.__dict__: if section == 'testing': continue if not config.has_section(section): continue for key in self.defaults.__dict__[section]: retval = False if not config.has_option(section, key): continue if isinstance(self.defaults.__dict__[section][key], int): value = config.getint(section, key) elif isinstance(self.defaults.__dict__[section][key], bool): value = config.getboolean(section, key) elif isinstance(self.defaults.__dict__[section][key], str): value = config.get(section, key) elif isinstance(self.defaults.__dict__[section][key], list): value = eval(config.get(section, key)) elif isinstance(self.defaults.__dict__[section][key], dict): value = eval(config.get(section, key)) if hasattr(self, "check_setting_%s_%s" % (section, key)): retval = eval("self.check_setting_%s_%s(%r)" % (section, key, value)) if not retval: # We just don't set it, check_setting_%s should have # taken care of the error messages continue if not self.defaults.__dict__[section][key] == value: if key.count('password') >= 1: log.debug( _("Setting %s_%s to '****' (from configuration file)") % ( section, key ), level=8 ) else: log.debug( _("Setting %s_%s to %r (from configuration file)") % ( section, key, value ), level=8 ) setattr(self, "%s_%s" % (section, key), value) def options_set_from_config(self): """ Sets the default configuration options from a configuration file. Configuration file may be customized using the --config CLI option """ log.debug(_("Setting options from configuration file"), level=4) # Check from which configuration file we should get the defaults # Other then default? self.config_file = self.defaults.config_file if hasattr(self, 'cli_keywords') and self.cli_keywords is not None: if not self.cli_keywords.config_file == self.defaults.config_file: self.config_file = self.cli_keywords.config_file config = self.check_config() self.load_config(config) def set_options_from_testing_section(self): """ Go through the options in the [testing] section if it exists. """ config = self.check_config() if not config.has_section('testing'): return for key in config.options('testing'): retval = False if isinstance(self.defaults.__dict__['testing'][key], int): value = config.getint('testing', key) elif isinstance(self.defaults.__dict__['testing'][key], bool): value = config.getboolean('testing', key) elif isinstance(self.defaults.__dict__['testing'][key], str): value = config.get('testing', key) elif isinstance(self.defaults.__dict__['testing'][key], list): value = eval(config.get('testing', key)) elif isinstance(self.defaults.__dict__['testing'][key], dict): value = eval(config.get('testing', key)) if hasattr(self, "check_setting_%s_%s" % ('testing', key)): retval = eval("self.check_setting_%s_%s(%r)" % ('testing', key, value)) if not retval: # We just don't set it, check_setting_%s should have # taken care of the error messages continue setattr(self, "%s_%s" % ('testing', key), value) if key.count('password') >= 1: log.debug( _("Setting %s_%s to '****' (from configuration file)") % ('testing', key), level=8 ) else: log.debug( _("Setting %s_%s to %r (from configuration file)") % ('testing', key, value), level=8 ) def check_config(self, val=None): """ Checks self.config_file or the filename passed using 'val' and returns a SafeConfigParser instance if everything is OK. """ if val is not None: config_file = val else: config_file = self.config_file if not os.access(config_file, os.R_OK): log.error(_("Configuration file %s not readable") % config_file) config = SafeConfigParser() log.debug(_("Reading configuration file %s") % config_file, level=8) try: config.read(config_file) except Exception: log.error(_("Invalid configuration file %s") % config_file) if not config.has_section("kolab"): log.warning( _("No master configuration section [kolab] in configuration file %s") % config_file ) return config def add_cli_parser_option_group(self, name): return self.cli_parser.add_option_group(name) def create_options_from_plugins(self): """ Create options from plugins. This function must be called separately from Conf.__init__(), or the configuration store is not yet done initializing when the plugins class and the plugins themselves go look for it. """ import pykolab.plugins self.plugins = pykolab.plugins.KolabPlugins() self.plugins.add_options(self.cli_parser) def create_options(self, load_plugins=True): """ Create the OptionParser for the options passed to us from runtime Command Line Interface. """ # Enterprise Linux 5 does not have an "epilog" parameter to OptionParser try: self.cli_parser = OptionParser(epilog=epilog) except Exception: self.cli_parser = OptionParser() # # Runtime Options # runtime_group = self.cli_parser.add_option_group(_("Runtime Options")) runtime_group.add_option( "-c", "--config", dest="config_file", action="store", default="/etc/kolab/kolab.conf", help=_("Configuration file to use") ) runtime_group.add_option( "-d", "--debug", dest="debuglevel", type='int', default=0, help=_( "Set the debugging verbosity. Maximum is 9, tracing protocols LDAP, SQL and IMAP." ) ) runtime_group.add_option( "-e", "--default", dest="answer_default", action="store_true", default=False, help=_("Use the default answer to all questions.") ) runtime_group.add_option( "-l", dest="loglevel", type='str', default="CRITICAL", help=_("Set the logging level. One of info, warn, error, critical or debug") ) runtime_group.add_option( "--logfile", dest="logfile", action="store", default="/var/log/kolab/pykolab.log", help=_("Log file to use") ) runtime_group.add_option( "-q", "--quiet", dest="quiet", action="store_true", default=False, help=_("Be quiet.") ) runtime_group.add_option( "-y", "--yes", dest="answer_yes", action="store_true", default=False, help=_("Answer yes to all questions.") ) def parse_options(self, fatal=True): """ Parse options passed to our call. """ if fatal: (self.cli_keywords, self.cli_args) = self.cli_parser.parse_args() def run(self): """ Run Forest, RUN! """ if self.cli_args: if len(self.cli_args) >= 1: if hasattr(self, "command_%s" % self.cli_args[0].replace('-', '_')): exec( "self.command_%s(%r)" % ( self.cli_args[0].replace('-', '_'), self.cli_args[1:] ) ) else: print(_("No command supplied"), file=sys.stderr) def command_dump(self, *args, **kw): """ Dumps applicable, valid configuration that is not defaults. """ if not self.cfg_parser: self.read_config() if not self.cfg_parser.has_section('kolab'): print("No section found for kolab", file=sys.stderr) sys.exit(1) # Get the sections, and then walk through the sections in a # sensible way. items = self.cfg_parser.options('kolab') items.sort() for item in items: mode = self.cfg_parser.get('kolab', item) print("%s = %s" % (item, mode)) if not self.cfg_parser.has_section(mode): print("WARNING: No configuration section %s for item %s" % (mode, item)) continue - keys = self.cfg_parser.options(mode) - keys.sort() - if self.cfg_parser.has_option(mode, 'leave_this_one_to_me'): print("Ignoring section %s" % (mode)) continue - for key in keys: + for key in sorted(self.cfg_parser.options(mode)): print("%s_%s = %s" % (mode, key, self.cfg_parser.get(mode, key))) def read_config(self, value=None): """ Reads the configuration file, sets a self.cfg_parser. """ if not value: value = self.defaults.config_file if hasattr(self, 'cli_keywords') and self.cli_keywords is not None: value = self.cli_keywords.config_file self.cfg_parser = SafeConfigParser() self.cfg_parser.read(value) if hasattr(self, 'cli_keywords') and hasattr(self.cli_keywords, 'config_file'): self.cli_keywords.config_file = value self.defaults.config_file = value self.config_file = value def command_get(self, *args, **kw): """ Get a configuration option. Pass me a section and key please. """ args = eval("%r" % args) print("%s/%s: %r" % (args[0], args[1], self.get(args[0], args[1]))) # if len(args) == 3: # # Return non-zero if no match # # Return zero if match # # Improvised "check" function def command_set(self, *args, **kw): """ Set a configuration option. Pass me a section, key and value please. Note that the section should already exist. TODO: Add a strict parameter TODO: Add key value checking """ if not self.cfg_parser: self.read_config() if not len(args) == 3: log.error(_("Insufficient options. Need section, key and value -in that order.")) if not self.cfg_parser.has_section(args[0]): log.error(_("No section '%s' exists.") % (args[0])) if '%' in args[2]: value = args[2].replace('%', '%%') else: value = args[2] self.cfg_parser.set(args[0], args[1], value) if hasattr(self, 'cli_keywords') and hasattr(self.cli_keywords, 'config_file'): fp = open(self.cli_keywords.config_file, "w+") self.cfg_parser.write(fp) fp.close() else: fp = open(self.config_file, "w+") self.cfg_parser.write(fp) fp.close() def create_logger(self): """ Create a logger instance using cli_options.debuglevel """ global log if self.cli_keywords.debuglevel is not None: loglevel = logging.DEBUG else: loglevel = logging.INFO self.cli_keywords.debuglevel = 0 self.debuglevel = self.cli_keywords.debuglevel # Initialize logger log = pykolab.logger.Logger( loglevel=loglevel, debuglevel=self.cli_keywords.debuglevel, logfile=self.cli_keywords.logfile ) def set_defaults_from_cli_options(self): for long_opt in self.cli_parser.__dict__['_long_opt']: if long_opt == "--help": continue setattr( self.defaults, self.cli_parser._long_opt[long_opt].dest, self.cli_parser._long_opt[long_opt].default ) # But, they should be available in our class as well for option in self.cli_parser.defaults: log.debug( _("Setting %s to %r (from the default values for CLI options)") % ( option, self.cli_parser.defaults[option] ), level=8 ) setattr(self, option, self.cli_parser.defaults[option]) def has_section(self, section): if not self.cfg_parser: self.read_config() return self.cfg_parser.has_section(section) def has_option(self, section, option): if not self.cfg_parser: self.read_config() return self.cfg_parser.has_option(section, option) def get_list(self, section, key, default=None): """ Gets a comma and/or space separated list from the configuration file and returns a list. """ values = [] untrimmed_values = [] setting = self.get_raw(section, key) if setting is None: return default if default else [] raw_values = setting.split(',') if raw_values is None: return default if default else [] for raw_value in raw_values: untrimmed_values.extend(raw_value.split(' ')) for value in untrimmed_values: if not value.strip() == "": values.append(value.strip().lower()) return values def get_raw(self, section, key, default=None): if not self.cfg_parser: self.read_config() if self.cfg_parser.has_option(section, key): return self.cfg_parser.get(section, key, raw=1) return default def get(self, section, key, default=None, quiet=False): """ Get a configuration option from our store, the configuration file, or an external source if we have some sort of function for it. TODO: Include getting the value from plugins through a hook. """ retval = False if not self.cfg_parser: self.read_config() # log.debug(_("Obtaining value for section %r, key %r") % (section, key), level=8) if self.cfg_parser.has_option(section, key): try: return self.cfg_parser.get(section, key) except Exception: self.read_config() return self.cfg_parser.get(section, key) if hasattr(self, "get_%s_%s" % (section, key)): try: retval = eval("self.get_%s_%s(quiet)" % (section, key)) except Exception: log.error( _("Could not execute configuration function: %s") % ( "get_%s_%s(quiet=%r)" % ( section, key, quiet ) ) ) return default return retval if quiet: return "" else: log.debug( _("Option %s/%s does not exist in config file %s, pulling from defaults") % ( section, key, self.config_file ), level=8 ) if hasattr(self.defaults, "%s_%s" % (section, key)): return getattr(self.defaults, "%s_%s" % (section, key)) elif hasattr(self.defaults, "%s" % (section)): if key in getattr(self.defaults, "%s" % (section)): _dict = getattr(self.defaults, "%s" % (section)) return _dict[key] else: log.warning(_("Option does not exist in defaults.")) return default else: log.warning(_("Option does not exist in defaults.")) return default def check_setting_config_file(self, value): if os.path.isfile(value): if os.access(value, os.R_OK): self.read_config(value=value) self.config_file = value return True else: log.error(_("Configuration file %s not readable.") % (value)) return False else: log.error(_("Configuration file %s does not exist.") % (value)) return False def check_setting_debuglevel(self, value): if value < 0: log.info( _( "WARNING: A negative debug level value does not " + "make this program be any more silent." ) ) elif value == 0: return True elif value <= 9: return True else: log.warning(_("This program has 9 levels of verbosity. Using the maximum of 9.")) return True def check_setting_saslauth_mode(self, value): if value: # TODO: I suppose this is platform specific if os.path.isfile("/var/run/saslauthd/mux"): if os.path.isfile("/var/run/saslauthd/saslauthd.pid"): log.error(_("Cannot start SASL authentication daemon")) return False else: try: os.remove("/var/run/saslauthd/mux") except IOError: log.error(_("Cannot start SASL authentication daemon")) return False elif os.path.isfile("/var/run/sasl2/mux"): if os.path.isfile("/var/run/sasl2/saslauthd.pid"): log.error(_("Cannot start SASL authentication daemon")) return False else: try: os.remove("/var/run/sasl2/mux") except IOError: log.error(_("Cannot start SASL authentication daemon")) return False return True def check_setting_use_imap(self, value): if value: try: import imaplib self.use_imap = value return True except ImportError: log.error(_("No imaplib library found.")) return False def check_setting_use_lmtp(self, value): if value: try: from smtplib import LMTP self.use_lmtp = value return True except ImportError: log.error(_("No LMTP class found in the smtplib library.")) return False def check_setting_use_mail(self, value): if value: try: from smtplib import SMTP self.use_mail = value return True except ImportError: log.error(_("No SMTP class found in the smtplib library.")) return False def check_setting_test_suites(self, value): # Attempt to load the suite, # Get the suite's options, # Set them here. if not hasattr(self, 'test_suites'): self.test_suites = [] if "zpush" in value: selectively = False for item in ['calendar', 'contacts', 'mail']: if self.cli_keywords.__dict__[item]: log.debug( _("Found you specified a specific set of items to test: %s") % (item), level=8 ) selectively = item if not selectively: self.calendar = True self.contacts = True self.mail = True else: log.debug(_("Selectively selecting: %s") % (selectively), level=8) setattr(self, selectively, True) self.test_suites.append('zpush') def check_setting_calendar(self, value): if self.cli_parser._long_opt['--calendar'].default == value: return False else: return True def check_setting_contacts(self, value): if self.cli_parser._long_opt['--contacts'].default == value: return False else: return True def check_setting_mail(self, value): if self.cli_parser._long_opt['--mail'].default == value: return False else: return True diff --git a/pykolab/plugins/__init__.py b/pykolab/plugins/__init__.py index ab110fb..f2f1ef4 100644 --- a/pykolab/plugins/__init__.py +++ b/pykolab/plugins/__init__.py @@ -1,272 +1,272 @@ # -*- coding: utf-8 -*- # Copyright 2010-2013 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # import logging import os import pdb import sys import traceback import pykolab from pykolab.translate import _ log = pykolab.getLogger('pykolab.plugins') conf = pykolab.getConf() class KolabPlugins(object): """ Detects, loads and interfaces with plugins for different Kolab components. """ def __init__(self): """ Searches the plugin directory for plugins, and loads them into a list. """ self.plugins = {} for plugin_path in [ os.path.dirname(__file__), '/usr/share/pykolab/plugins', './pykolab/plugins' ]: if os.path.isdir(plugin_path): for plugin in os.listdir(plugin_path): if os.path.isdir('%s/%s/' % (plugin_path, plugin, )): self.plugins[plugin] = False self.check_plugins() def check_plugins(self): """ Checks all plugins in self.plugins and sets the values to True (loadable) or False -- not enabled, not installed or not loadable. """ for plugin in self.plugins: try: exec("from pykolab.plugins import %s" % (plugin)) self.plugins[plugin] = True self.load_plugins(plugins=[plugin]) except ImportError as e: log.error(_("ImportError for plugin %s: %s") % (plugin, e)) traceback.print_exc() self.plugins[plugin] = False except RuntimeError as e: log.error( _("RuntimeError for plugin %s: %s") % (plugin, e)) traceback.print_exc() self.plugins[plugin] = False except Exception as e: log.error(_("Plugin %s failed to load (%s: %s)") % (plugin, e.__class__, e)) traceback.print_exc() except: traceback.print_exc() def load_plugins(self, plugins=[]): """ Loads plugins specified by a list of plugins or loads them all """ if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins for plugin in plugins: if self.plugins[plugin]: try: exec("self.%s = %s.Kolab%s()" % (plugin, plugin, plugin.capitalize())) except: # TODO: A little better verbosity please! traceback.print_exc() def set_defaults(self, defaults, plugins=[]): """ Test for a function set_defaults() in all available and loaded plugins and execute plugin.set_defaults() """ if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins for plugin in plugins: if not self.plugins[plugin]: continue if not hasattr(self, plugin): continue if hasattr(getattr(self, plugin), "set_defaults"): try: getattr(self, plugin).set_defaults(defaults) except TypeError as e: log.error(_("Cannot set defaults for plugin %s: %s") % (plugin, e)) except RuntimeError as e: log.error(_("Cannot set defaults for plugin %s: %s") % (plugin, e)) except: log.error(_("Cannot set defaults for plugin %s: Unknown Error") % (plugin)) else: log.debug(_("Not setting defaults for plugin %s: No function 'set_defaults()'") % plugin, level=5) def set_runtime(self, runtime, plugins=[]): """ Set runtime variables from plugins, like 'i_did_all_this' """ if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins for plugin in plugins: if not self.plugins[plugin]: continue if not hasattr(self, plugin): continue if hasattr(getattr(self, plugin), "set_runtime"): try: getattr(self, plugin).set_runtime(runtime) except RuntimeError as e: log.error(_("Cannot set runtime for plugin %s: %s") % (plugin, e)) else: log.debug(_("Not setting runtime for plugin %s: No function 'set_runtime()'") % (plugin), level=5) def add_options(self, parser, plugins=[]): """ Add options specified in a plugin to parser. Takes a list of plugin names or does them all """ if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins for plugin in plugins: if not self.plugins[plugin]: continue if not hasattr(self, plugin): continue if hasattr(getattr(self, plugin), "add_options"): try: exec("self.%s.add_options(parser)" % plugin) except RuntimeError as e: log.error(_("Cannot add options for plugin %s: %s") % (plugin, e)) except TypeError as e: log.error(_("Cannot add options for plugin %s: %s") % (plugin, e)) else: log.debug(_("Not adding options for plugin %s: No function 'add_options()'") % plugin, level=5) def check_options(self, plugins=[]): """ Executes plugin.check_plugins() for all enabled plugins or the list of plugin names specified. """ if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins for plugin in plugins: if not self.plugins[plugin]: continue if not hasattr(self, plugin): continue if hasattr(getattr(self, plugin), "check_options"): try: exec("self.%s.check_options()" % plugin) except AttributeError as e: log.error(_("Cannot check options for plugin %s: %s") % (plugin, e)) else: log.debug(_("Not checking options for plugin %s: No function 'check_options()'") % (plugin), level=5) def plugin_check_setting(self, func, option, val, plugins=[]): """ Checks one setting specified by 'option' against the 'val' it is passed by all plugins or by the list of plugins specified """ if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins for plugin in plugins: if not self.plugins[plugin]: continue if not hasattr(self, plugin): continue if hasattr(getattr(self, plugin), "%s_%s" % (func, option)): retval = eval("getattr(self, plugin).%s_%s(val)" % (func, option)) return retval return False def exec_hook(self, hook, plugins=[], kw={}, args=()): """Execute a hook""" retval = None if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins for plugin in plugins: if not self.plugins[plugin]: continue if not hasattr(self, plugin): continue if hasattr(getattr(self, plugin), hook): try: log.debug(_("Executing hook %s for plugin %s") % (hook, plugin), level=8) func = getattr(getattr(self, plugin), hook) retval = func(*args, **kw) except TypeError as errmsg: log.error( _("Cannot execute hook %s for plugin %s: %s") % (hook, plugin, errmsg) ) log.error(traceback.format_exc()) except AttributeError as errmsg: log.error( _("Cannot execute hook %s for plugin %s: %s") % (hook, plugin, errmsg) ) log.error(traceback.format_exc()) return retval def return_true_boolean_from_plugins(self, bool, plugins=[]): """ Given the name of a boolean, walks all specified plugins, or all available plugins, and returns True if a plugin has it set to true """ if len(plugins) < 1: - plugins = self.plugins.keys() + plugins = self.plugins retval = False for plugin in plugins: if not self.plugins[plugin]: continue if not hasattr(self, plugin): continue if hasattr(getattr(self, plugin), bool): try: boolval = eval("self.%s.%s" % (plugin, bool)) except AttributeError: pass else: boolval = None if boolval: retval = True return retval diff --git a/pykolab/setup/components.py b/pykolab/setup/components.py index bb01f9c..45f7ff8 100644 --- a/pykolab/setup/components.py +++ b/pykolab/setup/components.py @@ -1,267 +1,259 @@ # -*- coding: utf-8 -*- # # Copyright 2010-2013 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from six import string_types import os import pykolab from pykolab.constants import * from pykolab.translate import _ log = pykolab.getLogger('pykolab.setup') conf = pykolab.getConf() components = {} component_groups = {} executed_components = [] components_included_in_cli = [] finalize_conf_ok = None def __init__(): # We only want the base path components_base_path = os.path.dirname(__file__) for components_path, dirnames, filenames in os.walk(components_base_path): if not components_path == components_base_path: continue for filename in filenames: if filename.startswith('setup_') and filename.endswith('.py'): module_name = filename.replace('.py','') component_name = module_name.replace('setup_', '') #print "exec(\"from %s import __init__ as %s_register\"" % (module_name,component_name) exec("from .%s import __init__ as %s_register" % (module_name,component_name)) exec("%s_register()" % (component_name)) for dirname in dirnames: register_group(components_path, dirname) register('help', list_components, description=_("Display this help.")) def list_components(*args, **kw): """ List components """ __components = {} for component in components: if isinstance(component, tuple): component_group, component = component __components[component_group] = { component: components[(component_group,component)] } else: __components[component] = components[component] - _components = __components.keys() - _components.sort() - - for _component in _components: + for _component in sorted(__components): if 'function' in __components[_component]: # This is a top-level component if not __components[_component]['description'] == None: print("%-25s - %s" % (_component.replace('_','-'),__components[_component]['description'])) else: print("%-25s" % (_component.replace('_','-'))) - for _component in _components: + for _component in sorted(__components): if 'function' not in __components[_component]: # This is a nested component print("\n" + _("Command Group: %s") % (_component) + "\n") - ___components = __components[_component].keys() - ___components.sort() - for __component in ___components: + for __component in sorted(__components[_component]): if not __components[_component][__component]['description'] == None: print("%-4s%-21s - %s" % ('',__component.replace('_','-'),__components[_component][__component]['description'])) else: print("%-4s%-21s" % ('',__component.replace('_','-'))) def _list_components(*args, **kw): """ List components and return API compatible, parseable lists and dictionaries. """ __components = {} for component in components: if isinstance(component, tuple): component_group, component = component __components[component_group] = { component: components[(component_group,component)] } else: __components[component] = components[component] - _components = __components.keys() - _components.sort() - - return _components + return sorted(list(__components.keys())) def cli_options_from_component(component_name, *args, **kw): global components_included_in_cli if component_name in components_included_in_cli: return if 'group' in components[component_name]: group = components[component_name]['group'] component_name = components[component_name]['component_name'] try: exec("from .%s.setup_%s import cli_options as %s_%s_cli_options" % (group,component_name,group,component_name)) exec("%s_%s_cli_options()" % (group,component_name)) except ImportError: pass else: try: exec("from .setup_%s import cli_options as %s_cli_options" % (component_name,component_name)) exec("%s_cli_options()" % (component_name)) except ImportError: pass components_included_in_cli.append(component_name) def execute(component_name, *args, **kw): if component_name == '': log.debug( _("No component selected, continuing for all components"), level=8 ) while 1: for component in _list_components(): execute_this = True if component in executed_components: execute_this = False if component == "help": execute_this = False if execute_this: if 'after' in components[component]: for _component in components[component]['after']: if not _component in executed_components: execute_this = False if execute_this: execute(component) executed_components.append(component) executed_all = True for component in _list_components(): if not component in executed_components and not component == "help": executed_all = False if executed_all: break return else: for component in _list_components(): cli_options_from_component(component) if component_name not in components: log.error(_("No such component.")) sys.exit(1) if 'function' not in components[component_name] and \ 'group' not in components[component_name]: log.error(_("No such component.")) sys.exit(1) conf.finalize_conf() if len(conf.cli_args) >= 1: _component_name = conf.cli_args.pop(0) else: _component_name = component_name components[component_name]['function'](conf.cli_args, kw) def register_group(dirname, module): components_base_path = os.path.join(os.path.dirname(__file__), module) components[module] = {} for components_path, dirnames, filenames in os.walk(components_base_path): if not components_path == components_base_path: continue for filename in filenames: if filename.startswith('setup_') and filename.endswith('.py'): module_name = filename.replace('.py','') component_name = module_name.replace('setup_', '') #print "exec(\"from %s.%s import __init__ as %s_%s_register\"" % (module,module_name,module,component_name) exec("from .%s.%s import __init__ as %s_%s_register" % (module,module_name,module,component_name)) exec("%s_%s_register()" % (module,component_name)) def register(component_name, func, group=None, description=None, aliases=[], after=[], before=[]): if not group == None: component = "%s_%s" % (group,component_name) else: component = component_name if isinstance(aliases, string_types): aliases = [aliases] if component in components: log.fatal(_("Command '%s' already registered") % (component)) sys.exit(1) if callable(func): if group == None: components[component_name] = { 'function': func, 'description': description, 'after': after, 'before': before, } else: components[group][component_name] = { 'function': func, 'description': description, 'after': after, 'before': before, } components[component] = components[group][component_name] components[component]['group'] = group components[component]['component_name'] = component_name for alias in aliases: components[alias] = { 'function': func, 'description': _("Alias for %s") % (component_name) } ## ## Commands not yet implemented ## def not_yet_implemented(*args, **kw): print(_("Not yet implemented")) sys.exit(1) diff --git a/pykolab/utils.py b/pykolab/utils.py index e84f13b..213cddc 100644 --- a/pykolab/utils.py +++ b/pykolab/utils.py @@ -1,635 +1,632 @@ # -*- coding: utf-8 -*- # Copyright 2010-2013 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from __future__ import print_function import base64 import getpass import grp import os import pwd from six import string_types import struct import sys import pykolab from pykolab import constants from pykolab.translate import _ as _l # pylint: disable=invalid-name log = pykolab.getLogger('pykolab.utils') conf = pykolab.getConf() try: # pylint: disable=redefined-builtin input = raw_input except NameError: pass try: unicode('') except NameError: unicode = str # pylint: disable=too-many-branches def ask_question(question, default="", password=False, confirm=False): """ Ask a question on stderr. Since the answer to the question may actually be a password, cover that case with a getpass.getpass() prompt. Accepts a default value, but ignores defaults for password prompts. Usage: pykolab.utils.ask_question("What is the server?", default="localhost") """ if default != "" and default is not None and conf.cli_keywords.answer_default: if not conf.cli_keywords.quiet: print("%s [%s]: " % (question, default)) return default if password: if default == "" or default is None: answer = getpass.getpass("%s: " % (question)) else: answer = getpass.getpass("%s [%s]: " % (question, default)) else: if default == "" or default is None: answer = input("%s: " % (question)) else: answer = input("%s [%s]: " % (question, default)) # pylint: disable=too-many-nested-blocks if not answer == "": if confirm: answer_confirm = None answer_confirmed = False while not answer_confirmed: if password: answer_confirm = getpass.getpass(_l("Confirm %s: ") % (question)) else: answer_confirm = input(_l("Confirm %s: ") % (question)) if not answer_confirm == answer: print(_l("Incorrect confirmation. Please try again."), file=sys.stderr) if password: if default == "" or default is None: answer = getpass.getpass(_l("%s: ") % (question)) else: answer = getpass.getpass(_l("%s [%s]: ") % (question, default)) else: if default == "" or default is None: answer = input(_l("%s: ") % (question)) else: answer = input(_l("%s [%s]: ") % (question, default)) else: answer_confirmed = True if answer == "": return default return answer # pylint: disable=too-many-return-statements def ask_confirmation(question, default="y", all_inclusive_no=True): """ Create a confirmation dialog, including a default option (capitalized), and a "yes" or "no" parsing that can either require an explicit, full "yes" or "no", or take the default or any YyNn answer. """ default_answer = None if default in ["y", "Y"]: default_answer = True default_no = "n" default_yes = "Y" elif default in ["n", "N"]: default_answer = False default_no = "N" default_yes = "y" else: # This is a 'yes' or 'no' question the user # needs to provide the full yes or no for. default_no = "'no'" default_yes = "Please type 'yes'" if conf.cli_keywords.answer_yes \ or (conf.cli_keywords.answer_default and default_answer is not None): if not conf.cli_keywords.quiet: print("%s [%s/%s]: " % (question, default_yes, default_no)) if conf.cli_keywords.answer_yes: return True if conf.cli_keywords.answer_default: return default_answer answer = False while not answer: answer = input("%s [%s/%s]: " % (question, default_yes, default_no)) # Parse answer and set back to False if not appropriate if all_inclusive_no: if answer == "" and default_answer is not None: return default_answer if answer in ["y", "Y", "yes"]: return True if answer in ["n", "N", "no"]: return False answer = False print(_l("Please answer 'yes' or 'no'."), file=sys.stderr) if answer not in ["y", "Y", "yes"]: return False return True # pylint: disable=dangerous-default-value def ask_menu(question, options={}, default=''): if default != '' and conf.cli_keywords.answer_default: if not conf.cli_keywords.quiet: print(question + " [" + default + "]:") return default if default != '': print(question + " [" + default + "]:") else: print(question) answer_correct = False max_key_length = 0 if isinstance(options, list): _options = options options = {} for key in _options: options[key] = key - keys = options.keys() - keys.sort() - while not answer_correct: - for key in keys: + for key in sorted(options): key_length = len("%s" % key) if key_length > max_key_length: max_key_length = key_length str_format = "%%%ds" % max_key_length if default == '' or default not in options: - for key in keys: + for key in sorted(options): if options[key] == key: print(" - " + key) else: print(" - " + str_format % key + ": " + options[key]) answer = input(_l("Choice") + ": ") else: answer = input(_l("Choice (type '?' for options)") + ": ") if answer == '?': - for key in keys: + for key in sorted(options): if options[key] == key: print(" - " + key) else: print(" - " + str_format % key + ": " + options[key]) continue if answer == '' and default in options: answer = default if answer in [str(x) for x in options]: answer_correct = True return answer def decode(key, enc): if key is None: return enc dec = [] enc = base64.urlsafe_b64decode(enc) for i in range(len(enc)): key_c = key[i % len(key)] dec_c = chr((256 + ord(enc[i]) - ord(key_c)) % 256) dec.append(dec_c) return "".join(dec) def encode(key, clear): if key is None: return clear enc = [] for i in range(len(clear)): key_c = key[i % len(key)] enc_c = chr((ord(clear[i]) + ord(key_c)) % 256) enc.append(enc_c) return base64.urlsafe_b64encode("".join(enc)) def ensure_directory(_dir, _user='root', _group='root'): if not os.path.isdir(_dir): os.makedirs(_dir) try: try: (ruid, _, _) = os.getresuid() (rgid, _, _) = os.getresgid() except AttributeError: ruid = os.getuid() rgid = os.getgid() if ruid == 0: # Means we can setreuid() / setregid() / setgroups() if rgid == 0: # Get group entry details try: (_, _, group_gid, _) = grp.getgrnam(_group) except KeyError: print(_l("Group %s does not exist") % (_group), file=sys.stderr) sys.exit(1) # Set real and effective group if not the same as current. if not group_gid == rgid: os.chown(_dir, -1, group_gid) if ruid == 0: # Means we haven't switched yet. try: (_, _, user_uid, _, _, _, _) = pwd.getpwnam(_user) except KeyError: print(_l("User %s does not exist") % (_user), file=sys.stderr) sys.exit(1) # Set real and effective user if not the same as current. if not user_uid == ruid: os.chown(_dir, user_uid, -1) except Exception: print(_l("Could not change the permissions on %s") % (_dir), file=sys.stderr) def generate_password(): import subprocess p1 = subprocess.Popen(['head', '-c', '200', '/dev/urandom'], stdout=subprocess.PIPE) p2 = subprocess.Popen(['tr', '-dc', '_A-Z-a-z-0-9'], stdin=p1.stdout, stdout=subprocess.PIPE) p3 = subprocess.Popen(['head', '-c', '15'], stdin=p2.stdout, stdout=subprocess.PIPE) p1.stdout.close() p2.stdout.close() output = p3.communicate()[0] return output def multiline_message(message): if hasattr(conf, 'cli_keywords') and hasattr(conf.cli_keywords, 'quiet'): if conf.cli_keywords.quiet: return "" column_width = 80 # First, replace all occurences of "\n" message = message.replace(" ", "") message = message.replace("\n", " ") lines = [] line = "" for word in message.split(): if (len(line) + len(word)) > column_width: lines.append(line) line = word else: if line == "": line = word else: line += " %s" % (word) lines.append(line) return "\n%s\n" % ("\n".join(lines)) def stripped_message(message): return "\n" + message.strip() + "\n" def str2unicode(s, encoding='utf-8'): if isinstance(s, unicode): return s try: return unicode(s, encoding) except Exception: pass return s def normalize(_object): if isinstance(_object, list): result = [] elif isinstance(_object, dict): result = {} else: return _object if isinstance(_object, list): for item in _object: result.append(item.lower()) result = list(set(result)) return result if isinstance(_object, dict): def _strip(value): try: return value.strip() except Exception: return value for key in _object: if isinstance(_object[key], list): if _object[key] is None: continue # Dont run strip anything from attributes which # hold byte strings if key.lower() in constants.BINARY_ATTRS: val = _object[key] else: val = map(_strip, _object[key]) if len(val) == 1: result[key.lower()] = ''.join(val) else: result[key.lower()] = val else: if _object[key] is None: continue result[key.lower()] = _strip(_object[key]) if 'objectsid' in result and not result['objectsid'][0] == "S": result['objectsid'] = sid_to_string(result['objectsid']) if 'sn' in result: result['surname'] = result['sn'].replace(' ', '') if 'mail' in result: if isinstance(result['mail'], list): result['mail'] = result['mail'][0] if result['mail']: if len(result['mail'].split('@')) > 1: result['domain'] = result['mail'].split('@')[1] if 'domain' not in result and 'standard_domain' in result: result['domain'] = result['standard_domain'] if 'objectclass' not in result: result['objectclass'] = [] if result['objectclass'] is None: result['objectclass'] = [] if not isinstance(result['objectclass'], list): result['objectclass'] = [result['objectclass']] result['objectclass'] = [x.lower() for x in result['objectclass']] return result def parse_input(_input, splitchars=[' ']): """ Split the input string using the split characters defined in splitchars, and remove the empty list items, then unique the list items. Takes a string as input, and a list of characters the string should be split with (list of delimiter characters). """ _parse_list = _input.split(splitchars.pop()) _output_list = [] for splitchar in splitchars: __parse_list = [] for item in _parse_list: __parse_list.extend(item.split(splitchar)) _parse_list = __parse_list for item in _parse_list: if not item == '': if _output_list.count(item) < 1: _output_list.append(item) return _output_list def parse_ldap_uri(uri): """ Parse an LDAP URI and return it's components. Returns a tuple containing; - protocol (ldap, ldaps), - server (address or None), - base_dn, - attrs (list of attributes length 1, or empty), - scope, - filter or None on failure """ _protocol = uri.split(':')[0] try: try: _ldap_uri, _attr, _scope, _filter = uri.split('?') _server = _ldap_uri.split('//')[1].split('/')[0] _base_dn = _ldap_uri.split('//')[1].split('/')[1] except Exception: _server = uri.split('//')[1].split('/')[0] _attr = None _scope = None _filter = None _base_dn = None if len(_server.split(':')) > 1: _port = _server.split(':')[1] _server = _server.split(':')[0] else: if _protocol == 'ldaps': _port = "636" else: _port = "389" if _server == '': _server = None if _attr == '': _attrs = [] else: _attrs = [_attr] if _scope == '': _scope = 'sub' if _filter == '': _filter = "(objectclass=*)" return (_protocol, _server, _port, _base_dn, _attrs, _scope, _filter) except Exception: return None def pop_empty_from_list(_input_list): _output_list = [] for item in _input_list: if not item == '': _output_list.append(item) def sid_to_string(sid): srl = ord(sid[0]) number_sub_id = ord(sid[1]) iav = struct.unpack('!Q', '\x00\x00' + sid[2:8])[0] sub_ids = [] for i in range(number_sub_id): sub_ids.append(struct.unpack(' 1: (locale_name, locale_charset) = locale.normalize(locale_name).split('.') else: locale_charset = 'utf-8' try: log.debug(_l("Attempting to set locale"), level=8) locale.setlocale(locale.LC_ALL, (locale_name, locale_charset)) log.debug(_l("Success setting locale"), level=8) except Exception: log.debug(_l("Failure to set locale"), level=8) command = ['/usr/bin/iconv', '-f', 'UTF-8', '-t', 'ASCII//TRANSLIT', '-s'] log.debug(_l("Executing '%s | %s'") % (r"%s" % (mystring), ' '.join(command)), level=8) process = subprocess.Popen( command, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, env={'LANG': locale.normalize(locale_name)} ) try: print(r"%s" % (mystring), file=process.stdin) except UnicodeEncodeError: pass result = process.communicate()[0].strip() if '?' in result or (result == '' and not mystring == ''): log.warning(_l("Could not translate %s using locale %s") % (mystring, locale_name)) from pykolab import translit result = translit.transliterate(mystring, locale_name) return result def true_or_false(val): if val is None: return False if isinstance(val, bool): return val if isinstance(val, string_types): val = val.lower() if val in ["true", "yes", "y", "1"]: return True else: return False if isinstance(val, int) or isinstance(val, float): if val >= 1: return True else: return False def is_service(services): """ Checks each item in list services to see if it has a RC script in pykolab.constants.RC_DIR to see if it's a service, and returns the name of the service for the first service it can find. However, it also checks whether the other services exist and issues a warning if more then one service exists. Usage: utils.is_service(['dirsrv', 'ldap']) """ _service = None _other_services = [] for service in services: if os.path.isfile(os.path.join(constants.RC_DIR, service)): if _service == '': _service = service else: _other_services.append(service) return (_service, _other_services) diff --git a/wallace/modules.py b/wallace/modules.py index a1eaa7a..812b03c 100644 --- a/wallace/modules.py +++ b/wallace/modules.py @@ -1,457 +1,452 @@ # -*- coding: utf-8 -*- # Copyright 2010-2019 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from __future__ import print_function from six import string_types import os import sys import time from email import message_from_string from email.message import Message from email.mime.base import MIMEBase from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.parser import Parser from email.utils import COMMASPACE from email.utils import formatdate from email.utils import formataddr from email.utils import getaddresses from email.utils import parsedate_tz import smtplib import pykolab from pykolab import constants from pykolab.translate import _ log = pykolab.getLogger('pykolab.wallace/modules') extra_log_params = {'qid': '-'} log = pykolab.logger.LoggerAdapter(log, extra_log_params) conf = pykolab.getConf() modules = {} def initialize(): # We only want the base path modules_base_path = os.path.dirname(__file__) for modules_path, dirnames, filenames in os.walk(modules_base_path): if not modules_path == modules_base_path: continue for filename in filenames: if filename.startswith('module_') and filename.endswith('.py'): module_name = filename.replace('.py', '') name = module_name.replace('module_', '') # print("exec(\"from %s import __init__ as %s_register\")" % (module_name,name)) exec("from .%s import __init__ as %s_register" % (module_name, name)) exec("%s_register()" % (name)) for dirname in dirnames: register_group(modules_path, dirname) def list_modules(*args, **kw): """ List modules """ __modules = {} for module in modules: if isinstance(module, tuple): module_group, module = module __modules[module_group] = { module: modules[(module_group, module)] } else: __modules[module] = modules[module] - _modules = __modules.keys() - _modules.sort() - - for _module in _modules: + for _module in sorted(__modules): if 'function' in __modules[_module]: # This is a top-level module if __modules[_module]['description'] is not None: print("%-25s - %s" % (_module.replace('_', '-'), __modules[_module]['description'])) else: print("%-25s" % (_module.replace('_', '-'))) - for _module in _modules: + for _module in sorted(__modules): if 'function' not in __modules[_module]: # This is a nested module print("\n" + _("Module Group: %s") % (_module) + "\n") - ___modules = __modules[_module].keys() - ___modules.sort() - for __module in ___modules: + for __module in sorted(__modules[_module]): if __modules[_module][__module]['description'] is not None: print( "%-4s%-21s - %s" % ( '', _module.replace('_', '-'), __modules[_module][__module]['description'] ) ) else: print("%-4s%-21s" % ('', __module.replace('_', '-'))) def execute(name, *args, **kw): if name not in modules: log.error(_("No such module %r in modules %r (1).") % (name, modules)) sys.exit(1) if 'function' not in modules[name] and 'group' not in modules[name]: log.error(_("No such module %r in modules %r (2).") % (name, modules)) sys.exit(1) try: return modules[name]['function'](*args, **kw) except Exception as errmsg: log.exception(_("Module %r - Unknown error occurred; %r") % (name, errmsg)) def heartbeat(name, *args, **kw): if name not in modules: log.warning(_("No such module %r in modules %r (1).") % (name, modules)) if 'heartbeat' in modules[name]: return modules[name]['heartbeat'](*args, **kw) def _sendmail(sender, recipients, msg): # NOTE: Use "127.0.0.1" here for IPv6 (see also the service # definition in master.cf). sl = pykolab.logger.StderrToLogger(log) smtplib.stderr = sl smtp = smtplib.SMTP(timeout=15) if conf.debuglevel > 8: smtp.set_debuglevel(1) success = False attempt = 1 while not success and attempt <= 5: try: log.debug(_("Sending email via smtplib from %r, to %r (Attempt %r)") % (sender, recipients, attempt), level=8) smtp.connect("127.0.0.1", 10027) _response = smtp.sendmail(sender, recipients, msg) if len(_response) == 0: log.debug(_("SMTP sendmail OK"), level=8) else: log.debug(_("SMTP sendmail returned: %r") % (_response), level=8) smtp.quit() success = True break except smtplib.SMTPServerDisconnected as errmsg: log.error("SMTP Server Disconnected Error, %r" % (errmsg)) except smtplib.SMTPConnectError as errmsg: # DEFER log.error("SMTP Connect Error, %r" % (errmsg)) except smtplib.SMTPDataError as errmsg: # DEFER log.error("SMTP Data Error, %r" % (errmsg)) except smtplib.SMTPHeloError as errmsg: # DEFER log.error("SMTP HELO Error, %r" % (errmsg)) except smtplib.SMTPRecipientsRefused as errmsg: # REJECT, send NDR log.error("SMTP Recipient(s) Refused, %r" % (errmsg)) except smtplib.SMTPSenderRefused as errmsg: # REJECT, send NDR log.error("SMTP Sender Refused, %r" % (errmsg)) except Exception as errmsg: log.exception(_("smtplib - Unknown error occurred: %r") % (errmsg)) try: smtp.quit() except Exception as errmsg: log.error("smtplib quit() error - %r" % errmsg) time.sleep(10) attempt += 1 return success def cb_action_HOLD(module, filepath): global extra_log_params extra_log_params['qid'] = os.path.basename(filepath) log.info(_("Holding message in queue for manual review (%s by %s)") % (filepath, module)) def cb_action_DEFER(module, filepath): global extra_log_params extra_log_params['qid'] = os.path.basename(filepath) log.info(_("Deferring message in %s (by module %s)") % (filepath, module)) # parse message headers message = Parser().parse(open(filepath, 'r'), True) internal_time = parsedate_tz(message.__getitem__('Date')) internal_time = time.mktime(internal_time[:9]) + internal_time[9] now_time = time.time() delta = now_time - internal_time log.debug(_("The time when the message was sent: %r") % (internal_time), level=8) log.debug(_("The time now: %r") % (now_time), level=8) log.debug(_("The time delta: %r") % (delta), level=8) if delta > 432000: # TODO: Send NDR back to user log.debug(_("Message in file %s older then 5 days, deleting") % (filepath), level=8) os.unlink(filepath) # Alternative method is file age. #Date sent(/var/spool/pykolab/wallace/optout/DEFER/tmpIv7pDl): 'Thu, 08 Mar 2012 11:51:03 +0000' #(2012, 3, 8, 11, 51, 3, 0, 1, -1) # YYYY M D H m s weekday, yearday #log.debug(datetime.datetime(*), level=8) #import os #stat = os.stat(filepath) #fileage = datetime.datetime.fromtimestamp(stat.st_mtime) #now = datetime.datetime.now() #delta = now - fileage #print("file:", filepath, "fileage:", fileage, "now:", now, "delta(seconds):", delta.seconds) #if delta.seconds > 1800: ## TODO: Send NDR back to user #log.debug(_("Message in file %s older then 1800 seconds, deleting") % (filepath), level=8) #os.unlink(filepath) def cb_action_REJECT(module, filepath): global extra_log_params extra_log_params['qid'] = os.path.basename(filepath) log.info(_("Rejecting message in %s (by module %s)") % (filepath, module)) log.debug(_("Rejecting message in: %r") %(filepath), level=8) # parse message headers message = Parser().parse(open(filepath, 'r'), True) envelope_sender = getaddresses(message.get_all('From', [])) recipients = getaddresses(message.get_all('To', [])) + \ getaddresses(message.get_all('Cc', [])) + \ getaddresses(message.get_all('X-Kolab-To', [])) _recipients = [] for recipient in recipients: if not recipient[0] == '': _recipients.append('%s <%s>' % (recipient[0], recipient[1])) else: _recipients.append('%s' % (recipient[1])) # TODO: Find the preferredLanguage for the envelope_sender user. ndr_message_subject = "Undelivered Mail Returned to Sender" ndr_message_text = _("""This is the email system Wallace at %s. I'm sorry to inform you we could not deliver the attached message to the following recipients: - %s Your message is being delivered to any other recipients you may have sent your message to. There is no need to resend the message to those recipients. """) % ( constants.fqdn, "\n- ".join(_recipients) ) diagnostics = _("""X-Wallace-Module: %s X-Wallace-Result: REJECT """) % ( module ) msg = MIMEMultipart("report") msg['From'] = "MAILER-DAEMON@%s" % (constants.fqdn) msg['To'] = formataddr(envelope_sender[0]) msg['Date'] = formatdate(localtime=True) msg['Subject'] = ndr_message_subject msg.preamble = "This is a MIME-encapsulated message." part = MIMEText(ndr_message_text) part.add_header("Content-Description", "Notification") msg.attach(part) _diag_message = Message() _diag_message.set_payload(diagnostics) part = MIMEMessage(_diag_message, "delivery-status") part.add_header("Content-Description", "Delivery Report") msg.attach(part) # @TODO: here I'm not sure message will contain the whole body # when we used headersonly argument of Parser().parse() above # delete X-Kolab-* headers del message['X-Kolab-From'] del message['X-Kolab-To'] part = MIMEMessage(message) part.add_header("Content-Description", "Undelivered Message") msg.attach(part) result = _sendmail( "MAILER-DAEMON@%s" % (constants.fqdn), [formataddr(envelope_sender[0])], msg.as_string() ) log.debug(_("Rejection message was sent successfully: %r") % result) if result: os.unlink(filepath) else: log.debug(_("Message %r was not removed from spool") % filepath) def cb_action_ACCEPT(module, filepath): global extra_log_params extra_log_params['qid'] = os.path.basename(filepath) log.info(_("Accepting message in %s (by module %s)") % (filepath, module)) log.debug(_("Accepting message in: %r") %(filepath), level=8) # parse message headers message = Parser().parse(open(filepath, 'r'), True) messageid = message['message-id'] if 'message-id' in message else None sender = [formataddr(x) for x in getaddresses(message.get_all('X-Kolab-From', []))] recipients = [formataddr(x) for x in getaddresses(message.get_all('X-Kolab-To', []))] log.debug( _("Message-ID: %s, sender: %r, recipients: %r") % (messageid, sender, recipients), level=6 ) # delete X-Kolab-* headers del message['X-Kolab-From'] del message['X-Kolab-To'] log.debug(_("Removed X-Kolab- headers"), level=8) result = _sendmail( sender, recipients, # - Make sure we do not send this as binary. # - Second, strip NUL characters - I don't know where they # come from (TODO) # - Third, a character return is inserted somewhere. It # divides the body from the headers - and we don't like (TODO) # @TODO: check if we need Parser().parse() to load the whole message message.as_string() ) log.debug(_("Message was sent successfully: %r") % result) if result: os.unlink(filepath) else: log.debug(_("Message %r was not removed from spool") % filepath) def register_group(dirname, module): modules_base_path = os.path.join(os.path.dirname(__file__), module) modules[module] = {} for modules_path, dirnames, filenames in os.walk(modules_base_path): if not modules_path == modules_base_path: continue for filename in filenames: if filename.startswith('module_') and filename.endswith('.py'): module_name = filename.replace('.py','') name = module_name.replace('module_', '') # TODO: Error recovery from incomplete / incorrect modules. exec( "from .%s.%s import __init__ as %s_%s_register" % ( module, module_name, module, name ) ) exec("%s_%s_register()" % (module,name)) def register(name, func, group=None, description=None, aliases=[], heartbeat=None): if not group == None: module = "%s_%s" % (group,name) else: module = name if isinstance(aliases, string_types): aliases = [aliases] if module in modules: log.fatal(_("Module '%s' already registered") % (module)) sys.exit(1) if callable(func): if group == None: modules[name] = { 'function': func, 'description': description } else: modules[group][name] = { 'function': func, 'description': description } modules[module] = modules[group][name] modules[module]['group'] = group modules[module]['name'] = name for alias in aliases: modules[alias] = { 'function': func, 'description': _("Alias for %s") % (name) } if callable(heartbeat): modules[module]['heartbeat'] = heartbeat