diff --git a/pykolab/cli/__init__.py b/pykolab/cli/__init__.py index 3aafd03..2b3c511 100644 --- a/pykolab/cli/__init__.py +++ b/pykolab/cli/__init__.py @@ -1,76 +1,80 @@ # -*- coding: utf-8 -*- # Copyright 2010-2013 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # - +try: + # Python 2: "reload" is built-in + reload +except NameError: + from importlib import reload import ldap import ldif import logging import traceback import shutil import sys import time import codecs import locale from ldap.modlist import addModlist import pykolab import pykolab.plugins from pykolab import utils from pykolab import conf from pykolab.constants import * from pykolab.translate import _ log = pykolab.getLogger('pykolab.cli') conf = pykolab.getConf() class Cli(object): def __init__(self): from . import commands commands.__init__() to_execute = [] arg_num = 0 for arg in sys.argv[1:]: arg_num += 1 if not arg.startswith('-') and len(sys.argv) >= arg_num: if sys.argv[arg_num].replace('-','_') in commands.commands: to_execute.append(sys.argv[arg_num].replace('-','_')) if "%s_%s" % ( '_'.join(to_execute),sys.argv[arg_num].replace('-','_') ) in commands.commands: to_execute.append(sys.argv[arg_num].replace('-','_')) for cmd_component in to_execute: sys.argv.pop(sys.argv.index(cmd_component.replace('_','-'))) # force default encoding to match the locale encoding (T249) reload(sys) sys.setdefaultencoding(locale.getpreferredencoding() or 'utf-8') # wrap sys.stdout in a locale-aware StreamWriter (#3983) sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout) commands.execute('_'.join(to_execute)) def run(self): pass diff --git a/pykolab/wap_client/__init__.py b/pykolab/wap_client/__init__.py index f8427c0..176ac91 100644 --- a/pykolab/wap_client/__init__.py +++ b/pykolab/wap_client/__init__.py @@ -1,664 +1,667 @@ import json -import httplib +try: + import httplib +except ImportError: + import http.client as httplib import urllib import sys try: from urlparse import urlparse except ImportError: from urllib.parse import urlparse import pykolab from pykolab import utils from pykolab.translate import _ log = pykolab.getLogger('pykolab.wap_client') conf = pykolab.getConf() if not hasattr(conf, 'defaults'): conf.finalize_conf() API_HOSTNAME = "localhost" API_SCHEME = "http" API_PORT = 80 API_SSL = False API_BASE = "/kolab-webadmin/api/" kolab_wap_url = conf.get('kolab_wap', 'api_url') if not kolab_wap_url == None: result = urlparse(kolab_wap_url) else: result = None if hasattr(result, 'scheme') and result.scheme == 'https': API_SSL = True API_PORT = 443 if hasattr(result, 'hostname'): API_HOSTNAME = result.hostname if hasattr(result, 'port'): API_PORT = result.port if hasattr(result, 'path'): API_BASE = result.path session_id = None conn = None def authenticate(username=None, password=None, domain=None): global session_id if username == None: username = conf.get('ldap', 'bind_dn') if password == None: password = conf.get('ldap', 'bind_pw') if domain == None: domain = conf.get('kolab', 'primary_domain') post = json.dumps( { 'username': username, 'password': password, 'domain': domain } ) response = request('POST', "system.authenticate", post=post) if not response: return False if 'session_token' in response: session_id = response['session_token'] return True def connect(uri=None): global conn, API_SSL, API_PORT, API_HOSTNAME, API_BASE if not uri == None: result = urlparse(uri) if hasattr(result, 'scheme') and result.scheme == 'https': API_SSL = True API_PORT = 443 if hasattr(result, 'hostname'): API_HOSTNAME = result.hostname if hasattr(result, 'port'): API_PORT = result.port if hasattr(result, 'path'): API_BASE = result.path if conn == None: if API_SSL: conn = httplib.HTTPSConnection(API_HOSTNAME, API_PORT) else: conn = httplib.HTTPConnection(API_HOSTNAME, API_PORT) conn.connect() return conn def disconnect(quit=False): global conn, session_id if quit and session_id: request('GET', 'system.quit') session_id = None if conn: conn.close() conn = None def domain_add(domain, aliases=[]): dna = conf.get('ldap', 'domain_name_attribute') post = json.dumps({ dna: [ domain ] + aliases }) return request('POST', 'domain.add', post=post) def domain_delete(domain, force=False): domain_id, domain_attrs = domain_find(domain).popitem() param = {} param['id'] = domain_id if force: param['force'] = force post = json.dumps(param) return request('POST', 'domain.delete', post=post) def domain_find(domain): dna = conf.get('ldap', 'domain_name_attribute') get = { dna: domain } return request('GET', 'domain.find', get=get) def domain_info(domain): domain_id, domain_attrs = domain_find(domain) get = { 'id': domain_id } return request('GET', 'domain.info', get=get) def domains_capabilities(): return request('GET', 'domains.capabilities') def domains_list(): return request('GET', 'domains.list') def form_value_generate(params): post = json.dumps(params) return request('POST', 'form_value.generate', post=post) def form_value_generate_password(*args, **kw): return request('GET', 'form_value.generate_password') def form_value_list_options(object_type, object_type_id, attribute): post = json.dumps( { 'object_type': object_type, 'type_id': object_type_id, 'attribute': attribute } ) return request('POST', 'form_value.list_options', post=post) def form_value_select_options(object_type, object_type_id, attribute): post = json.dumps( { 'object_type': object_type, 'type_id': object_type_id, 'attributes': [ attribute ] } ) return request('POST', 'form_value.select_options', post=post) def get_group_input(): group_types = group_types_list() if len(group_types) > 1: for key in group_types: if not key == "status": print("%s) %s" % (key,group_types[key]['name'])) group_type_id = utils.ask_question("Please select the group type") elif len(group_types) > 0: print("Automatically selected the only group type available") group_type_id = group_types.keys()[0] else: print("No group types available") sys.exit(1) if group_type_id in group_types: group_type_info = group_types[group_type_id]['attributes'] else: print("No such group type") sys.exit(1) params = { 'group_type_id': group_type_id } for attribute in group_type_info['form_fields']: params[attribute] = utils.ask_question(attribute) for attribute in group_type_info['auto_form_fields']: exec("retval = group_form_value_generate_%s(params)" % (attribute)) params[attribute] = retval[attribute] return params def get_user_input(): user_types = user_types_list() if user_types['count'] > 1: print("") for key in user_types['list']: if not key == "status": print("%s) %s" % (key,user_types['list'][key]['name'])) print("") user_type_id = utils.ask_question("Please select the user type") elif user_types['count'] > 0: print("Automatically selected the only user type available") user_type_id = user_types['list'].keys()[0] else: print("No user types available") sys.exit(1) if user_type_id in user_types['list']: user_type_info = user_types['list'][user_type_id]['attributes'] else: print("No such user type") sys.exit(1) params = { 'object_type': 'user', 'type_id': user_type_id } must_attrs = [] may_attrs = [] for attribute in user_type_info['form_fields']: if isinstance(user_type_info['form_fields'][attribute], dict): if 'optional' in user_type_info['form_fields'][attribute] and user_type_info['form_fields'][attribute]['optional']: may_attrs.append(attribute) else: must_attrs.append(attribute) else: must_attrs.append(attribute) for attribute in must_attrs: if isinstance(user_type_info['form_fields'][attribute], dict) and \ 'type' in user_type_info['form_fields'][attribute]: if user_type_info['form_fields'][attribute]['type'] == 'select': if 'values' not in user_type_info['form_fields'][attribute]: attribute_values = form_value_select_options('user', user_type_id, attribute) default = '' if 'default' in attribute_values[attribute]: default = attribute_values[attribute]['default'] params[attribute] = utils.ask_menu( "Choose the %s value" % (attribute), attribute_values[attribute]['list'], default=default ) else: default = '' if 'default' in user_type_info['form_fields'][attribute]: default = user_type_info['form_fields'][attribute]['default'] params[attribute] = utils.ask_menu( "Choose the %s value" % (attribute), user_type_info['form_fields'][attribute]['values'], default=default ) else: params[attribute] = utils.ask_question(attribute) else: params[attribute] = utils.ask_question(attribute) for attribute in user_type_info['fields']: params[attribute] = user_type_info['fields'][attribute] exec("retval = user_form_value_generate(params)") print(retval) return params def group_add(params=None): if params == None: params = get_group_input() post = json.dumps(params) return request('POST', 'group.add', post=post) def group_delete(params=None): if params == None: params = { 'id': utils.ask_question("Name of group to delete", "group") } post = json.dumps(params) return request('POST', 'group.delete', post=post) def group_form_value_generate_mail(params=None): if params == None: params = get_user_input() params = json.dumps(params) return request('POST', 'group_form_value.generate_mail', params) def group_find(params=None): post = { 'search': { 'params': {} } } for (k,v) in params.items(): post['search']['params'][k] = { 'value': v, 'type': 'exact' } return request('POST', 'group.find', post=json.dumps(post)) def group_info(group=None): if group == None: group = utils.ask_question("group DN") return request('GET', 'group.info', get={ 'id': group }) def group_members_list(group=None): if group == None: group = utils.ask_question("Group email address") group = request('GET', 'group.members_list?group=%s' % (group)) return group def group_types_list(): return request('GET', 'group_types.list') def groups_list(params={}): return request('POST', 'groups.list', post=json.dumps(params)) def ou_add(params={}): return request('POST', 'ou.add', post=json.dumps(params)) def ou_delete(params={}): return request('POST', 'ou.delete', post=json.dumps(params)) def ou_edit(params={}): return request('POST', 'ou.edit', post=json.dumps(params)) def ou_find(params=None): post = { 'search': { 'params': {} } } for (k,v) in params.items(): post['search']['params'][k] = { 'value': v, 'type': 'exact' } return request('POST', 'ou.find', post=json.dumps(post)) def ou_info(ou): _params = { 'id': ou } ou = request('GET', 'ou.info', get=_params) return ou def ous_list(params={}): return request('POST', 'ous.list', post=json.dumps(params)) def request(method, api_uri, get=None, post=None, headers={}): response_data = request_raw(method, api_uri, get, post, headers) if response_data['status'] == "OK": del response_data['status'] return response_data['result'] else: print("%s: %s (code %s)" % (response_data['status'], response_data['reason'], response_data['code'])) return False def request_raw(method, api_uri, get=None, post=None, headers={}, isretry=False): global session_id if not session_id == None: headers["X-Session-Token"] = session_id reconnect = False conn = connect() if conf.debuglevel > 8: conn.set_debuglevel(9) if not get == None: _get = "?%s" % (urllib.urlencode(get)) else: _get = "" log.debug(_("Requesting %r with params %r") % ("%s/%s" % (API_BASE,api_uri), (get, post)), level=8) try: conn.request(method.upper(), "%s/%s%s" % (API_BASE, api_uri, _get), post, headers) response = conn.getresponse() data = response.read() log.debug(_("Got response: %r") % (data), level=8) except (httplib.BadStatusLine, httplib.CannotSendRequest) as e: if isretry: raise e log.info(_("Connection error: %r; re-connecting..."), e) reconnect = True # retry with a new connection if reconnect: disconnect() return request_raw(method, api_uri, get, post, headers, True) try: response_data = json.loads(data) except ValueError: # Some data is not JSON log.error(_("Response data is not JSON")) return response_data def resource_add(params=None): if params == None: params = get_user_input() return request('POST', 'resource.add', post=json.dumps(params)) def resource_delete(params=None): if params == None: params = { 'id': utils.ask_question("Resource DN to delete", "resource") } return request('POST', 'resource.delete', post=json.dumps(params)) def resource_find(params=None): post = { 'search': { 'params': {} } } for (k,v) in params.items(): post['search']['params'][k] = { 'value': v, 'type': 'exact' } return request('POST', 'resource.find', post=json.dumps(post)) def resource_info(resource=None): if resource == None: resource = utils.ask_question("Resource DN") return request('GET', 'resource.info', get={ 'id': resource }) def resource_types_list(): return request('GET', 'resource_types.list') def resources_list(params={}): return request('POST', 'resources.list', post=json.dumps(params)) def role_add(params=None): if params == None: role_name = utils.ask_question("Role name") params = { 'cn': role_name } params = json.dumps(params) return request('POST', 'role.add', params) def role_capabilities(): return request('GET', 'role.capabilities') def role_delete(params=None): if params == None: role_name = utils.ask_question("Role name") role = role_find_by_attribute({'cn': role_name}) params = { 'role': role.keys()[0] } if 'role' not in params: role = role_find_by_attribute(params) params = { 'role': role.keys()[0] } post = json.dumps(params) return request('POST', 'role.delete', post=post) def role_find_by_attribute(params=None): if params == None: role_name = utils.ask_question("Role name") else: role_name = params['cn'] get = { 'cn': role_name } role = request('GET', 'role.find_by_attribute', get=get) return role def role_info(role_name): role = role_find_by_attribute({'cn': role_name}) get = { 'role': role['id'] } role = request('GET', 'role.info', get=get) return role def roles_list(): return request('GET', 'roles.list') def sharedfolder_add(params=None): if params == None: params = get_user_input() return request('POST', 'sharedfolder.add', post=json.dumps(params)) def sharedfolder_delete(params=None): if params == None: params = { 'id': utils.ask_question("Shared Folder DN to delete", "sharedfolder") } return request('POST', 'sharedfolder.delete', post=json.dumps(params)) def sharedfolders_list(params={}): return request('POST', 'sharedfolders.list', post=json.dumps(params)) def system_capabilities(domain=None): return request('GET', 'system.capabilities', get={'domain':domain}) def system_get_domain(): return request('GET', 'system.get_domain') def system_select_domain(domain=None): if domain == None: domain = utils.ask_question("Domain name") get = { 'domain': domain } return request('GET', 'system.select_domain', get=get) def user_add(params=None): if params == None: params = get_user_input() params = json.dumps(params) return request('POST', 'user.add', post=params) def user_delete(params=None): if params == None: params = { 'id': utils.ask_question("Username for user to delete", "user") } post = json.dumps(params) return request('POST', 'user.delete', post=post) def user_edit(user = None, attributes={}): if user == None: get = { 'id': utils.ask_question("Username for user to edit", "user") } else: get = { 'id': user } user_info = request('GET', 'user.info', get=get) for attribute in attributes: user_info[attribute] = attributes[attribute] post = json.dumps(user_info) user_edit = request('POST', 'user.edit', get=get, post=post) return user_edit def user_find(attribs=None): if attribs == None: post = { 'search': { 'params': { utils.ask_question("Attribute") : { 'value': utils.ask_question("value"), 'type': 'exact' } } } } else: post = { 'search': { 'params': {} } } for (k,v) in attribs.items(): post['search']['params'][k] = { 'value': v, 'type': 'exact' } post = json.dumps(post) user = request('POST', 'user.find', post=post) return user def user_form_value_generate(params=None): if params == None: params = get_user_input() post = json.dumps(params) return request('POST', 'form_value.generate', post=post) def user_form_value_generate_uid(params=None): if params == None: params = get_user_input() params = json.dumps(params) return request('POST', 'form_value.generate_uid', params) def user_form_value_generate_userpassword(*args, **kw): result = form_value_generate_password() return { 'userpassword': result['password'] } def user_info(user=None): if user == None: user = utils.ask_question("User email address") _params = { 'id': user } user = request('GET', 'user.info', get=_params) return user def users_list(params={}): return request('POST', 'users.list', post=json.dumps(params)) def user_types_list(): return request('GET', 'user_types.list') diff --git a/wallace/__init__.py b/wallace/__init__.py index 4f73ac6..6f11b01 100644 --- a/wallace/__init__.py +++ b/wallace/__init__.py @@ -1,680 +1,683 @@ # -*- coding: utf-8 -*- # Copyright 2010-2019 Kolab Systems AG (http://www.kolabsys.com) # # Jeroen van Meeuwen (Kolab Systems) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from __future__ import print_function import asyncore from distutils import version import grp import multiprocessing import os import pwd import traceback import smtpd import socket import struct import sys import tempfile -from threading import _Timer +try: + from threading import _Timer +except: + from threading import Timer as _Timer import time import pykolab from pykolab import utils from pykolab.logger import StderrToLogger from pykolab.translate import _ as _l from . import modules from .modules import cb_action_ACCEPT # pylint: disable=invalid-name log = pykolab.getLogger('pykolab.wallace') sys.stderr = StderrToLogger(log) conf = pykolab.getConf() def pickup_message(filepath, *args, **kwargs): wallace_modules = args[0] if 'module' in kwargs: # Cause the previous modules to be skipped wallace_modules = wallace_modules[(wallace_modules.index(kwargs['module']) + 1):] log.debug(_l("Wallace modules: %r") % (wallace_modules), level=8) # Execute the module if 'stage' in kwargs: modules.execute(kwargs['module'], filepath, stage=kwargs['stage']) else: modules.execute(kwargs['module'], filepath) # After all modules are executed, continue with a call to # accept the message and re-inject in to Postfix. continue_with_accept = True for module in wallace_modules: try: result_filepath = modules.execute(module, filepath) except Exception: log.error( "Module %s.execute() failed on message %r with error: %s" % ( module, filepath, traceback.format_exc() ) ) result_filepath = False if result_filepath is not None and result_filepath is not False: filepath = result_filepath else: # A module has returned False or None continue_with_accept = False # The message very likely has been consumed by the module that returned False if not os.path.isfile(filepath): break if continue_with_accept: cb_action_ACCEPT('wallace', filepath) def modules_heartbeat(wallace_modules): lastrun = 0 while not multiprocessing.current_process().finished.is_set(): try: for module in wallace_modules: try: modules.heartbeat(module, lastrun) except Exception: log.error( "Module %s.heartbeat() failed with error: %s" % ( module, traceback.format_exc() ) ) lastrun = int(time.time()) multiprocessing.current_process().finished.wait(60) except (SystemExit, KeyboardInterrupt) as errmsg: log.warning("Exiting %s, %s" % (multiprocessing.current_process().name, errmsg)) break def worker_process(*args, **kwargs): import signal signal.signal(signal.SIGINT, signal.SIG_IGN) log.debug("Worker process %s initializing" % (multiprocessing.current_process().name), level=1) # pylint: disable=too-few-public-methods class Timer(_Timer): def run(self): while True: while not self.finished.is_set(): self.finished.wait(self.interval) log.debug(_l("Timer looping function '%s' every %ss") % ( self.function.__name__, self.interval ), level=8) self.function(*self.args, **self.kwargs) self.finished.set() log.debug( _l("Timer loop %s") % ('still active', 'finished')[self.finished.is_set()], level=8 ) break class WallaceDaemon: heartbeat = None timer = None def __init__(self): self.current_connections = 0 self.max_connections = 24 self.parent_pid = None self.pool = None daemon_group = conf.add_cli_parser_option_group(_l("Daemon Options")) daemon_group.add_option( "--fork", dest="fork_mode", action="store_true", default=False, help=_l("Fork to the background.") ) daemon_group.add_option( "-b", "--bind", dest="wallace_bind_address", action="store", default="localhost", help=_l("Bind address for Wallace.") ) daemon_group.add_option( "-g", "--group", dest="process_groupname", action="store", default="kolab", help=_l("Run as group GROUPNAME"), metavar="GROUPNAME" ) daemon_group.add_option( "--threads", dest="max_threads", action="store", default=4, type=int, help=_l("Number of threads to use.") ) daemon_group.add_option( "--max-tasks", dest="max_tasks", action="store", default=None, type=int, help=_l("Number of tasks per process.") ) daemon_group.add_option( "-p", "--pid-file", dest="pidfile", action="store", default="/var/run/wallaced/wallaced.pid", help=_l("Path to the PID file to use.") ) daemon_group.add_option( "--port", dest="wallace_port", action="store", default=10026, type=int, help=_l("Port that Wallace is supposed to use.") ) daemon_group.add_option( "-u", "--user", dest="process_username", action="store", default="kolab", help=_l("Run as user USERNAME"), metavar="USERNAME" ) conf.finalize_conf() utils.ensure_directory( os.path.dirname(conf.pidfile), conf.process_username, conf.process_groupname ) if conf.debuglevel >= 9: mp_logger = multiprocessing.get_logger() mp_logger.setLevel(multiprocessing.SUBDEBUG) mp_logger.debug('Python multi-processing logger started') modules.initialize() self.modules = conf.get_list('wallace', 'modules') if not self.modules: self.modules = [] # pylint: disable=too-many-branches # pylint: disable=too-many-statements def do_wallace(self): self.parent_pid = os.getpid() if version.StrictVersion(sys.version[:3]) >= version.StrictVersion("2.7"): self.pool = multiprocessing.Pool(conf.max_threads, worker_process, (), conf.max_tasks) else: self.pool = multiprocessing.Pool(conf.max_threads, worker_process, ()) self.pickup_spool_messages(sync=True) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) bound = False shutdown = False while not bound: try: if shutdown: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((conf.wallace_bind_address, conf.wallace_port)) bound = True # pylint: disable=broad-except except Exception: log.warning( _l("Could not bind to socket on port %d on bind address %s") % ( conf.wallace_port, conf.wallace_bind_address ) ) while not shutdown: try: s.shutdown(socket.SHUT_RDWR) shutdown = True # pylint: disable=broad-except except Exception: log.warning(_l("Could not shut down socket")) time.sleep(1) s.close() time.sleep(1) s.listen(5) self.timer = Timer(180, self.pickup_spool_messages, args=[], kwargs={'sync': True}) # pylint: disable=attribute-defined-outside-init self.timer.daemon = True self.timer.start() # start background process to run periodic jobs in active modules try: self.heartbeat = multiprocessing.Process( target=modules_heartbeat, name='Wallace_Heartbeat', args=[self.modules] ) self.heartbeat.finished = multiprocessing.Event() self.heartbeat.daemon = True self.heartbeat.start() except Exception as errmsg: log.error("Failed to start heartbeat daemon: %s" % (errmsg)) finally: log.debug( "Wallace heartbeat is %s" % ('not alive', 'alive')[self.heartbeat.is_alive()], level=8 ) try: while 1: while self.current_connections >= self.max_connections: log.debug( _l("Reached limit of max connections of: %s. Sleeping for 0.5s") % ( self.max_connections ), level=6 ) time.sleep(0.5) pair = s.accept() log.debug( _l("Accepted connection %r with address %r") % ( pair if pair is not None else (None, None) ), level=8 ) if pair is not None: self.current_connections += 1 connection, address = pair _smtpd = smtpd # Set DEBUGSTREAM of smtpd to log to pykolab logger if conf.debuglevel > 8: _smtpd.DEBUGSTREAM = pykolab.logger.StderrToLogger(log) log.debug(_l("Creating SMTPChannel for accepted message"), level=8) _smtpd.SMTPChannel(self, connection, address) asyncore.loop() else: log.error(_l("Socket accepted, but (conn, address) tuple is None.")) # pylint: disable=broad-except except Exception: traceback.print_exc() s.shutdown(1) s.close() # shut down hearbeat process self.heartbeat.terminate() self.timer.cancel() self.timer.join() # pylint: disable=no-self-use def data_header(self, mailfrom, rcpttos): COMMASPACE = ', ' return "X-Kolab-From: " + mailfrom + "\r\n" + \ "X-Kolab-To: " + COMMASPACE.join(rcpttos) + "\r\n" def pickup_spool_messages(self, sync=False): # Mind you to include the trailing slash pickup_path = '/var/spool/pykolab/wallace/' messages = [] for root, _, files in os.walk(pickup_path): for filename in files: messages.append((root, filename)) for root, filename in messages: filepath = os.path.join(root, filename) try: # ignore calls on too young files if os.stat(filepath).st_mtime + 150 > time.time(): log.debug("File not more than 150s old. Skipping %s" % (filepath), level=8) continue # ignore calls on lock files if '/locks/' in filepath: log.debug("File is in locks directory. Skipping %s" % (filepath), level=8) continue # pylint: disable=broad-except except Exception as errmsg: log.error("Error: %s. Skipping %s" % (errmsg, filepath)) continue if not root == pickup_path: module = os.path.dirname(root).replace(pickup_path, '') # Compare uppercase status (specifically, DEFER) with # lowercase (plugin names). # # The messages in DEFER are supposed to be picked up by # another thread, whereas the messages in other directories # are pending being handled by their respective plugins. # # TODO: Handle messages in spool directories for which a # plugin had been enabled, but is not enabled any longer. # if module.lower() == "defer": # Wallace was unable to deliver to re-injection smtpd. # Skip it, another thread is picking up the deferred # messages. continue stage = root.replace(pickup_path, '').split('/') if len(stage) < 2: stage = None else: stage = stage[1] if stage.lower() == "hold": continue # Do not handle messages in a defer state. if stage.lower() == "defer": continue self.current_connections += 1 if sync: pickup_message(filepath, self.modules, module=module, stage=stage) else: self.pool.apply_async( pickup_message, ( filepath, (self.modules), {'module': module, 'stage': stage} ) ) self.current_connections -= 1 continue self.current_connections += 1 if sync: pickup_message(filepath, self.modules) else: self.pool.apply_async(pickup_message, (filepath, (self.modules))) self.current_connections -= 1 def process_message(self, peer, mailfrom, rcpttos, data): """ We have retrieved the message. This should be as fast as possible, and not ever block. """ header = self.data_header(mailfrom, rcpttos) (fp, filename) = tempfile.mkstemp(dir="/var/spool/pykolab/wallace/") # @TODO: and add line separator (\n or \r\n?) # we should make sure there's only one line separator between # kolab headers and the original message (data) os.write(fp, header) os.write(fp, data) os.close(fp) log.debug(_l("Started processing accepted message %s") % filename, level=8) self.pool.apply_async(pickup_message, (filename, (self.modules))) self.current_connections -= 1 return "250 OK Message %s queued" % (filename) def reload_config(self, *args, **kwargs): pass def remove_pid(self, *args, **kwargs): try: if os.getpid() == self.parent_pid: log.debug("Stopping process %s" % multiprocessing.current_process().name, level=8) log.debug(_l("Terminating processes pool"), level=8) self.pool.close() if hasattr(self, 'timer'): if not self.timer.finished.is_set(): log.debug("Canceling Wallace Timer", level=8) self.timer.finished.set() self.timer.cancel() log.debug(_l("Terminating heartbeat process"), level=8) self.heartbeat.finished.set() self.heartbeat.terminate() self.pool.close() self.pool.join(5) self.timer.join(5) self.heartbeat.join(5) if os.access(conf.pidfile, os.R_OK): log.warning(_l("Removing PID file %s") % conf.pidfile) os.remove(conf.pidfile) log.warning("Exiting!") sys.exit() else: sys.exit(0) except Exception as errmsg: log.debug( "Exception while trying to stop %s: %s" % ( multiprocessing.current_process().name, errmsg ), level=8 ) sys.exit(1) sys.exit(0) # pylint: disable=too-many-locals def run(self): # noqa: C901 """ Run the Wallace daemon. """ exitcode = 0 try: try: (ruid, _, _) = os.getresuid() (rgid, _, _) = os.getresgid() except AttributeError: ruid = os.getuid() rgid = os.getgid() if ruid == 0: # Means we can setreuid() / setregid() / setgroups() if rgid == 0: # Get group entry details try: (_, _, group_gid, _) = grp.getgrnam(conf.process_groupname) except KeyError: print(_l("Group %s does not exist") % (conf.process_groupname)) sys.exit(1) # Set real and effective group if not the same as current. if not group_gid == rgid: log.debug( _l("Switching real and effective group id to %d") % ( group_gid ), level=8 ) os.setregid(group_gid, group_gid) if ruid == 0: # Means we haven't switched yet. try: (_, _, user_uid, _, _, _, _) = pwd.getpwnam(conf.process_username) except KeyError: print(_l("User %s does not exist") % (conf.process_username)) sys.exit(1) # Set real and effective user if not the same as current. if not user_uid == ruid: log.debug( _l("Switching real and effective user id to %d") % ( user_uid ), level=8 ) os.setreuid(user_uid, user_uid) # pylint: disable=broad-except except Exception: log.error(_l("Could not change real and effective uid and/or gid")) try: pid = os.getpid() if conf.fork_mode: pid = os.fork() if pid > 0 and not conf.fork_mode: self.do_wallace() elif pid > 0: sys.exit(0) else: # Give up the session, all control, # all open file descriptors, see #5151 os.chdir("/") old_umask = os.umask(0) os.setsid() pid = os.fork() if pid > 0: sys.exit(0) sys.stderr.flush() sys.stdout.flush() os.close(0) os.close(1) os.close(2) os.open(os.devnull, os.O_RDONLY) os.open(os.devnull, os.O_WRONLY) os.open(os.devnull, os.O_WRONLY) os.umask(old_umask) log.remove_stdout_handler() self.set_signal_handlers() self.write_pid() self.do_wallace() except SystemExit as errmsg: exitcode = errmsg except KeyboardInterrupt: exitcode = 1 log.info(_l("Interrupted by user")) except AttributeError: exitcode = 1 traceback.print_exc() print(_l("Traceback occurred, please report a bug.")) except TypeError as errmsg: exitcode = 1 traceback.print_exc() log.error(_l("Type Error: %s") % errmsg) except Exception: exitcode = 2 traceback.print_exc() print(_l("Traceback occurred, please report a bug.")) sys.exit(exitcode) def set_signal_handlers(self): import signal signal.signal(signal.SIGHUP, self.reload_config) signal.signal(signal.SIGTERM, self.remove_pid) def write_pid(self): pid = os.getpid() if os.access(os.path.dirname(conf.pidfile), os.W_OK): fp = open(conf.pidfile, 'w') fp.write("%d\n" % (pid)) fp.close() else: print(_l("Could not write pid file %s") % (conf.pidfile))