diff --git a/app/model/kolabobject.py b/app/model/kolabobject.py index 250c317..e948382 100644 --- a/app/model/kolabobject.py +++ b/app/model/kolabobject.py @@ -1,369 +1,374 @@ # -*- coding: utf-8 -*- # # Copyright 2014 Kolab Systems AG (http://www.kolabsys.com) # # Thomas Bruederli # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # import json import pytz import hashlib import datetime import logging from dateutil.parser import parse as parse_date from pykolab.xml.utils import compute_diff from collections import OrderedDict from email import message_from_string from app import storage log = logging.getLogger('model') class KolabObject(object): """ Base Model class for accessing Kolab Groupware Object data """ folder_type = 'unknown' x_kolab_type = 'application/x-vnd.kolab.*' def __init__(self, env={}): from flask import current_app self.env = env self.config = current_app.config self.storage = storage.factory() def created(self, uid, mailbox, msguid=None): """ Provide created date and user """ changelog = self._object_changelog(uid, mailbox, msguid, 1) if changelog and len(changelog) > 0: for change in changelog: if change['op'] == 'APPEND': change['uid'] = uid change.pop('op', None) return change return False def lastmodified(self, uid, mailbox, msguid=None): """ Provide last change information """ changelog = self._object_changelog(uid, mailbox, msguid, -3) if changelog and len(changelog) > 0: for change in changelog: if change['op'] == 'APPEND': change['uid'] = uid change.pop('op', None) return change return False def changelog(self, uid, mailbox, msguid=None): """ Full changelog """ changelog = self._object_changelog(uid, mailbox, msguid) if changelog: return dict(uid=uid, changes=changelog) return False def get(self, uid, rev, mailbox, msguid=None): """ Retrieve an old revision """ obj = self._get(uid, mailbox, msguid, rev) if obj is not None: return dict(uid=uid, rev=rev, xml=str(obj), mailbox=mailbox) return False def _get(self, uid, mailbox, msguid, rev): """ Get an old revision and return the pykolab.xml object """ obj = False rec = self.storage.get_revision(uid, self._resolve_mailbox_uri(mailbox), msguid, rev) if rec is not None: raw = self.storage.get_message_data(rec) try: message = message_from_string(raw.encode('utf8','replace')) obj = self._object_from_message(message) or False except Exception, e: log.warning("Failed to parse mime message for UID %s @%s: %r", uid, rev, e) if obj is False: log.warning("Failed to parse mime message for UID %s @%s", uid, rev) return obj def diff(self, uid, rev1, rev2, mailbox, msguid=None, instance=None): """ Compare two revisions of an object and return a list of property changes """ rev_old = rev1 rev_new = rev2 if rev_old >= rev_new: raise ValueError("Invalid argument 'rev'") old = self._get(uid, mailbox, msguid, rev_old) if old == False: raise ValueError("Object %s @rev:%s not found" % (uid, str(rev_old))) new = self._get(uid, mailbox, msguid, rev_new) if new == False: raise ValueError("Object %s @rev:%s not found" % (uid, str(rev_new))) # compute diff for the requested recurrence instance if instance is not None and hasattr(old, 'get_instance') and hasattr(new, 'get_instance'): log.debug("Get recurrence instance %s for object %s", instance, uid) try: recurrence_date = datetime.datetime.strptime(str(instance), "%Y%m%dT%H%M%S") except: try: recurrence_date = datetime.date.strptime(str(instance), "%Y%m%d").date() except: raise ValueError("Invalid isntance identifier %r" % (instance)) old = old.get_instance(recurrence_date) if old == None: raise ValueError("Object instance %s-%s @rev:%s not found" % (uid, instance, str(rev_old))) new = new.get_instance(recurrence_date) if new == None: raise ValueError("Object instance %s-%s @rev:%s not found" % (uid, instance, str(rev_new))) - return dict(uid=uid, rev=rev_new, changes=convert2primitives(compute_diff(old.to_dict(), new.to_dict(), False))) + result = dict(uid=uid, rev=rev_new, changes=convert2primitives(compute_diff(old.to_dict(), new.to_dict(), False))) + + if instance is not None: + result['instance'] = instance + + return result def rawdata(self, uid, mailbox, rev, msguid=None): """ Get the full message payload of an old revision """ rec = self.storage.get_revision(uid, self._resolve_mailbox_uri(mailbox), msguid, rev) if rec is not None: return self.storage.get_message_data(rec) return False def _object_from_message(self, message): """ To be implemented in derived classes """ return None def _object_changelog(self, uid, mailbox, msguid, limit=None): """ Query storage for changelog events related to the given UID """ # this requires a user context if not self.env.has_key('REQUEST_USER') or not self.env['REQUEST_USER']: return None # fetch event log from storage eventlog = self.storage.get_events(uid, self._resolve_mailbox_uri(mailbox), msguid, limit) # convert logstash entries into a sane changelog event_op_map = { 'MessageNew': 'APPEND', 'MessageAppend': 'APPEND', 'MessageTrash': 'DELETE', 'MessageMove': 'MOVE', } last_append_uid = 0 result = [] if eventlog is not None: for log in eventlog: # filter MessageTrash following a MessageAppend event (which is an update operation) if log['event'] == 'MessageTrash' and last_append_uid > int(log['uidset']): continue # remember last appended message uid if log['event'] == 'MessageAppend' and log.has_key('uidset'): last_append_uid = int(log['uidset']) # compose log entry to return logentry = { 'rev': log.get('revision', None), 'op': event_op_map.get(log['event'], 'UNKNOWN'), 'mailbox': self._convert_mailbox_uri(log.get('mailbox', None)) } try: timestamp = parse_date(log['timestamp']) logentry['date'] = datetime.datetime.strftime(timestamp, "%Y-%m-%dT%H:%M:%SZ") except: logentry['date'] = log['timestamp'] # TODO: translate mailbox identifier back to a relative folder path? logentry['user'] = self._get_user_info(log) result.append(logentry) return result def _resolve_username(self, user): """ Resovle the given username to the corresponding nsuniqueid from LDAP """ # find existing entry in our storage backend result = self.storage.get_user(username=user) if result and result.has_key('id'): # TODO: cache this lookup in memory? return result['id'] # fall-back: return md5 sum of the username to make usernames work as fields/keys in elasticsearch return hashlib.md5(user).hexdigest() def _get_user_info(self, rec): """ Return user information (name, email) related to the given log entry """ if rec.has_key('user_id'): # get real user name from rec['user_id'] user = self.storage.get_user(id=rec['user_id']) if user is not None: return "%(cn)s <%(user)s>" % user if rec.has_key('user'): return rec['user'] elif rec['event'] == 'MessageAppend' and rec['headers'].has_key('From'): # fallback to message headers return rec['headers']['From'][0] return 'unknown' def _resolve_mailbox_uri(self, mailbox): """ Convert the given mailbox string into an absolute URI regarding the context of the requesting user. """ # this requires a user context if not self.env.has_key('REQUEST_USER') or not self.env['REQUEST_USER']: return mailbox if mailbox is None: return None # mailbox already is an absolute path if mailbox.startswith('user/') or mailbox.startswith('shared/'): return mailbox domain = '' user = self.env['REQUEST_USER'] if '@' in user: (user,_domain) = user.split('@', 1) domain = '@' + _domain owner = user path = '/' + mailbox # TODO: make this configurable or read from IMAP shared_prefix = 'Shared Folders/' others_prefix = 'Other Users/' imap_delimiter = '/' # case: shared folder if mailbox.startswith(shared_prefix): return mailbox[len(shared_prefix):] + domain # case: other users folder if mailbox.startswith(others_prefix): (owner, subpath) = mailbox[len(others_prefix):].split(imap_delimiter, 1) path = imap_delimiter + subpath if mailbox.upper() == 'INBOX': path = '' # default: personal namespace folder return 'user/' + owner + path + domain def _convert_mailbox_uri(self, mailbox): """ Convert the given absolute mailbox URI into a relative folder name regarding the context of the requesting user. """ if mailbox is None: return None # this requires a user context request_user = str(self.env.get('REQUEST_USER', '')).lower() # TODO: make this configurable or read from IMAP shared_prefix = 'Shared Folders' others_prefix = 'Other Users' imap_delimiter = '/' domain = '' if '@' in mailbox: (folder,domain) = mailbox.split('@', 1) else: folder = mailbox if folder.startswith('user/'): parts = folder.split(imap_delimiter, 2) if len(parts) > 2: (prefix,user,path) = parts else: (prefix,user) = parts path = '' if len(path) == 0: path = 'INBOX' if not (user + '@' + domain).lower() == request_user: folder = imap_delimiter.join([others_prefix, user, path]) else: folder = path elif folder.startswith('shared/'): folder = imap_delimiter.join([shared_prefix, folder]) return folder ##### Utility functions def convert2primitives(struct): """ Convert complex types like datetime into primitives which can be serialized into JSON """ out = None if isinstance(struct, datetime.datetime): tz = 'Z' if struct.tzinfo == pytz.utc else '%z' out = struct.strftime('%Y-%m-%dT%H:%M:%S' + tz) elif isinstance(struct, datetime.date): out = struct.strftime('%Y-%m-%d') elif isinstance(struct, list): out = [convert2primitives(x) for x in struct] elif isinstance(struct, OrderedDict): out = OrderedDict([(key,convert2primitives(struct[key])) for key in struct.keys()]) elif isinstance(struct, dict): out = dict(zip(struct.keys(), map(convert2primitives, struct.values()))) else: out = struct return out diff --git a/tests/test_api.py b/tests/test_api.py index 8f44f4b..0c74e9d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,200 +1,240 @@ import httplib, json, base64, hmac, hashlib, email from twisted.trial import unittest class TestAPI(unittest.TestCase): reqid = 0 # test config config = { 'api_host': '127.0.0.1', 'api_port': 8080, 'api_user': 'webclient', 'api_pass': 'Welcome2KolabSystems', 'api_secret': '8431f19170e7f90d4107bf4b169baf' } @classmethod def setUp(self): # TODO: reset riak buckets and fill with sample data # TODO: fire up the web service pass def _api_request(self, user, method, sign=True, **kwargs): """ Helper method to send JSON-RPC calls to the API """ headers = { 'Authorization': 'Basic ' + base64.b64encode(self.config['api_user'] + ':' + self.config['api_pass']), 'Content-type': "application/json", 'X-Request-User': user } self.rpcerror = None self.reqid += 1 body = { 'jsonrpc': '2.0', 'id': self.reqid, 'method': method, 'params': kwargs } sbody = json.dumps(body) if sign: headers['X-Request-Sign'] = hmac.new( key=self.config['api_secret'], msg=headers.get('X-Request-User') + ':' + sbody, digestmod=hashlib.sha256 ).hexdigest() try: conn = httplib.HTTPConnection(self.config['api_host'], self.config['api_port']) conn.request('POST', '/api/rpc', sbody, headers) result = None response = conn.getresponse() if response.status == 200: result = json.loads(response.read()) if result.has_key('error'): self.rpcerror = result['error'] if result.has_key('result') and result.has_key('id') and result['id'] == self.reqid: return result['result'] # print "JSON-RPC response: %d %s; %r" % (response.status, response.reason, result) except Exception, e: print "JSON-RPC error: %r" % (e) return False def assertRPCError(self, code=None): self.assertIsInstance(self.rpcerror, dict) if code is not None: self.assertEqual(self.rpcerror['code'], code) def test_001_json_rpc(self): res = self._api_request('', 'system.keygen', sign=False) self.assertFalse(res) self.assertRPCError(-32600) res = self._api_request('', 'system.keygen', sign=True) self.assertIsInstance(res, dict) self.assertTrue(res.has_key('key')) self.assertTrue(res.has_key('expires')) def test_002_invalid_request(self): res = self._api_request('john.doe@example.org', 'event.bogus', uid='6EE0570E8CA21DDB67FC9ADE5EE38E7F-XXXXXXXXXXXXXXX' ) self.assertFalse(res) self.assertRPCError(-32601) # Method not found res = self._api_request('john.doe@example.org', 'event.created', uid='6EE0570E8CA21DDB67FC9ADE5EE38E7F-A4BF5BBB9FEAA271' ) self.assertFalse(res) self.assertRPCError(-32602) # Invalid params res = self._api_request('john.doe@example.org', 'event.created', uid='6EE0570E8CA21DDB67FC9ADE5EE38E7F-INVALID-UID', mailbox='Calendar' ) self.assertFalse(res) def test_003_created_lastmodified(self): created = self._api_request('john.doe@example.org', 'event.created', uid='6EE0570E8CA21DDB67FC9ADE5EE38E7F-A4BF5BBB9FEAA271', mailbox='Testing', msguid=2 ) self.assertIsInstance(created, dict) self.assertTrue(created.has_key('date')) self.assertTrue(created.has_key('user')) self.assertTrue(created.has_key('rev')) self.assertIn('john.doe@example.org', created['user']) changed = self._api_request('john.doe@example.org', 'event.lastmodified', uid='6EE0570E8CA21DDB67FC9ADE5EE38E7F-A4BF5BBB9FEAA271', mailbox='Testing' ) self.assertIsInstance(changed, dict) self.assertTrue(changed.has_key('rev')) self.assertTrue(changed['rev'] > created['rev']) self.assertTrue(changed['date'] > created['date']) def test_004_changelog(self): changelog = self._api_request('john.doe@example.org', 'event.changelog', uid='5A637BE7895D785671E1732356E65CC8-A4BF5BBB9FEAA271', mailbox='Calendar', msguid=22 ) # print json.dumps(changelog, indent=4) self.assertIsInstance(changelog, dict) self.assertTrue(changelog.has_key('changes')) self.assertTrue(changelog.has_key('uid')) self.assertEqual(len(changelog['changes']), 2) one = changelog['changes'][0] two = changelog['changes'][1] self.assertTrue(two['rev'] > one['rev']) def test_005_get_revision(self): changelog = self._api_request('john.doe@example.org', 'event.changelog', uid='5A637BE7895D785671E1732356E65CC8-A4BF5BBB9FEAA271', mailbox='Calendar' ) self.assertIsInstance(changelog, dict) self.assertEqual(len(changelog.get('changes', [])), 2) rev1 = self._api_request('john.doe@example.org', 'event.get', uid='5A637BE7895D785671E1732356E65CC8-A4BF5BBB9FEAA271', mailbox='Calendar', rev=changelog['changes'][0]['rev'] ) self.assertIsInstance(rev1, dict) self.assertTrue(rev1.has_key('xml')) self.assertNotIn('', rev1['xml']) rev2 = self._api_request('john.doe@example.org', 'event.get', uid='5A637BE7895D785671E1732356E65CC8-A4BF5BBB9FEAA271', mailbox='Calendar', rev=changelog['changes'][1]['rev'] ) self.assertIsInstance(rev2, dict) self.assertTrue(rev2.has_key('xml')) self.assertIn('Somewhere else', rev2['xml']) msgdata = self._api_request('john.doe@example.org', 'event.rawdata', uid='5A637BE7895D785671E1732356E65CC8-A4BF5BBB9FEAA271', mailbox='Calendar', rev=changelog['changes'][0]['rev'] ) self.assertIsInstance(msgdata, unicode) message = email.message_from_string(msgdata.encode('utf8','replace')) self.assertIsInstance(message, email.message.Message) self.assertTrue(message.is_multipart()) def test_006_diff(self): changelog = self._api_request('john.doe@example.org', 'event.changelog', uid='5A637BE7895D785671E1732356E65CC8-A4BF5BBB9FEAA271', mailbox='Calendar' ) self.assertIsInstance(changelog, dict) self.assertEqual(len(changelog.get('changes', [])), 2) rev1 = changelog['changes'][0]['rev'] rev2 = changelog['changes'][1]['rev'] diff = self._api_request('john.doe@example.org', 'event.diff', - uid='5A637BE7895D785671E1732356E65CC8-A4BF5BBB9FEAA271', + uid=changelog['uid'], mailbox='Calendar', rev1=rev1, rev2=rev2 ) # print json.dumps(diff, indent=4) self.assertIsInstance(diff, dict) self.assertTrue(diff.has_key('changes')) self.assertTrue(diff.has_key('uid')) self.assertEqual(len(diff['changes']), 6) - \ No newline at end of file + def test_007_diff_instance(self): + changelog = self._api_request('john.doe@example.org', 'event.changelog', + uid='8B3B2C54C5218FC09EBC840E6289F5E5-A4BF5BBB9FEAA271', + mailbox='Calendar' + ) + self.assertIsInstance(changelog, dict) + self.assertEqual(len(changelog.get('changes', [])), 2) + + rev1 = changelog['changes'][0]['rev'] + rev2 = changelog['changes'][1]['rev'] + + diff = self._api_request('john.doe@example.org', 'event.diff', + uid=changelog['uid'], + mailbox='Calendar', + rev1=rev1, + rev2=rev2, + instance='20150324T210000' + ) + self.assertIsInstance(diff, dict) + self.assertTrue(diff.has_key('instance')) + self.assertEqual(len(diff['changes']), 1) # no change (except lastmodified-date) in this instance + + diff = self._api_request('john.doe@example.org', 'event.diff', + uid=changelog['uid'], + mailbox='Calendar', + rev1=rev1, + rev2=rev2, + instance='20150325T210000' + ) + self.assertIsInstance(diff, dict) + self.assertEqual(len(diff['changes']), 4) # changes for start,end,sequence,lastmodified-date + + diff = self._api_request('john.doe@example.org', 'event.diff', + uid=changelog['uid'], + mailbox='Calendar', + rev1=rev1, + rev2=rev2, + instance='20150330T210000' + ) + self.assertFalse(diff) + self.assertRPCError(-32603) # report error for invalid instance