diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php index 6295e237..3ae5d3ab 100644 --- a/src/app/DataMigrator/EWS/Appointment.php +++ b/src/app/DataMigrator/EWS/Appointment.php @@ -1,101 +1,101 @@ 'calendar:UID']; return $request; } /** * Process event object */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Initialize $this->itemId (for some unit tests) $this->getUID($item); // Decode MIME content $ical = base64_decode((string) $item->getMimeContent()); $itemId = implode("\r\n ", str_split($this->itemId, 75 - strlen('X-MS-ID:'))); $ical = preg_replace('/\r\nBEGIN:VEVENT\r\n/', "\r\nBEGIN:VEVENT\r\nX-MS-ID:{$itemId}\r\n", $ical, 1); // TODO: replace source email with destination email address in ORGANIZER/ATTENDEE // Inject attachment bodies into the iCalendar content // Calendar event attachments are exported as: // ATTACH:CID:81490FBA13A3DC2BF071B894C96B44BA51BEAAED@eurprd05.prod.outlook.com if ($item->getHasAttachments()) { // FIXME: I've tried hard and no matter what ContentId property is always empty // This means we can't match the CID from iCalendar with the attachment. // That's why we'll just remove all ATTACH:CID:... occurrences // and inject attachments to the main event $ical = preg_replace('/\r\nATTACH:CID:[^\r]+\r\n(\r\n [^\r\n]*)?/', '', $ical); foreach ((array) $item->getAttachments()->getFileAttachment() as $attachment) { $_attachment = $this->getAttachment($attachment); $ctype = $_attachment->getContentType(); $body = $_attachment->getContent(); // It looks like Exchange may have an issue with plain text files. // We'll skip empty files if (!strlen($body)) { continue; } // FIXME: This is imo inconsistence on php-ews side that MimeContent // is base64 encoded, but Content isn't // TODO: We should not do it in memory to not exceed the memory limit $body = base64_encode($body); $body = rtrim(chunk_split($body, 74, "\r\n "), ' '); // Inject the attachment at the end of the first VEVENT block // TODO: We should not do it in memory to not exceed the memory limit $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;FMTTYPE={$ctype}:\r\n {$body}"; $pos = strpos($ical, "\r\nEND:VEVENT"); $ical = substr_replace($ical, $append, $pos + 2, 0); } } return $ical; } /** * Get Item UID (Generate a new one if needed) */ protected function getUID(Type $item): string { // Only appointments have UID property $this->uid = $item->getUID(); // This also sets $this->itemId; return parent::getUID($item); } } diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php index 2cbe6d21..a003c1fd 100644 --- a/src/app/DataMigrator/EWS/Contact.php +++ b/src/app/DataMigrator/EWS/Contact.php @@ -1,86 +1,86 @@ getMimeContent()); // Remove empty properties that EWS is exporting $vcard = preg_replace('|\n[^:]+:;*\r|', '', $vcard); // Inject UID (and Exchange item ID) to the vCard $uid = $this->getUID($item); $itemId = implode("\r\n ", str_split($this->itemId, 75 - strlen('X-MS-ID:'))); // TODO: Use DAV\Vcard instead of string matching and replacement $vcard = str_replace("BEGIN:VCARD", "BEGIN:VCARD\r\nUID:{$uid}\r\nX-MS-ID:{$itemId}", $vcard); // Note: Looks like PHOTO property is exported properly, so we // don't have to handle attachments as we do for calendar items // TODO: Use vCard v4 for anniversary and spouse? Roundcube works with what's below // Spouse: X-MS-SPOUSE;TYPE=N:Partner Name if (preg_match('/(X-MS-SPOUSE[;:][^\r\n]+)/', $vcard, $matches)) { $spouse = preg_replace('/^[^:]+:/', '', $matches[1]); $vcard = str_replace($matches[1], "X-SPOUSE:{$spouse}", $vcard); } // Anniversary: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12 if (preg_match('/(X-MS-ANNIVERSARY[;:][^\r\n]+)/', $vcard, $matches)) { $date = preg_replace('/^[^:]+:/', '', $matches[1]); $vcard = str_replace($matches[1], "X-ANNIVERSARY:{$date}", $vcard); } // Exchange 2010 for some reason do not include email addresses in the vCard if (!preg_match('/\nEMAIL[^:]*:[^\r\n]+/', $vcard) && ($emailEntries = $item->getEmailAddresses())) { $emails = []; // Note that the Entry property is either an array (multiple addresses) // or an object (single address). Not a great API design. if (!is_array($emailEntries->Entry)) { $emailEntries->Entry = [$emailEntries->Entry]; } foreach ($emailEntries->Entry as $email) { $emails[] = 'EMAIL;TYPE=internet:' . strval($email); } if ($emails) { $vcard = str_replace("BEGIN:VCARD\r\n", "BEGIN:VCARD\r\n" . implode("\r\n", $emails) . "\r\n", $vcard); } } return $vcard; } } diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php index 83ef5ef4..ddeb4a6d 100644 --- a/src/app/DataMigrator/EWS/DistList.php +++ b/src/app/DataMigrator/EWS/DistList.php @@ -1,90 +1,90 @@ 'item:Body']; return $request; } /** * Convert distribution list object to vCard */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Groups (Distribution Lists) are not exported in vCard format, they use eml $data = [ 'UID' => [$this->getUID($item)], 'KIND' => ['group'], 'FN' => [$item->getDisplayName()], 'REV' => [$item->getLastModifiedTime(), ['VALUE' => 'DATE-TIME']], 'X-MS-ID' => [$this->itemId], ]; $vcard = "BEGIN:VCARD\r\nVERSION:4.0\r\nPRODID:Kolab EWS Data Migrator\r\n"; foreach ($data as $key => $prop) { $vcard .= $this->formatProp($key, $prop[0], isset($prop[1]) ? $prop[1] : []); } // Process list members if ($members = $item->getMembers()) { // The Member property is either array (multiple members) or Type\MemberType // object (a group with just a one member). if (!is_array($members->Member)) { $members->Member = [$members->Member]; } foreach ($members->Member as $member) { $mailbox = $member->getMailbox(); $mailto = $mailbox->getEmailAddress(); $name = $mailbox->getName(); $id = $mailbox->getItemId(); // "mailto:" members are not fully supported by Kolab Webclient. // For members that are contacts (have ItemId specified) we use urn:uuid: // syntax that has good support. if ($id) { $contactUID = sha1($id->toArray()['Id']); $vcard .= $this->formatProp('MEMBER', "urn:uuid:{$contactUID}"); } elseif ($mailto) { if ($name && $name != $mailto) { $mailto = urlencode(sprintf('"%s" <%s>', addcslashes($name, '"'), $mailto)); } $vcard .= $this->formatProp('MEMBER', "mailto:{$mailto}"); } } } // Note: Kolab Webclient does not make any use of the NOTE property for contact groups if ($body = (string) $item->getBody()) { $vcard .= $this->formatProp('NOTE', $body); } $vcard .= "END:VCARD\r\n"; return $vcard; } } diff --git a/src/app/DataMigrator/EWS/Email.php b/src/app/DataMigrator/EWS/Email.php index 621daf5b..af526b8f 100644 --- a/src/app/DataMigrator/EWS/Email.php +++ b/src/app/DataMigrator/EWS/Email.php @@ -1,59 +1,73 @@ 'calendar:UID']; return $request; } /** * Process event object */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Initialize $this->itemId (for some unit tests) $this->getUID($item); // Decode MIME content $mime = base64_decode((string) $item->getMimeContent()); + //FIXME that's IMAP specific, and thus breaking the abstraction + $flags = []; + if ($item->getIsRead()) { + $flags[] = 'SEEN'; + } + if ($internaldate = $item->getDateTimeReceived()) { + $internaldate = (new \DateTime($internaldate))->format('d-M-Y H:i:s O'); + } + + $targetItem->data = [ + 'flags' => $flags, + 'internaldate' => $internaldate, + ]; + return $mime; } /** * Get Item UID (Generate a new one if needed) */ protected function getUID(Type $item): string { // Only appointments have UID property $this->uid = $item->getUID(); // This also sets $this->itemId; return parent::getUID($item); } } diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php index 50c6dbd4..dd90bf55 100644 --- a/src/app/DataMigrator/EWS/Item.php +++ b/src/app/DataMigrator/EWS/Item.php @@ -1,193 +1,194 @@ driver = $driver; $this->folder = $folder; } /** * Factory method. * Returns object suitable to handle specified item type. */ public static function factory(EWS $driver, ItemInterface $item) { $item_class = str_replace('IPM.', '', $item->class); $item_class = "\App\DataMigrator\EWS\\{$item_class}"; if (class_exists($item_class)) { return new $item_class($driver, $item->folder); } } /** * Validate that specified EWS Item is of supported type */ public static function isValidItem(Type $item): bool { $item_class = str_replace('IPM.', '', $item->getItemClass()); $item_class = "\App\DataMigrator\EWS\\{$item_class}"; return class_exists($item_class); } /** * Process an item (fetch data and convert it) */ public function processItem(ItemInterface $item): void { $itemId = ['Id' => $item->id]; \Log::debug("[EWS] Fetching item {$item->id}..."); // Fetch the item $ewsItem = $this->driver->api->getItem($itemId, $this->getItemRequest()); $uid = $this->getUID($ewsItem); \Log::debug("[EWS] Saving item {$uid}..."); + // Apply type-specific format converters - $content = $this->convertItem($ewsItem); + $content = $this->convertItem($ewsItem, $item); if (!is_string($content)) { throw new \Exception("Failed to fetch EWS item {$this->itemId}"); } $filename = $uid . '.' . $this->fileExtension(); if (strlen($content) > Engine::MAX_ITEM_SIZE) { $location = $this->folder->tempFileLocation($filename); if (file_put_contents($location, $content) === false) { throw new \Exception("Failed to write to file at {$location}"); } $item->filename = $location; } else { $item->content = $content; $item->filename = $filename; } } /** * Item conversion code */ - abstract protected function convertItem(Type $item); + abstract protected function convertItem(Type $item, $targetItem); /** * Get GetItem request parameters */ protected static function getItemRequest(): array { $request = [ 'ItemShape' => [ // Reqest default set of properties 'BaseShape' => 'Default', // Additional properties, e.g. LastModifiedTime 'AdditionalProperties' => [ 'FieldURI' => [ ['FieldURI' => 'item:LastModifiedTime'], ], ], ] ]; return $request; } /** * Fetch attachment object from Exchange */ protected function getAttachment(Type\FileAttachmentType $attachment) { $request = [ 'AttachmentIds' => [ $attachment->getAttachmentId()->toXmlObject() ], 'AttachmentShape' => [ 'IncludeMimeContent' => true, ] ]; return $this->driver->api->getClient()->GetAttachment($request); } /** * Get Item UID (Generate a new one if needed) */ protected function getUID(Type $item): string { $itemId = $item->getItemId()->toArray(); if ($this->uid === null) { // Tasks, contacts, distlists do not have an UID. We have to generate one // and inject it into the output file. // We'll use the ItemId (excluding the ChangeKey part) as a base for the UID, // this way we can e.g. get distlist members references working. $this->uid = sha1($itemId['Id']); // $this->uid = \App\Utils::uuidStr(); } $this->itemId = implode('!', $itemId); return $this->uid; } /** * Filename extension for cached file in-processing */ protected function fileExtension(): string { return constant(static::class . '::FILE_EXT') ?: 'txt'; } /** * VCard/iCal property formatting */ protected function formatProp($name, $value, array $params = []): string { $cal = new \Sabre\VObject\Component\VCalendar(); $prop = new \Sabre\VObject\Property\Text($cal, $name, $value, $params); $value = $prop->serialize(); // Revert escaping for some props if ($name == 'RRULE') { $value = str_replace("\\", '', $value); } return $value; } } diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php index abb1607b..021260f9 100644 --- a/src/app/DataMigrator/EWS/Note.php +++ b/src/app/DataMigrator/EWS/Note.php @@ -1,41 +1,41 @@ getMimeContent()); return $email; } } diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php index e7f46e37..e62b0bc8 100644 --- a/src/app/DataMigrator/EWS/Task.php +++ b/src/app/DataMigrator/EWS/Task.php @@ -1,319 +1,319 @@ [$this->getUID($item)], 'DTSTAMP' => [$this->formatDate($item->getLastModifiedTime()), ['VALUE' => 'DATE-TIME']], 'CREATED' => [$this->formatDate($item->getDateTimeCreated()), ['VALUE' => 'DATE-TIME']], 'SEQUENCE' => [intval($item->getChangeCount())], 'SUMMARY' => [$item->getSubject()], 'DESCRIPTION' => [(string) $item->getBody()], 'PERCENT-COMPLETE' => [intval($item->getPercentComplete())], 'X-MS-ID' => [$this->itemId], ]; if ($dueDate = $item->getDueDate()) { $data['DUE'] = [$this->formatDate($dueDate), ['VALUE' => 'DATE-TIME']]; } if ($startDate = $item->getStartDate()) { $data['DTSTART'] = [$this->formatDate($startDate), ['VALUE' => 'DATE-TIME']]; } if ($status = $item->getStatus()) { $status = strtoupper($status); $status_map = [ 'COMPLETED' => 'COMPLETED', 'INPROGRESS' => 'IN-PROGRESS', 'DEFERRED' => 'X-DEFERRED', 'NOTSTARTED' => 'X-NOTSTARTED', 'WAITINGONOTHERS' => 'X-WAITINGFOROTHERS', ]; if (isset($status_map[$status])) { $data['STATUS'] = [$status_map[$status]]; } } if (($categories = $item->getCategories()) && $categories->String) { $data['CATEGORIES'] = [$categories->String]; } if ($sensitivity = $item->getSensitivity()) { $sensitivity_map = [ 'CONFIDENTIAL' => 'CONFIDENTIAL', 'NORMAL' => 'PUBLIC', 'PERSONAL' => 'PUBLIC', 'PRIVATE' => 'PRIVATE', ]; $data['CLASS'] = [$sensitivity_map[strtoupper($sensitivity)] ?? 'PUBLIC']; } if ($importance = $item->getImportance()) { $importance_map = [ 'HIGH' => '9', 'NORMAL' => '5', 'LOW' => '1', ]; $data['PRIORITY'] = [$importance_map[strtoupper($importance)] ?? '0']; } $this->setTaskOrganizer($data, $item); $this->setTaskRecurrence($data, $item); $ical = "BEGIN:VCALENDAR\r\nMETHOD:PUBLISH\r\nVERSION:2.0\r\nPRODID:Kolab EWS Data Migrator\r\nBEGIN:VTODO\r\n"; foreach ($data as $key => $prop) { $ical .= $this->formatProp($key, $prop[0], isset($prop[1]) ? $prop[1] : []); } // Attachments if ($item->getHasAttachments()) { foreach ((array) $item->getAttachments()->getFileAttachment() as $attachment) { $_attachment = $this->getAttachment($attachment); $ctype = $_attachment->getContentType(); $body = $_attachment->getContent(); // It looks like Exchange may have an issue with plain text files. // We'll skip empty files if (!strlen($body)) { continue; } // FIXME: This is imo inconsistence on php-ews side that MimeContent // is base64 encoded, but Content isn't // TODO: We should not do it in memory to not exceed the memory limit $body = base64_encode($body); $body = rtrim(chunk_split($body, 74, "\r\n "), ' '); // Inject the attachment at the end of the VTODO block // TODO: We should not do it in memory to not exceed the memory limit $ical .= "ATTACH;VALUE=BINARY;ENCODING=BASE64;FMTTYPE={$ctype}:\r\n {$body}"; } } $ical .= $this->getVAlarm($item); $ical .= "END:VTODO\r\n"; $ical .= "END:VCALENDAR\r\n"; return $ical; } /** * Set task organizer/attendee */ protected function setTaskOrganizer(array &$data, Type $task) { // FIXME: Looks like the owner might be an email address or just a full user name $owner = $task->getOwner(); $source = $this->driver->getSourceAccount(); $destination = $this->driver->getDestinationAccount(); if (strpos($owner, '@') && $owner != $source->email) { // Task owned by another person $data['ORGANIZER'] = ["mailto:{$owner}"]; // FIXME: Because attendees are not specified in EWS, assume the user is an attendee if ($destination->email) { $params = ['ROLE' => 'REQ-PARTICIPANT', 'CUTYPE' => 'INDIVIDUAL']; $data['ATTENDEE'] = ["mailto:{$destination->email}", $params]; } return; } // Otherwise it must be owned by the user if ($destination->email) { $data['ORGANIZER'] = ["mailto:{$destination->email}"]; } } /** * Set task recurrence rule */ protected function setTaskRecurrence(array &$data, Type $task) { if (empty($task->getIsRecurring()) || empty($task->getRecurrence())) { return; } $r = $task->getRecurrence(); $rrule = []; if ($recurrence = $r->getDailyRecurrence()) { $rrule['FREQ'] = 'DAILY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; } elseif ($recurrence = $r->getWeeklyRecurrence()) { $rrule['FREQ'] = 'WEEKLY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; $rrule['BYDAY'] = $this->mapDays($recurrence->getDaysOfWeek()); $rrule['WKST'] = $this->mapDays($recurrence->getFirstDayOfWeek()); } elseif ($recurrence = $r->getAbsoluteMonthlyRecurrence()) { $rrule['FREQ'] = 'MONTHLY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; $rrule['BYMONTHDAY'] = $recurrence->getDayOfMonth(); } elseif ($recurrence = $r->getRelativeMonthlyRecurrence()) { $rrule['FREQ'] = 'MONTHLY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; $rrule['BYDAY'] = $this->mapDays($recurrence->getDaysOfWeek(), $recurrence->getDayOfWeekIndex()); } elseif ($recurrence = $r->getAbsoluteYearlyRecurrence()) { $rrule['FREQ'] = 'YEARLY'; $rrule['BYMONTH'] = $this->mapMonths($recurrence->getMonth()); $rrule['BYMONTHDAY'] = $recurrence->getDayOfMonth(); } elseif ($recurrence = $r->getRelativeYearlyRecurrence()) { $rrule['FREQ'] = 'YEARLY'; $rrule['BYMONTH'] = $this->mapMonths($recurrence->getMonth()); $rrule['BYDAY'] = $this->mapDays($recurrence->getDaysOfWeek(), $recurrence->getDayOfWeekIndex()); } else { // There might be *Regeneration rules that we don't support \Log::debug("[EWS] Unsupported Recurrence property value. Ignored."); } if (!empty($rrule)) { if ($recurrence = $r->getNumberedRecurrence()) { $rrule['COUNT'] = $recurrence->getNumberOfOccurrences(); } elseif ($recurrence = $r->getEndDateRecurrence()) { $rrule['UNTIL'] = $this->formatDate($recurrence->getEndDate()); } $rrule = array_filter($rrule); $rrule = trim(array_reduce( array_keys($rrule), function ($carry, $key) use ($rrule) { return $carry . ';' . $key . '=' . $rrule[$key]; } ), ';'); $data['RRULE'] = [$rrule]; } } /** * Get VALARM block for the task Reminder */ protected function getVAlarm(Type $task): string { // FIXME: To me it looks like ReminderMinutesBeforeStart property is not used $date = $this->formatDate($task->getReminderDueBy()); if (empty($task->getReminderIsSet()) || empty($date)) { return ''; } return "BEGIN:VALARM\r\n" . "ACTION:DISPLAY\r\nTRIGGER;VALUE=DATE-TIME:{$date}\r\n" . "END:VALARM\r\n"; } /** * Convert EWS representation of recurrence days to iCal */ protected function mapDays(string $days, string $index = ''): string { if (preg_match('/(Day|Weekday|WeekendDay)/', $days)) { // not supported return ''; } $days_map = [ 'Sunday' => 'SU', 'Monday' => 'MO', 'Tuesday' => 'TU', 'Wednesday' => 'WE', 'Thursday' => 'TH', 'Friday' => 'FR', 'Saturday' => 'SA', ]; $index_map = [ 'First' => 1, 'Second' => 2, 'Third' => 3, 'Fourth' => 4, 'Last' => -1, ]; $days = explode(' ', $days); $days = array_map( function ($day) use ($days_map, $index_map, $index) { return ($index ? $index_map[$index] : '') . $days_map[$day]; }, $days ); return implode(',', $days); } /** * Convert EWS representation of recurrence month to iCal */ protected function mapMonths(string $months): string { $months_map = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; $months = explode(' ', $months); $months = array_map( function ($month) use ($months_map) { return array_search($month, $months_map) + 1; }, $months ); return implode(',', $months); } /** * Format EWS date-time into a iCalendar date-time */ protected function formatDate($datetime) { if (empty($datetime)) { return null; } return str_replace(['Z', '-', ':'], '', $datetime); } }