diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php index 36b24212..cbbea171 100644 --- a/src/app/DataMigrator/EWS.php +++ b/src/app/DataMigrator/EWS.php @@ -1,518 +1,540 @@ Engine::TYPE_EVENT, EWS\Contact::FOLDER_TYPE => Engine::TYPE_CONTACT, EWS\Task::FOLDER_TYPE => Engine::TYPE_TASK, + EWS\Email::FOLDER_TYPE => Engine::TYPE_MAIL, ]; /** @var Account Account to operate on */ protected $account; /** @var Engine Data migrator engine */ protected $engine; /** * Object constructor */ public function __construct(Account $account, Engine $engine) { $this->account = $account; $this->engine = $engine; } /** * Server autodiscovery */ public static function autodiscover(string $user, string $password): ?string { // You should never run the Autodiscover more than once. // It can make between 1 and 5 calls before giving up, or before finding your server, // depending on how many different attempts it needs to make. // TODO: Autodiscovery may fail with an exception thrown. Handle this nicely. // TODO: Looks like this autodiscovery also does not work w/Basic Auth? $api = API\ExchangeAutodiscover::getAPI($user, $password); $server = $api->getClient()->getServer(); $version = $api->getClient()->getVersion(); return sprintf('ews://%s:%s@%s', urlencode($user), urlencode($password), $server); } /** * Authenticate to EWS (initialize the EWS client) */ public function authenticate(): void { if (!empty($this->account->params['client_id'])) { $this->api = $this->authenticateWithOAuth2( $this->account->host, $this->account->username, $this->account->params['client_id'], $this->account->params['client_secret'], $this->account->params['tenant_id'] ); } else { // Note: This initializes the client, but not yet connects to the server // TODO: To know that the credentials work we'll have to do some API call. $this->api = $this->authenticateWithPassword( $this->account->host, $this->account->username, $this->account->password, $this->account->loginas ); } } /** * Autodiscover the server and authenticate the user */ protected function authenticateWithPassword(string $server, string $user, string $password, string $loginas = null) { // Note: Since 2023-01-01 EWS at Office365 requires OAuth2, no way back to basic auth. \Log::debug("[EWS] Using basic authentication on $server..."); $options = []; if ($loginas) { $options['impersonation'] = $loginas; } $this->engine->setOption('ews', [ 'options' => $options, 'server' => $server, ]); return API::withUsernameAndPassword($server, $user, $password, $this->apiOptions($options)); } /** * Authenticate with a token (Office365) */ protected function authenticateWithToken(string $server, string $user, string $token, $expires_at = null) { \Log::debug("[EWS] Using token authentication on $server..."); $options = ['impersonation' => $user]; $this->engine->setOption('ews', [ 'options' => $options, 'server' => $server, 'token' => $token, 'expires_at' => $expires_at, ]); return API::withCallbackToken($server, $token, $this->apiOptions($options)); } /** * Authenticate with OAuth2 (Office365) - get the token */ protected function authenticateWithOAuth2( string $server, string $user, string $client_id, string $client_secret, string $tenant_id ) { // See https://github.com/Garethp/php-ews/blob/master/examples/basic/authenticatingWithOAuth.php // See https://github.com/Garethp/php-ews/issues/236#issuecomment-1292521527 // To register OAuth2 app goto https://entra.microsoft.com > Applications > App registrations \Log::debug("[EWS] Fetching OAuth2 token from $server..."); $scope = 'https://outlook.office365.com/.default'; $token_uri = "https://login.microsoftonline.com/{$tenant_id}/oauth2/v2.0/token"; // $authUri = "https://login.microsoftonline.com/{$tenant_id}/oauth2/authorize"; $response = Http::asForm() ->timeout(5) ->post($token_uri, [ 'client_id' => $client_id, 'client_secret' => $client_secret, 'scope' => $scope, 'grant_type' => 'client_credentials', ]) ->throwUnlessStatus(200); $token = $response->json('access_token'); // Note: Office365 default token expiration time is ~1h, $expires_in = $response->json('expires_in'); $expires_at = now()->addSeconds($expires_in)->toDateTimeString(); return $this->authenticateWithToken($server, $user, $token, $expires_at); } /** * Get folders hierarchy */ public function getFolders($types = []): array { if (empty($types)) { $types = array_values($this->type_map); } // Create FolderClass filter $search = new Type\OrType(); foreach ($types as $type) { $type = array_search($type, $this->type_map); $search->addContains(Type\Contains::buildFromArray([ 'FieldURI' => [ Type\FieldURI::buildFromArray(['FieldURI' => 'folder:FolderClass']), ], 'Constant' => Type\ConstantValueType::buildFromArray([ 'Value' => $type, ]), 'ContainmentComparison' => 'Exact', 'ContainmentMode' => 'FullString', ])); } // Get full folders hierarchy (filtered by folder class) // Use of the filter reduces the response size by excluding system folders $options = [ 'Traversal' => 'Deep', 'Restriction' => ['Or' => $search], ]; $folders = $this->api->getChildrenFolders('root', $options); $result = []; foreach ($folders as $folder) { $class = $folder->getFolderClass(); $type = $this->type_map[$class] ?? null; // Skip folder types we do not support (need) if (empty($type) || (!empty($types) && !in_array($type, $types))) { continue; } // Note: Folder names are localized, even INBOX $name = $fullname = $folder->getDisplayName(); $id = $folder->getFolderId()->getId(); $parentId = $folder->getParentFolderId()->getId(); // Create folder name with full path if ($parentId && !empty($result[$parentId])) { $fullname = $result[$parentId]->fullname . '/' . $name; } // Top-level folder, check if it's a special folder we should ignore // FIXME: Is there a better way to distinguish user folders from system ones? if ( in_array($fullname, $this->folder_exceptions) || strpos($fullname, 'OwaFV15.1All') === 0 ) { continue; } $result[$id] = Folder::fromArray([ 'id' => $folder->getFolderId()->toArray(true), 'total' => $folder->getTotalCount(), 'class' => $class, 'type' => $this->type_map[$class] ?? null, 'name' => $name, 'fullname' => $fullname, ]); } return $result; } /** * Fetch a list of folder items */ public function fetchItemList(Folder $folder, $callback, Interface\ImporterInterface $importer): void { // Job processing - initialize environment $this->initEnv($this->engine->queue); // The folder is empty, we can stop here if (empty($folder->total)) { // TODO: Delete all existing items? return; } // Get items already imported // TODO: This might be slow and/or memory expensive, we should consider // whether storing list of imported items in some cache wouldn't be a better // solution. Of course, cache would not get changes in the destination account. $existing = $importer->getItems($folder); // Create X-MS-ID index for easier search in existing items // Note: For some objects we could use UID (events), but for some we don't have UID in Exchange. // Also because fetching extra properties here is problematic, we use X-MS-ID. $existingIndex = []; array_walk( $existing, function (&$item, $idx) use (&$existingIndex) { if (!empty($item['x-ms-id'])) { [$id, $changeKey] = explode('!', $item['x-ms-id']); $item['changeKey'] = $changeKey; $existingIndex[$id] = $idx; unset($item['x-ms-id']); + } else { + $existingIndex[$idx] = $idx; } } ); $request = [ // Exchange's maximum is 1000 'IndexedPageItemView' => ['MaxEntriesReturned' => 100, 'Offset' => 0, 'BasePoint' => 'Beginning'], 'ParentFolderIds' => $folder->id, 'Traversal' => 'Shallow', 'ItemShape' => [ 'BaseShape' => 'IdOnly', 'AdditionalProperties' => [ 'FieldURI' => [ ['FieldURI' => 'item:ItemClass'], // ['FieldURI' => 'item:Size'], + ['FieldURI' => 'message:InternetMessageId'], //For mail only? ], ], ], ]; $request = Type::buildFromArray($request); // Note: It is not possible to get mimeContent with FindItem request // That's why we first get the list of object identifiers and // then call GetItem on each separately. // TODO: It might be feasible to get all properties for object types // for which we don't use MimeContent, for better performance. // Request first page $response = $this->api->getClient()->FindItem($request); $set = new ItemSet(); // @phpstan-ignore-next-line foreach ($response->getItems() as $item) { if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { $set->items[] = $item; if (count($set->items) == self::CHUNK_SIZE) { $callback($set); $set = new ItemSet(); } } } // Request other pages until we got all while (!$response->isIncludesLastItemInRange()) { // @phpstan-ignore-next-line $response = $this->api->getNextPage($response); foreach ($response->getItems() as $item) { if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { $set->items[] = $item; if (count($set->items) == self::CHUNK_SIZE) { $callback($set); $set = new ItemSet(); } } } } if (count($set->items)) { $callback($set); } // TODO: Delete items that do not exist anymore? } /** * Fetching an item */ public function fetchItem(Item $item): void { // Job processing - initialize environment $this->initEnv($this->engine->queue); if ($driver = EWS\Item::factory($this, $item)) { $driver->processItem($item); return; } throw new \Exception("Failed to fetch an item from EWS"); } /** * Get the source account */ public function getSourceAccount(): Account { return $this->engine->source; } /** * Get the destination account */ public function getDestinationAccount(): Account { return $this->engine->destination; } /** * Synchronize specified object */ protected function toItem(Type $item, Folder $folder, $existing, $existingIndex): ?Item { $id = $item->getItemId()->toArray(); $exists = null; // Detect an existing item, skip if nothing changed if (isset($existingIndex[$id['Id']])) { $idx = $existingIndex[$id['Id']]; if ($existing[$idx]['changeKey'] == $id['ChangeKey']) { + \Log::debug("[EWS] Skipping over already existing message $idx..."); return null; } $exists = $existing[$idx]['href']; + } else { + $msgid = null; + try { + $msgid = $item->getInternetMessageId(); + } catch (\Exception $e) { + //Ignore + } + if (isset($existingIndex[$msgid])) { + // If the messageid already exists, we assume it's the same email. + // Flag/size changes are ignored for now. + // Otherwise we should set uid/size/flags on exists, so the IMAP implementation can pick it up. + \Log::debug("[EWS] Skipping over already existing message $msgid..."); + return null; + } } if (!EWS\Item::isValidItem($item)) { return null; } return Item::fromArray([ 'id' => $id['Id'], 'class' => $item->getItemClass(), 'folder' => $folder, 'existing' => $exists, ]); } /** * Set common API options */ protected function apiOptions(array $options): array { if (empty($options['version'])) { $options['version'] = API\ExchangeWebServices::VERSION_2013; } // In debug mode record all responses if (\config('app.debug')) { $options['httpPlayback'] = [ 'mode' => 'record', 'recordLocation' => \storage_path('ews'), ]; } // Options for testing foreach (['httpClient', 'httpPlayback'] as $opt) { if (($val = $this->engine->getOption($opt)) !== null) { $options[$opt] = $val; } } return $options; } /** * Initialize environment for job execution * * @param Queue $queue Queue */ protected function initEnv(Queue $queue): void { $ews = $queue->data['options']['ews']; if (!empty($ews['token'])) { // TODO: Refresh the token if needed $this->api = API::withCallbackToken( $ews['server'], $ews['token'], $this->apiOptions($ews['options']) ); } else { $this->api = API::withUsernameAndPassword( $ews['server'], $this->account->username, $this->account->password, $this->apiOptions($ews['options']) ); } } } diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php index 6295e237..3ae5d3ab 100644 --- a/src/app/DataMigrator/EWS/Appointment.php +++ b/src/app/DataMigrator/EWS/Appointment.php @@ -1,101 +1,101 @@ 'calendar:UID']; return $request; } /** * Process event object */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Initialize $this->itemId (for some unit tests) $this->getUID($item); // Decode MIME content $ical = base64_decode((string) $item->getMimeContent()); $itemId = implode("\r\n ", str_split($this->itemId, 75 - strlen('X-MS-ID:'))); $ical = preg_replace('/\r\nBEGIN:VEVENT\r\n/', "\r\nBEGIN:VEVENT\r\nX-MS-ID:{$itemId}\r\n", $ical, 1); // TODO: replace source email with destination email address in ORGANIZER/ATTENDEE // Inject attachment bodies into the iCalendar content // Calendar event attachments are exported as: // ATTACH:CID:81490FBA13A3DC2BF071B894C96B44BA51BEAAED@eurprd05.prod.outlook.com if ($item->getHasAttachments()) { // FIXME: I've tried hard and no matter what ContentId property is always empty // This means we can't match the CID from iCalendar with the attachment. // That's why we'll just remove all ATTACH:CID:... occurrences // and inject attachments to the main event $ical = preg_replace('/\r\nATTACH:CID:[^\r]+\r\n(\r\n [^\r\n]*)?/', '', $ical); foreach ((array) $item->getAttachments()->getFileAttachment() as $attachment) { $_attachment = $this->getAttachment($attachment); $ctype = $_attachment->getContentType(); $body = $_attachment->getContent(); // It looks like Exchange may have an issue with plain text files. // We'll skip empty files if (!strlen($body)) { continue; } // FIXME: This is imo inconsistence on php-ews side that MimeContent // is base64 encoded, but Content isn't // TODO: We should not do it in memory to not exceed the memory limit $body = base64_encode($body); $body = rtrim(chunk_split($body, 74, "\r\n "), ' '); // Inject the attachment at the end of the first VEVENT block // TODO: We should not do it in memory to not exceed the memory limit $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;FMTTYPE={$ctype}:\r\n {$body}"; $pos = strpos($ical, "\r\nEND:VEVENT"); $ical = substr_replace($ical, $append, $pos + 2, 0); } } return $ical; } /** * Get Item UID (Generate a new one if needed) */ protected function getUID(Type $item): string { // Only appointments have UID property $this->uid = $item->getUID(); // This also sets $this->itemId; return parent::getUID($item); } } diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php index 2cbe6d21..a003c1fd 100644 --- a/src/app/DataMigrator/EWS/Contact.php +++ b/src/app/DataMigrator/EWS/Contact.php @@ -1,86 +1,86 @@ getMimeContent()); // Remove empty properties that EWS is exporting $vcard = preg_replace('|\n[^:]+:;*\r|', '', $vcard); // Inject UID (and Exchange item ID) to the vCard $uid = $this->getUID($item); $itemId = implode("\r\n ", str_split($this->itemId, 75 - strlen('X-MS-ID:'))); // TODO: Use DAV\Vcard instead of string matching and replacement $vcard = str_replace("BEGIN:VCARD", "BEGIN:VCARD\r\nUID:{$uid}\r\nX-MS-ID:{$itemId}", $vcard); // Note: Looks like PHOTO property is exported properly, so we // don't have to handle attachments as we do for calendar items // TODO: Use vCard v4 for anniversary and spouse? Roundcube works with what's below // Spouse: X-MS-SPOUSE;TYPE=N:Partner Name if (preg_match('/(X-MS-SPOUSE[;:][^\r\n]+)/', $vcard, $matches)) { $spouse = preg_replace('/^[^:]+:/', '', $matches[1]); $vcard = str_replace($matches[1], "X-SPOUSE:{$spouse}", $vcard); } // Anniversary: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12 if (preg_match('/(X-MS-ANNIVERSARY[;:][^\r\n]+)/', $vcard, $matches)) { $date = preg_replace('/^[^:]+:/', '', $matches[1]); $vcard = str_replace($matches[1], "X-ANNIVERSARY:{$date}", $vcard); } // Exchange 2010 for some reason do not include email addresses in the vCard if (!preg_match('/\nEMAIL[^:]*:[^\r\n]+/', $vcard) && ($emailEntries = $item->getEmailAddresses())) { $emails = []; // Note that the Entry property is either an array (multiple addresses) // or an object (single address). Not a great API design. if (!is_array($emailEntries->Entry)) { $emailEntries->Entry = [$emailEntries->Entry]; } foreach ($emailEntries->Entry as $email) { $emails[] = 'EMAIL;TYPE=internet:' . strval($email); } if ($emails) { $vcard = str_replace("BEGIN:VCARD\r\n", "BEGIN:VCARD\r\n" . implode("\r\n", $emails) . "\r\n", $vcard); } } return $vcard; } } diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php index 83ef5ef4..ddeb4a6d 100644 --- a/src/app/DataMigrator/EWS/DistList.php +++ b/src/app/DataMigrator/EWS/DistList.php @@ -1,90 +1,90 @@ 'item:Body']; return $request; } /** * Convert distribution list object to vCard */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Groups (Distribution Lists) are not exported in vCard format, they use eml $data = [ 'UID' => [$this->getUID($item)], 'KIND' => ['group'], 'FN' => [$item->getDisplayName()], 'REV' => [$item->getLastModifiedTime(), ['VALUE' => 'DATE-TIME']], 'X-MS-ID' => [$this->itemId], ]; $vcard = "BEGIN:VCARD\r\nVERSION:4.0\r\nPRODID:Kolab EWS Data Migrator\r\n"; foreach ($data as $key => $prop) { $vcard .= $this->formatProp($key, $prop[0], isset($prop[1]) ? $prop[1] : []); } // Process list members if ($members = $item->getMembers()) { // The Member property is either array (multiple members) or Type\MemberType // object (a group with just a one member). if (!is_array($members->Member)) { $members->Member = [$members->Member]; } foreach ($members->Member as $member) { $mailbox = $member->getMailbox(); $mailto = $mailbox->getEmailAddress(); $name = $mailbox->getName(); $id = $mailbox->getItemId(); // "mailto:" members are not fully supported by Kolab Webclient. // For members that are contacts (have ItemId specified) we use urn:uuid: // syntax that has good support. if ($id) { $contactUID = sha1($id->toArray()['Id']); $vcard .= $this->formatProp('MEMBER', "urn:uuid:{$contactUID}"); } elseif ($mailto) { if ($name && $name != $mailto) { $mailto = urlencode(sprintf('"%s" <%s>', addcslashes($name, '"'), $mailto)); } $vcard .= $this->formatProp('MEMBER', "mailto:{$mailto}"); } } } // Note: Kolab Webclient does not make any use of the NOTE property for contact groups if ($body = (string) $item->getBody()) { $vcard .= $this->formatProp('NOTE', $body); } $vcard .= "END:VCARD\r\n"; return $vcard; } } diff --git a/src/app/DataMigrator/EWS/Email.php b/src/app/DataMigrator/EWS/Email.php new file mode 100644 index 00000000..af526b8f --- /dev/null +++ b/src/app/DataMigrator/EWS/Email.php @@ -0,0 +1,73 @@ + 'calendar:UID']; + + return $request; + } + + /** + * Process event object + */ + protected function convertItem(Type $item, $targetItem) + { + // Initialize $this->itemId (for some unit tests) + $this->getUID($item); + + // Decode MIME content + $mime = base64_decode((string) $item->getMimeContent()); + + //FIXME that's IMAP specific, and thus breaking the abstraction + $flags = []; + if ($item->getIsRead()) { + $flags[] = 'SEEN'; + } + if ($internaldate = $item->getDateTimeReceived()) { + $internaldate = (new \DateTime($internaldate))->format('d-M-Y H:i:s O'); + } + + $targetItem->data = [ + 'flags' => $flags, + 'internaldate' => $internaldate, + ]; + + return $mime; + } + + /** + * Get Item UID (Generate a new one if needed) + */ + protected function getUID(Type $item): string + { + // Only appointments have UID property + $this->uid = $item->getUID(); + + // This also sets $this->itemId; + return parent::getUID($item); + } +} diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php index 50c6dbd4..dd90bf55 100644 --- a/src/app/DataMigrator/EWS/Item.php +++ b/src/app/DataMigrator/EWS/Item.php @@ -1,193 +1,194 @@ driver = $driver; $this->folder = $folder; } /** * Factory method. * Returns object suitable to handle specified item type. */ public static function factory(EWS $driver, ItemInterface $item) { $item_class = str_replace('IPM.', '', $item->class); $item_class = "\App\DataMigrator\EWS\\{$item_class}"; if (class_exists($item_class)) { return new $item_class($driver, $item->folder); } } /** * Validate that specified EWS Item is of supported type */ public static function isValidItem(Type $item): bool { $item_class = str_replace('IPM.', '', $item->getItemClass()); $item_class = "\App\DataMigrator\EWS\\{$item_class}"; return class_exists($item_class); } /** * Process an item (fetch data and convert it) */ public function processItem(ItemInterface $item): void { $itemId = ['Id' => $item->id]; \Log::debug("[EWS] Fetching item {$item->id}..."); // Fetch the item $ewsItem = $this->driver->api->getItem($itemId, $this->getItemRequest()); $uid = $this->getUID($ewsItem); \Log::debug("[EWS] Saving item {$uid}..."); + // Apply type-specific format converters - $content = $this->convertItem($ewsItem); + $content = $this->convertItem($ewsItem, $item); if (!is_string($content)) { throw new \Exception("Failed to fetch EWS item {$this->itemId}"); } $filename = $uid . '.' . $this->fileExtension(); if (strlen($content) > Engine::MAX_ITEM_SIZE) { $location = $this->folder->tempFileLocation($filename); if (file_put_contents($location, $content) === false) { throw new \Exception("Failed to write to file at {$location}"); } $item->filename = $location; } else { $item->content = $content; $item->filename = $filename; } } /** * Item conversion code */ - abstract protected function convertItem(Type $item); + abstract protected function convertItem(Type $item, $targetItem); /** * Get GetItem request parameters */ protected static function getItemRequest(): array { $request = [ 'ItemShape' => [ // Reqest default set of properties 'BaseShape' => 'Default', // Additional properties, e.g. LastModifiedTime 'AdditionalProperties' => [ 'FieldURI' => [ ['FieldURI' => 'item:LastModifiedTime'], ], ], ] ]; return $request; } /** * Fetch attachment object from Exchange */ protected function getAttachment(Type\FileAttachmentType $attachment) { $request = [ 'AttachmentIds' => [ $attachment->getAttachmentId()->toXmlObject() ], 'AttachmentShape' => [ 'IncludeMimeContent' => true, ] ]; return $this->driver->api->getClient()->GetAttachment($request); } /** * Get Item UID (Generate a new one if needed) */ protected function getUID(Type $item): string { $itemId = $item->getItemId()->toArray(); if ($this->uid === null) { // Tasks, contacts, distlists do not have an UID. We have to generate one // and inject it into the output file. // We'll use the ItemId (excluding the ChangeKey part) as a base for the UID, // this way we can e.g. get distlist members references working. $this->uid = sha1($itemId['Id']); // $this->uid = \App\Utils::uuidStr(); } $this->itemId = implode('!', $itemId); return $this->uid; } /** * Filename extension for cached file in-processing */ protected function fileExtension(): string { return constant(static::class . '::FILE_EXT') ?: 'txt'; } /** * VCard/iCal property formatting */ protected function formatProp($name, $value, array $params = []): string { $cal = new \Sabre\VObject\Component\VCalendar(); $prop = new \Sabre\VObject\Property\Text($cal, $name, $value, $params); $value = $prop->serialize(); // Revert escaping for some props if ($name == 'RRULE') { $value = str_replace("\\", '', $value); } return $value; } } diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php index abb1607b..dc839fcc 100644 --- a/src/app/DataMigrator/EWS/Note.php +++ b/src/app/DataMigrator/EWS/Note.php @@ -1,41 +1,56 @@ getMimeContent()); + $flags = []; + if ($item->getIsRead()) { + $flags[] = 'SEEN'; + } + + $internaldate = null; + if ($internaldate = $item->getDateTimeReceived()) { + $internaldate = (new \DateTime($internaldate))->format('d-M-Y H:i:s O'); + } + + $targetItem->data = [ + 'flags' => $flags, + 'internaldate' => $internaldate, + ]; + return $email; } } diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php index e7f46e37..e62b0bc8 100644 --- a/src/app/DataMigrator/EWS/Task.php +++ b/src/app/DataMigrator/EWS/Task.php @@ -1,319 +1,319 @@ [$this->getUID($item)], 'DTSTAMP' => [$this->formatDate($item->getLastModifiedTime()), ['VALUE' => 'DATE-TIME']], 'CREATED' => [$this->formatDate($item->getDateTimeCreated()), ['VALUE' => 'DATE-TIME']], 'SEQUENCE' => [intval($item->getChangeCount())], 'SUMMARY' => [$item->getSubject()], 'DESCRIPTION' => [(string) $item->getBody()], 'PERCENT-COMPLETE' => [intval($item->getPercentComplete())], 'X-MS-ID' => [$this->itemId], ]; if ($dueDate = $item->getDueDate()) { $data['DUE'] = [$this->formatDate($dueDate), ['VALUE' => 'DATE-TIME']]; } if ($startDate = $item->getStartDate()) { $data['DTSTART'] = [$this->formatDate($startDate), ['VALUE' => 'DATE-TIME']]; } if ($status = $item->getStatus()) { $status = strtoupper($status); $status_map = [ 'COMPLETED' => 'COMPLETED', 'INPROGRESS' => 'IN-PROGRESS', 'DEFERRED' => 'X-DEFERRED', 'NOTSTARTED' => 'X-NOTSTARTED', 'WAITINGONOTHERS' => 'X-WAITINGFOROTHERS', ]; if (isset($status_map[$status])) { $data['STATUS'] = [$status_map[$status]]; } } if (($categories = $item->getCategories()) && $categories->String) { $data['CATEGORIES'] = [$categories->String]; } if ($sensitivity = $item->getSensitivity()) { $sensitivity_map = [ 'CONFIDENTIAL' => 'CONFIDENTIAL', 'NORMAL' => 'PUBLIC', 'PERSONAL' => 'PUBLIC', 'PRIVATE' => 'PRIVATE', ]; $data['CLASS'] = [$sensitivity_map[strtoupper($sensitivity)] ?? 'PUBLIC']; } if ($importance = $item->getImportance()) { $importance_map = [ 'HIGH' => '9', 'NORMAL' => '5', 'LOW' => '1', ]; $data['PRIORITY'] = [$importance_map[strtoupper($importance)] ?? '0']; } $this->setTaskOrganizer($data, $item); $this->setTaskRecurrence($data, $item); $ical = "BEGIN:VCALENDAR\r\nMETHOD:PUBLISH\r\nVERSION:2.0\r\nPRODID:Kolab EWS Data Migrator\r\nBEGIN:VTODO\r\n"; foreach ($data as $key => $prop) { $ical .= $this->formatProp($key, $prop[0], isset($prop[1]) ? $prop[1] : []); } // Attachments if ($item->getHasAttachments()) { foreach ((array) $item->getAttachments()->getFileAttachment() as $attachment) { $_attachment = $this->getAttachment($attachment); $ctype = $_attachment->getContentType(); $body = $_attachment->getContent(); // It looks like Exchange may have an issue with plain text files. // We'll skip empty files if (!strlen($body)) { continue; } // FIXME: This is imo inconsistence on php-ews side that MimeContent // is base64 encoded, but Content isn't // TODO: We should not do it in memory to not exceed the memory limit $body = base64_encode($body); $body = rtrim(chunk_split($body, 74, "\r\n "), ' '); // Inject the attachment at the end of the VTODO block // TODO: We should not do it in memory to not exceed the memory limit $ical .= "ATTACH;VALUE=BINARY;ENCODING=BASE64;FMTTYPE={$ctype}:\r\n {$body}"; } } $ical .= $this->getVAlarm($item); $ical .= "END:VTODO\r\n"; $ical .= "END:VCALENDAR\r\n"; return $ical; } /** * Set task organizer/attendee */ protected function setTaskOrganizer(array &$data, Type $task) { // FIXME: Looks like the owner might be an email address or just a full user name $owner = $task->getOwner(); $source = $this->driver->getSourceAccount(); $destination = $this->driver->getDestinationAccount(); if (strpos($owner, '@') && $owner != $source->email) { // Task owned by another person $data['ORGANIZER'] = ["mailto:{$owner}"]; // FIXME: Because attendees are not specified in EWS, assume the user is an attendee if ($destination->email) { $params = ['ROLE' => 'REQ-PARTICIPANT', 'CUTYPE' => 'INDIVIDUAL']; $data['ATTENDEE'] = ["mailto:{$destination->email}", $params]; } return; } // Otherwise it must be owned by the user if ($destination->email) { $data['ORGANIZER'] = ["mailto:{$destination->email}"]; } } /** * Set task recurrence rule */ protected function setTaskRecurrence(array &$data, Type $task) { if (empty($task->getIsRecurring()) || empty($task->getRecurrence())) { return; } $r = $task->getRecurrence(); $rrule = []; if ($recurrence = $r->getDailyRecurrence()) { $rrule['FREQ'] = 'DAILY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; } elseif ($recurrence = $r->getWeeklyRecurrence()) { $rrule['FREQ'] = 'WEEKLY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; $rrule['BYDAY'] = $this->mapDays($recurrence->getDaysOfWeek()); $rrule['WKST'] = $this->mapDays($recurrence->getFirstDayOfWeek()); } elseif ($recurrence = $r->getAbsoluteMonthlyRecurrence()) { $rrule['FREQ'] = 'MONTHLY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; $rrule['BYMONTHDAY'] = $recurrence->getDayOfMonth(); } elseif ($recurrence = $r->getRelativeMonthlyRecurrence()) { $rrule['FREQ'] = 'MONTHLY'; $rrule['INTERVAL'] = $recurrence->getInterval() ?: 1; $rrule['BYDAY'] = $this->mapDays($recurrence->getDaysOfWeek(), $recurrence->getDayOfWeekIndex()); } elseif ($recurrence = $r->getAbsoluteYearlyRecurrence()) { $rrule['FREQ'] = 'YEARLY'; $rrule['BYMONTH'] = $this->mapMonths($recurrence->getMonth()); $rrule['BYMONTHDAY'] = $recurrence->getDayOfMonth(); } elseif ($recurrence = $r->getRelativeYearlyRecurrence()) { $rrule['FREQ'] = 'YEARLY'; $rrule['BYMONTH'] = $this->mapMonths($recurrence->getMonth()); $rrule['BYDAY'] = $this->mapDays($recurrence->getDaysOfWeek(), $recurrence->getDayOfWeekIndex()); } else { // There might be *Regeneration rules that we don't support \Log::debug("[EWS] Unsupported Recurrence property value. Ignored."); } if (!empty($rrule)) { if ($recurrence = $r->getNumberedRecurrence()) { $rrule['COUNT'] = $recurrence->getNumberOfOccurrences(); } elseif ($recurrence = $r->getEndDateRecurrence()) { $rrule['UNTIL'] = $this->formatDate($recurrence->getEndDate()); } $rrule = array_filter($rrule); $rrule = trim(array_reduce( array_keys($rrule), function ($carry, $key) use ($rrule) { return $carry . ';' . $key . '=' . $rrule[$key]; } ), ';'); $data['RRULE'] = [$rrule]; } } /** * Get VALARM block for the task Reminder */ protected function getVAlarm(Type $task): string { // FIXME: To me it looks like ReminderMinutesBeforeStart property is not used $date = $this->formatDate($task->getReminderDueBy()); if (empty($task->getReminderIsSet()) || empty($date)) { return ''; } return "BEGIN:VALARM\r\n" . "ACTION:DISPLAY\r\nTRIGGER;VALUE=DATE-TIME:{$date}\r\n" . "END:VALARM\r\n"; } /** * Convert EWS representation of recurrence days to iCal */ protected function mapDays(string $days, string $index = ''): string { if (preg_match('/(Day|Weekday|WeekendDay)/', $days)) { // not supported return ''; } $days_map = [ 'Sunday' => 'SU', 'Monday' => 'MO', 'Tuesday' => 'TU', 'Wednesday' => 'WE', 'Thursday' => 'TH', 'Friday' => 'FR', 'Saturday' => 'SA', ]; $index_map = [ 'First' => 1, 'Second' => 2, 'Third' => 3, 'Fourth' => 4, 'Last' => -1, ]; $days = explode(' ', $days); $days = array_map( function ($day) use ($days_map, $index_map, $index) { return ($index ? $index_map[$index] : '') . $days_map[$day]; }, $days ); return implode(',', $days); } /** * Convert EWS representation of recurrence month to iCal */ protected function mapMonths(string $months): string { $months_map = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; $months = explode(' ', $months); $months = array_map( function ($month) use ($months_map) { return array_search($month, $months_map) + 1; }, $months ); return implode(',', $months); } /** * Format EWS date-time into a iCalendar date-time */ protected function formatDate($datetime) { if (empty($datetime)) { return null; } return str_replace(['Z', '-', ':'], '', $datetime); } } diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php index 4405e47a..bc29b022 100644 --- a/src/app/DataMigrator/Engine.php +++ b/src/app/DataMigrator/Engine.php @@ -1,368 +1,372 @@ source = $source; $this->destination = $destination; $this->options = $options; // Create a unique identifier for the migration request $queue_id = md5(strval($source) . strval($destination) . ($options['type'] ?? '')); // TODO: When running in 'sync' mode we shouldn't create a queue at all // If queue exists, we'll display the progress only if ($queue = Queue::find($queue_id)) { // If queue contains no jobs, assume invalid // TODO: An better API to manage (reset) queues if (!$queue->jobs_started || !empty($options['force'])) { $queue->delete(); } else { while (true) { $this->debug(sprintf("Progress [%d of %d]\n", $queue->jobs_finished, $queue->jobs_started)); if ($queue->jobs_started == $queue->jobs_finished) { break; } sleep(1); $queue->refresh(); } return; } } // Initialize the source $this->exporter = $this->initDriver($source, ExporterInterface::class); $this->exporter->authenticate(); // Initialize the destination $this->importer = $this->initDriver($destination, ImporterInterface::class); $this->importer->authenticate(); // Create a queue $this->createQueue($queue_id); // We'll store temp files in storage/ tree $location = storage_path('export/') . $source->email; if (!file_exists($location)) { mkdir($location, 0740, true); } $types = empty($options['type']) ? [] : preg_split('/\s*,\s*/', strtolower($options['type'])); $this->debug("Fetching folders hierarchy..."); $folders = $this->exporter->getFolders($types); $count = 0; $async = empty($options['sync']); $folderMapping = $this->options['folderMapping']; foreach ($folders as $folder) { $this->debug("Processing folder {$folder->fullname}..."); $folder->queueId = $queue_id; $folder->location = $location; - if (array_key_exists($folder->fullname, $folderMapping)) { - $folder->targetname = $folderMapping[$folder->fullname]; - } else { - $folder->targetname = $folder->fullname; + // Apply name replacements + $folder->targetname = $folder->fullname; + foreach ($folderMapping as $key => $value) { + if (str_contains($folder->targetname, $key)) { + $folder->targetname = str_replace($key, $value, $folder->targetname); + $this->debug("Replacing {$folder->fullname} with {$folder->targetname}"); + break; + } } if ($async) { // Dispatch the job (for async execution) Jobs\FolderJob::dispatch($folder); $count++; } else { $this->processFolder($folder); } } if ($count) { $this->queue->bumpJobsStarted($count); } if ($async) { $this->debug(sprintf('Done. %d %s created in queue: %s.', $count, Str::plural('job', $count), $queue_id)); } else { $this->debug(sprintf('Done (queue: %s).', $queue_id)); } } /** * Processing of a folder synchronization */ public function processFolder(Folder $folder): void { // Job processing - initialize environment if (!$this->queue) { $this->envFromQueue($folder->queueId); } // Create the folder on the destination server $this->importer->createFolder($folder); $count = 0; $async = empty($this->options['sync']); // Fetch items from the source $this->exporter->fetchItemList( $folder, function ($item_or_set) use (&$count, $async) { if ($async) { // Dispatch the job (for async execution) if ($item_or_set instanceof ItemSet) { Jobs\ItemSetJob::dispatch($item_or_set); } else { Jobs\ItemJob::dispatch($item_or_set); } $count++; } else { if ($item_or_set instanceof ItemSet) { $this->processItemSet($item_or_set); } else { $this->processItem($item_or_set); } } }, $this->importer ); if ($count) { $this->queue->bumpJobsStarted($count); } if ($async) { $this->queue->bumpJobsFinished(); } } /** * Processing of item synchronization */ public function processItem(Item $item): void { // Job processing - initialize environment if (!$this->queue) { $this->envFromQueue($item->folder->queueId); } $this->exporter->fetchItem($item); $this->importer->createItem($item); if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) { @unlink($item->filename); } if (empty($this->options['sync'])) { $this->queue->bumpJobsFinished(); } } /** * Processing of item-set synchronization */ public function processItemSet(ItemSet $set): void { // Job processing - initialize environment if (!$this->queue) { $this->envFromQueue($set->items[0]->folder->queueId); } $importItem = function (Item $item) { $this->importer->createItem($item); if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) { @unlink($item->filename); } }; // Some exporters, e.g. DAV, might optimize fetching multiple items in one go if ($this->exporter instanceof FetchItemSetInterface) { $this->exporter->fetchItemSet($set, $importItem); } else { foreach ($set->items as $item) { $this->exporter->fetchItem($item); $importItem($item); } } // TODO: We should probably also track number of items migrated if (empty($this->options['sync'])) { $this->queue->bumpJobsFinished(); } } /** * Print progress/debug information */ public function debug($line) { if (!empty($this->options['stdout'])) { $output = new \Symfony\Component\Console\Output\ConsoleOutput(); $output->writeln("$line"); } else { \Log::debug("[DataMigrator] $line"); } } /** * Get migration option value. */ public function getOption(string $name) { return $this->options[$name] ?? null; } /** * Set migration queue option. Use this if you need to pass * some data between queue processes. */ public function setOption(string $name, $value): void { $this->options[$name] = $value; if ($this->queue) { $this->queue->data = $this->queueData(); $this->queue->save(); } } /** * Create a queue for the request * * @param string $queue_id Unique queue identifier */ protected function createQueue(string $queue_id): void { $this->queue = new Queue(); $this->queue->id = $queue_id; $this->queue->data = $this->queueData(); $this->queue->save(); } /** * Prepare queue data */ protected function queueData() { $options = $this->options; unset($options['stdout']); // jobs aren't in stdout anymore // TODO: data should be encrypted return [ 'source' => (string) $this->source, 'destination' => (string) $this->destination, 'options' => $options, ]; } /** * Initialize environment for job execution * * @param string $queueId Queue identifier */ protected function envFromQueue(string $queueId): void { $this->queue = Queue::findOrFail($queueId); $this->source = new Account($this->queue->data['source']); $this->destination = new Account($this->queue->data['destination']); $this->options = $this->queue->data['options']; $this->importer = $this->initDriver($this->destination, ImporterInterface::class); $this->exporter = $this->initDriver($this->source, ExporterInterface::class); } /** * Initialize (and select) migration driver */ protected function initDriver(Account $account, string $interface) { switch ($account->scheme) { case 'ews': $driver = new EWS($account, $this); break; case 'dav': case 'davs': $driver = new DAV($account, $this); break; case 'imap': case 'imaps': case 'tls': case 'ssl': $driver = new IMAP($account, $this); break; case 'test': $driver = new Test($account, $this); break; default: throw new \Exception("Failed to init driver for '{$account->scheme}'"); } // Make sure driver is used in the direction it supports if (!is_a($driver, $interface)) { throw new \Exception(sprintf( "'%s' driver does not implement %s", class_basename($driver), class_basename($interface) )); } return $driver; } } diff --git a/src/app/DataMigrator/IMAP.php b/src/app/DataMigrator/IMAP.php index 9fb580be..18fac312 100644 --- a/src/app/DataMigrator/IMAP.php +++ b/src/app/DataMigrator/IMAP.php @@ -1,464 +1,476 @@ account = $account; $this->engine = $engine; // TODO: Move this to self::authenticate()? $config = self::getConfig($account); $this->imap = self::initIMAP($config); } /** * Object destructor */ public function __destruct() { try { $this->imap->closeConnection(); } catch (\Throwable $e) { // Ignore. It may throw when destructing the object in tests // We also don't really care abount an error on this operation } } /** * Authenticate */ public function authenticate(): void { } /** * Create a folder. * * @param Folder $folder Folder data * * @throws \Exception on error */ public function createFolder(Folder $folder): void { if ($folder->type != 'mail') { throw new \Exception("IMAP does not support folder of type {$folder->type}"); } if ($folder->targetname == 'INBOX') { // INBOX always exists return; } - if (!$this->imap->createFolder($folder->targetname)) { + if (!$this->imap->createFolder(self::toUTF7($folder->targetname))) { \Log::warning("Failed to create the folder: {$this->imap->error}"); if (str_contains($this->imap->error, "Mailbox already exists")) { // Not an error } else { throw new \Exception("Failed to create an IMAP folder {$folder->targetname}"); } } // TODO: Migrate folder subscription state } + /** + * Convert UTF8 string to UTF7-IMAP encoding + */ + private static function toUTF7(string $string): string + { + return \mb_convert_encoding($string, 'UTF7-IMAP', 'UTF8'); + } + /** * Create an item in a folder. * * @param Item $item Item to import * * @throws \Exception */ public function createItem(Item $item): void { - $mailbox = $item->folder->targetname; + $mailbox = self::toUTF7($item->folder->targetname); if (strlen($item->content)) { $result = $this->imap->append( $mailbox, $item->content, - $item->data['flags'], - $item->data['internaldate'], + $item->data['flags'] ?? [], + $item->data['internaldate'] ?? null, true ); if ($result === false) { throw new \Exception("Failed to append IMAP message into {$mailbox}"); } } elseif ($item->filename) { $result = $this->imap->appendFromFile( $mailbox, $item->filename, null, - $item->data['flags'], - $item->data['internaldate'], + $item->data['flags'] ?? [], + $item->data['internaldate'] ?? null, true ); if ($result === false) { throw new \Exception("Failed to append IMAP message into {$mailbox}"); } } // When updating an existing email message we have to... if ($item->existing) { if (!empty($result)) { // Remove the old one $this->imap->flag($mailbox, $item->existing['uid'], 'DELETED'); $this->imap->expunge($mailbox, $item->existing['uid']); } else { // Update flags foreach ($item->existing['flags'] as $flag) { if (!in_array($flag, $item->data['flags'])) { $this->imap->unflag($mailbox, $item->existing['uid'], $flag); } } foreach ($item->data['flags'] as $flag) { if (!in_array($flag, $item->existing['flags'])) { $this->imap->flag($mailbox, $item->existing['uid'], $flag); } } } } } /** * Fetching an item */ public function fetchItem(Item $item): void { [$uid, $messageId] = explode(':', $item->id, 2); $mailbox = $item->folder->fullname; // Get message flags $header = $this->imap->fetchHeader($mailbox, (int) $uid, true, false, ['FLAGS']); if ($header === false) { throw new \Exception("Failed to get IMAP message headers for {$mailbox}/{$uid}"); } // Remove flags that we can't append (e.g. RECENT) $flags = $this->filterImapFlags(array_keys($header->flags)); // If message already exists in the destination account we should update only flags // and be done with it. On the other hand for Drafts it's not unusual to get completely // different body for the same Message-ID. Same can happen not only in Drafts, I suppose. // So, we compare size and INTERNALDATE timestamp. if ( !$item->existing || $header->timestamp != $item->existing['timestamp'] || $header->size != $item->existing['size'] ) { // Handle message content in memory (up to 20MB), bigger messages will use a temp file if ($header->size > Engine::MAX_ITEM_SIZE) { // Save the message content to a file $location = $item->folder->tempFileLocation($uid . '.eml'); $fp = fopen($location, 'w'); if (!$fp) { throw new \Exception("Failed to open 'php://temp' stream"); } $result = $this->imap->handlePartBody($mailbox, $uid, true, '', null, null, $fp); } else { $result = $this->imap->handlePartBody($mailbox, $uid, true); } if ($result === false) { if (!empty($fp)) { fclose($fp); } throw new \Exception("Failed to fetch IMAP message for {$mailbox}/{$uid}"); } if (!empty($fp) && !empty($location)) { $item->filename = $location; fclose($fp); } else { $item->content = $result; } } $item->data = [ 'flags' => $flags, 'internaldate' => $header->internaldate, ]; } /** * Fetch a list of folder items */ public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void { // Get existing messages' headers from the destination mailbox $existing = $importer->getItems($folder); $mailbox = $folder->fullname; // TODO: We should probably first use SEARCH/SORT to skip messages marked as \Deleted // It would also allow us to get headers in chunks 200 messages at a time, or so. // TODO: fetchHeaders() fetches too many headers, we should slim-down, here we need // only UID FLAGS INTERNALDATE BODY.PEEK[HEADER.FIELDS (DATE FROM MESSAGE-ID)] $messages = $this->imap->fetchHeaders($mailbox, '1:*', true, false, ['Message-Id']); if ($messages === false) { throw new \Exception("Failed to get all IMAP message headers for {$mailbox}"); } if (empty($messages)) { \Log::debug("Nothing to migrate for {$mailbox}"); return; } $set = new ItemSet(); foreach ($messages as $message) { // If Message-Id header does not exist create it based on internaldate/From/Date $id = $this->getMessageId($message, $mailbox); // Skip message that exists and did not change $exists = null; if (isset($existing[$id])) { $flags = $this->filterImapFlags(array_keys($message->flags)); if ( $flags == $existing[$id]['flags'] && $message->timestamp == $existing[$id]['timestamp'] && $message->size == $existing[$id]['size'] ) { continue; } $exists = $existing[$id]; } $set->items[] = Item::fromArray([ 'id' => $message->uid . ':' . $id, 'folder' => $folder, 'existing' => $exists, ]); if (count($set->items) == self::CHUNK_SIZE) { $callback($set); $set = new ItemSet(); } } if (count($set->items)) { $callback($set); } // TODO: Delete messages that do not exist anymore? } /** * Get folders hierarchy */ public function getFolders($types = []): array { $folders = $this->imap->listMailboxes('', ''); if ($folders === false) { throw new \Exception("Failed to get list of IMAP folders"); } // TODO: Migrate folder subscription state $result = []; foreach ($folders as $folder) { if ($this->shouldSkip($folder)) { \Log::debug("Skipping folder {$folder}."); continue; } $result[] = Folder::fromArray([ 'fullname' => $folder, 'type' => 'mail' ]); } return $result; } /** * Get a list of folder items, limited to their essential propeties * used in incremental migration to skip unchanged items. */ public function getItems(Folder $folder): array { - $mailbox = $folder->fullname; + if ($folder->targetname) { + $mailbox = self::toUTF7($folder->targetname); + } else { + $mailbox = $folder->fullname; + } // TODO: We should probably first use SEARCH/SORT to skip messages marked as \Deleted // TODO: fetchHeaders() fetches too many headers, we should slim-down, here we need // only UID FLAGS INTERNALDATE BODY.PEEK[HEADER.FIELDS (DATE FROM MESSAGE-ID)] $messages = $this->imap->fetchHeaders($mailbox, '1:*', true, false, ['Message-Id']); if ($messages === false) { throw new \Exception("Failed to get IMAP message headers in {$mailbox}"); } $result = []; foreach ($messages as $message) { // Remove flags that we can't append (e.g. RECENT) $flags = $this->filterImapFlags(array_keys($message->flags)); // Generate message ID if the header does not exist $id = $this->getMessageId($message, $mailbox); $result[$id] = [ 'uid' => $message->uid, 'flags' => $flags, 'size' => $message->size, 'timestamp' => $message->timestamp, ]; } return $result; } /** * Initialize IMAP connection and authenticate the user */ private static function initIMAP(array $config): \rcube_imap_generic { $imap = new \rcube_imap_generic(); if (\config('app.debug')) { $imap->setDebug(true, 'App\Backends\IMAP::logDebug'); } $imap->connect($config['host'], $config['user'], $config['password'], $config['options']); if (!$imap->connected()) { $message = sprintf("Login failed for %s against %s. %s", $config['user'], $config['host'], $imap->error); \Log::error($message); throw new \Exception("Connection to IMAP failed"); } return $imap; } /** * Get IMAP configuration */ private static function getConfig(Account $account): array { $uri = \parse_url($account->uri); $default_port = 143; $ssl_mode = null; if (isset($uri['scheme'])) { if (preg_match('/^(ssl|imaps)/', $uri['scheme'])) { $default_port = 993; $ssl_mode = 'ssl'; } elseif ($uri['scheme'] === 'tls') { $ssl_mode = 'tls'; } } $config = [ 'host' => $uri['host'], 'user' => $account->username, 'password' => $account->password, 'options' => [ 'port' => !empty($uri['port']) ? $uri['port'] : $default_port, 'ssl_mode' => $ssl_mode, 'socket_options' => [ 'ssl' => [ // TODO: These configuration options make sense for "local" Kolab IMAP, // but when connecting to external one we might want to just disable // cert validation, or make it optional via Account URI parameters 'verify_peer' => \config('services.imap.verify_peer'), 'verify_peer_name' => \config('services.imap.verify_peer'), 'verify_host' => \config('services.imap.verify_host') ], ], ], ]; // User impersonation. Example URI: imap://admin:password@hostname:143?user=user%40domain.tld if ($account->loginas) { $config['options']['auth_cid'] = $config['user']; $config['options']['auth_pw'] = $config['password']; $config['options']['auth_type'] = 'PLAIN'; $config['user'] = $account->loginas; } return $config; } /** * Limit IMAP flags to these that can be migrated */ private function filterImapFlags($flags) { // TODO: Support custom flags migration return array_filter( $flags, function ($flag) { return isset($this->imap->flags[$flag]); } ); } /** * Check if the folder should not be migrated */ private function shouldSkip($folder): bool { // TODO: This should probably use NAMESPACE information if (preg_match('~(Shared Folders|Other Users)/.*~', $folder)) { return true; } return false; } /** * Return Message-Id, generate unique identifier if Message-Id does not exist */ private function getMessageId($message, $folder): string { if (!empty($message->messageID)) { return $message->messageID; } return md5($folder . $message->from . ($message->date ?: $message->timestamp)); } }