diff --git a/src/app/Console/Commands/Data/MigrateCommand.php b/src/app/Console/Commands/Data/MigrateCommand.php --- a/src/app/Console/Commands/Data/MigrateCommand.php +++ b/src/app/Console/Commands/Data/MigrateCommand.php @@ -30,7 +30,8 @@ {dst : Destination account} {--type= : Object type(s)} {--sync : Execute migration synchronously} - {--force : Force existing queue removal}'; + {--force : Force existing queue removal} + {--folder-mapping=* : Folder mapping in the form "source:target"}'; // {--export-only : Only export data} // {--import-only : Only import previously exported data}'; @@ -50,10 +51,18 @@ { $src = new DataMigrator\Account($this->argument('src')); $dst = new DataMigrator\Account($this->argument('dst')); + + $folderMapping = []; + foreach ($this->option('folder-mapping') as $mapping) { + $arr = explode(":", $mapping); + $folderMapping[$arr[0]] = $arr[1]; + } + $options = [ 'type' => $this->option('type'), 'force' => $this->option('force'), 'sync' => $this->option('sync'), + 'folderMapping' => $folderMapping, 'stdout' => true, ]; diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php --- a/src/app/DataMigrator/DAV.php +++ b/src/app/DataMigrator/DAV.php @@ -183,6 +183,8 @@ $dav_type = $this->type2DAV($folder->type); $location = $this->getFolderPath($folder); $search = new DAVSearch($dav_type, false); + // FIXME this avoid a crash of iRony on empty collections? + $search->depth = "infinity"; // TODO: We request only properties relevant to incremental migration, // i.e. to find that something exists and its last update time. @@ -256,6 +258,8 @@ $location = $this->getFolderPath($folder); $search = new DAVSearch($dav_type); + // FIXME this avoid a crash of iRony on empty collections? + $search->depth = "infinity"; // TODO: We request only properties relevant to incremental migration, // i.e. to find that something exists and its last update time. @@ -342,19 +346,22 @@ return $this->folderPaths[$cache_key]; } - $folders = $this->client->listFolders($this->type2DAV($folder->type)); + for ($i = 0; $i < 5; $i++) { + $folders = $this->client->listFolders($this->type2DAV($folder->type)); - if ($folders === false) { - throw new \Exception("Failed to list folders on the DAV server"); - } + if ($folders === false) { + throw new \Exception("Failed to list folders on the DAV server"); + } - // Note: iRony flattens the list by modifying the folder name - // This is not going to work with Cyrus DAV, but anyway folder - // hierarchies support is not full in Kolab 4. - foreach ($folders as $dav_folder) { - if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) { - return $this->folderPaths[$cache_key] = rtrim($dav_folder->href, '/'); + // Note: iRony flattens the list by modifying the folder name + // This is not going to work with Cyrus DAV, but anyway folder + // hierarchies support is not full in Kolab 4. + foreach ($folders as $dav_folder) { + if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) { + return $this->folderPaths[$cache_key] = rtrim($dav_folder->href, '/'); + } } + sleep(1); } throw new \Exception("Folder not found: {$folder->fullname}"); diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php --- a/src/app/DataMigrator/EWS.php +++ b/src/app/DataMigrator/EWS.php @@ -25,6 +25,7 @@ EWS\Appointment::FOLDER_TYPE, EWS\Contact::FOLDER_TYPE, EWS\Task::FOLDER_TYPE, + EWS\Email::FOLDER_TYPE, // TODO: mail and sticky notes are exported as eml files. // We could use imapsync to synchronize mail, but for notes // the only option will be to convert them to Kolab format here @@ -61,6 +62,9 @@ 'XrmCompanySearch', 'XrmDealSearch', 'XrmSearch', + 'MS-OLK-AllCalendarItems', + 'MS-OLK-AllContactItems', + 'MS-OLK-AllMailItems', // TODO: These are different depending on a user locale and it's not possible // to switch to English other than changing the user locale in OWA/Exchange. 'Calendar/United States holidays', @@ -69,6 +73,8 @@ 'Kalendarz/Polska — dni wolne od pracy', // pl 'Ulubione', // pl 'Moje kontakty', // pl + 'Aufgabensuche', // de + 'Postausgang', // de ]; /** @var array Map of EWS folder types to Kolab types */ @@ -76,6 +82,7 @@ EWS\Appointment::FOLDER_TYPE => Engine::TYPE_EVENT, EWS\Contact::FOLDER_TYPE => Engine::TYPE_CONTACT, EWS\Task::FOLDER_TYPE => Engine::TYPE_TASK, + EWS\Email::FOLDER_TYPE => Engine::TYPE_MAIL, ]; /** @var Account Account to operate on */ @@ -329,6 +336,8 @@ $item['changeKey'] = $changeKey; $existingIndex[$id] = $idx; unset($item['x-ms-id']); + } else { + $existingIndex[$idx] = $idx; } } ); @@ -344,6 +353,7 @@ 'FieldURI' => [ ['FieldURI' => 'item:ItemClass'], // ['FieldURI' => 'item:Size'], + ['FieldURI' => 'message:InternetMessageId'], //For mail only? ], ], ], @@ -442,10 +452,25 @@ $idx = $existingIndex[$id['Id']]; if ($existing[$idx]['changeKey'] == $id['ChangeKey']) { + \Log::debug("[EWS] Skipping over already existing message $idx..."); return null; } $exists = $existing[$idx]['href']; + } else { + $msgid = null; + try { + $msgid = $item->getInternetMessageId(); + } catch (\Exception $e) { + //Ignore + } + if (isset($existingIndex[$msgid])) { + // If the messageid already exists, we assume it's the same email. + // Flag/size changes are ignored for now. + // Otherwise we should set uid/size/flags on exists, so the IMAP implementation can pick it up. + \Log::debug("[EWS] Skipping over already existing message $msgid..."); + return null; + } } if (!EWS\Item::isValidItem($item)) { diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php --- a/src/app/DataMigrator/EWS/Appointment.php +++ b/src/app/DataMigrator/EWS/Appointment.php @@ -34,7 +34,7 @@ /** * Process event object */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Initialize $this->itemId (for some unit tests) $this->getUID($item); @@ -57,12 +57,18 @@ // That's why we'll just remove all ATTACH:CID:... occurrences // and inject attachments to the main event $ical = preg_replace('/\r\nATTACH:CID:[^\r]+\r\n(\r\n [^\r\n]*)?/', '', $ical); + // We seem to get some weird ATTACH parts as part of ORGANIZER sometimes. + // Looks like this (when printing $ical to console): + // ORGANIZER;CN="Doe, John":MAILTO:John.Doe@example.comATTACH:CID:2388A81D6CB99E09E72ACF2D192043D418FD86B8@example.com + // DESCRIPTION;LANGUAGE=de-DE:@ ... + $ical = preg_replace('/ATTACH:CID:[^\r]+\r\n/', "\r\n", $ical); foreach ((array) $item->getAttachments()->getFileAttachment() as $attachment) { $_attachment = $this->getAttachment($attachment); $ctype = $_attachment->getContentType(); $body = $_attachment->getContent(); + $name = $_attachment->getName(); // It looks like Exchange may have an issue with plain text files. // We'll skip empty files @@ -78,7 +84,7 @@ // Inject the attachment at the end of the first VEVENT block // TODO: We should not do it in memory to not exceed the memory limit - $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;FMTTYPE={$ctype}:\r\n {$body}"; + $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;X-LABEL={$name};FMTTYPE={$ctype}:\r\n {$body}"; $pos = strpos($ical, "\r\nEND:VEVENT"); $ical = substr_replace($ical, $append, $pos + 2, 0); } diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php --- a/src/app/DataMigrator/EWS/Contact.php +++ b/src/app/DataMigrator/EWS/Contact.php @@ -29,7 +29,7 @@ /** * Process contact object */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Decode MIME content $vcard = base64_decode((string) $item->getMimeContent()); diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php --- a/src/app/DataMigrator/EWS/DistList.php +++ b/src/app/DataMigrator/EWS/DistList.php @@ -29,7 +29,7 @@ /** * Convert distribution list object to vCard */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Groups (Distribution Lists) are not exported in vCard format, they use eml diff --git a/src/app/DataMigrator/EWS/Email.php b/src/app/DataMigrator/EWS/Email.php new file mode 100644 --- /dev/null +++ b/src/app/DataMigrator/EWS/Email.php @@ -0,0 +1,53 @@ + 'calendar:UID']; + + return $request; + } + + /** + * Process event object + */ + protected function convertItem(Type $item, $targetItem) + { + // This is not actually called, emails are migrate with type Note, because of the FOLDER_TYPE + return null; + } + + /** + * Get Item UID (Generate a new one if needed) + */ + protected function getUID(Type $item): string + { + // Only appointments have UID property + $this->uid = $item->getUID(); + + // This also sets $this->itemId; + return parent::getUID($item); + } +} diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php --- a/src/app/DataMigrator/EWS/Item.php +++ b/src/app/DataMigrator/EWS/Item.php @@ -79,7 +79,7 @@ \Log::debug("[EWS] Saving item {$uid}..."); // Apply type-specific format converters - $content = $this->convertItem($ewsItem); + $content = $this->convertItem($ewsItem, $item); if (!is_string($content)) { throw new \Exception("Failed to fetch EWS item {$this->itemId}"); @@ -104,7 +104,7 @@ /** * Item conversion code */ - abstract protected function convertItem(Type $item); + abstract protected function convertItem(Type $item, $targetItem); /** * Get GetItem request parameters diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php --- a/src/app/DataMigrator/EWS/Note.php +++ b/src/app/DataMigrator/EWS/Note.php @@ -32,10 +32,25 @@ /** * Process contact object */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { $email = base64_decode((string) $item->getMimeContent()); + $flags = []; + if ($item->getIsRead()) { + $flags[] = 'SEEN'; + } + + $internaldate = null; + if ($internaldate = $item->getDateTimeReceived()) { + $internaldate = (new \DateTime($internaldate))->format('d-M-Y H:i:s O'); + } + + $targetItem->data = [ + 'flags' => $flags, + 'internaldate' => $internaldate, + ]; + return $email; } } diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php --- a/src/app/DataMigrator/EWS/Task.php +++ b/src/app/DataMigrator/EWS/Task.php @@ -29,7 +29,7 @@ /** * Process task object */ - protected function convertItem(Type $item) + protected function convertItem(Type $item, $targetItem) { // Tasks are exported as Email messages in useless format // (does not contain all relevant properties) diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php --- a/src/app/DataMigrator/Engine.php +++ b/src/app/DataMigrator/Engine.php @@ -105,6 +105,7 @@ $folders = $this->exporter->getFolders($types); $count = 0; $async = empty($options['sync']); + $folderMapping = $this->options['folderMapping'] ?? []; foreach ($folders as $folder) { $this->debug("Processing folder {$folder->fullname}..."); @@ -112,6 +113,16 @@ $folder->queueId = $queue_id; $folder->location = $location; + // Apply name replacements + $folder->targetname = $folder->fullname; + foreach ($folderMapping as $key => $value) { + if (str_contains($folder->targetname, $key)) { + $folder->targetname = str_replace($key, $value, $folder->targetname); + $this->debug("Replacing {$folder->fullname} with {$folder->targetname}"); + break; + } + } + if ($async) { // Dispatch the job (for async execution) Jobs\FolderJob::dispatch($folder); diff --git a/src/app/DataMigrator/IMAP.php b/src/app/DataMigrator/IMAP.php --- a/src/app/DataMigrator/IMAP.php +++ b/src/app/DataMigrator/IMAP.php @@ -72,22 +72,33 @@ throw new \Exception("IMAP does not support folder of type {$folder->type}"); } - if ($folder->fullname == 'INBOX') { + if ($folder->targetname == 'INBOX') { // INBOX always exists return; } - if (!$this->imap->createFolder($folder->fullname)) { + if (!$this->imap->createFolder(self::toUTF7($folder->targetname))) { \Log::warning("Failed to create the folder: {$this->imap->error}"); if (str_contains($this->imap->error, "Mailbox already exists")) { // Not an error } else { - throw new \Exception("Failed to create an IMAP folder {$folder->fullname}"); + throw new \Exception("Failed to create an IMAP folder {$folder->targetname}"); } } - // TODO: Migrate folder subscription state + // TODO: Migrate folder subscription state. For now we just subscribe. + if (!$this->imap->subscribe(self::toUTF7($folder->targetname))) { + \Log::warning("Failed to subscribe to the folder: {$this->imap->error}"); + } + } + + /** + * Convert UTF8 string to UTF7-IMAP encoding + */ + private static function toUTF7(string $string): string + { + return \mb_convert_encoding($string, 'UTF7-IMAP', 'UTF8'); } /** @@ -99,14 +110,14 @@ */ public function createItem(Item $item): void { - $mailbox = $item->folder->fullname; + $mailbox = self::toUTF7($item->folder->targetname); if (strlen($item->content)) { $result = $this->imap->append( $mailbox, $item->content, - $item->data['flags'], - $item->data['internaldate'], + $item->data['flags'] ?? [], + $item->data['internaldate'] ?? null, true ); @@ -118,8 +129,8 @@ $mailbox, $item->filename, null, - $item->data['flags'], - $item->data['internaldate'], + $item->data['flags'] ?? [], + $item->data['internaldate'] ?? null, true ); @@ -317,7 +328,11 @@ */ public function getItems(Folder $folder): array { - $mailbox = $folder->fullname; + if ($folder->targetname) { + $mailbox = self::toUTF7($folder->targetname); + } else { + $mailbox = $folder->fullname; + } // TODO: We should probably first use SEARCH/SORT to skip messages marked as \Deleted // TODO: fetchHeaders() fetches too many headers, we should slim-down, here we need diff --git a/src/app/DataMigrator/Interface/Folder.php b/src/app/DataMigrator/Interface/Folder.php --- a/src/app/DataMigrator/Interface/Folder.php +++ b/src/app/DataMigrator/Interface/Folder.php @@ -27,6 +27,9 @@ /** @var string Folder name with path */ public $fullname; + /** @var string Target folder name with path */ + public $targetname; + /** @var string Storage location (for temporary data) */ public $location; diff --git a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php --- a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php +++ b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php @@ -7,6 +7,7 @@ use App\DataMigrator\Engine; use App\DataMigrator\EWS; use App\DataMigrator\Interface\Folder; +use App\DataMigrator\Interface\Item; use garethp\ews\API\Type; use Tests\TestCase; @@ -21,6 +22,7 @@ $engine = new Engine(); $ews = new EWS($account, $engine); $folder = Folder::fromArray(['id' => 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $appointment = new EWS\Appointment($ews, $folder); $ical = file_get_contents(__DIR__ . '/../../../data/ews/event/1.ics'); @@ -78,7 +80,7 @@ ]); // Convert the Exchange item into iCalendar - $ical = $this->invokeMethod($appointment, 'convertItem', [$item]); + $ical = $this->invokeMethod($appointment, 'convertItem', [$item, $targetItem]); // Parse the iCalendar output $event = new Vevent(); diff --git a/src/tests/Unit/DataMigrator/EWS/ContactTest.php b/src/tests/Unit/DataMigrator/EWS/ContactTest.php --- a/src/tests/Unit/DataMigrator/EWS/ContactTest.php +++ b/src/tests/Unit/DataMigrator/EWS/ContactTest.php @@ -7,6 +7,7 @@ use App\DataMigrator\Engine; use App\DataMigrator\EWS; use App\DataMigrator\Interface\Folder; +use App\DataMigrator\Interface\Item; use garethp\ews\API\Type; use Tests\TestCase; @@ -21,6 +22,7 @@ $engine = new Engine(); $ews = new EWS($account, $engine); $folder = Folder::fromArray(['id' => 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $contact = new EWS\Contact($ews, $folder); $vcard = file_get_contents(__DIR__ . '/../../../data/ews/contact/1.vcf'); @@ -113,7 +115,7 @@ ]); // Convert the Exchange item into vCard - $vcard = $this->invokeMethod($contact, 'convertItem', [$item]); + $vcard = $this->invokeMethod($contact, 'convertItem', [$item, $targetItem]); // Parse the vCard $contact = new Vcard(); diff --git a/src/tests/Unit/DataMigrator/EWS/DistListTest.php b/src/tests/Unit/DataMigrator/EWS/DistListTest.php --- a/src/tests/Unit/DataMigrator/EWS/DistListTest.php +++ b/src/tests/Unit/DataMigrator/EWS/DistListTest.php @@ -7,6 +7,7 @@ use App\DataMigrator\Engine; use App\DataMigrator\EWS; use App\DataMigrator\Interface\Folder; +use App\DataMigrator\Interface\Item; use garethp\ews\API\Type; use Tests\TestCase; @@ -21,6 +22,7 @@ $engine = new Engine(); $ews = new EWS($account, $engine); $folder = Folder::fromArray(['id' => 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $distlist = new EWS\DistList($ews, $folder); // FIXME: I haven't found a way to convert xml content into a Type instance @@ -72,7 +74,7 @@ ]); // Convert the Exchange item into vCard - $vcard = $this->invokeMethod($distlist, 'convertItem', [$item]); + $vcard = $this->invokeMethod($distlist, 'convertItem', [$item, $targetItem]); // Parse the vCard $distlist = new Vcard(); diff --git a/src/tests/Unit/DataMigrator/EWS/TaskTest.php b/src/tests/Unit/DataMigrator/EWS/TaskTest.php --- a/src/tests/Unit/DataMigrator/EWS/TaskTest.php +++ b/src/tests/Unit/DataMigrator/EWS/TaskTest.php @@ -7,6 +7,7 @@ use App\DataMigrator\Engine; use App\DataMigrator\EWS; use App\DataMigrator\Interface\Folder; +use App\DataMigrator\Interface\Item; use garethp\ews\API\Type; use Tests\TestCase; @@ -24,6 +25,7 @@ $engine->destination = $destination; $ews = new EWS($source, $engine); $folder = Folder::fromArray(['id' => 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $task = new EWS\Task($ews, $folder); // FIXME: I haven't found a way to convert xml content into a Type instance @@ -118,7 +120,7 @@ ]); // Convert the Exchange item into iCalendar - $ical = $this->invokeMethod($task, 'convertItem', [$item]); + $ical = $this->invokeMethod($task, 'convertItem', [$item, $targetItem]); // Parse the iCalendar output $task = new Vtodo();