diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php --- a/src/app/DataMigrator/DAV.php +++ b/src/app/DataMigrator/DAV.php @@ -67,13 +67,21 @@ */ public function createItem(Item $item): void { - if ($item->existing) { - $href = $item->existing; + $is_file = false; + $href = $item->existing ?: null; + + if (strlen($item->content)) { + $content = $item->content; } else { - $href = $this->getFolderPath($item->folder) . '/' . pathinfo($item->filename, PATHINFO_BASENAME); + $content = $item->filename; + $is_file = true; + } + + if (empty($href)) { + $href = $this->getFolderPath($item->folder) . '/' . basename($item->filename); } - $object = new DAVOpaque($item->filename, true); + $object = new DAVOpaque($content, $is_file); $object->href = $href; switch ($item->folder->type) { @@ -139,15 +147,6 @@ */ public function fetchItem(Item $item): void { - // Save the item content to a file - $location = $item->folder->location; - - if (!file_exists($location)) { - mkdir($location, 0740, true); - } - - $location .= '/' . basename($item->id); - $result = $this->client->getObjects(dirname($item->id), $this->type2DAV($item->folder->type), [$item->id]); if ($result === false) { @@ -156,11 +155,21 @@ // TODO: Do any content changes, e.g. organizer/attendee email migration - if (file_put_contents($location, (string) $result[0]) === false) { - throw new \Exception("Failed to write to {$location}"); - } + $content = (string) $result[0]; + + if (strlen($content) > Engine::MAX_ITEM_SIZE) { + // Save the item content to a file + $location = $item->folder->tempFileLocation(basename($item->id)); + + if (file_put_contents($location, $content) === false) { + throw new \Exception("Failed to write to {$location}"); + } - $item->filename = $location; + $item->filename = $location; + } else { + $item->content = $content; + $item->filename = basename($item->id); + } } /** diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php --- a/src/app/DataMigrator/EWS.php +++ b/src/app/DataMigrator/EWS.php @@ -4,6 +4,7 @@ use App\DataMigrator\Interface\Folder; use App\DataMigrator\Interface\Item; +use App\DataMigrator\Interface\ItemSet; use garethp\ews\API; use garethp\ews\API\Type; use Illuminate\Support\Facades\Http; @@ -13,6 +14,9 @@ */ class EWS implements Interface\ExporterInterface { + /** @const int Max number of items to migrate in one go */ + protected const CHUNK_SIZE = 20; + /** @var API EWS API object */ public $api; @@ -31,15 +35,16 @@ /** @var array Interal folders to skip */ protected $folder_exceptions = [ + 'AllCategorizedItems', 'AllContacts', + 'AllContactsExtended', 'AllPersonMetadata', 'AllTodoTasks', 'Document Centric Conversations', 'ExternalContacts', - 'Favorites', 'Flagged Emails', + 'Folder Memberships', 'GraphFilesAndWorkingSetSearchFolder', - 'My Contacts', 'MyContactsExtended', 'Orion Notes', 'Outbox', @@ -48,16 +53,22 @@ 'RelevantContacts', 'SharedFilesSearchFolder', 'Sharing', + 'SpoolsPresentSharedItemsSearchFolder', + 'SpoolsSearchFolder', 'To-Do Search', 'UserCuratedContacts', 'XrmActivityStreamSearch', 'XrmCompanySearch', 'XrmDealSearch', 'XrmSearch', - 'Folder Memberships', - // TODO: These are different depending on a user locale + // TODO: These are different depending on a user locale and it's not possible + // to switch to English other than changing the user locale in OWA/Exchange. 'Calendar/United States holidays', + 'Favorites', + 'My Contacts', 'Kalendarz/Polska — dni wolne od pracy', // pl + 'Ulubione', // pl + 'Moje kontakty', // pl ]; /** @var array Map of EWS folder types to Kolab types */ @@ -214,9 +225,31 @@ */ public function getFolders($types = []): array { - // Get full folders hierarchy + if (empty($types)) { + $types = array_values($this->type_map); + } + + // Create FolderClass filter + $search = new Type\OrType(); + foreach ($types as $type) { + $type = array_search($type, $this->type_map); + $search->addContains(Type\Contains::buildFromArray([ + 'FieldURI' => [ + Type\FieldURI::buildFromArray(['FieldURI' => 'folder:FolderClass']), + ], + 'Constant' => Type\ConstantValueType::buildFromArray([ + 'Value' => $type, + ]), + 'ContainmentComparison' => 'Exact', + 'ContainmentMode' => 'FullString', + ])); + } + + // Get full folders hierarchy (filtered by folder class) + // Use of the filter reduces the response size by excluding system folders $options = [ 'Traversal' => 'Deep', + 'Restriction' => ['Or' => $search], ]; $folders = $this->api->getChildrenFolders('root', $options); @@ -232,7 +265,7 @@ continue; } - // Note: Folder names are localized + // Note: Folder names are localized, even INBOX $name = $fullname = $folder->getDisplayName(); $id = $folder->getFolderId()->getId(); $parentId = $folder->getParentFolderId()->getId(); @@ -308,7 +341,10 @@ 'ItemShape' => [ 'BaseShape' => 'IdOnly', 'AdditionalProperties' => [ - 'FieldURI' => ['FieldURI' => 'item:ItemClass'], + 'FieldURI' => [ + ['FieldURI' => 'item:ItemClass'], + // ['FieldURI' => 'item:Size'], + ], ], ], ]; @@ -325,10 +361,16 @@ // Request first page $response = $this->api->getClient()->FindItem($request); + $set = new ItemSet(); + // @phpstan-ignore-next-line - foreach ($response as $item) { + foreach ($response->getItems() as $item) { if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { - $callback($item); + $set->items[] = $item; + if (count($set->items) == self::CHUNK_SIZE) { + $callback($set); + $set = new ItemSet(); + } } } @@ -337,13 +379,21 @@ // @phpstan-ignore-next-line $response = $this->api->getNextPage($response); - foreach ($response as $item) { + foreach ($response->getItems() as $item) { if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { - $callback($item); + $set->items[] = $item; + if (count($set->items) == self::CHUNK_SIZE) { + $callback($set); + $set = new ItemSet(); + } } } } + if (count($set->items)) { + $callback($set); + } + // TODO: Delete items that do not exist anymore? } @@ -356,12 +406,11 @@ $this->initEnv($this->engine->queue); if ($driver = EWS\Item::factory($this, $item)) { - $item->filename = $driver->fetchItem($item); + $driver->processItem($item); + return; } - if (empty($item->filename)) { - throw new \Exception("Failed to fetch an item from EWS"); - } + throw new \Exception("Failed to fetch an item from EWS"); } /** @@ -399,20 +448,16 @@ $exists = $existing[$idx]['href']; } - $item = Item::fromArray([ - 'id' => $id, + if (!EWS\Item::isValidItem($item)) { + return null; + } + + return Item::fromArray([ + 'id' => $id['Id'], 'class' => $item->getItemClass(), 'folder' => $folder, 'existing' => $exists, ]); - - // TODO: We don't need to instantiate Item at this point, instead - // implement EWS\Item::validateClass() method - if ($driver = EWS\Item::factory($this, $item)) { - return $item; - } - - return null; } /** diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php --- a/src/app/DataMigrator/EWS/Appointment.php +++ b/src/app/DataMigrator/EWS/Appointment.php @@ -18,7 +18,7 @@ /** * Get GetItem request parameters */ - protected function getItemRequest(): array + protected static function getItemRequest(): array { $request = parent::getItemRequest(); @@ -26,8 +26,7 @@ $request['ItemShape']['IncludeMimeContent'] = true; // Get UID property, it's not included in the Default set - // FIXME: How to add the property to the set, not replace the whole set - $request['ItemShape']['AdditionalProperties']['FieldURI'] = ['FieldURI' => 'calendar:UID']; + $request['ItemShape']['AdditionalProperties']['FieldURI'][] = ['FieldURI' => 'calendar:UID']; return $request; } @@ -35,7 +34,7 @@ /** * Process event object */ - protected function processItem(Type $item) + protected function convertItem(Type $item) { // Initialize $this->itemId (for some unit tests) $this->getUID($item); diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php --- a/src/app/DataMigrator/EWS/Contact.php +++ b/src/app/DataMigrator/EWS/Contact.php @@ -16,7 +16,7 @@ /** * Get GetItem request parameters */ - protected function getItemRequest(): array + protected static function getItemRequest(): array { $request = parent::getItemRequest(); @@ -29,7 +29,7 @@ /** * Process contact object */ - protected function processItem(Type $item) + protected function convertItem(Type $item) { // Decode MIME content $vcard = base64_decode((string) $item->getMimeContent()); @@ -56,7 +56,7 @@ $vcard = str_replace($matches[1], "X-SPOUSE:{$spouse}", $vcard); } - // TODO: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12 + // Anniversary: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12 if (preg_match('/(X-MS-ANNIVERSARY[;:][^\r\n]+)/', $vcard, $matches)) { $date = preg_replace('/^[^:]+:/', '', $matches[1]); $vcard = str_replace($matches[1], "X-ANNIVERSARY:{$date}", $vcard); diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php --- a/src/app/DataMigrator/EWS/DistList.php +++ b/src/app/DataMigrator/EWS/DistList.php @@ -13,10 +13,23 @@ public const TYPE = 'IPM.DistList'; public const FILE_EXT = 'vcf'; + /** + * Get GetItem request parameters + */ + protected static function getItemRequest(): array + { + $request = parent::getItemRequest(); + + // Get Body property, it's not included in the Default set + $request['ItemShape']['AdditionalProperties']['FieldURI'] = ['FieldURI' => 'item:Body']; + + return $request; + } + /** * Convert distribution list object to vCard */ - protected function processItem(Type $item) + protected function convertItem(Type $item) { // Groups (Distribution Lists) are not exported in vCard format, they use eml @@ -34,11 +47,6 @@ $vcard .= $this->formatProp($key, $prop[0], isset($prop[1]) ? $prop[1] : []); } - // TODO: The group description property in Exchange is not available via EWS XML, - // at least not at outlook.office.com (Exchange 2010). It is available in the - // MimeContent which is in email message format. However, Kolab Webclient does not - // make any use of the NOTE property for contact groups. - // Process list members if ($members = $item->getMembers()) { // The Member property is either array (multiple members) or Type\MemberType @@ -70,6 +78,11 @@ } } + // Note: Kolab Webclient does not make any use of the NOTE property for contact groups + if ($body = (string) $item->getBody()) { + $vcard .= $this->formatProp('NOTE', $body); + } + $vcard .= "END:VCARD\r\n"; return $vcard; diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php --- a/src/app/DataMigrator/EWS/Item.php +++ b/src/app/DataMigrator/EWS/Item.php @@ -3,8 +3,10 @@ namespace App\DataMigrator\EWS; use App\DataMigrator\EWS; +use App\DataMigrator\Engine; use App\DataMigrator\Interface\Folder as FolderInterface; use App\DataMigrator\Interface\Item as ItemInterface; +use App\DataMigrator\Interface\ItemSet as ItemSetInterface; use garethp\ews\API; use garethp\ews\API\Type; @@ -50,11 +52,24 @@ } /** - * Fetch the specified object and put into a file + * Validate that specified EWS Item is of supported type */ - public function fetchItem(ItemInterface $item) + public static function isValidItem(Type $item): bool { - $itemId = $item->id; + $item_class = str_replace('IPM.', '', $item->getItemClass()); + $item_class = "\App\DataMigrator\EWS\\{$item_class}"; + + return class_exists($item_class); + } + + /** + * Process an item (fetch data and convert it) + */ + public function processItem(ItemInterface $item): void + { + $itemId = ['Id' => $item->id]; + + \Log::debug("[EWS] Fetching item {$item->id}..."); // Fetch the item $ewsItem = $this->driver->api->getItem($itemId, $this->getItemRequest()); @@ -64,46 +79,48 @@ \Log::debug("[EWS] Saving item {$uid}..."); // Apply type-specific format converters - $content = $this->processItem($ewsItem); + $content = $this->convertItem($ewsItem); if (!is_string($content)) { - return; + throw new \Exception("Failed to fetch EWS item {$this->itemId}"); } - $uid = preg_replace('/[^a-zA-Z0-9_:@-]/', '', $uid); - - $location = $this->folder->location; + $filename = $uid . '.' . $this->fileExtension(); - if (!file_exists($location)) { - mkdir($location, 0740, true); - } - - $location .= '/' . $uid . '.' . $this->fileExtension(); + if (strlen($content) > Engine::MAX_ITEM_SIZE) { + $location = $this->folder->tempFileLocation($filename); - file_put_contents($location, $content); + if (file_put_contents($location, $content) === false) { + throw new \Exception("Failed to write to file at {$location}"); + } - return $location; + $item->filename = $location; + } else { + $item->content = $content; + $item->filename = $filename; + } } /** * Item conversion code */ - abstract protected function processItem(Type $item); + abstract protected function convertItem(Type $item); /** * Get GetItem request parameters */ - protected function getItemRequest(): array + protected static function getItemRequest(): array { $request = [ 'ItemShape' => [ // Reqest default set of properties 'BaseShape' => 'Default', // Additional properties, e.g. LastModifiedTime - // FIXME: How to add multiple properties here? 'AdditionalProperties' => [ - 'FieldURI' => ['FieldURI' => 'item:LastModifiedTime'], - ] + 'FieldURI' => [ + ['FieldURI' => 'item:LastModifiedTime'], + ], + ], ] ]; diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php --- a/src/app/DataMigrator/EWS/Note.php +++ b/src/app/DataMigrator/EWS/Note.php @@ -16,7 +16,7 @@ /** * Get GetItem request parameters */ - protected function getItemRequest(): array + protected static function getItemRequest(): array { $request = parent::getItemRequest(); @@ -32,7 +32,7 @@ /** * Process contact object */ - protected function processItem(Type $item) + protected function convertItem(Type $item) { $email = base64_decode((string) $item->getMimeContent()); diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php --- a/src/app/DataMigrator/EWS/Task.php +++ b/src/app/DataMigrator/EWS/Task.php @@ -16,7 +16,7 @@ /** * Get GetItem request parameters */ - protected function getItemRequest(): array + protected static function getItemRequest(): array { $request = parent::getItemRequest(); @@ -29,7 +29,7 @@ /** * Process task object */ - protected function processItem(Type $item) + protected function convertItem(Type $item) { // Tasks are exported as Email messages in useless format // (does not contain all relevant properties) diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php --- a/src/app/DataMigrator/Engine.php +++ b/src/app/DataMigrator/Engine.php @@ -3,6 +3,7 @@ namespace App\DataMigrator; use App\DataMigrator\Interface\ExporterInterface; +use App\DataMigrator\Interface\FetchItemSetInterface; use App\DataMigrator\Interface\Folder; use App\DataMigrator\Interface\ImporterInterface; use App\DataMigrator\Interface\Item; @@ -21,6 +22,9 @@ public const TYPE_NOTE = 'note'; public const TYPE_TASK = 'task'; + /** @const int Max item size to handle in-memory, bigger will be handled with temp files */ + public const MAX_ITEM_SIZE = 20 * 1024 * 1024; + /** @var Account Source account */ public $source; @@ -43,7 +47,7 @@ /** * Execute migration for the specified user */ - public function migrate(Account $source, Account $destination, array $options = []) + public function migrate(Account $source, Account $destination, array $options = []): void { $this->source = $source; $this->destination = $destination; @@ -87,14 +91,14 @@ // Create a queue $this->createQueue($queue_id); - // We'll store output in storage/ tree + // We'll store temp files in storage/ tree $location = storage_path('export/') . $source->email; if (!file_exists($location)) { mkdir($location, 0740, true); } - $types = preg_split('/\s*,\s*/', strtolower($options['type'] ?? '')); + $types = empty($options['type']) ? [] : preg_split('/\s*,\s*/', strtolower($options['type'])); $this->debug("Fetching folders hierarchy..."); @@ -189,8 +193,8 @@ $this->exporter->fetchItem($item); $this->importer->createItem($item); - if (!empty($item->filename)) { - unlink($item->filename); + if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) { + @unlink($item->filename); } if (empty($this->options['sync'])) { @@ -208,15 +212,21 @@ $this->envFromQueue($set->items[0]->folder->queueId); } - // TODO: Some exporters, e.g. DAV, might optimize fetching multiple items in one go, - // we'll need a new API to do that - - foreach ($set->items as $item) { - $this->exporter->fetchItem($item); + $importItem = function (Item $item) { $this->importer->createItem($item); - if (!empty($item->filename)) { - unlink($item->filename); + if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) { + @unlink($item->filename); + } + }; + + // Some exporters, e.g. DAV, might optimize fetching multiple items in one go + if ($this->exporter instanceof FetchItemSetInterface) { + $this->exporter->fetchItemSet($set, $importItem); + } else { + foreach ($set->items as $item) { + $this->exporter->fetchItem($item); + $importItem($item); } } @@ -329,6 +339,10 @@ $driver = new IMAP($account, $this); break; + case 'test': + $driver = new Test($account, $this); + break; + default: throw new \Exception("Failed to init driver for '{$account->scheme}'"); } diff --git a/src/app/DataMigrator/IMAP.php b/src/app/DataMigrator/IMAP.php --- a/src/app/DataMigrator/IMAP.php +++ b/src/app/DataMigrator/IMAP.php @@ -88,7 +88,19 @@ { $mailbox = $item->folder->fullname; - if ($item->filename) { + if (strlen($item->content)) { + $result = $this->imap->append( + $mailbox, + $item->content, + $item->data['flags'], + $item->data['internaldate'], + true + ); + + if ($result === false) { + throw new \Exception("Failed to append IMAP message into {$mailbox}"); + } + } elseif ($item->filename) { $result = $this->imap->appendFromFile( $mailbox, $item->filename, @@ -153,35 +165,36 @@ || $header->timestamp != $item->existing['timestamp'] || $header->size != $item->existing['size'] ) { - // Save the message content to a file - $location = $item->folder->location; - - if (!file_exists($location)) { - mkdir($location, 0740, true); - } + // Handle message content in memory (up to 20MB), bigger messages will use a temp file + if ($header->size > Engine::MAX_ITEM_SIZE) { + // Save the message content to a file + $location = $item->folder->tempFileLocation($uid . '.eml'); - // TODO: What if parent folder not yet exists? - $location .= '/' . $uid . '.eml'; + $fp = fopen($location, 'w'); - // TODO: We should consider streaming the message, it should be possible - // with append() and handlePartBody(), but I don't know if anyone tried that. - - $fp = fopen($location, 'w'); + if (!$fp) { + throw new \Exception("Failed to open 'php://temp' stream"); + } - if (!$fp) { - throw new \Exception("Failed to write to {$location}"); + $result = $this->imap->handlePartBody($mailbox, $uid, true, '', null, null, $fp); + } else { + $result = $this->imap->handlePartBody($mailbox, $uid, true); } - $result = $this->imap->handlePartBody($mailbox, $uid, true, '', null, null, $fp); - if ($result === false) { - fclose($fp); + if (!empty($fp)) { + fclose($fp); + } + throw new \Exception("Failed to fetch IMAP message for {$mailbox}/{$uid}"); } - $item->filename = $location; - - fclose($fp); + if (!empty($fp) && !empty($location)) { + $item->filename = $location; + fclose($fp); + } else { + $item->content = $result; + } } $item->data = [ diff --git a/src/app/DataMigrator/Interface/FetchItemSetInterface.php b/src/app/DataMigrator/Interface/FetchItemSetInterface.php new file mode 100644 --- /dev/null +++ b/src/app/DataMigrator/Interface/FetchItemSetInterface.php @@ -0,0 +1,14 @@ +location; + + // TODO: What if parent folder not yet exists? + if (!file_exists($location)) { + mkdir($location, 0740, true); + } + + $location .= '/' . $filename; + + return $location; + } } diff --git a/src/app/DataMigrator/Interface/Item.php b/src/app/DataMigrator/Interface/Item.php --- a/src/app/DataMigrator/Interface/Item.php +++ b/src/app/DataMigrator/Interface/Item.php @@ -10,7 +10,7 @@ /** @var mixed Item identifier */ public $id; - /** @var Folder Folder */ + /** @var ?Folder Folder */ public $folder; /** @var string Object class */ @@ -24,13 +24,19 @@ */ public $existing; - /** @var ?string Exported object location in the local storage */ + /** @var string Exported object content */ + public $content = ''; + + /** @var ?string Exported object content location */ public $filename; /** @var array Extra data to migrate (like email flags, internaldate, etc.) */ public $data = []; + /** + * Create Item object from an array + */ public static function fromArray(array $data = []): Item { $obj = new self(); diff --git a/src/app/DataMigrator/Interface/ItemSet.php b/src/app/DataMigrator/Interface/ItemSet.php --- a/src/app/DataMigrator/Interface/ItemSet.php +++ b/src/app/DataMigrator/Interface/ItemSet.php @@ -5,16 +5,11 @@ /** * Data object representing a set of data items */ -class ItemSet +class ItemSet implements \Serializable { /** @var array Items list */ public $items = []; - // TODO: Every item has a $folder property, this makes the set - // needlesly big when serialized. We should probably store $folder - // once with the set and remove it from an item on serialize - // and back in unserialize. - /** * Create an ItemSet instance */ @@ -25,4 +20,26 @@ return $obj; } + + public function serialize(): ?string + { + // Every item has a Folder property, this makes the set + // needlesly big when serialized. Make the size more compact. + $folder = count($this->items) ? $this->items[0]->folder : null; + + foreach ($this->items as $item) { + $item->folder = null; + } + + return serialize([$folder, $this->items]); + } + + public function unserialize(string $data): void + { + [$folder, $this->items] = unserialize($data); + + foreach ($this->items as $item) { + $item->folder = $folder; + } + } } diff --git a/src/app/DataMigrator/Test.php b/src/app/DataMigrator/Test.php new file mode 100644 --- /dev/null +++ b/src/app/DataMigrator/Test.php @@ -0,0 +1,153 @@ +engine = $engine; + $this->account = $account; + } + + public static function init($folders) + { + self::$folders = $folders; + + self::$fetchedItems = []; + self::$createdItems = []; + self::$createdFolders = []; + } + + /** + * Check user credentials. + * + * @throws \Exception + */ + public function authenticate(): void + { + } + + /** + * Create an item in a folder. + * + * @param Item $item Item to import + * + * @throws \Exception + */ + public function createItem(Item $item): void + { + self::$createdItems[] = $item; + } + + /** + * Create a folder. + * + * @param Folder $folder Folder data + * + * @throws \Exception on error + */ + public function createFolder(Folder $folder): void + { + self::$createdFolders[] = $folder; + } + + /** + * Fetching an item + */ + public function fetchItem(Item $item): void + { + $item->content = 'content'; + $item->filename = 'test.eml'; + + self::$fetchedItems[] = $item; + } + + /** + * Fetch a list of folder items + */ + public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void + { + // Get existing messages' headers from the destination mailbox + $existing = $importer->getItems($folder); + + $set = new ItemSet(); + + foreach ((self::$folders[$folder->id]['items'] ?? []) as $itemId => $item) { + $exists = null; // TODO + + $item['id'] = $itemId; + $item['folder'] = $folder; + $item['existing'] = $exists; + + $set->items[] = Item::fromArray($item); + + if (count($set->items) == self::CHUNK_SIZE) { + $callback($set); + $set = new ItemSet(); + } + } + + if (count($set->items)) { + $callback($set); + } + } + + /** + * Get a list of items, limited to their essential propeties + * used in incremental migration. + * + * @param Folder $folder Folder data + * + * @throws \Exception on error + */ + public function getItems(Folder $folder): array + { + return self::$folders[$folder->id]['existing_items'] ?? []; + } + + /** + * Get folders hierarchy + */ + public function getFolders($types = []): array + { + $result = []; + + foreach (self::$folders as $folderId => $folder) { + // Skip folder types we do not support (need) + if (!empty($types) && !in_array($folder['type'], $types)) { + continue; + } + + $folder['id'] = $folderId; + $folder['total'] = count($folder['items']); + + $result[] = Folder::fromArray($folder); + } + + return $result; + } +} diff --git a/src/tests/Feature/DataMigrator/EngineTest.php b/src/tests/Feature/DataMigrator/EngineTest.php --- a/src/tests/Feature/DataMigrator/EngineTest.php +++ b/src/tests/Feature/DataMigrator/EngineTest.php @@ -4,11 +4,53 @@ use App\DataMigrator\Account; use App\DataMigrator\Engine; +use App\DataMigrator\Jobs\FolderJob; +use App\DataMigrator\Jobs\ItemJob; use App\DataMigrator\Queue as MigratorQueue; +use App\DataMigrator\Test; +use Illuminate\Support\Facades\Queue; use Tests\TestCase; class EngineTest extends TestCase { + protected $data = [ + 'Inbox' => [ + 'type' => Engine::TYPE_MAIL, + 'name' => 'Inbox', + 'fullname' => 'Inbox', + 'items' => [ + 'm1' => [], + 'm2' => [], + 'm3' => [], + 'm4' => [], + ], + 'existing_items' => [ + ], + ], + 'Contacts' => [ + 'type' => Engine::TYPE_CONTACT, + 'name' => 'Contacts', + 'fullname' => 'Contacts', + 'items' => [ + 'c1' => [], + 'c2' => [], + ], + 'existing_items' => [ + ], + ], + 'Calendar' => [ + 'type' => Engine::TYPE_EVENT, + 'name' => 'Calendar', + 'fullname' => 'Calendar', + 'items' => [ + 'e1' => [], + 'e2' => [], + ], + 'existing_items' => [ + ], + ], + ]; + /** * {@inheritDoc} */ @@ -34,6 +76,48 @@ */ public function testAsyncMigration(): void { + $source = new Account('test://test%40domain.tld:test@test'); + $destination = new Account('test://test%40kolab.org:test@test'); + $engine = new Engine(); + + Test::init($this->data); + Queue::fake(); + + // Migration initial run + $engine->migrate($source, $destination, []); + + $queue = MigratorQueue::first(); + + Queue::assertPushed(FolderJob::class, 3); + Queue::assertPushed( + FolderJob::class, + function ($job) use ($queue) { + $folder = TestCase::getObjectProperty($job, 'folder'); + return $folder->id === 'Inbox' && $folder->queueId = $queue->id; + } + ); + Queue::assertPushed( + FolderJob::class, + function ($job) use ($queue) { + $folder = TestCase::getObjectProperty($job, 'folder'); + return $folder->id === 'Contacts' && $folder->queueId = $queue->id; + } + ); + Queue::assertPushed( + FolderJob::class, + function ($job) use ($queue) { + $folder = TestCase::getObjectProperty($job, 'folder'); + return $folder->id === 'Calendar' && $folder->queueId = $queue->id; + } + ); + + $this->assertCount(0, Test::$createdFolders); + $this->assertSame(3, $queue->jobs_started); + $this->assertSame(0, $queue->jobs_finished); + $this->assertSame([], $queue->data['options']); + // TODO: Assert Source and destination in the queue + // TODO: Test 'force' option, test executing with an existing queue + // TODO: Test jobs execution $this->markTestIncomplete(); } @@ -42,6 +126,42 @@ */ public function testSyncMigration(): void { - $this->markTestIncomplete(); + $source = new Account('test://test%40domain.tld:test@test'); + $destination = new Account('test://test%40kolab.org:test@test'); + $engine = new Engine(); + + Test::init($this->data); + Queue::fake(); + + $engine->migrate($source, $destination, ['sync' => true]); + + $queue = MigratorQueue::first(); + + Queue::assertNothingPushed(); + + $this->assertSame(0, $queue->jobs_started); + $this->assertSame(0, $queue->jobs_finished); + $this->assertSame(['sync' => true], $queue->data['options']); + + $this->assertCount(3, Test::$createdFolders); + $this->assertCount(8, Test::$createdItems); + $this->assertCount(8, Test::$fetchedItems); + + Test::init($this->data); + + // Test 'type' argument + $engine->migrate($source, $destination, ['sync' => true, 'type' => 'contact,event']); + + $queue = MigratorQueue::whereNot('id', $queue->id)->first(); + + Queue::assertNothingPushed(); + + $this->assertSame(0, $queue->jobs_started); + $this->assertSame(0, $queue->jobs_finished); + $this->assertSame(['sync' => true, 'type' => 'contact,event'], $queue->data['options']); + + $this->assertCount(2, Test::$createdFolders); + $this->assertCount(4, Test::$createdItems); + $this->assertCount(4, Test::$fetchedItems); } } diff --git a/src/tests/Feature/DomainTest.php b/src/tests/Feature/DomainTest.php --- a/src/tests/Feature/DomainTest.php +++ b/src/tests/Feature/DomainTest.php @@ -138,9 +138,6 @@ $domainNamespace === $domain->namespace; } ); - - $job = new \App\Jobs\Domain\CreateJob($domain->id); - $job->handle(); } /** diff --git a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php --- a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php +++ b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php @@ -15,7 +15,7 @@ /** * Test appointment item processing */ - public function testProcessItem(): void + public function testConvertItem(): void { $account = new Account('ews://test:test@test'); $engine = new Engine(); @@ -78,7 +78,7 @@ ]); // Convert the Exchange item into iCalendar - $ical = $this->invokeMethod($appointment, 'processItem', [$item]); + $ical = $this->invokeMethod($appointment, 'convertItem', [$item]); // Parse the iCalendar output $event = new Vevent(); diff --git a/src/tests/Unit/DataMigrator/EWS/ContactTest.php b/src/tests/Unit/DataMigrator/EWS/ContactTest.php --- a/src/tests/Unit/DataMigrator/EWS/ContactTest.php +++ b/src/tests/Unit/DataMigrator/EWS/ContactTest.php @@ -15,7 +15,7 @@ /** * Test contact item processing */ - public function testProcessItem(): void + public function testConvertItem(): void { $account = new Account('ews://test:test@test'); $engine = new Engine(); @@ -113,7 +113,7 @@ ]); // Convert the Exchange item into vCard - $vcard = $this->invokeMethod($contact, 'processItem', [$item]); + $vcard = $this->invokeMethod($contact, 'convertItem', [$item]); // Parse the vCard $contact = new Vcard(); diff --git a/src/tests/Unit/DataMigrator/EWS/DistListTest.php b/src/tests/Unit/DataMigrator/EWS/DistListTest.php --- a/src/tests/Unit/DataMigrator/EWS/DistListTest.php +++ b/src/tests/Unit/DataMigrator/EWS/DistListTest.php @@ -15,7 +15,7 @@ /** * Test contact item processing */ - public function testProcessItem(): void + public function testConvertItem(): void { $account = new Account('ews://test:test@test'); $engine = new Engine(); @@ -37,6 +37,11 @@ 'LastModifiedTime' => '2024-06-27T13:44:32Z', 'DisplayName' => 'Lista', 'FileAs' => 'lista', + 'Body' => [ + 'BodyType' => 'Text', + 'IsTruncated' => false, + '_value' => 'distlist body', + ], 'Members' => (object) [ 'Member' => [ Type\MemberType::buildFromArray([ @@ -67,7 +72,7 @@ ]); // Convert the Exchange item into vCard - $vcard = $this->invokeMethod($distlist, 'processItem', [$item]); + $vcard = $this->invokeMethod($distlist, 'convertItem', [$item]); // Parse the vCard $distlist = new Vcard(); @@ -78,6 +83,7 @@ $this->assertMatchesRegularExpression('/^[a-f0-9]{40}$/', $distlist->uid); $this->assertSame('group', $distlist->kind); $this->assertSame('Lista', $distlist->fn); + $this->assertSame('distlist body', $distlist->note); $this->assertSame('Kolab EWS Data Migrator', $distlist->prodid); $this->assertSame('2024-06-27T13:44:32Z', $distlist->rev); diff --git a/src/tests/Unit/DataMigrator/EWS/TaskTest.php b/src/tests/Unit/DataMigrator/EWS/TaskTest.php --- a/src/tests/Unit/DataMigrator/EWS/TaskTest.php +++ b/src/tests/Unit/DataMigrator/EWS/TaskTest.php @@ -15,7 +15,7 @@ /** * Test task item processing */ - public function testProcessItem(): void + public function testConvertItem(): void { $source = new Account('ews://test%40domain.tld:test@test'); $destination = new Account('dav://test%40kolab.org:test@test'); @@ -118,7 +118,7 @@ ]); // Convert the Exchange item into iCalendar - $ical = $this->invokeMethod($task, 'processItem', [$item]); + $ical = $this->invokeMethod($task, 'convertItem', [$item]); // Parse the iCalendar output $task = new Vtodo(); @@ -154,7 +154,7 @@ /** * Test processing Recurrence property */ - public function testProcessItemRecurrence(): void + public function testConvertItemRecurrence(): void { $this->markTestIncomplete(); }