diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php index bc29b022..60db7dcf 100644 --- a/src/app/DataMigrator/Engine.php +++ b/src/app/DataMigrator/Engine.php @@ -1,372 +1,372 @@ source = $source; $this->destination = $destination; $this->options = $options; // Create a unique identifier for the migration request $queue_id = md5(strval($source) . strval($destination) . ($options['type'] ?? '')); // TODO: When running in 'sync' mode we shouldn't create a queue at all // If queue exists, we'll display the progress only if ($queue = Queue::find($queue_id)) { // If queue contains no jobs, assume invalid // TODO: An better API to manage (reset) queues if (!$queue->jobs_started || !empty($options['force'])) { $queue->delete(); } else { while (true) { $this->debug(sprintf("Progress [%d of %d]\n", $queue->jobs_finished, $queue->jobs_started)); if ($queue->jobs_started == $queue->jobs_finished) { break; } sleep(1); $queue->refresh(); } return; } } // Initialize the source $this->exporter = $this->initDriver($source, ExporterInterface::class); $this->exporter->authenticate(); // Initialize the destination $this->importer = $this->initDriver($destination, ImporterInterface::class); $this->importer->authenticate(); // Create a queue $this->createQueue($queue_id); // We'll store temp files in storage/ tree $location = storage_path('export/') . $source->email; if (!file_exists($location)) { mkdir($location, 0740, true); } $types = empty($options['type']) ? [] : preg_split('/\s*,\s*/', strtolower($options['type'])); $this->debug("Fetching folders hierarchy..."); $folders = $this->exporter->getFolders($types); $count = 0; $async = empty($options['sync']); - $folderMapping = $this->options['folderMapping']; + $folderMapping = $this->options['folderMapping'] ?? []; foreach ($folders as $folder) { $this->debug("Processing folder {$folder->fullname}..."); $folder->queueId = $queue_id; $folder->location = $location; // Apply name replacements $folder->targetname = $folder->fullname; foreach ($folderMapping as $key => $value) { if (str_contains($folder->targetname, $key)) { $folder->targetname = str_replace($key, $value, $folder->targetname); $this->debug("Replacing {$folder->fullname} with {$folder->targetname}"); break; } } if ($async) { // Dispatch the job (for async execution) Jobs\FolderJob::dispatch($folder); $count++; } else { $this->processFolder($folder); } } if ($count) { $this->queue->bumpJobsStarted($count); } if ($async) { $this->debug(sprintf('Done. %d %s created in queue: %s.', $count, Str::plural('job', $count), $queue_id)); } else { $this->debug(sprintf('Done (queue: %s).', $queue_id)); } } /** * Processing of a folder synchronization */ public function processFolder(Folder $folder): void { // Job processing - initialize environment if (!$this->queue) { $this->envFromQueue($folder->queueId); } // Create the folder on the destination server $this->importer->createFolder($folder); $count = 0; $async = empty($this->options['sync']); // Fetch items from the source $this->exporter->fetchItemList( $folder, function ($item_or_set) use (&$count, $async) { if ($async) { // Dispatch the job (for async execution) if ($item_or_set instanceof ItemSet) { Jobs\ItemSetJob::dispatch($item_or_set); } else { Jobs\ItemJob::dispatch($item_or_set); } $count++; } else { if ($item_or_set instanceof ItemSet) { $this->processItemSet($item_or_set); } else { $this->processItem($item_or_set); } } }, $this->importer ); if ($count) { $this->queue->bumpJobsStarted($count); } if ($async) { $this->queue->bumpJobsFinished(); } } /** * Processing of item synchronization */ public function processItem(Item $item): void { // Job processing - initialize environment if (!$this->queue) { $this->envFromQueue($item->folder->queueId); } $this->exporter->fetchItem($item); $this->importer->createItem($item); if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) { @unlink($item->filename); } if (empty($this->options['sync'])) { $this->queue->bumpJobsFinished(); } } /** * Processing of item-set synchronization */ public function processItemSet(ItemSet $set): void { // Job processing - initialize environment if (!$this->queue) { $this->envFromQueue($set->items[0]->folder->queueId); } $importItem = function (Item $item) { $this->importer->createItem($item); if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) { @unlink($item->filename); } }; // Some exporters, e.g. DAV, might optimize fetching multiple items in one go if ($this->exporter instanceof FetchItemSetInterface) { $this->exporter->fetchItemSet($set, $importItem); } else { foreach ($set->items as $item) { $this->exporter->fetchItem($item); $importItem($item); } } // TODO: We should probably also track number of items migrated if (empty($this->options['sync'])) { $this->queue->bumpJobsFinished(); } } /** * Print progress/debug information */ public function debug($line) { if (!empty($this->options['stdout'])) { $output = new \Symfony\Component\Console\Output\ConsoleOutput(); $output->writeln("$line"); } else { \Log::debug("[DataMigrator] $line"); } } /** * Get migration option value. */ public function getOption(string $name) { return $this->options[$name] ?? null; } /** * Set migration queue option. Use this if you need to pass * some data between queue processes. */ public function setOption(string $name, $value): void { $this->options[$name] = $value; if ($this->queue) { $this->queue->data = $this->queueData(); $this->queue->save(); } } /** * Create a queue for the request * * @param string $queue_id Unique queue identifier */ protected function createQueue(string $queue_id): void { $this->queue = new Queue(); $this->queue->id = $queue_id; $this->queue->data = $this->queueData(); $this->queue->save(); } /** * Prepare queue data */ protected function queueData() { $options = $this->options; unset($options['stdout']); // jobs aren't in stdout anymore // TODO: data should be encrypted return [ 'source' => (string) $this->source, 'destination' => (string) $this->destination, 'options' => $options, ]; } /** * Initialize environment for job execution * * @param string $queueId Queue identifier */ protected function envFromQueue(string $queueId): void { $this->queue = Queue::findOrFail($queueId); $this->source = new Account($this->queue->data['source']); $this->destination = new Account($this->queue->data['destination']); $this->options = $this->queue->data['options']; $this->importer = $this->initDriver($this->destination, ImporterInterface::class); $this->exporter = $this->initDriver($this->source, ExporterInterface::class); } /** * Initialize (and select) migration driver */ protected function initDriver(Account $account, string $interface) { switch ($account->scheme) { case 'ews': $driver = new EWS($account, $this); break; case 'dav': case 'davs': $driver = new DAV($account, $this); break; case 'imap': case 'imaps': case 'tls': case 'ssl': $driver = new IMAP($account, $this); break; case 'test': $driver = new Test($account, $this); break; default: throw new \Exception("Failed to init driver for '{$account->scheme}'"); } // Make sure driver is used in the direction it supports if (!is_a($driver, $interface)) { throw new \Exception(sprintf( "'%s' driver does not implement %s", class_basename($driver), class_basename($interface) )); } return $driver; } } diff --git a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php index 89cf961e..65e9e95c 100644 --- a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php +++ b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php @@ -1,113 +1,115 @@ 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $appointment = new EWS\Appointment($ews, $folder); $ical = file_get_contents(__DIR__ . '/../../../data/ews/event/1.ics'); $ical = preg_replace('/\r?\n/', "\r\n", $ical); // FIXME: I haven't found a way to convert xml content into a Type instance // therefore we create it "manually", but it would be better to have both // vcard and xml in a single data file that we could just get content from. $item = Type::buildFromArray([ 'MimeContent' => base64_encode($ical), 'ItemId' => new Type\ItemIdType( 'AAMkAGEzOGRlODRiLTBkN2ItNDgwZS04ZDJmLTM5NDEyY2Q0NGQ0OABGAAAAAAC9tlDYSlG2TaxWBr' . 'A1OzWtBwAs2ajhknXlRYN/pbC8JqblAAAAAAEOAAAs2ajhknXlRYN/pbC8JqblAAJnrWkBAAA=', 'EQAAABYAAAAs2ajhknXlRYN/pbC8JqblAAJnqlKm', ), 'UID' => '1F3C13D7E99642A75ABE23D50487B454-8FE68B2E68E1B348', 'Subject' => 'test subject', 'HasAttachments' => false, 'IsAssociated' => false, 'Start' => '2023-11-21T11:00:00Z', 'End' => '2023-11-21T11:30:00Z', 'LegacyFreeBusyStatus' => 'Tentative', 'CalendarItemType' => 'Single', 'Organizer' => [ 'Mailbox' => [ 'Name' => 'Aleksander Machniak', 'EmailAddress' => 'test@kolab.org', 'RoutingType' => 'SMTP', 'MailboxType' => 'Contact', ], ], 'RequiredAttendees' => (object) [ 'Attendee' => [ Type\AttendeeType::buildFromArray([ 'Mailbox' => [ 'Name' => 'Aleksander Machniak', 'EmailAddress' => 'test@kolab.org', 'RoutingType' => 'SMTP', 'MailboxType' => 'Contact', ], 'ResponseType' => 'Unknown', ]), Type\AttendeeType::buildFromArray([ 'Mailbox' => [ 'Name' => 'Alec Machniak', 'EmailAddress' => 'test@outlook.com', 'RoutingType' => 'SMTP', 'MailboxType' => 'Mailbox', ], 'ResponseType' => 'Unknown', ]), ], ], ]); // Convert the Exchange item into iCalendar - $ical = $this->invokeMethod($appointment, 'convertItem', [$item]); + $ical = $this->invokeMethod($appointment, 'convertItem', [$item, $targetItem]); // Parse the iCalendar output $event = new Vevent(); $this->invokeMethod($event, 'fromIcal', [$ical]); $msId = implode('!', $item->getItemId()->toArray()); $this->assertSame($msId, $event->custom['X-MS-ID']); $this->assertSame($item->getUID(), $event->uid); $this->assertSame('test description', $event->description); $this->assertSame('test subject', $event->summary); $this->assertSame('CONFIRMED', $event->status); $this->assertSame('PUBLIC', $event->class); $this->assertSame('Microsoft Exchange Server 2010', $event->prodid); $this->assertSame('2023-11-20T14:50:05+00:00', $event->dtstamp->getDateTime()->format('c')); $this->assertSame('2023-11-21T12:00:00+01:00', $event->dtstart->getDateTime()->format('c')); $this->assertSame('2023-11-21T12:30:00+01:00', $event->dtend->getDateTime()->format('c')); // Organizer/attendees $this->assertSame('test@kolab.org', $event->organizer['email']); $this->assertSame('Aleksander Machniak', $event->organizer['cn']); $this->assertSame('ORGANIZER', $event->organizer['role']); $this->assertSame('ACCEPTED', $event->organizer['partstat']); $this->assertSame(false, $event->organizer['rsvp']); $this->assertCount(1, $event->attendees); $this->assertSame('alec@outlook.com', $event->attendees[0]['email']); $this->assertSame('Alec Machniak', $event->attendees[0]['cn']); $this->assertSame('REQ-PARTICIPANT', $event->attendees[0]['role']); $this->assertSame('NEEDS-ACTION', $event->attendees[0]['partstat']); $this->assertSame(true, $event->attendees[0]['rsvp']); } } diff --git a/src/tests/Unit/DataMigrator/EWS/ContactTest.php b/src/tests/Unit/DataMigrator/EWS/ContactTest.php index ac4cb767..7d6a1d09 100644 --- a/src/tests/Unit/DataMigrator/EWS/ContactTest.php +++ b/src/tests/Unit/DataMigrator/EWS/ContactTest.php @@ -1,141 +1,143 @@ 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $contact = new EWS\Contact($ews, $folder); $vcard = file_get_contents(__DIR__ . '/../../../data/ews/contact/1.vcf'); $vcard = preg_replace('/\r?\n/', "\r\n", $vcard); // FIXME: I haven't found a way to convert xml content into a Type instance // therefore we create it "manually", but it would be better to have both // vcard and xml in a single data file that we could just get content from. $item = Type::buildFromArray([ 'MimeContent' => base64_encode($vcard), 'ItemId' => new Type\ItemIdType( 'AAMkAGEzOGRlODRiLTBkN2ItNDgwZS04ZDJmLTM5NDEyY2Q0NGQ0OABGAAAAAAC9tlDYSlG2TaxWBr' . 'A1OzWtBwAs2ajhknXlRYN/pbC8JqblAAAAAAEOAAAs2ajhknXlRYN/pbC8JqblAAJnrWkBAAA=', 'EQAAABYAAAAs2ajhknXlRYN/pbC8JqblAAJnqlKm', ), 'HasAttachments' => false, 'LastModifiedTime' => '2024-07-15T11:17:39,701Z', 'DisplayName' => 'Nowy Nazwisko', 'GivenName' => 'Nowy', 'Surname' => 'Nazwisko', 'EmailAddresses' => (object) [ 'Entry' => [ Type\EmailAddressDictionaryEntryType::buildFromArray([ 'Key' => 'EmailAddress1', 'Name' => 'test1@outlook.com', 'RoutingType' => 'SMTP', 'MailboxType' => 'Contact', '_value' => 'christian1@outlook.com', ]), Type\EmailAddressDictionaryEntryType::buildFromArray([ 'Key' => 'EmailAddress2', 'Name' => 'test2@outlook.com', 'RoutingType' => 'SMTP', 'MailboxType' => 'Contact', '_value' => 'test2@outlook.com', ]), ], ], /* ContactPicture.jpg image/jpeg 2081 2024-07-15T11:17:38 false true category en-US Mr Nowy Bartosz Nazwisko Jr. Nowy Nazwisko alec Company Testowa Warsaw mazowickie Poland 00-001 home123456 1234556679200 2014-10-11T11:59:00Z IT Developer Office Location 2020-11-12T11:59:00Z true */ ]); // Convert the Exchange item into vCard - $vcard = $this->invokeMethod($contact, 'convertItem', [$item]); + $vcard = $this->invokeMethod($contact, 'convertItem', [$item, $targetItem]); // Parse the vCard $contact = new Vcard(); $this->invokeMethod($contact, 'fromVcard', [$vcard]); $this->assertMatchesRegularExpression('/^[a-f0-9]{40}$/', $contact->uid); $this->assertSame('PUBLIC', $contact->class); $this->assertSame('Nowy Nazwisko', $contact->fn); $this->assertSame(null, $contact->kind); $this->assertSame('Microsoft Exchange', $contact->prodid); $this->assertSame('2024-07-15T11:17:39,701Z', $contact->rev); $this->assertSame('Notatki do kontaktu', $contact->note); // EWS Properties with special handling $msId = implode('!', $item->getItemId()->toArray()); $this->assertSame($msId, $contact->custom['X-MS-ID']); $this->assertSame('Partner Name', $contact->custom['X-SPOUSE']); $this->assertSame('2020-11-12', $contact->custom['X-ANNIVERSARY']); $this->assertCount(2, $contact->email); $this->assertSame('internet', $contact->email[0]['type']); $this->assertSame('christian1@outlook.com', $contact->email[0]['email']); $this->assertSame('internet', $contact->email[1]['type']); $this->assertSame('test2@outlook.com', $contact->email[1]['email']); } } diff --git a/src/tests/Unit/DataMigrator/EWS/DistListTest.php b/src/tests/Unit/DataMigrator/EWS/DistListTest.php index 5665e38f..c9260a77 100644 --- a/src/tests/Unit/DataMigrator/EWS/DistListTest.php +++ b/src/tests/Unit/DataMigrator/EWS/DistListTest.php @@ -1,96 +1,98 @@ 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $distlist = new EWS\DistList($ews, $folder); // FIXME: I haven't found a way to convert xml content into a Type instance // therefore we create it "manually", but it would be better to have both // vcard and xml in a single data file that we could just get content from. $item = Type::buildFromArray([ 'ItemId' => new Type\ItemIdType( 'AAMkAGEzOGRlODRiLTBkN2ItNDgwZS04ZDJmLTM5NDEyY2Q0NGQ0OABGAAAAAAC9tlDYSlG2TaxWBr' . 'A1OzWtBwAs2ajhknXlRYN/pbC8JqblAAAAAAEOAAAs2ajhknXlRYN/pbC8JqblAAJnrWkBAAA=', 'EQAAABYAAAAs2ajhknXlRYN/pbC8JqblAAJnqlKm', ), 'Subject' => 'subject list', 'LastModifiedTime' => '2024-06-27T13:44:32Z', 'DisplayName' => 'Lista', 'FileAs' => 'lista', 'Body' => [ 'BodyType' => 'Text', 'IsTruncated' => false, '_value' => 'distlist body', ], 'Members' => (object) [ 'Member' => [ Type\MemberType::buildFromArray([ 'Key' => 'AAAAAIErH6S+oxAZnW4A3QEPVAIAAAGAYQBsAGUAYwBAAGEAbABlAGMALgBw' . 'AGwAAABTAE0AVABQAAAAYQBsAGUAYwBAAGEAbABlAGMALgBwAGwAAAA=', 'Mailbox' => Type\Mailbox::buildFromArray([ 'Name' => 'Alec', 'EmailAddress' => 'alec@kolab.org', 'RoutingType' => 'SMTP', 'MailboxType' => 'OneOff', ]), 'Status' => 'Normal', ]), Type\MemberType::buildFromArray([ 'Key' => 'AAAAAIErH6S+oxAZnW4A3QEPVAIAAAGAYQBsAGUAYwBAAGEAbABlAGMALgBw' . 'AGwAAABTAE0AVABQAAAAYQBsAGUAYwBAAGEAbABlAGMALgBwAGwAAAB=', 'Mailbox' => Type\Mailbox::buildFromArray([ 'Name' => 'Christian', 'EmailAddress' => 'christian@kolab.org', 'RoutingType' => 'SMTP', 'MailboxType' => 'OneOff', 'ItemId' => new Type\ItemIdType('AAA', 'BBB'), ]), 'Status' => 'Normal', ]), ], ], ]); // Convert the Exchange item into vCard - $vcard = $this->invokeMethod($distlist, 'convertItem', [$item]); + $vcard = $this->invokeMethod($distlist, 'convertItem', [$item, $targetItem]); // Parse the vCard $distlist = new Vcard(); $this->invokeMethod($distlist, 'fromVcard', [$vcard]); $msId = implode('!', $item->getItemId()->toArray()); $this->assertSame(['X-MS-ID' => $msId], $distlist->custom); $this->assertMatchesRegularExpression('/^[a-f0-9]{40}$/', $distlist->uid); $this->assertSame('group', $distlist->kind); $this->assertSame('Lista', $distlist->fn); $this->assertSame('distlist body', $distlist->note); $this->assertSame('Kolab EWS Data Migrator', $distlist->prodid); $this->assertSame('2024-06-27T13:44:32Z', $distlist->rev); $members = [ 'mailto:%22Alec%22+%3Calec%40kolab.org%3E', 'urn:uuid:' . sha1('AAA'), ]; $this->assertSame($members, $distlist->member); } } diff --git a/src/tests/Unit/DataMigrator/EWS/TaskTest.php b/src/tests/Unit/DataMigrator/EWS/TaskTest.php index 4d3d120a..5dc14a1b 100644 --- a/src/tests/Unit/DataMigrator/EWS/TaskTest.php +++ b/src/tests/Unit/DataMigrator/EWS/TaskTest.php @@ -1,161 +1,163 @@ source = $source; $engine->destination = $destination; $ews = new EWS($source, $engine); $folder = Folder::fromArray(['id' => 'test']); + $targetItem = Item::fromArray(['id' => 'test']); $task = new EWS\Task($ews, $folder); // FIXME: I haven't found a way to convert xml content into a Type instance // therefore we create it "manually", but it would be better to have it in XML. $html = '' . '
task notes
'; $item = Type\TaskType::buildFromArray([ 'ItemId' => new Type\ItemIdType( 'AAMkAGEzOGRlODRiLTBkN2ItNDgwZS04ZDJmLTM5NDEyY2Q0NGQ0OABGAAAAAAC9tlDYSlG2TaxWBr' . 'A1OzWtBwAs2ajhknXlRYN/pbC8JqblAAAAAAEOAAAs2ajhknXlRYN/pbC8JqblAAJnrWkBAAA=', 'EQAAABYAAAAs2ajhknXlRYN/pbC8JqblAAJnqlKm', ), 'ItemClass' => 'IPM.Task', 'Subject' => 'Nowe zadanie', 'LastModifiedTime' => '2024-06-27T13:44:32Z', 'Sensitivity' => 'Private', // TODO: Looks like EWS has Body->IsTruncated property, but is it relevant? 'Body' => new Type\BodyType($html, 'HTML'), 'Importance' => 'High', 'DateTimeCreated' => '2024-06-27T08:58:05Z', 'ReminderDueBy' => '2024-07-17T07:00:00Z', 'ReminderIsSet' => true, 'ReminderNextTime' => '2024-07-17T07:00:00Z', 'ReminderMinutesBeforeStart' => '0', 'DueDate' => '2024-06-26T22:00:00Z', 'IsComplete' => false, 'IsRecurring' => true, 'Owner' => 'Alec Machniak', 'PercentComplete' => '10', 'Recurrence' => Type\TaskRecurrenceType::buildFromArray([ 'WeeklyRecurrence' => [ 'Interval' => '1', 'DaysOfWeek' => 'Thursday', 'FirstDayOfWeek' => 'Sunday', ], 'NoEndRecurrence' => [ 'StartDate' => '2024-06-27Z', ], ]), 'Status' => 'NotStarted', 'ChangeCount' => '2', /* testdisk.log application/octet-stream 299368b3-06e4-42df-959e-d428046f55e6 249 2024-07-16T12:13:58 false false 2024-06-27T08:58:05Z 3041 Kategoria Niebieski false false false false false 2024-06-27T08:58:05Z true en-US false false false true true true true Alec Machniak 2024-07-16T12:14:38Z false NotFlagged AQEAAAAAAAESAQAAAmjiC08AAAAA 1 Not Started */ ]); // Convert the Exchange item into iCalendar - $ical = $this->invokeMethod($task, 'convertItem', [$item]); + $ical = $this->invokeMethod($task, 'convertItem', [$item, $targetItem]); // Parse the iCalendar output $task = new Vtodo(); $this->invokeMethod($task, 'fromIcal', [$ical]); $msId = implode('!', $item->getItemId()->toArray()); $this->assertSame($msId, $task->custom['X-MS-ID']); $this->assertMatchesRegularExpression('/^[a-f0-9]{40}$/', $task->uid); $this->assertSame('Nowe zadanie', $task->summary); $this->assertSame($html, $task->description); $this->assertSame('Kolab EWS Data Migrator', $task->prodid); $this->assertSame('2', $task->sequence); $this->assertSame('9', $task->priority); $this->assertSame('PRIVATE', $task->class); $this->assertSame(10, $task->percentComplete); $this->assertSame('X-NOTSTARTED', $task->status); $this->assertSame('2024-06-27T13:44:32+00:00', $task->dtstamp->getDateTime()->format('c')); $this->assertSame('2024-06-27T08:58:05+00:00', $task->created->getDateTime()->format('c')); $this->assertSame('2024-06-26T22:00:00+00:00', $task->due->getDateTime()->format('c')); $this->assertSame('test@kolab.org', $task->organizer['email']); $this->assertSame('WEEKLY', $task->rrule['freq']); $this->assertSame('1', $task->rrule['interval']); $this->assertSame('TH', $task->rrule['byday']); $this->assertSame('SU', $task->rrule['wkst']); $this->assertCount(1, $task->valarms); $this->assertCount(2, $task->valarms[0]); $this->assertSame('DISPLAY', $task->valarms[0]['action']); $this->assertSame('2024-07-17T07:00:00+00:00', $task->valarms[0]['trigger']->format('c')); } /** * Test processing Recurrence property */ public function testConvertItemRecurrence(): void { $this->markTestIncomplete(); } }