diff --git a/src/app/Console/Commands/UserMigrate.php b/src/app/Console/Commands/UserMigrate.php index 07d0d0cb..fa37a0af 100644 --- a/src/app/Console/Commands/UserMigrate.php +++ b/src/app/Console/Commands/UserMigrate.php @@ -1,52 +1,53 @@ argument('src')); $dst = new DataMigrator\Account($this->argument('dst')); DataMigrator::migrate($src, $dst, $this->options()); } } diff --git a/src/app/DataMigrator/Account.php b/src/app/DataMigrator/Account.php index 70c9fc0e..be1e3fee 100644 --- a/src/app/DataMigrator/Account.php +++ b/src/app/DataMigrator/Account.php @@ -1,66 +1,82 @@ :". * For proxy authentication use: "**" as username. * * @param string $input Account specification */ public function __construct(string $input) { $url = parse_url($input); // Not valid URI, try the other form of input if ($url === false || !array_key_exists('scheme', $url)) { list($user, $password) = explode(':', $input, 2); $url = ['user' => $user, 'pass' => $password]; } if (isset($url['user'])) { $this->username = urldecode($url['user']); if (strpos($this->username, '**')) { list($this->username, $this->loginas) = explode('**', $this->username, 2); } } if (isset($url['pass'])) { $this->password = urldecode($url['pass']); } if (isset($url['host'])) { $this->uri = preg_replace('/\?.*$/', '', $input); } if (strpos($this->loginas, '@')) { $this->email = $this->loginas; } elseif (strpos($this->username, '@')) { $this->email = $this->username; } + + $this->input = $input; + } + + /** + * Returns string representation of the object. + * You can use the result as an input to the object constructor. + * + * @return string Account string representation + */ + public function __toString(): string + { + return $this->input; } } diff --git a/src/app/DataMigrator/DAVClient.php b/src/app/DataMigrator/DAVClient.php index 09073ae3..43134eac 100644 --- a/src/app/DataMigrator/DAVClient.php +++ b/src/app/DataMigrator/DAVClient.php @@ -1,234 +1,248 @@ username . ($account->loginas ? "**{$account->loginas}" : ''); $this->settings = [ 'baseUri' => rtrim($account->uri, '/') . '/', 'userName' => $username, 'password' => $account->password, 'authType' => Client::AUTH_BASIC, ]; $this->client = new Client($this->settings); } + /** + * Check user credentials. + * + * @throws Exception + */ + public function authenticate() + { + $result = $this->client->options(); + + if (empty($result)) { + throw new Exception("Invalid DAV credentials or server."); + } + } + /** * Create an object. * * @param string $filename File location * @param array $folder Folder name * * @throws Exception */ public function createObjectFromFile(string $filename, string $folder) { $data = fopen($filename, 'r'); /* // Need to tell Curl the attachments size, so it properly // sets Content-Length header, that is required in PUT // request by some webdav servers (#2978) $stat = fstat($data); $this->client->addCurlSetting(CURLOPT_INFILESIZE, $stat['size']); */ $path = $this->getFolderPath($folder) . '/' . pathinfo($filename, PATHINFO_BASENAME); $response = $this->client->request('PUT', $path, $data); fclose($data); if ($response['statusCode'] != 201) { throw new \Exception("Storage error. " . $response['body']); } } /** * Create a folder. * * @param string $folder Name of a folder with full path * @param string $type Folder type * * @throws Exception on error */ public function createFolder(string $folder, string $type) { $folders = $this->listFolders(); if (array_key_exists($folder, $folders)) { // do nothing, folder already exists return; } $types = [['DAV:', 'collection']]; $prefix = ''; if ($type == self::TYPE_CONTACT) { $types[] = ['urn:ietf:params:xml:ns:carddav', 'addressbook']; $prefix = 'addressbooks/' . urlencode($this->settings['userName']) . '/'; } elseif ($type == self::TYPE_EVENT || $type == self::TYPE_TASK) { $types[] = ['urn:ietf:params:xml:ns:caldav', 'calendar']; $prefix = 'calendars/' . urlencode($this->settings['userName']) . '/'; } // Create XML request $xml = new \DOMDocument('1.0', 'UTF-8'); $xml->formatOutput = true; $root = $xml->createElementNS('DAV:', 'mkcol'); $set = $xml->createElementNS('DAV:', 'set'); $prop = $xml->createElementNS('DAV:', 'prop'); // Folder display name $prop->appendChild($xml->createElementNS('DAV:', 'displayname', $folder)); // Folder type $resource_type = $xml->createElementNS('DAV:', 'resourcetype'); foreach ($types as $rt) { $resource_type->appendChild($xml->createElementNS($rt[0], $rt[1])); } $prop->appendChild($resource_type); if ($type == self::TYPE_TASK) { // Extra property needed for task folders $cset = $xml->createElementNS('urn:ietf:params:xml:ns:caldav', 'supported-calendar-component-set'); $comp = $xml->createElementNS('urn:ietf:params:xml:ns:caldav', 'comp'); $comp->setAttribute('name', $type == self::TYPE_TASK ? 'VTODO' : 'VEVENT'); $cset->appendChild($comp); $prop->appendChild($cset); } $xml->appendChild($root)->appendChild($set)->appendChild($prop); $body = $xml->saveXML(); $folder_id = Utils::uuidStr(); $path = $prefix . $folder_id; // Send the request $response = $this->client->request('MKCOL', $path, $body, ['Content-Type' => 'text/xml']); if ($response['statusCode'] != 201) { throw new \Exception("Storage error: " . $response['body']); } $this->folders[$folder] = [ 'id' => $folder_id, 'type' => $type, ]; } /** * Returns list of folders. * * @return array List of folders */ public function listFolders(): array { if ($this->folders !== null) { return $this->folders; } $request = [ '{DAV:}displayname', '{DAV:}resourcetype', '{urn:ietf:params:xml:ns:caldav}supported-calendar-component-set', ]; // Get addressbook folders $root = 'addressbooks/' . urlencode($this->settings['userName']); $collections = $this->client->propFind($root, $request, 1); // Get calendar and task folders $root = 'calendars/' . urlencode($this->settings['userName']); $calendars = $this->client->propFind($root, $request, 'infinity'); $collections = array_merge($collections, $calendars); $this->folders = []; foreach ($collections as $key => $props) { if ($type = $this->collectionType($props)) { $path = explode('/', rtrim($key, '/')); $id = $path[count($path)-1]; // Note that in CalDAV/CardDAV folder names directly the same is in IMAP // especially talking about shared/other users folders $name = $props['{DAV:}displayname']; $name = str_replace(' ยป ', '/', $name); $this->folders[$name] = [ 'id' => $id, 'type' => $type, ]; } } return $this->folders; } /** * Detect folder type from collection properties. * Special collections (ldap addressbooks, calendar inbox/outbox) will be ignored */ protected function collectionType(array $props) { if ($props['{DAV:}resourcetype']->is('{urn:ietf:params:xml:ns:caldav}calendar')) { foreach ($props['{urn:ietf:params:xml:ns:caldav}supported-calendar-component-set'] as $set) { if (isset($set['attributes']) && isset($set['attributes']['name']) && $set['attributes']['name'] === 'VTODO') { return self::TYPE_TASK; } } return self::TYPE_EVENT; } if ($props['{DAV:}resourcetype']->is('{urn:ietf:params:xml:ns:carddav}addressbook') && !$props['{DAV:}resourcetype']->is('{urn:ietf:params:xml:ns:carddav}directory') ) { return self::TYPE_CONTACT; } } /** * Get folder relative URI */ protected function getFolderPath(string $folder): string { $folders = $this->listFolders(); if (array_key_exists($folder, $folders)) { $data = $folders[$folder]; if ($data['type'] == self::TYPE_CONTACT) { return 'addressbooks/' . urlencode($this->settings['userName']) . '/' . urlencode($data['id']); } if ($data['type'] == self::TYPE_EVENT || $data['type'] == self::TYPE_TASK) { return 'calendars/' . urlencode($this->settings['userName']) . '/' . urlencode($data['id']); } } throw new \Exception("Folder not found: {$folder}"); } } diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php index f89633ee..4b15a16e 100644 --- a/src/app/DataMigrator/EWS.php +++ b/src/app/DataMigrator/EWS.php @@ -1,353 +1,469 @@ DAVClient::TYPE_EVENT, EWS\Contact::FOLDER_TYPE => DAVClient::TYPE_CONTACT, EWS\Task::FOLDER_TYPE => DAVClient::TYPE_TASK, ]; /** @var string Output location */ protected $location; /** @var Account Source account */ protected $source; /** @var Account Destination account */ protected $destination; /** @var array Migration options */ protected $options = []; /** @var DAVClient Data importer */ protected $importer; + /** @var \App\DataMigratorQueue Migrator jobs queue */ + protected $queue; + + /** @var array EWS server setup (after autodiscovery) */ + protected $ews = []; + /** * Print progress/debug information */ public function debug($line) { // TODO: When not in console mode we should // not write to stdout, but to log $output = new \Symfony\Component\Console\Output\ConsoleOutput; $output->writeln($line); } /** * Return destination account */ public function getDestination() { return $this->destination; } /** * Return source account */ public function getSource() { return $this->source; } /** * Execute migration for the specified user */ public function migrate(Account $source, Account $destination, array $options = []): void { + // Create a unique identifier for the migration request + $queue_id = md5(strval($source).strval($destination).$options['type']); + + // If queue exists, we'll display the progress only + if ($queue = DataMigratorQueue::find($queue_id)) { + // If queue contains no jobs, assume invalid + // TODO: An better API to manage (reset) queues + if (!$queue->jobs_started || !empty($options['force'])) { + $queue->delete(); + } else { + while (true) { + printf("Progress [%d of %d]\n", $queue->jobs_finished, $queue->jobs_started); + + if ($queue->jobs_started == $queue->jobs_finished) { + break; + } + + sleep(1); + $queue->refresh(); + } + + return; + } + } + $this->source = $source; $this->destination = $destination; $this->options = $options; // We'll store output in storage/ tree $this->location = storage_path('export/') . $source->email; if (!file_exists($this->location)) { mkdir($this->location, 0740, true); } // Autodiscover and authenticate the user $this->authenticate($source->username, $source->password, $source->loginas); - $this->debug("Logged in. Fetching folders hierarchy..."); + // Also check user credentials for Kolab destination + $this->importer = new DAVClient($destination); + $this->importer->authenticate(); + + $this->debug("Source/destination user credentials verified."); + $this->debug("Fetching folders hierarchy..."); + + // Create a queue + $this->createQueue($queue_id); $folders = $this->getFolders(); + $count = 0; - if (empty($options['import-only'])) { - foreach ($folders as $folder) { - $this->debug("Syncing folder {$folder['fullname']}..."); + foreach ($folders as $folder) { + // Only supported folder types + if ($folder['type']) { + $this->debug("Processing folder {$folder['fullname']}..."); - if ($folder['total'] > 0) { - $this->syncItems($folder); - } + // Dispatch the job (for async execution) + DataMigratorEWSFolder::dispatch($folder); + $count++; } - - $this->debug("Done."); } - if (empty($options['export-only'])) { - $this->debug("Importing to Kolab account..."); - - $this->importer = new DAVClient($destination); + $this->queue->bumpJobsStarted($count); - // TODO: If we were to stay with this storage solution and need still - // the import mode, it should not require connecting again to - // Exchange. Now we do this for simplicity. - foreach ($folders as $folder) { - $this->debug("Syncing folder {$folder['fullname']}..."); - - if (empty($folder['type'])) { - // skip unsupported object type (e.g. mail) for now - continue; - } - - $this->importer->createFolder($folder['fullname'], $folder['type']); - - if ($folder['total'] > 0) { - $files = array_diff(scandir($folder['location']), ['.', '..']); - foreach ($files as $file) { - $this->debug("* Pushing item {$file}..."); - $this->importer->createObjectFromFile($folder['location'] . '/' . $file, $folder['fullname']); - // TODO: remove the file/folder? - } - } - } - - $this->debug("Done."); - } + $this->debug("Done. {$count} jobs created in queue: {$queue_id}."); } /** * Autodiscover the server and authenticate the user */ protected function authenticate(string $user, string $password, string $loginas = null): void { // You should never run the Autodiscover more than once. // It can make between 1 and 5 calls before giving up, or before finding your server, // depending on how many different attempts it needs to make. + // TODO: After 2020-10-13 EWS at Office365 will require OAuth + $api = API\ExchangeAutodiscover::getAPI($user, $password); $server = $api->getClient()->getServer(); $version = $api->getClient()->getVersion(); $options = ['version' => $version]; if ($loginas) { $options['impersonation'] = $loginas; } $this->debug("Connected to $server ($version). Authenticating..."); + $this->ews = [ + 'options' => $options, + 'server' => $server, + ]; + $this->api = API::withUsernameAndPassword($server, $user, $password, $options); } /** * Get folders hierarchy */ protected function getFolders(): array { // Folder types we're ineterested in $folder_classes = $this->folderClasses(); // Get full folders hierarchy $options = [ 'Traversal' => 'Deep', ]; $folders = $this->api->getChildrenFolders('root', $options); $result = []; foreach ($folders as $folder) { $class = $folder->getFolderClass(); // Skip folder types we do not support if (!in_array($class, $folder_classes)) { continue; } $name = $fullname = $folder->getDisplayName(); $id = $folder->getFolderId()->getId(); $parentId = $folder->getParentFolderId()->getId(); // Create folder name with full path if ($parentId && !empty($result[$parentId])) { $fullname = $result[$parentId]['fullname'] . '/' . $name; } // Top-level folder, check if it's a special folder we should ignore // FIXME: Is there a better way to distinguish user folders from system ones? if (in_array($fullname, $this->folder_exceptions) || strpos($fullname, 'OwaFV15.1All') === 0 ) { continue; } $result[$id] = [ 'id' => $folder->getFolderId(), 'total' => $folder->getTotalCount(), 'class' => $class, 'type' => array_key_exists($class, $this->type_map) ? $this->type_map[$class] : null, 'name' => $name, 'fullname' => $fullname, 'location' => $this->location . '/' . $fullname, + 'queue_id' => $this->queue->id, ]; } return $result; } /** - * Synchronize specified folder + * Processing of a folder synchronization */ - protected function syncItems(array $folder): void + public function processFolder(array $folder): void { + // Job processing - initialize environment + if (!empty($folder['queue_id'])) { + $this->initEnv($folder['queue_id']); + } + + // Create the folder on destination server + $this->importer->createFolder($folder['fullname'], $folder['type']); + + // The folder is empty, we can stop here + if (empty($folder['total'])) { + $this->queue->bumpJobsFinished(); + return; + } + $request = [ // Exchange's maximum is 1000 'IndexedPageItemView' => ['MaxEntriesReturned' => 100, 'Offset' => 0, 'BasePoint' => 'Beginning'], 'ParentFolderIds' => $folder['id']->toArray(true), 'Traversal' => 'Shallow', 'ItemShape' => [ 'BaseShape' => 'IdOnly', 'AdditionalProperties' => [ 'FieldURI' => ['FieldURI' => 'item:ItemClass'], ], ], ]; $request = Type::buildFromArray($request); // Note: It is not possible to get mimeContent with FindItem request // That's why we first get the list of object identifiers and // then call GetItem on each separately. // TODO: It might be feasible to get all properties for object types // for which we don't use MimeContent, for better performance. // Request first page $response = $this->api->getClient()->FindItem($request); + $count = 0; foreach ($response as $item) { - $this->syncItem($item, $folder); + $count += (int) $this->syncItem($item, $folder); } + $this->queue->bumpJobsStarted($count); + // Request other pages until we got all while (!$response->isIncludesLastItemInRange()) { $response = $this->api->getNextPage($response); + $count = 0; + foreach ($response as $item) { - $this->syncItem($item, $folder); + $count += (int) $this->syncItem($item, $folder); + } + + $this->queue->bumpJobsStarted($count); + } + + $this->queue->bumpJobsFinished(); + } + + /** + * Processing of item synchronization + */ + public function processItem(array $item): void + { + // Job processing - initialize environment + if (!empty($item['queue_id'])) { + $this->initEnv($item['queue_id']); + } + + if ($driver = EWS\Item::factory($this, $item['item'], $item)) { + if ($file = $driver->syncItem($item['item'])) { + $this->importer->createObjectFromFile($file, $item['fullname']); + // TODO: remove the file } } + + $this->queue->bumpJobsFinished(); } /** * Synchronize specified object */ - protected function syncItem(Type $item, array $folder): void + protected function syncItem(Type $item, array $folder): bool { if ($driver = EWS\Item::factory($this, $item, $folder)) { - $driver->syncItem($item); - return; + // TODO: This object could probably be streamlined down to save some space + // All we need is item ID and class. + $folder['item'] = $item; + + // Dispatch the job (for async execution) + DataMigratorEWSItem::dispatch($folder); + + return true; } // TODO IPM.Note (email) and IPM.StickyNote // Note: iTip messages in mail folders may have different class assigned // https://docs.microsoft.com/en-us/office/vba/outlook/Concepts/Forms/item-types-and-message-classes $this->debug("Unsupported object type: {$item->getItemClass()}. Skiped."); + + return false; } /** * Return list of folder classes for current migrate operation */ protected function folderClasses(): array { if (!empty($this->options['type'])) { $types = preg_split('/\s*,\s*/', strtolower($this->options['type'])); $result = []; foreach ($types as $type) { switch ($type) { case 'event': $result[] = EWS\Appointment::FOLDER_TYPE; break; case 'contact': $result[] = EWS\Contact::FOLDER_TYPE; break; case 'task': $result[] = EWS\Task::FOLDER_TYPE; break; /* case 'note': $result[] = EWS\StickyNote::FOLDER_TYPE; break; */ case 'mail': $result[] = EWS\Note::FOLDER_TYPE; break; default: throw new \Exception("Unsupported type: {$type}"); } } return $result; } return $this->folder_classes; } + + /** + * Create a queue for the request + * + * @param string $queue_id Unique queue identifier + */ + protected function createQueue(string $queue_id): void + { + $this->queue = new DataMigratorQueue; + $this->queue->id = $queue_id; + + // TODO: data should be encrypted + $this->queue->data = [ + 'source' => (string) $this->source, + 'destination' => (string) $this->destination, + 'options' => $this->options, + 'ews' => $this->ews, + ]; + + $this->queue->save(); + } + + /** + * Initialize environment for job execution + * + * @param string $queue_id Queue identifier + */ + protected function initEnv(string $queue_id): void + { + $this->queue = DataMigratorQueue::findOrFail($queue_id); + $this->source = new Account($this->queue->data['source']); + $this->destination = new Account($this->queue->data['destination']); + $this->options = $this->queue->data['options']; + $this->importer = new DAVClient($this->destination); + $this->api = API::withUsernameAndPassword( + $this->queue->data['ews']['server'], + $this->source->username, + $this->source->password, + $this->queue->data['ews']['options'] + ); + } } diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php index a2e154bd..71986a85 100644 --- a/src/app/DataMigrator/EWS/Item.php +++ b/src/app/DataMigrator/EWS/Item.php @@ -1,152 +1,154 @@ engine = $engine; $this->folder = $folder; } /** * Factory method. * Returns object suitable to handle specified item type. */ public static function factory(EWS $engine, Type $item, array $folder) { $item_class = str_replace('IPM.', '', $item->getItemClass()); $item_class = "\App\DataMigrator\EWS\\{$item_class}"; if (class_exists($item_class)) { return new $item_class($engine, $folder); } } /** * Synchronize specified object */ - public function syncItem(Type $item): void + public function syncItem(Type $item) { // Fetch the item $item = $this->engine->api->getItem($item->getItemId(), $this->getItemRequest()); $uid = $this->getUID($item); $this->engine->debug("* Saving item {$uid}..."); // Apply type-specific format converters if ($this->processItem($item) === false) { return; } $uid = preg_replace('/[^a-zA-Z0-9_:@-]/', '', $uid); $location = $this->folder['location']; if (!file_exists($location)) { mkdir($location, 0740, true); } $location .= '/' . $uid . '.' . $this::FILE_EXT; file_put_contents($location, (string) $item->getMimeContent()); + + return $location; } /** * Item conversion code */ abstract protected function processItem(Type $item): bool; /** * Get GetItem request parameters */ protected function getItemRequest(): array { $request = [ 'ItemShape' => [ // Reqest default set of properties 'BaseShape' => 'Default', // Additional properties, e.g. LastModifiedTime // FIXME: How to add multiple properties here? 'AdditionalProperties' => [ 'FieldURI' => ['FieldURI' => 'item:LastModifiedTime'], ] ] ]; return $request; } /** * Fetch attachment object from Exchange */ protected function getAttachment(Type\FileAttachmentType $attachment) { $request = [ 'AttachmentIds' => [ $attachment->getAttachmentId()->toXmlObject() ], 'AttachmentShape' => [ 'IncludeMimeContent' => true, ] ]; return $this->engine->api->getClient()->GetAttachment($request); } /** * Get Item UID (Generate a new one if needed) */ protected function getUID(Type $item): string { if ($this->uid === null) { // We should generate an UID for objects that do not have it // and inject it into the output file // FIXME: Should we use e.g. md5($itemId->getId()) instead? $this->uid = \App\Utils::uuidStr(); } return $this->uid; } /** * VCard/iCal property formatting */ protected function formatProp($name, $value, array $params = []): string { $cal = new \Sabre\VObject\Component\VCalendar(); $prop = new \Sabre\VObject\Property\Text($cal, $name, $value, $params); $value = $prop->serialize(); // Revert escaping for some props if ($name == 'RRULE') { $value = str_replace("\\", '', $value); } return $value; } } diff --git a/src/app/DataMigratorQueue.php b/src/app/DataMigratorQueue.php new file mode 100644 index 00000000..d520cc16 --- /dev/null +++ b/src/app/DataMigratorQueue.php @@ -0,0 +1,82 @@ + 'array']; + + /** + * The model's default values for attributes. + * + * @var array + */ + protected $attributes = [ + 'jobs_started' => 0, + 'jobs_finished' => 0, + 'data' => '', // must not be [] + ]; + + + /** + * Fast and race-condition free method of bumping the jobs_started value + */ + public function bumpJobsStarted(int $num = null) + { + DB::update( + "update data_migrator_queues set jobs_started = jobs_started + ? where id = ?", + [$num ?: 1, $this->id] + ); + } + + /** + * Fast and race-condition free method of bumping the jobs_finished value + */ + public function bumpJobsFinished(int $num = null) + { + DB::update( + "update data_migrator_queues set jobs_finished = jobs_finished + ? where id = ?", + [$num ?: 1, $this->id] + ); + } +} diff --git a/src/app/Jobs/DataMigratorEWSFolder.php b/src/app/Jobs/DataMigratorEWSFolder.php new file mode 100644 index 00000000..6ead37e5 --- /dev/null +++ b/src/app/Jobs/DataMigratorEWSFolder.php @@ -0,0 +1,19 @@ +processFolder($this->data); + } +} diff --git a/src/app/Jobs/DataMigratorEWSItem.php b/src/app/Jobs/DataMigratorEWSItem.php new file mode 100644 index 00000000..d0e956dc --- /dev/null +++ b/src/app/Jobs/DataMigratorEWSItem.php @@ -0,0 +1,61 @@ +data = $data; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $ews = new EWS; + $ews->processItem($this->data); + } + + /** + * The job failed to process. + * + * @param Exception $exception + * + * @return void + */ + public function failed(Exception $exception) + { + // TODO: Count failed jobs in the queue + // I'm not sure how to do this after the final failure (after X tries) + // In other words how do we know all jobs in a queue finished (successfully or not) + // Probably we have to set $tries = 1 + } +} diff --git a/src/database/migrations/2019_11_21_110000_create_data_migrator_queues_table.php b/src/database/migrations/2019_11_21_110000_create_data_migrator_queues_table.php new file mode 100644 index 00000000..e7b90194 --- /dev/null +++ b/src/database/migrations/2019_11_21_110000_create_data_migrator_queues_table.php @@ -0,0 +1,35 @@ +string('id', 32); + $table->integer('jobs_started'); + $table->integer('jobs_finished'); + $table->text('data'); + $table->timestamps(); + + $table->primary('id'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::drop('data_migrator_queues'); + } +}