diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php index e7133acc..0799aa91 100644 --- a/src/app/DataMigrator/DAV.php +++ b/src/app/DataMigrator/DAV.php @@ -1,423 +1,429 @@ username . ($account->loginas ? "**{$account->loginas}" : ''); $baseUri = rtrim($account->uri, '/'); $baseUri = preg_replace('|^dav|', 'http', $baseUri); $this->client = new DAVClient($username, $account->password, $baseUri); $this->engine = $engine; $this->account = $account; } /** * Check user credentials. * * @throws \Exception */ public function authenticate(): void { try { $this->client->options(); } catch (\Exception $e) { throw new \Exception("Invalid DAV credentials or server."); } } /** * Create an item in a folder. * * @param Item $item Item to import * * @throws \Exception */ public function createItem(Item $item): void { $is_file = false; $href = $item->existing ?: null; if (strlen($item->content)) { $content = $item->content; } else { $content = $item->filename; $is_file = true; } if (empty($href)) { $href = $this->getFolderPath($item->folder) . '/' . basename($item->filename); } $object = new DAVOpaque($content, $is_file); $object->href = $href; switch ($item->folder->type) { case Engine::TYPE_EVENT: case Engine::TYPE_TASK: $object->contentType = 'text/calendar; charset=utf-8'; break; case Engine::TYPE_CONTACT: $object->contentType = 'text/vcard; charset=utf-8'; break; } if ($this->client->create($object) === false) { throw new \Exception("Failed to save DAV object at {$href}"); } } /** * Create a folder. * * @param Folder $folder Folder data * * @throws \Exception on error */ public function createFolder(Folder $folder): void { $dav_type = $this->type2DAV($folder->type); $folders = $this->client->listFolders($dav_type); if ($folders === false) { throw new \Exception("Failed to list folders on the DAV server"); } // Note: iRony flattens the list by modifying the folder name // This is not going to work with Cyrus DAV, but anyway folder // hierarchies support is not full in Kolab 4. foreach ($folders as $dav_folder) { if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) { // do nothing, folder already exists return; } } $home = $this->client->getHome($dav_type); $folder_id = Utils::uuidStr(); $collection_type = $dav_type == DAVClient::TYPE_VCARD ? 'addressbook' : 'calendar'; // We create all folders on the top-level $dav_folder = new DAVFolder(); $dav_folder->name = $folder->fullname; $dav_folder->href = rtrim($home, '/') . '/' . $folder_id; $dav_folder->components = [$dav_type]; $dav_folder->types = ['collection', $collection_type]; if ($this->client->folderCreate($dav_folder) === false) { throw new \Exception("Failed to create a DAV folder {$dav_folder->href}"); } } /** * Fetching an item */ public function fetchItem(Item $item): void { $result = $this->client->getObjects(dirname($item->id), $this->type2DAV($item->folder->type), [$item->id]); if ($result === false) { throw new \Exception("Failed to fetch DAV item for {$item->id}"); } // TODO: Do any content changes, e.g. organizer/attendee email migration $content = (string) $result[0]; if (strlen($content) > Engine::MAX_ITEM_SIZE) { // Save the item content to a file $location = $item->folder->tempFileLocation(basename($item->id)); if (file_put_contents($location, $content) === false) { throw new \Exception("Failed to write to {$location}"); } $item->filename = $location; } else { $item->content = $content; $item->filename = basename($item->id); } } /** * Fetch a list of folder items */ public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void { // Get existing messages' headers from the destination mailbox $existing = $importer->getItems($folder); $dav_type = $this->type2DAV($folder->type); $location = $this->getFolderPath($folder); $search = new DAVSearch($dav_type, false); + // FIXME this avoids a crash of iRony on empty collections? + // It does not request items on subfolders in iRony (I tried), but maybe that's just an iRony defect? + $search->depth = "infinity"; // TODO: We request only properties relevant to incremental migration, // i.e. to find that something exists and its last update time. // Some servers (iRony) do ignore that and return full VCARD/VEVENT/VTODO // content, if there's many objects we'll have a memory limit issue. // Also, this list should be controlled by the exporter. $search->dataProperties = ['UID', 'REV', 'DTSTAMP']; $set = new ItemSet(); $result = $this->client->search( $location, $search, function ($item) use (&$set, $dav_type, $folder, $existing, $callback) { // Skip an item that exists and did not change $exists = null; if (!empty($existing[$item->uid])) { $exists = $existing[$item->uid]['href']; switch ($dav_type) { case DAVClient::TYPE_VCARD: if ($existing[$item->uid]['rev'] == $item->rev) { return null; } break; case DAVClient::TYPE_VEVENT: case DAVClient::TYPE_VTODO: if ($existing[$item->uid]['dtstamp'] == (string) $item->dtstamp) { return null; } break; } } $set->items[] = Item::fromArray([ 'id' => $item->href, 'folder' => $folder, 'existing' => $exists, ]); if (count($set->items) == self::CHUNK_SIZE) { $callback($set); $set = new ItemSet(); } return null; } ); if (count($set->items)) { $callback($set); } if ($result === false) { throw new \Exception("Failed to get items from a DAV folder {$location}"); } // TODO: Delete items that do not exist anymore? } /** * Get a list of items, limited to their essential propeties * used in incremental migration. * * @param Folder $folder Folder data * * @throws \Exception on error */ public function getItems(Folder $folder): array { $dav_type = $this->type2DAV($folder->type); $location = $this->getFolderPath($folder); $search = new DAVSearch($dav_type); + // FIXME this avoids a crash of iRony on empty collections? + // It does not request items on subfolders in iRony (I tried), but maybe that's just an iRony defect? + $search->depth = "infinity"; // TODO: We request only properties relevant to incremental migration, // i.e. to find that something exists and its last update time. // Some servers (iRony) do ignore that and return full VCARD/VEVENT/VTODO // content, if there's many objects we'll have a memory limit issue. // Also, this list should be controlled by the exporter. $search->dataProperties = ['UID', 'X-MS-ID', 'REV', 'DTSTAMP']; $items = $this->client->search( $location, $search, function ($item) use ($dav_type) { // Slim down the result to properties we might need $object = [ 'href' => $item->href, 'uid' => $item->uid, ]; if (!empty($item->custom['X-MS-ID'])) { $object['x-ms-id'] = $item->custom['X-MS-ID']; } switch ($dav_type) { case DAVClient::TYPE_VCARD: $object['rev'] = $item->rev; break; case DAVClient::TYPE_VEVENT: case DAVClient::TYPE_VTODO: $object['dtstamp'] = (string) $item->dtstamp; break; } return [$item->uid, $object]; } ); if ($items === false) { throw new \Exception("Failed to get items from a DAV folder {$location}"); } return $items; } /** * Get folders hierarchy */ public function getFolders($types = []): array { $result = []; foreach (['VEVENT', 'VTODO', 'VCARD'] as $component) { $type = $this->typeFromDAV($component); // Skip folder types we do not support (need) if (!empty($types) && !in_array($type, $types)) { continue; } $folders = $this->client->listFolders($component); foreach ($folders as $folder) { // Skip other users/shared folders if ($this->shouldSkip($folder)) { continue; } $result[$folder->href] = Folder::fromArray([ 'fullname' => str_replace(' » ', '/', $folder->name), 'href' => $folder->href, 'type' => $type, ]); } } return $result; } /** * Get folder relative URI */ protected function getFolderPath(Folder $folder): string { $cache_key = $folder->type . '!' . $folder->fullname; if (isset($this->folderPaths[$cache_key])) { return $this->folderPaths[$cache_key]; } for ($i = 0; $i < 5; $i++) { $folders = $this->client->listFolders($this->type2DAV($folder->type)); if ($folders === false) { throw new \Exception("Failed to list folders on the DAV server"); } // Note: iRony flattens the list by modifying the folder name // This is not going to work with Cyrus DAV, but anyway folder // hierarchies support is not full in Kolab 4. foreach ($folders as $dav_folder) { if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) { return $this->folderPaths[$cache_key] = rtrim($dav_folder->href, '/'); } } sleep(1); } throw new \Exception("Folder not found: {$folder->fullname}"); } /** * Map Kolab type into DAV object type */ protected static function type2DAV(string $type): string { switch ($type) { case Engine::TYPE_EVENT: return DAVClient::TYPE_VEVENT; case Engine::TYPE_TASK: return DAVClient::TYPE_VTODO; case Engine::TYPE_CONTACT: case Engine::TYPE_GROUP: return DAVClient::TYPE_VCARD; default: throw new \Exception("Cannot map type '{$type}' to DAV"); } } /** * Map DAV object type into Kolab type */ protected static function typeFromDAV(string $type): string { switch ($type) { case DAVClient::TYPE_VEVENT: return Engine::TYPE_EVENT; case DAVClient::TYPE_VTODO: return Engine::TYPE_TASK; case DAVClient::TYPE_VCARD: // TODO what about groups return Engine::TYPE_CONTACT; default: throw new \Exception("Cannot map type '{$type}' from DAV"); } } /** * Check if the folder should not be migrated */ private function shouldSkip($folder): bool { // When dealing with iRony DAV other user folders names have distinct names // there's no other way to recognize them than by the name pattern. // ;et's hope that users do not have personal folders with names starting with a bracket. if (preg_match('~\(.*\) .*~', $folder->name)) { return true; } if (str_starts_with($folder->name, 'shared » ')) { return true; } // TODO: Cyrus DAV shared folders return false; } } diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php index 3fc44b54..36b24212 100644 --- a/src/app/DataMigrator/EWS.php +++ b/src/app/DataMigrator/EWS.php @@ -1,516 +1,518 @@ Engine::TYPE_EVENT, EWS\Contact::FOLDER_TYPE => Engine::TYPE_CONTACT, EWS\Task::FOLDER_TYPE => Engine::TYPE_TASK, ]; /** @var Account Account to operate on */ protected $account; /** @var Engine Data migrator engine */ protected $engine; /** * Object constructor */ public function __construct(Account $account, Engine $engine) { $this->account = $account; $this->engine = $engine; } /** * Server autodiscovery */ public static function autodiscover(string $user, string $password): ?string { // You should never run the Autodiscover more than once. // It can make between 1 and 5 calls before giving up, or before finding your server, // depending on how many different attempts it needs to make. // TODO: Autodiscovery may fail with an exception thrown. Handle this nicely. // TODO: Looks like this autodiscovery also does not work w/Basic Auth? $api = API\ExchangeAutodiscover::getAPI($user, $password); $server = $api->getClient()->getServer(); $version = $api->getClient()->getVersion(); return sprintf('ews://%s:%s@%s', urlencode($user), urlencode($password), $server); } /** * Authenticate to EWS (initialize the EWS client) */ public function authenticate(): void { if (!empty($this->account->params['client_id'])) { $this->api = $this->authenticateWithOAuth2( $this->account->host, $this->account->username, $this->account->params['client_id'], $this->account->params['client_secret'], $this->account->params['tenant_id'] ); } else { // Note: This initializes the client, but not yet connects to the server // TODO: To know that the credentials work we'll have to do some API call. $this->api = $this->authenticateWithPassword( $this->account->host, $this->account->username, $this->account->password, $this->account->loginas ); } } /** * Autodiscover the server and authenticate the user */ protected function authenticateWithPassword(string $server, string $user, string $password, string $loginas = null) { // Note: Since 2023-01-01 EWS at Office365 requires OAuth2, no way back to basic auth. \Log::debug("[EWS] Using basic authentication on $server..."); $options = []; if ($loginas) { $options['impersonation'] = $loginas; } $this->engine->setOption('ews', [ 'options' => $options, 'server' => $server, ]); return API::withUsernameAndPassword($server, $user, $password, $this->apiOptions($options)); } /** * Authenticate with a token (Office365) */ protected function authenticateWithToken(string $server, string $user, string $token, $expires_at = null) { \Log::debug("[EWS] Using token authentication on $server..."); $options = ['impersonation' => $user]; $this->engine->setOption('ews', [ 'options' => $options, 'server' => $server, 'token' => $token, 'expires_at' => $expires_at, ]); return API::withCallbackToken($server, $token, $this->apiOptions($options)); } /** * Authenticate with OAuth2 (Office365) - get the token */ protected function authenticateWithOAuth2( string $server, string $user, string $client_id, string $client_secret, string $tenant_id ) { // See https://github.com/Garethp/php-ews/blob/master/examples/basic/authenticatingWithOAuth.php // See https://github.com/Garethp/php-ews/issues/236#issuecomment-1292521527 // To register OAuth2 app goto https://entra.microsoft.com > Applications > App registrations \Log::debug("[EWS] Fetching OAuth2 token from $server..."); $scope = 'https://outlook.office365.com/.default'; $token_uri = "https://login.microsoftonline.com/{$tenant_id}/oauth2/v2.0/token"; // $authUri = "https://login.microsoftonline.com/{$tenant_id}/oauth2/authorize"; $response = Http::asForm() ->timeout(5) ->post($token_uri, [ 'client_id' => $client_id, 'client_secret' => $client_secret, 'scope' => $scope, 'grant_type' => 'client_credentials', ]) ->throwUnlessStatus(200); $token = $response->json('access_token'); // Note: Office365 default token expiration time is ~1h, $expires_in = $response->json('expires_in'); $expires_at = now()->addSeconds($expires_in)->toDateTimeString(); return $this->authenticateWithToken($server, $user, $token, $expires_at); } /** * Get folders hierarchy */ public function getFolders($types = []): array { if (empty($types)) { $types = array_values($this->type_map); } // Create FolderClass filter $search = new Type\OrType(); foreach ($types as $type) { $type = array_search($type, $this->type_map); $search->addContains(Type\Contains::buildFromArray([ 'FieldURI' => [ Type\FieldURI::buildFromArray(['FieldURI' => 'folder:FolderClass']), ], 'Constant' => Type\ConstantValueType::buildFromArray([ 'Value' => $type, ]), 'ContainmentComparison' => 'Exact', 'ContainmentMode' => 'FullString', ])); } // Get full folders hierarchy (filtered by folder class) // Use of the filter reduces the response size by excluding system folders $options = [ 'Traversal' => 'Deep', 'Restriction' => ['Or' => $search], ]; $folders = $this->api->getChildrenFolders('root', $options); $result = []; foreach ($folders as $folder) { $class = $folder->getFolderClass(); $type = $this->type_map[$class] ?? null; // Skip folder types we do not support (need) if (empty($type) || (!empty($types) && !in_array($type, $types))) { continue; } // Note: Folder names are localized, even INBOX $name = $fullname = $folder->getDisplayName(); $id = $folder->getFolderId()->getId(); $parentId = $folder->getParentFolderId()->getId(); // Create folder name with full path if ($parentId && !empty($result[$parentId])) { $fullname = $result[$parentId]->fullname . '/' . $name; } // Top-level folder, check if it's a special folder we should ignore // FIXME: Is there a better way to distinguish user folders from system ones? if ( in_array($fullname, $this->folder_exceptions) || strpos($fullname, 'OwaFV15.1All') === 0 ) { continue; } $result[$id] = Folder::fromArray([ 'id' => $folder->getFolderId()->toArray(true), 'total' => $folder->getTotalCount(), 'class' => $class, 'type' => $this->type_map[$class] ?? null, 'name' => $name, 'fullname' => $fullname, ]); } return $result; } /** * Fetch a list of folder items */ public function fetchItemList(Folder $folder, $callback, Interface\ImporterInterface $importer): void { // Job processing - initialize environment $this->initEnv($this->engine->queue); // The folder is empty, we can stop here if (empty($folder->total)) { // TODO: Delete all existing items? return; } // Get items already imported // TODO: This might be slow and/or memory expensive, we should consider // whether storing list of imported items in some cache wouldn't be a better // solution. Of course, cache would not get changes in the destination account. $existing = $importer->getItems($folder); // Create X-MS-ID index for easier search in existing items // Note: For some objects we could use UID (events), but for some we don't have UID in Exchange. // Also because fetching extra properties here is problematic, we use X-MS-ID. $existingIndex = []; array_walk( $existing, function (&$item, $idx) use (&$existingIndex) { if (!empty($item['x-ms-id'])) { [$id, $changeKey] = explode('!', $item['x-ms-id']); $item['changeKey'] = $changeKey; $existingIndex[$id] = $idx; unset($item['x-ms-id']); } } ); $request = [ // Exchange's maximum is 1000 'IndexedPageItemView' => ['MaxEntriesReturned' => 100, 'Offset' => 0, 'BasePoint' => 'Beginning'], 'ParentFolderIds' => $folder->id, 'Traversal' => 'Shallow', 'ItemShape' => [ 'BaseShape' => 'IdOnly', 'AdditionalProperties' => [ 'FieldURI' => [ ['FieldURI' => 'item:ItemClass'], // ['FieldURI' => 'item:Size'], ], ], ], ]; $request = Type::buildFromArray($request); // Note: It is not possible to get mimeContent with FindItem request // That's why we first get the list of object identifiers and // then call GetItem on each separately. // TODO: It might be feasible to get all properties for object types // for which we don't use MimeContent, for better performance. // Request first page $response = $this->api->getClient()->FindItem($request); $set = new ItemSet(); // @phpstan-ignore-next-line foreach ($response->getItems() as $item) { if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { $set->items[] = $item; if (count($set->items) == self::CHUNK_SIZE) { $callback($set); $set = new ItemSet(); } } } // Request other pages until we got all while (!$response->isIncludesLastItemInRange()) { // @phpstan-ignore-next-line $response = $this->api->getNextPage($response); foreach ($response->getItems() as $item) { if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { $set->items[] = $item; if (count($set->items) == self::CHUNK_SIZE) { $callback($set); $set = new ItemSet(); } } } } if (count($set->items)) { $callback($set); } // TODO: Delete items that do not exist anymore? } /** * Fetching an item */ public function fetchItem(Item $item): void { // Job processing - initialize environment $this->initEnv($this->engine->queue); if ($driver = EWS\Item::factory($this, $item)) { $driver->processItem($item); return; } throw new \Exception("Failed to fetch an item from EWS"); } /** * Get the source account */ public function getSourceAccount(): Account { return $this->engine->source; } /** * Get the destination account */ public function getDestinationAccount(): Account { return $this->engine->destination; } /** * Synchronize specified object */ protected function toItem(Type $item, Folder $folder, $existing, $existingIndex): ?Item { $id = $item->getItemId()->toArray(); $exists = null; // Detect an existing item, skip if nothing changed if (isset($existingIndex[$id['Id']])) { $idx = $existingIndex[$id['Id']]; if ($existing[$idx]['changeKey'] == $id['ChangeKey']) { return null; } $exists = $existing[$idx]['href']; } if (!EWS\Item::isValidItem($item)) { return null; } return Item::fromArray([ 'id' => $id['Id'], 'class' => $item->getItemClass(), 'folder' => $folder, 'existing' => $exists, ]); } /** * Set common API options */ protected function apiOptions(array $options): array { if (empty($options['version'])) { $options['version'] = API\ExchangeWebServices::VERSION_2013; } // In debug mode record all responses if (\config('app.debug')) { $options['httpPlayback'] = [ 'mode' => 'record', 'recordLocation' => \storage_path('ews'), ]; } // Options for testing foreach (['httpClient', 'httpPlayback'] as $opt) { if (($val = $this->engine->getOption($opt)) !== null) { $options[$opt] = $val; } } return $options; } /** * Initialize environment for job execution * * @param Queue $queue Queue */ protected function initEnv(Queue $queue): void { $ews = $queue->data['options']['ews']; if (!empty($ews['token'])) { // TODO: Refresh the token if needed $this->api = API::withCallbackToken( $ews['server'], $ews['token'], $this->apiOptions($ews['options']) ); } else { $this->api = API::withUsernameAndPassword( $ews['server'], $this->account->username, $this->account->password, $this->apiOptions($ews['options']) ); } } }