diff --git a/src/app/Backends/DAV.php b/src/app/Backends/DAV.php index 5fd88d8e..f0f86f33 100644 --- a/src/app/Backends/DAV.php +++ b/src/app/Backends/DAV.php @@ -1,586 +1,569 @@ 'urn:ietf:params:xml:ns:caldav', self::TYPE_VTODO => 'urn:ietf:params:xml:ns:caldav', self::TYPE_VCARD => 'urn:ietf:params:xml:ns:carddav', ]; protected $url; protected $user; protected $password; protected $responseHeaders = []; protected $homes; /** * Object constructor */ public function __construct($user, $password, $url = null) { $this->url = $url ?: \config('services.dav.uri'); $this->user = $user; $this->password = $password; } /** * Discover DAV home (root) collection of a specified type. * * @return array|false Home locations or False on error */ public function discover() { if (is_array($this->homes)) { return $this->homes; } $path = parse_url($this->url, PHP_URL_PATH); $body = '' . '' . '' . '' . '' . ''; // Note: Cyrus CardDAV service requires Depth:1 (CalDAV works without it) $response = $this->request('/', 'PROPFIND', $body, ['Depth' => 1, 'Prefer' => 'return-minimal']); if (empty($response)) { \Log::error("Failed to get current-user-principal from the DAV server."); return false; } $elements = $response->getElementsByTagName('response'); $principal_href = ''; foreach ($elements as $element) { foreach ($element->getElementsByTagName('current-user-principal') as $prop) { $principal_href = $prop->nodeValue; break; } } if ($path && str_starts_with($principal_href, $path)) { $principal_href = substr($principal_href, strlen($path)); } $body = '' . '' . '' . '' . '' . '' . '' . ''; $response = $this->request($principal_href, 'PROPFIND', $body); if (empty($response)) { \Log::error("Failed to get home collections from the DAV server."); return false; } $elements = $response->getElementsByTagName('response'); $homes = []; if ($element = $response->getElementsByTagName('response')->item(0)) { if ($prop = $element->getElementsByTagName('prop')->item(0)) { foreach ($prop->childNodes as $home) { if ($home->firstChild && $home->firstChild->localName == 'href') { $href = $home->firstChild->nodeValue; if ($path && str_starts_with($href, $path)) { $href = substr($href, strlen($path)); } $homes[$home->localName] = $href; } } } } return $this->homes = $homes; } /** * Get user home folder of specified type * * @param string $type Home type or component name * * @return string|null Folder location href */ public function getHome($type) { $options = [ self::TYPE_VEVENT => 'calendar-home-set', self::TYPE_VTODO => 'calendar-home-set', self::TYPE_VCARD => 'addressbook-home-set', self::TYPE_NOTIFICATION => 'notification-URL', ]; $homes = $this->discover(); if (is_array($homes) && isset($options[$type])) { return $homes[$options[$type]] ?? null; } return null; } /** * Check if we can connect to the DAV server * * @return bool True on success, False otherwise */ public static function healthcheck(): bool { // TODO return true; } /** * Get list of folders of specified type. * * @param string $component Component to filter by (VEVENT, VTODO, VCARD) * * @return false|array List of folders' metadata or False on error */ public function listFolders(string $component) { $root_href = $this->getHome($component); if ($root_href === null) { return false; } $ns = 'xmlns:d="DAV:" xmlns:cs="http://calendarserver.org/ns/"'; $props = ''; if ($component != self::TYPE_VCARD) { $ns .= ' xmlns:c="urn:ietf:params:xml:ns:caldav" xmlns:a="http://apple.com/ns/ical/" xmlns:k="Kolab:"'; $props = '' . '' . ''; } $body = '' . '' . '' . '' . '' . '' . $props . '' . ''; // Note: Cyrus CardDAV service requires Depth:1 (CalDAV works without it) $headers = ['Depth' => 1, 'Prefer' => 'return-minimal']; $response = $this->request($root_href, 'PROPFIND', $body, $headers); if (empty($response)) { \Log::error("Failed to get folders list from the DAV server."); return false; } $folders = []; foreach ($response->getElementsByTagName('response') as $element) { $folder = DAV\Folder::fromDomElement($element); // Note: Addressbooks don't have 'type' specified if ( ($component == self::TYPE_VCARD && in_array('addressbook', $folder->types)) || in_array($component, $folder->components) ) { $folders[] = $folder; } } return $folders; } /** * Create a DAV object in a folder * * @param DAV\CommonObject $object Object * * @return false|DAV\CommonObject Object on success, False on error */ public function create(DAV\CommonObject $object) { $headers = ['Content-Type' => $object->contentType]; $response = $this->request($object->href, 'PUT', $object, $headers); if ($response !== false) { if (!empty($this->responseHeaders['ETag'])) { $etag = $this->responseHeaders['ETag'][0]; if (preg_match('|^".*"$|', $etag)) { $etag = substr($etag, 1, -1); } $object->etag = $etag; } return $object; } return false; } /** * Update a DAV object in a folder * * @param DAV\CommonObject $object Object * * @return false|DAV\CommonObject Object on success, False on error */ public function update(DAV\CommonObject $object) { return $this->create($object); } /** * Delete a DAV object from a folder * * @param string $location Object location * * @return bool True on success, False on error */ public function delete(string $location) { $response = $this->request($location, 'DELETE', '', ['Depth' => 1, 'Prefer' => 'return-minimal']); return $response !== false; } /** * Create a DAV folder (collection) * * @param DAV\Folder $folder Folder object * * @return bool True on success, False on error */ public function folderCreate(DAV\Folder $folder) { $response = $this->request($folder->href, 'MKCOL', $folder->toXML('mkcol')); return $response !== false; } /** * Delete a DAV folder (collection) * * @param string $location Folder location * * @return bool True on success, False on error */ public function folderDelete($location) { $response = $this->request($location, 'DELETE'); return $response !== false; } /** * Get all properties of a folder. * * @param string $location Object location * * @return false|DAV\Folder Folder metadata or False on error */ public function folderInfo(string $location) { $body = DAV\Folder::propfindXML(); // Note: Cyrus CardDAV service requires Depth:1 (CalDAV works without it) $response = $this->request($location, 'PROPFIND', $body, ['Depth' => 0, 'Prefer' => 'return-minimal']); if (!empty($response) && ($element = $response->getElementsByTagName('response')->item(0))) { return DAV\Folder::fromDomElement($element); } return false; } /** * Update a DAV folder (collection) * * @param DAV\Folder $folder Folder object * * @return bool True on success, False on error */ public function folderUpdate(DAV\Folder $folder) { // Note: Changing resourcetype property is forbidden (at least by Cyrus) $response = $this->request($folder->href, 'PROPPATCH', $folder->toXML('propertyupdate')); return $response !== false; } /** * Check server options (and authentication) * * @return false|array DAV capabilities on success, False on error */ public function options() { $response = $this->request('', 'OPTIONS'); if ($response !== false) { return preg_split('/,\s+/', implode(',', $this->responseHeaders['DAV'] ?? [])); } return false; } /** * Search DAV objects in a folder. * - * @param string $location Folder location - * @param string $component Object type (VEVENT, VTODO, VCARD) + * @param string $location Folder location + * @param DAV\Search $search Search request parameters + * @param ?callable $callback Callback to execute on every object * * @return false|array Objects metadata on success, False on error */ - public function search(string $location, string $component) + public function search(string $location, DAV\Search $search, $callback = null) { - $queries = [ - self::TYPE_VEVENT => 'calendar-query', - self::TYPE_VTODO => 'calendar-query', - self::TYPE_VCARD => 'addressbook-query', - ]; - - $filter = ''; - if ($component != self::TYPE_VCARD) { - $filter = '' - . '' - . ''; - } - - // TODO: Make filter an argument of this function to build all kind of queries. - // It probably should be a separate object e.g. DAV\Filter. - // TODO: List of object props to return should also be an argument, so we not only - // could fetch "an index" but also any of object's data. - - $body = '' - . ' ' - . '' - . '' - . '' - . ($filter ? "$filter" : '') - . ''; + $headers = ['Depth' => $search->depth, 'Prefer' => 'return-minimal']; - $response = $this->request($location, 'REPORT', $body, ['Depth' => 1, 'Prefer' => 'return-minimal']); + $response = $this->request($location, 'REPORT', $search, $headers); if (empty($response)) { \Log::error("Failed to get objects from the DAV server."); return false; } $objects = []; foreach ($response->getElementsByTagName('response') as $element) { - $objects[] = $this->objectFromElement($element, $component); + $object = $this->objectFromElement($element, $search->component); + + if ($callback) { + $object = $callback($object); + } + + $objects[] = $object; } return $objects; } /** * Fetch DAV objects data from a folder * * @param string $location Folder location * @param string $component Object type (VEVENT, VTODO, VCARD) * @param array $hrefs List of objects' locations to fetch (empty for all objects) * * @return false|array Objects metadata on success, False on error */ public function getObjects(string $location, string $component, array $hrefs = []) { if (empty($hrefs)) { return []; } $body = ''; foreach ($hrefs as $href) { $body .= '' . $href . ''; } $queries = [ self::TYPE_VEVENT => 'calendar-multiget', self::TYPE_VTODO => 'calendar-multiget', self::TYPE_VCARD => 'addressbook-multiget', ]; $types = [ self::TYPE_VEVENT => 'calendar-data', self::TYPE_VTODO => 'calendar-data', self::TYPE_VCARD => 'address-data', ]; $body = '' . ' ' . '' . '' . '' . '' . $body . ''; $response = $this->request($location, 'REPORT', $body, ['Depth' => 1, 'Prefer' => 'return-minimal']); if (empty($response)) { \Log::error("Failed to get objects from the DAV server."); return false; } $objects = []; foreach ($response->getElementsByTagName('response') as $element) { $objects[] = $this->objectFromElement($element, $component); } return $objects; } /** * Parse XML content */ protected function parseXML($xml) { $doc = new \DOMDocument('1.0', 'UTF-8'); if (str_starts_with($xml, 'loadXML($xml)) { throw new \Exception("Failed to parse XML"); } $doc->formatOutput = true; } return $doc; } /** * Parse request/response body for debug purposes */ protected function debugBody($body, $headers) { $head = ''; foreach ($headers as $header_name => $header_value) { if (is_array($header_value)) { $header_value = implode("\n\t", $header_value); } $head .= "{$header_name}: {$header_value}\n"; } if ($body instanceof DAV\CommonObject) { $body = (string) $body; } if (str_starts_with($body, 'formatOutput = true; $doc->preserveWhiteSpace = false; if (!$doc->loadXML($body)) { throw new \Exception("Failed to parse XML"); } $body = $doc->saveXML(); } return $head . (is_string($body) && strlen($body) > 0 ? "\n{$body}" : ''); } /** * Create DAV\CommonObject from a DOMElement */ protected function objectFromElement($element, $component) { switch ($component) { case self::TYPE_VEVENT: $object = DAV\Vevent::fromDomElement($element); break; case self::TYPE_VTODO: $object = DAV\Vtodo::fromDomElement($element); break; case self::TYPE_VCARD: $object = DAV\Vcard::fromDomElement($element); break; default: throw new \Exception("Unknown component: {$component}"); } return $object; } /** * Execute HTTP request to a DAV server */ protected function request($path, $method, $body = '', $headers = []) { $debug = \config('app.debug'); $url = $this->url; $this->responseHeaders = []; if ($path && ($rootPath = parse_url($url, PHP_URL_PATH)) && str_starts_with($path, $rootPath)) { $path = substr($path, strlen($rootPath)); } $url .= $path; $client = Http::withBasicAuth($this->user, $this->password); // $client = Http::withToken($token); // Bearer token if ($body) { if (!isset($headers['Content-Type'])) { $headers['Content-Type'] = 'application/xml; charset=utf-8'; } $client->withBody($body, $headers['Content-Type']); } if (!empty($headers)) { $client->withHeaders($headers); } if ($debug) { $body = $this->debugBody($body, $headers); \Log::debug("C: {$method}: {$url}" . (strlen($body) > 0 ? "\n$body" : '')); } $response = $client->send($method, $url); $body = $response->body(); $code = $response->status(); if ($debug) { \Log::debug("S: [{$code}]\n" . $this->debugBody($body, $response->headers())); } // Throw an exception if a client or server error occurred... $response->throw(); $this->responseHeaders = $response->headers(); return $this->parseXML($body); } } diff --git a/src/app/Backends/DAV/CommonObject.php b/src/app/Backends/DAV/CommonObject.php index 16d2ee82..3b2aa421 100644 --- a/src/app/Backends/DAV/CommonObject.php +++ b/src/app/Backends/DAV/CommonObject.php @@ -1,58 +1,61 @@ value) */ + public $custom = []; + /** * Create DAV object from a DOMElement element * * @param \DOMElement $element DOM element with object properties * * @return CommonObject */ public static function fromDomElement(\DOMElement $element) { $object = new static(); // @phpstan-ignore-line if ($href = $element->getElementsByTagName('href')->item(0)) { $object->href = $href->nodeValue; // Extract UID from the URL $href_parts = explode('/', $object->href); $object->uid = preg_replace('/\.[a-z]+$/', '', $href_parts[count($href_parts) - 1]); } if ($etag = $element->getElementsByTagName('getetag')->item(0)) { $object->etag = $etag->nodeValue; if (preg_match('|^".*"$|', $object->etag)) { $object->etag = substr($object->etag, 1, -1); } } return $object; } /** * Create string representation of the DAV object * * @return string */ public function __toString() { return ''; } } diff --git a/src/app/Backends/DAV/Search.php b/src/app/Backends/DAV/Search.php new file mode 100644 index 00000000..3c9bc7be --- /dev/null +++ b/src/app/Backends/DAV/Search.php @@ -0,0 +1,79 @@ +component = $component; + } + + /** + * Create string representation of the search + * + * @return string + */ + public function __toString() + { + $ns = implode(' ', [ + 'xmlns:d="DAV:"', + 'xmlns:c="' . DAV::NAMESPACES[$this->component] . '"', + ]); + + // Return properties + $props = []; + foreach ($this->properties as $prop) { + $props[] = '<' . $prop . '/>'; + } + + // Warning: Looks like some servers (iRony) ignore address-data/calendar-data + // and return full VCARD/VCALENDAR. Which leads to unwanted loads of data in a response. + if (!empty($this->dataProperties)) { + $more_props = []; + foreach ($this->dataProperties as $prop) { + $more_props[] = ''; + } + + if ($this->component == DAV::TYPE_VCARD) { + $props[] = '' . implode('', $more_props) . ''; + } else { + $props[] = '' + . '' + . '' + . '' . implode('', $more_props) . '' + . ''; + } + } + + // Search filter + $filter = ''; + if ($this->component == DAV::TYPE_VCARD) { + $query = 'addressbook-query'; + } else { + $query = 'calendar-query'; + $filter = ''; + $filter = "{$filter}"; + } + + if (empty($props)) { + $props = ''; + } else { + $props = '' . implode('', $props) . ''; + } + + return '' + . "" . $props . $filter . ""; + } +} diff --git a/src/app/Backends/DAV/Vcard.php b/src/app/Backends/DAV/Vcard.php index ab3f2e1d..b71edb39 100644 --- a/src/app/Backends/DAV/Vcard.php +++ b/src/app/Backends/DAV/Vcard.php @@ -1,49 +1,90 @@ getElementsByTagName('address-data')->item(0)) { $object->fromVcard($data->nodeValue); } return $object; } /** * Set object properties from a vcard * * @param string $vcard vCard string */ protected function fromVcard(string $vcard): void { - // TODO + $vobject = Reader::read($vcard, Reader::OPTION_FORGIVING | Reader::OPTION_IGNORE_INVALID_LINES); + + if ($vobject->name != 'VCARD') { + // FIXME: throw an exception? + return; + } + + $string_properties = [ + 'FN', + 'REV', + 'UID', + ]; + + foreach ($vobject->children() as $prop) { + if (!($prop instanceof Property)) { + continue; + } + + switch ($prop->name) { + // TODO: Map all vCard properties to class properties + + default: + // map string properties + if (in_array($prop->name, $string_properties)) { + $key = Str::camel(strtolower($prop->name)); + $this->{$key} = (string) $prop; + } + + // custom properties + if (\str_starts_with($prop->name, 'X-')) { + $this->custom[$prop->name] = (string) $prop; + } + } + } } /** * Create string representation of the DAV object (vcard) * * @return string */ public function __toString() { // TODO: This will be needed when we want to create/update objects return ''; } } diff --git a/src/app/Backends/DAV/Vevent.php b/src/app/Backends/DAV/Vevent.php index d2b7b627..e0774e5b 100644 --- a/src/app/Backends/DAV/Vevent.php +++ b/src/app/Backends/DAV/Vevent.php @@ -1,279 +1,284 @@ getElementsByTagName('calendar-data')->item(0)) { $object->fromIcal($data->nodeValue); } return $object; } /** * Set object properties from an iCalendar * * @param string $ical iCalendar string */ protected function fromIcal(string $ical): void { $options = VObject\Reader::OPTION_FORGIVING | VObject\Reader::OPTION_IGNORE_INVALID_LINES; $vobject = VObject\Reader::read($ical, $options); if ($vobject->name != 'VCALENDAR') { return; } $selfType = strtoupper(class_basename(get_class($this))); foreach ($vobject->getComponents() as $component) { if ($component->name == $selfType) { $this->fromVObject($component); return; } } } /** * Set object properties from a Sabre/VObject component object * * @param VObject\Component $vobject Sabre/VObject component */ protected function fromVObject(VObject\Component $vobject): void { $string_properties = [ 'COMMENT', 'DESCRIPTION', 'LOCATION', 'SEQUENCE', 'STATUS', 'SUMMARY', 'TRANSP', 'UID', 'URL', ]; // map string properties foreach ($string_properties as $prop) { if (isset($vobject->{$prop})) { $key = Str::camel(strtolower($prop)); $this->{$key} = (string) $vobject->{$prop}; } } // map other properties foreach ($vobject->children() as $prop) { if (!($prop instanceof VObject\Property)) { continue; } switch ($prop->name) { case 'DTSTART': case 'DTEND': case 'DUE': case 'CREATED': case 'LAST-MODIFIED': case 'DTSTAMP': $key = Str::camel(strtolower($prop->name)); // These are of type Sabre\VObject\Property\ICalendar\DateTime $this->{$key} = $prop; break; case 'RRULE': $params = !empty($this->recurrence) ? $this->recurrence : []; foreach ($prop->getParts() as $k => $v) { $params[Str::camel(strtolower($k))] = is_array($v) ? implode(',', $v) : $v; } if (!empty($params['until'])) { $params['until'] = new \DateTime($params['until']); } if (empty($params['interval'])) { $params['interval'] = 1; } $this->recurrence = array_filter($params); break; case 'EXDATE': case 'RDATE': $key = strtolower($prop->name); $dates = []; // TODO if (!empty($this->recurrence[$key])) { $this->recurrence[$key] = array_merge($this->recurrence[$key], $dates); } else { $this->recurrence[$key] = $dates; } break; case 'ATTENDEE': case 'ORGANIZER': $attendee = [ 'rsvp' => false, 'email' => preg_replace('!^mailto:!i', '', (string) $prop), ]; $attendeeProps = ['CN', 'PARTSTAT', 'ROLE', 'CUTYPE', 'RSVP', 'DELEGATED-FROM', 'DELEGATED-TO', 'SCHEDULE-STATUS', 'SCHEDULE-AGENT', 'SENT-BY']; foreach ($prop->parameters() as $name => $value) { $key = Str::camel(strtolower($name)); switch ($name) { case 'RSVP': $params[$key] = strtolower($value) == 'true'; break; case 'CN': $params[$key] = str_replace('\,', ',', strval($value)); break; default: if (in_array($name, $attendeeProps)) { $params[$key] = strval($value); } break; } } if ($prop->name == 'ORGANIZER') { $attendee['role'] = 'ORGANIZER'; $attendee['partstat'] = 'ACCEPTED'; $this->organizer = $attendee; } elseif (empty($this->organizer) || $attendee['email'] != $this->organizer['email']) { $this->attendees[] = $attendee; } break; + + default: + if (\str_starts_with($prop->name, 'X-')) { + $this->custom[$prop->name] = (string) $prop; + } } } // Check DURATION property if no end date is set /* if (empty($this->dtend) && !empty($this->dtstart) && !empty($vobject->DURATION)) { try { $duration = new \DateInterval((string) $vobject->DURATION); $end = clone $this->dtstart; $end->add($duration); $this->dtend = $end; } catch (\Exception $e) { // TODO: Error? } } */ // Find alarms foreach ($vobject->select('VALARM') as $valarm) { $action = 'DISPLAY'; $trigger = null; $alarm = []; foreach ($valarm->children() as $prop) { $value = strval($prop); switch ($prop->name) { case 'TRIGGER': foreach ($prop->parameters as $param) { if ($param->name == 'VALUE' && $param->getValue() == 'DATE-TIME') { $trigger = '@' . $prop->getDateTime()->format('U'); $alarm['trigger'] = $prop->getDateTime(); } elseif ($param->name == 'RELATED') { $alarm['related'] = $param->getValue(); } } /* if (!$trigger && ($values = libcalendaring::parse_alarm_value($value))) { $trigger = $values[2]; } */ if (empty($alarm['trigger'])) { $alarm['trigger'] = rtrim(preg_replace('/([A-Z])0[WDHMS]/', '\\1', $value), 'T'); // if all 0-values have been stripped, assume 'at time' if ($alarm['trigger'] == 'P') { $alarm['trigger'] = 'PT0S'; } } break; case 'ACTION': $action = $alarm['action'] = strtoupper($value); break; case 'SUMMARY': case 'DESCRIPTION': case 'DURATION': $alarm[strtolower($prop->name)] = $value; break; case 'REPEAT': $alarm['repeat'] = (int) $value; break; case 'ATTENDEE': $alarm['attendees'][] = preg_replace('!^mailto:!i', '', $value); break; } } if ($action != 'NONE') { if (!empty($alarm['trigger'])) { $this->valarms[] = $alarm; } } } } /** * Create string representation of the DAV object (iCalendar) * * @return string */ public function __toString() { // TODO: This will be needed when we want to create/update objects return ''; } } diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php index 069879e4..27934428 100644 --- a/src/app/DataMigrator/DAV.php +++ b/src/app/DataMigrator/DAV.php @@ -1,173 +1,240 @@ username . ($account->loginas ? "**{$account->loginas}" : ''); $baseUri = rtrim($account->uri, '/'); $baseUri = preg_replace('|^dav|', 'http', $baseUri); $this->settings = [ 'baseUri' => $baseUri, 'userName' => $username, 'password' => $account->password, ]; $this->client = new DAVClient($username, $account->password, $baseUri); $this->engine = $engine; $this->account = $account; } /** * Check user credentials. * * @throws \Exception */ public function authenticate() { try { $result = $this->client->options(); } catch (\Exception $e) { throw new \Exception("Invalid DAV credentials or server."); } } /** * Create an item in a folder. * - * @param string $filename File location - * @param Folder $folder Folder + * @param Item $item Item to import * * @throws \Exception */ - public function createItemFromFile(string $filename, Folder $folder): void + public function createItem(Item $item): void { - $href = $this->getFolderPath($folder) . '/' . pathinfo($filename, PATHINFO_BASENAME); + // TODO: For now we do DELETE + PUT. It's because the UID might have changed (which + // is the case with e.g. contacts from EWS) causing a discrepancy between UID and href. + // This is not necessarily a problem and would not happen to calendar events. + // So, maybe we could improve that so DELETE is not needed. + if ($item->existing) { + try { + $this->client->delete($item->existing); + } catch (\Illuminate\Http\Client\RequestException $e) { + // ignore 404 response, item removed in meantime? + if ($e->getCode() != 404) { + throw $e; + } + } + } + + $href = $this->getFolderPath($item->folder) . '/' . pathinfo($item->filename, PATHINFO_BASENAME); - $object = new DAVOpaque($filename); + $object = new DAVOpaque($item->filename); $object->href = $href; - switch ($folder->type) { + switch ($item->folder->type) { case Engine::TYPE_EVENT: case Engine::TYPE_TASK: $object->contentType = 'text/calendar; charset=utf-8'; break; case Engine::TYPE_CONTACT: $object->contentType = 'text/vcard; charset=utf-8'; break; } if ($this->client->create($object) === false) { - throw new \Exception("Failed to save object into DAV server at {$href}"); + throw new \Exception("Failed to save DAV object at {$href}"); } } /** * Create a folder. * * @param Folder $folder Folder data * * @throws \Exception on error */ public function createFolder(Folder $folder): void { $dav_type = $this->type2DAV($folder->type); $folders = $this->client->listFolders($dav_type); if ($folders === false) { throw new \Exception("Failed to list folders on the DAV server"); } // Note: iRony flattens the list by modifying the folder name // This is not going to work with Cyrus DAV, but anyway folder // hierarchies support is not full in Kolab 4. foreach ($folders as $dav_folder) { if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) { // do nothing, folder already exists return; } } $home = $this->client->getHome($dav_type); $folder_id = Utils::uuidStr(); $collection_type = $dav_type == DAVClient::TYPE_VCARD ? 'addressbook' : 'calendar'; // We create all folders on the top-level $dav_folder = new DAVFolder(); $dav_folder->name = $folder->fullname; $dav_folder->href = rtrim($home, '/') . '/' . $folder_id; $dav_folder->components = [$dav_type]; $dav_folder->types = ['collection', $collection_type]; if ($this->client->folderCreate($dav_folder) === false) { throw new \Exception("Failed to create a DAV folder {$dav_folder->href}"); } } + /** + * Get a list of folder items, limited to their essential propeties + * used in incremental migration. + * + * @param Folder $folder Folder data + * + * @throws \Exception on error + */ + public function getItems(Folder $folder): array + { + $dav_type = $this->type2DAV($folder->type); + $location = $this->getFolderPath($folder); + + $search = new DAVSearch($dav_type); + + // TODO: We request only properties relevant to incremental migration, + // i.e. to find that something exists and its last update time. + // Some servers (iRony) do ignore that and return full VCARD/VEVENT/VTODO + // content, if there's many objects we'll have a memory limit issue. + // Also, this list should be controlled by the exporter. + $search->dataProperties = ['UID', 'X-MS-ID', 'REV']; + + $items = $this->client->search( + $location, + $search, + function ($item) use ($dav_type) { + // Slim down the result to properties we might need + $result = [ + 'href' => $item->href, + 'uid' => $item->uid, + 'x-ms-id' => $item->custom['X-MS-ID'] ?? null, + ]; + /* + switch ($dav_type) { + case DAVClient::TYPE_VCARD: + $result['rev'] = $item->rev; + break; + } + */ + + return $result; + } + ); + + if ($items === false) { + throw new \Exception("Failed to get items from a DAV folder {$location}"); + } + + return $items; + } + /** * Get folder relative URI */ protected function getFolderPath(Folder $folder): string { $folders = $this->client->listFolders($this->type2DAV($folder->type)); if ($folders === false) { throw new \Exception("Failed to list folders on the DAV server"); } // Note: iRony flattens the list by modifying the folder name // This is not going to work with Cyrus DAV, but anyway folder // hierarchies support is not full in Kolab 4. foreach ($folders as $dav_folder) { if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) { return rtrim($dav_folder->href, '/'); } } throw new \Exception("Folder not found: {$folder->fullname}"); } /** * Map Kolab type into DAV object type */ protected static function type2DAV(string $type): string { switch ($type) { case Engine::TYPE_EVENT: return DAVClient::TYPE_VEVENT; case Engine::TYPE_TASK: return DAVClient::TYPE_VTODO; case Engine::TYPE_CONTACT: case Engine::TYPE_GROUP: return DAVClient::TYPE_VCARD; default: throw new \Exception("Cannot map type '{$type}' to DAV"); } } } diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php index 6d1d6d2d..92d327a3 100644 --- a/src/app/DataMigrator/EWS.php +++ b/src/app/DataMigrator/EWS.php @@ -1,422 +1,462 @@ Engine::TYPE_EVENT, EWS\Contact::FOLDER_TYPE => Engine::TYPE_CONTACT, EWS\Task::FOLDER_TYPE => Engine::TYPE_TASK, ]; /** @var Account Account to operate on */ protected $account; /** @var Engine Data migrator engine */ protected $engine; /** * Object constructor */ public function __construct(Account $account, Engine $engine) { $this->account = $account; $this->engine = $engine; } /** * Server autodiscovery */ public static function autodiscover(string $user, string $password): ?string { // You should never run the Autodiscover more than once. // It can make between 1 and 5 calls before giving up, or before finding your server, // depending on how many different attempts it needs to make. // TODO: Autodiscovery may fail with an exception thrown. Handle this nicely. // TODO: Looks like this autodiscovery also does not work w/Basic Auth? $api = API\ExchangeAutodiscover::getAPI($user, $password); $server = $api->getClient()->getServer(); $version = $api->getClient()->getVersion(); return sprintf('ews://%s:%s@%s', urlencode($user), urlencode($password), $server); } /** * Authenticate to EWS (initialize the EWS client) */ public function authenticate() { if (!empty($this->account->params['client_id'])) { $this->api = $this->authenticateWithOAuth2( $this->account->host, $this->account->username, $this->account->params['client_id'], $this->account->params['client_secret'], $this->account->params['tenant_id'] ); } else { // Note: This initializes the client, but not yet connects to the server // TODO: To know that the credentials work we'll have to do some API call. $this->api = $this->authenticateWithPassword( $this->account->host, $this->account->username, $this->account->password, $this->account->loginas ); } } /** * Autodiscover the server and authenticate the user */ protected function authenticateWithPassword(string $server, string $user, string $password, string $loginas = null) { // Note: Since 2023-01-01 EWS at Office365 requires OAuth2, no way back to basic auth. \Log::debug("[EWS] Using basic authentication on $server..."); $options = []; if ($loginas) { $options['impersonation'] = $loginas; } $this->engine->setOption('ews', [ 'options' => $options, 'server' => $server, ]); return API::withUsernameAndPassword($server, $user, $password, $this->apiOptions($options)); } /** * Authenticate with a token (Office365) */ protected function authenticateWithToken(string $server, string $user, string $token, $expires_at = null) { \Log::debug("[EWS] Using token authentication on $server..."); $options = ['impersonation' => $user]; $this->engine->setOption('ews', [ 'options' => $options, 'server' => $server, 'token' => $token, 'expires_at' => $expires_at, ]); return API::withCallbackToken($server, $token, $this->apiOptions($options)); } /** * Authenticate with OAuth2 (Office365) - get the token */ protected function authenticateWithOAuth2(string $server, string $user, string $client_id, string $client_secret, string $tenant_id) { // See https://github.com/Garethp/php-ews/blob/master/examples/basic/authenticatingWithOAuth.php // See https://github.com/Garethp/php-ews/issues/236#issuecomment-1292521527 // To register OAuth2 app goto https://entra.microsoft.com > Applications > App registrations \Log::debug("[EWS] Fetching OAuth2 token from $server..."); $scope = 'https://outlook.office365.com/.default'; $token_uri = "https://login.microsoftonline.com/{$tenant_id}/oauth2/v2.0/token"; // $authUri = "https://login.microsoftonline.com/{$tenant_id}/oauth2/authorize"; $response = Http::asForm() ->timeout(5) ->post($token_uri, [ 'client_id' => $client_id, 'client_secret' => $client_secret, 'scope' => $scope, 'grant_type' => 'client_credentials', ]) ->throwUnlessStatus(200); $token = $response->json('access_token'); // Note: Office365 default token expiration time is ~1h, $expires_in = $response->json('expires_in'); $expires_at = now()->addSeconds($expires_in)->toDateTimeString(); return $this->authenticateWithToken($server, $user, $token, $expires_at); } /** * Get folders hierarchy */ public function getFolders($types = []): array { // Get full folders hierarchy $options = [ 'Traversal' => 'Deep', ]; $folders = $this->api->getChildrenFolders('root', $options); $result = []; foreach ($folders as $folder) { $class = $folder->getFolderClass(); $type = $this->type_map[$class] ?? null; // Skip folder types we do not support (need) if (empty($type) || (!empty($types) && !in_array($type, $types))) { continue; } // Note: Folder names are localized $name = $fullname = $folder->getDisplayName(); $id = $folder->getFolderId()->getId(); $parentId = $folder->getParentFolderId()->getId(); // Create folder name with full path if ($parentId && !empty($result[$parentId])) { $fullname = $result[$parentId]->fullname . '/' . $name; } // Top-level folder, check if it's a special folder we should ignore // FIXME: Is there a better way to distinguish user folders from system ones? if ( in_array($fullname, $this->folder_exceptions) || strpos($fullname, 'OwaFV15.1All') === 0 ) { continue; } $result[$id] = Folder::fromArray([ 'id' => $folder->getFolderId()->toArray(true), 'total' => $folder->getTotalCount(), 'class' => $class, 'type' => $this->type_map[$class] ?? null, 'name' => $name, 'fullname' => $fullname, ]); } return $result; } /** * Fetch a list of folder items */ - public function fetchItemList(Folder $folder, $callback): void + public function fetchItemList(Folder $folder, $callback, Interface\ImporterInterface $importer): void { // Job processing - initialize environment $this->initEnv($this->engine->queue); // The folder is empty, we can stop here if (empty($folder->total)) { + // TODO: Delete all existing items? return; } + // Get items already imported + // TODO: This might be slow and/or memory expensive, we should consider + // whether storing list of imported items in some cache wouldn't be a better + // solution. Of course, cache would not get changes in the destination account. + $existing = $importer->getItems($folder); + + // Create X-MS-ID index for easier search in existing items + // Note: For some objects we could use UID (events), but for some we don't have UID in Exchange. + // Also because fetching extra properties here is problematic, we use X-MS-ID. + $existingIndex = []; + array_walk( + $existing, + function (&$item, $idx) use (&$existingIndex) { + if (!empty($item['x-ms-id'])) { + [$id, $changeKey] = explode('!', $item['x-ms-id']); + $item['changeKey'] = $changeKey; + $existingIndex[$id] = $idx; + unset($item['x-ms-id']); + } + } + ); + $request = [ // Exchange's maximum is 1000 'IndexedPageItemView' => ['MaxEntriesReturned' => 100, 'Offset' => 0, 'BasePoint' => 'Beginning'], 'ParentFolderIds' => $folder->id, 'Traversal' => 'Shallow', 'ItemShape' => [ 'BaseShape' => 'IdOnly', 'AdditionalProperties' => [ 'FieldURI' => ['FieldURI' => 'item:ItemClass'], ], ], ]; $request = Type::buildFromArray($request); // Note: It is not possible to get mimeContent with FindItem request // That's why we first get the list of object identifiers and // then call GetItem on each separately. // TODO: It might be feasible to get all properties for object types // for which we don't use MimeContent, for better performance. // Request first page $response = $this->api->getClient()->FindItem($request); // @phpstan-ignore-next-line foreach ($response as $item) { - if ($item = $this->toItem($item, $folder)) { + if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { $callback($item); } } // Request other pages until we got all while (!$response->isIncludesLastItemInRange()) { // @phpstan-ignore-next-line $response = $this->api->getNextPage($response); foreach ($response as $item) { - if ($item = $this->toItem($item, $folder)) { + if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) { $callback($item); } } } + + // TODO: Delete items that do not exist anymore? } /** * Fetching an item */ public function fetchItem(Item $item): string { // Job processing - initialize environment $this->initEnv($this->engine->queue); if ($driver = EWS\Item::factory($this, $item)) { return $driver->fetchItem($item); } throw new \Exception("Failed to fetch an item from EWS"); } /** * Get the source account */ public function getSourceAccount(): Account { return $this->engine->source; } /** * Get the destination account */ public function getDestinationAccount(): Account { return $this->engine->destination; } /** * Synchronize specified object */ - protected function toItem(Type $item, Folder $folder): ?Item + protected function toItem(Type $item, Folder $folder, $existing, $existingIndex): ?Item { + $id = $item->getItemId()->toArray(); + $exists = false; + + // Detect an existing item, skip if nothing changed + if (isset($existingIndex[$id['Id']])) { + $idx = $existingIndex[$id['Id']]; + + if ($existing[$idx]['changeKey'] == $id['ChangeKey']) { + return null; + } + + $existing = $existing[$idx]['href']; + } + $item = Item::fromArray([ - 'id' => $item->getItemId()->toArray(), + 'id' => $id, 'class' => $item->getItemClass(), 'folder' => $folder, + 'existing' => $existing, ]); // TODO: We don't need to instantiate Item at this point, instead // implement EWS\Item::validateClass() method if ($driver = EWS\Item::factory($this, $item)) { return $item; } return null; } /** * Set common API options */ protected function apiOptions(array $options): array { if (empty($options['version'])) { $options['version'] = API\ExchangeWebServices::VERSION_2013; } // If you want to inject your own GuzzleClient for the requests // $options['httpClient]' = $client; // In debug mode record all responses /* if (\config('app.debug')) { $options['httpPlayback'] = [ 'mode' => 'record', 'recordLocation' => \storage_path('ews'), ]; } */ return $options; } /** * Initialize environment for job execution * * @param Queue $queue Queue */ protected function initEnv(Queue $queue): void { $ews = $queue->data['options']['ews']; if (!empty($ews['token'])) { // TODO: Refresh the token if needed $this->api = API::withCallbackToken( $ews['server'], $ews['token'], $this->apiOptions($ews['options']) ); } else { $this->api = API::withUsernameAndPassword( $ews['server'], $this->account->username, $this->account->password, $this->apiOptions($ews['options']) ); } } } diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php index e98ca2a8..a9691438 100644 --- a/src/app/DataMigrator/EWS/Item.php +++ b/src/app/DataMigrator/EWS/Item.php @@ -1,172 +1,174 @@ driver = $driver; $this->folder = $folder; } /** * Factory method. * Returns object suitable to handle specified item type. */ public static function factory(EWS $driver, ItemInterface $item) { $item_class = str_replace('IPM.', '', $item->class); $item_class = "\App\DataMigrator\EWS\\{$item_class}"; if (class_exists($item_class)) { return new $item_class($driver, $item->folder); } } /** * Fetch the specified object and put into a file */ public function fetchItem(ItemInterface $item) { $itemId = $item->id; // Fetch the item $item = $this->driver->api->getItem($itemId, $this->getItemRequest()); $this->itemId = implode('!', $itemId); $uid = $this->getUID($item); \Log::debug("[EWS] Saving item {$uid}..."); // Apply type-specific format converters $content = $this->processItem($item); if (!is_string($content)) { return; } $uid = preg_replace('/[^a-zA-Z0-9_:@-]/', '', $uid); $location = $this->folder->location; if (!file_exists($location)) { mkdir($location, 0740, true); } $location .= '/' . $uid . '.' . $this->fileExtension(); file_put_contents($location, $content); return $location; } /** * Item conversion code */ abstract protected function processItem(Type $item); /** * Get GetItem request parameters */ protected function getItemRequest(): array { $request = [ 'ItemShape' => [ // Reqest default set of properties 'BaseShape' => 'Default', // Additional properties, e.g. LastModifiedTime // FIXME: How to add multiple properties here? 'AdditionalProperties' => [ 'FieldURI' => ['FieldURI' => 'item:LastModifiedTime'], ] ] ]; return $request; } /** * Fetch attachment object from Exchange */ protected function getAttachment(Type\FileAttachmentType $attachment) { $request = [ 'AttachmentIds' => [ $attachment->getAttachmentId()->toXmlObject() ], 'AttachmentShape' => [ 'IncludeMimeContent' => true, ] ]; return $this->driver->api->getClient()->GetAttachment($request); } /** * Get Item UID (Generate a new one if needed) */ protected function getUID(Type $item): string { if ($this->uid === null) { // We should generate an UID for objects that do not have it // and inject it into the output file // FIXME: Should we use e.g. md5($itemId->getId()) instead? + // It looks that ItemId on EWS consists of three parts separated with a slash, + // maybe using the last part as UID would be a solution $this->uid = \App\Utils::uuidStr(); } return $this->uid; } /** * Filename extension for cached file in-processing */ protected function fileExtension(): string { return constant(static::class . '::FILE_EXT') ?: 'txt'; } /** * VCard/iCal property formatting */ protected function formatProp($name, $value, array $params = []): string { $cal = new \Sabre\VObject\Component\VCalendar(); $prop = new \Sabre\VObject\Property\Text($cal, $name, $value, $params); $value = $prop->serialize(); // Revert escaping for some props if ($name == 'RRULE') { $value = str_replace("\\", '', $value); } return $value; } } diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php index 240f57a1..56f09d30 100644 --- a/src/app/DataMigrator/Engine.php +++ b/src/app/DataMigrator/Engine.php @@ -1,273 +1,275 @@ source = $source; $this->destination = $destination; $this->options = $options; // Create a unique identifier for the migration request $queue_id = md5(strval($source) . strval($destination) . $options['type']); // If queue exists, we'll display the progress only if ($queue = Queue::find($queue_id)) { // If queue contains no jobs, assume invalid // TODO: An better API to manage (reset) queues if (!$queue->jobs_started || !empty($options['force'])) { $queue->delete(); } else { while (true) { $this->debug(sprintf("Progress [%d of %d]\n", $queue->jobs_finished, $queue->jobs_started)); if ($queue->jobs_started == $queue->jobs_finished) { break; } sleep(1); $queue->refresh(); } return; } } // Initialize the source $this->exporter = $this->initDriver($source, ExporterInterface::class); $this->exporter->authenticate(); // Initialize the destination $this->importer = $this->initDriver($destination, ImporterInterface::class); $this->importer->authenticate(); // $this->debug("Source/destination user credentials verified."); $this->debug("Fetching folders hierarchy..."); // Create a queue $this->createQueue($queue_id); // We'll store output in storage/ tree $location = storage_path('export/') . $source->email; if (!file_exists($location)) { mkdir($location, 0740, true); } $types = preg_split('/\s*,\s*/', strtolower($this->options['type'] ?? '')); $folders = $this->exporter->getFolders($types); $count = 0; foreach ($folders as $folder) { $this->debug("Processing folder {$folder->fullname}..."); $folder->queueId = $queue_id; $folder->location = $location; // Dispatch the job (for async execution) Jobs\FolderJob::dispatch($folder); $count++; } $this->queue->bumpJobsStarted($count); $this->debug(sprintf('Done. %d %s created in queue: %s.', $count, Str::plural('job', $count), $queue_id)); } /** * Processing of a folder synchronization */ public function processFolder(Folder $folder): void { // Job processing - initialize environment $this->envFromQueue($folder->queueId); // Create the folder on the destination server $this->importer->createFolder($folder); $count = 0; // Fetch items from the source $this->exporter->fetchItemList( $folder, function (Item $item) use (&$count) { // Dispatch the job (for async execution) Jobs\ItemJob::dispatch($item); $count++; - } + }, + $this->importer ); if ($count) { $this->queue->bumpJobsStarted($count); } $this->queue->bumpJobsFinished(); } /** * Processing of item synchronization */ public function processItem(Item $item): void { // Job processing - initialize environment $this->envFromQueue($item->folder->queueId); if ($filename = $this->exporter->fetchItem($item)) { - $this->importer->createItemFromFile($filename, $item->folder); + $item->filename = $filename; + $this->importer->createItem($item); // TODO: remove the file } $this->queue->bumpJobsFinished(); } /** * Print progress/debug information */ public function debug($line) { if (!empty($this->options['stdout'])) { $output = new \Symfony\Component\Console\Output\ConsoleOutput(); $output->writeln("$line"); } else { \Log::debug("[DataMigrator] $line"); } } /** * Set migration queue option. Use this if you need to pass * some data between queue processes. */ public function setOption(string $name, $value): void { $this->options[$name] = $value; if ($this->queue) { $this->queue->data = $this->queueData(); $this->queue->save(); } } /** * Create a queue for the request * * @param string $queue_id Unique queue identifier */ protected function createQueue(string $queue_id): void { $this->queue = new Queue(); $this->queue->id = $queue_id; $this->queue->data = $this->queueData(); $this->queue->save(); } /** * Prepare queue data */ protected function queueData() { $options = $this->options; unset($options['stdout']); // jobs aren't in stdout anymore // TODO: data should be encrypted return [ 'source' => (string) $this->source, 'destination' => (string) $this->destination, 'options' => $options, ]; } /** * Initialize environment for job execution * * @param string $queueId Queue identifier */ protected function envFromQueue(string $queueId): void { $this->queue = Queue::findOrFail($queueId); $this->source = new Account($this->queue->data['source']); $this->destination = new Account($this->queue->data['destination']); $this->options = $this->queue->data['options']; $this->importer = $this->initDriver($this->destination, ImporterInterface::class); $this->exporter = $this->initDriver($this->source, ExporterInterface::class); } /** * Initialize (and select) migration driver */ protected function initDriver(Account $account, string $interface) { switch ($account->scheme) { case 'ews': $driver = new EWS($account, $this); break; case 'dav': case 'davs': $driver = new DAV($account, $this); break; /* case 'imap': case 'imaps': $driver = new IMAP($account, $this); break; */ default: throw new \Exception("Failed to init driver for '{$account->scheme}'"); } // Make sure driver is used in the direction it supports if (!is_a($driver, $interface)) { throw new \Exception(sprintf( "'%s' driver does not implement %s", class_basename($driver), class_basename($interface) )); } return $driver; } } diff --git a/src/app/DataMigrator/Interface/ExporterInterface.php b/src/app/DataMigrator/Interface/ExporterInterface.php index 76316443..250106ef 100644 --- a/src/app/DataMigrator/Interface/ExporterInterface.php +++ b/src/app/DataMigrator/Interface/ExporterInterface.php @@ -1,36 +1,36 @@ $value) { $obj->{$key} = $value; } return $obj; } }