diff --git a/src/app/Backends/DAV.php b/src/app/Backends/DAV.php
--- a/src/app/Backends/DAV.php
+++ b/src/app/Backends/DAV.php
@@ -11,7 +11,7 @@
public const TYPE_VCARD = 'VCARD';
public const TYPE_NOTIFICATION = 'NOTIFICATION';
- protected const NAMESPACES = [
+ public const NAMESPACES = [
self::TYPE_VEVENT => 'urn:ietf:params:xml:ns:caldav',
self::TYPE_VTODO => 'urn:ietf:params:xml:ns:caldav',
self::TYPE_VCARD => 'urn:ietf:params:xml:ns:carddav',
@@ -350,40 +350,17 @@
/**
* Search DAV objects in a folder.
*
- * @param string $location Folder location
- * @param string $component Object type (VEVENT, VTODO, VCARD)
+ * @param string $location Folder location
+ * @param DAV\Search $search Search request parameters
+ * @param ?callable $callback Callback to execute on every object
*
* @return false|array Objects metadata on success, False on error
*/
- public function search(string $location, string $component)
+ public function search(string $location, DAV\Search $search, $callback = null)
{
- $queries = [
- self::TYPE_VEVENT => 'calendar-query',
- self::TYPE_VTODO => 'calendar-query',
- self::TYPE_VCARD => 'addressbook-query',
- ];
-
- $filter = '';
- if ($component != self::TYPE_VCARD) {
- $filter = ''
- . ''
- . '';
- }
-
- // TODO: Make filter an argument of this function to build all kind of queries.
- // It probably should be a separate object e.g. DAV\Filter.
- // TODO: List of object props to return should also be an argument, so we not only
- // could fetch "an index" but also any of object's data.
-
- $body = ''
- . ' '
- . ''
- . ''
- . ''
- . ($filter ? "$filter" : '')
- . '';
+ $headers = ['Depth' => $search->depth, 'Prefer' => 'return-minimal'];
- $response = $this->request($location, 'REPORT', $body, ['Depth' => 1, 'Prefer' => 'return-minimal']);
+ $response = $this->request($location, 'REPORT', $search, $headers);
if (empty($response)) {
\Log::error("Failed to get objects from the DAV server.");
@@ -393,7 +370,13 @@
$objects = [];
foreach ($response->getElementsByTagName('response') as $element) {
- $objects[] = $this->objectFromElement($element, $component);
+ $object = $this->objectFromElement($element, $search->component);
+
+ if ($callback) {
+ $object = $callback($object);
+ }
+
+ $objects[] = $object;
}
return $objects;
diff --git a/src/app/Backends/DAV/CommonObject.php b/src/app/Backends/DAV/CommonObject.php
--- a/src/app/Backends/DAV/CommonObject.php
+++ b/src/app/Backends/DAV/CommonObject.php
@@ -16,6 +16,9 @@
/** @var ?string Object UID */
public $uid;
+ /** @var array Custom properties (key->value) */
+ public $custom = [];
+
/**
* Create DAV object from a DOMElement element
diff --git a/src/app/Backends/DAV/Vcard.php b/src/app/Backends/DAV/Vcard.php
--- a/src/app/Backends/DAV/Vcard.php
+++ b/src/app/Backends/DAV/Vcard.php
@@ -2,11 +2,19 @@
namespace App\Backends\DAV;
+use Illuminate\Support\Str;
+use Sabre\VObject\Reader;
+use Sabre\VObject\Property;
+
class Vcard extends CommonObject
{
/** @var string Object content type (of the string representation) */
public $contentType = 'text/vcard; charset=utf-8';
+ public $fn;
+ public $rev;
+
+
/**
* Create event object from a DOMElement element
*
@@ -33,7 +41,40 @@
*/
protected function fromVcard(string $vcard): void
{
- // TODO
+ $vobject = Reader::read($vcard, Reader::OPTION_FORGIVING | Reader::OPTION_IGNORE_INVALID_LINES);
+
+ if ($vobject->name != 'VCARD') {
+ // FIXME: throw an exception?
+ return;
+ }
+
+ $string_properties = [
+ 'FN',
+ 'REV',
+ 'UID',
+ ];
+
+ foreach ($vobject->children() as $prop) {
+ if (!($prop instanceof Property)) {
+ continue;
+ }
+
+ switch ($prop->name) {
+ // TODO: Map all vCard properties to class properties
+
+ default:
+ // map string properties
+ if (in_array($prop->name, $string_properties)) {
+ $key = Str::camel(strtolower($prop->name));
+ $this->{$key} = (string) $prop;
+ }
+
+ // custom properties
+ if (\str_starts_with($prop->name, 'X-')) {
+ $this->custom[$prop->name] = (string) $prop;
+ }
+ }
+ }
}
/**
diff --git a/src/app/Backends/DAV/Vevent.php b/src/app/Backends/DAV/Vevent.php
--- a/src/app/Backends/DAV/Vevent.php
+++ b/src/app/Backends/DAV/Vevent.php
@@ -187,6 +187,11 @@
}
break;
+
+ default:
+ if (\str_starts_with($prop->name, 'X-')) {
+ $this->custom[$prop->name] = (string) $prop;
+ }
}
}
diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php
--- a/src/app/DataMigrator/DAV.php
+++ b/src/app/DataMigrator/DAV.php
@@ -5,7 +5,9 @@
use App\Backends\DAV as DAVClient;
use App\Backends\DAV\Opaque as DAVOpaque;
use App\Backends\DAV\Folder as DAVFolder;
+use App\Backends\DAV\Search as DAVSearch;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use App\Utils;
class DAV implements Interface\ImporterInterface
@@ -60,19 +62,33 @@
/**
* Create an item in a folder.
*
- * @param string $filename File location
- * @param Folder $folder Folder
+ * @param Item $item Item to import
*
* @throws \Exception
*/
- public function createItemFromFile(string $filename, Folder $folder): void
+ public function createItem(Item $item): void
{
- $href = $this->getFolderPath($folder) . '/' . pathinfo($filename, PATHINFO_BASENAME);
+ // TODO: For now we do DELETE + PUT. It's because the UID might have changed (which
+ // is the case with e.g. contacts from EWS) causing a discrepancy between UID and href.
+ // This is not necessarily a problem and would not happen to calendar events.
+ // So, maybe we could improve that so DELETE is not needed.
+ if ($item->existing) {
+ try {
+ $this->client->delete($item->existing);
+ } catch (\Illuminate\Http\Client\RequestException $e) {
+ // ignore 404 response, item removed in meantime?
+ if ($e->getCode() != 404) {
+ throw $e;
+ }
+ }
+ }
+
+ $href = $this->getFolderPath($item->folder) . '/' . pathinfo($item->filename, PATHINFO_BASENAME);
- $object = new DAVOpaque($filename);
+ $object = new DAVOpaque($item->filename);
$object->href = $href;
- switch ($folder->type) {
+ switch ($item->folder->type) {
case Engine::TYPE_EVENT:
case Engine::TYPE_TASK:
$object->contentType = 'text/calendar; charset=utf-8';
@@ -84,7 +100,7 @@
}
if ($this->client->create($object) === false) {
- throw new \Exception("Failed to save object into DAV server at {$href}");
+ throw new \Exception("Failed to save DAV object at {$href}");
}
}
@@ -130,6 +146,57 @@
}
}
+ /**
+ * Get a list of folder items, limited to their essential propeties
+ * used in incremental migration.
+ *
+ * @param Folder $folder Folder data
+ *
+ * @throws \Exception on error
+ */
+ public function getItems(Folder $folder): array
+ {
+ $dav_type = $this->type2DAV($folder->type);
+ $location = $this->getFolderPath($folder);
+
+ $search = new DAVSearch($dav_type);
+
+ // TODO: We request only properties relevant to incremental migration,
+ // i.e. to find that something exists and its last update time.
+ // Some servers (iRony) do ignore that and return full VCARD/VEVENT/VTODO
+ // content, if there's many objects we'll have a memory limit issue.
+ // Also, this list should be controlled by the exporter.
+ $search->dataProperties = ['UID', 'X-MS-ID', 'REV'];
+
+ $items = $this->client->search(
+ $location,
+ $search,
+ function ($item) use ($dav_type) {
+ // Slim down the result to properties we might need
+ $result = [
+ 'href' => $item->href,
+ 'uid' => $item->uid,
+ 'x-ms-id' => $item->custom['X-MS-ID'] ?? null,
+ ];
+ /*
+ switch ($dav_type) {
+ case DAVClient::TYPE_VCARD:
+ $result['rev'] = $item->rev;
+ break;
+ }
+ */
+
+ return $result;
+ }
+ );
+
+ if ($items === false) {
+ throw new \Exception("Failed to get items from a DAV folder {$location}");
+ }
+
+ return $items;
+ }
+
/**
* Get folder relative URI
*/
diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php
--- a/src/app/DataMigrator/EWS.php
+++ b/src/app/DataMigrator/EWS.php
@@ -263,16 +263,39 @@
/**
* Fetch a list of folder items
*/
- public function fetchItemList(Folder $folder, $callback): void
+ public function fetchItemList(Folder $folder, $callback, Interface\ImporterInterface $importer): void
{
// Job processing - initialize environment
$this->initEnv($this->engine->queue);
// The folder is empty, we can stop here
if (empty($folder->total)) {
+ // TODO: Delete all existing items?
return;
}
+ // Get items already imported
+ // TODO: This might be slow and/or memory expensive, we should consider
+ // whether storing list of imported items in some cache wouldn't be a better
+ // solution. Of course, cache would not get changes in the destination account.
+ $existing = $importer->getItems($folder);
+
+ // Create X-MS-ID index for easier search in existing items
+ // Note: For some objects we could use UID (events), but for some we don't have UID in Exchange.
+ // Also because fetching extra properties here is problematic, we use X-MS-ID.
+ $existingIndex = [];
+ array_walk(
+ $existing,
+ function (&$item, $idx) use (&$existingIndex) {
+ if (!empty($item['x-ms-id'])) {
+ [$id, $changeKey] = explode('!', $item['x-ms-id']);
+ $item['changeKey'] = $changeKey;
+ $existingIndex[$id] = $idx;
+ unset($item['x-ms-id']);
+ }
+ }
+ );
+
$request = [
// Exchange's maximum is 1000
'IndexedPageItemView' => ['MaxEntriesReturned' => 100, 'Offset' => 0, 'BasePoint' => 'Beginning'],
@@ -300,7 +323,7 @@
// @phpstan-ignore-next-line
foreach ($response as $item) {
- if ($item = $this->toItem($item, $folder)) {
+ if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
$callback($item);
}
}
@@ -311,11 +334,13 @@
$response = $this->api->getNextPage($response);
foreach ($response as $item) {
- if ($item = $this->toItem($item, $folder)) {
+ if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
$callback($item);
}
}
}
+
+ // TODO: Delete items that do not exist anymore?
}
/**
@@ -352,12 +377,27 @@
/**
* Synchronize specified object
*/
- protected function toItem(Type $item, Folder $folder): ?Item
+ protected function toItem(Type $item, Folder $folder, $existing, $existingIndex): ?Item
{
+ $id = $item->getItemId()->toArray();
+ $exists = false;
+
+ // Detect an existing item, skip if nothing changed
+ if (isset($existingIndex[$id['Id']])) {
+ $idx = $existingIndex[$id['Id']];
+
+ if ($existing[$idx]['changeKey'] == $id['ChangeKey']) {
+ return null;
+ }
+
+ $existing = $existing[$idx]['href'];
+ }
+
$item = Item::fromArray([
- 'id' => $item->getItemId()->toArray(),
+ 'id' => $id,
'class' => $item->getItemClass(),
'folder' => $folder,
+ 'existing' => $existing,
]);
// TODO: We don't need to instantiate Item at this point, instead
diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php
--- a/src/app/DataMigrator/EWS/Item.php
+++ b/src/app/DataMigrator/EWS/Item.php
@@ -138,6 +138,8 @@
// We should generate an UID for objects that do not have it
// and inject it into the output file
// FIXME: Should we use e.g. md5($itemId->getId()) instead?
+ // It looks that ItemId on EWS consists of three parts separated with a slash,
+ // maybe using the last part as UID would be a solution
$this->uid = \App\Utils::uuidStr();
}
diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
--- a/src/app/DataMigrator/Engine.php
+++ b/src/app/DataMigrator/Engine.php
@@ -135,7 +135,8 @@
// Dispatch the job (for async execution)
Jobs\ItemJob::dispatch($item);
$count++;
- }
+ },
+ $this->importer
);
if ($count) {
@@ -154,7 +155,8 @@
$this->envFromQueue($item->folder->queueId);
if ($filename = $this->exporter->fetchItem($item)) {
- $this->importer->createItemFromFile($filename, $item->folder);
+ $item->filename = $filename;
+ $this->importer->createItem($item);
// TODO: remove the file
}
diff --git a/src/app/DataMigrator/Interface/ExporterInterface.php b/src/app/DataMigrator/Interface/ExporterInterface.php
--- a/src/app/DataMigrator/Interface/ExporterInterface.php
+++ b/src/app/DataMigrator/Interface/ExporterInterface.php
@@ -27,7 +27,7 @@
/**
* Fetch a list of folder items
*/
- public function fetchItemList(Folder $folder, $callback): void;
+ public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void;
/**
* Fetching an item
diff --git a/src/app/DataMigrator/Interface/ImporterInterface.php b/src/app/DataMigrator/Interface/ImporterInterface.php
--- a/src/app/DataMigrator/Interface/ImporterInterface.php
+++ b/src/app/DataMigrator/Interface/ImporterInterface.php
@@ -22,12 +22,11 @@
/**
* Create an item in a folder.
*
- * @param string $filename File location
- * @param Folder $folder Folder object
+ * @param Item $item Item to import
*
* @throws \Exception
*/
- public function createItemFromFile(string $filename, Folder $folder): void;
+ public function createItem(Item $item): void;
/**
* Create a folder.
@@ -37,4 +36,10 @@
* @throws \Exception
*/
public function createFolder(Folder $folder): void;
+
+ /**
+ * Get a list of folder items, limited to their essential propeties
+ * used in incremental migration to skip unchanged items.
+ */
+ public function getItems(Folder $folder): array;
}
diff --git a/src/app/DataMigrator/Interface/Item.php b/src/app/DataMigrator/Interface/Item.php
--- a/src/app/DataMigrator/Interface/Item.php
+++ b/src/app/DataMigrator/Interface/Item.php
@@ -16,6 +16,12 @@
/** @var string Object class */
public $class;
+ /** @var false|string Identifier/Location of the item if exists in the destination folder */
+ public $existing = false;
+
+ /** @var ?string Exported object location in the local storage */
+ public $filename;
+
public static function fromArray(array $data = [])
{