Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F15398913
D4836.id13836.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
16 KB
Referenced Files
None
Subscribers
None
D4836.id13836.diff
View Options
diff --git a/src/app/Backends/DAV.php b/src/app/Backends/DAV.php
--- a/src/app/Backends/DAV.php
+++ b/src/app/Backends/DAV.php
@@ -11,7 +11,7 @@
public const TYPE_VCARD = 'VCARD';
public const TYPE_NOTIFICATION = 'NOTIFICATION';
- protected const NAMESPACES = [
+ public const NAMESPACES = [
self::TYPE_VEVENT => 'urn:ietf:params:xml:ns:caldav',
self::TYPE_VTODO => 'urn:ietf:params:xml:ns:caldav',
self::TYPE_VCARD => 'urn:ietf:params:xml:ns:carddav',
@@ -350,40 +350,17 @@
/**
* Search DAV objects in a folder.
*
- * @param string $location Folder location
- * @param string $component Object type (VEVENT, VTODO, VCARD)
+ * @param string $location Folder location
+ * @param DAV\Search $search Search request parameters
+ * @param ?callable $callback Callback to execute on every object
*
* @return false|array Objects metadata on success, False on error
*/
- public function search(string $location, string $component)
+ public function search(string $location, DAV\Search $search, $callback = null)
{
- $queries = [
- self::TYPE_VEVENT => 'calendar-query',
- self::TYPE_VTODO => 'calendar-query',
- self::TYPE_VCARD => 'addressbook-query',
- ];
-
- $filter = '';
- if ($component != self::TYPE_VCARD) {
- $filter = '<c:comp-filter name="VCALENDAR">'
- . '<c:comp-filter name="' . $component . '" />'
- . '</c:comp-filter>';
- }
-
- // TODO: Make filter an argument of this function to build all kind of queries.
- // It probably should be a separate object e.g. DAV\Filter.
- // TODO: List of object props to return should also be an argument, so we not only
- // could fetch "an index" but also any of object's data.
-
- $body = '<?xml version="1.0" encoding="utf-8"?>'
- . ' <c:' . $queries[$component] . ' xmlns:d="DAV:" xmlns:c="' . self::NAMESPACES[$component] . '">'
- . '<d:prop>'
- . '<d:getetag />'
- . '</d:prop>'
- . ($filter ? "<c:filter>$filter</c:filter>" : '')
- . '</c:' . $queries[$component] . '>';
+ $headers = ['Depth' => $search->depth, 'Prefer' => 'return-minimal'];
- $response = $this->request($location, 'REPORT', $body, ['Depth' => 1, 'Prefer' => 'return-minimal']);
+ $response = $this->request($location, 'REPORT', $search, $headers);
if (empty($response)) {
\Log::error("Failed to get objects from the DAV server.");
@@ -393,7 +370,13 @@
$objects = [];
foreach ($response->getElementsByTagName('response') as $element) {
- $objects[] = $this->objectFromElement($element, $component);
+ $object = $this->objectFromElement($element, $search->component);
+
+ if ($callback) {
+ $object = $callback($object);
+ }
+
+ $objects[] = $object;
}
return $objects;
diff --git a/src/app/Backends/DAV/CommonObject.php b/src/app/Backends/DAV/CommonObject.php
--- a/src/app/Backends/DAV/CommonObject.php
+++ b/src/app/Backends/DAV/CommonObject.php
@@ -16,6 +16,9 @@
/** @var ?string Object UID */
public $uid;
+ /** @var array Custom properties (key->value) */
+ public $custom = [];
+
/**
* Create DAV object from a DOMElement element
diff --git a/src/app/Backends/DAV/Vcard.php b/src/app/Backends/DAV/Vcard.php
--- a/src/app/Backends/DAV/Vcard.php
+++ b/src/app/Backends/DAV/Vcard.php
@@ -2,11 +2,19 @@
namespace App\Backends\DAV;
+use Illuminate\Support\Str;
+use Sabre\VObject\Reader;
+use Sabre\VObject\Property;
+
class Vcard extends CommonObject
{
/** @var string Object content type (of the string representation) */
public $contentType = 'text/vcard; charset=utf-8';
+ public $fn;
+ public $rev;
+
+
/**
* Create event object from a DOMElement element
*
@@ -33,7 +41,40 @@
*/
protected function fromVcard(string $vcard): void
{
- // TODO
+ $vobject = Reader::read($vcard, Reader::OPTION_FORGIVING | Reader::OPTION_IGNORE_INVALID_LINES);
+
+ if ($vobject->name != 'VCARD') {
+ // FIXME: throw an exception?
+ return;
+ }
+
+ $string_properties = [
+ 'FN',
+ 'REV',
+ 'UID',
+ ];
+
+ foreach ($vobject->children() as $prop) {
+ if (!($prop instanceof Property)) {
+ continue;
+ }
+
+ switch ($prop->name) {
+ // TODO: Map all vCard properties to class properties
+
+ default:
+ // map string properties
+ if (in_array($prop->name, $string_properties)) {
+ $key = Str::camel(strtolower($prop->name));
+ $this->{$key} = (string) $prop;
+ }
+
+ // custom properties
+ if (\str_starts_with($prop->name, 'X-')) {
+ $this->custom[$prop->name] = (string) $prop;
+ }
+ }
+ }
}
/**
diff --git a/src/app/Backends/DAV/Vevent.php b/src/app/Backends/DAV/Vevent.php
--- a/src/app/Backends/DAV/Vevent.php
+++ b/src/app/Backends/DAV/Vevent.php
@@ -187,6 +187,11 @@
}
break;
+
+ default:
+ if (\str_starts_with($prop->name, 'X-')) {
+ $this->custom[$prop->name] = (string) $prop;
+ }
}
}
diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php
--- a/src/app/DataMigrator/DAV.php
+++ b/src/app/DataMigrator/DAV.php
@@ -5,7 +5,9 @@
use App\Backends\DAV as DAVClient;
use App\Backends\DAV\Opaque as DAVOpaque;
use App\Backends\DAV\Folder as DAVFolder;
+use App\Backends\DAV\Search as DAVSearch;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use App\Utils;
class DAV implements Interface\ImporterInterface
@@ -60,19 +62,33 @@
/**
* Create an item in a folder.
*
- * @param string $filename File location
- * @param Folder $folder Folder
+ * @param Item $item Item to import
*
* @throws \Exception
*/
- public function createItemFromFile(string $filename, Folder $folder): void
+ public function createItem(Item $item): void
{
- $href = $this->getFolderPath($folder) . '/' . pathinfo($filename, PATHINFO_BASENAME);
+ // TODO: For now we do DELETE + PUT. It's because the UID might have changed (which
+ // is the case with e.g. contacts from EWS) causing a discrepancy between UID and href.
+ // This is not necessarily a problem and would not happen to calendar events.
+ // So, maybe we could improve that so DELETE is not needed.
+ if ($item->existing) {
+ try {
+ $this->client->delete($item->existing);
+ } catch (\Illuminate\Http\Client\RequestException $e) {
+ // ignore 404 response, item removed in meantime?
+ if ($e->getCode() != 404) {
+ throw $e;
+ }
+ }
+ }
+
+ $href = $this->getFolderPath($item->folder) . '/' . pathinfo($item->filename, PATHINFO_BASENAME);
- $object = new DAVOpaque($filename);
+ $object = new DAVOpaque($item->filename);
$object->href = $href;
- switch ($folder->type) {
+ switch ($item->folder->type) {
case Engine::TYPE_EVENT:
case Engine::TYPE_TASK:
$object->contentType = 'text/calendar; charset=utf-8';
@@ -84,7 +100,7 @@
}
if ($this->client->create($object) === false) {
- throw new \Exception("Failed to save object into DAV server at {$href}");
+ throw new \Exception("Failed to save DAV object at {$href}");
}
}
@@ -130,6 +146,57 @@
}
}
+ /**
+ * Get a list of folder items, limited to their essential propeties
+ * used in incremental migration.
+ *
+ * @param Folder $folder Folder data
+ *
+ * @throws \Exception on error
+ */
+ public function getItems(Folder $folder): array
+ {
+ $dav_type = $this->type2DAV($folder->type);
+ $location = $this->getFolderPath($folder);
+
+ $search = new DAVSearch($dav_type);
+
+ // TODO: We request only properties relevant to incremental migration,
+ // i.e. to find that something exists and its last update time.
+ // Some servers (iRony) do ignore that and return full VCARD/VEVENT/VTODO
+ // content, if there's many objects we'll have a memory limit issue.
+ // Also, this list should be controlled by the exporter.
+ $search->dataProperties = ['UID', 'X-MS-ID', 'REV'];
+
+ $items = $this->client->search(
+ $location,
+ $search,
+ function ($item) use ($dav_type) {
+ // Slim down the result to properties we might need
+ $result = [
+ 'href' => $item->href,
+ 'uid' => $item->uid,
+ 'x-ms-id' => $item->custom['X-MS-ID'] ?? null,
+ ];
+ /*
+ switch ($dav_type) {
+ case DAVClient::TYPE_VCARD:
+ $result['rev'] = $item->rev;
+ break;
+ }
+ */
+
+ return $result;
+ }
+ );
+
+ if ($items === false) {
+ throw new \Exception("Failed to get items from a DAV folder {$location}");
+ }
+
+ return $items;
+ }
+
/**
* Get folder relative URI
*/
diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php
--- a/src/app/DataMigrator/EWS.php
+++ b/src/app/DataMigrator/EWS.php
@@ -263,16 +263,39 @@
/**
* Fetch a list of folder items
*/
- public function fetchItemList(Folder $folder, $callback): void
+ public function fetchItemList(Folder $folder, $callback, Interface\ImporterInterface $importer): void
{
// Job processing - initialize environment
$this->initEnv($this->engine->queue);
// The folder is empty, we can stop here
if (empty($folder->total)) {
+ // TODO: Delete all existing items?
return;
}
+ // Get items already imported
+ // TODO: This might be slow and/or memory expensive, we should consider
+ // whether storing list of imported items in some cache wouldn't be a better
+ // solution. Of course, cache would not get changes in the destination account.
+ $existing = $importer->getItems($folder);
+
+ // Create X-MS-ID index for easier search in existing items
+ // Note: For some objects we could use UID (events), but for some we don't have UID in Exchange.
+ // Also because fetching extra properties here is problematic, we use X-MS-ID.
+ $existingIndex = [];
+ array_walk(
+ $existing,
+ function (&$item, $idx) use (&$existingIndex) {
+ if (!empty($item['x-ms-id'])) {
+ [$id, $changeKey] = explode('!', $item['x-ms-id']);
+ $item['changeKey'] = $changeKey;
+ $existingIndex[$id] = $idx;
+ unset($item['x-ms-id']);
+ }
+ }
+ );
+
$request = [
// Exchange's maximum is 1000
'IndexedPageItemView' => ['MaxEntriesReturned' => 100, 'Offset' => 0, 'BasePoint' => 'Beginning'],
@@ -300,7 +323,7 @@
// @phpstan-ignore-next-line
foreach ($response as $item) {
- if ($item = $this->toItem($item, $folder)) {
+ if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
$callback($item);
}
}
@@ -311,11 +334,13 @@
$response = $this->api->getNextPage($response);
foreach ($response as $item) {
- if ($item = $this->toItem($item, $folder)) {
+ if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
$callback($item);
}
}
}
+
+ // TODO: Delete items that do not exist anymore?
}
/**
@@ -352,12 +377,27 @@
/**
* Synchronize specified object
*/
- protected function toItem(Type $item, Folder $folder): ?Item
+ protected function toItem(Type $item, Folder $folder, $existing, $existingIndex): ?Item
{
+ $id = $item->getItemId()->toArray();
+ $exists = false;
+
+ // Detect an existing item, skip if nothing changed
+ if (isset($existingIndex[$id['Id']])) {
+ $idx = $existingIndex[$id['Id']];
+
+ if ($existing[$idx]['changeKey'] == $id['ChangeKey']) {
+ return null;
+ }
+
+ $existing = $existing[$idx]['href'];
+ }
+
$item = Item::fromArray([
- 'id' => $item->getItemId()->toArray(),
+ 'id' => $id,
'class' => $item->getItemClass(),
'folder' => $folder,
+ 'existing' => $existing,
]);
// TODO: We don't need to instantiate Item at this point, instead
diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php
--- a/src/app/DataMigrator/EWS/Item.php
+++ b/src/app/DataMigrator/EWS/Item.php
@@ -138,6 +138,8 @@
// We should generate an UID for objects that do not have it
// and inject it into the output file
// FIXME: Should we use e.g. md5($itemId->getId()) instead?
+ // It looks that ItemId on EWS consists of three parts separated with a slash,
+ // maybe using the last part as UID would be a solution
$this->uid = \App\Utils::uuidStr();
}
diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
--- a/src/app/DataMigrator/Engine.php
+++ b/src/app/DataMigrator/Engine.php
@@ -135,7 +135,8 @@
// Dispatch the job (for async execution)
Jobs\ItemJob::dispatch($item);
$count++;
- }
+ },
+ $this->importer
);
if ($count) {
@@ -154,7 +155,8 @@
$this->envFromQueue($item->folder->queueId);
if ($filename = $this->exporter->fetchItem($item)) {
- $this->importer->createItemFromFile($filename, $item->folder);
+ $item->filename = $filename;
+ $this->importer->createItem($item);
// TODO: remove the file
}
diff --git a/src/app/DataMigrator/Interface/ExporterInterface.php b/src/app/DataMigrator/Interface/ExporterInterface.php
--- a/src/app/DataMigrator/Interface/ExporterInterface.php
+++ b/src/app/DataMigrator/Interface/ExporterInterface.php
@@ -27,7 +27,7 @@
/**
* Fetch a list of folder items
*/
- public function fetchItemList(Folder $folder, $callback): void;
+ public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void;
/**
* Fetching an item
diff --git a/src/app/DataMigrator/Interface/ImporterInterface.php b/src/app/DataMigrator/Interface/ImporterInterface.php
--- a/src/app/DataMigrator/Interface/ImporterInterface.php
+++ b/src/app/DataMigrator/Interface/ImporterInterface.php
@@ -22,12 +22,11 @@
/**
* Create an item in a folder.
*
- * @param string $filename File location
- * @param Folder $folder Folder object
+ * @param Item $item Item to import
*
* @throws \Exception
*/
- public function createItemFromFile(string $filename, Folder $folder): void;
+ public function createItem(Item $item): void;
/**
* Create a folder.
@@ -37,4 +36,10 @@
* @throws \Exception
*/
public function createFolder(Folder $folder): void;
+
+ /**
+ * Get a list of folder items, limited to their essential propeties
+ * used in incremental migration to skip unchanged items.
+ */
+ public function getItems(Folder $folder): array;
}
diff --git a/src/app/DataMigrator/Interface/Item.php b/src/app/DataMigrator/Interface/Item.php
--- a/src/app/DataMigrator/Interface/Item.php
+++ b/src/app/DataMigrator/Interface/Item.php
@@ -16,6 +16,12 @@
/** @var string Object class */
public $class;
+ /** @var false|string Identifier/Location of the item if exists in the destination folder */
+ public $existing = false;
+
+ /** @var ?string Exported object location in the local storage */
+ public $filename;
+
public static function fromArray(array $data = [])
{
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Fri, Sep 20, 3:04 AM (9 h, 25 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
9466278
Default Alt Text
D4836.id13836.diff (16 KB)
Attached To
Mode
D4836: Incremental migration EWS -> DAV
Attached
Detach File
Event Timeline
Log In to Comment