D4857.id13917.diff
No OneTemporary
Actions

Size

40 KB

Referenced Files

None

Subscribers

None

D4857.id13917.diff
View Options

	diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php
	--- a/src/app/DataMigrator/DAV.php
	+++ b/src/app/DataMigrator/DAV.php
	@@ -67,13 +67,21 @@
	*/
	public function createItem(Item $item): void
	{
	- if ($item->existing) {
	- $href = $item->existing;
	+ $is_file = false;
	+ $href = $item->existing ?: null;
	+
	+ if (strlen($item->content)) {
	+ $content = $item->content;
	} else {
	- $href = $this->getFolderPath($item->folder) . '/' . pathinfo($item->filename, PATHINFO_BASENAME);
	+ $content = $item->filename;
	+ $is_file = true;
	+ }
	+
	+ if (empty($href)) {
	+ $href = $this->getFolderPath($item->folder) . '/' . basename($item->filename);
	}

	- $object = new DAVOpaque($item->filename, true);
	+ $object = new DAVOpaque($content, $is_file);
	$object->href = $href;

	switch ($item->folder->type) {
	@@ -139,15 +147,6 @@
	*/
	public function fetchItem(Item $item): void
	{
	- // Save the item content to a file
	- $location = $item->folder->location;
	-
	- if (!file_exists($location)) {
	- mkdir($location, 0740, true);
	- }
	-
	- $location .= '/' . basename($item->id);
	-
	$result = $this->client->getObjects(dirname($item->id), $this->type2DAV($item->folder->type), [$item->id]);

	if ($result === false) {
	@@ -156,11 +155,21 @@

	// TODO: Do any content changes, e.g. organizer/attendee email migration

	- if (file_put_contents($location, (string) $result[0]) === false) {
	- throw new \Exception("Failed to write to {$location}");
	- }
	+ $content = (string) $result[0];
	+
	+ if (strlen($content) > Engine::MAX_ITEM_SIZE) {
	+ // Save the item content to a file
	+ $location = $item->folder->tempFileLocation(basename($item->id));
	+
	+ if (file_put_contents($location, $content) === false) {
	+ throw new \Exception("Failed to write to {$location}");
	+ }

	- $item->filename = $location;
	+ $item->filename = $location;
	+ } else {
	+ $item->content = $content;
	+ $item->filename = basename($item->id);
	+ }
	}

	/**
	diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php
	--- a/src/app/DataMigrator/EWS.php
	+++ b/src/app/DataMigrator/EWS.php
	@@ -4,6 +4,7 @@

	use App\DataMigrator\Interface\Folder;
	use App\DataMigrator\Interface\Item;
	+use App\DataMigrator\Interface\ItemSet;
	use garethp\ews\API;
	use garethp\ews\API\Type;
	use Illuminate\Support\Facades\Http;
	@@ -13,6 +14,9 @@
	*/
	class EWS implements Interface\ExporterInterface
	{
	+ /** @const int Max number of items to migrate in one go */
	+ protected const CHUNK_SIZE = 20;
	+
	/** @var API EWS API object */
	public $api;

	@@ -31,15 +35,16 @@

	/** @var array Interal folders to skip */
	protected $folder_exceptions = [
	+ 'AllCategorizedItems',
	'AllContacts',
	+ 'AllContactsExtended',
	'AllPersonMetadata',
	'AllTodoTasks',
	'Document Centric Conversations',
	'ExternalContacts',
	- 'Favorites',
	'Flagged Emails',
	+ 'Folder Memberships',
	'GraphFilesAndWorkingSetSearchFolder',
	- 'My Contacts',
	'MyContactsExtended',
	'Orion Notes',
	'Outbox',
	@@ -48,16 +53,22 @@
	'RelevantContacts',
	'SharedFilesSearchFolder',
	'Sharing',
	+ 'SpoolsPresentSharedItemsSearchFolder',
	+ 'SpoolsSearchFolder',
	'To-Do Search',
	'UserCuratedContacts',
	'XrmActivityStreamSearch',
	'XrmCompanySearch',
	'XrmDealSearch',
	'XrmSearch',
	- 'Folder Memberships',
	- // TODO: These are different depending on a user locale
	+ // TODO: These are different depending on a user locale and it's not possible
	+ // to switch to English other than changing the user locale in OWA/Exchange.
	'Calendar/United States holidays',
	+ 'Favorites',
	+ 'My Contacts',
	'Kalendarz/Polska — dni wolne od pracy', // pl
	+ 'Ulubione', // pl
	+ 'Moje kontakty', // pl
	];

	/** @var array Map of EWS folder types to Kolab types */
	@@ -214,9 +225,31 @@
	*/
	public function getFolders($types = []): array
	{
	- // Get full folders hierarchy
	+ if (empty($types)) {
	+ $types = array_values($this->type_map);
	+ }
	+
	+ // Create FolderClass filter
	+ $search = new Type\OrType();
	+ foreach ($types as $type) {
	+ $type = array_search($type, $this->type_map);
	+ $search->addContains(Type\Contains::buildFromArray([
	+ 'FieldURI' => [
	+ Type\FieldURI::buildFromArray(['FieldURI' => 'folder:FolderClass']),
	+ ],
	+ 'Constant' => Type\ConstantValueType::buildFromArray([
	+ 'Value' => $type,
	+ ]),
	+ 'ContainmentComparison' => 'Exact',
	+ 'ContainmentMode' => 'FullString',
	+ ]));
	+ }
	+
	+ // Get full folders hierarchy (filtered by folder class)
	+ // Use of the filter reduces the response size by excluding system folders
	$options = [
	'Traversal' => 'Deep',
	+ 'Restriction' => ['Or' => $search],
	];

	$folders = $this->api->getChildrenFolders('root', $options);
	@@ -232,7 +265,7 @@
	continue;
	}

	- // Note: Folder names are localized
	+ // Note: Folder names are localized, even INBOX
	$name = $fullname = $folder->getDisplayName();
	$id = $folder->getFolderId()->getId();
	$parentId = $folder->getParentFolderId()->getId();
	@@ -308,7 +341,10 @@
	'ItemShape' => [
	'BaseShape' => 'IdOnly',
	'AdditionalProperties' => [
	- 'FieldURI' => ['FieldURI' => 'item:ItemClass'],
	+ 'FieldURI' => [
	+ ['FieldURI' => 'item:ItemClass'],
	+ // ['FieldURI' => 'item:Size'],
	+ ],
	],
	],
	];
	@@ -325,10 +361,16 @@
	// Request first page
	$response = $this->api->getClient()->FindItem($request);

	+ $set = new ItemSet();
	+
	// @phpstan-ignore-next-line
	- foreach ($response as $item) {
	+ foreach ($response->getItems() as $item) {
	if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
	- $callback($item);
	+ $set->items[] = $item;
	+ if (count($set->items) == self::CHUNK_SIZE) {
	+ $callback($set);
	+ $set = new ItemSet();
	+ }
	}
	}

	@@ -337,13 +379,21 @@
	// @phpstan-ignore-next-line
	$response = $this->api->getNextPage($response);

	- foreach ($response as $item) {
	+ foreach ($response->getItems() as $item) {
	if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
	- $callback($item);
	+ $set->items[] = $item;
	+ if (count($set->items) == self::CHUNK_SIZE) {
	+ $callback($set);
	+ $set = new ItemSet();
	+ }
	}
	}
	}

	+ if (count($set->items)) {
	+ $callback($set);
	+ }
	+
	// TODO: Delete items that do not exist anymore?
	}

	@@ -356,12 +406,11 @@
	$this->initEnv($this->engine->queue);

	if ($driver = EWS\Item::factory($this, $item)) {
	- $item->filename = $driver->fetchItem($item);
	+ $driver->processItem($item);
	+ return;
	}

	- if (empty($item->filename)) {
	- throw new \Exception("Failed to fetch an item from EWS");
	- }
	+ throw new \Exception("Failed to fetch an item from EWS");
	}

	/**
	@@ -399,20 +448,16 @@
	$exists = $existing[$idx]['href'];
	}

	- $item = Item::fromArray([
	- 'id' => $id,
	+ if (!EWS\Item::isValidItem($item)) {
	+ return null;
	+ }
	+
	+ return Item::fromArray([
	+ 'id' => $id['Id'],
	'class' => $item->getItemClass(),
	'folder' => $folder,
	'existing' => $exists,
	]);
	-
	- // TODO: We don't need to instantiate Item at this point, instead
	- // implement EWS\Item::validateClass() method
	- if ($driver = EWS\Item::factory($this, $item)) {
	- return $item;
	- }
	-
	- return null;
	}

	/**
	diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php
	--- a/src/app/DataMigrator/EWS/Appointment.php
	+++ b/src/app/DataMigrator/EWS/Appointment.php
	@@ -18,7 +18,7 @@
	/**
	* Get GetItem request parameters
	*/
	- protected function getItemRequest(): array
	+ protected static function getItemRequest(): array
	{
	$request = parent::getItemRequest();

	@@ -26,8 +26,7 @@
	$request['ItemShape']['IncludeMimeContent'] = true;

	// Get UID property, it's not included in the Default set
	- // FIXME: How to add the property to the set, not replace the whole set
	- $request['ItemShape']['AdditionalProperties']['FieldURI'] = ['FieldURI' => 'calendar:UID'];
	+ $request['ItemShape']['AdditionalProperties']['FieldURI'][] = ['FieldURI' => 'calendar:UID'];

	return $request;
	}
	@@ -35,7 +34,7 @@
	/**
	* Process event object
	*/
	- protected function processItem(Type $item)
	+ protected function convertItem(Type $item)
	{
	// Initialize $this->itemId (for some unit tests)
	$this->getUID($item);
	diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php
	--- a/src/app/DataMigrator/EWS/Contact.php
	+++ b/src/app/DataMigrator/EWS/Contact.php
	@@ -16,7 +16,7 @@
	/**
	* Get GetItem request parameters
	*/
	- protected function getItemRequest(): array
	+ protected static function getItemRequest(): array
	{
	$request = parent::getItemRequest();

	@@ -29,7 +29,7 @@
	/**
	* Process contact object
	*/
	- protected function processItem(Type $item)
	+ protected function convertItem(Type $item)
	{
	// Decode MIME content
	$vcard = base64_decode((string) $item->getMimeContent());
	@@ -56,7 +56,7 @@
	$vcard = str_replace($matches[1], "X-SPOUSE:{$spouse}", $vcard);
	}

	- // TODO: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12
	+ // Anniversary: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12
	if (preg_match('/(X-MS-ANNIVERSARY[;:][^\r\n]+)/', $vcard, $matches)) {
	$date = preg_replace('/^[^:]+:/', '', $matches[1]);
	$vcard = str_replace($matches[1], "X-ANNIVERSARY:{$date}", $vcard);
	diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php
	--- a/src/app/DataMigrator/EWS/DistList.php
	+++ b/src/app/DataMigrator/EWS/DistList.php
	@@ -13,10 +13,23 @@
	public const TYPE = 'IPM.DistList';
	public const FILE_EXT = 'vcf';

	+ /**
	+ * Get GetItem request parameters
	+ */
	+ protected static function getItemRequest(): array
	+ {
	+ $request = parent::getItemRequest();
	+
	+ // Get Body property, it's not included in the Default set
	+ $request['ItemShape']['AdditionalProperties']['FieldURI'] = ['FieldURI' => 'item:Body'];
	+
	+ return $request;
	+ }
	+
	/**
	* Convert distribution list object to vCard
	*/
	- protected function processItem(Type $item)
	+ protected function convertItem(Type $item)
	{
	// Groups (Distribution Lists) are not exported in vCard format, they use eml

	@@ -34,11 +47,6 @@
	$vcard .= $this->formatProp($key, $prop[0], isset($prop[1]) ? $prop[1] : []);
	}

	- // TODO: The group description property in Exchange is not available via EWS XML,
	- // at least not at outlook.office.com (Exchange 2010). It is available in the
	- // MimeContent which is in email message format. However, Kolab Webclient does not
	- // make any use of the NOTE property for contact groups.
	-
	// Process list members
	if ($members = $item->getMembers()) {
	// The Member property is either array (multiple members) or Type\MemberType
	@@ -70,6 +78,11 @@
	}
	}

	+ // Note: Kolab Webclient does not make any use of the NOTE property for contact groups
	+ if ($body = (string) $item->getBody()) {
	+ $vcard .= $this->formatProp('NOTE', $body);
	+ }
	+
	$vcard .= "END:VCARD\r\n";

	return $vcard;
	diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php
	--- a/src/app/DataMigrator/EWS/Item.php
	+++ b/src/app/DataMigrator/EWS/Item.php
	@@ -3,8 +3,10 @@
	namespace App\DataMigrator\EWS;

	use App\DataMigrator\EWS;
	+use App\DataMigrator\Engine;
	use App\DataMigrator\Interface\Folder as FolderInterface;
	use App\DataMigrator\Interface\Item as ItemInterface;
	+use App\DataMigrator\Interface\ItemSet as ItemSetInterface;
	use garethp\ews\API;
	use garethp\ews\API\Type;

	@@ -50,11 +52,24 @@
	}

	/**
	- * Fetch the specified object and put into a file
	+ * Validate that specified EWS Item is of supported type
	*/
	- public function fetchItem(ItemInterface $item)
	+ public static function isValidItem(Type $item): bool
	{
	- $itemId = $item->id;
	+ $item_class = str_replace('IPM.', '', $item->getItemClass());
	+ $item_class = "\App\DataMigrator\EWS\\{$item_class}";
	+
	+ return class_exists($item_class);
	+ }
	+
	+ /**
	+ * Process an item (fetch data and convert it)
	+ */
	+ public function processItem(ItemInterface $item): void
	+ {
	+ $itemId = ['Id' => $item->id];
	+
	+ \Log::debug("[EWS] Fetching item {$item->id}...");

	// Fetch the item
	$ewsItem = $this->driver->api->getItem($itemId, $this->getItemRequest());
	@@ -64,46 +79,48 @@
	\Log::debug("[EWS] Saving item {$uid}...");

	// Apply type-specific format converters
	- $content = $this->processItem($ewsItem);
	+ $content = $this->convertItem($ewsItem);

	if (!is_string($content)) {
	- return;
	+ throw new \Exception("Failed to fetch EWS item {$this->itemId}");
	}

	- $uid = preg_replace('/[^a-zA-Z0-9_:@-]/', '', $uid);
	-
	- $location = $this->folder->location;
	+ $filename = $uid . '.' . $this->fileExtension();

	- if (!file_exists($location)) {
	- mkdir($location, 0740, true);
	- }
	-
	- $location .= '/' . $uid . '.' . $this->fileExtension();
	+ if (strlen($content) > Engine::MAX_ITEM_SIZE) {
	+ $location = $this->folder->tempFileLocation($filename);

	- file_put_contents($location, $content);
	+ if (file_put_contents($location, $content) === false) {
	+ throw new \Exception("Failed to write to file at {$location}");
	+ }

	- return $location;
	+ $item->filename = $location;
	+ } else {
	+ $item->content = $content;
	+ $item->filename = $filename;
	+ }
	}

	/**
	* Item conversion code
	*/
	- abstract protected function processItem(Type $item);
	+ abstract protected function convertItem(Type $item);

	/**
	* Get GetItem request parameters
	*/
	- protected function getItemRequest(): array
	+ protected static function getItemRequest(): array
	{
	$request = [
	'ItemShape' => [
	// Reqest default set of properties
	'BaseShape' => 'Default',
	// Additional properties, e.g. LastModifiedTime
	- // FIXME: How to add multiple properties here?
	'AdditionalProperties' => [
	- 'FieldURI' => ['FieldURI' => 'item:LastModifiedTime'],
	- ]
	+ 'FieldURI' => [
	+ ['FieldURI' => 'item:LastModifiedTime'],
	+ ],
	+ ],
	]
	];

	diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php
	--- a/src/app/DataMigrator/EWS/Note.php
	+++ b/src/app/DataMigrator/EWS/Note.php
	@@ -16,7 +16,7 @@
	/**
	* Get GetItem request parameters
	*/
	- protected function getItemRequest(): array
	+ protected static function getItemRequest(): array
	{
	$request = parent::getItemRequest();

	@@ -32,7 +32,7 @@
	/**
	* Process contact object
	*/
	- protected function processItem(Type $item)
	+ protected function convertItem(Type $item)
	{
	$email = base64_decode((string) $item->getMimeContent());

	diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php
	--- a/src/app/DataMigrator/EWS/Task.php
	+++ b/src/app/DataMigrator/EWS/Task.php
	@@ -16,7 +16,7 @@
	/**
	* Get GetItem request parameters
	*/
	- protected function getItemRequest(): array
	+ protected static function getItemRequest(): array
	{
	$request = parent::getItemRequest();

	@@ -29,7 +29,7 @@
	/**
	* Process task object
	*/
	- protected function processItem(Type $item)
	+ protected function convertItem(Type $item)
	{
	// Tasks are exported as Email messages in useless format
	// (does not contain all relevant properties)
	diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
	--- a/src/app/DataMigrator/Engine.php
	+++ b/src/app/DataMigrator/Engine.php
	@@ -3,6 +3,7 @@
	namespace App\DataMigrator;

	use App\DataMigrator\Interface\ExporterInterface;
	+use App\DataMigrator\Interface\FetchItemSetInterface;
	use App\DataMigrator\Interface\Folder;
	use App\DataMigrator\Interface\ImporterInterface;
	use App\DataMigrator\Interface\Item;
	@@ -21,6 +22,9 @@
	public const TYPE_NOTE = 'note';
	public const TYPE_TASK = 'task';

	+ /** @const int Max item size to handle in-memory, bigger will be handled with temp files */
	+ public const MAX_ITEM_SIZE = 20 * 1024 * 1024;
	+
	/** @var Account Source account */
	public $source;

	@@ -43,7 +47,7 @@
	/**
	* Execute migration for the specified user
	*/
	- public function migrate(Account $source, Account $destination, array $options = [])
	+ public function migrate(Account $source, Account $destination, array $options = []): void
	{
	$this->source = $source;
	$this->destination = $destination;
	@@ -87,14 +91,14 @@
	// Create a queue
	$this->createQueue($queue_id);

	- // We'll store output in storage/<username> tree
	+ // We'll store temp files in storage/<username> tree
	$location = storage_path('export/') . $source->email;

	if (!file_exists($location)) {
	mkdir($location, 0740, true);
	}

	- $types = preg_split('/\s,\s/', strtolower($options['type'] ?? ''));
	+ $types = empty($options['type']) ? [] : preg_split('/\s,\s/', strtolower($options['type']));

	$this->debug("Fetching folders hierarchy...");

	@@ -189,8 +193,8 @@
	$this->exporter->fetchItem($item);
	$this->importer->createItem($item);

	- if (!empty($item->filename)) {
	- unlink($item->filename);
	+ if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) {
	+ @unlink($item->filename);
	}

	if (empty($this->options['sync'])) {
	@@ -208,15 +212,21 @@
	$this->envFromQueue($set->items[0]->folder->queueId);
	}

	- // TODO: Some exporters, e.g. DAV, might optimize fetching multiple items in one go,
	- // we'll need a new API to do that
	-
	- foreach ($set->items as $item) {
	- $this->exporter->fetchItem($item);
	+ $importItem = function (Item $item) {
	$this->importer->createItem($item);

	- if (!empty($item->filename)) {
	- unlink($item->filename);
	+ if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) {
	+ @unlink($item->filename);
	+ }
	+ };
	+
	+ // Some exporters, e.g. DAV, might optimize fetching multiple items in one go
	+ if ($this->exporter instanceof FetchItemSetInterface) {
	+ $this->exporter->fetchItemSet($set, $importItem);
	+ } else {
	+ foreach ($set->items as $item) {
	+ $this->exporter->fetchItem($item);
	+ $importItem($item);
	}
	}

	@@ -329,6 +339,10 @@
	$driver = new IMAP($account, $this);
	break;

	+ case 'test':
	+ $driver = new Test($account, $this);
	+ break;
	+
	default:
	throw new \Exception("Failed to init driver for '{$account->scheme}'");
	}
	diff --git a/src/app/DataMigrator/IMAP.php b/src/app/DataMigrator/IMAP.php
	--- a/src/app/DataMigrator/IMAP.php
	+++ b/src/app/DataMigrator/IMAP.php
	@@ -88,7 +88,19 @@
	{
	$mailbox = $item->folder->fullname;

	- if ($item->filename) {
	+ if (strlen($item->content)) {
	+ $result = $this->imap->append(
	+ $mailbox,
	+ $item->content,
	+ $item->data['flags'],
	+ $item->data['internaldate'],
	+ true
	+ );
	+
	+ if ($result === false) {
	+ throw new \Exception("Failed to append IMAP message into {$mailbox}");
	+ }
	+ } elseif ($item->filename) {
	$result = $this->imap->appendFromFile(
	$mailbox,
	$item->filename,
	@@ -153,35 +165,36 @@
	\|\| $header->timestamp != $item->existing['timestamp']
	\|\| $header->size != $item->existing['size']
	) {
	- // Save the message content to a file
	- $location = $item->folder->location;
	-
	- if (!file_exists($location)) {
	- mkdir($location, 0740, true);
	- }
	+ // Handle message content in memory (up to 20MB), bigger messages will use a temp file
	+ if ($header->size > Engine::MAX_ITEM_SIZE) {
	+ // Save the message content to a file
	+ $location = $item->folder->tempFileLocation($uid . '.eml');

	- // TODO: What if parent folder not yet exists?
	- $location .= '/' . $uid . '.eml';
	+ $fp = fopen($location, 'w');

	- // TODO: We should consider streaming the message, it should be possible
	- // with append() and handlePartBody(), but I don't know if anyone tried that.
	-
	- $fp = fopen($location, 'w');
	+ if (!$fp) {
	+ throw new \Exception("Failed to open 'php://temp' stream");
	+ }

	- if (!$fp) {
	- throw new \Exception("Failed to write to {$location}");
	+ $result = $this->imap->handlePartBody($mailbox, $uid, true, '', null, null, $fp);
	+ } else {
	+ $result = $this->imap->handlePartBody($mailbox, $uid, true);
	}

	- $result = $this->imap->handlePartBody($mailbox, $uid, true, '', null, null, $fp);
	-
	if ($result === false) {
	- fclose($fp);
	+ if (!empty($fp)) {
	+ fclose($fp);
	+ }
	+
	throw new \Exception("Failed to fetch IMAP message for {$mailbox}/{$uid}");
	}

	- $item->filename = $location;
	-
	- fclose($fp);
	+ if (!empty($fp) && !empty($location)) {
	+ $item->filename = $location;
	+ fclose($fp);
	+ } else {
	+ $item->content = $result;
	+ }
	}

	$item->data = [
	diff --git a/src/app/DataMigrator/Interface/FetchItemSetInterface.php b/src/app/DataMigrator/Interface/FetchItemSetInterface.php
	new file mode 100644
	--- /dev/null
	+++ b/src/app/DataMigrator/Interface/FetchItemSetInterface.php
	@@ -0,0 +1,14 @@
	+<?php
	+
	+namespace App\DataMigrator\Interface;
	+
	+interface FetchItemSetInterface
	+{
	+ /**
	+ * Fetching a set of items in one operation
	+ *
	+ * @param ItemSet $set Set of items
	+ * @param callable $callback A callback to execute on every Item
	+ */
	+ public function fetchItemSet(ItemSet $set, $callback): void;
	+}
	diff --git a/src/app/DataMigrator/Interface/Folder.php b/src/app/DataMigrator/Interface/Folder.php
	--- a/src/app/DataMigrator/Interface/Folder.php
	+++ b/src/app/DataMigrator/Interface/Folder.php
	@@ -34,6 +34,9 @@
	public $queueId;


	+ /**
	+ * Create Folder instance from an array
	+ */
	public static function fromArray(array $data = []): Folder
	{
	$obj = new self();
	@@ -44,4 +47,23 @@

	return $obj;
	}
	+
	+ /**
	+ * Returns location of a temp file for an Item content
	+ */
	+ public function tempFileLocation(string $filename): string
	+ {
	+ $filename = preg_replace('/[^a-zA-Z0-9_:@.-]/', '', $filename);
	+
	+ $location = $this->location;
	+
	+ // TODO: What if parent folder not yet exists?
	+ if (!file_exists($location)) {
	+ mkdir($location, 0740, true);
	+ }
	+
	+ $location .= '/' . $filename;
	+
	+ return $location;
	+ }
	}
	diff --git a/src/app/DataMigrator/Interface/Item.php b/src/app/DataMigrator/Interface/Item.php
	--- a/src/app/DataMigrator/Interface/Item.php
	+++ b/src/app/DataMigrator/Interface/Item.php
	@@ -10,7 +10,7 @@
	/** @var mixed Item identifier */
	public $id;

	- /** @var Folder Folder */
	+ /** @var ?Folder Folder */
	public $folder;

	/** @var string Object class */
	@@ -24,13 +24,19 @@
	*/
	public $existing;

	- /** @var ?string Exported object location in the local storage */
	+ /** @var string Exported object content */
	+ public $content = '';
	+
	+ /** @var ?string Exported object content location */
	public $filename;

	/** @var array Extra data to migrate (like email flags, internaldate, etc.) */
	public $data = [];


	+ /**
	+ * Create Item object from an array
	+ */
	public static function fromArray(array $data = []): Item
	{
	$obj = new self();
	diff --git a/src/app/DataMigrator/Interface/ItemSet.php b/src/app/DataMigrator/Interface/ItemSet.php
	--- a/src/app/DataMigrator/Interface/ItemSet.php
	+++ b/src/app/DataMigrator/Interface/ItemSet.php
	@@ -5,16 +5,11 @@
	/**
	* Data object representing a set of data items
	*/
	-class ItemSet
	+class ItemSet implements \Serializable
	{
	/** @var array<Item> Items list */
	public $items = [];

	- // TODO: Every item has a $folder property, this makes the set
	- // needlesly big when serialized. We should probably store $folder
	- // once with the set and remove it from an item on serialize
	- // and back in unserialize.
	-
	/**
	* Create an ItemSet instance
	*/
	@@ -25,4 +20,26 @@

	return $obj;
	}
	+
	+ public function serialize(): ?string
	+ {
	+ // Every item has a Folder property, this makes the set
	+ // needlesly big when serialized. Make the size more compact.
	+ $folder = count($this->items) ? $this->items[0]->folder : null;
	+
	+ foreach ($this->items as $item) {
	+ $item->folder = null;
	+ }
	+
	+ return serialize([$folder, $this->items]);
	+ }
	+
	+ public function unserialize(string $data): void
	+ {
	+ [$folder, $this->items] = unserialize($data);
	+
	+ foreach ($this->items as $item) {
	+ $item->folder = $folder;
	+ }
	+ }
	}
	diff --git a/src/app/DataMigrator/Test.php b/src/app/DataMigrator/Test.php
	new file mode 100644
	--- /dev/null
	+++ b/src/app/DataMigrator/Test.php
	@@ -0,0 +1,153 @@
	+<?php
	+
	+namespace App\DataMigrator;
	+
	+use App\DataMigrator\Interface\Folder;
	+use App\DataMigrator\Interface\ExporterInterface;
	+use App\DataMigrator\Interface\ImporterInterface;
	+use App\DataMigrator\Interface\Item;
	+use App\DataMigrator\Interface\ItemSet;
	+
	+class Test implements ExporterInterface, ImporterInterface
	+{
	+ protected const CHUNK_SIZE = 3;
	+
	+ public static $fetchedItems = [];
	+ public static $createdItems = [];
	+ public static $createdFolders = [];
	+ public static $folders = [];
	+
	+ /** @var Account Account to operate on */
	+ protected $account;
	+
	+ /** @var Engine Data migrator engine */
	+ protected $engine;
	+
	+
	+ /**
	+ * Object constructor
	+ */
	+ public function __construct(Account $account, Engine $engine)
	+ {
	+ $this->engine = $engine;
	+ $this->account = $account;
	+ }
	+
	+ public static function init($folders)
	+ {
	+ self::$folders = $folders;
	+
	+ self::$fetchedItems = [];
	+ self::$createdItems = [];
	+ self::$createdFolders = [];
	+ }
	+
	+ /**
	+ * Check user credentials.
	+ *
	+ * @throws \Exception
	+ */
	+ public function authenticate(): void
	+ {
	+ }
	+
	+ /**
	+ * Create an item in a folder.
	+ *
	+ * @param Item $item Item to import
	+ *
	+ * @throws \Exception
	+ */
	+ public function createItem(Item $item): void
	+ {
	+ self::$createdItems[] = $item;
	+ }
	+
	+ /**
	+ * Create a folder.
	+ *
	+ * @param Folder $folder Folder data
	+ *
	+ * @throws \Exception on error
	+ */
	+ public function createFolder(Folder $folder): void
	+ {
	+ self::$createdFolders[] = $folder;
	+ }
	+
	+ /**
	+ * Fetching an item
	+ */
	+ public function fetchItem(Item $item): void
	+ {
	+ $item->content = 'content';
	+ $item->filename = 'test.eml';
	+
	+ self::$fetchedItems[] = $item;
	+ }
	+
	+ /**
	+ * Fetch a list of folder items
	+ */
	+ public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void
	+ {
	+ // Get existing messages' headers from the destination mailbox
	+ $existing = $importer->getItems($folder);
	+
	+ $set = new ItemSet();
	+
	+ foreach ((self::$folders[$folder->id]['items'] ?? []) as $itemId => $item) {
	+ $exists = null; // TODO
	+
	+ $item['id'] = $itemId;
	+ $item['folder'] = $folder;
	+ $item['existing'] = $exists;
	+
	+ $set->items[] = Item::fromArray($item);
	+
	+ if (count($set->items) == self::CHUNK_SIZE) {
	+ $callback($set);
	+ $set = new ItemSet();
	+ }
	+ }
	+
	+ if (count($set->items)) {
	+ $callback($set);
	+ }
	+ }
	+
	+ /**
	+ * Get a list of items, limited to their essential propeties
	+ * used in incremental migration.
	+ *
	+ * @param Folder $folder Folder data
	+ *
	+ * @throws \Exception on error
	+ */
	+ public function getItems(Folder $folder): array
	+ {
	+ return self::$folders[$folder->id]['existing_items'] ?? [];
	+ }
	+
	+ /**
	+ * Get folders hierarchy
	+ */
	+ public function getFolders($types = []): array
	+ {
	+ $result = [];
	+
	+ foreach (self::$folders as $folderId => $folder) {
	+ // Skip folder types we do not support (need)
	+ if (!empty($types) && !in_array($folder['type'], $types)) {
	+ continue;
	+ }
	+
	+ $folder['id'] = $folderId;
	+ $folder['total'] = count($folder['items']);
	+
	+ $result[] = Folder::fromArray($folder);
	+ }
	+
	+ return $result;
	+ }
	+}
	diff --git a/src/tests/Feature/DataMigrator/EngineTest.php b/src/tests/Feature/DataMigrator/EngineTest.php
	--- a/src/tests/Feature/DataMigrator/EngineTest.php
	+++ b/src/tests/Feature/DataMigrator/EngineTest.php
	@@ -4,11 +4,53 @@

	use App\DataMigrator\Account;
	use App\DataMigrator\Engine;
	+use App\DataMigrator\Jobs\FolderJob;
	+use App\DataMigrator\Jobs\ItemJob;
	use App\DataMigrator\Queue as MigratorQueue;
	+use App\DataMigrator\Test;
	+use Illuminate\Support\Facades\Queue;
	use Tests\TestCase;

	class EngineTest extends TestCase
	{
	+ protected $data = [
	+ 'Inbox' => [
	+ 'type' => Engine::TYPE_MAIL,
	+ 'name' => 'Inbox',
	+ 'fullname' => 'Inbox',
	+ 'items' => [
	+ 'm1' => [],
	+ 'm2' => [],
	+ 'm3' => [],
	+ 'm4' => [],
	+ ],
	+ 'existing_items' => [
	+ ],
	+ ],
	+ 'Contacts' => [
	+ 'type' => Engine::TYPE_CONTACT,
	+ 'name' => 'Contacts',
	+ 'fullname' => 'Contacts',
	+ 'items' => [
	+ 'c1' => [],
	+ 'c2' => [],
	+ ],
	+ 'existing_items' => [
	+ ],
	+ ],
	+ 'Calendar' => [
	+ 'type' => Engine::TYPE_EVENT,
	+ 'name' => 'Calendar',
	+ 'fullname' => 'Calendar',
	+ 'items' => [
	+ 'e1' => [],
	+ 'e2' => [],
	+ ],
	+ 'existing_items' => [
	+ ],
	+ ],
	+ ];
	+
	/**
	* {@inheritDoc}
	*/
	@@ -34,6 +76,48 @@
	*/
	public function testAsyncMigration(): void
	{
	+ $source = new Account('test://test%40domain.tld:test@test');
	+ $destination = new Account('test://test%40kolab.org:test@test');
	+ $engine = new Engine();
	+
	+ Test::init($this->data);
	+ Queue::fake();
	+
	+ // Migration initial run
	+ $engine->migrate($source, $destination, []);
	+
	+ $queue = MigratorQueue::first();
	+
	+ Queue::assertPushed(FolderJob::class, 3);
	+ Queue::assertPushed(
	+ FolderJob::class,
	+ function ($job) use ($queue) {
	+ $folder = TestCase::getObjectProperty($job, 'folder');
	+ return $folder->id === 'Inbox' && $folder->queueId = $queue->id;
	+ }
	+ );
	+ Queue::assertPushed(
	+ FolderJob::class,
	+ function ($job) use ($queue) {
	+ $folder = TestCase::getObjectProperty($job, 'folder');
	+ return $folder->id === 'Contacts' && $folder->queueId = $queue->id;
	+ }
	+ );
	+ Queue::assertPushed(
	+ FolderJob::class,
	+ function ($job) use ($queue) {
	+ $folder = TestCase::getObjectProperty($job, 'folder');
	+ return $folder->id === 'Calendar' && $folder->queueId = $queue->id;
	+ }
	+ );
	+
	+ $this->assertCount(0, Test::$createdFolders);
	+ $this->assertSame(3, $queue->jobs_started);
	+ $this->assertSame(0, $queue->jobs_finished);
	+ $this->assertSame([], $queue->data['options']);
	+ // TODO: Assert Source and destination in the queue
	+ // TODO: Test 'force' option, test executing with an existing queue
	+ // TODO: Test jobs execution
	$this->markTestIncomplete();
	}

	@@ -42,6 +126,42 @@
	*/
	public function testSyncMigration(): void
	{
	- $this->markTestIncomplete();
	+ $source = new Account('test://test%40domain.tld:test@test');
	+ $destination = new Account('test://test%40kolab.org:test@test');
	+ $engine = new Engine();
	+
	+ Test::init($this->data);
	+ Queue::fake();
	+
	+ $engine->migrate($source, $destination, ['sync' => true]);
	+
	+ $queue = MigratorQueue::first();
	+
	+ Queue::assertNothingPushed();
	+
	+ $this->assertSame(0, $queue->jobs_started);
	+ $this->assertSame(0, $queue->jobs_finished);
	+ $this->assertSame(['sync' => true], $queue->data['options']);
	+
	+ $this->assertCount(3, Test::$createdFolders);
	+ $this->assertCount(8, Test::$createdItems);
	+ $this->assertCount(8, Test::$fetchedItems);
	+
	+ Test::init($this->data);
	+
	+ // Test 'type' argument
	+ $engine->migrate($source, $destination, ['sync' => true, 'type' => 'contact,event']);
	+
	+ $queue = MigratorQueue::whereNot('id', $queue->id)->first();
	+
	+ Queue::assertNothingPushed();
	+
	+ $this->assertSame(0, $queue->jobs_started);
	+ $this->assertSame(0, $queue->jobs_finished);
	+ $this->assertSame(['sync' => true, 'type' => 'contact,event'], $queue->data['options']);
	+
	+ $this->assertCount(2, Test::$createdFolders);
	+ $this->assertCount(4, Test::$createdItems);
	+ $this->assertCount(4, Test::$fetchedItems);
	}
	}
	diff --git a/src/tests/Feature/DomainTest.php b/src/tests/Feature/DomainTest.php
	--- a/src/tests/Feature/DomainTest.php
	+++ b/src/tests/Feature/DomainTest.php
	@@ -138,9 +138,6 @@
	$domainNamespace === $domain->namespace;
	}
	);
	-
	- $job = new \App\Jobs\Domain\CreateJob($domain->id);
	- $job->handle();
	}

	/**
	diff --git a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
	--- a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
	+++ b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
	@@ -15,7 +15,7 @@
	/**
	* Test appointment item processing
	*/
	- public function testProcessItem(): void
	+ public function testConvertItem(): void
	{
	$account = new Account('ews://test:test@test');
	$engine = new Engine();
	@@ -78,7 +78,7 @@
	]);

	// Convert the Exchange item into iCalendar
	- $ical = $this->invokeMethod($appointment, 'processItem', [$item]);
	+ $ical = $this->invokeMethod($appointment, 'convertItem', [$item]);

	// Parse the iCalendar output
	$event = new Vevent();
	diff --git a/src/tests/Unit/DataMigrator/EWS/ContactTest.php b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
	--- a/src/tests/Unit/DataMigrator/EWS/ContactTest.php
	+++ b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
	@@ -15,7 +15,7 @@
	/**
	* Test contact item processing
	*/
	- public function testProcessItem(): void
	+ public function testConvertItem(): void
	{
	$account = new Account('ews://test:test@test');
	$engine = new Engine();
	@@ -113,7 +113,7 @@
	]);

	// Convert the Exchange item into vCard
	- $vcard = $this->invokeMethod($contact, 'processItem', [$item]);
	+ $vcard = $this->invokeMethod($contact, 'convertItem', [$item]);

	// Parse the vCard
	$contact = new Vcard();
	diff --git a/src/tests/Unit/DataMigrator/EWS/DistListTest.php b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
	--- a/src/tests/Unit/DataMigrator/EWS/DistListTest.php
	+++ b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
	@@ -15,7 +15,7 @@
	/**
	* Test contact item processing
	*/
	- public function testProcessItem(): void
	+ public function testConvertItem(): void
	{
	$account = new Account('ews://test:test@test');
	$engine = new Engine();
	@@ -37,6 +37,11 @@
	'LastModifiedTime' => '2024-06-27T13:44:32Z',
	'DisplayName' => 'Lista',
	'FileAs' => 'lista',
	+ 'Body' => [
	+ 'BodyType' => 'Text',
	+ 'IsTruncated' => false,
	+ '_value' => 'distlist body',
	+ ],
	'Members' => (object) [
	'Member' => [
	Type\MemberType::buildFromArray([
	@@ -67,7 +72,7 @@
	]);

	// Convert the Exchange item into vCard
	- $vcard = $this->invokeMethod($distlist, 'processItem', [$item]);
	+ $vcard = $this->invokeMethod($distlist, 'convertItem', [$item]);

	// Parse the vCard
	$distlist = new Vcard();
	@@ -78,6 +83,7 @@
	$this->assertMatchesRegularExpression('/^[a-f0-9]{40}$/', $distlist->uid);
	$this->assertSame('group', $distlist->kind);
	$this->assertSame('Lista', $distlist->fn);
	+ $this->assertSame('distlist body', $distlist->note);
	$this->assertSame('Kolab EWS Data Migrator', $distlist->prodid);
	$this->assertSame('2024-06-27T13:44:32Z', $distlist->rev);

	diff --git a/src/tests/Unit/DataMigrator/EWS/TaskTest.php b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
	--- a/src/tests/Unit/DataMigrator/EWS/TaskTest.php
	+++ b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
	@@ -15,7 +15,7 @@
	/**
	* Test task item processing
	*/
	- public function testProcessItem(): void
	+ public function testConvertItem(): void
	{
	$source = new Account('ews://test%40domain.tld:test@test');
	$destination = new Account('dav://test%40kolab.org:test@test');
	@@ -118,7 +118,7 @@
	]);

	// Convert the Exchange item into iCalendar
	- $ical = $this->invokeMethod($task, 'processItem', [$item]);
	+ $ical = $this->invokeMethod($task, 'convertItem', [$item]);

	// Parse the iCalendar output
	$task = new Vtodo();
	@@ -154,7 +154,7 @@
	/**
	* Test processing Recurrence property
	*/
	- public function testProcessItemRecurrence(): void
	+ public function testConvertItemRecurrence(): void
	{
	$this->markTestIncomplete();
	}

File Metadata

Mime Type: text/plain
Expires: Wed, Oct 30, 9:29 PM (8 h, 57 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 10061772
Default Alt Text: D4857.id13917.diff (40 KB)

D4857.id13917.diffNo OneTemporaryActions

D4857.id13917.diffView Options

File Metadata

Event Timeline

D4857.id13917.diff
No OneTemporary
Actions

D4857.id13917.diff
View Options