Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F16558533
D4857.id13917.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
40 KB
Referenced Files
None
Subscribers
None
D4857.id13917.diff
View Options
diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php
--- a/src/app/DataMigrator/DAV.php
+++ b/src/app/DataMigrator/DAV.php
@@ -67,13 +67,21 @@
*/
public function createItem(Item $item): void
{
- if ($item->existing) {
- $href = $item->existing;
+ $is_file = false;
+ $href = $item->existing ?: null;
+
+ if (strlen($item->content)) {
+ $content = $item->content;
} else {
- $href = $this->getFolderPath($item->folder) . '/' . pathinfo($item->filename, PATHINFO_BASENAME);
+ $content = $item->filename;
+ $is_file = true;
+ }
+
+ if (empty($href)) {
+ $href = $this->getFolderPath($item->folder) . '/' . basename($item->filename);
}
- $object = new DAVOpaque($item->filename, true);
+ $object = new DAVOpaque($content, $is_file);
$object->href = $href;
switch ($item->folder->type) {
@@ -139,15 +147,6 @@
*/
public function fetchItem(Item $item): void
{
- // Save the item content to a file
- $location = $item->folder->location;
-
- if (!file_exists($location)) {
- mkdir($location, 0740, true);
- }
-
- $location .= '/' . basename($item->id);
-
$result = $this->client->getObjects(dirname($item->id), $this->type2DAV($item->folder->type), [$item->id]);
if ($result === false) {
@@ -156,11 +155,21 @@
// TODO: Do any content changes, e.g. organizer/attendee email migration
- if (file_put_contents($location, (string) $result[0]) === false) {
- throw new \Exception("Failed to write to {$location}");
- }
+ $content = (string) $result[0];
+
+ if (strlen($content) > Engine::MAX_ITEM_SIZE) {
+ // Save the item content to a file
+ $location = $item->folder->tempFileLocation(basename($item->id));
+
+ if (file_put_contents($location, $content) === false) {
+ throw new \Exception("Failed to write to {$location}");
+ }
- $item->filename = $location;
+ $item->filename = $location;
+ } else {
+ $item->content = $content;
+ $item->filename = basename($item->id);
+ }
}
/**
diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php
--- a/src/app/DataMigrator/EWS.php
+++ b/src/app/DataMigrator/EWS.php
@@ -4,6 +4,7 @@
use App\DataMigrator\Interface\Folder;
use App\DataMigrator\Interface\Item;
+use App\DataMigrator\Interface\ItemSet;
use garethp\ews\API;
use garethp\ews\API\Type;
use Illuminate\Support\Facades\Http;
@@ -13,6 +14,9 @@
*/
class EWS implements Interface\ExporterInterface
{
+ /** @const int Max number of items to migrate in one go */
+ protected const CHUNK_SIZE = 20;
+
/** @var API EWS API object */
public $api;
@@ -31,15 +35,16 @@
/** @var array Interal folders to skip */
protected $folder_exceptions = [
+ 'AllCategorizedItems',
'AllContacts',
+ 'AllContactsExtended',
'AllPersonMetadata',
'AllTodoTasks',
'Document Centric Conversations',
'ExternalContacts',
- 'Favorites',
'Flagged Emails',
+ 'Folder Memberships',
'GraphFilesAndWorkingSetSearchFolder',
- 'My Contacts',
'MyContactsExtended',
'Orion Notes',
'Outbox',
@@ -48,16 +53,22 @@
'RelevantContacts',
'SharedFilesSearchFolder',
'Sharing',
+ 'SpoolsPresentSharedItemsSearchFolder',
+ 'SpoolsSearchFolder',
'To-Do Search',
'UserCuratedContacts',
'XrmActivityStreamSearch',
'XrmCompanySearch',
'XrmDealSearch',
'XrmSearch',
- 'Folder Memberships',
- // TODO: These are different depending on a user locale
+ // TODO: These are different depending on a user locale and it's not possible
+ // to switch to English other than changing the user locale in OWA/Exchange.
'Calendar/United States holidays',
+ 'Favorites',
+ 'My Contacts',
'Kalendarz/Polska — dni wolne od pracy', // pl
+ 'Ulubione', // pl
+ 'Moje kontakty', // pl
];
/** @var array Map of EWS folder types to Kolab types */
@@ -214,9 +225,31 @@
*/
public function getFolders($types = []): array
{
- // Get full folders hierarchy
+ if (empty($types)) {
+ $types = array_values($this->type_map);
+ }
+
+ // Create FolderClass filter
+ $search = new Type\OrType();
+ foreach ($types as $type) {
+ $type = array_search($type, $this->type_map);
+ $search->addContains(Type\Contains::buildFromArray([
+ 'FieldURI' => [
+ Type\FieldURI::buildFromArray(['FieldURI' => 'folder:FolderClass']),
+ ],
+ 'Constant' => Type\ConstantValueType::buildFromArray([
+ 'Value' => $type,
+ ]),
+ 'ContainmentComparison' => 'Exact',
+ 'ContainmentMode' => 'FullString',
+ ]));
+ }
+
+ // Get full folders hierarchy (filtered by folder class)
+ // Use of the filter reduces the response size by excluding system folders
$options = [
'Traversal' => 'Deep',
+ 'Restriction' => ['Or' => $search],
];
$folders = $this->api->getChildrenFolders('root', $options);
@@ -232,7 +265,7 @@
continue;
}
- // Note: Folder names are localized
+ // Note: Folder names are localized, even INBOX
$name = $fullname = $folder->getDisplayName();
$id = $folder->getFolderId()->getId();
$parentId = $folder->getParentFolderId()->getId();
@@ -308,7 +341,10 @@
'ItemShape' => [
'BaseShape' => 'IdOnly',
'AdditionalProperties' => [
- 'FieldURI' => ['FieldURI' => 'item:ItemClass'],
+ 'FieldURI' => [
+ ['FieldURI' => 'item:ItemClass'],
+ // ['FieldURI' => 'item:Size'],
+ ],
],
],
];
@@ -325,10 +361,16 @@
// Request first page
$response = $this->api->getClient()->FindItem($request);
+ $set = new ItemSet();
+
// @phpstan-ignore-next-line
- foreach ($response as $item) {
+ foreach ($response->getItems() as $item) {
if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
- $callback($item);
+ $set->items[] = $item;
+ if (count($set->items) == self::CHUNK_SIZE) {
+ $callback($set);
+ $set = new ItemSet();
+ }
}
}
@@ -337,13 +379,21 @@
// @phpstan-ignore-next-line
$response = $this->api->getNextPage($response);
- foreach ($response as $item) {
+ foreach ($response->getItems() as $item) {
if ($item = $this->toItem($item, $folder, $existing, $existingIndex)) {
- $callback($item);
+ $set->items[] = $item;
+ if (count($set->items) == self::CHUNK_SIZE) {
+ $callback($set);
+ $set = new ItemSet();
+ }
}
}
}
+ if (count($set->items)) {
+ $callback($set);
+ }
+
// TODO: Delete items that do not exist anymore?
}
@@ -356,12 +406,11 @@
$this->initEnv($this->engine->queue);
if ($driver = EWS\Item::factory($this, $item)) {
- $item->filename = $driver->fetchItem($item);
+ $driver->processItem($item);
+ return;
}
- if (empty($item->filename)) {
- throw new \Exception("Failed to fetch an item from EWS");
- }
+ throw new \Exception("Failed to fetch an item from EWS");
}
/**
@@ -399,20 +448,16 @@
$exists = $existing[$idx]['href'];
}
- $item = Item::fromArray([
- 'id' => $id,
+ if (!EWS\Item::isValidItem($item)) {
+ return null;
+ }
+
+ return Item::fromArray([
+ 'id' => $id['Id'],
'class' => $item->getItemClass(),
'folder' => $folder,
'existing' => $exists,
]);
-
- // TODO: We don't need to instantiate Item at this point, instead
- // implement EWS\Item::validateClass() method
- if ($driver = EWS\Item::factory($this, $item)) {
- return $item;
- }
-
- return null;
}
/**
diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php
--- a/src/app/DataMigrator/EWS/Appointment.php
+++ b/src/app/DataMigrator/EWS/Appointment.php
@@ -18,7 +18,7 @@
/**
* Get GetItem request parameters
*/
- protected function getItemRequest(): array
+ protected static function getItemRequest(): array
{
$request = parent::getItemRequest();
@@ -26,8 +26,7 @@
$request['ItemShape']['IncludeMimeContent'] = true;
// Get UID property, it's not included in the Default set
- // FIXME: How to add the property to the set, not replace the whole set
- $request['ItemShape']['AdditionalProperties']['FieldURI'] = ['FieldURI' => 'calendar:UID'];
+ $request['ItemShape']['AdditionalProperties']['FieldURI'][] = ['FieldURI' => 'calendar:UID'];
return $request;
}
@@ -35,7 +34,7 @@
/**
* Process event object
*/
- protected function processItem(Type $item)
+ protected function convertItem(Type $item)
{
// Initialize $this->itemId (for some unit tests)
$this->getUID($item);
diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php
--- a/src/app/DataMigrator/EWS/Contact.php
+++ b/src/app/DataMigrator/EWS/Contact.php
@@ -16,7 +16,7 @@
/**
* Get GetItem request parameters
*/
- protected function getItemRequest(): array
+ protected static function getItemRequest(): array
{
$request = parent::getItemRequest();
@@ -29,7 +29,7 @@
/**
* Process contact object
*/
- protected function processItem(Type $item)
+ protected function convertItem(Type $item)
{
// Decode MIME content
$vcard = base64_decode((string) $item->getMimeContent());
@@ -56,7 +56,7 @@
$vcard = str_replace($matches[1], "X-SPOUSE:{$spouse}", $vcard);
}
- // TODO: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12
+ // Anniversary: X-MS-ANNIVERSARY;VALUE=DATE:2020-11-12
if (preg_match('/(X-MS-ANNIVERSARY[;:][^\r\n]+)/', $vcard, $matches)) {
$date = preg_replace('/^[^:]+:/', '', $matches[1]);
$vcard = str_replace($matches[1], "X-ANNIVERSARY:{$date}", $vcard);
diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php
--- a/src/app/DataMigrator/EWS/DistList.php
+++ b/src/app/DataMigrator/EWS/DistList.php
@@ -13,10 +13,23 @@
public const TYPE = 'IPM.DistList';
public const FILE_EXT = 'vcf';
+ /**
+ * Get GetItem request parameters
+ */
+ protected static function getItemRequest(): array
+ {
+ $request = parent::getItemRequest();
+
+ // Get Body property, it's not included in the Default set
+ $request['ItemShape']['AdditionalProperties']['FieldURI'] = ['FieldURI' => 'item:Body'];
+
+ return $request;
+ }
+
/**
* Convert distribution list object to vCard
*/
- protected function processItem(Type $item)
+ protected function convertItem(Type $item)
{
// Groups (Distribution Lists) are not exported in vCard format, they use eml
@@ -34,11 +47,6 @@
$vcard .= $this->formatProp($key, $prop[0], isset($prop[1]) ? $prop[1] : []);
}
- // TODO: The group description property in Exchange is not available via EWS XML,
- // at least not at outlook.office.com (Exchange 2010). It is available in the
- // MimeContent which is in email message format. However, Kolab Webclient does not
- // make any use of the NOTE property for contact groups.
-
// Process list members
if ($members = $item->getMembers()) {
// The Member property is either array (multiple members) or Type\MemberType
@@ -70,6 +78,11 @@
}
}
+ // Note: Kolab Webclient does not make any use of the NOTE property for contact groups
+ if ($body = (string) $item->getBody()) {
+ $vcard .= $this->formatProp('NOTE', $body);
+ }
+
$vcard .= "END:VCARD\r\n";
return $vcard;
diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php
--- a/src/app/DataMigrator/EWS/Item.php
+++ b/src/app/DataMigrator/EWS/Item.php
@@ -3,8 +3,10 @@
namespace App\DataMigrator\EWS;
use App\DataMigrator\EWS;
+use App\DataMigrator\Engine;
use App\DataMigrator\Interface\Folder as FolderInterface;
use App\DataMigrator\Interface\Item as ItemInterface;
+use App\DataMigrator\Interface\ItemSet as ItemSetInterface;
use garethp\ews\API;
use garethp\ews\API\Type;
@@ -50,11 +52,24 @@
}
/**
- * Fetch the specified object and put into a file
+ * Validate that specified EWS Item is of supported type
*/
- public function fetchItem(ItemInterface $item)
+ public static function isValidItem(Type $item): bool
{
- $itemId = $item->id;
+ $item_class = str_replace('IPM.', '', $item->getItemClass());
+ $item_class = "\App\DataMigrator\EWS\\{$item_class}";
+
+ return class_exists($item_class);
+ }
+
+ /**
+ * Process an item (fetch data and convert it)
+ */
+ public function processItem(ItemInterface $item): void
+ {
+ $itemId = ['Id' => $item->id];
+
+ \Log::debug("[EWS] Fetching item {$item->id}...");
// Fetch the item
$ewsItem = $this->driver->api->getItem($itemId, $this->getItemRequest());
@@ -64,46 +79,48 @@
\Log::debug("[EWS] Saving item {$uid}...");
// Apply type-specific format converters
- $content = $this->processItem($ewsItem);
+ $content = $this->convertItem($ewsItem);
if (!is_string($content)) {
- return;
+ throw new \Exception("Failed to fetch EWS item {$this->itemId}");
}
- $uid = preg_replace('/[^a-zA-Z0-9_:@-]/', '', $uid);
-
- $location = $this->folder->location;
+ $filename = $uid . '.' . $this->fileExtension();
- if (!file_exists($location)) {
- mkdir($location, 0740, true);
- }
-
- $location .= '/' . $uid . '.' . $this->fileExtension();
+ if (strlen($content) > Engine::MAX_ITEM_SIZE) {
+ $location = $this->folder->tempFileLocation($filename);
- file_put_contents($location, $content);
+ if (file_put_contents($location, $content) === false) {
+ throw new \Exception("Failed to write to file at {$location}");
+ }
- return $location;
+ $item->filename = $location;
+ } else {
+ $item->content = $content;
+ $item->filename = $filename;
+ }
}
/**
* Item conversion code
*/
- abstract protected function processItem(Type $item);
+ abstract protected function convertItem(Type $item);
/**
* Get GetItem request parameters
*/
- protected function getItemRequest(): array
+ protected static function getItemRequest(): array
{
$request = [
'ItemShape' => [
// Reqest default set of properties
'BaseShape' => 'Default',
// Additional properties, e.g. LastModifiedTime
- // FIXME: How to add multiple properties here?
'AdditionalProperties' => [
- 'FieldURI' => ['FieldURI' => 'item:LastModifiedTime'],
- ]
+ 'FieldURI' => [
+ ['FieldURI' => 'item:LastModifiedTime'],
+ ],
+ ],
]
];
diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php
--- a/src/app/DataMigrator/EWS/Note.php
+++ b/src/app/DataMigrator/EWS/Note.php
@@ -16,7 +16,7 @@
/**
* Get GetItem request parameters
*/
- protected function getItemRequest(): array
+ protected static function getItemRequest(): array
{
$request = parent::getItemRequest();
@@ -32,7 +32,7 @@
/**
* Process contact object
*/
- protected function processItem(Type $item)
+ protected function convertItem(Type $item)
{
$email = base64_decode((string) $item->getMimeContent());
diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php
--- a/src/app/DataMigrator/EWS/Task.php
+++ b/src/app/DataMigrator/EWS/Task.php
@@ -16,7 +16,7 @@
/**
* Get GetItem request parameters
*/
- protected function getItemRequest(): array
+ protected static function getItemRequest(): array
{
$request = parent::getItemRequest();
@@ -29,7 +29,7 @@
/**
* Process task object
*/
- protected function processItem(Type $item)
+ protected function convertItem(Type $item)
{
// Tasks are exported as Email messages in useless format
// (does not contain all relevant properties)
diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
--- a/src/app/DataMigrator/Engine.php
+++ b/src/app/DataMigrator/Engine.php
@@ -3,6 +3,7 @@
namespace App\DataMigrator;
use App\DataMigrator\Interface\ExporterInterface;
+use App\DataMigrator\Interface\FetchItemSetInterface;
use App\DataMigrator\Interface\Folder;
use App\DataMigrator\Interface\ImporterInterface;
use App\DataMigrator\Interface\Item;
@@ -21,6 +22,9 @@
public const TYPE_NOTE = 'note';
public const TYPE_TASK = 'task';
+ /** @const int Max item size to handle in-memory, bigger will be handled with temp files */
+ public const MAX_ITEM_SIZE = 20 * 1024 * 1024;
+
/** @var Account Source account */
public $source;
@@ -43,7 +47,7 @@
/**
* Execute migration for the specified user
*/
- public function migrate(Account $source, Account $destination, array $options = [])
+ public function migrate(Account $source, Account $destination, array $options = []): void
{
$this->source = $source;
$this->destination = $destination;
@@ -87,14 +91,14 @@
// Create a queue
$this->createQueue($queue_id);
- // We'll store output in storage/<username> tree
+ // We'll store temp files in storage/<username> tree
$location = storage_path('export/') . $source->email;
if (!file_exists($location)) {
mkdir($location, 0740, true);
}
- $types = preg_split('/\s*,\s*/', strtolower($options['type'] ?? ''));
+ $types = empty($options['type']) ? [] : preg_split('/\s*,\s*/', strtolower($options['type']));
$this->debug("Fetching folders hierarchy...");
@@ -189,8 +193,8 @@
$this->exporter->fetchItem($item);
$this->importer->createItem($item);
- if (!empty($item->filename)) {
- unlink($item->filename);
+ if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) {
+ @unlink($item->filename);
}
if (empty($this->options['sync'])) {
@@ -208,15 +212,21 @@
$this->envFromQueue($set->items[0]->folder->queueId);
}
- // TODO: Some exporters, e.g. DAV, might optimize fetching multiple items in one go,
- // we'll need a new API to do that
-
- foreach ($set->items as $item) {
- $this->exporter->fetchItem($item);
+ $importItem = function (Item $item) {
$this->importer->createItem($item);
- if (!empty($item->filename)) {
- unlink($item->filename);
+ if (!empty($item->filename) && str_starts_with($item->filename, storage_path('export/'))) {
+ @unlink($item->filename);
+ }
+ };
+
+ // Some exporters, e.g. DAV, might optimize fetching multiple items in one go
+ if ($this->exporter instanceof FetchItemSetInterface) {
+ $this->exporter->fetchItemSet($set, $importItem);
+ } else {
+ foreach ($set->items as $item) {
+ $this->exporter->fetchItem($item);
+ $importItem($item);
}
}
@@ -329,6 +339,10 @@
$driver = new IMAP($account, $this);
break;
+ case 'test':
+ $driver = new Test($account, $this);
+ break;
+
default:
throw new \Exception("Failed to init driver for '{$account->scheme}'");
}
diff --git a/src/app/DataMigrator/IMAP.php b/src/app/DataMigrator/IMAP.php
--- a/src/app/DataMigrator/IMAP.php
+++ b/src/app/DataMigrator/IMAP.php
@@ -88,7 +88,19 @@
{
$mailbox = $item->folder->fullname;
- if ($item->filename) {
+ if (strlen($item->content)) {
+ $result = $this->imap->append(
+ $mailbox,
+ $item->content,
+ $item->data['flags'],
+ $item->data['internaldate'],
+ true
+ );
+
+ if ($result === false) {
+ throw new \Exception("Failed to append IMAP message into {$mailbox}");
+ }
+ } elseif ($item->filename) {
$result = $this->imap->appendFromFile(
$mailbox,
$item->filename,
@@ -153,35 +165,36 @@
|| $header->timestamp != $item->existing['timestamp']
|| $header->size != $item->existing['size']
) {
- // Save the message content to a file
- $location = $item->folder->location;
-
- if (!file_exists($location)) {
- mkdir($location, 0740, true);
- }
+ // Handle message content in memory (up to 20MB), bigger messages will use a temp file
+ if ($header->size > Engine::MAX_ITEM_SIZE) {
+ // Save the message content to a file
+ $location = $item->folder->tempFileLocation($uid . '.eml');
- // TODO: What if parent folder not yet exists?
- $location .= '/' . $uid . '.eml';
+ $fp = fopen($location, 'w');
- // TODO: We should consider streaming the message, it should be possible
- // with append() and handlePartBody(), but I don't know if anyone tried that.
-
- $fp = fopen($location, 'w');
+ if (!$fp) {
+ throw new \Exception("Failed to open 'php://temp' stream");
+ }
- if (!$fp) {
- throw new \Exception("Failed to write to {$location}");
+ $result = $this->imap->handlePartBody($mailbox, $uid, true, '', null, null, $fp);
+ } else {
+ $result = $this->imap->handlePartBody($mailbox, $uid, true);
}
- $result = $this->imap->handlePartBody($mailbox, $uid, true, '', null, null, $fp);
-
if ($result === false) {
- fclose($fp);
+ if (!empty($fp)) {
+ fclose($fp);
+ }
+
throw new \Exception("Failed to fetch IMAP message for {$mailbox}/{$uid}");
}
- $item->filename = $location;
-
- fclose($fp);
+ if (!empty($fp) && !empty($location)) {
+ $item->filename = $location;
+ fclose($fp);
+ } else {
+ $item->content = $result;
+ }
}
$item->data = [
diff --git a/src/app/DataMigrator/Interface/FetchItemSetInterface.php b/src/app/DataMigrator/Interface/FetchItemSetInterface.php
new file mode 100644
--- /dev/null
+++ b/src/app/DataMigrator/Interface/FetchItemSetInterface.php
@@ -0,0 +1,14 @@
+<?php
+
+namespace App\DataMigrator\Interface;
+
+interface FetchItemSetInterface
+{
+ /**
+ * Fetching a set of items in one operation
+ *
+ * @param ItemSet $set Set of items
+ * @param callable $callback A callback to execute on every Item
+ */
+ public function fetchItemSet(ItemSet $set, $callback): void;
+}
diff --git a/src/app/DataMigrator/Interface/Folder.php b/src/app/DataMigrator/Interface/Folder.php
--- a/src/app/DataMigrator/Interface/Folder.php
+++ b/src/app/DataMigrator/Interface/Folder.php
@@ -34,6 +34,9 @@
public $queueId;
+ /**
+ * Create Folder instance from an array
+ */
public static function fromArray(array $data = []): Folder
{
$obj = new self();
@@ -44,4 +47,23 @@
return $obj;
}
+
+ /**
+ * Returns location of a temp file for an Item content
+ */
+ public function tempFileLocation(string $filename): string
+ {
+ $filename = preg_replace('/[^a-zA-Z0-9_:@.-]/', '', $filename);
+
+ $location = $this->location;
+
+ // TODO: What if parent folder not yet exists?
+ if (!file_exists($location)) {
+ mkdir($location, 0740, true);
+ }
+
+ $location .= '/' . $filename;
+
+ return $location;
+ }
}
diff --git a/src/app/DataMigrator/Interface/Item.php b/src/app/DataMigrator/Interface/Item.php
--- a/src/app/DataMigrator/Interface/Item.php
+++ b/src/app/DataMigrator/Interface/Item.php
@@ -10,7 +10,7 @@
/** @var mixed Item identifier */
public $id;
- /** @var Folder Folder */
+ /** @var ?Folder Folder */
public $folder;
/** @var string Object class */
@@ -24,13 +24,19 @@
*/
public $existing;
- /** @var ?string Exported object location in the local storage */
+ /** @var string Exported object content */
+ public $content = '';
+
+ /** @var ?string Exported object content location */
public $filename;
/** @var array Extra data to migrate (like email flags, internaldate, etc.) */
public $data = [];
+ /**
+ * Create Item object from an array
+ */
public static function fromArray(array $data = []): Item
{
$obj = new self();
diff --git a/src/app/DataMigrator/Interface/ItemSet.php b/src/app/DataMigrator/Interface/ItemSet.php
--- a/src/app/DataMigrator/Interface/ItemSet.php
+++ b/src/app/DataMigrator/Interface/ItemSet.php
@@ -5,16 +5,11 @@
/**
* Data object representing a set of data items
*/
-class ItemSet
+class ItemSet implements \Serializable
{
/** @var array<Item> Items list */
public $items = [];
- // TODO: Every item has a $folder property, this makes the set
- // needlesly big when serialized. We should probably store $folder
- // once with the set and remove it from an item on serialize
- // and back in unserialize.
-
/**
* Create an ItemSet instance
*/
@@ -25,4 +20,26 @@
return $obj;
}
+
+ public function serialize(): ?string
+ {
+ // Every item has a Folder property, this makes the set
+ // needlesly big when serialized. Make the size more compact.
+ $folder = count($this->items) ? $this->items[0]->folder : null;
+
+ foreach ($this->items as $item) {
+ $item->folder = null;
+ }
+
+ return serialize([$folder, $this->items]);
+ }
+
+ public function unserialize(string $data): void
+ {
+ [$folder, $this->items] = unserialize($data);
+
+ foreach ($this->items as $item) {
+ $item->folder = $folder;
+ }
+ }
}
diff --git a/src/app/DataMigrator/Test.php b/src/app/DataMigrator/Test.php
new file mode 100644
--- /dev/null
+++ b/src/app/DataMigrator/Test.php
@@ -0,0 +1,153 @@
+<?php
+
+namespace App\DataMigrator;
+
+use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\ExporterInterface;
+use App\DataMigrator\Interface\ImporterInterface;
+use App\DataMigrator\Interface\Item;
+use App\DataMigrator\Interface\ItemSet;
+
+class Test implements ExporterInterface, ImporterInterface
+{
+ protected const CHUNK_SIZE = 3;
+
+ public static $fetchedItems = [];
+ public static $createdItems = [];
+ public static $createdFolders = [];
+ public static $folders = [];
+
+ /** @var Account Account to operate on */
+ protected $account;
+
+ /** @var Engine Data migrator engine */
+ protected $engine;
+
+
+ /**
+ * Object constructor
+ */
+ public function __construct(Account $account, Engine $engine)
+ {
+ $this->engine = $engine;
+ $this->account = $account;
+ }
+
+ public static function init($folders)
+ {
+ self::$folders = $folders;
+
+ self::$fetchedItems = [];
+ self::$createdItems = [];
+ self::$createdFolders = [];
+ }
+
+ /**
+ * Check user credentials.
+ *
+ * @throws \Exception
+ */
+ public function authenticate(): void
+ {
+ }
+
+ /**
+ * Create an item in a folder.
+ *
+ * @param Item $item Item to import
+ *
+ * @throws \Exception
+ */
+ public function createItem(Item $item): void
+ {
+ self::$createdItems[] = $item;
+ }
+
+ /**
+ * Create a folder.
+ *
+ * @param Folder $folder Folder data
+ *
+ * @throws \Exception on error
+ */
+ public function createFolder(Folder $folder): void
+ {
+ self::$createdFolders[] = $folder;
+ }
+
+ /**
+ * Fetching an item
+ */
+ public function fetchItem(Item $item): void
+ {
+ $item->content = 'content';
+ $item->filename = 'test.eml';
+
+ self::$fetchedItems[] = $item;
+ }
+
+ /**
+ * Fetch a list of folder items
+ */
+ public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void
+ {
+ // Get existing messages' headers from the destination mailbox
+ $existing = $importer->getItems($folder);
+
+ $set = new ItemSet();
+
+ foreach ((self::$folders[$folder->id]['items'] ?? []) as $itemId => $item) {
+ $exists = null; // TODO
+
+ $item['id'] = $itemId;
+ $item['folder'] = $folder;
+ $item['existing'] = $exists;
+
+ $set->items[] = Item::fromArray($item);
+
+ if (count($set->items) == self::CHUNK_SIZE) {
+ $callback($set);
+ $set = new ItemSet();
+ }
+ }
+
+ if (count($set->items)) {
+ $callback($set);
+ }
+ }
+
+ /**
+ * Get a list of items, limited to their essential propeties
+ * used in incremental migration.
+ *
+ * @param Folder $folder Folder data
+ *
+ * @throws \Exception on error
+ */
+ public function getItems(Folder $folder): array
+ {
+ return self::$folders[$folder->id]['existing_items'] ?? [];
+ }
+
+ /**
+ * Get folders hierarchy
+ */
+ public function getFolders($types = []): array
+ {
+ $result = [];
+
+ foreach (self::$folders as $folderId => $folder) {
+ // Skip folder types we do not support (need)
+ if (!empty($types) && !in_array($folder['type'], $types)) {
+ continue;
+ }
+
+ $folder['id'] = $folderId;
+ $folder['total'] = count($folder['items']);
+
+ $result[] = Folder::fromArray($folder);
+ }
+
+ return $result;
+ }
+}
diff --git a/src/tests/Feature/DataMigrator/EngineTest.php b/src/tests/Feature/DataMigrator/EngineTest.php
--- a/src/tests/Feature/DataMigrator/EngineTest.php
+++ b/src/tests/Feature/DataMigrator/EngineTest.php
@@ -4,11 +4,53 @@
use App\DataMigrator\Account;
use App\DataMigrator\Engine;
+use App\DataMigrator\Jobs\FolderJob;
+use App\DataMigrator\Jobs\ItemJob;
use App\DataMigrator\Queue as MigratorQueue;
+use App\DataMigrator\Test;
+use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
class EngineTest extends TestCase
{
+ protected $data = [
+ 'Inbox' => [
+ 'type' => Engine::TYPE_MAIL,
+ 'name' => 'Inbox',
+ 'fullname' => 'Inbox',
+ 'items' => [
+ 'm1' => [],
+ 'm2' => [],
+ 'm3' => [],
+ 'm4' => [],
+ ],
+ 'existing_items' => [
+ ],
+ ],
+ 'Contacts' => [
+ 'type' => Engine::TYPE_CONTACT,
+ 'name' => 'Contacts',
+ 'fullname' => 'Contacts',
+ 'items' => [
+ 'c1' => [],
+ 'c2' => [],
+ ],
+ 'existing_items' => [
+ ],
+ ],
+ 'Calendar' => [
+ 'type' => Engine::TYPE_EVENT,
+ 'name' => 'Calendar',
+ 'fullname' => 'Calendar',
+ 'items' => [
+ 'e1' => [],
+ 'e2' => [],
+ ],
+ 'existing_items' => [
+ ],
+ ],
+ ];
+
/**
* {@inheritDoc}
*/
@@ -34,6 +76,48 @@
*/
public function testAsyncMigration(): void
{
+ $source = new Account('test://test%40domain.tld:test@test');
+ $destination = new Account('test://test%40kolab.org:test@test');
+ $engine = new Engine();
+
+ Test::init($this->data);
+ Queue::fake();
+
+ // Migration initial run
+ $engine->migrate($source, $destination, []);
+
+ $queue = MigratorQueue::first();
+
+ Queue::assertPushed(FolderJob::class, 3);
+ Queue::assertPushed(
+ FolderJob::class,
+ function ($job) use ($queue) {
+ $folder = TestCase::getObjectProperty($job, 'folder');
+ return $folder->id === 'Inbox' && $folder->queueId = $queue->id;
+ }
+ );
+ Queue::assertPushed(
+ FolderJob::class,
+ function ($job) use ($queue) {
+ $folder = TestCase::getObjectProperty($job, 'folder');
+ return $folder->id === 'Contacts' && $folder->queueId = $queue->id;
+ }
+ );
+ Queue::assertPushed(
+ FolderJob::class,
+ function ($job) use ($queue) {
+ $folder = TestCase::getObjectProperty($job, 'folder');
+ return $folder->id === 'Calendar' && $folder->queueId = $queue->id;
+ }
+ );
+
+ $this->assertCount(0, Test::$createdFolders);
+ $this->assertSame(3, $queue->jobs_started);
+ $this->assertSame(0, $queue->jobs_finished);
+ $this->assertSame([], $queue->data['options']);
+ // TODO: Assert Source and destination in the queue
+ // TODO: Test 'force' option, test executing with an existing queue
+ // TODO: Test jobs execution
$this->markTestIncomplete();
}
@@ -42,6 +126,42 @@
*/
public function testSyncMigration(): void
{
- $this->markTestIncomplete();
+ $source = new Account('test://test%40domain.tld:test@test');
+ $destination = new Account('test://test%40kolab.org:test@test');
+ $engine = new Engine();
+
+ Test::init($this->data);
+ Queue::fake();
+
+ $engine->migrate($source, $destination, ['sync' => true]);
+
+ $queue = MigratorQueue::first();
+
+ Queue::assertNothingPushed();
+
+ $this->assertSame(0, $queue->jobs_started);
+ $this->assertSame(0, $queue->jobs_finished);
+ $this->assertSame(['sync' => true], $queue->data['options']);
+
+ $this->assertCount(3, Test::$createdFolders);
+ $this->assertCount(8, Test::$createdItems);
+ $this->assertCount(8, Test::$fetchedItems);
+
+ Test::init($this->data);
+
+ // Test 'type' argument
+ $engine->migrate($source, $destination, ['sync' => true, 'type' => 'contact,event']);
+
+ $queue = MigratorQueue::whereNot('id', $queue->id)->first();
+
+ Queue::assertNothingPushed();
+
+ $this->assertSame(0, $queue->jobs_started);
+ $this->assertSame(0, $queue->jobs_finished);
+ $this->assertSame(['sync' => true, 'type' => 'contact,event'], $queue->data['options']);
+
+ $this->assertCount(2, Test::$createdFolders);
+ $this->assertCount(4, Test::$createdItems);
+ $this->assertCount(4, Test::$fetchedItems);
}
}
diff --git a/src/tests/Feature/DomainTest.php b/src/tests/Feature/DomainTest.php
--- a/src/tests/Feature/DomainTest.php
+++ b/src/tests/Feature/DomainTest.php
@@ -138,9 +138,6 @@
$domainNamespace === $domain->namespace;
}
);
-
- $job = new \App\Jobs\Domain\CreateJob($domain->id);
- $job->handle();
}
/**
diff --git a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
--- a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
@@ -15,7 +15,7 @@
/**
* Test appointment item processing
*/
- public function testProcessItem(): void
+ public function testConvertItem(): void
{
$account = new Account('ews://test:test@test');
$engine = new Engine();
@@ -78,7 +78,7 @@
]);
// Convert the Exchange item into iCalendar
- $ical = $this->invokeMethod($appointment, 'processItem', [$item]);
+ $ical = $this->invokeMethod($appointment, 'convertItem', [$item]);
// Parse the iCalendar output
$event = new Vevent();
diff --git a/src/tests/Unit/DataMigrator/EWS/ContactTest.php b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
--- a/src/tests/Unit/DataMigrator/EWS/ContactTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
@@ -15,7 +15,7 @@
/**
* Test contact item processing
*/
- public function testProcessItem(): void
+ public function testConvertItem(): void
{
$account = new Account('ews://test:test@test');
$engine = new Engine();
@@ -113,7 +113,7 @@
]);
// Convert the Exchange item into vCard
- $vcard = $this->invokeMethod($contact, 'processItem', [$item]);
+ $vcard = $this->invokeMethod($contact, 'convertItem', [$item]);
// Parse the vCard
$contact = new Vcard();
diff --git a/src/tests/Unit/DataMigrator/EWS/DistListTest.php b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
--- a/src/tests/Unit/DataMigrator/EWS/DistListTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
@@ -15,7 +15,7 @@
/**
* Test contact item processing
*/
- public function testProcessItem(): void
+ public function testConvertItem(): void
{
$account = new Account('ews://test:test@test');
$engine = new Engine();
@@ -37,6 +37,11 @@
'LastModifiedTime' => '2024-06-27T13:44:32Z',
'DisplayName' => 'Lista',
'FileAs' => 'lista',
+ 'Body' => [
+ 'BodyType' => 'Text',
+ 'IsTruncated' => false,
+ '_value' => 'distlist body',
+ ],
'Members' => (object) [
'Member' => [
Type\MemberType::buildFromArray([
@@ -67,7 +72,7 @@
]);
// Convert the Exchange item into vCard
- $vcard = $this->invokeMethod($distlist, 'processItem', [$item]);
+ $vcard = $this->invokeMethod($distlist, 'convertItem', [$item]);
// Parse the vCard
$distlist = new Vcard();
@@ -78,6 +83,7 @@
$this->assertMatchesRegularExpression('/^[a-f0-9]{40}$/', $distlist->uid);
$this->assertSame('group', $distlist->kind);
$this->assertSame('Lista', $distlist->fn);
+ $this->assertSame('distlist body', $distlist->note);
$this->assertSame('Kolab EWS Data Migrator', $distlist->prodid);
$this->assertSame('2024-06-27T13:44:32Z', $distlist->rev);
diff --git a/src/tests/Unit/DataMigrator/EWS/TaskTest.php b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
--- a/src/tests/Unit/DataMigrator/EWS/TaskTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
@@ -15,7 +15,7 @@
/**
* Test task item processing
*/
- public function testProcessItem(): void
+ public function testConvertItem(): void
{
$source = new Account('ews://test%40domain.tld:test@test');
$destination = new Account('dav://test%40kolab.org:test@test');
@@ -118,7 +118,7 @@
]);
// Convert the Exchange item into iCalendar
- $ical = $this->invokeMethod($task, 'processItem', [$item]);
+ $ical = $this->invokeMethod($task, 'convertItem', [$item]);
// Parse the iCalendar output
$task = new Vtodo();
@@ -154,7 +154,7 @@
/**
* Test processing Recurrence property
*/
- public function testProcessItemRecurrence(): void
+ public function testConvertItemRecurrence(): void
{
$this->markTestIncomplete();
}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Wed, Oct 30, 9:29 PM (8 h, 57 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
10061772
Default Alt Text
D4857.id13917.diff (40 KB)
Attached To
Mode
D4857: Data Migrator: Improvements and tests
Attached
Detach File
Event Timeline
Log In to Comment