Page MenuHomePhorge
Files F16219659

D4995.id14295.diff
No OneTemporary
Actions

D4995.id14295.diff
View Options

diff --git a/src/app/Console/Commands/Data/MigrateCommand.php b/src/app/Console/Commands/Data/MigrateCommand.php
--- a/src/app/Console/Commands/Data/MigrateCommand.php
+++ b/src/app/Console/Commands/Data/MigrateCommand.php
@@ -30,7 +30,8 @@
{dst : Destination account}
{--type= : Object type(s)}
{--sync : Execute migration synchronously}
- {--force : Force existing queue removal}';
+ {--force : Force existing queue removal}
+ {--folder-mapping=* : Folder mapping in the form "source:target"}';
// {--export-only : Only export data}
// {--import-only : Only import previously exported data}';
@@ -50,10 +51,18 @@
{
$src = new DataMigrator\Account($this->argument('src'));
$dst = new DataMigrator\Account($this->argument('dst'));
+
+ $folderMapping = [];
+ foreach ($this->option('folder-mapping') as $mapping) {
+ $arr = explode(":", $mapping);
+ $folderMapping[$arr[0]] = $arr[1];
+ }
+
$options = [
'type' => $this->option('type'),
'force' => $this->option('force'),
'sync' => $this->option('sync'),
+ 'folderMapping' => $folderMapping,
'stdout' => true,
];
diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php
--- a/src/app/DataMigrator/DAV.php
+++ b/src/app/DataMigrator/DAV.php
@@ -183,6 +183,8 @@
$dav_type = $this->type2DAV($folder->type);
$location = $this->getFolderPath($folder);
$search = new DAVSearch($dav_type, false);
+ // FIXME this avoid a crash of iRony on empty collections?
+ $search->depth = "infinity";
// TODO: We request only properties relevant to incremental migration,
// i.e. to find that something exists and its last update time.
@@ -256,6 +258,8 @@
$location = $this->getFolderPath($folder);
$search = new DAVSearch($dav_type);
+ // FIXME this avoid a crash of iRony on empty collections?
+ $search->depth = "infinity";
// TODO: We request only properties relevant to incremental migration,
// i.e. to find that something exists and its last update time.
@@ -342,19 +346,22 @@
return $this->folderPaths[$cache_key];
}
- $folders = $this->client->listFolders($this->type2DAV($folder->type));
+ for ($i = 0; $i < 5; $i++) {
+ $folders = $this->client->listFolders($this->type2DAV($folder->type));
- if ($folders === false) {
- throw new \Exception("Failed to list folders on the DAV server");
- }
+ if ($folders === false) {
+ throw new \Exception("Failed to list folders on the DAV server");
+ }
- // Note: iRony flattens the list by modifying the folder name
- // This is not going to work with Cyrus DAV, but anyway folder
- // hierarchies support is not full in Kolab 4.
- foreach ($folders as $dav_folder) {
- if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) {
- return $this->folderPaths[$cache_key] = rtrim($dav_folder->href, '/');
+ // Note: iRony flattens the list by modifying the folder name
+ // This is not going to work with Cyrus DAV, but anyway folder
+ // hierarchies support is not full in Kolab 4.
+ foreach ($folders as $dav_folder) {
+ if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) {
+ return $this->folderPaths[$cache_key] = rtrim($dav_folder->href, '/');
+ }
}
+ sleep(1);
}
throw new \Exception("Folder not found: {$folder->fullname}");
diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php
--- a/src/app/DataMigrator/EWS.php
+++ b/src/app/DataMigrator/EWS.php
@@ -25,6 +25,7 @@
EWS\Appointment::FOLDER_TYPE,
EWS\Contact::FOLDER_TYPE,
EWS\Task::FOLDER_TYPE,
+ EWS\Email::FOLDER_TYPE,
// TODO: mail and sticky notes are exported as eml files.
// We could use imapsync to synchronize mail, but for notes
// the only option will be to convert them to Kolab format here
@@ -61,6 +62,9 @@
'XrmCompanySearch',
'XrmDealSearch',
'XrmSearch',
+ 'MS-OLK-AllCalendarItems',
+ 'MS-OLK-AllContactItems',
+ 'MS-OLK-AllMailItems',
// TODO: These are different depending on a user locale and it's not possible
// to switch to English other than changing the user locale in OWA/Exchange.
'Calendar/United States holidays',
@@ -69,6 +73,8 @@
'Kalendarz/Polska — dni wolne od pracy', // pl
'Ulubione', // pl
'Moje kontakty', // pl
+ 'Aufgabensuche', // de
+ 'Postausgang', // de
];
/** @var array Map of EWS folder types to Kolab types */
@@ -76,6 +82,7 @@
EWS\Appointment::FOLDER_TYPE => Engine::TYPE_EVENT,
EWS\Contact::FOLDER_TYPE => Engine::TYPE_CONTACT,
EWS\Task::FOLDER_TYPE => Engine::TYPE_TASK,
+ EWS\Email::FOLDER_TYPE => Engine::TYPE_MAIL,
];
/** @var Account Account to operate on */
@@ -329,6 +336,8 @@
$item['changeKey'] = $changeKey;
$existingIndex[$id] = $idx;
unset($item['x-ms-id']);
+ } else {
+ $existingIndex[$idx] = $idx;
}
}
);
@@ -344,6 +353,7 @@
'FieldURI' => [
['FieldURI' => 'item:ItemClass'],
// ['FieldURI' => 'item:Size'],
+ ['FieldURI' => 'message:InternetMessageId'], //For mail only?
],
],
],
@@ -442,10 +452,25 @@
$idx = $existingIndex[$id['Id']];
if ($existing[$idx]['changeKey'] == $id['ChangeKey']) {
+ \Log::debug("[EWS] Skipping over already existing message $idx...");
return null;
}
$exists = $existing[$idx]['href'];
+ } else {
+ $msgid = null;
+ try {
+ $msgid = $item->getInternetMessageId();
+ } catch (\Exception $e) {
+ //Ignore
+ }
+ if (isset($existingIndex[$msgid])) {
+ // If the messageid already exists, we assume it's the same email.
+ // Flag/size changes are ignored for now.
+ // Otherwise we should set uid/size/flags on exists, so the IMAP implementation can pick it up.
+ \Log::debug("[EWS] Skipping over already existing message $msgid...");
+ return null;
+ }
}
if (!EWS\Item::isValidItem($item)) {
diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php
--- a/src/app/DataMigrator/EWS/Appointment.php
+++ b/src/app/DataMigrator/EWS/Appointment.php
@@ -34,7 +34,7 @@
/**
* Process event object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Initialize $this->itemId (for some unit tests)
$this->getUID($item);
@@ -57,12 +57,18 @@
// That's why we'll just remove all ATTACH:CID:... occurrences
// and inject attachments to the main event
$ical = preg_replace('/\r\nATTACH:CID:[^\r]+\r\n(\r\n [^\r\n]*)?/', '', $ical);
+ // We seem to get some weird ATTACH parts as part of ORGANIZER sometimes.
+ // Looks like this (when printing $ical to console):
+ // ORGANIZER;CN="Doe, John":MAILTO:John.Doe@example.comATTACH:CID:2388A81D6CB99E09E72ACF2D192043D418FD86B8@example.com
+ // DESCRIPTION;LANGUAGE=de-DE:@ ...
+ $ical = preg_replace('/ATTACH:CID:[^\r]+\r\n/', "\r\n", $ical);
foreach ((array) $item->getAttachments()->getFileAttachment() as $attachment) {
$_attachment = $this->getAttachment($attachment);
$ctype = $_attachment->getContentType();
$body = $_attachment->getContent();
+ $name = $_attachment->getName();
// It looks like Exchange may have an issue with plain text files.
// We'll skip empty files
@@ -78,7 +84,7 @@
// Inject the attachment at the end of the first VEVENT block
// TODO: We should not do it in memory to not exceed the memory limit
- $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;FMTTYPE={$ctype}:\r\n {$body}";
+ $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;X-LABEL={$name};FMTTYPE={$ctype}:\r\n {$body}";
$pos = strpos($ical, "\r\nEND:VEVENT");
$ical = substr_replace($ical, $append, $pos + 2, 0);
}
diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php
--- a/src/app/DataMigrator/EWS/Contact.php
+++ b/src/app/DataMigrator/EWS/Contact.php
@@ -29,7 +29,7 @@
/**
* Process contact object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Decode MIME content
$vcard = base64_decode((string) $item->getMimeContent());
diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php
--- a/src/app/DataMigrator/EWS/DistList.php
+++ b/src/app/DataMigrator/EWS/DistList.php
@@ -29,7 +29,7 @@
/**
* Convert distribution list object to vCard
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Groups (Distribution Lists) are not exported in vCard format, they use eml
diff --git a/src/app/DataMigrator/EWS/Email.php b/src/app/DataMigrator/EWS/Email.php
new file mode 100644
--- /dev/null
+++ b/src/app/DataMigrator/EWS/Email.php
@@ -0,0 +1,53 @@
+<?php
+
+namespace App\DataMigrator\EWS;
+
+use garethp\ews\API\Type;
+
+/**
+ * Email object handler
+ */
+class Email extends Item
+{
+ public const FOLDER_TYPE = 'IPF.Note';
+ public const TYPE = 'IPM.Email';
+ public const FILE_EXT = 'mime';
+
+
+ /**
+ * Get GetItem request parameters
+ */
+ protected static function getItemRequest(): array
+ {
+ $request = parent::getItemRequest();
+
+ // Request IncludeMimeContent as it's not included by default
+ $request['ItemShape']['IncludeMimeContent'] = true;
+
+ // Get UID property, it's not included in the Default set
+ // $request['ItemShape']['AdditionalProperties']['FieldURI'][] = ['FieldURI' => 'calendar:UID'];
+
+ return $request;
+ }
+
+ /**
+ * Process event object
+ */
+ protected function convertItem(Type $item, $targetItem)
+ {
+ // This is not actually called, emails are migrate with type Note, because of the FOLDER_TYPE
+ return null;
+ }
+
+ /**
+ * Get Item UID (Generate a new one if needed)
+ */
+ protected function getUID(Type $item): string
+ {
+ // Only appointments have UID property
+ $this->uid = $item->getUID();
+
+ // This also sets $this->itemId;
+ return parent::getUID($item);
+ }
+}
diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php
--- a/src/app/DataMigrator/EWS/Item.php
+++ b/src/app/DataMigrator/EWS/Item.php
@@ -79,7 +79,7 @@
\Log::debug("[EWS] Saving item {$uid}...");
// Apply type-specific format converters
- $content = $this->convertItem($ewsItem);
+ $content = $this->convertItem($ewsItem, $item);
if (!is_string($content)) {
throw new \Exception("Failed to fetch EWS item {$this->itemId}");
@@ -104,7 +104,7 @@
/**
* Item conversion code
*/
- abstract protected function convertItem(Type $item);
+ abstract protected function convertItem(Type $item, $targetItem);
/**
* Get GetItem request parameters
diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php
--- a/src/app/DataMigrator/EWS/Note.php
+++ b/src/app/DataMigrator/EWS/Note.php
@@ -32,10 +32,25 @@
/**
* Process contact object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
$email = base64_decode((string) $item->getMimeContent());
+ $flags = [];
+ if ($item->getIsRead()) {
+ $flags[] = 'SEEN';
+ }
+
+ $internaldate = null;
+ if ($internaldate = $item->getDateTimeReceived()) {
+ $internaldate = (new \DateTime($internaldate))->format('d-M-Y H:i:s O');
+ }
+
+ $targetItem->data = [
+ 'flags' => $flags,
+ 'internaldate' => $internaldate,
+ ];
+
return $email;
}
}
diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php
--- a/src/app/DataMigrator/EWS/Task.php
+++ b/src/app/DataMigrator/EWS/Task.php
@@ -29,7 +29,7 @@
/**
* Process task object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Tasks are exported as Email messages in useless format
// (does not contain all relevant properties)
diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
--- a/src/app/DataMigrator/Engine.php
+++ b/src/app/DataMigrator/Engine.php
@@ -105,6 +105,7 @@
$folders = $this->exporter->getFolders($types);
$count = 0;
$async = empty($options['sync']);
+ $folderMapping = $this->options['folderMapping'] ?? [];
foreach ($folders as $folder) {
$this->debug("Processing folder {$folder->fullname}...");
@@ -112,6 +113,16 @@
$folder->queueId = $queue_id;
$folder->location = $location;
+ // Apply name replacements
+ $folder->targetname = $folder->fullname;
+ foreach ($folderMapping as $key => $value) {
+ if (str_contains($folder->targetname, $key)) {
+ $folder->targetname = str_replace($key, $value, $folder->targetname);
+ $this->debug("Replacing {$folder->fullname} with {$folder->targetname}");
+ break;
+ }
+ }
+
if ($async) {
// Dispatch the job (for async execution)
Jobs\FolderJob::dispatch($folder);
diff --git a/src/app/DataMigrator/IMAP.php b/src/app/DataMigrator/IMAP.php
--- a/src/app/DataMigrator/IMAP.php
+++ b/src/app/DataMigrator/IMAP.php
@@ -72,22 +72,33 @@
throw new \Exception("IMAP does not support folder of type {$folder->type}");
}
- if ($folder->fullname == 'INBOX') {
+ if ($folder->targetname == 'INBOX') {
// INBOX always exists
return;
}
- if (!$this->imap->createFolder($folder->fullname)) {
+ if (!$this->imap->createFolder(self::toUTF7($folder->targetname))) {
\Log::warning("Failed to create the folder: {$this->imap->error}");
if (str_contains($this->imap->error, "Mailbox already exists")) {
// Not an error
} else {
- throw new \Exception("Failed to create an IMAP folder {$folder->fullname}");
+ throw new \Exception("Failed to create an IMAP folder {$folder->targetname}");
}
}
- // TODO: Migrate folder subscription state
+ // TODO: Migrate folder subscription state. For now we just subscribe.
+ if (!$this->imap->subscribe(self::toUTF7($folder->targetname))) {
+ \Log::warning("Failed to subscribe to the folder: {$this->imap->error}");
+ }
+ }
+
+ /**
+ * Convert UTF8 string to UTF7-IMAP encoding
+ */
+ private static function toUTF7(string $string): string
+ {
+ return \mb_convert_encoding($string, 'UTF7-IMAP', 'UTF8');
}
/**
@@ -99,14 +110,14 @@
*/
public function createItem(Item $item): void
{
- $mailbox = $item->folder->fullname;
+ $mailbox = self::toUTF7($item->folder->targetname);
if (strlen($item->content)) {
$result = $this->imap->append(
$mailbox,
$item->content,
- $item->data['flags'],
- $item->data['internaldate'],
+ $item->data['flags'] ?? [],
+ $item->data['internaldate'] ?? null,
true
);
@@ -118,8 +129,8 @@
$mailbox,
$item->filename,
null,
- $item->data['flags'],
- $item->data['internaldate'],
+ $item->data['flags'] ?? [],
+ $item->data['internaldate'] ?? null,
true
);
@@ -317,7 +328,11 @@
*/
public function getItems(Folder $folder): array
{
- $mailbox = $folder->fullname;
+ if ($folder->targetname) {
+ $mailbox = self::toUTF7($folder->targetname);
+ } else {
+ $mailbox = $folder->fullname;
+ }
// TODO: We should probably first use SEARCH/SORT to skip messages marked as \Deleted
// TODO: fetchHeaders() fetches too many headers, we should slim-down, here we need
diff --git a/src/app/DataMigrator/Interface/Folder.php b/src/app/DataMigrator/Interface/Folder.php
--- a/src/app/DataMigrator/Interface/Folder.php
+++ b/src/app/DataMigrator/Interface/Folder.php
@@ -27,6 +27,9 @@
/** @var string Folder name with path */
public $fullname;
+ /** @var string Target folder name with path */
+ public $targetname;
+
/** @var string Storage location (for temporary data) */
public $location;
diff --git a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
--- a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -21,6 +22,7 @@
$engine = new Engine();
$ews = new EWS($account, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$appointment = new EWS\Appointment($ews, $folder);
$ical = file_get_contents(__DIR__ . '/../../../data/ews/event/1.ics');
@@ -78,7 +80,7 @@
]);
// Convert the Exchange item into iCalendar
- $ical = $this->invokeMethod($appointment, 'convertItem', [$item]);
+ $ical = $this->invokeMethod($appointment, 'convertItem', [$item, $targetItem]);
// Parse the iCalendar output
$event = new Vevent();
diff --git a/src/tests/Unit/DataMigrator/EWS/ContactTest.php b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
--- a/src/tests/Unit/DataMigrator/EWS/ContactTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -21,6 +22,7 @@
$engine = new Engine();
$ews = new EWS($account, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$contact = new EWS\Contact($ews, $folder);
$vcard = file_get_contents(__DIR__ . '/../../../data/ews/contact/1.vcf');
@@ -113,7 +115,7 @@
]);
// Convert the Exchange item into vCard
- $vcard = $this->invokeMethod($contact, 'convertItem', [$item]);
+ $vcard = $this->invokeMethod($contact, 'convertItem', [$item, $targetItem]);
// Parse the vCard
$contact = new Vcard();
diff --git a/src/tests/Unit/DataMigrator/EWS/DistListTest.php b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
--- a/src/tests/Unit/DataMigrator/EWS/DistListTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -21,6 +22,7 @@
$engine = new Engine();
$ews = new EWS($account, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$distlist = new EWS\DistList($ews, $folder);
// FIXME: I haven't found a way to convert xml content into a Type instance
@@ -72,7 +74,7 @@
]);
// Convert the Exchange item into vCard
- $vcard = $this->invokeMethod($distlist, 'convertItem', [$item]);
+ $vcard = $this->invokeMethod($distlist, 'convertItem', [$item, $targetItem]);
// Parse the vCard
$distlist = new Vcard();
diff --git a/src/tests/Unit/DataMigrator/EWS/TaskTest.php b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
--- a/src/tests/Unit/DataMigrator/EWS/TaskTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -24,6 +25,7 @@
$engine->destination = $destination;
$ews = new EWS($source, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$task = new EWS\Task($ews, $folder);
// FIXME: I haven't found a way to convert xml content into a Type instance
@@ -118,7 +120,7 @@
]);
// Convert the Exchange item into iCalendar
- $ical = $this->invokeMethod($task, 'convertItem', [$item]);
+ $ical = $this->invokeMethod($task, 'convertItem', [$item, $targetItem]);
// Parse the iCalendar output
$task = new Vtodo();

File Metadata

Mime Type
text/plain
Expires
Sat, Oct 19, 10:46 AM (21 h, 18 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
9882597
Default Alt Text
D4995.id14295.diff (22 KB)

Event Timeline