Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F16200930
D4995.id14292.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
22 KB
Referenced Files
None
Subscribers
None
D4995.id14292.diff
View Options
diff --git a/src/app/Console/Commands/Data/MigrateCommand.php b/src/app/Console/Commands/Data/MigrateCommand.php
--- a/src/app/Console/Commands/Data/MigrateCommand.php
+++ b/src/app/Console/Commands/Data/MigrateCommand.php
@@ -30,7 +30,8 @@
{dst : Destination account}
{--type= : Object type(s)}
{--sync : Execute migration synchronously}
- {--force : Force existing queue removal}';
+ {--force : Force existing queue removal}
+ {--folder-mapping=* : Folder mapping in the form "source:target"}';
// {--export-only : Only export data}
// {--import-only : Only import previously exported data}';
@@ -50,10 +51,18 @@
{
$src = new DataMigrator\Account($this->argument('src'));
$dst = new DataMigrator\Account($this->argument('dst'));
+
+ $folderMapping = [];
+ foreach ($this->option('folder-mapping') as $mapping) {
+ $arr = explode(":", $mapping);
+ $folderMapping[$arr[0]] = $arr[1];
+ }
+
$options = [
'type' => $this->option('type'),
'force' => $this->option('force'),
'sync' => $this->option('sync'),
+ 'folderMapping' => $folderMapping,
'stdout' => true,
];
diff --git a/src/app/DataMigrator/DAV.php b/src/app/DataMigrator/DAV.php
--- a/src/app/DataMigrator/DAV.php
+++ b/src/app/DataMigrator/DAV.php
@@ -183,6 +183,8 @@
$dav_type = $this->type2DAV($folder->type);
$location = $this->getFolderPath($folder);
$search = new DAVSearch($dav_type, false);
+ // FIXME this avoid a crash of iRony on empty collections?
+ $search->depth = "infinity";
// TODO: We request only properties relevant to incremental migration,
// i.e. to find that something exists and its last update time.
@@ -256,6 +258,8 @@
$location = $this->getFolderPath($folder);
$search = new DAVSearch($dav_type);
+ // FIXME this avoid a crash of iRony on empty collections?
+ $search->depth = "infinity";
// TODO: We request only properties relevant to incremental migration,
// i.e. to find that something exists and its last update time.
@@ -342,19 +346,22 @@
return $this->folderPaths[$cache_key];
}
- $folders = $this->client->listFolders($this->type2DAV($folder->type));
+ for ($i = 0; $i < 5; $i++) {
+ $folders = $this->client->listFolders($this->type2DAV($folder->type));
- if ($folders === false) {
- throw new \Exception("Failed to list folders on the DAV server");
- }
+ if ($folders === false) {
+ throw new \Exception("Failed to list folders on the DAV server");
+ }
- // Note: iRony flattens the list by modifying the folder name
- // This is not going to work with Cyrus DAV, but anyway folder
- // hierarchies support is not full in Kolab 4.
- foreach ($folders as $dav_folder) {
- if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) {
- return $this->folderPaths[$cache_key] = rtrim($dav_folder->href, '/');
+ // Note: iRony flattens the list by modifying the folder name
+ // This is not going to work with Cyrus DAV, but anyway folder
+ // hierarchies support is not full in Kolab 4.
+ foreach ($folders as $dav_folder) {
+ if (str_replace(' » ', '/', $dav_folder->name) === $folder->fullname) {
+ return $this->folderPaths[$cache_key] = rtrim($dav_folder->href, '/');
+ }
}
+ sleep(1);
}
throw new \Exception("Folder not found: {$folder->fullname}");
diff --git a/src/app/DataMigrator/EWS.php b/src/app/DataMigrator/EWS.php
--- a/src/app/DataMigrator/EWS.php
+++ b/src/app/DataMigrator/EWS.php
@@ -25,6 +25,7 @@
EWS\Appointment::FOLDER_TYPE,
EWS\Contact::FOLDER_TYPE,
EWS\Task::FOLDER_TYPE,
+ EWS\Email::FOLDER_TYPE,
// TODO: mail and sticky notes are exported as eml files.
// We could use imapsync to synchronize mail, but for notes
// the only option will be to convert them to Kolab format here
@@ -61,6 +62,9 @@
'XrmCompanySearch',
'XrmDealSearch',
'XrmSearch',
+ 'MS-OLK-AllCalendarItems',
+ 'MS-OLK-AllContactItems',
+ 'MS-OLK-AllMailItems',
// TODO: These are different depending on a user locale and it's not possible
// to switch to English other than changing the user locale in OWA/Exchange.
'Calendar/United States holidays',
@@ -69,6 +73,8 @@
'Kalendarz/Polska — dni wolne od pracy', // pl
'Ulubione', // pl
'Moje kontakty', // pl
+ 'Aufgabensuche', // de
+ 'Postausgang', // de
];
/** @var array Map of EWS folder types to Kolab types */
@@ -76,6 +82,7 @@
EWS\Appointment::FOLDER_TYPE => Engine::TYPE_EVENT,
EWS\Contact::FOLDER_TYPE => Engine::TYPE_CONTACT,
EWS\Task::FOLDER_TYPE => Engine::TYPE_TASK,
+ EWS\Email::FOLDER_TYPE => Engine::TYPE_MAIL,
];
/** @var Account Account to operate on */
@@ -329,6 +336,8 @@
$item['changeKey'] = $changeKey;
$existingIndex[$id] = $idx;
unset($item['x-ms-id']);
+ } else {
+ $existingIndex[$idx] = $idx;
}
}
);
@@ -344,6 +353,7 @@
'FieldURI' => [
['FieldURI' => 'item:ItemClass'],
// ['FieldURI' => 'item:Size'],
+ ['FieldURI' => 'message:InternetMessageId'], //For mail only?
],
],
],
@@ -442,10 +452,25 @@
$idx = $existingIndex[$id['Id']];
if ($existing[$idx]['changeKey'] == $id['ChangeKey']) {
+ \Log::debug("[EWS] Skipping over already existing message $idx...");
return null;
}
$exists = $existing[$idx]['href'];
+ } else {
+ $msgid = null;
+ try {
+ $msgid = $item->getInternetMessageId();
+ } catch (\Exception $e) {
+ //Ignore
+ }
+ if (isset($existingIndex[$msgid])) {
+ // If the messageid already exists, we assume it's the same email.
+ // Flag/size changes are ignored for now.
+ // Otherwise we should set uid/size/flags on exists, so the IMAP implementation can pick it up.
+ \Log::debug("[EWS] Skipping over already existing message $msgid...");
+ return null;
+ }
}
if (!EWS\Item::isValidItem($item)) {
diff --git a/src/app/DataMigrator/EWS/Appointment.php b/src/app/DataMigrator/EWS/Appointment.php
--- a/src/app/DataMigrator/EWS/Appointment.php
+++ b/src/app/DataMigrator/EWS/Appointment.php
@@ -34,7 +34,7 @@
/**
* Process event object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Initialize $this->itemId (for some unit tests)
$this->getUID($item);
@@ -57,12 +57,18 @@
// That's why we'll just remove all ATTACH:CID:... occurrences
// and inject attachments to the main event
$ical = preg_replace('/\r\nATTACH:CID:[^\r]+\r\n(\r\n [^\r\n]*)?/', '', $ical);
+ // We seem to get some weird ATTACH parts as part of ORGANIZER sometimes.
+ // Looks like this (when printing $ical to console):
+ // ORGANIZER;CN="Doe, John":MAILTO:John.Doe@example.comATTACH:CID:2388A81D6CB99E09E72ACF2D192043D418FD86B8@example.com
+ // DESCRIPTION;LANGUAGE=de-DE:@ ...
+ $ical = preg_replace('/ATTACH:CID:[^\r]+\r\n/', "\r\n", $ical);
foreach ((array) $item->getAttachments()->getFileAttachment() as $attachment) {
$_attachment = $this->getAttachment($attachment);
$ctype = $_attachment->getContentType();
$body = $_attachment->getContent();
+ $name = $_attachment->getName();
// It looks like Exchange may have an issue with plain text files.
// We'll skip empty files
@@ -78,7 +84,7 @@
// Inject the attachment at the end of the first VEVENT block
// TODO: We should not do it in memory to not exceed the memory limit
- $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;FMTTYPE={$ctype}:\r\n {$body}";
+ $append = "ATTACH;VALUE=BINARY;ENCODING=BASE64;X-LABEL={$name};FMTTYPE={$ctype}:\r\n {$body}";
$pos = strpos($ical, "\r\nEND:VEVENT");
$ical = substr_replace($ical, $append, $pos + 2, 0);
}
diff --git a/src/app/DataMigrator/EWS/Contact.php b/src/app/DataMigrator/EWS/Contact.php
--- a/src/app/DataMigrator/EWS/Contact.php
+++ b/src/app/DataMigrator/EWS/Contact.php
@@ -29,7 +29,7 @@
/**
* Process contact object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Decode MIME content
$vcard = base64_decode((string) $item->getMimeContent());
diff --git a/src/app/DataMigrator/EWS/DistList.php b/src/app/DataMigrator/EWS/DistList.php
--- a/src/app/DataMigrator/EWS/DistList.php
+++ b/src/app/DataMigrator/EWS/DistList.php
@@ -29,7 +29,7 @@
/**
* Convert distribution list object to vCard
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Groups (Distribution Lists) are not exported in vCard format, they use eml
diff --git a/src/app/DataMigrator/EWS/Email.php b/src/app/DataMigrator/EWS/Email.php
new file mode 100644
--- /dev/null
+++ b/src/app/DataMigrator/EWS/Email.php
@@ -0,0 +1,73 @@
+<?php
+
+namespace App\DataMigrator\EWS;
+
+use garethp\ews\API;
+use garethp\ews\API\Type;
+
+/**
+ * Email object handler
+ */
+class Email extends Item
+{
+ public const FOLDER_TYPE = 'IPF.Note';
+ public const TYPE = 'IPM.Email';
+ public const FILE_EXT = 'mime';
+
+
+ /**
+ * Get GetItem request parameters
+ */
+ protected static function getItemRequest(): array
+ {
+ $request = parent::getItemRequest();
+
+ // Request IncludeMimeContent as it's not included by default
+ $request['ItemShape']['IncludeMimeContent'] = true;
+
+ // Get UID property, it's not included in the Default set
+ // $request['ItemShape']['AdditionalProperties']['FieldURI'][] = ['FieldURI' => 'calendar:UID'];
+
+ return $request;
+ }
+
+ /**
+ * Process event object
+ */
+ protected function convertItem(Type $item, $targetItem)
+ {
+ // Initialize $this->itemId (for some unit tests)
+ $this->getUID($item);
+
+ // Decode MIME content
+ $mime = base64_decode((string) $item->getMimeContent());
+
+ //FIXME that's IMAP specific, and thus breaking the abstraction
+ $flags = [];
+ if ($item->getIsRead()) {
+ $flags[] = 'SEEN';
+ }
+ if ($internaldate = $item->getDateTimeReceived()) {
+ $internaldate = (new \DateTime($internaldate))->format('d-M-Y H:i:s O');
+ }
+
+ $targetItem->data = [
+ 'flags' => $flags,
+ 'internaldate' => $internaldate,
+ ];
+
+ return $mime;
+ }
+
+ /**
+ * Get Item UID (Generate a new one if needed)
+ */
+ protected function getUID(Type $item): string
+ {
+ // Only appointments have UID property
+ $this->uid = $item->getUID();
+
+ // This also sets $this->itemId;
+ return parent::getUID($item);
+ }
+}
diff --git a/src/app/DataMigrator/EWS/Item.php b/src/app/DataMigrator/EWS/Item.php
--- a/src/app/DataMigrator/EWS/Item.php
+++ b/src/app/DataMigrator/EWS/Item.php
@@ -78,8 +78,9 @@
\Log::debug("[EWS] Saving item {$uid}...");
+
// Apply type-specific format converters
- $content = $this->convertItem($ewsItem);
+ $content = $this->convertItem($ewsItem, $item);
if (!is_string($content)) {
throw new \Exception("Failed to fetch EWS item {$this->itemId}");
@@ -104,7 +105,7 @@
/**
* Item conversion code
*/
- abstract protected function convertItem(Type $item);
+ abstract protected function convertItem(Type $item, $targetItem);
/**
* Get GetItem request parameters
diff --git a/src/app/DataMigrator/EWS/Note.php b/src/app/DataMigrator/EWS/Note.php
--- a/src/app/DataMigrator/EWS/Note.php
+++ b/src/app/DataMigrator/EWS/Note.php
@@ -32,10 +32,25 @@
/**
* Process contact object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
$email = base64_decode((string) $item->getMimeContent());
+ $flags = [];
+ if ($item->getIsRead()) {
+ $flags[] = 'SEEN';
+ }
+
+ $internaldate = null;
+ if ($internaldate = $item->getDateTimeReceived()) {
+ $internaldate = (new \DateTime($internaldate))->format('d-M-Y H:i:s O');
+ }
+
+ $targetItem->data = [
+ 'flags' => $flags,
+ 'internaldate' => $internaldate,
+ ];
+
return $email;
}
}
diff --git a/src/app/DataMigrator/EWS/Task.php b/src/app/DataMigrator/EWS/Task.php
--- a/src/app/DataMigrator/EWS/Task.php
+++ b/src/app/DataMigrator/EWS/Task.php
@@ -29,7 +29,7 @@
/**
* Process task object
*/
- protected function convertItem(Type $item)
+ protected function convertItem(Type $item, $targetItem)
{
// Tasks are exported as Email messages in useless format
// (does not contain all relevant properties)
diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
--- a/src/app/DataMigrator/Engine.php
+++ b/src/app/DataMigrator/Engine.php
@@ -105,6 +105,7 @@
$folders = $this->exporter->getFolders($types);
$count = 0;
$async = empty($options['sync']);
+ $folderMapping = $this->options['folderMapping'] ?? [];
foreach ($folders as $folder) {
$this->debug("Processing folder {$folder->fullname}...");
@@ -112,6 +113,16 @@
$folder->queueId = $queue_id;
$folder->location = $location;
+ // Apply name replacements
+ $folder->targetname = $folder->fullname;
+ foreach ($folderMapping as $key => $value) {
+ if (str_contains($folder->targetname, $key)) {
+ $folder->targetname = str_replace($key, $value, $folder->targetname);
+ $this->debug("Replacing {$folder->fullname} with {$folder->targetname}");
+ break;
+ }
+ }
+
if ($async) {
// Dispatch the job (for async execution)
Jobs\FolderJob::dispatch($folder);
diff --git a/src/app/DataMigrator/IMAP.php b/src/app/DataMigrator/IMAP.php
--- a/src/app/DataMigrator/IMAP.php
+++ b/src/app/DataMigrator/IMAP.php
@@ -72,22 +72,33 @@
throw new \Exception("IMAP does not support folder of type {$folder->type}");
}
- if ($folder->fullname == 'INBOX') {
+ if ($folder->targetname == 'INBOX') {
// INBOX always exists
return;
}
- if (!$this->imap->createFolder($folder->fullname)) {
+ if (!$this->imap->createFolder(self::toUTF7($folder->targetname))) {
\Log::warning("Failed to create the folder: {$this->imap->error}");
if (str_contains($this->imap->error, "Mailbox already exists")) {
// Not an error
} else {
- throw new \Exception("Failed to create an IMAP folder {$folder->fullname}");
+ throw new \Exception("Failed to create an IMAP folder {$folder->targetname}");
}
}
- // TODO: Migrate folder subscription state
+ // TODO: Migrate folder subscription state. For now we just subscribe.
+ if (!$this->imap->subscribe(self::toUTF7($folder->targetname))) {
+ \Log::warning("Failed to subscribe to the folder: {$this->imap->error}");
+ }
+ }
+
+ /**
+ * Convert UTF8 string to UTF7-IMAP encoding
+ */
+ private static function toUTF7(string $string): string
+ {
+ return \mb_convert_encoding($string, 'UTF7-IMAP', 'UTF8');
}
/**
@@ -99,14 +110,14 @@
*/
public function createItem(Item $item): void
{
- $mailbox = $item->folder->fullname;
+ $mailbox = self::toUTF7($item->folder->targetname);
if (strlen($item->content)) {
$result = $this->imap->append(
$mailbox,
$item->content,
- $item->data['flags'],
- $item->data['internaldate'],
+ $item->data['flags'] ?? [],
+ $item->data['internaldate'] ?? null,
true
);
@@ -118,8 +129,8 @@
$mailbox,
$item->filename,
null,
- $item->data['flags'],
- $item->data['internaldate'],
+ $item->data['flags'] ?? [],
+ $item->data['internaldate'] ?? null,
true
);
@@ -317,7 +328,11 @@
*/
public function getItems(Folder $folder): array
{
- $mailbox = $folder->fullname;
+ if ($folder->targetname) {
+ $mailbox = self::toUTF7($folder->targetname);
+ } else {
+ $mailbox = $folder->fullname;
+ }
// TODO: We should probably first use SEARCH/SORT to skip messages marked as \Deleted
// TODO: fetchHeaders() fetches too many headers, we should slim-down, here we need
diff --git a/src/app/DataMigrator/Interface/Folder.php b/src/app/DataMigrator/Interface/Folder.php
--- a/src/app/DataMigrator/Interface/Folder.php
+++ b/src/app/DataMigrator/Interface/Folder.php
@@ -27,6 +27,9 @@
/** @var string Folder name with path */
public $fullname;
+ /** @var string Target folder name with path */
+ public $targetname;
+
/** @var string Storage location (for temporary data) */
public $location;
diff --git a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
--- a/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/AppointmentTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -21,6 +22,7 @@
$engine = new Engine();
$ews = new EWS($account, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$appointment = new EWS\Appointment($ews, $folder);
$ical = file_get_contents(__DIR__ . '/../../../data/ews/event/1.ics');
@@ -78,7 +80,7 @@
]);
// Convert the Exchange item into iCalendar
- $ical = $this->invokeMethod($appointment, 'convertItem', [$item]);
+ $ical = $this->invokeMethod($appointment, 'convertItem', [$item, $targetItem]);
// Parse the iCalendar output
$event = new Vevent();
diff --git a/src/tests/Unit/DataMigrator/EWS/ContactTest.php b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
--- a/src/tests/Unit/DataMigrator/EWS/ContactTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/ContactTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -21,6 +22,7 @@
$engine = new Engine();
$ews = new EWS($account, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$contact = new EWS\Contact($ews, $folder);
$vcard = file_get_contents(__DIR__ . '/../../../data/ews/contact/1.vcf');
@@ -113,7 +115,7 @@
]);
// Convert the Exchange item into vCard
- $vcard = $this->invokeMethod($contact, 'convertItem', [$item]);
+ $vcard = $this->invokeMethod($contact, 'convertItem', [$item, $targetItem]);
// Parse the vCard
$contact = new Vcard();
diff --git a/src/tests/Unit/DataMigrator/EWS/DistListTest.php b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
--- a/src/tests/Unit/DataMigrator/EWS/DistListTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/DistListTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -21,6 +22,7 @@
$engine = new Engine();
$ews = new EWS($account, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$distlist = new EWS\DistList($ews, $folder);
// FIXME: I haven't found a way to convert xml content into a Type instance
@@ -72,7 +74,7 @@
]);
// Convert the Exchange item into vCard
- $vcard = $this->invokeMethod($distlist, 'convertItem', [$item]);
+ $vcard = $this->invokeMethod($distlist, 'convertItem', [$item, $targetItem]);
// Parse the vCard
$distlist = new Vcard();
diff --git a/src/tests/Unit/DataMigrator/EWS/TaskTest.php b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
--- a/src/tests/Unit/DataMigrator/EWS/TaskTest.php
+++ b/src/tests/Unit/DataMigrator/EWS/TaskTest.php
@@ -7,6 +7,7 @@
use App\DataMigrator\Engine;
use App\DataMigrator\EWS;
use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
use garethp\ews\API\Type;
use Tests\TestCase;
@@ -24,6 +25,7 @@
$engine->destination = $destination;
$ews = new EWS($source, $engine);
$folder = Folder::fromArray(['id' => 'test']);
+ $targetItem = Item::fromArray(['id' => 'test']);
$task = new EWS\Task($ews, $folder);
// FIXME: I haven't found a way to convert xml content into a Type instance
@@ -118,7 +120,7 @@
]);
// Convert the Exchange item into iCalendar
- $ical = $this->invokeMethod($task, 'convertItem', [$item]);
+ $ical = $this->invokeMethod($task, 'convertItem', [$item, $targetItem]);
// Parse the iCalendar output
$task = new Vtodo();
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Fri, Oct 18, 8:48 PM (12 h, 34 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
9872781
Default Alt Text
D4995.id14292.diff (22 KB)
Attached To
Mode
D4995: Required changes for EWS import, including email
Attached
Detach File
Event Timeline
Log In to Comment