Page MenuHomePhorge

D5088.1775284995.diff
No OneTemporary

Authored By
Unknown
Size
45 KB
Referenced Files
None
Subscribers
None

D5088.1775284995.diff

diff --git a/src/app/DataMigrator/Account.php b/src/app/DataMigrator/Account.php
--- a/src/app/DataMigrator/Account.php
+++ b/src/app/DataMigrator/Account.php
@@ -50,7 +50,7 @@
{
$this->input = $input;
- if (!preg_match('|^[a-z]+://.*|', $input)) {
+ if (!preg_match('|^[a-z0-9]+://.*|', $input)) {
throw new \Exception("Invalid URI specified");
}
diff --git a/src/app/DataMigrator/Driver/IMAP.php b/src/app/DataMigrator/Driver/IMAP.php
--- a/src/app/DataMigrator/Driver/IMAP.php
+++ b/src/app/DataMigrator/Driver/IMAP.php
@@ -79,30 +79,25 @@
return;
}
- if (!$this->imap->createFolder(self::toUTF7($folder->targetname))) {
+ $mailbox = self::toUTF7($folder->targetname);
+
+ if (!$this->imap->createFolder($mailbox)) {
if (str_contains($this->imap->error, "Mailbox already exists")) {
// Not an error
- \Log::debug("Folder already exists: {$folder->targetname}");
+ \Log::debug("Folder already exists: {$mailbox}");
} else {
\Log::warning("Failed to create the folder: {$this->imap->error}");
- throw new \Exception("Failed to create an IMAP folder {$folder->targetname}");
+ throw new \Exception("Failed to create an IMAP folder {$mailbox}");
}
}
- // TODO: Migrate folder subscription state. For now we just subscribe.
- if (!$this->imap->subscribe(self::toUTF7($folder->targetname))) {
- \Log::warning("Failed to subscribe to the folder: {$this->imap->error}");
+ if ($folder->subscribed) {
+ if (!$this->imap->subscribe($mailbox)) {
+ \Log::warning("Failed to subscribe to the folder: {$this->imap->error}");
+ }
}
}
- /**
- * Convert UTF8 string to UTF7-IMAP encoding
- */
- private static function toUTF7(string $string): string
- {
- return \mb_convert_encoding($string, 'UTF7-IMAP', 'UTF8');
- }
-
/**
* Create an item in a folder.
*
@@ -172,7 +167,7 @@
{
[$uid, $messageId] = explode(':', $item->id, 2);
- $mailbox = $item->folder->fullname;
+ $mailbox = self::toUTF7($item->folder->fullname);
// Get message flags
$header = $this->imap->fetchHeader($mailbox, (int) $uid, true, false, ['FLAGS']);
@@ -239,7 +234,7 @@
// Get existing messages' headers from the destination mailbox
$existing = $importer->getItems($folder);
- $mailbox = $folder->fullname;
+ $mailbox = self::toUTF7($folder->fullname);
// TODO: We should probably first use SEARCH/SORT to skip messages marked as \Deleted
// It would also allow us to get headers in chunks 200 messages at a time, or so.
@@ -304,7 +299,11 @@
throw new \Exception("Failed to get list of IMAP folders");
}
- // TODO: Migrate folder subscription state
+ $subscribed = $this->imap->listSubscribed('', '');
+
+ if ($subscribed === false) {
+ throw new \Exception("Failed to get list of subscribed IMAP folders");
+ }
$result = [];
@@ -315,8 +314,9 @@
}
$result[] = Folder::fromArray([
- 'fullname' => $folder,
- 'type' => 'mail'
+ 'fullname' => self::fromUTF7($folder),
+ 'type' => 'mail',
+ 'subscribed' => in_array($folder, $subscribed) || $folder === 'INBOX',
]);
}
@@ -329,11 +329,7 @@
*/
public function getItems(Folder $folder): array
{
- if ($folder->targetname) {
- $mailbox = self::toUTF7($folder->targetname);
- } else {
- $mailbox = $folder->fullname;
- }
+ $mailbox = self::toUTF7($folder->targetname ? $folder->targetname : $folder->fullname);
// TODO: We should probably first use SEARCH/SORT to skip messages marked as \Deleted
// TODO: fetchHeaders() fetches too many headers, we should slim-down, here we need
@@ -477,4 +473,20 @@
return md5($folder . $message->from . $message->timestamp);
}
+
+ /**
+ * Convert UTF8 string to UTF7-IMAP encoding
+ */
+ protected static function toUTF7(string $string): string
+ {
+ return \mb_convert_encoding($string, 'UTF7-IMAP', 'UTF8');
+ }
+
+ /**
+ * Convert UTF7-IMAP string to UTF8 encoding
+ */
+ protected static function fromUTF7(string $string): string
+ {
+ return \mb_convert_encoding($string, 'UTF8', 'UTF7-IMAP');
+ }
}
diff --git a/src/app/DataMigrator/Driver/Kolab.php b/src/app/DataMigrator/Driver/Kolab.php
--- a/src/app/DataMigrator/Driver/Kolab.php
+++ b/src/app/DataMigrator/Driver/Kolab.php
@@ -72,6 +72,10 @@
*/
public function createFolder(Folder $folder): void
{
+ if ($this->account->scheme == 'kolab3') {
+ throw new \Exception("Kolab v3 destination not supported");
+ }
+
// IMAP
if ($folder->type == Engine::TYPE_MAIL) {
parent::createFolder($folder);
@@ -94,6 +98,11 @@
*/
public function createItem(Item $item): void
{
+ // Destination server is always Kolab4, we don't support migration into Kolab3
+ if ($this->account->scheme == 'kolab3') {
+ throw new \Exception("Kolab v3 destination not supported");
+ }
+
// IMAP
if ($item->folder->type == Engine::TYPE_MAIL) {
parent::createItem($item);
@@ -106,7 +115,11 @@
return;
}
- // TODO: Configuration (v3 tags)
+ // Configuration (v3 tags)
+ if ($item->folder->type == Engine::TYPE_CONFIGURATION) {
+ Kolab\Tags::migrateKolab3Tag($this->imap, $item);
+ return;
+ }
}
/**
@@ -114,6 +127,11 @@
*/
public function fetchItem(Item $item): void
{
+ // No support for migration from Kolab4 yet
+ if ($this->account->scheme != 'kolab3') {
+ throw new \Exception("Kolab v4 source not supported");
+ }
+
// IMAP
if ($item->folder->type == Engine::TYPE_MAIL) {
parent::fetchItem($item);
@@ -126,14 +144,23 @@
return;
}
- // TODO: Configuration (v3 tags)
+ // Configuration (v3 tags)
+ if ($item->folder->type == Engine::TYPE_CONFIGURATION) {
+ Kolab\Tags::fetchKolab3Tag($this->imap, $item);
+ return;
+ }
}
/**
- * Fetch a list of folder items
+ * Fetch a list of folder items from the source server
*/
public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void
{
+ // Note: No support for migration from Kolab4 yet
+ if ($this->account->scheme != 'kolab3') {
+ throw new \Exception("Kolab v4 source not supported");
+ }
+
// IMAP
if ($folder->type == Engine::TYPE_MAIL) {
parent::fetchItemList($folder, $callback, $importer);
@@ -146,7 +173,18 @@
return;
}
- // TODO: Configuration (v3 tags)
+ // Configuration (v3 tags)
+ if ($folder->type == Engine::TYPE_CONFIGURATION) {
+ // Get existing tags from the destination account
+ $existing = $importer->getItems($folder);
+
+ $mailbox = self::toUTF7($folder->fullname);
+ foreach (Kolab\Tags::getKolab3Tags($this->imap, $mailbox, $existing) as $tag) {
+ $tag['folder'] = $folder;
+ $item = Item::fromArray($tag);
+ $callback($item);
+ }
+ }
}
/**
@@ -154,6 +192,11 @@
*/
public function getFolders($types = []): array
{
+ // Note: No support for migration from Kolab4 yet.
+ if ($this->account->scheme != 'kolab3') {
+ throw new \Exception("Kolab v4 source not supported");
+ }
+
// Using only IMAP to get the list of all folders works with Kolab v3, but not v4.
// We could use IMAP, extract the XML, convert to iCal/vCard format and pass to DAV.
// But it will be easier to use DAV for contact/task/event folders migration.
@@ -169,7 +212,7 @@
}
// Get IMAP (mail and configuration) folders
- $folders = $this->imap->listMailboxes('', '');
+ $folders = $this->imap->listMailboxes('', '', ['SUBSCRIBED']);
if ($folders === false) {
throw new \Exception("Failed to get list of IMAP folders");
@@ -207,9 +250,14 @@
continue;
}
+ $is_subscribed = !empty($this->imap->data['LIST'])
+ && !empty($this->imap->data['LIST'][$folder])
+ && in_array('\Subscribed', $this->imap->data['LIST'][$folder]);
+
$folder = Folder::fromArray([
- 'fullname' => $folder,
+ 'fullname' => self::fromUTF7($folder),
'type' => $type,
+ 'subscribed' => $is_subscribed || $folder === 'INBOX',
]);
if ($type == Engine::TYPE_CONFIGURATION) {
@@ -229,11 +277,16 @@
}
/**
- * Get a list of folder items, limited to their essential propeties
+ * Get a list of folder items from the destination server, limited to their essential propeties
* used in incremental migration to skip unchanged items.
*/
public function getItems(Folder $folder): array
{
+ // Destination server is always Kolab4, we don't support migration into Kolab3
+ if ($this->account->scheme == 'kolab3') {
+ throw new \Exception("Kolab v3 destination not supported");
+ }
+
// IMAP
if ($folder->type == Engine::TYPE_MAIL) {
return parent::getItems($folder);
@@ -246,8 +299,7 @@
// Configuration folder (v3 tags)
if ($folder->type == Engine::TYPE_CONFIGURATION) {
- // X-Kolab-Type: application/x-vnd.kolab.configuration.relation
- // TODO
+ return Kolab\Tags::getKolab4Tags($this->imap);
}
return [];
diff --git a/src/app/DataMigrator/Driver/Kolab/Tags.php b/src/app/DataMigrator/Driver/Kolab/Tags.php
new file mode 100644
--- /dev/null
+++ b/src/app/DataMigrator/Driver/Kolab/Tags.php
@@ -0,0 +1,299 @@
+<?php
+
+namespace App\DataMigrator\Driver\Kolab;
+
+use App\DataMigrator\Interface\Item;
+
+/**
+ * Utilities to handle/migrate Kolab (v3 and v4) tags
+ */
+class Tags
+{
+ private const ANNOTATE_KEY_PREFIX = '/vendor/kolab/tag/v1/';
+ private const ANNOTATE_VALUE = '1';
+ private const METADATA_ROOT = 'INBOX';
+ private const METADATA_TAGS_KEY = '/private/vendor/kolab/tags/v1';
+
+
+ /**
+ * Get all tag properties, resolve tag members
+ *
+ * @param \rcube_imap_generic $imap IMAP client (account)
+ * @param Item $item Tag item
+ */
+ public static function fetchKolab3Tag($imap, Item $item): void
+ {
+ // Note: We already have all tag properties, we need to resolve
+ // member URLs into folder+message-id pairs so we can later annotate
+ // mail messages
+
+ $user = $imap->getUser();
+ $members = [];
+
+ foreach (($item->data['member'] ?? []) as $member) {
+ // TODO: For now we'll ignore tasks/notes, but in the future we
+ // should migrate task tags into task CATEGORIES property (dunno notes).
+ if (!str_starts_with($member, 'imap://')) {
+ continue;
+ }
+
+ // Sample member: imap:///(user/username@domain|shared)/<folder>/<UID>?<search_params>
+ // parse_url does not work with imap:/// prefix
+ $url = parse_url(substr($member, 8));
+ $path = explode('/', $url['path']);
+ parse_str($url['query'], $params);
+
+ // Skip members without Message-ID
+ if (empty($params['message-id'])) {
+ continue;
+ }
+
+ $uid = array_pop($path);
+ $ns = array_shift($path);
+
+ // TODO: For now we ignore shared folders
+ if ($ns != 'user') {
+ continue;
+ }
+
+ $path = array_map('rawurldecode', $path);
+ $username = array_shift($path);
+ $folder = implode('/', $path);
+
+ if ($username == $user) {
+ if (!strlen($folder)) {
+ $folder = 'INBOX';
+ }
+ }
+
+ if (!isset($members[$folder])) {
+ $members[$folder] = [];
+ }
+
+ // Get the folder and message-id, we can't use UIDs, they are different in the target account
+ $members[$folder][] = $params['message-id'];
+ }
+
+ $item->data['member'] = $members;
+ }
+
+ /**
+ * Get tags from Kolab3 folder
+ *
+ * @param \rcube_imap_generic $imap IMAP client (account)
+ * @param string $mailbox Configuration folder name
+ * @param array $existing Tags existing at the destination account
+ */
+ public static function getKolab3Tags($imap, $mailbox, $existing = []): array
+ {
+ // Find relation objects
+ $search = 'NOT DELETED HEADER X-Kolab-Type "application/x-vnd.kolab.configuration.relation"';
+ $search = $imap->search($mailbox, $search, true);
+ if ($search->is_empty()) {
+ return [];
+ }
+
+ // Get messages' basic headers, include headers for the XML attachment
+ $uids = $search->get_compressed();
+ $messages = $imap->fetchHeaders($mailbox, $uids, true, false, [], ['BODY.PEEK[2.MIME]']);
+ $tags = [];
+
+ foreach ($messages as $message) {
+ $headers = \rcube_mime::parse_headers($message->bodypart['2.MIME'] ?? '');
+
+ // Sanity check, part 2 is expected to be Kolab XML attachment
+ if (stripos($headers['content-type'] ?? '', 'application/vnd.kolab+xml') === false) {
+ continue;
+ }
+
+ // Get the XML content
+ $encoding = $headers['content-transfer-encoding'] ?? '8bit';
+ $xml = $imap->handlePartBody($mailbox, $message->uid, true, 2, $encoding);
+
+ // Remove namespace so xpath queries below are short
+ $xml = str_replace(' xmlns="http://kolab.org"', '', $xml);
+ $xml = simplexml_load_string($xml);
+
+ if ($xml === false) {
+ throw new \Exception("Failed to parse XML for {$mailbox}/{$message->uid}");
+ }
+
+ $tag = ['member' => []];
+ foreach ($xml->xpath('/configuration/*') as $node) {
+ $nodeName = $node->getName();
+ if (in_array($nodeName, ['relationType', 'name', 'color', 'last-modification-date', 'member'])) {
+ if ($nodeName == 'member') {
+ $tag['member'][] = (string) $node;
+ } else {
+ $tag[$nodeName] = (string) $node;
+ }
+ }
+ }
+
+ if ($tag['relationType'] === 'tag') {
+ if (empty($tag['last-modification-date'])) {
+ $tag['last-modification-date'] = $message->internaldate;
+ }
+
+ $exists = null;
+ foreach ($existing as $existing_tag) {
+ if ($existing_tag['name'] == $tag['name']) {
+ if (isset($existing_tag['mtime']) && $existing_tag['mtime'] == $tag['last-modification-date']) {
+ // No changes to the tag, skip it
+ continue 2;
+ }
+
+ $exists = $existing_tag;
+ break;
+ }
+ }
+
+ $tags[] = [
+ 'id' => $message->uid,
+ 'class' => 'tag',
+ 'existing' => $exists,
+ 'data' => $tag,
+ ];
+ }
+ }
+
+ return $tags;
+ }
+
+ /**
+ * Get list of Kolab4 tags
+ *
+ * @param \rcube_imap_generic $imap IMAP client (account)
+ */
+ public static function getKolab4Tags($imap): array
+ {
+ $tags = [];
+
+ if ($meta = $imap->getMetadata(self::METADATA_ROOT, self::METADATA_TAGS_KEY)) {
+ $tags = json_decode($meta[self::METADATA_ROOT][self::METADATA_TAGS_KEY], true);
+ }
+
+ return $tags;
+ }
+
+ /**
+ * Get list of Kolab4 tag members (message UIDs)
+ *
+ * @param \rcube_imap_generic $imap IMAP client (account)
+ * @param string $tag_name Tag name
+ * @param string $folder IMAP folder name
+ */
+ public static function getKolab4TagMembers($imap, string $tag_name, string $folder): array
+ {
+ $criteria = sprintf(
+ 'ANNOTATION %s value.priv %s',
+ $imap->escape(self::ANNOTATE_KEY_PREFIX . $tag_name),
+ $imap->escape(self::ANNOTATE_VALUE, true)
+ );
+
+ $search = $imap->search($folder, $criteria, true);
+
+ if ($search->is_error()) {
+ throw new \Exception("Failed to SEARCH in {$folder}. Error: {$imap->error}");
+ }
+
+ return $search->get();
+ }
+
+ /**
+ * Migrate Kolab3 tag into Kolab4 tag (including tag associations)
+ *
+ * @param \rcube_imap_generic $imap IMAP client (target account)
+ * @param Item $item Tag item
+ */
+ public static function migrateKolab3Tag($imap, Item $item): void
+ {
+ $tags = self::getKolab4Tags($imap);
+
+ $tag_name = $item->data['name'];
+ $found = false;
+
+ // Find the tag
+ foreach ($tags as &$tag) {
+ if ($tag['name'] == $tag_name) {
+ $tag['color'] = $item->data['color'] ?? null;
+ $tag['mtime'] = $item->data['last-modification-date'] ?? null;
+ $tag = array_filter($tag);
+ $found = true;
+ break;
+ }
+ }
+
+ if (!$found) {
+ $tags[] = array_filter([
+ 'name' => $tag_name,
+ 'color' => $item->data['color'] ?? null,
+ 'mtime' => $item->data['last-modification-date'] ?? null,
+ ]);
+ }
+
+ self::saveKolab4Tags($imap, $tags);
+
+ // Migrate members
+ // For each folder, search messages by Message-ID and annotate them
+ // TODO: In incremental migration (if tag already exists) we probably should
+ // remove the annotation from all messages in all folders first.
+ foreach (($item->data['member'] ?? []) as $folder => $ids) {
+ $uids = [];
+ $search = '';
+ $prefix = '';
+ // Do search in chunks
+ foreach ($ids as $idx => $id) {
+ $str = ' HEADER MESSAGE-ID ' . $imap->escape($id);
+ $len = strlen($search) + strlen($str) + strlen($prefix) + 100;
+ $is_last = $idx == count($ids) - 1;
+
+ if ($len > 65536 || $is_last) {
+ if ($is_last) {
+ $prefix .= strlen($search) ? ' OR' : '';
+ $search .= $str;
+ }
+
+ $search = $imap->search($folder, $prefix . $search, true);
+ if ($search->is_error()) {
+ throw new \Exception("Failed to SEARCH in {$folder}. Error: {$imap->error}");
+ }
+
+ $uids = array_merge($uids, $search->get());
+ $search = '';
+ $prefix = '';
+ }
+
+ $prefix .= strlen($search) ? ' OR' : '';
+ $search .= $str;
+ }
+
+ if (!empty($uids)) {
+ $annotation = [];
+ $annotation[self::ANNOTATE_KEY_PREFIX . $tag_name] = ['value.priv' => self::ANNOTATE_VALUE];
+
+ $uids = $imap->compressMessageSet(array_unique($uids));
+ $res = $imap->storeMessageAnnotation($folder, $uids, $annotation);
+
+ if (!$res) {
+ throw new \Exception("Failed to ANNOTATE in {$folder}. Error: {$imap->error}");
+ }
+ }
+ }
+ }
+
+ /**
+ * Get list of Kolab4 tags
+ *
+ * @param \rcube_imap_generic $imap IMAP client (account)
+ * @param array $tags List of tags
+ */
+ public static function saveKolab4Tags($imap, array $tags): void
+ {
+ $metadata = json_encode($tags, JSON_INVALID_UTF8_IGNORE | JSON_UNESCAPED_UNICODE);
+
+ if (!$imap->setMetadata(self::METADATA_ROOT, [self::METADATA_TAGS_KEY => $metadata])) {
+ throw new \Exception("Failed to store tags in IMAP. Error: {$imap->error}");
+ }
+ }
+}
diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
--- a/src/app/DataMigrator/Engine.php
+++ b/src/app/DataMigrator/Engine.php
@@ -382,6 +382,8 @@
break;
case 'kolab':
+ case 'kolab3':
+ case 'kolab4':
$driver = new Driver\Kolab($account, $this);
break;
diff --git a/src/app/DataMigrator/Interface/Folder.php b/src/app/DataMigrator/Interface/Folder.php
--- a/src/app/DataMigrator/Interface/Folder.php
+++ b/src/app/DataMigrator/Interface/Folder.php
@@ -21,13 +21,13 @@
/** @var string Folder Kolab object type */
public $type;
- /** @var string Folder name */
+ /** @var string Folder name (UTF-8) */
public $name;
- /** @var string Folder name with path */
+ /** @var string Folder name with path (UTF-8) */
public $fullname;
- /** @var string Target folder name with path */
+ /** @var string Target folder name with path (UTF-8) */
public $targetname;
/** @var string Storage location (for temporary data) */
@@ -36,6 +36,9 @@
/** @var string Migration queue identifier */
public $queueId;
+ /** @var bool Folder subscription state */
+ public $subscribed = true;
+
/**
* Create Folder instance from an array
diff --git a/src/include/rcube_imap_generic.php b/src/include/rcube_imap_generic.php
--- a/src/include/rcube_imap_generic.php
+++ b/src/include/rcube_imap_generic.php
@@ -1176,6 +1176,16 @@
$this->clearCapability();
}
+ /**
+ * Get crrent user.
+ *
+ * @return string|null Current username set for the connection
+ */
+ public function getUser()
+ {
+ return $this->user;
+ }
+
/**
* Executes SELECT command (if mailbox is already not in selected state)
*
diff --git a/src/include/rcube_result_index.php b/src/include/rcube_result_index.php
--- a/src/include/rcube_result_index.php
+++ b/src/include/rcube_result_index.php
@@ -19,23 +19,6 @@
+-----------------------------------------------------------------------+
*/
-/**
- * Get first element from an array
- *
- * @param array $array Input array
- *
- * @return mixed First element if found, Null otherwise
- */
-function array_first($array)
-{
- if (is_array($array)) {
- reset($array);
- foreach ($array as $element) {
- return $element;
- }
- }
-}
-
/**
* Class for accessing IMAP's SORT/SEARCH/ESEARCH result
*
diff --git a/src/tests/BackendsTrait.php b/src/tests/BackendsTrait.php
--- a/src/tests/BackendsTrait.php
+++ b/src/tests/BackendsTrait.php
@@ -296,7 +296,7 @@
/**
* Create an IMAP folder
*/
- protected function imapCreateFolder(Account $account, $folder): void
+ protected function imapCreateFolder(Account $account, $folder, bool $subscribe = false): void
{
$imap = $this->getImapClient($account);
@@ -307,6 +307,12 @@
throw new \Exception("Failed to create an IMAP folder {$account}/{$folder}");
}
}
+
+ if ($subscribe) {
+ if (!$imap->subscribe($folder)) {
+ throw new \Exception("Failed to subscribe an IMAP folder {$account}/{$folder}");
+ }
+ }
}
/**
@@ -316,13 +322,6 @@
{
$imap = $this->getImapClient($account);
- // Check the folder existence first, to prevent Cyrus IMAP fatal error when
- // attempting to delete a non-existing folder
- $existing = $imap->listMailboxes('', $folder);
- if (is_array($existing) && in_array($folder, $existing)) {
- return;
- }
-
if (!$imap->deleteFolder($folder)) {
if (str_contains($imap->error, "Mailbox does not exist")) {
// Ignore
@@ -370,11 +369,11 @@
/**
* List IMAP folders
*/
- protected function imapListFolders(Account $account): array
+ protected function imapListFolders(Account $account, bool $subscribed = false): array
{
$imap = $this->getImapClient($account);
- $folders = $imap->listMailboxes('', '');
+ $folders = $subscribed ? $imap->listSubscribed('', '') : $imap->listMailboxes('', '');
if ($folders === false) {
throw new \Exception("Failed to list IMAP folders for {$account}");
diff --git a/src/tests/Feature/DataMigrator/IMAPTest.php b/src/tests/Feature/DataMigrator/IMAPTest.php
--- a/src/tests/Feature/DataMigrator/IMAPTest.php
+++ b/src/tests/Feature/DataMigrator/IMAPTest.php
@@ -53,14 +53,17 @@
$this->initAccount($dst);
// Add some mail to the source account
+ $utf7_folder = \mb_convert_encoding('ImapDataMigrator/&kość', 'UTF7-IMAP', 'UTF8');
$this->imapAppend($src, 'INBOX', 'mail/1.eml');
$this->imapAppend($src, 'INBOX', 'mail/2.eml', ['SEEN']);
$this->imapCreateFolder($src, 'ImapDataMigrator');
$this->imapCreateFolder($src, 'ImapDataMigrator/Test');
- $this->imapAppend($src, 'ImapDataMigrator/Test', 'mail/1.eml');
- $this->imapAppend($src, 'ImapDataMigrator/Test', 'mail/2.eml');
+ $this->imapCreateFolder($src, $utf7_folder, true);
+ $this->imapAppend($src, $utf7_folder, 'mail/1.eml', [], '10-Jan-2024 09:09:09 +0000');
+ $this->imapAppend($src, $utf7_folder, 'mail/2.eml', [], '10-Jan-2024 09:09:09 +0000');
// Clean up the destination folders structure
+ $this->imapDeleteFolder($dst, $utf7_folder);
$this->imapDeleteFolder($dst, 'ImapDataMigrator/Test');
$this->imapDeleteFolder($dst, 'ImapDataMigrator');
@@ -72,6 +75,10 @@
$dstFolders = $this->imapListFolders($dst);
$this->assertContains('ImapDataMigrator', $dstFolders);
$this->assertContains('ImapDataMigrator/Test', $dstFolders);
+ $this->assertContains($utf7_folder, $dstFolders);
+ $subscribed = $this->imapListFolders($dst, true);
+ $this->assertContains($utf7_folder, $subscribed);
+ $this->assertNotContains('ImapDataMigrator/Test', $subscribed);
// Assert the migrated messages
$dstMessages = $this->imapList($dst, 'INBOX');
@@ -83,16 +90,16 @@
$this->assertSame('<sync2@kolab.org>', $msg->messageID);
$this->assertSame(['SEEN'], array_keys($msg->flags));
- $dstMessages = $this->imapList($dst, 'ImapDataMigrator/Test');
+ $dstMessages = $this->imapList($dst, $utf7_folder);
$this->assertCount(2, $dstMessages);
$msg = array_shift($dstMessages);
$this->assertSame('<sync1@kolab.org>', $msg->messageID);
$this->assertSame([], $msg->flags);
+ $this->assertSame('10-Jan-2024 09:09:09 +0000', $msg->internaldate);
$msg = array_shift($dstMessages);
$this->assertSame('<sync2@kolab.org>', $msg->messageID);
$this->assertSame([], $msg->flags);
-
- // TODO: Test INTERNALDATE migration
+ $this->assertSame('10-Jan-2024 09:09:09 +0000', $msg->internaldate);
}
/**
@@ -142,7 +149,8 @@
$this->assertSame(['<sync3@kolab.org>','<sync4@kolab.org>'], $ids);
// Nothing changed in the other folder
- $dstMessages = $this->imapList($dst, 'ImapDataMigrator/Test');
+ $utf7_folder = \mb_convert_encoding('ImapDataMigrator/&kość', 'UTF7-IMAP', 'UTF8');
+ $dstMessages = $this->imapList($dst, $utf7_folder);
$this->assertCount(2, $dstMessages);
$msg = array_shift($dstMessages);
$this->assertSame('<sync1@kolab.org>', $msg->messageID);
diff --git a/src/tests/Feature/DataMigrator/KolabTest.php b/src/tests/Feature/DataMigrator/KolabTest.php
new file mode 100644
--- /dev/null
+++ b/src/tests/Feature/DataMigrator/KolabTest.php
@@ -0,0 +1,277 @@
+<?php
+
+namespace Tests\Feature\DataMigrator;
+
+use App\DataMigrator\Account;
+use App\DataMigrator\Driver\Kolab\Tags as KolabTags;
+use App\DataMigrator\Engine;
+use App\DataMigrator\Queue as MigratorQueue;
+use Tests\BackendsTrait;
+use Tests\TestCase;
+
+/**
+ * @group slow
+ * @group dav
+ * @group imap
+ */
+class KolabTest extends TestCase
+{
+ use BackendsTrait;
+
+ /**
+ * {@inheritDoc}
+ */
+ public function setUp(): void
+ {
+ parent::setUp();
+
+ MigratorQueue::truncate();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public function tearDown(): void
+ {
+ MigratorQueue::truncate();
+
+ parent::tearDown();
+ }
+
+ /**
+ * Test Kolab3 to Kolab4 migration
+ */
+ public function testInitialMigration3to4(): void
+ {
+ [$src, $src_imap, $src_dav, $dst, $dst_imap, $dst_dav] = $this->getTestAccounts();
+
+ $this->prepareKolab3Account($src_imap, $src_dav);
+ $this->prepareKolab4Account($dst_imap, $dst_dav);
+
+ // Run the migration
+ $migrator = new Engine();
+ $migrator->migrate($src, $dst, ['force' => true, 'sync' => true]);
+
+ // Assert the migrated mail
+ $messages = $this->imapList($dst_imap, 'INBOX');
+ $messages = \collect($messages)->keyBy('messageID')->all();
+ $this->assertCount(2, $messages);
+ $this->assertSame([], $messages['<sync1@kolab.org>']->flags);
+ $this->assertSame(['SEEN'], array_keys($messages['<sync2@kolab.org>']->flags));
+ $sync1_uid = $messages['<sync1@kolab.org>']->uid;
+ $sync2_uid = $messages['<sync2@kolab.org>']->uid;
+
+ $messages = $this->imapList($dst_imap, 'Drafts');
+ $messages = \collect($messages)->keyBy('messageID')->all();
+ $this->assertCount(1, $messages);
+ $this->assertSame(['SEEN'], array_keys($messages['<sync3@kolab.org>']->flags));
+
+ $this->assertCount(0, $this->imapList($dst_imap, 'Spam'));
+ $this->assertCount(0, $this->imapList($dst_imap, 'Trash'));
+
+ // Assert non-mail folders were not migrated
+ $mail_folders = $this->imapListFolders($dst_imap);
+ $this->assertNotContains('Configuration', $mail_folders);
+ $this->assertNotContains('Test', $mail_folders);
+
+ // Extra IMAP folder with non-ascii characters
+ $utf7_folder = \mb_convert_encoding('&kość', 'UTF7-IMAP', 'UTF8');
+ $this->assertContains($utf7_folder, $mail_folders);
+
+ // Check migration of folder subscription state
+ $subscribed = $this->imapListFolders($dst_imap, true);
+ $this->assertNotContains($utf7_folder, $subscribed);
+ $this->assertContains('Test2', $subscribed);
+
+ // Assert migrated tags
+ $imap = $this->getImapClient($dst_imap);
+ $tags = KolabTags::getKolab4Tags($imap);
+ $this->assertCount(1, $tags);
+ $this->assertSame('tag', $tags[0]['name']);
+ $this->assertSame('#E0431B', $tags[0]['color']);
+ $members = KolabTags::getKolab4TagMembers($imap, 'tag', 'INBOX');
+ $this->assertSame([(string) $sync1_uid, (string) $sync2_uid], $members);
+
+ // Assert the migrated events
+ $events = $this->davList($dst_dav, 'Calendar', Engine::TYPE_EVENT);
+ $events = \collect($events)->keyBy('uid')->all();
+ $this->assertCount(2, $events);
+ $this->assertSame('Party', $events['abcdef']->summary);
+ $this->assertSame('Meeting', $events['123456']->summary);
+
+ $events = $this->davList($dst_dav, 'Custom Calendar', Engine::TYPE_EVENT);
+ $events = \collect($events)->keyBy('uid')->all();
+ $this->assertCount(1, $events);
+ $this->assertSame('Test Summary', $events['aaa-aaa']->summary);
+
+ // Assert the migrated contacts
+ $contacts = $this->davList($dst_dav, 'Contacts', Engine::TYPE_CONTACT);
+ $contacts = \collect($contacts)->keyBy('uid')->all();
+ $this->assertCount(2, $contacts);
+ $this->assertSame('Jane Doe', $contacts['uid1']->fn);
+ $this->assertSame('Jack Strong', $contacts['uid2']->fn);
+
+ // Assert the migrated tasks
+ $tasks = $this->davList($dst_dav, 'Tasks', Engine::TYPE_TASK);
+ $tasks = \collect($tasks)->keyBy('uid')->all();
+ $this->assertCount(2, $tasks);
+ $this->assertSame('Task1', $tasks['ccc-ccc']->summary);
+ $this->assertSame('Task2', $tasks['ddd-ddd']->summary);
+ }
+
+ /**
+ * Test Kolab3 to Kolab4 incremental migration run
+ *
+ * @depends testInitialMigration3to4
+ */
+ public function testIncrementalMigration3to4(): void
+ {
+ [$src, $src_imap, $src_dav, $dst, $dst_imap, $dst_dav] = $this->getTestAccounts();
+
+ // Add/modify some source account data
+ $messages = $this->imapList($src_imap, 'INBOX');
+ $existing = \collect($messages)->keyBy('messageID')->all();
+ $this->imapFlagAs($src_imap, 'INBOX', $existing['<sync1@kolab.org>']->uid, ['SEEN', 'FLAGGED']);
+ $this->imapFlagAs($src_imap, 'INBOX', $existing['<sync2@kolab.org>']->uid, ['UNSEEN']);
+ $this->imapAppend($src_imap, 'Configuration', 'kolab3/tag2.eml');
+ $this->imapAppend($src_imap, 'Drafts', 'mail/4.eml', ['SEEN']);
+ $replace = [
+ // '/john@kolab.org/' => 'ned@kolab.org',
+ '/DTSTAMP:19970714T170000Z/' => 'DTSTAMP:20240714T170000Z',
+ '/SUMMARY:Party/' => 'SUMMARY:Test'
+ ];
+ $this->davAppend($src_dav, 'Calendar', ['event/1.ics'], Engine::TYPE_EVENT, $replace);
+
+ // Run the migration
+ $migrator = new Engine();
+ $migrator->migrate($src, $dst, ['force' => true,'sync' => true]);
+
+ // Assert the migrated mail
+ $messages = $this->imapList($dst_imap, 'INBOX');
+ $messages = \collect($messages)->keyBy('messageID')->all();
+ $this->assertCount(2, $messages);
+ $this->assertSame(['FLAGGED', 'SEEN'], array_keys($messages['<sync1@kolab.org>']->flags));
+ $this->assertSame([], $messages['<sync2@kolab.org>']->flags);
+ $sync2_uid = $messages['<sync2@kolab.org>']->uid;
+
+ $messages = $this->imapList($dst_imap, 'Drafts');
+ $messages = \collect($messages)->keyBy('messageID')->all();
+ $this->assertCount(2, $messages);
+ $this->assertSame(['SEEN'], array_keys($messages['<sync3@kolab.org>']->flags));
+ $this->assertSame(['SEEN'], array_keys($messages['<sync4@kolab.org>']->flags));
+ $sync3_uid = $messages['<sync3@kolab.org>']->uid;
+ $sync4_uid = $messages['<sync4@kolab.org>']->uid;
+
+ // Assert migrated tags
+ $imap = $this->getImapClient($dst_imap);
+ $tags = KolabTags::getKolab4Tags($imap);
+ $this->assertCount(2, $tags);
+ $this->assertSame('tag', $tags[0]['name']);
+ $this->assertSame('#E0431B', $tags[0]['color']);
+ $this->assertSame('test', $tags[1]['name']);
+ $this->assertSame('#FFFFFF', $tags[1]['color']);
+ $members = KolabTags::getKolab4TagMembers($imap, 'test', 'INBOX');
+ $this->assertSame([(string) $sync2_uid], $members);
+ $members = KolabTags::getKolab4TagMembers($imap, 'test', 'Drafts');
+ $this->assertSame([(string) $sync3_uid, (string) $sync4_uid], $members);
+
+ // Assert the migrated events
+ $events = $this->davList($dst_dav, 'Calendar', Engine::TYPE_EVENT);
+ $events = \collect($events)->keyBy('uid')->all();
+ $this->assertCount(2, $events);
+ $this->assertSame('Test', $events['abcdef']->summary);
+ }
+
+ /**
+ * Initialize accounts for tests
+ */
+ private function getTestAccounts()
+ {
+ $dav_uri = \config('services.dav.uri');
+ $dav_uri = preg_replace('|^http|', 'dav', $dav_uri);
+ $imap_uri = \config('services.imap.uri');
+ if (strpos($imap_uri, '://') === false) {
+ $imap_uri = 'imap://' . $imap_uri;
+ }
+
+ $kolab3_uri = preg_replace('|^[a-z]+://|', 'kolab3://ned%40kolab.org:simple123@', $imap_uri)
+ . '?dav_host=' . preg_replace('|^davs?://|', '', $dav_uri);
+ $kolab4_uri = preg_replace('|^[a-z]+://|', 'kolab4://jack%40kolab.org:simple123@', $imap_uri)
+ . '?dav_host=' . preg_replace('|^davs?://|', '', $dav_uri);
+
+ // Note: These are Kolab4 accounts, we'll modify the src account to imitate a Kolab3 account
+ // as much as we can. See self::prepareKolab3Account()
+
+ $src_imap = new Account(preg_replace('|://|', '://ned%40kolab.org:simple123@', $imap_uri));
+ $src_dav = new Account(preg_replace('|://|', '://ned%40kolab.org:simple123@', $dav_uri));
+ $src = new Account($kolab3_uri);
+ $dst_imap = new Account(preg_replace('|://|', '://jack%40kolab.org:simple123@', $imap_uri));
+ $dst_dav = new Account(preg_replace('|://|', '://jack%40kolab.org:simple123@', $dav_uri));
+ $dst = new Account($kolab4_uri);
+
+ return [$src, $src_imap, $src_dav, $dst, $dst_imap, $dst_dav];
+ }
+
+ /**
+ * Initial preparation of a Kolab v3 account for tests
+ */
+ private function prepareKolab3Account(Account $imap_account, Account $dav_account)
+ {
+ // Cleanup the account
+ $this->initAccount($imap_account);
+ $this->initAccount($dav_account);
+ $this->davDeleteFolder($dav_account, 'Custom Calendar', Engine::TYPE_EVENT);
+
+ $imap = $this->getImapClient($imap_account);
+
+ // Create Configuration folder with sample relation (tag) object
+ $this->imapCreateFolder($imap_account, 'Configuration');
+ $this->imapEmptyFolder($imap_account, 'Configuration');
+ if (!$imap->setMetadata('Configuration', ['/private/vendor/kolab/folder-type' => 'configuration'])) {
+ throw new \Exception("Failed to set metadata");
+ }
+ $this->imapAppend($imap_account, 'Configuration', 'kolab3/tag1.eml');
+ $this->imapAppend($imap_account, 'Configuration', 'kolab3/tag2.eml', ['DELETED']);
+
+ // Create a non-mail folder, we'll assert that it was skipped in migration
+ $this->imapCreateFolder($imap_account, 'Test');
+ if (!$imap->setMetadata('Test', ['/private/vendor/kolab/folder-type' => 'contact'])) {
+ throw new \Exception("Failed to set metadata");
+ }
+ if (!$imap->setMetadata('INBOX', ['/private/vendor/kolab/folder-type' => 'mail.inbox'])) {
+ throw new \Exception("Failed to set metadata");
+ }
+
+ // Create an IMAP folder with non-ascii characters, unsubscribed
+ $this->imapCreateFolder($imap_account, \mb_convert_encoding('&kość', 'UTF7-IMAP', 'UTF8'));
+
+ // One more IMAP folder, subscribed
+ $this->imapCreateFolder($imap_account, 'Test2', true);
+
+ // Insert some other data to migrate
+ $this->imapAppend($imap_account, 'INBOX', 'mail/1.eml');
+ $this->imapAppend($imap_account, 'INBOX', 'mail/2.eml', ['SEEN']);
+ $this->imapAppend($imap_account, 'Drafts', 'mail/3.eml', ['SEEN']);
+ $this->davAppend($dav_account, 'Calendar', ['event/1.ics', 'event/2.ics'], Engine::TYPE_EVENT);
+ $this->davCreateFolder($dav_account, 'Custom Calendar', Engine::TYPE_EVENT);
+ $this->davAppend($dav_account, 'Custom Calendar', ['event/3.ics'], Engine::TYPE_EVENT);
+ $this->davAppend($dav_account, 'Contacts', ['contact/1.vcf', 'contact/2.vcf'], Engine::TYPE_CONTACT);
+ $this->davAppend($dav_account, 'Tasks', ['task/1.ics', 'task/2.ics'], Engine::TYPE_TASK);
+ }
+
+ /**
+ * Initial preparation of a Kolab v4 account for tests
+ */
+ private function prepareKolab4Account(Account $imap_account, Account $dav_account)
+ {
+ $this->initAccount($imap_account);
+ $this->initAccount($dav_account);
+ $this->davDeleteFolder($dav_account, 'Custom Calendar', Engine::TYPE_EVENT);
+ $this->imapDeleteFolder($imap_account, 'Test');
+ $this->imapDeleteFolder($imap_account, 'Test2');
+ $this->imapDeleteFolder($imap_account, \mb_convert_encoding('&kość', 'UTF7-IMAP', 'UTF8'));
+ $this->imapDeleteFolder($imap_account, 'Configuration');
+ $imap = $this->getImapClient($imap_account);
+ KolabTags::saveKolab4Tags($imap, []);
+ }
+}
diff --git a/src/tests/data/kolab3/tag1.eml b/src/tests/data/kolab3/tag1.eml
new file mode 100644
--- /dev/null
+++ b/src/tests/data/kolab3/tag1.eml
@@ -0,0 +1,46 @@
+MIME-Version: 1.0
+From: ned@kolab.org
+To: ned@kolab.org
+Date: Fri, 10 Jan 2025 15:24:06 +0100
+X-Kolab-Type: application/x-vnd.kolab.configuration.relation
+X-Kolab-Mime-Version: 3.0
+Subject: a597dfc8-9876-4b1d-964d-f4fafc6f4481
+User-Agent: Kolab 16/Roundcube 1.5-git
+Content-Type: multipart/mixed;
+ boundary="=_9abdaa1a17f700a3819efeb6c73ee9fa"
+
+--=_9abdaa1a17f700a3819efeb6c73ee9fa
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a Kolab Groupware object. To view this object you will need an emai=
+l client that understands the Kolab Groupware format. For a list of such em=
+ail clients please visit https://www.kolab.org/
+
+
+--=_9abdaa1a17f700a3819efeb6c73ee9fa
+Content-Transfer-Encoding: 8bit
+Content-Type: application/vnd.kolab+xml; charset=UTF-8;
+ name=kolab.xml
+Content-Disposition: attachment;
+ filename=kolab.xml;
+ size=869
+
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<configuration xmlns="http://kolab.org" version="3.0">
+ <uid>a597dfc8-9876-4b1d-964d-f4fafc6f4481</uid>
+ <prodid>Roundcube-libkolab-1.1 Libkolabxml-1.3.1</prodid>
+ <creation-date>2024-10-01T07:40:45Z</creation-date>
+ <last-modification-date>2025-01-10T14:24:06Z</last-modification-date>
+ <type>relation</type>
+ <name>tag</name>
+ <relationType>tag</relationType>
+ <color>#E0431B</color>
+ <priority>-1444709376</priority>
+ <member>urn:uuid:ccc-ccc</member>
+ <member>urn:uuid:ddd-ddd</member>
+ <member>imap:///user/ned%40kolab.org/INBOX/10?message-id=%3Csync1%40kolab.org%3E&amp;date=Thu%2C+09+Aug+2012+13%3A18%3A31+%2B0000&amp;subject=test+sync</member>
+ <member>imap:///user/ned%40kolab.org/INBOX/11?message-id=%3Csync2%40kolab.org%3E&amp;date=Thu%2C+09+Aug+2012+13%3A18%3A31+%2B0000&amp;subject=test+sync+with+attachment</member>
+</configuration>
+
+--=_9abdaa1a17f700a3819efeb6c73ee9fa--
diff --git a/src/tests/data/kolab3/tag2.eml b/src/tests/data/kolab3/tag2.eml
new file mode 100644
--- /dev/null
+++ b/src/tests/data/kolab3/tag2.eml
@@ -0,0 +1,46 @@
+MIME-Version: 1.0
+From: ned@kolab.org
+To: ned@kolab.org
+Date: Fri, 10 Jan 2025 15:24:06 +0100
+X-Kolab-Type: application/x-vnd.kolab.configuration.relation
+X-Kolab-Mime-Version: 3.0
+Subject: a597dfc8-9876-4b1d-964d-f4fafc6f4482
+User-Agent: Kolab 16/Roundcube 1.5-git
+Content-Type: multipart/mixed;
+ boundary="=_9abdaa1a17f700a3819efeb6c73ee9fa"
+
+--=_9abdaa1a17f700a3819efeb6c73ee9fa
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a Kolab Groupware object. To view this object you will need an emai=
+l client that understands the Kolab Groupware format. For a list of such em=
+ail clients please visit https://www.kolab.org/
+
+
+--=_9abdaa1a17f700a3819efeb6c73ee9fa
+Content-Transfer-Encoding: 8bit
+Content-Type: application/vnd.kolab+xml; charset=UTF-8;
+ name=kolab.xml
+Content-Disposition: attachment;
+ filename=kolab.xml;
+ size=869
+
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<configuration xmlns="http://kolab.org" version="3.0">
+ <uid>a597dfc8-9876-4b1d-964d-f4fafc6f4482</uid>
+ <prodid>Roundcube-libkolab-1.1 Libkolabxml-1.3.1</prodid>
+ <creation-date>2024-10-01T07:40:45Z</creation-date>
+ <last-modification-date>2025-01-10T14:24:06Z</last-modification-date>
+ <type>relation</type>
+ <name>test</name>
+ <relationType>tag</relationType>
+ <color>#FFFFFF</color>
+ <priority>1</priority>
+ <member>urn:uuid:ccc-ccc</member>
+ <member>imap:///user/ned%40kolab.org/INBOX/11?message-id=%3Csync2%40kolab.org%3E&amp;date=Thu%2C+09+Aug+2012+13%3A18%3A31+%2B0000&amp;subject=test+sync+with+attachment</member>
+ <member>imap:///user/ned%40kolab.org/Drafts/120?message-id=%3Csync3%40kolab.org%3E&amp;date=Thu%2C+09+Aug+2012+13%3A18%3A31+%2B0000&amp;subject=test+migrator</member>
+ <member>imap:///user/ned%40kolab.org/Drafts/121?message-id=%3Csync4%40kolab.org%3E&amp;date=Thu%2C+09+Aug+2012+13%3A18%3A31+%2B0000&amp;subject=test+sync+4th</member>
+</configuration>
+
+--=_9abdaa1a17f700a3819efeb6c73ee9fa--
diff --git a/src/tests/data/task/1.ics b/src/tests/data/task/1.ics
new file mode 100644
--- /dev/null
+++ b/src/tests/data/task/1.ics
@@ -0,0 +1,14 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Roundcube 1.5-git//Sabre VObject 4.5.5//EN
+CALSCALE:GREGORIAN
+BEGIN:VTODO
+UID:ccc-ccc
+DTSTAMP:20241218T101054Z
+CREATED:20241218T101054Z
+LAST-MODIFIED:20241218T101054Z
+SUMMARY:Task1
+SEQUENCE:0
+ORGANIZER;CN=Test:mailto:ned@kolab.org
+END:VTODO
+END:VCALENDAR
diff --git a/src/tests/data/task/2.ics b/src/tests/data/task/2.ics
new file mode 100644
--- /dev/null
+++ b/src/tests/data/task/2.ics
@@ -0,0 +1,14 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Roundcube 1.5-git//Sabre VObject 4.5.5//EN
+CALSCALE:GREGORIAN
+BEGIN:VTODO
+UID:ddd-ddd
+DTSTAMP:20241218T101054Z
+CREATED:20241218T101054Z
+LAST-MODIFIED:20241218T101054Z
+SUMMARY:Task2
+SEQUENCE:0
+ORGANIZER;CN=Test:mailto:ned@kolab.org
+END:VTODO
+END:VCALENDAR

File Metadata

Mime Type
text/plain
Expires
Sat, Apr 4, 6:43 AM (7 h, 8 m ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18828408
Default Alt Text
D5088.1775284995.diff (45 KB)

Event Timeline