Page MenuHomePhorge
Files F117820270

D5076.1775294728.diff
No OneTemporary
Actions

Authored By
Unknown
Size
36 KB
Referenced Files
None
Subscribers
None

D5076.1775294728.diff
View Options

diff --git a/src/app/Backends/DAV.php b/src/app/Backends/DAV.php
--- a/src/app/Backends/DAV.php
+++ b/src/app/Backends/DAV.php
@@ -608,7 +608,7 @@
protected function request($path, $method, $body = '', $headers = [])
{
$debug = \config('app.debug');
- $url = trim($this->url, '/');
+ $url = $this->url;
$this->responseHeaders = [];
diff --git a/src/app/Backends/DAV/Vcard.php b/src/app/Backends/DAV/Vcard.php
--- a/src/app/Backends/DAV/Vcard.php
+++ b/src/app/Backends/DAV/Vcard.php
@@ -18,6 +18,7 @@
public $kind;
public $note;
public $member = [];
+ public $photo;
public $prodid;
public $rev;
public $version;
@@ -65,6 +66,7 @@
'FN',
'KIND',
'NOTE',
+ 'PHOTO',
'PRODID',
'REV',
'UID',
diff --git a/src/app/Console/Commands/Data/MigrateCommand.php b/src/app/Console/Commands/Data/MigrateCommand.php
--- a/src/app/Console/Commands/Data/MigrateCommand.php
+++ b/src/app/Console/Commands/Data/MigrateCommand.php
@@ -13,10 +13,10 @@
* ```
* php artisan data:migrate \
* "ews://$user@$server?client_id=$client_id&client_secret=$client_secret&tenant_id=$tenant_id" \
- * "dav://$dest_user:$dest_pass@$dest_server"
+ * "kolab://$dest_user:$dest_pass@$dest_server"
* ```
*
- * Supported account types: ews, dav (davs), imap (tls, ssl, imaps)
+ * For supported migration driver names look into DataMigrator\Engine::initDriver()
*/
class MigrateCommand extends Command
{
diff --git a/src/app/DataMigrator/Account.php b/src/app/DataMigrator/Account.php
--- a/src/app/DataMigrator/Account.php
+++ b/src/app/DataMigrator/Account.php
@@ -48,6 +48,8 @@
*/
public function __construct(string $input)
{
+ $this->input = $input;
+
if (!preg_match('|^[a-z]+://.*|', $input)) {
throw new \Exception("Invalid URI specified");
}
@@ -56,6 +58,11 @@
// Not valid URI
if (!is_array($url) || empty($url)) {
+ if (preg_match('|^[a-z]+:///.*|', $input)) {
+ $this->parseFileUri($input);
+ return;
+ }
+
throw new \Exception("Invalid URI specified");
}
@@ -95,8 +102,6 @@
} elseif (strpos($this->username, '@')) {
$this->email = $this->username;
}
-
- $this->input = $input;
}
/**
@@ -109,4 +114,37 @@
{
return $this->input;
}
+
+ /**
+ * Parse file URI
+ */
+ protected function parseFileUri($input)
+ {
+ if (!preg_match('|^[a-z]+://(/[^?]+)|', $input, $matches)) {
+ throw new \Exception("Invalid URI specified");
+ }
+
+ // Replace file+path with a fake host name so the URI can be parsed
+ $input = str_replace($matches[1], 'fake.host', $input);
+ $url = parse_url($input);
+
+ // Not valid URI
+ if (!is_array($url) || empty($url)) {
+ throw new \Exception("Invalid URI specified");
+ }
+
+ $this->uri = $matches[1];
+
+ if (isset($url['scheme'])) {
+ $this->scheme = strtolower($url['scheme']);
+ }
+
+ if (!empty($url['query'])) {
+ parse_str($url['query'], $this->params);
+ }
+
+ if (!empty($this->params['user']) && strpos($this->params['user'], '@')) {
+ $this->email = $this->params['user'];
+ }
+ }
}
diff --git a/src/app/DataMigrator/Driver/IMAP.php b/src/app/DataMigrator/Driver/IMAP.php
--- a/src/app/DataMigrator/Driver/IMAP.php
+++ b/src/app/DataMigrator/Driver/IMAP.php
@@ -149,6 +149,8 @@
$this->imap->expunge($mailbox, $item->existing['uid']);
} else {
// Update flags
+ // TODO: Here we may produce more STORE commands than necessary, we could improve this
+ // if we make flag()/unflag() methods to accept multiple flags
foreach ($item->existing['flags'] as $flag) {
if (!in_array($flag, $item->data['flags'])) {
$this->imap->unflag($mailbox, $item->existing['uid'], $flag);
@@ -188,8 +190,8 @@
// So, we compare size and INTERNALDATE timestamp.
if (
!$item->existing
- || $header->timestamp != $item->existing['timestamp']
|| $header->size != $item->existing['size']
+ || \rcube_utils::strtotime($header->internaldate) != \rcube_utils::strtotime($item->existing['date'])
) {
// Handle message content in memory (up to 20MB), bigger messages will use a temp file
if ($header->size > Engine::MAX_ITEM_SIZE) {
@@ -261,18 +263,15 @@
$id = $this->getMessageId($message, $mailbox);
// Skip message that exists and did not change
- $exists = null;
- if (isset($existing[$id])) {
- $flags = $this->filterImapFlags(array_keys($message->flags));
+ $exists = $existing[$id] ?? null;
+ if ($exists) {
if (
- $flags == $existing[$id]['flags']
- && $message->timestamp == $existing[$id]['timestamp']
- && $message->size == $existing[$id]['size']
+ $message->size == $exists['size']
+ && \rcube_utils::strtotime($message->internaldate) == \rcube_utils::strtotime($exists['date'])
+ && $this->filterImapFlags(array_keys($message->flags)) == $exists['flags']
) {
continue;
}
-
- $exists = $existing[$id];
}
$set->items[] = Item::fromArray([
@@ -358,7 +357,7 @@
'uid' => $message->uid,
'flags' => $flags,
'size' => $message->size,
- 'timestamp' => $message->timestamp,
+ 'date' => $message->internaldate,
];
}
@@ -476,6 +475,6 @@
return $message->messageID;
}
- return md5($folder . $message->from . ($message->date ?: $message->timestamp));
+ return md5($folder . $message->from . $message->timestamp);
}
}
diff --git a/src/app/DataMigrator/Driver/Kolab.php b/src/app/DataMigrator/Driver/Kolab.php
--- a/src/app/DataMigrator/Driver/Kolab.php
+++ b/src/app/DataMigrator/Driver/Kolab.php
@@ -32,8 +32,7 @@
{
// TODO: Right now we require IMAP server and DAV host names, but for Kolab we should be able
// to detect IMAP and DAV locations, e.g. so we can just provide "kolabnow.com" as an input
- // Note: E.g. KolabNow uses different hostname for DAV, pass it as a query parameter
- // 'dav_host' and 'dav_path'.
+ // Note: E.g. KolabNow uses different hostname for DAV, pass it as a query parameter 'dav_host'.
// Setup IMAP connection
$uri = (string) $account;
@@ -43,11 +42,10 @@
// Setup DAV connection
$uri = sprintf(
- 'davs://%s:%s@%s/%s',
+ 'davs://%s:%s@%s',
urlencode($account->username),
urlencode($account->password),
$account->params['dav_host'] ?? $account->host,
- $account->params['dav_path'] ?? '', // e.g. 'dav'
);
$this->davDriver = new DAV(new Account($uri), $engine);
diff --git a/src/app/DataMigrator/Driver/Takeout.php b/src/app/DataMigrator/Driver/Takeout.php
new file mode 100644
--- /dev/null
+++ b/src/app/DataMigrator/Driver/Takeout.php
@@ -0,0 +1,520 @@
+<?php
+
+namespace App\DataMigrator\Driver;
+
+use App\DataMigrator\Account;
+use App\DataMigrator\Engine;
+use App\DataMigrator\Interface\ExporterInterface;
+use App\DataMigrator\Interface\ImporterInterface;
+use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
+use App\DataMigrator\Interface\ItemSet;
+
+/**
+ * Data migration from Google Takeout archive file
+ */
+class Takeout implements ExporterInterface
+{
+ /** @var Account Account (local file) to operate on */
+ protected $account;
+
+ /** @var Engine Data migrator engine */
+ protected $engine;
+
+ /** @var string Local folder with folders/files (extracted from the Takeout archive) */
+ protected $location;
+
+
+ /**
+ * Object constructor
+ */
+ public function __construct(Account $account, Engine $engine)
+ {
+ $this->account = $account;
+ $this->engine = $engine;
+
+ if (!file_exists($account->uri)) {
+ throw new \Exception("File does not exists: {$account->uri}");
+ }
+ }
+
+ /**
+ * Authenticate
+ */
+ public function authenticate(): void
+ {
+ // NOP
+ }
+
+ /**
+ * Get folders hierarchy
+ */
+ public function getFolders($types = []): array
+ {
+ $this->extractArchive();
+
+ $folders = [];
+
+ // Mail folders
+ if (empty($types) || in_array(Engine::TYPE_MAIL, $types)) {
+ // GMail has no custom folders, it has labels and categories (we could import them too, as tags)
+ // All mail is exported into a single mbox file.
+ if (file_exists("{$this->location}/Mail/All mail Including Spam and Trash.mbox")) {
+ foreach (['INBOX', 'Sent', 'Drafts', 'Spam', 'Trash'] as $folder) {
+ $folders[] = Folder::fromArray([
+ 'fullname' => $folder,
+ 'type' => Engine::TYPE_MAIL,
+ ]);
+ }
+ }
+ }
+
+ // Contacts folder
+ if (empty($types) || in_array(Engine::TYPE_CONTACT, $types)) {
+ if (@filesize("{$this->location}/Contacts/My Contacts/My Contacts.vcf")) {
+ $folders[] = Folder::fromArray([
+ 'fullname' => 'Contacts',
+ 'id' => "{$this->location}/Contacts/My Contacts",
+ 'type' => Engine::TYPE_CONTACT,
+ ]);
+ }
+ }
+
+ // Calendars
+ if (empty($types) || in_array(Engine::TYPE_EVENT, $types)) {
+ if (file_exists("{$this->location}/Calendar")) {
+ foreach (glob("{$this->location}/Calendar/*.ics") as $filename) {
+ $folder = preg_replace('/\.ics$/', '', pathinfo($filename, \PATHINFO_BASENAME));
+ $folders[] = Folder::fromArray([
+ // Note: The default calendar is exported into <email>.ics file
+ 'fullname' => $folder == $this->account->email ? 'Calendar' : $folder,
+ 'id' => $filename,
+ 'type' => Engine::TYPE_EVENT,
+ ]);
+ }
+ }
+ }
+
+ // TODO: Tasks
+ // TODO: Files
+
+ return $folders;
+ }
+
+ /**
+ * Fetch a list of folder items
+ */
+ public function fetchItemList(Folder $folder, $callback, ImporterInterface $importer): void
+ {
+ $this->extractArchive();
+
+ // Get existing objects from the destination folder
+ $existing = $importer->getItems($folder);
+
+ // Mail
+ if ($folder->type == Engine::TYPE_MAIL) {
+ if (!file_exists("{$this->location}/Mail/All mail Including Spam and Trash.mbox")) {
+ return;
+ }
+
+ // Read mbox file line by line
+ $fp = fopen("{$this->location}/Mail/All mail Including Spam and Trash.mbox", 'r');
+ $msg = '';
+ $headline = '';
+
+ while (($line = fgets($fp)) !== false) {
+ if (str_starts_with($line, 'From ') && preg_match('/^From [^\s]+ [a-zA-Z]{3} [a-zA-Z]{3}/', $line)) {
+ $this->mailItemHandler($folder, $headline, $msg, $existing, $callback);
+ $msg = '';
+ $headline = $line;
+ continue;
+ }
+
+ // TODO: Probably stream_get_contents() once per message would be faster than concatenating lines
+ $msg .= $line;
+ }
+
+ fclose($fp);
+ $this->mailItemHandler($folder, $headline, $msg, $existing, $callback);
+ return;
+ }
+
+ // Calendar(s)
+ if ($folder->type == Engine::TYPE_EVENT) {
+ $foldername = $folder->fullname == 'Calendar' ? $this->account->email : $folder->fullname;
+ if (!file_exists("{$this->location}/Calendar/{$foldername}.ics")) {
+ return;
+ }
+
+ // Read iCalendar file line by line
+ // Note: We assume that event exceptions are always in order after the master event
+ $fp = fopen("{$this->location}/Calendar/{$foldername}.ics", 'r');
+ $event = '';
+ $previous = '';
+ $head = '';
+ $got_head = false;
+
+ while (($line = fgets($fp)) !== false) {
+ if (str_starts_with($line, 'BEGIN:VEVENT')) {
+ $got_head = true;
+ if (strlen($event)) {
+ if (strpos($event, "\nRECURRENCE-ID")) {
+ $previous .= $event;
+ } else {
+ if (strlen($previous)) {
+ $_event = $head . $previous . "END:VCALENDAR\r\n";
+ $this->eventItemHandler($folder, $_event, $existing, $callback);
+ }
+ $previous = $event;
+ }
+ $event = '';
+ }
+ } elseif (!$got_head) {
+ $head .= $line;
+ continue;
+ }
+
+ // TODO: Probably stream_get_contents() once per event would be faster than concatenating lines
+ if (!str_starts_with($line, 'END:VCALENDAR')) {
+ $event .= $line;
+ }
+ }
+
+ fclose($fp);
+
+ if (strlen($event)) {
+ if (strpos($event, "\nRECURRENCE-ID")) {
+ $previous .= $event;
+ } else {
+ $this->eventItemHandler($folder, $head . $event . "END:VCALENDAR\r\n", $existing, $callback);
+ }
+ }
+ if (strlen($previous)) {
+ $this->eventItemHandler($folder, $head . $previous . "END:VCALENDAR\r\n", $existing, $callback);
+ }
+
+ return;
+ }
+
+ // Contacts
+ if ($folder->type == Engine::TYPE_CONTACT) {
+ if (!file_exists("{$this->location}/Contacts/My Contacts/My Contacts.vcf")) {
+ return;
+ }
+
+ // Read vCard file line by line
+ $fp = fopen("{$this->location}/Contacts/My Contacts/My Contacts.vcf", 'r');
+ $vcard = '';
+
+ while (($line = fgets($fp)) !== false) {
+ if (str_starts_with($line, 'END:VCARD')) {
+ $this->contactItemHandler($folder, $vcard . $line, $existing, $callback);
+ $vcard = '';
+ continue;
+ }
+
+ // TODO: Probably stream_get_contents() once per event would be faster than concatenating lines
+ $vcard .= $line;
+ }
+
+ // TODO: Takeout does not include vCards for groups (labels), but we could consider
+ // creating them for all CATEGORIES mentioned in all contacts.
+
+ fclose($fp);
+ return;
+ }
+
+ // TODO: Tasks (JSON format, is there any spec?)
+ // TODO: Files
+ // TODO: Filters (JSON format, is there any spec?)
+ }
+
+ /**
+ * Fetching an item
+ */
+ public function fetchItem(Item $item): void
+ {
+ // Do nothing, we do all data processing in fetchItemList()
+ }
+
+ /**
+ * Extract the ZIP archive into temp storage location
+ */
+ protected function extractArchive(): void
+ {
+ if ($this->location) {
+ return;
+ }
+
+ // Use the same location as in the DataMigrator Engine
+ $location = storage_path('export/') . $this->account->email;
+
+ // TODO: Skip this if the file was already extracted, e.g. in async mode
+
+ $this->engine->debug("Extracting ZIP archive...");
+
+ $zip = new \ZipArchive();
+ if (!$zip->open($this->account->uri)) {
+ throw new \Exception("Failed to extract Takeout archive file. " . $zip->getStatusString());
+ }
+
+ // This will create storage/export/<email>/Takeout folder with
+ // Calendar, Contacts, Drive, Mail folders
+ $zip->extractTo($location);
+ $zip->close();
+
+ $this->location = "{$location}/Takeout";
+ }
+
+ /**
+ * Handle vCard item
+ */
+ protected function contactItemHandler(Folder $folder, string $vcard, array $existing, $callback): void
+ {
+ // Let's try to handle the content without an expensive parser
+
+ $inject = [];
+ $fn = null;
+ if (preg_match('/\nFN:([^\r\n]+)/', $vcard, $matches)) {
+ $fn = $matches[1];
+ }
+
+ if ($fn === null) {
+ return;
+ }
+
+ // It looks like Takeout's contacts do not include UID nor REV properties, we'll add these
+ // as they are vital for incremental migration
+ if (preg_match('/\nUID:([^\r\n]+)/', $vcard, $matches)) {
+ $uid = $matches[1];
+ } else {
+ // FIXME: FN might not be unique enough probably?
+ $uid = md5($fn);
+ $inject[] = "UID:{$uid}";
+ }
+
+ if (preg_match('/\nREV:([^\r\n]+)/', $vcard, $matches)) {
+ $rev = $matches[1];
+ } else {
+ $rev = date('Y-m-d', crc32($vcard));
+ $inject[] = "REV:{$rev}";
+ }
+
+ // Skip message that exists and did not change
+ $exists = $existing[$uid] ?? null;
+ if ($exists) {
+ if ($exists['rev'] === $rev) {
+ return;
+ }
+ }
+
+ if (!empty($inject)) {
+ $vcard = str_replace("\r\nEND:VCARD", "\r\n" . implode("\r\n", $inject) . "\r\nEND:VCARD", $vcard);
+ }
+
+ // Replace PHOTO url with the file content if included outside the .vcf file
+ if ($pos = strpos($vcard, "\nPHOTO:https:")) {
+ // FIXME: Are these only jpegs?
+ // Note: If there's two contacts with the same FN there will be two images:
+ // "First Last.jpg" and "First Last(1).jpg", in random order. We ignore all of them.
+ $photo = "{$this->location}/Contacts/My Contacts/{$fn}.jpg";
+ if (file_exists($photo) && !file_exists("{$this->location}/Contacts/My Contacts/{$fn}(1).jpg")) {
+ $content = file_get_contents($photo);
+ $content = rtrim(chunk_split(base64_encode($content), 76, "\r\n "));
+
+ $endpos = strpos($vcard, "\r\n", $pos);
+ $vcard = substr_replace($vcard, "PHOTO:{$content}", $pos + 1, $endpos - $pos);
+ }
+ }
+
+ $item = Item::fromArray([
+ 'id' => $uid,
+ 'folder' => $folder,
+ 'existing' => $exists ? $exists['href'] : null,
+ ]);
+
+ $this->storeItemContent($item, $vcard, $uid . '.vcf');
+
+ $callback($item);
+ }
+
+ /**
+ * Handle VEVENT item
+ */
+ protected function eventItemHandler(Folder $folder, string $event, array $existing, $callback): void
+ {
+ // Let's try to handle the content without an expensive parser
+
+ if (!preg_match('/\nUID:([^\r\n]+)/', $event, $matches)) {
+ return;
+ }
+
+ $uid = $matches[1];
+ $dtstamp = null;
+
+ if (preg_match('/\nDTSTAMP:([^\r\n]+)/', $event, $matches)) {
+ $dtstamp = $matches[1];
+ }
+
+ // Skip message that exists and did not change
+ $exists = $existing[$uid] ?? null;
+ if ($exists) {
+ if ($exists['dtstamp'] === $dtstamp) {
+ return;
+ }
+ }
+
+ // Takeout can include events without ORGANIZER, but Cyrus requires it (also in events with RECURRENCE-ID)
+ // TODO: Replace existing organizer/attendee email with destination email?
+ if (!strpos($event, "\nORGANIZER")) {
+ $organizer = 'mailto:' . $this->account->email;
+ $event = str_replace("\r\nEND:VEVENT", "\r\nORGANIZER:{$organizer}\r\nEND:VEVENT", $event);
+ }
+
+ // TODO: Takeout's VCALENDAR files lack VTIMEZONE block, we should use X-WR-TIMEZONE property
+ // to fill timezone into the event.
+
+ $item = Item::fromArray([
+ 'id' => $uid,
+ 'folder' => $folder,
+ 'existing' => $exists ? $exists['href'] : null,
+ ]);
+
+ $this->storeItemContent($item, $event, $uid . '.ics');
+
+ $callback($item);
+ }
+
+ /**
+ * Handle mail message
+ */
+ protected function mailItemHandler(Folder $folder, string $headline, string $msg, array $existing, $callback): void
+ {
+ if (!($length = strlen($msg))) {
+ return;
+ }
+
+ $pos = strpos($msg, "\r\n\r\n");
+ $head = $pos ? substr($msg, 0, $pos) : $msg;
+
+ [$foldername, $date, $id, $flags] = self::parseMailHead($head, $headline);
+
+ if ($folder->fullname !== $foldername) {
+ return;
+ }
+ // Skip message that exists and did not change
+ $exists = $existing[$id] ?? null;
+ $changed = true;
+ if ($exists) {
+ $changed = $length != $exists['size']
+ || \rcube_utils::strtotime($date) != \rcube_utils::strtotime($exists['date']);
+
+ if (!$changed && $flags == array_values(array_intersect($exists['flags'], ['SEEN', 'FLAGGED']))) {
+ return;
+ }
+ }
+
+ $item = Item::fromArray([
+ 'id' => $id,
+ 'folder' => $folder,
+ 'existing' => $exists,
+ 'data' => [
+ 'flags' => $flags,
+ 'internaldate' => $date,
+ ],
+ ]);
+
+ if ($changed) {
+ // We need mail content for new or changed messages
+ $this->storeItemContent($item, $msg, $id . '.eml');
+ }
+
+ $callback($item);
+ }
+
+ /**
+ * Extract some details from mail message headers
+ *
+ * @param string $head Mail headers
+ * @param string $headline MBOX format header (separator) line
+ */
+ protected static function parseMailHead($head, $headline): array
+ {
+ $from = explode(' ', $headline, 3);
+
+ $date = isset($from[2]) ? trim($from[2]) : null;
+ $folder = 'INBOX';
+ $flags = [];
+ $id = $from[1] ?? '';
+
+ // Get folder and message state/flags
+ if (preg_match('/\nX-Gmail-Labels:([^\r\n]+)/', $head, $matches)) {
+ $labels = explode(',', $matches[1]);
+ $labels = array_map('trim', $labels);
+
+ if (in_array('Trash', $labels) || in_array('[Imap]/Trash', $labels)) {
+ $folder = 'Trash';
+ } elseif (in_array('Drafts', $labels) || in_array('[Imap]/Drafts', $labels)) {
+ $folder = 'Drafts';
+ } elseif (in_array('Sent', $labels) || in_array('[Imap]/Sent', $labels)) {
+ $folder = 'Sent';
+ }
+
+ // Note: It doesn't look like Google supports ANSWERED, FORWARDED and MDNSENT state
+ if (!in_array('Unread', $labels)) {
+ $flags[] = 'SEEN';
+ }
+ if (in_array('Starred', $labels)) {
+ $flags[] = 'FLAGGED';
+ }
+ }
+
+ if (preg_match('/\nMessage-Id:([^\r\n]+)/i', $head, $matches)) {
+ // substr() for sanity and compatibility with the IMAP driver
+ $id = substr(trim($matches[1]), 0, 2048);
+ }
+
+ // Convert date into IMAP format, Takeout uses custom format "Sat Jan 3 01:05:34 +0200 1996"
+ try {
+ $dt = new \DateTime($date);
+ $date = sprintf('%2s', $dt->format('j')) . $dt->format('-M-Y H:i:s O');
+ } catch (\Exception $e) {
+ \Log::warning("Failed to convert date format for: {$date}");
+ $date = null;
+ }
+
+ // If Message-ID header does not exist we need to make $id compatible with
+ // the one generated by the IMAP driver (for incremental migration comparisons)
+ if (empty($id)) {
+ if (preg_match('/\nFrom:([^\r\n]+)/i', $head, $matches)) {
+ $from_header = $matches[1];
+ }
+ if (preg_match('/\nDate:([^\r\n]+)/i', $head, $matches)) {
+ $date_header = $matches[1];
+ }
+
+ $id = md5($folder . ($from_header ?? '') . \rcube_utils::strtotime($date_header ?? $date));
+ }
+
+ return [$folder, $date, $id, $flags];
+ }
+
+ /**
+ * Store item content in a temp file so it can be used by an importer
+ */
+ protected function storeItemContent($item, $content, $filename)
+ {
+ if (strlen($content) > Engine::MAX_ITEM_SIZE) {
+ $location = $item->folder->tempFileLocation($filename);
+
+ if (file_put_contents($location, $content) === false) {
+ throw new \Exception("Failed to write to file at {$location}");
+ }
+
+ $item->filename = $location;
+ } else {
+ $item->content = $content;
+ $item->filename = $filename;
+ }
+ }
+}
diff --git a/src/app/DataMigrator/Engine.php b/src/app/DataMigrator/Engine.php
--- a/src/app/DataMigrator/Engine.php
+++ b/src/app/DataMigrator/Engine.php
@@ -385,6 +385,10 @@
$driver = new Driver\Kolab($account, $this);
break;
+ case 'takeout':
+ $driver = new Driver\Takeout($account, $this);
+ break;
+
case 'test':
$driver = new Driver\Test($account, $this);
break;
diff --git a/src/app/DataMigrator/Interface/Item.php b/src/app/DataMigrator/Interface/Item.php
--- a/src/app/DataMigrator/Interface/Item.php
+++ b/src/app/DataMigrator/Interface/Item.php
@@ -19,6 +19,7 @@
/**
* Identifier/Location of the item if exists in the destination folder.
* And/or some metadata on the existing item. This information is driver specific.
+ * @TODO: Unify this to be always an array (or object) for easier cross-driver interop.
*
* @var string|array|null
*/
diff --git a/src/tests/BackendsTrait.php b/src/tests/BackendsTrait.php
--- a/src/tests/BackendsTrait.php
+++ b/src/tests/BackendsTrait.php
@@ -22,7 +22,7 @@
/**
* Append an DAV object to a DAV folder
*/
- protected function davAppend(Account $account, $foldername, $filenames, $type): void
+ protected function davAppend(Account $account, $foldername, $filenames, $type, $replace = []): void
{
$dav = $this->getDavClient($account);
@@ -40,7 +40,12 @@
}
$content = file_get_contents($path);
- $uid = preg_match('/\nUID:(?:urn:uuid:)?([a-z0-9-]+)/', $content, $m) ? $m[1] : null;
+
+ foreach ($replace as $from => $to) {
+ $content = preg_replace($from, $to, $content);
+ }
+
+ $uid = preg_match('/\nUID:(?:urn:uuid:)?(\S+)/', $content, $m) ? $m[1] : null;
if (empty($uid)) {
throw new \Exception("Filed to find UID in {$path}");
diff --git a/src/tests/Feature/DataMigrator/TakeoutTest.php b/src/tests/Feature/DataMigrator/TakeoutTest.php
new file mode 100644
--- /dev/null
+++ b/src/tests/Feature/DataMigrator/TakeoutTest.php
@@ -0,0 +1,197 @@
+<?php
+
+namespace Tests\Feature\DataMigrator;
+
+use App\DataMigrator\Account;
+use App\DataMigrator\Engine;
+use App\DataMigrator\Queue as MigratorQueue;
+use Tests\BackendsTrait;
+use Tests\TestCase;
+
+/**
+ * @group slow
+ * @group dav
+ * @group imap
+ */
+class TakeoutTest extends TestCase
+{
+ use BackendsTrait;
+
+ /**
+ * {@inheritDoc}
+ */
+ public function setUp(): void
+ {
+ parent::setUp();
+
+ MigratorQueue::truncate();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public function tearDown(): void
+ {
+ MigratorQueue::truncate();
+
+ exec('rm -rf ' . storage_path('export/test@gmail.com'));
+
+ parent::tearDown();
+ }
+
+ /**
+ * Test Google Takeout to Kolab migration
+ */
+ public function testInitialMigration(): void
+ {
+ [$imap, $dav, $src, $dst] = $this->getTestAccounts();
+
+ // Cleanup the Kolab account
+ $this->initAccount($imap);
+ $this->initAccount($dav);
+ $this->davDeleteFolder($dav, 'Custom Calendar', Engine::TYPE_EVENT);
+
+ // Run the migration
+ $migrator = new Engine();
+ $migrator->migrate($src, $dst, ['force' => true, 'sync' => true]);
+
+ // Assert the migrated mail
+ $messages = $this->imapList($imap, 'INBOX');
+ $this->assertCount(3, $messages);
+ $msg = array_shift($messages);
+ $this->assertSame('<1@google.com>', $msg->messageID);
+ $this->assertSame([], $msg->flags);
+ // Note: Cyrus returns INTERNALDATE in UTC even though we used different TZ in APPEND
+ $this->assertSame(' 8-Jun-2024 13:37:46 +0000', $msg->internaldate);
+ $msg = array_shift($messages);
+ $this->assertSame('<3@google.com>', $msg->messageID);
+ $this->assertSame(['FLAGGED'], array_keys($msg->flags));
+ $this->assertSame('30-Jun-2022 17:45:58 +0000', $msg->internaldate);
+ $msg = array_shift($messages);
+ $this->assertSame('<5@google.com>', $msg->messageID);
+ $this->assertSame(['SEEN'], array_keys($msg->flags));
+ $this->assertSame('30-Jun-2022 19:45:58 +0000', $msg->internaldate);
+
+ $messages = $this->imapList($imap, 'Sent');
+ $this->assertCount(1, $messages);
+ $msg = array_shift($messages);
+ $this->assertSame('<4@google.com>', $msg->messageID);
+ $this->assertSame(['SEEN'], array_keys($msg->flags));
+
+ $messages = $this->imapList($imap, 'Drafts');
+ $this->assertCount(2, $messages);
+ $msg = array_shift($messages);
+ $this->assertSame('<6@google.com>', $msg->messageID);
+ $this->assertSame(['SEEN'], array_keys($msg->flags));
+ $msg = array_shift($messages);
+ $this->assertSame('<7@google.com>', $msg->messageID);
+ $this->assertSame(['SEEN'], array_keys($msg->flags));
+
+ $messages = $this->imapList($imap, 'Spam');
+ $this->assertCount(0, $messages);
+
+ $messages = $this->imapList($imap, 'Trash');
+ $this->assertCount(1, $messages);
+ $msg = array_shift($messages);
+ $this->assertSame('<2@google.com>', $msg->messageID);
+ $this->assertSame(['SEEN'], array_keys($msg->flags));
+
+ // Assert the migrated events
+ $events = $this->davList($dav, 'Calendar', Engine::TYPE_EVENT);
+ $events = \collect($events)->keyBy('uid')->all();
+ $this->assertCount(4, $events);
+ $this->assertSame('testt', $events['m3gkk34n6t7spu6b8li4hvraug@google.com']->summary);
+ $this->assertSame('TestX', $events['mpm0q3ki8plp8d7s3uagag989k@google.com']->summary);
+ $this->assertSame('ssss', $events['44gvk1rth37n1qk8nsml26o7og@google.com']->summary);
+ $this->assertSame('recur', $events['0o2o2fnfdajjsnt2dnt50vckpf@google.com']->summary);
+ $this->assertCount(1, $events['0o2o2fnfdajjsnt2dnt50vckpf@google.com']->exceptions);
+
+ $events = $this->davList($dav, 'Custom Calendar', Engine::TYPE_EVENT);
+ $events = \collect($events)->keyBy('uid')->all();
+ $this->assertCount(1, $events);
+ $this->assertSame('TestY', $events['ps1tmklc3gvao@google.com']->summary);
+
+ // Assert the migrated contacts
+ // Note: Contacts do not have UID in Takeout so it's generated
+ $contacts = $this->davList($dav, 'Contacts', Engine::TYPE_CONTACT);
+ $contacts = \collect($contacts)->keyBy('fn')->all();
+ $this->assertCount(2, $contacts);
+ $this->assertSame('test note', $contacts['Test']->note);
+ $this->assertSame('erwe note', $contacts['Nameof']->note);
+ $this->assertTrue(strlen($contacts['Nameof']->photo) == 5630);
+ }
+
+ /**
+ * Test Google Takeout to Kolab incremental migration run
+ *
+ * @depends testInitialMigration
+ */
+ public function testIncrementalMigration(): void
+ {
+ [$imap, $dav, $src, $dst] = $this->getTestAccounts();
+
+ // We modify a some mail messages and single event to make sure they get updated/overwritten by migration
+ $messages = $this->imapList($imap, 'INBOX');
+ $existing = \collect($messages)->keyBy('messageID')->all();
+ $this->imapFlagAs($imap, 'INBOX', $existing['<1@google.com>']->uid, ['SEEN', 'FLAGGED']);
+ $this->imapFlagAs($imap, 'INBOX', $existing['<5@google.com>']->uid, ['UNSEEN']);
+ $this->imapEmptyFolder($imap, 'Sent');
+ $this->davEmptyFolder($dav, 'Custom Calendar', Engine::TYPE_EVENT);
+ $replace = [
+ '/UID:aaa-aaa/' => 'UID:44gvk1rth37n1qk8nsml26o7og@google.com',
+ '/john@kolab.org/' => 'test@gmail.com'
+ ];
+ $this->davAppend($dav, 'Calendar', ['event/3.ics'], Engine::TYPE_EVENT, $replace);
+
+ // Run the migration
+ $migrator = new Engine();
+ $migrator->migrate($src, $dst, ['force' => true,'sync' => true]);
+
+ // Assert the migrated mail
+ $messages = $this->imapList($imap, 'INBOX');
+ $messages = \collect($messages)->keyBy('messageID')->all();
+ $this->assertCount(3, $messages);
+ $this->assertSame([], $messages['<1@google.com>']->flags);
+ $this->assertSame(['FLAGGED'], array_keys($messages['<3@google.com>']->flags));
+ $this->assertSame(['SEEN'], array_keys($messages['<5@google.com>']->flags));
+ $messages = $this->imapList($imap, 'Sent');
+ $this->assertCount(1, $messages);
+ $msg = array_shift($messages);
+ $this->assertSame('<4@google.com>', $msg->messageID);
+
+ // Assert the migrated events
+ $events = $this->davList($dav, 'Calendar', Engine::TYPE_EVENT);
+ $events = \collect($events)->keyBy('uid')->all();
+ $this->assertCount(4, $events);
+ $this->assertSame('ssss', $events['44gvk1rth37n1qk8nsml26o7og@google.com']->summary);
+
+ $events = $this->davList($dav, 'Custom Calendar', Engine::TYPE_EVENT);
+ $events = \collect($events)->keyBy('uid')->all();
+ $this->assertCount(1, $events);
+ $this->assertSame('TestY', $events['ps1tmklc3gvao@google.com']->summary);
+ }
+
+ /**
+ * Initialize accounts for tests
+ */
+ private function getTestAccounts()
+ {
+ $dav_uri = \config('services.dav.uri');
+ $dav_uri = preg_replace('|^http|', 'dav', $dav_uri);
+ $imap_uri = \config('services.imap.uri');
+ if (strpos($imap_uri, '://') === false) {
+ $imap_uri = 'imap://' . $imap_uri;
+ }
+
+ $takeout_uri = 'takeout://' . self::BASE_DIR . '/data/takeout.zip?user=test@gmail.com';
+ $kolab_uri = preg_replace('|^[a-z]+://|', 'kolab://jack%40kolab.org:simple123@', $imap_uri)
+ . '?dav_host=' . preg_replace('|^davs?://|', '', $dav_uri);
+
+ $imap = new Account(preg_replace('|://|', '://jack%40kolab.org:simple123@', $imap_uri));
+ $dav = new Account(preg_replace('|://|', '://jack%40kolab.org:simple123@', $dav_uri));
+ $src = new Account($takeout_uri);
+ $dst = new Account($kolab_uri);
+
+ return [$imap, $dav, $src, $dst];
+ }
+}
diff --git a/src/tests/Unit/DataMigrator/AccountTest.php b/src/tests/Unit/DataMigrator/AccountTest.php
--- a/src/tests/Unit/DataMigrator/AccountTest.php
+++ b/src/tests/Unit/DataMigrator/AccountTest.php
@@ -29,5 +29,16 @@
// Invalid input
$this->expectException(\Exception::class);
$account = new Account(str_replace('imap://', '', $uri));
+
+ // Local file URI
+ $uri = 'takeout://' . ($file = self::BASE_DIR . '/data/takeout.zip');
+ $account = new Account($uri);
+
+ $this->assertSame($uri, (string) $account);
+ $this->assertSame('takeout', $account->scheme);
+ $this->assertSame($file, $account->uri);
+ $this->assertNull($account->username);
+ $this->assertNull($account->password);
+ $this->assertNull($account->host);
}
}
diff --git a/src/tests/data/takeout.zip b/src/tests/data/takeout.zip
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001

File Metadata

Mime Type
text/plain
Expires
Sat, Apr 4, 9:25 AM (1 d, 21 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18828783
Default Alt Text
D5076.1775294728.diff (36 KB)

Event Timeline