Page MenuHomePhorge
Files F117846084

D5092.1775311434.diff
No OneTemporary
Actions

Authored By
Unknown
Size
35 KB
Referenced Files
None
Subscribers
None

D5092.1775311434.diff
View Options

diff --git a/src/app/Backends/Storage.php b/src/app/Backends/Storage.php
--- a/src/app/Backends/Storage.php
+++ b/src/app/Backends/Storage.php
@@ -106,6 +106,27 @@
return $response;
}
+ /**
+ * File content getter.
+ *
+ * @param \App\Fs\Item $file File object
+ *
+ * @throws \Exception
+ */
+ public static function fileFetch(Item $file): string
+ {
+ $output = '';
+
+ $file->chunks()->orderBy('sequence')->get()->each(function ($chunk) use ($file, &$output) {
+ $disk = LaravelStorage::disk(\config('filesystems.default'));
+ $path = Storage::chunkLocation($chunk->chunk_id, $file);
+
+ $output .= $disk->read($path);
+ });
+
+ return $output;
+ }
+
/**
* File upload handler
*
@@ -137,15 +158,27 @@
$file->save();
}
+ $properties = [
+ 'size' => $fileSize,
+ // Pick the client-supplied mimetype if available, otherwise detect.
+ 'mimetype' => !empty($params['mimetype']) ? $params['mimetype'] : self::mimetype($path),
+ ];
+
// Update the file type and size information
- $file->setProperties([
- 'size' => $fileSize,
- // Pick the client-supplied mimetype if available, otherwise detect.
- 'mimetype' => !empty($params['mimetype']) ? $params['mimetype'] : self::mimetype($path),
- ]);
+ // We can optimize this if we just created a new file and we know there's no property records yet.
+ if (!empty($params['isNew'])) {
+ foreach ($properties as $key => &$value) {
+ $value = ['key' => $key, 'value' => $value];
+ }
+ $file->properties()->createMany($properties);
+ } else {
+ $file->setProperties($properties);
- // Assign the node to the file, "unlink" any old nodes of this file
- $file->chunks()->delete();
+ // "Unlink" any old nodes of this file
+ $file->chunks()->delete();
+ }
+
+ // Assign the node to the file
$file->chunks()->create([
'chunk_id' => $chunkId,
'sequence' => 0,
@@ -212,7 +245,6 @@
throw new \Exception("Invalid 'from' parameter for resumable file upload.");
}
-
$disk = LaravelStorage::disk(\config('filesystems.default'));
$chunkId = \App\Utils::uuidStr();
diff --git a/src/app/DataMigrator/Account.php b/src/app/DataMigrator/Account.php
--- a/src/app/DataMigrator/Account.php
+++ b/src/app/DataMigrator/Account.php
@@ -2,6 +2,8 @@
namespace App\DataMigrator;
+use App\User;
+
/**
* Data object representing user account on an external service
*/
@@ -37,6 +39,9 @@
/** @var string Full account definition */
protected $input;
+ /** @var ?User User object */
+ protected $user;
+
/**
* Object constructor
@@ -115,6 +120,20 @@
return $this->input;
}
+ /**
+ * Returns User object assiciated with the account (if it is a local account)
+ *
+ * @return ?User User object if found
+ */
+ public function getUser(): ?User
+ {
+ if (!$this->user && $this->email) {
+ $this->user = User::where('email', $this->email)->first();
+ }
+
+ return $this->user;
+ }
+
/**
* Parse file URI
*/
diff --git a/src/app/DataMigrator/Driver/Kolab.php b/src/app/DataMigrator/Driver/Kolab.php
--- a/src/app/DataMigrator/Driver/Kolab.php
+++ b/src/app/DataMigrator/Driver/Kolab.php
@@ -21,6 +21,11 @@
Engine::TYPE_EVENT,
Engine::TYPE_TASK,
];
+ protected const IMAP_TYPES = [
+ Engine::TYPE_MAIL,
+ Engine::TYPE_CONFIGURATION,
+ Engine::TYPE_FILE
+ ];
/** @var DAV DAV importer/exporter engine */
protected $davDriver;
@@ -87,6 +92,12 @@
$this->davDriver->createFolder($folder);
return;
}
+
+ // Files
+ if ($folder->type == Engine::TYPE_FILE) {
+ Kolab\Files::createFolder($this->account, $folder);
+ return;
+ }
}
/**
@@ -115,6 +126,12 @@
return;
}
+ // Files
+ if ($item->folder->type == Engine::TYPE_FILE) {
+ Kolab\Files::saveKolab4File($this->account, $item);
+ return;
+ }
+
// Configuration (v3 tags)
if ($item->folder->type == Engine::TYPE_CONFIGURATION) {
Kolab\Tags::migrateKolab3Tag($this->imap, $item);
@@ -144,6 +161,12 @@
return;
}
+ // Files (IMAP)
+ if ($item->folder->type == Engine::TYPE_FILE) {
+ Kolab\Files::fetchKolab3File($this->imap, $item);
+ return;
+ }
+
// Configuration (v3 tags)
if ($item->folder->type == Engine::TYPE_CONFIGURATION) {
Kolab\Tags::fetchKolab3Tag($this->imap, $item);
@@ -173,6 +196,19 @@
return;
}
+ // Files
+ if ($folder->type == Engine::TYPE_FILE) {
+ // Get existing files from the destination account
+ $existing = $importer->getItems($folder);
+
+ $mailbox = self::toUTF7($folder->fullname);
+ foreach (Kolab\Files::getKolab3Files($this->imap, $mailbox, $existing) as $file) {
+ $file['folder'] = $folder;
+ $item = Item::fromArray($file);
+ $callback($item);
+ }
+ }
+
// Configuration (v3 tags)
if ($folder->type == Engine::TYPE_CONFIGURATION) {
// Get existing tags from the destination account
@@ -204,10 +240,11 @@
// Get DAV folders
if (empty($types) || count(array_intersect($types, self::DAV_TYPES)) > 0) {
+ $types = array_intersect($types, self::DAV_TYPES);
$result = $this->davDriver->getFolders($types);
}
- if (!empty($types) && count(array_intersect($types, [Engine::TYPE_MAIL, Engine::TYPE_CONFIGURATION])) == 0) {
+ if (!empty($types) && count(array_intersect($types, self::IMAP_TYPES)) == 0) {
return $result;
}
@@ -237,7 +274,7 @@
[$type] = explode('.', $type);
// These types we do not support
- if ($type != Engine::TYPE_MAIL && $type != Engine::TYPE_CONFIGURATION) {
+ if (!in_array($type, self::IMAP_TYPES)) {
continue;
}
@@ -297,7 +334,12 @@
return $this->davDriver->getItems($folder);
}
- // Configuration folder (v3 tags)
+ // Files
+ if ($folder->type == Engine::TYPE_FILE) {
+ return Kolab\Files::getKolab4Files($this->account, $folder);
+ }
+
+ // Configuration folder (tags)
if ($folder->type == Engine::TYPE_CONFIGURATION) {
return Kolab\Tags::getKolab4Tags($this->imap);
}
diff --git a/src/app/DataMigrator/Driver/Kolab/Files.php b/src/app/DataMigrator/Driver/Kolab/Files.php
new file mode 100644
--- /dev/null
+++ b/src/app/DataMigrator/Driver/Kolab/Files.php
@@ -0,0 +1,309 @@
+<?php
+
+namespace App\DataMigrator\Driver\Kolab;
+
+use App\Backends\Storage;
+use App\DataMigrator\Account;
+use App\DataMigrator\Engine;
+use App\DataMigrator\Interface\Folder;
+use App\DataMigrator\Interface\Item;
+use App\Fs\Item as FsItem;
+use Illuminate\Support\Facades\DB;
+
+/**
+ * Utilities to handle/migrate Kolab (v3 and v4) files
+ */
+class Files
+{
+ /**
+ * Create a Kolab4 files collection (folder)
+ *
+ * @param Account $account Destination account
+ * @param Folder $folder Folder object
+ */
+ public static function createFolder(Account $account, Folder $folder): void
+ {
+ // We assume destination is the local server. Maybe we should be using Cockpit API?
+ self::getFsCollection($account, $folder, true);
+ }
+
+ /**
+ * Get file properties/content
+ *
+ * @param \rcube_imap_generic $imap IMAP client (account)
+ * @param Item $item File item
+ */
+ public static function fetchKolab3File($imap, Item $item): void
+ {
+ // Handle file content in memory (up to 20MB), bigger files will use a temp file
+ if (!isset($item->data['size']) || $item->data['size'] > Engine::MAX_ITEM_SIZE) {
+ // Save the message content to a file
+ $location = $item->folder->tempFileLocation($item->id . '.eml');
+
+ $fp = fopen($location, 'w');
+
+ if (!$fp) {
+ throw new \Exception("Failed to open 'php://temp' stream");
+ }
+ }
+
+ $result = $imap->handlePartBody($item->data['mailbox'], $item->id, true, 3,
+ $item->data['encoding'], null, $fp ?? null);
+
+ if ($result === false) {
+ if (!empty($fp)) {
+ fclose($fp);
+ }
+
+ throw new \Exception("Failed to fetch IMAP message attachment for {$item->data['mailbox']}/{$item->id}");
+ }
+
+ if (!empty($fp) && !empty($location)) {
+ $item->filename = $location;
+ fclose($fp);
+ } else {
+ $item->content = $result;
+ }
+ }
+
+ /**
+ * Get files from Kolab3 (IMAP) folder
+ *
+ * @param \rcube_imap_generic $imap IMAP client (account)
+ * @param string $mailbox Folder name
+ * @param array $existing Files existing at the destination account
+ */
+ public static function getKolab3Files($imap, $mailbox, $existing = []): array
+ {
+ // Find file objects
+ $search = 'NOT DELETED HEADER X-Kolab-Type "application/x-vnd.kolab.file"';
+ $search = $imap->search($mailbox, $search, true);
+ if ($search->is_empty()) {
+ return [];
+ }
+
+ // Get messages' basic headers, include headers for the XML attachment
+ // TODO: Limit data in FETCH, we need only INTERNALDATE and BODY.PEEK[3.MIME].
+ $uids = $search->get_compressed();
+ $messages = $imap->fetchHeaders($mailbox, $uids, true, false, [], ['BODY.PEEK[3.MIME]']);
+ $files = [];
+
+ foreach ($messages as $message) {
+ [$type, $name, $size, $encoding] = self::parseHeaders($message->bodypart['3.MIME'] ?? '');
+
+ // Sanity check
+ if ($name === null || $type === null) {
+ continue;
+ }
+
+ // Note: We do not really need to fetch and parse Kolab XML
+
+ $mtime = \rcube_utils::anytodatetime($message->internaldate, new \DateTimeZone('UTC'));
+ $exists = $existing[$name] ?? null;
+
+ if ($exists && $exists->updated_at == $mtime) {
+ // No changes to the file, skip it
+ continue;
+ }
+
+ $files[] = [
+ 'id' => $message->uid,
+ 'existing' => $exists,
+ 'data' => [
+ 'name' => $name,
+ 'mailbox' => $mailbox,
+ 'encoding' => $encoding,
+ 'type' => $type,
+ 'size' => $size,
+ 'mtime' => $mtime,
+ ],
+ ];
+ }
+
+ return $files;
+ }
+
+ /**
+ * Get list of Kolab4 files
+ *
+ * @param Account $account Destination account
+ * @param Folder $folder Folder
+ */
+ public static function getKolab4Files(Account $account, Folder $folder): array
+ {
+ // We assume destination is the local server. Maybe we should be using Cockpit API?
+ $collection = self::getFsCollection($account, $folder, false);
+
+ if (!$collection) {
+ return [];
+ }
+
+ return $collection->children()
+ ->select('fs_items.*')
+ ->addSelect(DB::raw("(select value from fs_properties where fs_properties.item_id = fs_items.id"
+ . " and fs_properties.key = 'name') as name"))
+ ->where('type', '&', FsItem::TYPE_FILE)
+ ->whereNot('type', '&', FsItem::TYPE_INCOMPLETE)
+ ->get()
+ ->keyBy('name')
+ ->all();
+ }
+
+ /**
+ * Save a file into Kolab4 storage
+ *
+ * @param Account $account Destination account
+ * @param Item $item File item
+ */
+ public static function saveKolab4File(Account $account, Item $item): void
+ {
+ // We assume destination is the local server. Maybe we should be using Cockpit API?
+ $collection = self::getFsCollection($account, $item->folder, false);
+
+ if (!$collection) {
+ throw new \Exception("Failed to find destination collection for {$item->folder->fullname}");
+ }
+
+ $params = ['mimetype' => $item->data['type']];
+
+ DB::beginTransaction();
+
+ if ($item->existing) {
+ /** @var FsItem $file */
+ $file = $item->existing;
+ $file->updated_at = $item->data['mtime'];
+ $file->timestamps = false;
+ $file->save();
+ } else {
+ $file = new FsItem();
+ $file->user_id = $account->getUser()->id;
+ $file->type = FsItem::TYPE_FILE;
+ $file->updated_at = $item->data['mtime'];
+ $file->timestamps = false;
+ $file->save();
+
+ $file->properties()->create(['key' => 'name', 'value' => $item->data['name']]);
+ $collection->relations()->create(['related_id' => $file->id]);
+ $params['isNew'] = true;
+ }
+
+ if ($item->filename) {
+ $fp = fopen($item->filename, 'r');
+ } else {
+ $fp = fopen('php://memory', 'r+');
+ fwrite($fp, $item->content);
+ rewind($fp);
+ }
+
+ // TODO: Use proper size chunks
+
+ Storage::fileInput($fp, $params, $file);
+
+ DB::commit();
+
+ fclose($fp);
+ }
+
+ /**
+ * Find (and optionally create) a Kolab4 files collection
+ *
+ * @param Account $account Destination account
+ * @param Folder $folder Folder object
+ * @param bool $create Create collection(s) if it does not exist
+ *
+ * @return ?FsItem Collection object if found
+ */
+ protected static function getFsCollection(Account $account, Folder $folder, bool $create = false)
+ {
+ if (!empty($folder->data['collection'])) {
+ return $folder->data['collection'];
+ }
+
+ // We assume destination is the local server. Maybe we should be using Cockpit API?
+ $user = $account->getUser();
+
+ // TODO: For now we assume '/' is the IMAP hierarchy separator. This may not work with dovecot.
+ $path = explode('/', $folder->fullname);
+ $collection = null;
+
+ // Create folder (and the whole tree) if it does not exist yet
+ foreach ($path as $name) {
+ $result = $user->fsItems()->select('fs_items.*');
+
+ if ($collection) {
+ $result->join('fs_relations', 'fs_items.id', '=', 'fs_relations.related_id')
+ ->where('fs_relations.item_id', $collection->id);
+ } else {
+ $result->leftJoin('fs_relations', 'fs_items.id', '=', 'fs_relations.related_id')
+ ->whereNull('fs_relations.related_id');
+ }
+
+ $found = $result->join('fs_properties', 'fs_items.id', '=', 'fs_properties.item_id')
+ ->where('type', '&', FsItem::TYPE_COLLECTION)
+ ->where('key', 'name')
+ ->where('value', $name)
+ ->first();
+
+ if (!$found) {
+ if ($create) {
+ DB::beginTransaction();
+ $col = $user->fsItems()->create(['type' => FsItem::TYPE_COLLECTION]);
+ $col->properties()->create(['key' => 'name', 'value' => $name]);
+ if ($collection) {
+ $collection->relations()->create(['related_id' => $col->id]);
+ }
+ $collection = $col;
+ DB::commit();
+ } else {
+ return null;
+ }
+ } else {
+ $collection = $found;
+ }
+ }
+
+ $folder->data['collection'] = $collection;
+
+ return $collection;
+ }
+
+ /**
+ * Parse mail attachment headers (get content type, file name and size)
+ */
+ protected static function parseHeaders(string $input): array
+ {
+ // Parse headers
+ $headers = \rcube_mime::parse_headers($input);
+
+ $ctype = $headers['content-type'] ?? '';
+ $disposition = $headers['content-disposition'] ?? '';
+ $tokens = preg_split('/;[\s\r\n\t]*/', $ctype . ';' . $disposition);
+ $params = [];
+
+ // Extract file parameters
+ foreach ($tokens as $token) {
+ // TODO: Use order defined by the parameter name not order of occurrence in the header
+ if (preg_match('/^(name|filename)\*([0-9]*)\*?="*([^"]+)"*/i', $token, $matches)) {
+ $key = strtolower($matches[1]);
+ $params["{$key}*"] = ($params["{$key}*"] ?? '') . $matches[3];
+ } elseif (str_starts_with($token, 'size=')) {
+ $params['size'] = (int) trim(substr($token, 5));
+ } elseif (str_starts_with($token, 'filename=')) {
+ $params['filename'] = trim(substr($token, 9), '" ');
+ }
+ }
+
+ $type = explode(';', $ctype)[0] ?? null;
+ $size = $params['size'] ?? null;
+ $encoding = $headers['content-transfer-encoding'] ?? null;
+ $name = $params['filename'] ?? null;
+
+ // Decode file name according to RFC 2231, Section 4
+ if (isset($params['filename*']) && preg_match("/^([^']*)'[^']*'(.*)$/", $params['filename*'], $m)) {
+ // Note: We ignore charset as it should be always UTF-8
+ $name = rawurldecode($m[2]);
+ }
+
+ return [$type, $name, $size, $encoding];
+ }
+}
diff --git a/src/app/DataMigrator/Interface/Folder.php b/src/app/DataMigrator/Interface/Folder.php
--- a/src/app/DataMigrator/Interface/Folder.php
+++ b/src/app/DataMigrator/Interface/Folder.php
@@ -39,6 +39,9 @@
/** @var bool Folder subscription state */
public $subscribed = true;
+ /** @var array Extra data to migrate (like ACL) or some temporary data */
+ public $data = [];
+
/**
* Create Folder instance from an array
diff --git a/src/app/Fs/Item.php b/src/app/Fs/Item.php
--- a/src/app/Fs/Item.php
+++ b/src/app/Fs/Item.php
@@ -41,7 +41,7 @@
/**
- * COntent chunks of this item (file).
+ * Content chunks of this item (file).
*
* @return \Illuminate\Database\Eloquent\Relations\HasMany
*/
diff --git a/src/app/Fs/Relation.php b/src/app/Fs/Relation.php
--- a/src/app/Fs/Relation.php
+++ b/src/app/Fs/Relation.php
@@ -18,4 +18,7 @@
/** @var string Database table name */
protected $table = 'fs_relations';
+
+ /** @var bool Indicates if the model should be timestamped. */
+ public $timestamps = false;
}
diff --git a/src/app/Http/Controllers/API/V4/FsController.php b/src/app/Http/Controllers/API/V4/FsController.php
--- a/src/app/Http/Controllers/API/V4/FsController.php
+++ b/src/app/Http/Controllers/API/V4/FsController.php
@@ -292,8 +292,8 @@
// Add properties
$result->join('fs_properties', 'fs_items.id', '=', 'fs_properties.item_id')
- ->whereNot('type', '&', Item::TYPE_INCOMPLETE)
- ->where('key', 'name');
+ ->whereNot('type', '&', Item::TYPE_INCOMPLETE)
+ ->where('key', 'name');
if ($type) {
if ($type == self::TYPE_COLLECTION) {
@@ -320,6 +320,8 @@
// Process the result
$result = $result->map(
function ($file) {
+ // TODO: This is going to be 100 SELECT queries (with pageSize=100), we should get
+ // file properties using the main query
$result = $this->objectToClient($file);
$result['name'] = $file->name; // @phpstan-ignore-line
diff --git a/src/tests/BackendsTrait.php b/src/tests/BackendsTrait.php
--- a/src/tests/BackendsTrait.php
+++ b/src/tests/BackendsTrait.php
@@ -271,7 +271,7 @@
/**
* Append an email message to the IMAP folder
*/
- protected function imapAppend(Account $account, $folder, $filename, $flags = [], $date = null): string
+ protected function imapAppend(Account $account, $folder, $filename, $flags = [], $date = null, $replace = []): string
{
$imap = $this->getImapClient($account);
@@ -284,6 +284,10 @@
$source = file_get_contents($source);
$source = preg_replace('/\r?\n/', "\r\n", $source);
+ foreach ($replace as $from => $to) {
+ $source = preg_replace($from, $to, $source);
+ }
+
$uid = $imap->append($folder, $source, $flags, $date, true);
if ($uid === false) {
diff --git a/src/tests/Feature/DataMigrator/KolabTest.php b/src/tests/Feature/DataMigrator/KolabTest.php
--- a/src/tests/Feature/DataMigrator/KolabTest.php
+++ b/src/tests/Feature/DataMigrator/KolabTest.php
@@ -2,10 +2,14 @@
namespace Tests\Feature\DataMigrator;
+use App\Backends\Storage;
use App\DataMigrator\Account;
use App\DataMigrator\Driver\Kolab\Tags as KolabTags;
use App\DataMigrator\Engine;
use App\DataMigrator\Queue as MigratorQueue;
+use App\Fs\Item as FsItem;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Storage as LaravelStorage;
use Tests\BackendsTrait;
use Tests\TestCase;
@@ -18,6 +22,9 @@
{
use BackendsTrait;
+ private static $skipTearDown = false;
+ private static $skipSetUp = false;
+
/**
* {@inheritDoc}
*/
@@ -25,7 +32,12 @@
{
parent::setUp();
- MigratorQueue::truncate();
+ if (!self::$skipSetUp) {
+ MigratorQueue::truncate();
+ FsItem::query()->forceDelete();
+ }
+
+ self::$skipSetUp = false;
}
/**
@@ -33,7 +45,17 @@
*/
public function tearDown(): void
{
- MigratorQueue::truncate();
+ if (!self::$skipTearDown) {
+ MigratorQueue::truncate();
+ FsItem::query()->forceDelete();
+
+ $disk = LaravelStorage::disk(\config('filesystems.default'));
+ foreach ($disk->listContents('') as $dir) {
+ $disk->deleteDirectory($dir->path());
+ }
+ }
+
+ self::$skipTearDown = false;
parent::tearDown();
}
@@ -117,6 +139,45 @@
$this->assertCount(2, $tasks);
$this->assertSame('Task1', $tasks['ccc-ccc']->summary);
$this->assertSame('Task2', $tasks['ddd-ddd']->summary);
+
+ // Assert migrated files
+ $user = $dst_imap->getUser();
+ $folders = $user->fsItems()->where('type', '&', FsItem::TYPE_COLLECTION)
+ ->select('fs_items.*')
+ ->addSelect(DB::raw("(select value from fs_properties where fs_properties.item_id = fs_items.id"
+ . " and fs_properties.key = 'name') as name"))
+ ->get()
+ ->keyBy('name')
+ ->all();
+ $files = $user->fsItems()->whereNot('type', '&', FsItem::TYPE_COLLECTION)
+ ->select('fs_items.*')
+ ->addSelect(DB::raw("(select value from fs_properties where fs_properties.item_id = fs_items.id"
+ . " and fs_properties.key = 'name') as name"))
+ ->get()
+ ->keyBy('name')
+ ->all();
+ $this->assertSame(2, count($folders));
+ $this->assertSame(3, count($files));
+ $this->assertArrayHasKey('A€B', $folders);
+ $this->assertArrayHasKey('Files', $folders);
+ $this->assertTrue($folders['Files']->children->contains($folders['A€B']));
+ $this->assertTrue($folders['Files']->children->contains($files['empty.txt']));
+ $this->assertTrue($folders['Files']->children->contains($files['&kość.odt']));
+ $this->assertTrue($folders['A€B']->children->contains($files['test2.odt']));
+ $this->assertEquals(10000, $files['&kość.odt']->getProperty('size'));
+ $this->assertEquals(0, $files['empty.txt']->getProperty('size'));
+ $this->assertEquals(10000, $files['test2.odt']->getProperty('size'));
+ $this->assertSame('application/vnd.oasis.opendocument.odt', $files['&kość.odt']->getProperty('mimetype'));
+ $this->assertSame('text/plain', $files['empty.txt']->getProperty('mimetype'));
+ $this->assertSame('application/vnd.oasis.opendocument.odt', $files['test2.odt']->getProperty('mimetype'));
+ $this->assertSame('2024-01-10 09:09:09', $files['&kość.odt']->updated_at->toDateTimeString());
+ $file_content = str_repeat('1234567890', 1000);
+ $this->assertSame($file_content, Storage::fileFetch($files['&kość.odt']));
+ $this->assertSame($file_content, Storage::fileFetch($files['test2.odt']));
+ $this->assertSame('', Storage::fileFetch($files['empty.txt']));
+
+ self::$skipTearDown = true;
+ self::$skipSetUp = true;
}
/**
@@ -141,6 +202,10 @@
'/SUMMARY:Party/' => 'SUMMARY:Test'
];
$this->davAppend($src_dav, 'Calendar', ['event/1.ics'], Engine::TYPE_EVENT, $replace);
+ $this->imapEmptyFolder($src_imap, 'Files');
+ $file_content = rtrim(chunk_split(base64_encode('123'), 76, "\r\n"));
+ $replaces = ['/%FILE%/' => $file_content];
+ $this->imapAppend($src_imap, 'Files', 'kolab3/file1.eml', [], '12-Jan-2024 09:09:09 +0000', $replaces);
// Run the migration
$migrator = new Engine();
@@ -180,6 +245,21 @@
$events = \collect($events)->keyBy('uid')->all();
$this->assertCount(2, $events);
$this->assertSame('Test', $events['abcdef']->summary);
+
+ // Assert the migrated files
+ $user = $dst_imap->getUser();
+ $files = $user->fsItems()->whereNot('type', '&', FsItem::TYPE_COLLECTION)
+ ->select('fs_items.*')
+ ->addSelect(DB::raw("(select value from fs_properties where fs_properties.item_id = fs_items.id"
+ . " and fs_properties.key = 'name') as name"))
+ ->get()
+ ->keyBy('name')
+ ->all();
+ $this->assertSame(3, count($files));
+ $this->assertEquals(3, $files['&kość.odt']->getProperty('size'));
+ $this->assertSame('application/vnd.oasis.opendocument.odt', $files['&kość.odt']->getProperty('mimetype'));
+ $this->assertSame('2024-01-12 09:09:09', $files['&kość.odt']->updated_at->toDateTimeString());
+ $this->assertSame('123', Storage::fileFetch($files['&kość.odt']));
}
/**
@@ -248,10 +328,33 @@
// One more IMAP folder, subscribed
$this->imapCreateFolder($imap_account, 'Test2', true);
- // Insert some other data to migrate
+ // File storage in IMAP
+ $utf7_folder = \mb_convert_encoding('Files/A€B', 'UTF7-IMAP', 'UTF8');
+ $this->imapCreateFolder($imap_account, 'Files');
+ $this->imapCreateFolder($imap_account, $utf7_folder);
+ $this->imapEmptyFolder($imap_account, 'Files');
+ $this->imapEmptyFolder($imap_account, $utf7_folder);
+ if (!$imap->setMetadata('Files', ['/private/vendor/kolab/folder-type' => 'file.default'])) {
+ throw new \Exception("Failed to set metadata");
+ }
+ if (!$imap->setMetadata($utf7_folder, ['/private/vendor/kolab/folder-type' => 'file'])) {
+ throw new \Exception("Failed to set metadata");
+ }
+ $file_content = str_repeat('1234567890', 1000);
+ $file_content = rtrim(chunk_split(base64_encode($file_content), 76, "\r\n"));
+ $replaces = ['/%FILE%/' => $file_content];
+ $this->imapAppend($imap_account, 'Files', 'kolab3/file1.eml', [], '10-Jan-2024 09:09:09 +0000', $replaces);
+ $this->imapAppend($imap_account, 'Files', 'kolab3/file2.eml', [], '11-Jan-2024 09:09:09 +0000');
+ $replaces['/&amp;kość.odt/'] = 'test2.odt';
+ $replaces['/&ko%C5%9B%C4%87.odt/'] = 'test2.odt';
+ $this->imapAppend($imap_account, $utf7_folder, 'kolab3/file1.eml', [], '12-Jan-2024 09:09:09 +0000', $replaces);
+
+ // Insert some mail to migrate
$this->imapAppend($imap_account, 'INBOX', 'mail/1.eml');
$this->imapAppend($imap_account, 'INBOX', 'mail/2.eml', ['SEEN']);
$this->imapAppend($imap_account, 'Drafts', 'mail/3.eml', ['SEEN']);
+
+ // Insert some DAV data to migrate
$this->davAppend($dav_account, 'Calendar', ['event/1.ics', 'event/2.ics'], Engine::TYPE_EVENT);
$this->davCreateFolder($dav_account, 'Custom Calendar', Engine::TYPE_EVENT);
$this->davAppend($dav_account, 'Custom Calendar', ['event/3.ics'], Engine::TYPE_EVENT);
diff --git a/src/tests/Unit/DataMigrator/Driver/Kolab/FilesTest.php b/src/tests/Unit/DataMigrator/Driver/Kolab/FilesTest.php
new file mode 100644
--- /dev/null
+++ b/src/tests/Unit/DataMigrator/Driver/Kolab/FilesTest.php
@@ -0,0 +1,58 @@
+<?php
+
+namespace Tests\Unit\DataMigrator\Driver\Kolab;
+
+use App\DataMigrator\Driver\Kolab\Files;
+use Tests\TestCase;
+
+class FilesTest extends TestCase
+{
+ /**
+ * Test attachment part headers processing
+ */
+ public function testParseHeaders(): void
+ {
+ $files = new Files();
+
+ $headers = "Content-ID: <ko.1732537341.48230.odt>\r\n"
+ . "Content-Transfer-Encoding: base64\r\n"
+ . "Content-Type: application/vnd.oasis.opendocument.odt;\r\n"
+ . " name*=UTF-8''&ko%C5%9B%C4%87.txt\r\n"
+ . "Content-Disposition: attachment;\r\n"
+ . " filename*=UTF-8''&ko%C5%9B%C4%87.odt;\r\n"
+ . " size=30068\r\n";
+
+ $result = $this->invokeMethod($files, 'parseHeaders', [$headers]);
+ $this->assertSame('application/vnd.oasis.opendocument.odt', $result[0]);
+ $this->assertSame('&kość.odt', $result[1]);
+ $this->assertSame(30068, $result[2]);
+ $this->assertSame('base64', $result[3]);
+
+ $headers = "Content-ID: <ko.1732537341.48230.odt>\r\n"
+ . "Content-Transfer-Encoding: quoted-printable\r\n"
+ . "Content-Type: text/plain;\r\n"
+ . " name*0*=UTF-8''a%20very%20long%20name%20for%20the%20attachment%20to%20tes;\r\n"
+ . " name*1*=t%20%C4%87%C4%87%C4%87%20will%20see%20how%20it%20goes%20with%20so;\r\n"
+ . " name*2*=me%20non-ascii%20ko%C5%9B%C4%87%20characters.txt\r\n"
+ . "Content-Disposition: attachment;\r\n"
+ . " filename*0*=UTF-8''a%20very%20long%20name%20for%20the%20attachment%20to;\r\n"
+ . " filename*1*=%20test%20%C4%87%C4%87%C4%87%20will%20see%20how%20it%20goes;\r\n"
+ . " filename*2*=%20with%20some%20non-ascii%20ko%C5%9B%C4%87%20characters.txt\r\n";
+
+ $result = $this->invokeMethod($files, 'parseHeaders', [$headers]);
+ $this->assertSame('text/plain', $result[0]);
+ $this->assertSame('a very long name for the attachment to test ććć will see how '
+ . 'it goes with some non-ascii kość characters.txt', $result[1]);
+ $this->assertSame(null, $result[2]);
+ $this->assertSame('quoted-printable', $result[3]);
+
+ $headers = "Content-Type: text/plain; name=test.txt\r\n"
+ . "Content-Disposition: attachment; filename=test.txt\r\n";
+
+ $result = $this->invokeMethod($files, 'parseHeaders', [$headers]);
+ $this->assertSame('text/plain', $result[0]);
+ $this->assertSame('test.txt', $result[1]);
+ $this->assertSame(null, $result[2]);
+ $this->assertSame(null, $result[3]);
+ }
+}
diff --git a/src/tests/data/kolab3/file1.eml b/src/tests/data/kolab3/file1.eml
new file mode 100644
--- /dev/null
+++ b/src/tests/data/kolab3/file1.eml
@@ -0,0 +1,56 @@
+MIME-Version: 1.0
+From: ned@kolab.org
+To: ned@kolab.org
+Date: Mon, 25 Nov 2024 13:22:21 +0100
+X-Kolab-Type: application/x-vnd.kolab.file
+X-Kolab-Mime-Version: 3.0
+Subject: f76382d6-651d-43ea-ae64-eca3884aed04
+User-Agent: Kolab 16/Roundcube 1.5-git
+Content-Type: multipart/mixed;
+ boundary="=_f70c0dafff53b07d8e6e5a83f1430da1"
+
+--=_f70c0dafff53b07d8e6e5a83f1430da1
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a Kolab Groupware object. To view this object you will need an emai=
+l client that understands the Kolab Groupware format. For a list of such em=
+ail clients please visit https://www.kolab.org/
+
+
+--=_f70c0dafff53b07d8e6e5a83f1430da1
+Content-Transfer-Encoding: 8bit
+Content-Type: application/vnd.kolab+xml; charset=UTF-8;
+ name=kolab.xml
+Content-Disposition: attachment;
+ filename=kolab.xml;
+ size=618
+
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<file xmlns="http://kolab.org" version="3.0">
+ <uid>f76382d6-651d-43ea-ae64-eca3884aed04</uid>
+ <prodid>Roundcube-libkolab-1.1 Libkolabxml-1.3.1</prodid>
+ <creation-date>2019-03-12T09:35:08Z</creation-date>
+ <last-modification-date>2024-11-25T12:22:21Z</last-modification-date>
+ <classification>PUBLIC</classification>
+ <file>
+ <parameters>
+ <fmttype>application/vnd.oasis.opendocument.text</fmttype>
+ <x-label>&amp;kość.odt</x-label>
+ </parameters>
+ <uri>cid:ko.1732537341.48230.odt</uri>
+ </file>
+ <note/>
+</file>
+
+--=_f70c0dafff53b07d8e6e5a83f1430da1
+Content-ID: <ko.1732537341.48230.odt>
+Content-Transfer-Encoding: base64
+Content-Type: application/vnd.oasis.opendocument.odt;
+ name*=UTF-8''&ko%C5%9B%C4%87.odt
+Content-Disposition: attachment;
+ filename*=UTF-8''&ko%C5%9B%C4%87.odt;
+ size=30068
+
+%FILE%
+--=_f70c0dafff53b07d8e6e5a83f1430da1--
diff --git a/src/tests/data/kolab3/file2.eml b/src/tests/data/kolab3/file2.eml
new file mode 100644
--- /dev/null
+++ b/src/tests/data/kolab3/file2.eml
@@ -0,0 +1,52 @@
+MIME-Version: 1.0
+From: ned@kolab.org
+To: ned@kolab.org
+Date: Mon, 25 Nov 2024 13:22:21 +0100
+X-Kolab-Type: application/x-vnd.kolab.file
+X-Kolab-Mime-Version: 3.0
+Subject: f76382d6-651d-43ea-ae64-eca3884aed05
+User-Agent: Kolab 16/Roundcube 1.5-git
+Content-Type: multipart/mixed;
+ boundary="=_f70c0dafff53b07d8e6e5a83f1430da1"
+
+--=_f70c0dafff53b07d8e6e5a83f1430da1
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a Kolab Groupware object. To view this object you will need an emai=
+l client that understands the Kolab Groupware format. For a list of such em=
+ail clients please visit https://www.kolab.org/
+
+
+--=_f70c0dafff53b07d8e6e5a83f1430da1
+Content-Transfer-Encoding: 8bit
+Content-Type: application/vnd.kolab+xml; charset=UTF-8;
+ name=kolab.xml
+Content-Disposition: attachment;
+ filename=kolab.xml
+
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<file xmlns="http://kolab.org" version="3.0">
+ <uid>f76382d6-651d-43ea-ae64-eca3884aed05</uid>
+ <prodid>Roundcube-libkolab-1.1 Libkolabxml-1.3.1</prodid>
+ <creation-date>2024-03-12T09:35:08Z</creation-date>
+ <last-modification-date>2024-11-27T12:22:21Z</last-modification-date>
+ <classification>PUBLIC</classification>
+ <file>
+ <parameters>
+ <fmttype>text/plain</fmttype>
+ <x-label>empty.txt</x-label>
+ </parameters>
+ <uri>cid:empty.1732537341.48230.txt</uri>
+ </file>
+ <note/>
+</file>
+
+--=_f70c0dafff53b07d8e6e5a83f1430da1
+Content-ID: <empty.1732537341.48230.txt>
+Content-Transfer-Encoding: base64
+Content-Type: text/plain; name=empty.txt
+Content-Disposition: attachment; filename=empty.txt; size=0
+
+
+--=_f70c0dafff53b07d8e6e5a83f1430da1--

File Metadata

Mime Type
text/plain
Expires
Sat, Apr 4, 2:03 PM (14 h, 36 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18829787
Default Alt Text
D5092.1775311434.diff (35 KB)

Event Timeline