diff --git a/docker/migrate/Dockerfile b/docker/migrate/Dockerfile new file mode 100644 --- /dev/null +++ b/docker/migrate/Dockerfile @@ -0,0 +1,42 @@ +FROM fedora:33 + +MAINTAINER Jeroen van Meeuwen + +ENV HOME=/opt/app-root/src + +RUN dnf -y install \ + composer \ + file \ + git \ + make \ + npm \ + openssl-devel \ + php-cli \ + php-common \ + php-ldap \ + php-opcache \ + php-pecl-apcu \ + php-mysqlnd \ + wget \ + sudo \ + libgsf \ + gettext \ + libtool \ + gcc-c++ \ + shtool \ + automake \ + gd-devel zlib-devel boost-devel libgsf-devel gettext-devel \ + python3 python3-devel boost-python3 boost-python3-devel + + +RUN id default || (groupadd -g 1001 default && useradd -d /opt/app-root/ -u 1001 -g 1001 default) +RUN usermod -aG wheel default +RUN echo '%wheel ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +RUN mkdir ${HOME} && chown 1001:1001 ${HOME} + +USER 1001 + +WORKDIR ${HOME} + +COPY /rootfs / diff --git a/docker/migrate/README.md b/docker/migrate/README.md new file mode 100644 --- /dev/null +++ b/docker/migrate/README.md @@ -0,0 +1,6 @@ +This is a docker container to run the migrate:userdata artisan command. + +The python script provided allows mass migrating pst files. + + + diff --git a/docker/migrate/do-it-again.sh b/docker/migrate/do-it-again.sh new file mode 100755 --- /dev/null +++ b/docker/migrate/do-it-again.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +export APP_DEBUG="true" +export APP_KEY= +export APP_PUBLIC_URL=http://127.0.0.1:8000/ +export APP_SRC=src/ +export APP_URL=http://127.0.0.1:8000/ +export CACHE_DRIVER="array" +export COMPOSER_ARGS="--no-dev" +export DB_CONNECTION="sqlite" +export DB_DATABASE=":memory:" +export GIT_URI=https://git.kolab.org/source/kolab.git +export GIT_BRANCH=dev/pstimport +export LARAVEL_ENV=production +export LOG_CHANNEL="stderr" +export MAIL_DRIVER="array" +export QUEUE_CONNECTION="sync" +export SESSION_DRIVER="array" + +docker build -t migrate . + +docker kill migrate + +docker rm migrate + +docker_opts="\ + -e APP_DEBUG=${APP_DEBUG} \ + -e APP_KEY=${APP_KEY} \ + -e APP_PUBLIC_URL=${APP_PUBLIC_URL} \ + -e APP_SRC=${APP_SRC} \ + -e APP_URL=${APP_URL} \ + -e CACHE_DRIVER=${CACHE_DRIVER} \ + -e COMPOSER_ARGS=${COMPOSER_ARGS} \ + -e DB_CONNECTION=${DB_CONNECTION} \ + -e DB_DATABASE=${DB_DATABASE} \ + -e GIT_URI=${GIT_URI} \ + -e GIT_BRANCH=${GIT_BRANCH} \ + -e LARAVEL_ENV=${LARAVEL_ENV} \ + -e LOG_CHANNEL=${LOG_CHANNEL} \ + -e MAIL_DRIVER=${MAIL_DRIVER} \ + -e QUEUE_CONNECTION=${QUEUE_CONNECTION} \ + -e SESSION_DRIVER=${SESSION_DRIVER}" + +docker run -it \ + ${docker_opts} \ + --name migrate migrate /usr/local/bin/build-image + +docker commit migrate migrate-s2i diff --git a/docker/migrate/input/john@kolab.org.pst b/docker/migrate/input/john@kolab.org.pst new file mode 100644 diff --git a/docker/migrate/massmigrate.py b/docker/migrate/massmigrate.py new file mode 100755 --- /dev/null +++ b/docker/migrate/massmigrate.py @@ -0,0 +1,136 @@ +#!/bin/env python3 + +# This script allows mass migrating pst files. +# Usage ./massmigrate.py +# --input ./input/ +# --output ./output/ +# --password $password +# --imap ssl://$domain:993 --dav https://$domain/iRony +# +# It requires an input and an output directory. +# +# The input directory contains files to import in the format: +# username@hostname.pst +# +# The output directory will receive all processed pst files. + +import subprocess +import argparse +import glob +import shutil +import os +import time +from threading import Thread +from queue import Queue + + +def transform_file(file): + dir_path, filename = os.path.split(os.path.realpath(file)) + + removepst = False + if '.zip' in file: + username = filename.replace('.zip', '') + parts = username.split('.') + username = f"{parts[1]}.{parts[2]}@{parts[0].lower()}.domain.com" + result = subprocess.run(f"unzip {filename}", + shell=True, + cwd=dir_path + ) + if result.returncode != 0: + return None, None, None, None + removepst = True + filename = f"{parts[1]}.{parts[2]}.pst" + else: + username = filename.replace('.pst', '') + + return dir_path, filename, username, removepst + + +def import_file(file, password, imap_uri, dav_uri, type_filter, type_blacklist): + image = "migrate-s2i" + start = time.time() + + dir_path, filename, username, removepst = transform_file(file) + if not username: + print(f"Failed to extract the file {file}") + return False + + print("Starting " + username) + + DOCKER_OPTIONS = ' '.join([ + "--network=host", "--rm", "-ti", + "-v", f"{dir_path}:/opt/app-root/input/", + "-w", "/opt/app-root/src/kolab/src", + ]) + + cmdargs = [ + "./artisan", "migrate:userdata", + f"--importpst=/opt/app-root/input/{filename}", + f"--username={username} --password={password}", + "--subscribe --debug", + f"--davUrl={dav_uri}", + f"--imapUrl={imap_uri}", + "--exclude-target='Sync Issues (This computer only)'", + "--exclude-target='Drafts (This computer only)'", + "--exclude-target='Sync Issues*'", + "--exclude-target='Recoverable Items*'", + ] + + if type_filter: + cmdargs.append(f"--type-filter='{type_filter}'") + + if type_blacklist: + cmdargs.append(f"--type-blacklist='{type_blacklist}'") + + CMD = ' '.join(cmdargs) + + with open(f"output/{username}.log", 'w') as logfile: + result = subprocess.run(f"docker run {DOCKER_OPTIONS} {image} {CMD}", + shell=True, + text=True, + stderr=subprocess.STDOUT, + stdout=logfile) + + executionTime = time.time() - start + print(f"Finished {username} in {executionTime}s") + if removepst: + os.remove(f"{dir_path}/{filename}") + return not result.returncode + return 1 + + +def main(): + parser = argparse.ArgumentParser("usage: %prog [options]") + parser.add_argument("--input", help="Input directory") + parser.add_argument("--output", help="Output directory") + parser.add_argument("--password", help="User password to use for all files") + parser.add_argument("--imap", help="IMAP URI") + parser.add_argument("--dav", help="DAV URI") + parser.add_argument("--typefilter", help="Folder type whitelist (not a list)") + parser.add_argument("--typeblacklist", help="Folder type blacklist (not a list)") + options = parser.parse_args() + + input_dir = os.path.expanduser(options.input) + output_dir = os.path.expanduser(options.output) + + q = Queue() + for file in glob.glob(f"{input_dir}*"): + q.put(file) + + def worker(): + while True: + file = q.get() + if import_file(file, options.password, options.imap, options.dav, options.typefilter, options.typeblacklist): + shutil.move(file, output_dir + os.path.basename(file)) + else: + print(f"Failed to import {file}") + q.task_done() + + for _i in range(3): + Thread(target=worker, daemon=True).start() + q.join() + print("Processing finished") + + +if __name__ == "__main__": + main() diff --git a/docker/migrate/rootfs/usr/local/bin/build-image b/docker/migrate/rootfs/usr/local/bin/build-image new file mode 100755 --- /dev/null +++ b/docker/migrate/rootfs/usr/local/bin/build-image @@ -0,0 +1,44 @@ +#!/bin/bash + +set -x + +set -e + +pwd + +if [ -z "${GIT_URI}" ]; then + echo "No GIT_URI specified. Exiting." + exit 1 +fi + +git clone -b ${GIT_BRANCH:-master} ${GIT_URI} + +cd $(basename ${GIT_URI} .git) + +if [ ! -z "${APP_SRC}" ]; then + cd ${APP_SRC} +fi + +if [ -f "composer.json" ]; then + echo "Detected composer.json, running install" + php -dmemory_limit=${COMPOSER_MEMORY_LIMIT:--1} /usr/bin/composer install ${COMPOSER_ARGS} + rm -rf ~/.cache/composer/ +fi + +if [ -z "${LARAVEL_ENV}" ]; then + LARAVEL_ENV=prod +fi + +npm install + +npm run ${LARAVEL_ENV} && rm -rf ~/.npm/ + +cd - + +# Build libpst +git clone -b stable/kolab-0.6.76 https://git.kolab.org/source/libpst.git +cd libpst +autoreconf -vif +./configure --enable-libpst-shared --with-boost-python=boost_python39 +make -j5 +sudo make install diff --git a/src/app/Console/Commands/MigrateUserdata.php b/src/app/Console/Commands/MigrateUserdata.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/MigrateUserdata.php @@ -0,0 +1,657 @@ +option('importpst')) { + return $this->importPst( + $this->option('importpst'), + $this->option('username'), + $this->option('password'), + $this->option('clear-target'), + $this->option('subscribe'), + $this->option('exclude-target'), + $this->option('include-target'), + $this->option('pickup-from'), + $this->option('debug'), + $this->option('davUrl'), + $this->option('imapUrl'), + $this->option('type-filter'), + $this->option('type-blacklist'), + ); + } + } + + + /** + * Get LDAP configuration for specified access level + */ + private static function getConfig($imapUrl, $username, $password, $verifyPeer, $verifyHost) + { + $uri = \parse_url($imapUrl ?? \config('imap.uri')); + $default_port = 143; + $ssl_mode = null; + + if (isset($uri['scheme'])) { + if (preg_match('/^(ssl|imaps)/', $uri['scheme'])) { + $default_port = 993; + $ssl_mode = 'ssl'; + } elseif ($uri['scheme'] === 'tls') { + $ssl_mode = 'tls'; + } + } + + $config = [ + 'host' => $uri['host'], + 'user' => $username ?? \config('imap.admin_login'), + 'password' => $password ?? \config('imap.admin_password'), + 'options' => [ + 'port' => !empty($uri['port']) ? $uri['port'] : $default_port, + 'ssl_mode' => $ssl_mode, + 'timeout' => 60, + //Work around guam bug + 'literal+' => false, + 'socket_options' => [ + 'ssl' => [ + 'verify_peer' => $verifyPeer ?? \config('imap.verify_peer'), + 'verify_peer_name' => $verifyPeer ?? \config('imap.verify_peer'), + 'verify_host' => $verifyHost ?? \config('imap.verify_host') + ], + ], + ], + ]; + + return $config; + } + + /** + * Initialize connection to IMAP + */ + private function initIMAP(array $config, string $login_as = null, $debug) + { + $imap = new \rcube_imap_generic(); + + if ($debug || \config('app.debug')) { + $imap->setDebug(true, 'App\Backends\IMAP::logDebug'); + } + + if ($login_as) { + $config['options']['auth_cid'] = $config['user']; + $config['options']['auth_pw'] = $config['password']; + $config['options']['auth_type'] = 'PLAIN'; + $config['user'] = $login_as; + } + + $imap->connect($config['host'], $config['user'], $config['password'], $config['options']); + + if (!$imap->connected()) { + $message = sprintf("Login failed for %s against %s. %s", $config['user'], $config['host'], $imap->error); + $this->error($message); + return null; + } + + return $imap; + } + + private static function imapFlagsFromStatus($status) + { + # libpst only sets R and O + $flags = []; + if (strpos($status, 'R') !== false || strpos($status, 'O') !== false) { + $flags[] = 'SEEN'; + } + /* if 'D' in flags: */ + /* $flags[] = '\Deleted' */ + /* if 'A' in flags: */ + /* $flags[] = '\Answered' */ + /* if 'F' in flags */ + /* $flags[] = '\Flagged' */ + return $flags; + } + + private static function appendFromFile($imap, $mailbox, $path) + { + // open message file + $fp = null; + if (file_exists(realpath($path))) { + $fp = fopen($path, 'r'); + } + + if (!$fp) { + print("Failed to open for reading: " . $path); + return false; + } + + $message = fread($fp, filesize($path)); + // IMAP requires CRLF + $message = str_replace("\n", "\r\n", str_replace("\r", '', $message)); + + $flags = []; + $matches = null; + //In practice we only seem to get RO + if (preg_match("/Status: (R?O?)\r\n/", $message, $matches)) { + /* print("Found matches:\n"); */ + /* print_r($matches); */ + $flags = self::imapFlagsFromStatus($matches[1]); + $message = preg_replace("/Status: (R?O?)\r\n/", "", $message); + } + $date = null; + $binary = false; + return $imap->append($mailbox, $message, $flags, $date, $binary); + } + + private static function listDirectoryTree($dir) + { + $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir)); + $directories = []; + foreach ($it as $fileinfo) { + //We look for all "." direcotires, because we somehow can't just iterate over directories without files. + if ($fileinfo->isDir() && $fileinfo->getFilename() == "." && dirname($fileinfo->getPath()) != dirname($dir)) { + $directories[] = $fileinfo->getPath(); + } + } + sort($directories); + return $directories; + } + + + private function createTemporaryDirectory() + { + $tempfile = tempnam(sys_get_temp_dir(), ''); + if (file_exists($tempfile)) { + unlink($tempfile); + } + mkdir($tempfile); + if (!is_dir($tempfile)) { + return null; + } + return $tempfile . "/"; + } + + + private static function fixICal($ical) + { + $lines = explode("\n", $ical); + + for ($i = 0; $i < count($lines); $i++) { + $line = $lines[$i]; + //Remove nonsensical NONE categories libpst by Outlook + if ($line == "CATEGORIES:NONE") { + $lines[$i] = null; + } + } + return implode("\n", array_filter($lines)); + } + + + private static function caldavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path) + { + $fp = null; + if (file_exists(realpath($path))) { + $fp = fopen($path, 'r'); + } + + if (!$fp) { + print("Failed to open for reading: " . $path); + return false; + } + + $putdata = self::fixICal(fread($fp, filesize($path))); + + print($putdata . "\n"); + + $urlParts = parse_url($serverUrl); + $newUrl = $urlParts['scheme'] . "://" . $urlParts['host']; + if (isset($urlParts['port'])) { + $newUrl = $newUrl . ":" . $urlParts['port']; + } + $url = $newUrl . $mailbox . basename($path); + + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $url); + curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/calendar; charset='UTF-8'")); + curl_setopt($c, CURLOPT_HEADER, 0); + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_setopt($c, CURLOPT_POSTFIELDS, $putdata); + + $data = curl_exec($c); + + if (!$data) { + print("Append failed: " . curl_error($c) . "\n"); + curl_close($c); + return false; + } + + curl_close($c); + return true; + } + + + private static function fixVCard($vcard) + { + $lines = explode("\n", $vcard); + + //Generate a UID if there isn't one + if (strpos($vcard, "UID") === false) { + $uid = uniqid(); + array_splice($lines, 1, null, ["UID:{$uid}"]); + } + + return implode("\n", $lines); + } + + private function carddavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path) + { + $fp = null; + if (file_exists(realpath($path))) { + $fp = fopen($path, 'r'); + } + + if (!$fp) { + print("Failed to open for reading: " . $path); + return false; + } + + $putdata = self::fixVCard(fread($fp, filesize($path))); + + print($putdata . "\n"); + + $urlParts = parse_url($serverUrl); + $newUrl = $urlParts['scheme'] . "://" . $urlParts['host']; + if (isset($urlParts['port'])) { + $newUrl = $newUrl . ":" . $urlParts['port']; + } + $url = $newUrl . $mailbox . basename($path); + + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $url); + curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/vcard; charset='UTF-8'")); + curl_setopt($c, CURLOPT_HEADER, 0); + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_setopt($c, CURLOPT_POSTFIELDS, $putdata); + + $data = curl_exec($c); + + if (!$data) { + print("Append failed: " . curl_error($c) . "\n"); + curl_close($c); + return false; + } + curl_close($c); + return true; + } + + private static function convertToMailbox($name) + { + // Replace forbidden characters + //List based on testing Kolab's Cyrus-IMAP 2.5 (according to the roundcube codebase) + $name = str_replace(';', '_', $name); + $name = str_replace('<', '_', $name); + $name = str_replace('?', '_', $name); + $name = str_replace('\\', '_', $name); + $name = str_replace('|', '_', $name); + $name = str_replace('`', '_', $name); + $name = str_replace('!', '_', $name); + $name = str_replace('{', '_', $name); + $name = str_replace('}', '_', $name); + $name = str_replace('(', '_', $name); + $name = str_replace(')', '_', $name); + $name = str_replace('@', 'at', $name); + return mb_convert_encoding($name, "UTF7-IMAP", "UTF-8"); + } + + private function findDavMailbox($url, $user, $pw, $mailbox) + { + $xml = ''; + + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $url); + curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal")); + curl_setopt($c, CURLOPT_HEADER, 0); + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND"); + curl_setopt($c, CURLOPT_POSTFIELDS, $xml); + curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); + + $data = curl_exec($c); + if (!$data) { + $this->error("Curl propfind failed on url {$url}: " . curl_error($c) . "\n"); + curl_close($c); + return null; + } + curl_close($c); + + try { + $xml = \simplexml_load_string($data, null, 0, "d", true); + } catch (\Exception $e) { + $this->error("Exception during xml parsing: {$e->getMessage()}\n"); + $xml = null; + } + + if (!$xml) { + $this->error("Failed to parse xml from {$url}\n"); + $this->error("\n" . $data . "\n"); + + $errors = \libxml_get_errors(); + + foreach ($errors as $error) { + $this->error("Error: " . $error->message . "\n"); + } + + \libxml_clear_errors(); + return null; + } + + foreach ($xml->response as $e) { + $displayname = (string)$e->propstat->prop->displayname; + + //Convert the hierarchy separator back from " » ", " : " or "/" + //The utf-8 modifier is required for the multibyte character "»" to work within [] + $parts = preg_split('!(*UTF8)(\s*/\s*|\s+[»:]\s+)!', $displayname); + $converted = self::convertToMailbox(join('/', $parts)); + $this->info("Found {$displayname} : {$converted}"); + + if ($converted == $mailbox) { + return (string)$e->href; + } + } + + $this->error("Could not find a matching dav collection for {$mailbox}"); + $this->error("\n" . $data . "\n"); + + return null; + } + + private static function mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter) + { + // Ignore the toplevel directory which is something like: "Outlook Data File" + $mailboxParts = array_slice(explode('/', $directory), $directoryRootDepth + 1); + + //Move folders out of inbox + if (count($mailboxParts) > 1 && strcasecmp($mailboxParts[0], "inbox") == 0) { + $mailboxParts = array_slice($mailboxParts, 1); + } + + $mailboxParts = array_map( + function ($part) { + return self::convertToMailbox($part); + }, + $mailboxParts + ); + return implode($hierarchyDelimiter, $mailboxParts); + } + + private static function inBlacklist($mailbox, $mailboxBlacklist) + { + foreach ($mailboxBlacklist as $entry) { + if (fnmatch($entry, $mailbox)) { + return true; + } + } + return false; + } + + private function importPst($file, $username, $password, $clearTarget, $subscribe, $mailboxBlacklist, $mailboxWhitelist, $pickupFrom, $debug, $davUrl, $imapUrl, $typeFilter, $typeBlacklist): int + { + if (!file_exists($file)) { + //TODO downlaod file (see Utils.php) + $this->error("File doesn't exist: " . $file); + return 1; + } + + $dir = $this->createTemporaryDirectory(); + if (!$dir) { + $this->error("Failed to create a temporary directory"); + return 1; + } + + $this->info("Importing pst: " . $file); + $this->info("Using temporary directory: " . $dir); + + // Extract the pst file + if (!exec("readpst -e {$file} -o {$dir}", $output, $resultCode)) { + $this->error("Failed to extract pst data"); + return 1; + } + + $this->info("Extracted pst file: \n" . implode("\n ", $output) . "\n"); + + $outputDirectories = self::listDirectoryTree($dir); + $directoryRoot = array_shift($outputDirectories); + $directoryRootDepth = substr_count($directoryRoot, '/'); + $this->info(" Directory root: {$directoryRoot}"); + $this->info(" Depth: {$directoryRootDepth}"); + $this->info(" Directories: \n" . var_export($outputDirectories, true) . "\n"); + + // Prepare imap connection + $imap = $this->initIMAP(self::getConfig($imapUrl, $username, $password, false, false), false, $debug); + if (!$imap) { + $this->error("Failed to connect to imap."); + return 1; + } + + $this->info("Logged in to imap as user: {$username}"); + + $hierarchyDelimiter = $imap->getHierarchyDelimiter(); + $this->info("Hierarchy delimiter: {$hierarchyDelimiter}"); + + //TODO config option + $importRoot = ''; + + $totalFolders = count($outputDirectories); + $folderCounter = 0; + + $skipBeforePickupfromFolder = !empty($pickupFrom); + + foreach ($outputDirectories as $directory) { + $mailbox = $importRoot . self::mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter); + $folderCounter++; + + if ( + (empty($mailboxWhitelist) && self::inBlacklist($mailbox, $mailboxBlacklist)) || + (!empty($mailboxWhitelist) && !self::inBlacklist($mailbox, $mailboxWhitelist)) + ) { + $this->info("Skipping blacklisted mailbox: {$mailbox}"); + continue; + } + + if ($skipBeforePickupfromFolder) { + if ($mailbox == $pickupFrom) { + $this->info("Picking up from mailbox: {$mailbox}"); + $skipBeforePickupfromFolder = false; + } else { + $this->info("Skipping before pickup-from mailbox: {$mailbox}"); + continue; + } + } + + $this->info("Importing folder: {$directory} to mailbox: {$mailbox}"); + $this->info("Folder {$folderCounter} out of {$totalFolders}"); + + $type = "mail"; + $folderType = null; + + //TODO exclude folders + $files = glob("{$directory}/*.ics"); + $fileCount = count($files); + if ($fileCount) { + $type = "calendar"; + $folderType = 'event'; + } else { + $files = glob("{$directory}/*.vcf"); + $fileCount = count($files); + if ($fileCount) { + $type = "addressbook"; + $folderType = 'contact'; + } + } + + //Filter folders that match the file naming scheme (e.g. a calendar with a .ics) + $files = array_filter( + $files, + function ($file) { + return !is_dir($file); + } + ); + + $this->info("Folder type is {$type}"); + + // This can happen if there e.g. is a calendar and mail folder of the same name. + // We prefer the calendar/addressbook and ignore the mail in this case. + if ($type != "mail") { + $_files = glob("{$directory}/*.eml"); + $_fileCount = count($_files); + if ($_fileCount) { + $this->info("Warning: there are {$_fileCount} .eml files in your {$type} folder that are not imported."); + } + } + + if ($typeFilter && $typeFilter != $type) { + $this->info("Skipping due to type filter"); + continue; + } + + if ($typeBlacklist && $typeBlacklist == $type) { + $this->info("Skipping due to type blacklist"); + continue; + } + + if (!$fileCount) { + $this->info("Nothing to do"); + continue; + } + + $count = $imap->countMessages($mailbox); + if ($count === false) { + $this->info("Creating mailbox: {$mailbox}"); + //TODO set specialuse; + if (!$imap->createFolder($mailbox)) { + $this->error("Failed to create mailbox: {$mailbox}"); + return 1; + } + } elseif ($count) { + if ($clearTarget) { + $imap->clearFolder($mailbox); + } + } + + if ($folderType) { + if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => $folderType])) { + $this->error("Failed to set metadata on: {$mailbox}"); + return 1; + } + } + + if ($subscribe) { + $imap->subscribe($mailbox); + } + + if ($type == "calendar") { + $mailbox = $this->findDavMailbox($davUrl . "/calendars/{$username}/", $username, $password, $mailbox); + } elseif ($type == "addressbook") { + $mailbox = $this->findDavMailbox($davUrl . "/addressbooks/{$username}/", $username, $password, $mailbox); + } else { + if (!$imap->select($mailbox)) { + $this->error("Failed to select: {$mailbox}"); + return 1; + } + } + + if (!$mailbox) { + $this->error("Failed to get the target mailbox."); + return 1; + } + + $bar = \App\Utils::createProgressBar($this->output, $fileCount, "Importing to " . $mailbox); + $bar->advance(); + foreach ($files as $file) { + /* $this->info("Processing file: {$file}"); */ + + if ($type == "calendar") { + if (!self::caldavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) { + $this->error("Append failed: {$file}"); + $this->error("IMAP error: {$imap->error}"); + return 1; + } + } elseif ($type == "addressbook") { + if (!self::carddavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) { + $this->error("Append failed: {$file}"); + $this->error("IMAP error: {$imap->error}"); + return 1; + } + } else { + if (!self::appendFromFile($imap, $mailbox, $file)) { + $this->error("Append failed: {$file}"); + $this->error("IMAP error: {$imap->error}"); + + //Whitelist non fatal errors + if (strpos($imap->error, "Message contains invalid header") != false) { + continue; + } + + return 1; + } + } + // TODO If the work above did not involve imap (e.g. caldav import) and took too long, + // it's possible that the imap connection times out and then is invalid in closeConnection() below, + // or for another imap import. Check connection and reestablish if necessary. + $bar->advance(); + } + $bar->finish(); + } + $imap->closeConnection(); + + $this->info("Finished importing folders."); + return 0; + } +}