Page MenuHomePhorge
Files F16571223

D2413.id14124.diff
No OneTemporary
Actions

D2413.id14124.diff
View Options

diff --git a/docker/migrate/Dockerfile b/docker/migrate/Dockerfile
new file mode 100644
--- /dev/null
+++ b/docker/migrate/Dockerfile
@@ -0,0 +1,18 @@
+FROM kolab-webapp:latest
+
+USER root
+
+RUN dnf -y install \
+ --setopt=install_weak_deps=False \
+ --setopt 'tsflags=nodocs' \
+ gcc-c++ \
+ libgsf \
+ gettext \
+ libtool \
+ automake \
+ gd-devel zlib-devel boost-devel libgsf-devel gettext-devel && \
+ dnf clean all
+
+ADD build.sh /build.sh
+RUN /build.sh
+COPY /rootfs /
diff --git a/docker/migrate/Makefile b/docker/migrate/Makefile
new file mode 100644
--- /dev/null
+++ b/docker/migrate/Makefile
@@ -0,0 +1,4 @@
+build:
+ podman build . -t migrate
+shell:
+ podman run --network=host --rm -ti -v /home/mollekopf/src/kolab/docker/migrate/input:/opt/app-root/input/ -w /opt/app-root/src migrate /bin/bash
diff --git a/docker/migrate/README.md b/docker/migrate/README.md
new file mode 100644
--- /dev/null
+++ b/docker/migrate/README.md
@@ -0,0 +1,17 @@
+This is a container to run the migrate:userdata artisan command, which uses libpst to import .pst files.
+This container builds on a regular kolab4 container for the artisan framework, but adds the migrate:userdata command which is only useful together with libpst (which is built in this container).
+This container is not part of a regular kolab4 setup.
+
+
+# Execution
+
+A typical individual import could look like this:
+
+podman run --network=host --rm -ti -v /home/mollekopf/src/kolab/docker/migrate/input:/opt/app-root/input/ -w /opt/app-root/src migrate ./artisan migrate:userdata --importpst="/opt/app-root/input/$INPUT" --username="$TARGETUSER" --password="$PASSWORD" --subscribe --debug --davUrl=https://apps.kolabnow.com --imapUrl=ssl://imap.kolabnow.com:993 --clear-target
+
+# massmigrate.py
+
+This python script was used to massmigrate .zip files with .pst data.
+It will require adjustment to be used, and should probably be executed inside the container.
+
+
diff --git a/docker/migrate/build.sh b/docker/migrate/build.sh
new file mode 100755
--- /dev/null
+++ b/docker/migrate/build.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -x
+set -e
+
+# Build libpst
+git clone -b stable/kolab-0.6.76 https://git.kolab.org/source/libpst.git
+cd libpst
+autoreconf -vif
+# ./configure --enable-libpst-shared --with-boost-python=boost_python39
+./configure --enable-python=no --prefix=/usr
+# Override the configure result
+echo "#define HAVE_ICONV 1" >> config.h
+make -j5
+make install
diff --git a/docker/migrate/input/.gitkeep b/docker/migrate/input/.gitkeep
new file mode 100644
diff --git a/docker/migrate/massmigrate.py b/docker/migrate/massmigrate.py
new file mode 100755
--- /dev/null
+++ b/docker/migrate/massmigrate.py
@@ -0,0 +1,136 @@
+#!/bin/env python3
+
+# This script allows mass migrating pst files.
+# Usage ./massmigrate.py
+# --input ./input/
+# --output ./output/
+# --password $password
+# --imap ssl://$domain:993 --dav https://$domain/iRony
+#
+# It requires an input and an output directory.
+#
+# The input directory contains files to import in the format:
+# username@hostname.pst
+#
+# The output directory will receive all processed pst files.
+
+import subprocess
+import argparse
+import glob
+import shutil
+import os
+import time
+from threading import Thread
+from queue import Queue
+
+
+def transform_file(file):
+ dir_path, filename = os.path.split(os.path.realpath(file))
+
+ removepst = False
+ if '.zip' in file:
+ username = filename.replace('.zip', '')
+ parts = username.split('.')
+ username = f"{parts[1]}.{parts[2]}@{parts[0].lower()}.domain.com"
+ result = subprocess.run(f"unzip {filename}",
+ shell=True,
+ cwd=dir_path
+ )
+ if result.returncode != 0:
+ return None, None, None, None
+ removepst = True
+ filename = f"{parts[1]}.{parts[2]}.pst"
+ else:
+ username = filename.replace('.pst', '')
+
+ return dir_path, filename, username, removepst
+
+
+def import_file(file, password, imap_uri, dav_uri, type_filter, type_blacklist):
+ image = "migrate"
+ start = time.time()
+
+ dir_path, filename, username, removepst = transform_file(file)
+ if not username:
+ print(f"Failed to extract the file {file}")
+ return False
+
+ print("Starting " + username)
+
+ DOCKER_OPTIONS = ' '.join([
+ "--network=host", "--rm", "-ti",
+ "-v", f"{dir_path}:/opt/app-root/input/",
+ "-w", "/opt/app-root/src",
+ ])
+
+ cmdargs = [
+ "./artisan", "migrate:userdata",
+ f"--importpst=/opt/app-root/input/{filename}",
+ f"--username={username} --password={password}",
+ "--subscribe --debug",
+ f"--davUrl={dav_uri}",
+ f"--imapUrl={imap_uri}",
+ "--exclude-target='Sync Issues (This computer only)'",
+ "--exclude-target='Drafts (This computer only)'",
+ "--exclude-target='Sync Issues*'",
+ "--exclude-target='Recoverable Items*'",
+ ]
+
+ if type_filter:
+ cmdargs.append(f"--type-filter='{type_filter}'")
+
+ if type_blacklist:
+ cmdargs.append(f"--type-blacklist='{type_blacklist}'")
+
+ CMD = ' '.join(cmdargs)
+
+ with open(f"output/{username}.log", 'w') as logfile:
+ result = subprocess.run(f"podman run {DOCKER_OPTIONS} {image} {CMD}",
+ shell=True,
+ text=True,
+ stderr=subprocess.STDOUT,
+ stdout=logfile)
+
+ executionTime = time.time() - start
+ print(f"Finished {username} in {executionTime}s")
+ if removepst:
+ os.remove(f"{dir_path}/{filename}")
+ return not result.returncode
+ return 1
+
+
+def main():
+ parser = argparse.ArgumentParser("usage: %prog [options]")
+ parser.add_argument("--input", help="Input directory")
+ parser.add_argument("--output", help="Output directory")
+ parser.add_argument("--password", help="User password to use for all files")
+ parser.add_argument("--imap", help="IMAP URI")
+ parser.add_argument("--dav", help="DAV URI")
+ parser.add_argument("--typefilter", help="Folder type whitelist (not a list)")
+ parser.add_argument("--typeblacklist", help="Folder type blacklist (not a list)")
+ options = parser.parse_args()
+
+ input_dir = os.path.expanduser(options.input)
+ output_dir = os.path.expanduser(options.output)
+
+ q = Queue()
+ for file in glob.glob(f"{input_dir}*"):
+ q.put(file)
+
+ def worker():
+ while True:
+ file = q.get()
+ if import_file(file, options.password, options.imap, options.dav, options.typefilter, options.typeblacklist):
+ shutil.move(file, output_dir + os.path.basename(file))
+ else:
+ print(f"Failed to import {file}")
+ q.task_done()
+
+ for _i in range(3):
+ Thread(target=worker, daemon=True).start()
+ q.join()
+ print("Processing finished")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docker/migrate/output/.gitkeep b/docker/migrate/output/.gitkeep
new file mode 100644
diff --git a/docker/migrate/rootfs/init.sh b/docker/migrate/rootfs/init.sh
new file mode 100755
--- /dev/null
+++ b/docker/migrate/rootfs/init.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+set -x
+
+cd /opt/app-root/src/
+
+rsync -av \
+ --exclude=vendor \
+ --exclude=composer.lock \
+ --exclude=node_modules \
+ --exclude=package-lock.json \
+ --exclude=public \
+ --exclude=storage \
+ --exclude=resources/build \
+ --exclude=bootstrap \
+ --exclude=.gitignore \
+ --exclude=.env \
+ /src/overlay/ /opt/app-root/src/ | tee /tmp/rsync-overlay.output
+
+# We rely on the environment for configuration
+# We have to do this before running composer, because that attempts to read the .env file too.
+rm -f .env
+
+
+rm -rf storage/framework
+mkdir -p storage/framework/{sessions,views,cache}
+
+if grep -e "composer.json" -e "app" /tmp/rsync.output; then
+ rm composer.lock
+ # Must be before the first artisan command because those can fail otherwise)
+ php -dmemory_limit=-1 $(command -v composer) install
+fi
+
+find bootstrap/cache/ -type f ! -name ".gitignore" -delete
+./artisan clear-compiled
+./artisan cache:clear || :
+
+exec $@
diff --git a/docker/migrate/rootfs/opt/app-root/src/app/Console/Commands/MigrateUserdata.php b/docker/migrate/rootfs/opt/app-root/src/app/Console/Commands/MigrateUserdata.php
new file mode 100644
--- /dev/null
+++ b/docker/migrate/rootfs/opt/app-root/src/app/Console/Commands/MigrateUserdata.php
@@ -0,0 +1,736 @@
+<?php
+
+namespace App\Console\Commands;
+
+
+use Illuminate\Console\Command;
+
+
+class MigrateUserdata extends Command
+{
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'migrate:userdata
+ {--importpst= : Pst file path}
+ {--extractonly : Only extract the pst file, then exit.}
+ {--username= : Target user}
+ {--password= : Password}
+ {--davUrl= : caldav server url}
+ {--imapUrl= : imap server url}
+ {--clear-target : Remove all messages from the target mailbox.}
+ {--subscribe : Subscribe to the target mailbox.}
+ {--exclude-target=* : Blacklist target mailbox.}
+ {--include-target=* : Whitelist target mailbox.}
+ {--pickup-from= : Pick-up import from a specific target mailbox.}
+ {--type-filter= : Whitelist folder type [calendar, mail, addressbook]. Not a list atm, only single value.}
+ {--type-blacklist= : Blacklist folder type [calendar, mail, addressbook]. Not a list atm, only single value.}
+ {--debug : Enable debug output}';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Migrate userdata';
+
+ /**
+ * Execute the console command.
+ *
+ * @return mixed
+ */
+ public function handle()
+ {
+ if ($this->option('importpst')) {
+ return $this->importPst(
+ $this->option('importpst'),
+ $this->option('extractonly'),
+ $this->option('username'),
+ $this->option('password'),
+ $this->option('clear-target'),
+ $this->option('subscribe'),
+ $this->option('exclude-target'),
+ $this->option('include-target'),
+ $this->option('pickup-from'),
+ $this->option('debug'),
+ $this->option('davUrl'),
+ $this->option('imapUrl'),
+ $this->option('type-filter'),
+ $this->option('type-blacklist'),
+ );
+ }
+ }
+
+
+ /**
+ * Shortcut to creating a progress bar of a particular format with a particular message.
+ *
+ * @param \Illuminate\Console\OutputStyle $output Console output object
+ * @param int $count Number of progress steps
+ * @param string $message The description
+ *
+ * @return \Symfony\Component\Console\Helper\ProgressBar
+ */
+ private static function createProgressBar($output, $count, $message = null)
+ {
+ $bar = $output->createProgressBar($count);
+
+ $bar->setFormat(
+ '%current:7s%/%max:7s% [%bar%] %percent:3s%% %elapsed:7s%/%estimated:-7s% %message% '
+ );
+
+ if ($message) {
+ $bar->setMessage($message . " ...");
+ }
+
+ $bar->start();
+
+ return $bar;
+ }
+
+
+ /**
+ * Get LDAP configuration for specified access level
+ */
+ private static function getConfig($imapUrl, $username, $password, $verifyPeer, $verifyHost)
+ {
+ $uri = \parse_url($imapUrl ?? \config('imap.uri'));
+ $default_port = 143;
+ $ssl_mode = null;
+
+ if (isset($uri['scheme'])) {
+ if (preg_match('/^(ssl|imaps)/', $uri['scheme'])) {
+ $default_port = 993;
+ $ssl_mode = 'ssl';
+ } elseif ($uri['scheme'] === 'tls') {
+ $ssl_mode = 'tls';
+ }
+ }
+
+ $config = [
+ 'host' => $uri['host'],
+ 'user' => $username ?? \config('imap.admin_login'),
+ 'password' => $password ?? \config('imap.admin_password'),
+ 'options' => [
+ 'port' => !empty($uri['port']) ? $uri['port'] : $default_port,
+ 'ssl_mode' => $ssl_mode,
+ 'timeout' => 60,
+ //Work around guam bug
+ 'literal+' => false,
+ 'socket_options' => [
+ 'ssl' => [
+ 'verify_peer' => $verifyPeer ?? \config('imap.verify_peer'),
+ 'verify_peer_name' => $verifyPeer ?? \config('imap.verify_peer'),
+ 'verify_host' => $verifyHost ?? \config('imap.verify_host')
+ ],
+ ],
+ ],
+ ];
+
+ return $config;
+ }
+
+ /**
+ * Initialize connection to IMAP
+ */
+ private function initIMAP(array $config, string $login_as = null, $debug)
+ {
+ $imap = new \rcube_imap_generic();
+
+ if ($debug || \config('app.debug')) {
+ $imap->setDebug(true, 'App\Backends\IMAP::logDebug');
+ }
+
+ if ($login_as) {
+ $config['options']['auth_cid'] = $config['user'];
+ $config['options']['auth_pw'] = $config['password'];
+ $config['options']['auth_type'] = 'PLAIN';
+ $config['user'] = $login_as;
+ }
+
+ $imap->connect($config['host'], $config['user'], $config['password'], $config['options']);
+
+ if (!$imap->connected()) {
+ $message = sprintf("Login failed for %s against %s. %s", $config['user'], $config['host'], $imap->error);
+ $this->error($message);
+ return null;
+ }
+
+ return $imap;
+ }
+
+ private static function imapFlagsFromStatus($status)
+ {
+ # libpst only sets R and O
+ $flags = [];
+ if (strpos($status, 'R') !== false || strpos($status, 'O') !== false) {
+ $flags[] = 'SEEN';
+ }
+ /* if 'D' in flags: */
+ /* $flags[] = '\Deleted' */
+ /* if 'A' in flags: */
+ /* $flags[] = '\Answered' */
+ /* if 'F' in flags */
+ /* $flags[] = '\Flagged' */
+ return $flags;
+ }
+
+ private static function appendFromFile($imap, $mailbox, $path)
+ {
+ // open message file
+ $fp = null;
+ if (file_exists(realpath($path))) {
+ $fp = fopen($path, 'r');
+ }
+
+ if (!$fp) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+ $filesize = filesize($path);
+ if ($filesize <= 0) {
+ print("Empty file: " . $path . "\n");
+ return true;
+ }
+
+ $message = fread($fp, $filesize);
+ // IMAP requires CRLF
+ $message = str_replace("\n", "\r\n", str_replace("\r", '', $message));
+
+ $flags = [];
+ $matches = null;
+ //In practice we only seem to get RO
+ if (preg_match("/Status: (R?O?)\r\n/", $message, $matches)) {
+ /* print("Found matches:\n"); */
+ /* print_r($matches); */
+ $flags = self::imapFlagsFromStatus($matches[1]);
+ $message = preg_replace("/Status: (R?O?)\r\n/", "", $message);
+ }
+ $date = null;
+ $binary = false;
+ return $imap->append($mailbox, $message, $flags, $date, $binary);
+ }
+
+ private static function listDirectoryTree($dir)
+ {
+ $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir));
+ $directories = [];
+ foreach ($it as $fileinfo) {
+ //We look for all "." direcotires, because we somehow can't just iterate over directories without files.
+ if ($fileinfo->isDir() && $fileinfo->getFilename() == "." && dirname($fileinfo->getPath()) != dirname($dir)) {
+ $directories[] = $fileinfo->getPath();
+ }
+ }
+ sort($directories);
+ return $directories;
+ }
+
+
+ private function createTemporaryDirectory()
+ {
+ $tempfile = tempnam(sys_get_temp_dir(), '');
+ if (file_exists($tempfile)) {
+ unlink($tempfile);
+ }
+ mkdir($tempfile);
+ if (!is_dir($tempfile)) {
+ return null;
+ }
+ return $tempfile . "/";
+ }
+
+
+ private static function fixICal($ical)
+ {
+ $lines = explode("\n", $ical);
+
+ for ($i = 0; $i < count($lines); $i++) {
+ $line = $lines[$i];
+ //Remove nonsensical NONE categories libpst by Outlook
+ if ($line == "CATEGORIES:NONE") {
+ $lines[$i] = null;
+ }
+ }
+ return implode("\n", array_filter($lines));
+ }
+
+
+ private static function caldavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path)
+ {
+ $content = file_get_contents($path);
+ if (!$content) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $putdata = self::fixICal($content);
+
+ // print($putdata . "\n");
+
+ $urlParts = parse_url($serverUrl);
+ $newUrl = $urlParts['scheme'] . "://" . $urlParts['host'];
+ if (isset($urlParts['port'])) {
+ $newUrl = $newUrl . ":" . $urlParts['port'];
+ }
+ $url = $newUrl . $mailbox . basename($path);
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/calendar; charset='UTF-8'"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $putdata);
+
+ $data = curl_exec($c);
+
+ if (!$data) {
+ print("Append failed: " . curl_error($c) . "\n");
+ curl_close($c);
+ return false;
+ }
+
+ curl_close($c);
+ return true;
+ }
+
+
+ private static function fixVCard($vcard)
+ {
+ $lines = explode("\n", $vcard);
+
+ $lines = array_filter($lines, function($line) {
+ if ($line == "FN:(null)") {
+ return false;
+ }
+ if ($line == "N:;;;;") {
+ return false;
+ }
+
+ return true;
+ });
+ //Generate a UID if there isn't one
+ if (strpos($vcard, "UID") === false) {
+ $uid = uniqid();
+ array_splice($lines, 1, 0, ["UID:{$uid}"]);
+ }
+
+ return implode("\n", $lines);
+ }
+
+ private function carddavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path)
+ {
+ $content = file_get_contents($path);
+ if (!$content) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $putdata = self::fixVCard($content);
+
+ // print($putdata . "\n");
+
+ $urlParts = parse_url($serverUrl);
+ $newUrl = $urlParts['scheme'] . "://" . $urlParts['host'];
+ if (isset($urlParts['port'])) {
+ $newUrl = $newUrl . ":" . $urlParts['port'];
+ }
+ $url = $newUrl . $mailbox . basename($path);
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/vcard; charset='UTF-8'"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $putdata);
+
+ $data = curl_exec($c);
+
+ if (!$data) {
+ print("Append failed: " . curl_error($c) . "\n");
+ curl_close($c);
+ return false;
+ }
+ curl_close($c);
+ return true;
+ }
+
+ private static function convertToMailbox($name)
+ {
+ // Replace forbidden characters
+ //List based on testing Kolab's Cyrus-IMAP 2.5 (according to the roundcube codebase)
+ $name = str_replace(';', '_', $name);
+ $name = str_replace('<', '_', $name);
+ $name = str_replace('?', '_', $name);
+ $name = str_replace('\\', '_', $name);
+ $name = str_replace('|', '_', $name);
+ $name = str_replace('`', '_', $name);
+ $name = str_replace('!', '_', $name);
+ $name = str_replace('{', '_', $name);
+ $name = str_replace('}', '_', $name);
+ $name = str_replace('(', '_', $name);
+ $name = str_replace(')', '_', $name);
+ $name = str_replace('@', 'at', $name);
+ return mb_convert_encoding($name, "UTF7-IMAP", "UTF-8");
+ }
+
+ private function findDavMailbox($url, $user, $pw, $mailbox)
+ {
+ $xml = '<d:propfind xmlns:d="DAV:"><d:prop><d:resourcetype /><d:displayname /></d:prop></d:propfind>';
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $xml);
+ curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
+
+ $data = curl_exec($c);
+ if (!$data) {
+ $this->error("Curl propfind failed on url {$url}: " . curl_error($c) . "\n");
+ curl_close($c);
+ return null;
+ }
+ curl_close($c);
+
+ try {
+ $xml = \simplexml_load_string($data, null, 0, "d", true);
+ } catch (\Exception $e) {
+ $this->error("Exception during xml parsing: {$e->getMessage()}\n");
+ $xml = null;
+ }
+
+ if (!$xml) {
+ $this->error("Failed to parse xml from {$url}\n");
+ $this->error("\n" . $data . "\n");
+
+ $errors = \libxml_get_errors();
+
+ foreach ($errors as $error) {
+ $this->error("Error: " . $error->message . "\n");
+ }
+
+ \libxml_clear_errors();
+ return null;
+ }
+
+ foreach ($xml->response as $e) {
+ $displayname = (string)$e->propstat->prop->displayname;
+
+ //Convert the hierarchy separator back from " » ", " : " or "/"
+ //The utf-8 modifier is required for the multibyte character "»" to work within []
+ $parts = preg_split('!(*UTF8)(\s*/\s*|\s+[»:]\s+)!', $displayname);
+ $converted = self::convertToMailbox(join('/', $parts));
+ $this->info("Found {$displayname} : {$converted}");
+
+ if ($converted == $mailbox) {
+ return (string)$e->href;
+ }
+ }
+
+ $this->error("Could not find a matching dav collection for {$mailbox}");
+ $this->error("\n" . $data . "\n");
+
+ return null;
+ }
+
+ private static function mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter)
+ {
+ // Ignore the toplevel directory which is something like: "Outlook Data File"
+ $mailboxParts = array_slice(explode('/', $directory), $directoryRootDepth + 1);
+
+ //Move folders out of inbox
+ if (count($mailboxParts) > 1 && strcasecmp($mailboxParts[0], "inbox") == 0) {
+ $mailboxParts = array_slice($mailboxParts, 1);
+ }
+
+ $mailboxParts = array_map(
+ function ($part) {
+ return self::convertToMailbox($part);
+ },
+ $mailboxParts
+ );
+ return implode($hierarchyDelimiter, $mailboxParts);
+ }
+
+ private static function inBlacklist($mailbox, $mailboxBlacklist)
+ {
+ foreach ($mailboxBlacklist as $entry) {
+ if (fnmatch($entry, $mailbox)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private function getFolderType($type)
+ {
+ if ($type == "addressbook") {
+ return 'contact';
+ }
+ if ($type == "calendar") {
+ return 'event';
+ }
+ return null;
+ }
+
+
+ private function createFolder($imap, $mailbox, $type, $clearTarget, $subscribe, $davUrl, $username, $password)
+ {
+ $count = $imap->countMessages($mailbox);
+ if ($count === false) {
+ $this->info("Creating mailbox: {$mailbox}");
+ //TODO set specialuse;
+ if (!$imap->createFolder($mailbox)) {
+ $this->error("Failed to create mailbox: {$mailbox}");
+ return null;
+ }
+ } elseif ($count) {
+ if ($clearTarget) {
+ $this->info("Clearing mailbox: {$mailbox}");
+ if (!$imap->clearFolder($mailbox)) {
+ $this->error("Error while clearing mailbox: {$mailbox}");
+ }
+ }
+ }
+
+ if ($folderType = $this->getFolderType($type)) {
+ $this->info("Setting folder type annotation: {$folderType}");
+ if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => $folderType])) {
+ $this->error("Failed to set metadata on: {$mailbox}");
+ return null;
+ }
+ }
+
+ if ($subscribe) {
+ $this->info("Subscribing to folder");
+ $imap->subscribe($mailbox);
+ }
+
+ $davPrefix = null;
+ if ($type == "calendar") {
+ $davPrefix = "calendars";
+ } elseif ($type == "addressbook") {
+ $davPrefix = "addressbooks";
+ }
+ $retries = 0;
+ while ($retries < 3) {
+ if ($davPrefix) {
+ if ($res = $this->findDavMailbox($davUrl . "/{$davPrefix}/{$username}/", $username, $password, $mailbox)) {
+ return $res;
+ }
+ } else {
+ if ($imap->select($mailbox)) {
+ return $mailbox;
+ }
+ }
+ sleep(5);
+ $retries++;
+ }
+ return null;
+ }
+
+
+ private function importPst($file, $extractOnly, $username, $password, $clearTarget, $subscribe, $mailboxBlacklist, $mailboxWhitelist, $pickupFrom, $debug, $davUrl, $imapUrl, $typeFilter, $typeBlacklist): int
+ {
+ if (!file_exists($file)) {
+ //TODO downlaod file (see Utils.php)
+ $this->error("File doesn't exist: " . $file);
+ return 1;
+ }
+
+ $dir = $this->createTemporaryDirectory();
+ if (!$dir) {
+ $this->error("Failed to create a temporary directory");
+ return 1;
+ }
+
+ $this->info("Importing pst: " . $file);
+ $this->info("Using temporary directory: " . $dir);
+
+ $debugArg = "";
+ if ($debug) {
+ $debugArg = "-d pstdebugoutput.txt -L 2";
+ }
+
+ // Extract the pst file
+ if (!exec("readpst $debugArg -e {$file} -o {$dir}", $output, $resultCode)) {
+ $this->error("Failed to extract pst data");
+ return 1;
+ }
+
+ $this->info("Extracted pst file: \n" . implode("\n ", $output) . "\n");
+
+ $outputDirectories = self::listDirectoryTree($dir);
+ $directoryRoot = array_shift($outputDirectories);
+ $directoryRootDepth = substr_count($directoryRoot, '/');
+ $this->info(" Directory root: {$directoryRoot}");
+ $this->info(" Depth: {$directoryRootDepth}");
+ $this->info(" Directories: \n" . var_export($outputDirectories, true) . "\n");
+
+ if ($extractOnly) {
+ return 0;
+ }
+
+ // Prepare imap connection
+ $imap = $this->initIMAP(self::getConfig($imapUrl, $username, $password, false, false), false, $debug);
+ if (!$imap) {
+ $this->error("Failed to connect to imap.");
+ return 1;
+ }
+
+ $this->info("Logged in to imap as user: {$username}");
+
+ $hierarchyDelimiter = $imap->getHierarchyDelimiter();
+ $this->info("Hierarchy delimiter: {$hierarchyDelimiter}");
+
+ //TODO config option
+ $importRoot = '';
+
+ $totalFolders = count($outputDirectories);
+ $folderCounter = 0;
+
+ $skipBeforePickupfromFolder = !empty($pickupFrom);
+
+ foreach ($outputDirectories as $directory) {
+ $this->info("");
+ $mailbox = $importRoot . self::mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter);
+ $folderCounter++;
+
+ if (
+ (empty($mailboxWhitelist) && self::inBlacklist($mailbox, $mailboxBlacklist)) ||
+ (!empty($mailboxWhitelist) && !self::inBlacklist($mailbox, $mailboxWhitelist))
+ ) {
+ $this->info("Skipping blacklisted mailbox: {$mailbox}");
+ continue;
+ }
+
+ if ($skipBeforePickupfromFolder) {
+ if ($mailbox == $pickupFrom) {
+ $this->info("Picking up from mailbox: {$mailbox}");
+ $skipBeforePickupfromFolder = false;
+ } else {
+ $this->info("Skipping before pickup-from mailbox: {$mailbox}");
+ continue;
+ }
+ }
+
+ $this->info("Importing folder: {$directory} to mailbox: {$mailbox}");
+ $this->info("Folder {$folderCounter} out of {$totalFolders}");
+
+ $type = "mail";
+
+ //TODO exclude folders
+ $files = glob("{$directory}/*.ics");
+ $fileCount = count($files);
+ if ($fileCount) {
+ $type = "calendar";
+ } else {
+ $files = glob("{$directory}/*.vcf");
+ $fileCount = count($files);
+ if ($fileCount) {
+ $type = "addressbook";
+ }
+ }
+
+ //Filter folders that match the file naming scheme (e.g. a calendar with a .ics)
+ $files = array_filter(
+ $files,
+ function ($file) {
+ return !is_dir($file);
+ }
+ );
+
+ $this->info("Folder type is {$type}");
+
+ // This can happen if there e.g. is a calendar and mail folder of the same name.
+ // We prefer the calendar/addressbook and ignore the mail in this case.
+ if ($type != "mail") {
+ $_files = glob("{$directory}/*.eml");
+ $_fileCount = count($_files);
+ if ($_fileCount) {
+ $this->info("Warning: there are {$_fileCount} .eml files in your {$type} folder that are not imported.");
+ }
+ } else {
+ $files = glob("{$directory}/*.eml");
+ $fileCount = count($files);
+ }
+
+ if ($typeFilter && $typeFilter != $type) {
+ $this->info("Skipping due to type filter");
+ continue;
+ }
+
+ if ($typeBlacklist && $typeBlacklist == $type) {
+ $this->info("Skipping due to type blacklist");
+ continue;
+ }
+
+ if (!$fileCount) {
+ $this->info("Nothing to do");
+ continue;
+ }
+
+ $mailbox = $this->createFolder($imap, $mailbox, $type, $clearTarget, $subscribe, $davUrl, $username, $password);
+ if (!$mailbox) {
+ $this->error("Failed to get the target mailbox.");
+ return 1;
+ }
+
+ $bar = self::createProgressBar($this->output, $fileCount, "Importing to " . $mailbox);
+ $bar->advance();
+ foreach ($files as $file) {
+ /* $this->info("Processing file: {$file}"); */
+
+ if ($type == "calendar") {
+ if (!self::caldavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+ return 1;
+ }
+ } elseif ($type == "addressbook") {
+ if (!self::carddavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+ return 1;
+ }
+ } else {
+ if (!self::appendFromFile($imap, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+
+ //Whitelist non fatal errors
+ if (strpos($imap->error, "Message contains invalid header") != false) {
+ continue;
+ }
+
+ return 1;
+ }
+ }
+ // TODO If the work above did not involve imap (e.g. caldav import) and took too long,
+ // it's possible that the imap connection times out and then is invalid in closeConnection() below,
+ // or for another imap import. Check connection and reestablish if necessary.
+ $bar->advance();
+ }
+ $bar->finish();
+ }
+ $imap->closeConnection();
+
+ $this->info("Finished importing folders.");
+ return 0;
+ }
+}

File Metadata

Mime Type
text/plain
Expires
Thu, Oct 31, 10:03 AM (21 h, 29 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
10076313
Default Alt Text
D2413.id14124.diff (34 KB)

Event Timeline