Page MenuHomePhorge
Files F16571286

No OneTemporary
Actions

View Options

diff --git a/docker/migrate/Dockerfile b/docker/migrate/Dockerfile
new file mode 100644
index 00000000..4ab76426
--- /dev/null
+++ b/docker/migrate/Dockerfile
@@ -0,0 +1,42 @@
+FROM fedora:33
+
+MAINTAINER Jeroen van Meeuwen <vanmeeuwen@apheleia-it.ch>
+
+ENV HOME=/opt/app-root/src
+
+RUN dnf -y install \
+ composer \
+ file \
+ git \
+ make \
+ npm \
+ openssl-devel \
+ php-cli \
+ php-common \
+ php-ldap \
+ php-opcache \
+ php-pecl-apcu \
+ php-mysqlnd \
+ wget \
+ sudo \
+ libgsf \
+ gettext \
+ libtool \
+ gcc-c++ \
+ shtool \
+ automake \
+ gd-devel zlib-devel boost-devel libgsf-devel gettext-devel \
+ python3 python3-devel boost-python3 boost-python3-devel
+
+
+RUN id default || (groupadd -g 1001 default && useradd -d /opt/app-root/ -u 1001 -g 1001 default)
+RUN usermod -aG wheel default
+RUN echo '%wheel ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+RUN mkdir ${HOME} && chown 1001:1001 ${HOME}
+
+USER 1001
+
+WORKDIR ${HOME}
+
+COPY /rootfs /
diff --git a/docker/migrate/README.md b/docker/migrate/README.md
new file mode 100644
index 00000000..84a93b82
--- /dev/null
+++ b/docker/migrate/README.md
@@ -0,0 +1,6 @@
+This is a docker container to run the migrate:userdata artisan command.
+
+The python script provided allows mass migrating pst files.
+
+
+
diff --git a/docker/migrate/do-it-again.sh b/docker/migrate/do-it-again.sh
new file mode 100755
index 00000000..c571cfb5
--- /dev/null
+++ b/docker/migrate/do-it-again.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+export APP_DEBUG="true"
+export APP_KEY=
+export APP_PUBLIC_URL=http://127.0.0.1:8000/
+export APP_SRC=src/
+export APP_URL=http://127.0.0.1:8000/
+export CACHE_DRIVER="array"
+export COMPOSER_ARGS="--no-dev"
+export DB_CONNECTION="sqlite"
+export DB_DATABASE=":memory:"
+export GIT_URI=https://git.kolab.org/source/kolab.git
+export GIT_BRANCH=dev/pstimport
+export LARAVEL_ENV=production
+export LOG_CHANNEL="stderr"
+export MAIL_DRIVER="array"
+export QUEUE_CONNECTION="sync"
+export SESSION_DRIVER="array"
+
+docker build -t migrate .
+
+docker kill migrate
+
+docker rm migrate
+
+docker_opts="\
+ -e APP_DEBUG=${APP_DEBUG} \
+ -e APP_KEY=${APP_KEY} \
+ -e APP_PUBLIC_URL=${APP_PUBLIC_URL} \
+ -e APP_SRC=${APP_SRC} \
+ -e APP_URL=${APP_URL} \
+ -e CACHE_DRIVER=${CACHE_DRIVER} \
+ -e COMPOSER_ARGS=${COMPOSER_ARGS} \
+ -e DB_CONNECTION=${DB_CONNECTION} \
+ -e DB_DATABASE=${DB_DATABASE} \
+ -e GIT_URI=${GIT_URI} \
+ -e GIT_BRANCH=${GIT_BRANCH} \
+ -e LARAVEL_ENV=${LARAVEL_ENV} \
+ -e LOG_CHANNEL=${LOG_CHANNEL} \
+ -e MAIL_DRIVER=${MAIL_DRIVER} \
+ -e QUEUE_CONNECTION=${QUEUE_CONNECTION} \
+ -e SESSION_DRIVER=${SESSION_DRIVER}"
+
+docker run -it \
+ ${docker_opts} \
+ --name migrate migrate /usr/local/bin/build-image
+
+docker commit migrate migrate-s2i
diff --git a/docker/migrate/input/john@kolab.org.pst b/docker/migrate/input/john@kolab.org.pst
new file mode 100644
index 00000000..e69de29b
diff --git a/docker/migrate/massmigrate.py b/docker/migrate/massmigrate.py
new file mode 100755
index 00000000..b0fc93c2
--- /dev/null
+++ b/docker/migrate/massmigrate.py
@@ -0,0 +1,136 @@
+#!/bin/env python3
+
+# This script allows mass migrating pst files.
+# Usage ./massmigrate.py
+# --input ./input/
+# --output ./output/
+# --password $password
+# --imap ssl://$domain:993 --dav https://$domain/iRony
+#
+# It requires an input and an output directory.
+#
+# The input directory contains files to import in the format:
+# username@hostname.pst
+#
+# The output directory will receive all processed pst files.
+
+import subprocess
+import argparse
+import glob
+import shutil
+import os
+import time
+from threading import Thread
+from queue import Queue
+
+
+def transform_file(file):
+ dir_path, filename = os.path.split(os.path.realpath(file))
+
+ removepst = False
+ if '.zip' in file:
+ username = filename.replace('.zip', '')
+ parts = username.split('.')
+ username = f"{parts[1]}.{parts[2]}@{parts[0].lower()}.domain.com"
+ result = subprocess.run(f"unzip {filename}",
+ shell=True,
+ cwd=dir_path
+ )
+ if result.returncode != 0:
+ return None, None, None, None
+ removepst = True
+ filename = f"{parts[1]}.{parts[2]}.pst"
+ else:
+ username = filename.replace('.pst', '')
+
+ return dir_path, filename, username, removepst
+
+
+def import_file(file, password, imap_uri, dav_uri, type_filter, type_blacklist):
+ image = "migrate-s2i"
+ start = time.time()
+
+ dir_path, filename, username, removepst = transform_file(file)
+ if not username:
+ print(f"Failed to extract the file {file}")
+ return False
+
+ print("Starting " + username)
+
+ DOCKER_OPTIONS = ' '.join([
+ "--network=host", "--rm", "-ti",
+ "-v", f"{dir_path}:/opt/app-root/input/",
+ "-w", "/opt/app-root/src/kolab/src",
+ ])
+
+ cmdargs = [
+ "./artisan", "migrate:userdata",
+ f"--importpst=/opt/app-root/input/{filename}",
+ f"--username={username} --password={password}",
+ "--subscribe --debug",
+ f"--davUrl={dav_uri}",
+ f"--imapUrl={imap_uri}",
+ "--exclude-target='Sync Issues (This computer only)'",
+ "--exclude-target='Drafts (This computer only)'",
+ "--exclude-target='Sync Issues*'",
+ "--exclude-target='Recoverable Items*'",
+ ]
+
+ if type_filter:
+ cmdargs.append(f"--type-filter='{type_filter}'")
+
+ if type_blacklist:
+ cmdargs.append(f"--type-blacklist='{type_blacklist}'")
+
+ CMD = ' '.join(cmdargs)
+
+ with open(f"output/{username}.log", 'w') as logfile:
+ result = subprocess.run(f"docker run {DOCKER_OPTIONS} {image} {CMD}",
+ shell=True,
+ text=True,
+ stderr=subprocess.STDOUT,
+ stdout=logfile)
+
+ executionTime = time.time() - start
+ print(f"Finished {username} in {executionTime}s")
+ if removepst:
+ os.remove(f"{dir_path}/{filename}")
+ return not result.returncode
+ return 1
+
+
+def main():
+ parser = argparse.ArgumentParser("usage: %prog [options]")
+ parser.add_argument("--input", help="Input directory")
+ parser.add_argument("--output", help="Output directory")
+ parser.add_argument("--password", help="User password to use for all files")
+ parser.add_argument("--imap", help="IMAP URI")
+ parser.add_argument("--dav", help="DAV URI")
+ parser.add_argument("--typefilter", help="Folder type whitelist (not a list)")
+ parser.add_argument("--typeblacklist", help="Folder type blacklist (not a list)")
+ options = parser.parse_args()
+
+ input_dir = os.path.expanduser(options.input)
+ output_dir = os.path.expanduser(options.output)
+
+ q = Queue()
+ for file in glob.glob(f"{input_dir}*"):
+ q.put(file)
+
+ def worker():
+ while True:
+ file = q.get()
+ if import_file(file, options.password, options.imap, options.dav, options.typefilter, options.typeblacklist):
+ shutil.move(file, output_dir + os.path.basename(file))
+ else:
+ print(f"Failed to import {file}")
+ q.task_done()
+
+ for _i in range(3):
+ Thread(target=worker, daemon=True).start()
+ q.join()
+ print("Processing finished")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docker/migrate/rootfs/usr/local/bin/build-image b/docker/migrate/rootfs/usr/local/bin/build-image
new file mode 100755
index 00000000..70b7ba8d
--- /dev/null
+++ b/docker/migrate/rootfs/usr/local/bin/build-image
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -x
+
+set -e
+
+pwd
+
+if [ -z "${GIT_URI}" ]; then
+ echo "No GIT_URI specified. Exiting."
+ exit 1
+fi
+
+git clone -b ${GIT_BRANCH:-master} ${GIT_URI}
+
+cd $(basename ${GIT_URI} .git)
+
+if [ ! -z "${APP_SRC}" ]; then
+ cd ${APP_SRC}
+fi
+
+if [ -f "composer.json" ]; then
+ echo "Detected composer.json, running install"
+ php -dmemory_limit=${COMPOSER_MEMORY_LIMIT:--1} /usr/bin/composer install ${COMPOSER_ARGS}
+ rm -rf ~/.cache/composer/
+fi
+
+if [ -z "${LARAVEL_ENV}" ]; then
+ LARAVEL_ENV=prod
+fi
+
+npm install
+
+npm run ${LARAVEL_ENV} && rm -rf ~/.npm/
+
+cd -
+
+# Build libpst
+git clone -b stable/kolab-0.6.76 https://git.kolab.org/source/libpst.git
+cd libpst
+autoreconf -vif
+./configure --enable-libpst-shared --with-boost-python=boost_python39
+make -j5
+sudo make install
diff --git a/src/app/Console/Commands/MigrateUserdata.php b/src/app/Console/Commands/MigrateUserdata.php
new file mode 100644
index 00000000..fd90754a
--- /dev/null
+++ b/src/app/Console/Commands/MigrateUserdata.php
@@ -0,0 +1,657 @@
+<?php
+
+namespace App\Console\Commands;
+
+
+use Illuminate\Console\Command;
+
+
+class MigrateUserdata extends Command
+{
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'migrate:userdata
+ {--importpst= : Pst file path}
+ {--username= : Target user}
+ {--password= : Password}
+ {--davUrl= : caldav server url}
+ {--imapUrl= : imap server url}
+ {--clear-target : Remove all messages from the target mailbox.}
+ {--subscribe : Subscribe to the target mailbox.}
+ {--exclude-target=* : Blacklist target mailbox.}
+ {--include-target=* : Whitelist target mailbox.}
+ {--pickup-from= : Pick-up import from a specific target mailbox.}
+ {--type-filter= : Whitelist folder type [calendar, mail, addressbook]. Not a list atm, only single value.}
+ {--type-blacklist= : Blacklist folder type [calendar, mail, addressbook]. Not a list atm, only single value.}
+ {--debug : Enable debug output}';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Migrate userdata';
+
+ /**
+ * Execute the console command.
+ *
+ * @return mixed
+ */
+ public function handle()
+ {
+ if ($this->option('importpst')) {
+ return $this->importPst(
+ $this->option('importpst'),
+ $this->option('username'),
+ $this->option('password'),
+ $this->option('clear-target'),
+ $this->option('subscribe'),
+ $this->option('exclude-target'),
+ $this->option('include-target'),
+ $this->option('pickup-from'),
+ $this->option('debug'),
+ $this->option('davUrl'),
+ $this->option('imapUrl'),
+ $this->option('type-filter'),
+ $this->option('type-blacklist'),
+ );
+ }
+ }
+
+
+ /**
+ * Get LDAP configuration for specified access level
+ */
+ private static function getConfig($imapUrl, $username, $password, $verifyPeer, $verifyHost)
+ {
+ $uri = \parse_url($imapUrl ?? \config('imap.uri'));
+ $default_port = 143;
+ $ssl_mode = null;
+
+ if (isset($uri['scheme'])) {
+ if (preg_match('/^(ssl|imaps)/', $uri['scheme'])) {
+ $default_port = 993;
+ $ssl_mode = 'ssl';
+ } elseif ($uri['scheme'] === 'tls') {
+ $ssl_mode = 'tls';
+ }
+ }
+
+ $config = [
+ 'host' => $uri['host'],
+ 'user' => $username ?? \config('imap.admin_login'),
+ 'password' => $password ?? \config('imap.admin_password'),
+ 'options' => [
+ 'port' => !empty($uri['port']) ? $uri['port'] : $default_port,
+ 'ssl_mode' => $ssl_mode,
+ 'timeout' => 60,
+ //Work around guam bug
+ 'literal+' => false,
+ 'socket_options' => [
+ 'ssl' => [
+ 'verify_peer' => $verifyPeer ?? \config('imap.verify_peer'),
+ 'verify_peer_name' => $verifyPeer ?? \config('imap.verify_peer'),
+ 'verify_host' => $verifyHost ?? \config('imap.verify_host')
+ ],
+ ],
+ ],
+ ];
+
+ return $config;
+ }
+
+ /**
+ * Initialize connection to IMAP
+ */
+ private function initIMAP(array $config, string $login_as = null, $debug)
+ {
+ $imap = new \rcube_imap_generic();
+
+ if ($debug || \config('app.debug')) {
+ $imap->setDebug(true, 'App\Backends\IMAP::logDebug');
+ }
+
+ if ($login_as) {
+ $config['options']['auth_cid'] = $config['user'];
+ $config['options']['auth_pw'] = $config['password'];
+ $config['options']['auth_type'] = 'PLAIN';
+ $config['user'] = $login_as;
+ }
+
+ $imap->connect($config['host'], $config['user'], $config['password'], $config['options']);
+
+ if (!$imap->connected()) {
+ $message = sprintf("Login failed for %s against %s. %s", $config['user'], $config['host'], $imap->error);
+ $this->error($message);
+ return null;
+ }
+
+ return $imap;
+ }
+
+ private static function imapFlagsFromStatus($status)
+ {
+ # libpst only sets R and O
+ $flags = [];
+ if (strpos($status, 'R') !== false || strpos($status, 'O') !== false) {
+ $flags[] = 'SEEN';
+ }
+ /* if 'D' in flags: */
+ /* $flags[] = '\Deleted' */
+ /* if 'A' in flags: */
+ /* $flags[] = '\Answered' */
+ /* if 'F' in flags */
+ /* $flags[] = '\Flagged' */
+ return $flags;
+ }
+
+ private static function appendFromFile($imap, $mailbox, $path)
+ {
+ // open message file
+ $fp = null;
+ if (file_exists(realpath($path))) {
+ $fp = fopen($path, 'r');
+ }
+
+ if (!$fp) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $message = fread($fp, filesize($path));
+ // IMAP requires CRLF
+ $message = str_replace("\n", "\r\n", str_replace("\r", '', $message));
+
+ $flags = [];
+ $matches = null;
+ //In practice we only seem to get RO
+ if (preg_match("/Status: (R?O?)\r\n/", $message, $matches)) {
+ /* print("Found matches:\n"); */
+ /* print_r($matches); */
+ $flags = self::imapFlagsFromStatus($matches[1]);
+ $message = preg_replace("/Status: (R?O?)\r\n/", "", $message);
+ }
+ $date = null;
+ $binary = false;
+ return $imap->append($mailbox, $message, $flags, $date, $binary);
+ }
+
+ private static function listDirectoryTree($dir)
+ {
+ $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir));
+ $directories = [];
+ foreach ($it as $fileinfo) {
+ //We look for all "." direcotires, because we somehow can't just iterate over directories without files.
+ if ($fileinfo->isDir() && $fileinfo->getFilename() == "." && dirname($fileinfo->getPath()) != dirname($dir)) {
+ $directories[] = $fileinfo->getPath();
+ }
+ }
+ sort($directories);
+ return $directories;
+ }
+
+
+ private function createTemporaryDirectory()
+ {
+ $tempfile = tempnam(sys_get_temp_dir(), '');
+ if (file_exists($tempfile)) {
+ unlink($tempfile);
+ }
+ mkdir($tempfile);
+ if (!is_dir($tempfile)) {
+ return null;
+ }
+ return $tempfile . "/";
+ }
+
+
+ private static function fixICal($ical)
+ {
+ $lines = explode("\n", $ical);
+
+ for ($i = 0; $i < count($lines); $i++) {
+ $line = $lines[$i];
+ //Remove nonsensical NONE categories libpst by Outlook
+ if ($line == "CATEGORIES:NONE") {
+ $lines[$i] = null;
+ }
+ }
+ return implode("\n", array_filter($lines));
+ }
+
+
+ private static function caldavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path)
+ {
+ $fp = null;
+ if (file_exists(realpath($path))) {
+ $fp = fopen($path, 'r');
+ }
+
+ if (!$fp) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $putdata = self::fixICal(fread($fp, filesize($path)));
+
+ print($putdata . "\n");
+
+ $urlParts = parse_url($serverUrl);
+ $newUrl = $urlParts['scheme'] . "://" . $urlParts['host'];
+ if (isset($urlParts['port'])) {
+ $newUrl = $newUrl . ":" . $urlParts['port'];
+ }
+ $url = $newUrl . $mailbox . basename($path);
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/calendar; charset='UTF-8'"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $putdata);
+
+ $data = curl_exec($c);
+
+ if (!$data) {
+ print("Append failed: " . curl_error($c) . "\n");
+ curl_close($c);
+ return false;
+ }
+
+ curl_close($c);
+ return true;
+ }
+
+
+ private static function fixVCard($vcard)
+ {
+ $lines = explode("\n", $vcard);
+
+ //Generate a UID if there isn't one
+ if (strpos($vcard, "UID") === false) {
+ $uid = uniqid();
+ array_splice($lines, 1, null, ["UID:{$uid}"]);
+ }
+
+ return implode("\n", $lines);
+ }
+
+ private function carddavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path)
+ {
+ $fp = null;
+ if (file_exists(realpath($path))) {
+ $fp = fopen($path, 'r');
+ }
+
+ if (!$fp) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $putdata = self::fixVCard(fread($fp, filesize($path)));
+
+ print($putdata . "\n");
+
+ $urlParts = parse_url($serverUrl);
+ $newUrl = $urlParts['scheme'] . "://" . $urlParts['host'];
+ if (isset($urlParts['port'])) {
+ $newUrl = $newUrl . ":" . $urlParts['port'];
+ }
+ $url = $newUrl . $mailbox . basename($path);
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/vcard; charset='UTF-8'"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $putdata);
+
+ $data = curl_exec($c);
+
+ if (!$data) {
+ print("Append failed: " . curl_error($c) . "\n");
+ curl_close($c);
+ return false;
+ }
+ curl_close($c);
+ return true;
+ }
+
+ private static function convertToMailbox($name)
+ {
+ // Replace forbidden characters
+ //List based on testing Kolab's Cyrus-IMAP 2.5 (according to the roundcube codebase)
+ $name = str_replace(';', '_', $name);
+ $name = str_replace('<', '_', $name);
+ $name = str_replace('?', '_', $name);
+ $name = str_replace('\\', '_', $name);
+ $name = str_replace('|', '_', $name);
+ $name = str_replace('`', '_', $name);
+ $name = str_replace('!', '_', $name);
+ $name = str_replace('{', '_', $name);
+ $name = str_replace('}', '_', $name);
+ $name = str_replace('(', '_', $name);
+ $name = str_replace(')', '_', $name);
+ $name = str_replace('@', 'at', $name);
+ return mb_convert_encoding($name, "UTF7-IMAP", "UTF-8");
+ }
+
+ private function findDavMailbox($url, $user, $pw, $mailbox)
+ {
+ $xml = '<d:propfind xmlns:d="DAV:"><d:prop><d:resourcetype /><d:displayname /></d:prop></d:propfind>';
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $xml);
+ curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
+
+ $data = curl_exec($c);
+ if (!$data) {
+ $this->error("Curl propfind failed on url {$url}: " . curl_error($c) . "\n");
+ curl_close($c);
+ return null;
+ }
+ curl_close($c);
+
+ try {
+ $xml = \simplexml_load_string($data, null, 0, "d", true);
+ } catch (\Exception $e) {
+ $this->error("Exception during xml parsing: {$e->getMessage()}\n");
+ $xml = null;
+ }
+
+ if (!$xml) {
+ $this->error("Failed to parse xml from {$url}\n");
+ $this->error("\n" . $data . "\n");
+
+ $errors = \libxml_get_errors();
+
+ foreach ($errors as $error) {
+ $this->error("Error: " . $error->message . "\n");
+ }
+
+ \libxml_clear_errors();
+ return null;
+ }
+
+ foreach ($xml->response as $e) {
+ $displayname = (string)$e->propstat->prop->displayname;
+
+ //Convert the hierarchy separator back from " » ", " : " or "/"
+ //The utf-8 modifier is required for the multibyte character "»" to work within []
+ $parts = preg_split('!(*UTF8)(\s*/\s*|\s+[»:]\s+)!', $displayname);
+ $converted = self::convertToMailbox(join('/', $parts));
+ $this->info("Found {$displayname} : {$converted}");
+
+ if ($converted == $mailbox) {
+ return (string)$e->href;
+ }
+ }
+
+ $this->error("Could not find a matching dav collection for {$mailbox}");
+ $this->error("\n" . $data . "\n");
+
+ return null;
+ }
+
+ private static function mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter)
+ {
+ // Ignore the toplevel directory which is something like: "Outlook Data File"
+ $mailboxParts = array_slice(explode('/', $directory), $directoryRootDepth + 1);
+
+ //Move folders out of inbox
+ if (count($mailboxParts) > 1 && strcasecmp($mailboxParts[0], "inbox") == 0) {
+ $mailboxParts = array_slice($mailboxParts, 1);
+ }
+
+ $mailboxParts = array_map(
+ function ($part) {
+ return self::convertToMailbox($part);
+ },
+ $mailboxParts
+ );
+ return implode($hierarchyDelimiter, $mailboxParts);
+ }
+
+ private static function inBlacklist($mailbox, $mailboxBlacklist)
+ {
+ foreach ($mailboxBlacklist as $entry) {
+ if (fnmatch($entry, $mailbox)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private function importPst($file, $username, $password, $clearTarget, $subscribe, $mailboxBlacklist, $mailboxWhitelist, $pickupFrom, $debug, $davUrl, $imapUrl, $typeFilter, $typeBlacklist): int
+ {
+ if (!file_exists($file)) {
+ //TODO downlaod file (see Utils.php)
+ $this->error("File doesn't exist: " . $file);
+ return 1;
+ }
+
+ $dir = $this->createTemporaryDirectory();
+ if (!$dir) {
+ $this->error("Failed to create a temporary directory");
+ return 1;
+ }
+
+ $this->info("Importing pst: " . $file);
+ $this->info("Using temporary directory: " . $dir);
+
+ // Extract the pst file
+ if (!exec("readpst -e {$file} -o {$dir}", $output, $resultCode)) {
+ $this->error("Failed to extract pst data");
+ return 1;
+ }
+
+ $this->info("Extracted pst file: \n" . implode("\n ", $output) . "\n");
+
+ $outputDirectories = self::listDirectoryTree($dir);
+ $directoryRoot = array_shift($outputDirectories);
+ $directoryRootDepth = substr_count($directoryRoot, '/');
+ $this->info(" Directory root: {$directoryRoot}");
+ $this->info(" Depth: {$directoryRootDepth}");
+ $this->info(" Directories: \n" . var_export($outputDirectories, true) . "\n");
+
+ // Prepare imap connection
+ $imap = $this->initIMAP(self::getConfig($imapUrl, $username, $password, false, false), false, $debug);
+ if (!$imap) {
+ $this->error("Failed to connect to imap.");
+ return 1;
+ }
+
+ $this->info("Logged in to imap as user: {$username}");
+
+ $hierarchyDelimiter = $imap->getHierarchyDelimiter();
+ $this->info("Hierarchy delimiter: {$hierarchyDelimiter}");
+
+ //TODO config option
+ $importRoot = '';
+
+ $totalFolders = count($outputDirectories);
+ $folderCounter = 0;
+
+ $skipBeforePickupfromFolder = !empty($pickupFrom);
+
+ foreach ($outputDirectories as $directory) {
+ $mailbox = $importRoot . self::mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter);
+ $folderCounter++;
+
+ if (
+ (empty($mailboxWhitelist) && self::inBlacklist($mailbox, $mailboxBlacklist)) ||
+ (!empty($mailboxWhitelist) && !self::inBlacklist($mailbox, $mailboxWhitelist))
+ ) {
+ $this->info("Skipping blacklisted mailbox: {$mailbox}");
+ continue;
+ }
+
+ if ($skipBeforePickupfromFolder) {
+ if ($mailbox == $pickupFrom) {
+ $this->info("Picking up from mailbox: {$mailbox}");
+ $skipBeforePickupfromFolder = false;
+ } else {
+ $this->info("Skipping before pickup-from mailbox: {$mailbox}");
+ continue;
+ }
+ }
+
+ $this->info("Importing folder: {$directory} to mailbox: {$mailbox}");
+ $this->info("Folder {$folderCounter} out of {$totalFolders}");
+
+ $type = "mail";
+ $folderType = null;
+
+ //TODO exclude folders
+ $files = glob("{$directory}/*.ics");
+ $fileCount = count($files);
+ if ($fileCount) {
+ $type = "calendar";
+ $folderType = 'event';
+ } else {
+ $files = glob("{$directory}/*.vcf");
+ $fileCount = count($files);
+ if ($fileCount) {
+ $type = "addressbook";
+ $folderType = 'contact';
+ }
+ }
+
+ //Filter folders that match the file naming scheme (e.g. a calendar with a .ics)
+ $files = array_filter(
+ $files,
+ function ($file) {
+ return !is_dir($file);
+ }
+ );
+
+ $this->info("Folder type is {$type}");
+
+ // This can happen if there e.g. is a calendar and mail folder of the same name.
+ // We prefer the calendar/addressbook and ignore the mail in this case.
+ if ($type != "mail") {
+ $_files = glob("{$directory}/*.eml");
+ $_fileCount = count($_files);
+ if ($_fileCount) {
+ $this->info("Warning: there are {$_fileCount} .eml files in your {$type} folder that are not imported.");
+ }
+ }
+
+ if ($typeFilter && $typeFilter != $type) {
+ $this->info("Skipping due to type filter");
+ continue;
+ }
+
+ if ($typeBlacklist && $typeBlacklist == $type) {
+ $this->info("Skipping due to type blacklist");
+ continue;
+ }
+
+ if (!$fileCount) {
+ $this->info("Nothing to do");
+ continue;
+ }
+
+ $count = $imap->countMessages($mailbox);
+ if ($count === false) {
+ $this->info("Creating mailbox: {$mailbox}");
+ //TODO set specialuse;
+ if (!$imap->createFolder($mailbox)) {
+ $this->error("Failed to create mailbox: {$mailbox}");
+ return 1;
+ }
+ } elseif ($count) {
+ if ($clearTarget) {
+ $imap->clearFolder($mailbox);
+ }
+ }
+
+ if ($folderType) {
+ if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => $folderType])) {
+ $this->error("Failed to set metadata on: {$mailbox}");
+ return 1;
+ }
+ }
+
+ if ($subscribe) {
+ $imap->subscribe($mailbox);
+ }
+
+ if ($type == "calendar") {
+ $mailbox = $this->findDavMailbox($davUrl . "/calendars/{$username}/", $username, $password, $mailbox);
+ } elseif ($type == "addressbook") {
+ $mailbox = $this->findDavMailbox($davUrl . "/addressbooks/{$username}/", $username, $password, $mailbox);
+ } else {
+ if (!$imap->select($mailbox)) {
+ $this->error("Failed to select: {$mailbox}");
+ return 1;
+ }
+ }
+
+ if (!$mailbox) {
+ $this->error("Failed to get the target mailbox.");
+ return 1;
+ }
+
+ $bar = \App\Utils::createProgressBar($this->output, $fileCount, "Importing to " . $mailbox);
+ $bar->advance();
+ foreach ($files as $file) {
+ /* $this->info("Processing file: {$file}"); */
+
+ if ($type == "calendar") {
+ if (!self::caldavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+ return 1;
+ }
+ } elseif ($type == "addressbook") {
+ if (!self::carddavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+ return 1;
+ }
+ } else {
+ if (!self::appendFromFile($imap, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+
+ //Whitelist non fatal errors
+ if (strpos($imap->error, "Message contains invalid header") != false) {
+ continue;
+ }
+
+ return 1;
+ }
+ }
+ // TODO If the work above did not involve imap (e.g. caldav import) and took too long,
+ // it's possible that the imap connection times out and then is invalid in closeConnection() below,
+ // or for another imap import. Check connection and reestablish if necessary.
+ $bar->advance();
+ }
+ $bar->finish();
+ }
+ $imap->closeConnection();
+
+ $this->info("Finished importing folders.");
+ return 0;
+ }
+}

File Metadata

Mime Type
text/x-diff
Expires
Fri, Nov 1, 10:05 AM (1 d, 21 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
10076358
Default Alt Text
(32 KB)

Event Timeline