Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F16571322
D2413.id7771.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
32 KB
Referenced Files
None
Subscribers
None
D2413.id7771.diff
View Options
diff --git a/docker/migrate/Dockerfile b/docker/migrate/Dockerfile
new file mode 100644
--- /dev/null
+++ b/docker/migrate/Dockerfile
@@ -0,0 +1,42 @@
+FROM fedora:33
+
+MAINTAINER Jeroen van Meeuwen <vanmeeuwen@apheleia-it.ch>
+
+ENV HOME=/opt/app-root/src
+
+RUN dnf -y install \
+ composer \
+ file \
+ git \
+ make \
+ npm \
+ openssl-devel \
+ php-cli \
+ php-common \
+ php-ldap \
+ php-opcache \
+ php-pecl-apcu \
+ php-mysqlnd \
+ wget \
+ sudo \
+ libgsf \
+ gettext \
+ libtool \
+ gcc-c++ \
+ shtool \
+ automake \
+ gd-devel zlib-devel boost-devel libgsf-devel gettext-devel \
+ python3 python3-devel boost-python3 boost-python3-devel
+
+
+RUN id default || (groupadd -g 1001 default && useradd -d /opt/app-root/ -u 1001 -g 1001 default)
+RUN usermod -aG wheel default
+RUN echo '%wheel ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+RUN mkdir ${HOME} && chown 1001:1001 ${HOME}
+
+USER 1001
+
+WORKDIR ${HOME}
+
+COPY /rootfs /
diff --git a/docker/migrate/README.md b/docker/migrate/README.md
new file mode 100644
--- /dev/null
+++ b/docker/migrate/README.md
@@ -0,0 +1,6 @@
+This is a docker container to run the migrate:userdata artisan command.
+
+The python script provided allows mass migrating pst files.
+
+
+
diff --git a/docker/migrate/do-it-again.sh b/docker/migrate/do-it-again.sh
new file mode 100755
--- /dev/null
+++ b/docker/migrate/do-it-again.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+export APP_DEBUG="true"
+export APP_KEY=
+export APP_PUBLIC_URL=http://127.0.0.1:8000/
+export APP_SRC=src/
+export APP_URL=http://127.0.0.1:8000/
+export CACHE_DRIVER="array"
+export COMPOSER_ARGS="--no-dev"
+export DB_CONNECTION="sqlite"
+export DB_DATABASE=":memory:"
+export GIT_URI=https://git.kolab.org/source/kolab.git
+export GIT_BRANCH=dev/pstimport
+export LARAVEL_ENV=production
+export LOG_CHANNEL="stderr"
+export MAIL_DRIVER="array"
+export QUEUE_CONNECTION="sync"
+export SESSION_DRIVER="array"
+
+docker build -t migrate .
+
+docker kill migrate
+
+docker rm migrate
+
+docker_opts="\
+ -e APP_DEBUG=${APP_DEBUG} \
+ -e APP_KEY=${APP_KEY} \
+ -e APP_PUBLIC_URL=${APP_PUBLIC_URL} \
+ -e APP_SRC=${APP_SRC} \
+ -e APP_URL=${APP_URL} \
+ -e CACHE_DRIVER=${CACHE_DRIVER} \
+ -e COMPOSER_ARGS=${COMPOSER_ARGS} \
+ -e DB_CONNECTION=${DB_CONNECTION} \
+ -e DB_DATABASE=${DB_DATABASE} \
+ -e GIT_URI=${GIT_URI} \
+ -e GIT_BRANCH=${GIT_BRANCH} \
+ -e LARAVEL_ENV=${LARAVEL_ENV} \
+ -e LOG_CHANNEL=${LOG_CHANNEL} \
+ -e MAIL_DRIVER=${MAIL_DRIVER} \
+ -e QUEUE_CONNECTION=${QUEUE_CONNECTION} \
+ -e SESSION_DRIVER=${SESSION_DRIVER}"
+
+docker run -it \
+ ${docker_opts} \
+ --name migrate migrate /usr/local/bin/build-image
+
+docker commit migrate migrate-s2i
diff --git a/docker/migrate/input/john@kolab.org.pst b/docker/migrate/input/john@kolab.org.pst
new file mode 100644
diff --git a/docker/migrate/massmigrate.py b/docker/migrate/massmigrate.py
new file mode 100755
--- /dev/null
+++ b/docker/migrate/massmigrate.py
@@ -0,0 +1,136 @@
+#!/bin/env python3
+
+# This script allows mass migrating pst files.
+# Usage ./massmigrate.py
+# --input ./input/
+# --output ./output/
+# --password $password
+# --imap ssl://$domain:993 --dav https://$domain/iRony
+#
+# It requires an input and an output directory.
+#
+# The input directory contains files to import in the format:
+# username@hostname.pst
+#
+# The output directory will receive all processed pst files.
+
+import subprocess
+import argparse
+import glob
+import shutil
+import os
+import time
+from threading import Thread
+from queue import Queue
+
+
+def transform_file(file):
+ dir_path, filename = os.path.split(os.path.realpath(file))
+
+ removepst = False
+ if '.zip' in file:
+ username = filename.replace('.zip', '')
+ parts = username.split('.')
+ username = f"{parts[1]}.{parts[2]}@{parts[0].lower()}.domain.com"
+ result = subprocess.run(f"unzip {filename}",
+ shell=True,
+ cwd=dir_path
+ )
+ if result.returncode != 0:
+ return None, None, None, None
+ removepst = True
+ filename = f"{parts[1]}.{parts[2]}.pst"
+ else:
+ username = filename.replace('.pst', '')
+
+ return dir_path, filename, username, removepst
+
+
+def import_file(file, password, imap_uri, dav_uri, type_filter, type_blacklist):
+ image = "migrate-s2i"
+ start = time.time()
+
+ dir_path, filename, username, removepst = transform_file(file)
+ if not username:
+ print(f"Failed to extract the file {file}")
+ return False
+
+ print("Starting " + username)
+
+ DOCKER_OPTIONS = ' '.join([
+ "--network=host", "--rm", "-ti",
+ "-v", f"{dir_path}:/opt/app-root/input/",
+ "-w", "/opt/app-root/src/kolab/src",
+ ])
+
+ cmdargs = [
+ "./artisan", "migrate:userdata",
+ f"--importpst=/opt/app-root/input/{filename}",
+ f"--username={username} --password={password}",
+ "--subscribe --debug",
+ f"--davUrl={dav_uri}",
+ f"--imapUrl={imap_uri}",
+ "--exclude-target='Sync Issues (This computer only)'",
+ "--exclude-target='Drafts (This computer only)'",
+ "--exclude-target='Sync Issues*'",
+ "--exclude-target='Recoverable Items*'",
+ ]
+
+ if type_filter:
+ cmdargs.append(f"--type-filter='{type_filter}'")
+
+ if type_blacklist:
+ cmdargs.append(f"--type-blacklist='{type_blacklist}'")
+
+ CMD = ' '.join(cmdargs)
+
+ with open(f"output/{username}.log", 'w') as logfile:
+ result = subprocess.run(f"docker run {DOCKER_OPTIONS} {image} {CMD}",
+ shell=True,
+ text=True,
+ stderr=subprocess.STDOUT,
+ stdout=logfile)
+
+ executionTime = time.time() - start
+ print(f"Finished {username} in {executionTime}s")
+ if removepst:
+ os.remove(f"{dir_path}/{filename}")
+ return not result.returncode
+ return 1
+
+
+def main():
+ parser = argparse.ArgumentParser("usage: %prog [options]")
+ parser.add_argument("--input", help="Input directory")
+ parser.add_argument("--output", help="Output directory")
+ parser.add_argument("--password", help="User password to use for all files")
+ parser.add_argument("--imap", help="IMAP URI")
+ parser.add_argument("--dav", help="DAV URI")
+ parser.add_argument("--typefilter", help="Folder type whitelist (not a list)")
+ parser.add_argument("--typeblacklist", help="Folder type blacklist (not a list)")
+ options = parser.parse_args()
+
+ input_dir = os.path.expanduser(options.input)
+ output_dir = os.path.expanduser(options.output)
+
+ q = Queue()
+ for file in glob.glob(f"{input_dir}*"):
+ q.put(file)
+
+ def worker():
+ while True:
+ file = q.get()
+ if import_file(file, options.password, options.imap, options.dav, options.typefilter, options.typeblacklist):
+ shutil.move(file, output_dir + os.path.basename(file))
+ else:
+ print(f"Failed to import {file}")
+ q.task_done()
+
+ for _i in range(3):
+ Thread(target=worker, daemon=True).start()
+ q.join()
+ print("Processing finished")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docker/migrate/rootfs/usr/local/bin/build-image b/docker/migrate/rootfs/usr/local/bin/build-image
new file mode 100755
--- /dev/null
+++ b/docker/migrate/rootfs/usr/local/bin/build-image
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -x
+
+set -e
+
+pwd
+
+if [ -z "${GIT_URI}" ]; then
+ echo "No GIT_URI specified. Exiting."
+ exit 1
+fi
+
+git clone -b ${GIT_BRANCH:-master} ${GIT_URI}
+
+cd $(basename ${GIT_URI} .git)
+
+if [ ! -z "${APP_SRC}" ]; then
+ cd ${APP_SRC}
+fi
+
+if [ -f "composer.json" ]; then
+ echo "Detected composer.json, running install"
+ php -dmemory_limit=${COMPOSER_MEMORY_LIMIT:--1} /usr/bin/composer install ${COMPOSER_ARGS}
+ rm -rf ~/.cache/composer/
+fi
+
+if [ -z "${LARAVEL_ENV}" ]; then
+ LARAVEL_ENV=prod
+fi
+
+npm install
+
+npm run ${LARAVEL_ENV} && rm -rf ~/.npm/
+
+cd -
+
+# Build libpst
+git clone -b stable/kolab-0.6.76 https://git.kolab.org/source/libpst.git
+cd libpst
+autoreconf -vif
+./configure --enable-libpst-shared --with-boost-python=boost_python39
+make -j5
+sudo make install
diff --git a/src/app/Console/Commands/MigrateUserdata.php b/src/app/Console/Commands/MigrateUserdata.php
new file mode 100644
--- /dev/null
+++ b/src/app/Console/Commands/MigrateUserdata.php
@@ -0,0 +1,657 @@
+<?php
+
+namespace App\Console\Commands;
+
+
+use Illuminate\Console\Command;
+
+
+class MigrateUserdata extends Command
+{
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'migrate:userdata
+ {--importpst= : Pst file path}
+ {--username= : Target user}
+ {--password= : Password}
+ {--davUrl= : caldav server url}
+ {--imapUrl= : imap server url}
+ {--clear-target : Remove all messages from the target mailbox.}
+ {--subscribe : Subscribe to the target mailbox.}
+ {--exclude-target=* : Blacklist target mailbox.}
+ {--include-target=* : Whitelist target mailbox.}
+ {--pickup-from= : Pick-up import from a specific target mailbox.}
+ {--type-filter= : Whitelist folder type [calendar, mail, addressbook]. Not a list atm, only single value.}
+ {--type-blacklist= : Blacklist folder type [calendar, mail, addressbook]. Not a list atm, only single value.}
+ {--debug : Enable debug output}';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Migrate userdata';
+
+ /**
+ * Execute the console command.
+ *
+ * @return mixed
+ */
+ public function handle()
+ {
+ if ($this->option('importpst')) {
+ return $this->importPst(
+ $this->option('importpst'),
+ $this->option('username'),
+ $this->option('password'),
+ $this->option('clear-target'),
+ $this->option('subscribe'),
+ $this->option('exclude-target'),
+ $this->option('include-target'),
+ $this->option('pickup-from'),
+ $this->option('debug'),
+ $this->option('davUrl'),
+ $this->option('imapUrl'),
+ $this->option('type-filter'),
+ $this->option('type-blacklist'),
+ );
+ }
+ }
+
+
+ /**
+ * Get LDAP configuration for specified access level
+ */
+ private static function getConfig($imapUrl, $username, $password, $verifyPeer, $verifyHost)
+ {
+ $uri = \parse_url($imapUrl ?? \config('imap.uri'));
+ $default_port = 143;
+ $ssl_mode = null;
+
+ if (isset($uri['scheme'])) {
+ if (preg_match('/^(ssl|imaps)/', $uri['scheme'])) {
+ $default_port = 993;
+ $ssl_mode = 'ssl';
+ } elseif ($uri['scheme'] === 'tls') {
+ $ssl_mode = 'tls';
+ }
+ }
+
+ $config = [
+ 'host' => $uri['host'],
+ 'user' => $username ?? \config('imap.admin_login'),
+ 'password' => $password ?? \config('imap.admin_password'),
+ 'options' => [
+ 'port' => !empty($uri['port']) ? $uri['port'] : $default_port,
+ 'ssl_mode' => $ssl_mode,
+ 'timeout' => 60,
+ //Work around guam bug
+ 'literal+' => false,
+ 'socket_options' => [
+ 'ssl' => [
+ 'verify_peer' => $verifyPeer ?? \config('imap.verify_peer'),
+ 'verify_peer_name' => $verifyPeer ?? \config('imap.verify_peer'),
+ 'verify_host' => $verifyHost ?? \config('imap.verify_host')
+ ],
+ ],
+ ],
+ ];
+
+ return $config;
+ }
+
+ /**
+ * Initialize connection to IMAP
+ */
+ private function initIMAP(array $config, string $login_as = null, $debug)
+ {
+ $imap = new \rcube_imap_generic();
+
+ if ($debug || \config('app.debug')) {
+ $imap->setDebug(true, 'App\Backends\IMAP::logDebug');
+ }
+
+ if ($login_as) {
+ $config['options']['auth_cid'] = $config['user'];
+ $config['options']['auth_pw'] = $config['password'];
+ $config['options']['auth_type'] = 'PLAIN';
+ $config['user'] = $login_as;
+ }
+
+ $imap->connect($config['host'], $config['user'], $config['password'], $config['options']);
+
+ if (!$imap->connected()) {
+ $message = sprintf("Login failed for %s against %s. %s", $config['user'], $config['host'], $imap->error);
+ $this->error($message);
+ return null;
+ }
+
+ return $imap;
+ }
+
+ private static function imapFlagsFromStatus($status)
+ {
+ # libpst only sets R and O
+ $flags = [];
+ if (strpos($status, 'R') !== false || strpos($status, 'O') !== false) {
+ $flags[] = 'SEEN';
+ }
+ /* if 'D' in flags: */
+ /* $flags[] = '\Deleted' */
+ /* if 'A' in flags: */
+ /* $flags[] = '\Answered' */
+ /* if 'F' in flags */
+ /* $flags[] = '\Flagged' */
+ return $flags;
+ }
+
+ private static function appendFromFile($imap, $mailbox, $path)
+ {
+ // open message file
+ $fp = null;
+ if (file_exists(realpath($path))) {
+ $fp = fopen($path, 'r');
+ }
+
+ if (!$fp) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $message = fread($fp, filesize($path));
+ // IMAP requires CRLF
+ $message = str_replace("\n", "\r\n", str_replace("\r", '', $message));
+
+ $flags = [];
+ $matches = null;
+ //In practice we only seem to get RO
+ if (preg_match("/Status: (R?O?)\r\n/", $message, $matches)) {
+ /* print("Found matches:\n"); */
+ /* print_r($matches); */
+ $flags = self::imapFlagsFromStatus($matches[1]);
+ $message = preg_replace("/Status: (R?O?)\r\n/", "", $message);
+ }
+ $date = null;
+ $binary = false;
+ return $imap->append($mailbox, $message, $flags, $date, $binary);
+ }
+
+ private static function listDirectoryTree($dir)
+ {
+ $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir));
+ $directories = [];
+ foreach ($it as $fileinfo) {
+ //We look for all "." direcotires, because we somehow can't just iterate over directories without files.
+ if ($fileinfo->isDir() && $fileinfo->getFilename() == "." && dirname($fileinfo->getPath()) != dirname($dir)) {
+ $directories[] = $fileinfo->getPath();
+ }
+ }
+ sort($directories);
+ return $directories;
+ }
+
+
+ private function createTemporaryDirectory()
+ {
+ $tempfile = tempnam(sys_get_temp_dir(), '');
+ if (file_exists($tempfile)) {
+ unlink($tempfile);
+ }
+ mkdir($tempfile);
+ if (!is_dir($tempfile)) {
+ return null;
+ }
+ return $tempfile . "/";
+ }
+
+
+ private static function fixICal($ical)
+ {
+ $lines = explode("\n", $ical);
+
+ for ($i = 0; $i < count($lines); $i++) {
+ $line = $lines[$i];
+ //Remove nonsensical NONE categories libpst by Outlook
+ if ($line == "CATEGORIES:NONE") {
+ $lines[$i] = null;
+ }
+ }
+ return implode("\n", array_filter($lines));
+ }
+
+
+ private static function caldavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path)
+ {
+ $fp = null;
+ if (file_exists(realpath($path))) {
+ $fp = fopen($path, 'r');
+ }
+
+ if (!$fp) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $putdata = self::fixICal(fread($fp, filesize($path)));
+
+ print($putdata . "\n");
+
+ $urlParts = parse_url($serverUrl);
+ $newUrl = $urlParts['scheme'] . "://" . $urlParts['host'];
+ if (isset($urlParts['port'])) {
+ $newUrl = $newUrl . ":" . $urlParts['port'];
+ }
+ $url = $newUrl . $mailbox . basename($path);
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/calendar; charset='UTF-8'"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $putdata);
+
+ $data = curl_exec($c);
+
+ if (!$data) {
+ print("Append failed: " . curl_error($c) . "\n");
+ curl_close($c);
+ return false;
+ }
+
+ curl_close($c);
+ return true;
+ }
+
+
+ private static function fixVCard($vcard)
+ {
+ $lines = explode("\n", $vcard);
+
+ //Generate a UID if there isn't one
+ if (strpos($vcard, "UID") === false) {
+ $uid = uniqid();
+ array_splice($lines, 1, null, ["UID:{$uid}"]);
+ }
+
+ return implode("\n", $lines);
+ }
+
+ private function carddavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path)
+ {
+ $fp = null;
+ if (file_exists(realpath($path))) {
+ $fp = fopen($path, 'r');
+ }
+
+ if (!$fp) {
+ print("Failed to open for reading: " . $path);
+ return false;
+ }
+
+ $putdata = self::fixVCard(fread($fp, filesize($path)));
+
+ print($putdata . "\n");
+
+ $urlParts = parse_url($serverUrl);
+ $newUrl = $urlParts['scheme'] . "://" . $urlParts['host'];
+ if (isset($urlParts['port'])) {
+ $newUrl = $newUrl . ":" . $urlParts['port'];
+ }
+ $url = $newUrl . $mailbox . basename($path);
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/vcard; charset='UTF-8'"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $putdata);
+
+ $data = curl_exec($c);
+
+ if (!$data) {
+ print("Append failed: " . curl_error($c) . "\n");
+ curl_close($c);
+ return false;
+ }
+ curl_close($c);
+ return true;
+ }
+
+ private static function convertToMailbox($name)
+ {
+ // Replace forbidden characters
+ //List based on testing Kolab's Cyrus-IMAP 2.5 (according to the roundcube codebase)
+ $name = str_replace(';', '_', $name);
+ $name = str_replace('<', '_', $name);
+ $name = str_replace('?', '_', $name);
+ $name = str_replace('\\', '_', $name);
+ $name = str_replace('|', '_', $name);
+ $name = str_replace('`', '_', $name);
+ $name = str_replace('!', '_', $name);
+ $name = str_replace('{', '_', $name);
+ $name = str_replace('}', '_', $name);
+ $name = str_replace('(', '_', $name);
+ $name = str_replace(')', '_', $name);
+ $name = str_replace('@', 'at', $name);
+ return mb_convert_encoding($name, "UTF7-IMAP", "UTF-8");
+ }
+
+ private function findDavMailbox($url, $user, $pw, $mailbox)
+ {
+ $xml = '<d:propfind xmlns:d="DAV:"><d:prop><d:resourcetype /><d:displayname /></d:prop></d:propfind>';
+
+ $c = curl_init();
+ curl_setopt($c, CURLOPT_URL, $url);
+ curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal"));
+ curl_setopt($c, CURLOPT_HEADER, 0);
+ curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false);
+ curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+ curl_setopt($c, CURLOPT_USERPWD, mb_strtolower($user) . ":" . $pw);
+ curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND");
+ curl_setopt($c, CURLOPT_POSTFIELDS, $xml);
+ curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
+
+ $data = curl_exec($c);
+ if (!$data) {
+ $this->error("Curl propfind failed on url {$url}: " . curl_error($c) . "\n");
+ curl_close($c);
+ return null;
+ }
+ curl_close($c);
+
+ try {
+ $xml = \simplexml_load_string($data, null, 0, "d", true);
+ } catch (\Exception $e) {
+ $this->error("Exception during xml parsing: {$e->getMessage()}\n");
+ $xml = null;
+ }
+
+ if (!$xml) {
+ $this->error("Failed to parse xml from {$url}\n");
+ $this->error("\n" . $data . "\n");
+
+ $errors = \libxml_get_errors();
+
+ foreach ($errors as $error) {
+ $this->error("Error: " . $error->message . "\n");
+ }
+
+ \libxml_clear_errors();
+ return null;
+ }
+
+ foreach ($xml->response as $e) {
+ $displayname = (string)$e->propstat->prop->displayname;
+
+ //Convert the hierarchy separator back from " » ", " : " or "/"
+ //The utf-8 modifier is required for the multibyte character "»" to work within []
+ $parts = preg_split('!(*UTF8)(\s*/\s*|\s+[»:]\s+)!', $displayname);
+ $converted = self::convertToMailbox(join('/', $parts));
+ $this->info("Found {$displayname} : {$converted}");
+
+ if ($converted == $mailbox) {
+ return (string)$e->href;
+ }
+ }
+
+ $this->error("Could not find a matching dav collection for {$mailbox}");
+ $this->error("\n" . $data . "\n");
+
+ return null;
+ }
+
+ private static function mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter)
+ {
+ // Ignore the toplevel directory which is something like: "Outlook Data File"
+ $mailboxParts = array_slice(explode('/', $directory), $directoryRootDepth + 1);
+
+ //Move folders out of inbox
+ if (count($mailboxParts) > 1 && strcasecmp($mailboxParts[0], "inbox") == 0) {
+ $mailboxParts = array_slice($mailboxParts, 1);
+ }
+
+ $mailboxParts = array_map(
+ function ($part) {
+ return self::convertToMailbox($part);
+ },
+ $mailboxParts
+ );
+ return implode($hierarchyDelimiter, $mailboxParts);
+ }
+
+ private static function inBlacklist($mailbox, $mailboxBlacklist)
+ {
+ foreach ($mailboxBlacklist as $entry) {
+ if (fnmatch($entry, $mailbox)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private function importPst($file, $username, $password, $clearTarget, $subscribe, $mailboxBlacklist, $mailboxWhitelist, $pickupFrom, $debug, $davUrl, $imapUrl, $typeFilter, $typeBlacklist): int
+ {
+ if (!file_exists($file)) {
+ //TODO downlaod file (see Utils.php)
+ $this->error("File doesn't exist: " . $file);
+ return 1;
+ }
+
+ $dir = $this->createTemporaryDirectory();
+ if (!$dir) {
+ $this->error("Failed to create a temporary directory");
+ return 1;
+ }
+
+ $this->info("Importing pst: " . $file);
+ $this->info("Using temporary directory: " . $dir);
+
+ // Extract the pst file
+ if (!exec("readpst -e {$file} -o {$dir}", $output, $resultCode)) {
+ $this->error("Failed to extract pst data");
+ return 1;
+ }
+
+ $this->info("Extracted pst file: \n" . implode("\n ", $output) . "\n");
+
+ $outputDirectories = self::listDirectoryTree($dir);
+ $directoryRoot = array_shift($outputDirectories);
+ $directoryRootDepth = substr_count($directoryRoot, '/');
+ $this->info(" Directory root: {$directoryRoot}");
+ $this->info(" Depth: {$directoryRootDepth}");
+ $this->info(" Directories: \n" . var_export($outputDirectories, true) . "\n");
+
+ // Prepare imap connection
+ $imap = $this->initIMAP(self::getConfig($imapUrl, $username, $password, false, false), false, $debug);
+ if (!$imap) {
+ $this->error("Failed to connect to imap.");
+ return 1;
+ }
+
+ $this->info("Logged in to imap as user: {$username}");
+
+ $hierarchyDelimiter = $imap->getHierarchyDelimiter();
+ $this->info("Hierarchy delimiter: {$hierarchyDelimiter}");
+
+ //TODO config option
+ $importRoot = '';
+
+ $totalFolders = count($outputDirectories);
+ $folderCounter = 0;
+
+ $skipBeforePickupfromFolder = !empty($pickupFrom);
+
+ foreach ($outputDirectories as $directory) {
+ $mailbox = $importRoot . self::mailboxFromDirectory($directory, $directoryRootDepth, $hierarchyDelimiter);
+ $folderCounter++;
+
+ if (
+ (empty($mailboxWhitelist) && self::inBlacklist($mailbox, $mailboxBlacklist)) ||
+ (!empty($mailboxWhitelist) && !self::inBlacklist($mailbox, $mailboxWhitelist))
+ ) {
+ $this->info("Skipping blacklisted mailbox: {$mailbox}");
+ continue;
+ }
+
+ if ($skipBeforePickupfromFolder) {
+ if ($mailbox == $pickupFrom) {
+ $this->info("Picking up from mailbox: {$mailbox}");
+ $skipBeforePickupfromFolder = false;
+ } else {
+ $this->info("Skipping before pickup-from mailbox: {$mailbox}");
+ continue;
+ }
+ }
+
+ $this->info("Importing folder: {$directory} to mailbox: {$mailbox}");
+ $this->info("Folder {$folderCounter} out of {$totalFolders}");
+
+ $type = "mail";
+ $folderType = null;
+
+ //TODO exclude folders
+ $files = glob("{$directory}/*.ics");
+ $fileCount = count($files);
+ if ($fileCount) {
+ $type = "calendar";
+ $folderType = 'event';
+ } else {
+ $files = glob("{$directory}/*.vcf");
+ $fileCount = count($files);
+ if ($fileCount) {
+ $type = "addressbook";
+ $folderType = 'contact';
+ }
+ }
+
+ //Filter folders that match the file naming scheme (e.g. a calendar with a .ics)
+ $files = array_filter(
+ $files,
+ function ($file) {
+ return !is_dir($file);
+ }
+ );
+
+ $this->info("Folder type is {$type}");
+
+ // This can happen if there e.g. is a calendar and mail folder of the same name.
+ // We prefer the calendar/addressbook and ignore the mail in this case.
+ if ($type != "mail") {
+ $_files = glob("{$directory}/*.eml");
+ $_fileCount = count($_files);
+ if ($_fileCount) {
+ $this->info("Warning: there are {$_fileCount} .eml files in your {$type} folder that are not imported.");
+ }
+ }
+
+ if ($typeFilter && $typeFilter != $type) {
+ $this->info("Skipping due to type filter");
+ continue;
+ }
+
+ if ($typeBlacklist && $typeBlacklist == $type) {
+ $this->info("Skipping due to type blacklist");
+ continue;
+ }
+
+ if (!$fileCount) {
+ $this->info("Nothing to do");
+ continue;
+ }
+
+ $count = $imap->countMessages($mailbox);
+ if ($count === false) {
+ $this->info("Creating mailbox: {$mailbox}");
+ //TODO set specialuse;
+ if (!$imap->createFolder($mailbox)) {
+ $this->error("Failed to create mailbox: {$mailbox}");
+ return 1;
+ }
+ } elseif ($count) {
+ if ($clearTarget) {
+ $imap->clearFolder($mailbox);
+ }
+ }
+
+ if ($folderType) {
+ if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => $folderType])) {
+ $this->error("Failed to set metadata on: {$mailbox}");
+ return 1;
+ }
+ }
+
+ if ($subscribe) {
+ $imap->subscribe($mailbox);
+ }
+
+ if ($type == "calendar") {
+ $mailbox = $this->findDavMailbox($davUrl . "/calendars/{$username}/", $username, $password, $mailbox);
+ } elseif ($type == "addressbook") {
+ $mailbox = $this->findDavMailbox($davUrl . "/addressbooks/{$username}/", $username, $password, $mailbox);
+ } else {
+ if (!$imap->select($mailbox)) {
+ $this->error("Failed to select: {$mailbox}");
+ return 1;
+ }
+ }
+
+ if (!$mailbox) {
+ $this->error("Failed to get the target mailbox.");
+ return 1;
+ }
+
+ $bar = \App\Utils::createProgressBar($this->output, $fileCount, "Importing to " . $mailbox);
+ $bar->advance();
+ foreach ($files as $file) {
+ /* $this->info("Processing file: {$file}"); */
+
+ if ($type == "calendar") {
+ if (!self::caldavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+ return 1;
+ }
+ } elseif ($type == "addressbook") {
+ if (!self::carddavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+ return 1;
+ }
+ } else {
+ if (!self::appendFromFile($imap, $mailbox, $file)) {
+ $this->error("Append failed: {$file}");
+ $this->error("IMAP error: {$imap->error}");
+
+ //Whitelist non fatal errors
+ if (strpos($imap->error, "Message contains invalid header") != false) {
+ continue;
+ }
+
+ return 1;
+ }
+ }
+ // TODO If the work above did not involve imap (e.g. caldav import) and took too long,
+ // it's possible that the imap connection times out and then is invalid in closeConnection() below,
+ // or for another imap import. Check connection and reestablish if necessary.
+ $bar->advance();
+ }
+ $bar->finish();
+ }
+ $imap->closeConnection();
+
+ $this->info("Finished importing folders.");
+ return 0;
+ }
+}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Oct 31, 10:07 AM (19 h, 40 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
10076385
Default Alt Text
D2413.id7771.diff (32 KB)
Attached To
Mode
D2413: Userdata migration for pst import
Attached
Detach File
Event Timeline
Log In to Comment