diff --git a/src/app/Console/Commands/MigrateUserdata.php b/src/app/Console/Commands/MigrateUserdata.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/MigrateUserdata.php @@ -0,0 +1,590 @@ +option('importpst')) { + $mailboxBlacklist = [ + "Sync Issues (This computer only)", + "Drafts (This computer only)" + ]; + + $this->importPst( + $this->option('importpst'), + $this->option('username'), + $this->option('password'), + $this->option('clear-target'), + $this->option('subscribe'), + $this->option('exclude-target'), + $this->option('debug'), + $this->option('davUrl'), + ); + } + } + + + /** + * Get LDAP configuration for specified access level + */ + private static function getConfig() + { + $uri = \parse_url(\config('imap.uri')); + $default_port = 143; + $ssl_mode = null; + + if (isset($uri['scheme'])) { + if (preg_match('/^(ssl|imaps)/', $uri['scheme'])) { + $default_port = 993; + $ssl_mode = 'ssl'; + } elseif ($uri['scheme'] === 'tls') { + $ssl_mode = 'tls'; + } + } + + $config = [ + 'host' => $uri['host'], + 'user' => \config('imap.admin_login'), + 'password' => \config('imap.admin_password'), + 'options' => [ + 'port' => !empty($uri['port']) ? $uri['port'] : $default_port, + 'ssl_mode' => $ssl_mode, + 'socket_options' => [ + 'ssl' => [ + 'verify_peer' => \config('imap.verify_peer'), + 'verify_peer_name' => \config('imap.verify_peer'), + 'verify_host' => \config('imap.verify_host') + ], + ], + ], + ]; + + return $config; + } + + /** + * Initialize connection to IMAP + */ + private static function initIMAP(array $config, string $login_as = null) + { + $imap = new \rcube_imap_generic(); + + if (\config('app.debug')) { + $imap->setDebug(true, 'App\Backends\IMAP::logDebug'); + } + + if ($login_as) { + $config['options']['auth_cid'] = $config['user']; + $config['options']['auth_pw'] = $config['password']; + $config['options']['auth_type'] = 'PLAIN'; + $config['user'] = $login_as; + } + + $imap->connect($config['host'], $config['user'], $config['password'], $config['options']); + + if (!$imap->connected()) { + $message = sprintf("Login failed for %s against %s. %s", $config['user'], $config['host'], $imap->error); + + \Log::error($message); + + throw new \Exception("Connection to IMAP failed"); + } + + return $imap; + } + + private static function imapFlagsFromStatus($status) + { + # libpst only sets R and O + $flags = []; + if (strpos($status, 'R') !== false || strpos($status, 'O') !== false) { + $flags[] = 'SEEN'; + } + /* if 'D' in flags: */ + /* $flags[] = '\Deleted' */ + /* if 'A' in flags: */ + /* $flags[] = '\Answered' */ + /* if 'F' in flags */ + /* $flags[] = '\Flagged' */ + return $flags; + } + + private static function appendFromFile($imap, $mailbox, $path) + { + // open message file + if (file_exists(realpath($path))) { + $fp = fopen($path, 'r'); + } + + if (!$fp) { + print("Failed to open for reading: " . $path); + return false; + } + + $message = fread($fp, filesize($path)); + // IMAP requires CRLF + $message = str_replace("\n", "\r\n", str_replace("\r", '', $message)); + + $flags = []; + $matches = null; + //In practice we only seem to get RO + if (preg_match("/Status: (R?O?)\r\n/", $message, $matches)) { + /* print("Found matches:\n"); */ + /* print_r($matches); */ + $flags = self::imapFlagsFromStatus($matches[1]); + } + $date = null; + $binary = false; + return $imap->append($mailbox, $message, $flags, $date, $binary); + } + + private static function listDirectoryTree($dir) + { + $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir)); + $directories = []; + foreach ($it as $fileinfo) { + //We look for all "." direcotires, because we somehow can't just iterate over directories without files. + if ($fileinfo->isDir() && $fileinfo->getFilename() == "." && dirname($fileinfo->getPath()) != dirname($dir)) { + $directories[] = $fileinfo->getPath(); + } + } + return $directories; + } + + + private function createTemporaryDirectory() + { + $tempfile = tempnam(sys_get_temp_dir(), ''); + if (file_exists($tempfile)) { + unlink($tempfile); + } + mkdir($tempfile); + if (!is_dir($tempfile)) { + return null; + } + return $tempfile . "/"; + } + + + private function fixICal($ical) + { + $lines = explode("\n", $ical); + + for ($i = 0; $i < count($lines); $i++) { + $line = $lines[$i]; + //Remove nonsensical NONE categories libpst by Outlook + if ($line == "CATEGORIES:NONE") { + $lines[$i] = null; + } + } + return implode("\n", array_filter($lines)); + } + + + private function caldavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path) + { + if (file_exists(realpath($path))) { + $fp = fopen($path, 'r'); + } + + if (!$fp) { + print("Failed to open for reading: " . $path); + return false; + } + + $putdata = self::fixICal(fread($fp, filesize($path))); + + print($putdata . "\n"); + + $urlParts = parse_url($serverUrl); + $newUrl = $urlParts['scheme'] . "://" . $urlParts['host']; + if (isset($urlParts['port'])) { + $newUrl = $newUrl . ":" . $urlParts['port']; + } + $url = $newUrl . $mailbox . basename($path); + + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $url); + curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/calendar; charset='UTF-8'")); + curl_setopt($c, CURLOPT_HEADER, 0); + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_setopt($c, CURLOPT_POSTFIELDS, $putdata); + + $data = curl_exec($c); + + if (!$data) { + print("Append failed: " . curl_error($c) . "\n"); + curl_close($c); + return false; + } + + curl_close($c); + return true; + } + + + private function fixVCard($vcard) + { + $lines = explode("\n", $vcard); + + //Generate a UID if there isn't one + if (strpos($vcard, "UID") === false) { + $uid = uniqid(); + array_splice($lines, 1, null, ["UID:{$uid}"]); + } + + return implode("\n", $lines); + } + + private function carddavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path) + { + if (file_exists(realpath($path))) { + $fp = fopen($path, 'r'); + } + + if (!$fp) { + print("Failed to open for reading: " . $path); + return false; + } + + $putdata = self::fixVCard(fread($fp, filesize($path))); + + print($putdata . "\n"); + + $urlParts = parse_url($serverUrl); + $newUrl = $urlParts['scheme'] . "://" . $urlParts['host']; + if (isset($urlParts['port'])) { + $newUrl = $newUrl . ":" . $urlParts['port']; + } + $url = $newUrl . $mailbox . basename($path); + + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $url); + curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/vcard; charset='UTF-8'")); + curl_setopt($c, CURLOPT_HEADER, 0); + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_setopt($c, CURLOPT_POSTFIELDS, $putdata); + + $data = curl_exec($c); + + if (!$data) { + print("Append failed: " . curl_error($c) . "\n"); + curl_close($c); + return false; + } + curl_close($c); + return true; + } + + private function findCaldavMailbox($serverUrl, $user, $pw, $mailbox) + { + $xml = ''; + $url = $serverUrl . "/calendars/{$user}/"; + + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $url); + curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal")); + curl_setopt($c, CURLOPT_HEADER, 0); + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND"); + curl_setopt($c, CURLOPT_POSTFIELDS, $xml); + curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); + + $data = curl_exec($c); + if (!$data) { + print("Curl failed\n"); + return null; + } + + $xml = \simplexml_load_string($data, null, 0, "d", true); + + if (!$xml) { + print("Failed to parse xml\n"); + $errors = \libxml_get_errors(); + + foreach ($errors as $error) { + print("Error: " . $error->message . "\n"); + } + + \libxml_clear_errors(); + return null; + } + + foreach ($xml->response as $e) { + $displayname = (string)$e->propstat->prop->displayname; + if ($displayname == $mailbox) { + return (string)$e->href; + } + } + + curl_close($c); + return null; + } + + private function findCarddavMailbox($serverUrl, $user, $pw, $mailbox) + { + $xml = ''; + $url = $serverUrl . "/addressbooks/{$user}/"; + + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $url); + curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal")); + curl_setopt($c, CURLOPT_HEADER, 0); + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND"); + curl_setopt($c, CURLOPT_POSTFIELDS, $xml); + curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); + + $data = curl_exec($c); + if (!$data) { + print("Curl failed\n"); + return null; + } + + $xml = \simplexml_load_string($data, null, 0, "d", true); + + if (!$xml) { + print("Failed to parse xml\n"); + $errors = \libxml_get_errors(); + + foreach ($errors as $error) { + print("Error: " . $error->message . "\n"); + } + + \libxml_clear_errors(); + return null; + } + + foreach ($xml->response as $e) { + $displayname = (string)$e->propstat->prop->displayname; + if ($displayname == $mailbox) { + return (string)$e->href; + } + } + + curl_close($c); + return null; + } + + private function importPst($file, $username, $password, $clearTarget, $subscribe, $mailboxBlacklist, $debug, $davUrl) + { + if (!file_exists($file)) { + //TODO downlaod file (see Utils.php) + $this->error("File doesn't exist: " . $file); + return; + } + + $dir = $this->createTemporaryDirectory(); + if (!$dir) { + $this->error("Failed to create a temporary directory"); + return; + } + + $this->info("Using temporary directory: " . $dir); + + // Extract the pst file + $output; + $resultCode; + if (!exec("readpst -e {$file} -o {$dir}", $output, $resultCode)) { + $this->error("Failed to extract pst data"); + return; + } + + $this->info("Extracted pst file: \n" . implode("\n ", $output) . "\n"); + + $outputDirectories = self::listDirectoryTree($dir); + $directoryRoot = array_shift($outputDirectories); + $depth = substr_count($directoryRoot, '/'); + /* $this->info("Directories: \n" . var_export($outputDirectories, true) . "\n"); */ + + // Prepare imap connection + $imap = self::initIMAP(self::getConfig(), $username); + $imap->setDebug($debug); + if (!$imap) { + $this->error("Failed to connect to imap."); + return; + } + + $this->info("Logged in to imap as user: {$username}"); + + $hierarchyDelimiter = $imap->getHierarchyDelimiter(); + + //TODO config option + $importRoot = ''; + + $totalFolders = count($outputDirectories); + $folderCounter = 0; + + foreach ($outputDirectories as $directory) { + // Ignore the toplevel directory which is something like: "Outlook Data File" + $mailboxParts = array_slice(explode('/', $directory), $depth + 1); + + //Move folders out of inbox + if (count($mailboxParts) > 1 && strcasecmp($mailboxParts[0], "inbox") == 0) { + $mailboxParts = array_slice($mailboxParts, 1); + } + + $mailbox = $importRoot . implode($hierarchyDelimiter, $mailboxParts); + $folderCounter++; + + if (in_array($mailbox, $mailboxBlacklist)) { + $this->info("Skipping blacklisted mailbox"); + continue; + } + + $this->info("Importing folder: {$directory} to mailbox: {$mailbox}"); + $this->info("Folder {$folderCounter} out of {$totalFolders}"); + + $files = glob("{$directory}/*.eml"); + $fileCount = count($files); + $type = "mail"; + if (!$fileCount) { + $files = glob("{$directory}/*.ics"); + $fileCount = count($files); + if ($fileCount) { + $type = "calendar"; + } else { + $files = glob("{$directory}/*.vcf"); + $fileCount = count($files); + if ($fileCount) { + $type = "addressbook"; + } + } + } + + if (!$fileCount) { + $this->info("Nothing to do"); + continue; + } + + $count = $imap->countMessages($mailbox); + if ($count === false) { + $this->info("Creating mailbox: {$directory}"); + //TODO set specialuse; + if (!$imap->createFolder($mailbox)) { + $this->error("Failed to create mailbox: {$mailbox}"); + return; + } + + if ($type == "calendar") { + if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => 'event'])) { + $this->error("Failed to set metadata on: {$mailbox}"); + return; + } + } + + if ($type == "addressbook") { + if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => 'contact'])) { + $this->error("Failed to set metadata on: {$mailbox}"); + return; + } + } + } elseif ($count) { + if ($clearTarget) { + $imap->clearFolder($mailbox); + } else { + $this->error("Target mailbox is not empty: {$mailbox}"); + return; + } + } + + if ($subscribe) { + $imap->subscribe($mailbox); + } + + if ($type == "calendar") { + $mailbox = self::findCaldavMailbox($davUrl, $username, $password, $mailbox); + } elseif ($type == "addressbook") { + $mailbox = self::findCarddavMailbox($davUrl, $username, $password, $mailbox); + } else { + if (!$imap->select($mailbox)) { + $this->error("Failed to select: {$mailbox}"); + return; + } + } + + if (!$mailbox) { + $this->error("Failed to get the target mailbox."); + return; + } + + $bar = \App\Utils::createProgressBar($this->output, $fileCount, "Importing to " . $mailbox); + $bar->advance(); + foreach ($files as $file) { + /* $this->info("Processing file: {$file}"); */ + + if ($type == "calendar") { + if (!self::caldavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) { + $this->error("Append failed: {$file}"); + $this->error("IMAP error: {$imap->error}"); + return; + } + } elseif ($type == "addressbook") { + if (!self::carddavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) { + $this->error("Append failed: {$file}"); + $this->error("IMAP error: {$imap->error}"); + return; + } + } else { + if (!self::appendFromFile($imap, $mailbox, $file)) { + $this->error("Append failed: {$file}"); + $this->error("IMAP error: {$imap->error}"); + return; + } + } + $bar->advance(); + } + $bar->finish(); + } + $imap->closeConnection(); + + $this->info("Finished importing folders."); + } +}