Changeset View
Changeset View
Standalone View
Standalone View
src/app/Console/Commands/MigrateUserdata.php
- This file was added.
| <?php | |||||
| namespace App\Console\Commands; | |||||
| use Illuminate\Console\Command; | |||||
| class MigrateUserdata extends Command | |||||
| { | |||||
| /** | |||||
| * The name and signature of the console command. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $signature = 'migrate:userdata | |||||
| {--importpst= : Pst file path} | |||||
| {--username= : Target user} | |||||
| {--password= : Password} | |||||
| {--davUrl= : caldav server url} | |||||
| {--clear-target : Remove all messages from the target mailbox.} | |||||
| {--subscribe : Subscribe to the target mailbox.} | |||||
| {--exclude-target=* : Do not process this target mailbox.} | |||||
| {--debug : Enable debug output}'; | |||||
| /** | |||||
| * The console command description. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $description = 'Migrate userdata'; | |||||
| /** | |||||
| * Execute the console command. | |||||
| * | |||||
| * @return mixed | |||||
| */ | |||||
| public function handle() | |||||
| { | |||||
| if ($this->option('importpst')) { | |||||
| $mailboxBlacklist = [ | |||||
| "Sync Issues (This computer only)", | |||||
| "Drafts (This computer only)" | |||||
| ]; | |||||
| $this->importPst( | |||||
| $this->option('importpst'), | |||||
| $this->option('username'), | |||||
| $this->option('password'), | |||||
| $this->option('clear-target'), | |||||
| $this->option('subscribe'), | |||||
| $this->option('exclude-target'), | |||||
| $this->option('debug'), | |||||
| $this->option('davUrl'), | |||||
| ); | |||||
| } | |||||
| } | |||||
| /** | |||||
| * Get LDAP configuration for specified access level | |||||
| */ | |||||
| private static function getConfig() | |||||
| { | |||||
| $uri = \parse_url(\config('imap.uri')); | |||||
| $default_port = 143; | |||||
| $ssl_mode = null; | |||||
| if (isset($uri['scheme'])) { | |||||
| if (preg_match('/^(ssl|imaps)/', $uri['scheme'])) { | |||||
| $default_port = 993; | |||||
| $ssl_mode = 'ssl'; | |||||
| } elseif ($uri['scheme'] === 'tls') { | |||||
| $ssl_mode = 'tls'; | |||||
| } | |||||
| } | |||||
| $config = [ | |||||
| 'host' => $uri['host'], | |||||
| 'user' => \config('imap.admin_login'), | |||||
| 'password' => \config('imap.admin_password'), | |||||
| 'options' => [ | |||||
| 'port' => !empty($uri['port']) ? $uri['port'] : $default_port, | |||||
| 'ssl_mode' => $ssl_mode, | |||||
| 'socket_options' => [ | |||||
| 'ssl' => [ | |||||
| 'verify_peer' => \config('imap.verify_peer'), | |||||
| 'verify_peer_name' => \config('imap.verify_peer'), | |||||
| 'verify_host' => \config('imap.verify_host') | |||||
| ], | |||||
| ], | |||||
| ], | |||||
| ]; | |||||
| return $config; | |||||
| } | |||||
| /** | |||||
| * Initialize connection to IMAP | |||||
| */ | |||||
| private static function initIMAP(array $config, string $login_as = null) | |||||
| { | |||||
| $imap = new \rcube_imap_generic(); | |||||
| if (\config('app.debug')) { | |||||
| $imap->setDebug(true, 'App\Backends\IMAP::logDebug'); | |||||
| } | |||||
| if ($login_as) { | |||||
| $config['options']['auth_cid'] = $config['user']; | |||||
| $config['options']['auth_pw'] = $config['password']; | |||||
| $config['options']['auth_type'] = 'PLAIN'; | |||||
| $config['user'] = $login_as; | |||||
| } | |||||
| $imap->connect($config['host'], $config['user'], $config['password'], $config['options']); | |||||
| if (!$imap->connected()) { | |||||
| $message = sprintf("Login failed for %s against %s. %s", $config['user'], $config['host'], $imap->error); | |||||
| \Log::error($message); | |||||
| throw new \Exception("Connection to IMAP failed"); | |||||
| } | |||||
| return $imap; | |||||
| } | |||||
| private static function imapFlagsFromStatus($status) | |||||
| { | |||||
| # libpst only sets R and O | |||||
| $flags = []; | |||||
| if (strpos($status, 'R') !== false || strpos($status, 'O') !== false) { | |||||
| $flags[] = 'SEEN'; | |||||
| } | |||||
| /* if 'D' in flags: */ | |||||
| /* $flags[] = '\Deleted' */ | |||||
| /* if 'A' in flags: */ | |||||
| /* $flags[] = '\Answered' */ | |||||
| /* if 'F' in flags */ | |||||
| /* $flags[] = '\Flagged' */ | |||||
| return $flags; | |||||
| } | |||||
| private static function appendFromFile($imap, $mailbox, $path) | |||||
| { | |||||
| // open message file | |||||
| if (file_exists(realpath($path))) { | |||||
| $fp = fopen($path, 'r'); | |||||
| } | |||||
| if (!$fp) { | |||||
| print("Failed to open for reading: " . $path); | |||||
| return false; | |||||
| } | |||||
| $message = fread($fp, filesize($path)); | |||||
| // IMAP requires CRLF | |||||
| $message = str_replace("\n", "\r\n", str_replace("\r", '', $message)); | |||||
| $flags = []; | |||||
| $matches = null; | |||||
| //In practice we only seem to get RO | |||||
| if (preg_match("/Status: (R?O?)\r\n/", $message, $matches)) { | |||||
| /* print("Found matches:\n"); */ | |||||
| /* print_r($matches); */ | |||||
| $flags = self::imapFlagsFromStatus($matches[1]); | |||||
| } | |||||
| $date = null; | |||||
| $binary = false; | |||||
| return $imap->append($mailbox, $message, $flags, $date, $binary); | |||||
| } | |||||
| private static function listDirectoryTree($dir) | |||||
| { | |||||
| $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir)); | |||||
| $directories = []; | |||||
| foreach ($it as $fileinfo) { | |||||
| //We look for all "." direcotires, because we somehow can't just iterate over directories without files. | |||||
| if ($fileinfo->isDir() && $fileinfo->getFilename() == "." && dirname($fileinfo->getPath()) != dirname($dir)) { | |||||
| $directories[] = $fileinfo->getPath(); | |||||
| } | |||||
| } | |||||
| return $directories; | |||||
| } | |||||
| private function createTemporaryDirectory() | |||||
| { | |||||
| $tempfile = tempnam(sys_get_temp_dir(), ''); | |||||
| if (file_exists($tempfile)) { | |||||
| unlink($tempfile); | |||||
| } | |||||
| mkdir($tempfile); | |||||
| if (!is_dir($tempfile)) { | |||||
| return null; | |||||
| } | |||||
| return $tempfile . "/"; | |||||
| } | |||||
| private function fixICal($ical) | |||||
| { | |||||
| $lines = explode("\n", $ical); | |||||
| for ($i = 0; $i < count($lines); $i++) { | |||||
| $line = $lines[$i]; | |||||
| //Remove nonsensical NONE categories libpst by Outlook | |||||
| if ($line == "CATEGORIES:NONE") { | |||||
| $lines[$i] = null; | |||||
| } | |||||
| } | |||||
| return implode("\n", array_filter($lines)); | |||||
| } | |||||
| private function caldavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path) | |||||
| { | |||||
| if (file_exists(realpath($path))) { | |||||
| $fp = fopen($path, 'r'); | |||||
| } | |||||
| if (!$fp) { | |||||
| print("Failed to open for reading: " . $path); | |||||
| return false; | |||||
| } | |||||
| $putdata = self::fixICal(fread($fp, filesize($path))); | |||||
| print($putdata . "\n"); | |||||
| $urlParts = parse_url($serverUrl); | |||||
| $newUrl = $urlParts['scheme'] . "://" . $urlParts['host']; | |||||
| if (isset($urlParts['port'])) { | |||||
| $newUrl = $newUrl . ":" . $urlParts['port']; | |||||
| } | |||||
| $url = $newUrl . $mailbox . basename($path); | |||||
| $c = curl_init(); | |||||
| curl_setopt($c, CURLOPT_URL, $url); | |||||
| curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/calendar; charset='UTF-8'")); | |||||
| curl_setopt($c, CURLOPT_HEADER, 0); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); | |||||
| curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); | |||||
| curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); | |||||
| curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT"); | |||||
| curl_setopt($c, CURLOPT_POSTFIELDS, $putdata); | |||||
| $data = curl_exec($c); | |||||
| if (!$data) { | |||||
| print("Append failed: " . curl_error($c) . "\n"); | |||||
| curl_close($c); | |||||
| return false; | |||||
| } | |||||
| curl_close($c); | |||||
| return true; | |||||
| } | |||||
| private function fixVCard($vcard) | |||||
| { | |||||
| $lines = explode("\n", $vcard); | |||||
| //Generate a UID if there isn't one | |||||
| if (strpos($vcard, "UID") === false) { | |||||
| $uid = uniqid(); | |||||
| array_splice($lines, 1, null, ["UID:{$uid}"]); | |||||
| } | |||||
| return implode("\n", $lines); | |||||
| } | |||||
| private function carddavAppendFromFile($serverUrl, $user, $pw, $mailbox, $path) | |||||
| { | |||||
| if (file_exists(realpath($path))) { | |||||
| $fp = fopen($path, 'r'); | |||||
| } | |||||
| if (!$fp) { | |||||
| print("Failed to open for reading: " . $path); | |||||
| return false; | |||||
| } | |||||
| $putdata = self::fixVCard(fread($fp, filesize($path))); | |||||
| print($putdata . "\n"); | |||||
| $urlParts = parse_url($serverUrl); | |||||
| $newUrl = $urlParts['scheme'] . "://" . $urlParts['host']; | |||||
| if (isset($urlParts['port'])) { | |||||
| $newUrl = $newUrl . ":" . $urlParts['port']; | |||||
| } | |||||
| $url = $newUrl . $mailbox . basename($path); | |||||
| $c = curl_init(); | |||||
| curl_setopt($c, CURLOPT_URL, $url); | |||||
| curl_setopt($c, CURLOPT_HTTPHEADER, array("Content-Type: text/vcard; charset='UTF-8'")); | |||||
| curl_setopt($c, CURLOPT_HEADER, 0); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); | |||||
| curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); | |||||
| curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); | |||||
| curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PUT"); | |||||
| curl_setopt($c, CURLOPT_POSTFIELDS, $putdata); | |||||
| $data = curl_exec($c); | |||||
| if (!$data) { | |||||
| print("Append failed: " . curl_error($c) . "\n"); | |||||
| curl_close($c); | |||||
| return false; | |||||
| } | |||||
| curl_close($c); | |||||
| return true; | |||||
| } | |||||
| private function findCaldavMailbox($serverUrl, $user, $pw, $mailbox) | |||||
| { | |||||
| $xml = '<d:propfind xmlns:d="DAV:" xmlns:cs="https://calendarserver.org/ns/"><d:prop><d:resourcetype /><d:displayname /></d:prop></d:propfind>'; | |||||
| $url = $serverUrl . "/calendars/{$user}/"; | |||||
| $c = curl_init(); | |||||
| curl_setopt($c, CURLOPT_URL, $url); | |||||
| curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal")); | |||||
| curl_setopt($c, CURLOPT_HEADER, 0); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); | |||||
| curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); | |||||
| curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); | |||||
| curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND"); | |||||
| curl_setopt($c, CURLOPT_POSTFIELDS, $xml); | |||||
| curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); | |||||
| $data = curl_exec($c); | |||||
| if (!$data) { | |||||
| print("Curl failed\n"); | |||||
| return null; | |||||
| } | |||||
| $xml = \simplexml_load_string($data, null, 0, "d", true); | |||||
| if (!$xml) { | |||||
| print("Failed to parse xml\n"); | |||||
| $errors = \libxml_get_errors(); | |||||
| foreach ($errors as $error) { | |||||
| print("Error: " . $error->message . "\n"); | |||||
| } | |||||
| \libxml_clear_errors(); | |||||
| return null; | |||||
| } | |||||
| foreach ($xml->response as $e) { | |||||
| $displayname = (string)$e->propstat->prop->displayname; | |||||
| if ($displayname == $mailbox) { | |||||
| return (string)$e->href; | |||||
| } | |||||
| } | |||||
| curl_close($c); | |||||
| return null; | |||||
| } | |||||
| private function findCarddavMailbox($serverUrl, $user, $pw, $mailbox) | |||||
| { | |||||
| $xml = '<d:propfind xmlns:d="DAV:" xmlns="urn:ietf:params:xml:ns:carddav"><d:prop><d:resourcetype /><d:displayname /></d:prop></d:propfind>'; | |||||
| $url = $serverUrl . "/addressbooks/{$user}/"; | |||||
| $c = curl_init(); | |||||
| curl_setopt($c, CURLOPT_URL, $url); | |||||
| curl_setopt($c, CURLOPT_HTTPHEADER, array("Depth: 1", "Content-Type: text/xml; charset='UTF-8'", "Prefer: return-minimal")); | |||||
| curl_setopt($c, CURLOPT_HEADER, 0); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); | |||||
| curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); | |||||
| curl_setopt($c, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); | |||||
| curl_setopt($c, CURLOPT_USERPWD, $user . ":" . $pw); | |||||
| curl_setopt($c, CURLOPT_CUSTOMREQUEST, "PROPFIND"); | |||||
| curl_setopt($c, CURLOPT_POSTFIELDS, $xml); | |||||
| curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); | |||||
| $data = curl_exec($c); | |||||
| if (!$data) { | |||||
| print("Curl failed\n"); | |||||
| return null; | |||||
| } | |||||
| $xml = \simplexml_load_string($data, null, 0, "d", true); | |||||
| if (!$xml) { | |||||
| print("Failed to parse xml\n"); | |||||
| $errors = \libxml_get_errors(); | |||||
| foreach ($errors as $error) { | |||||
| print("Error: " . $error->message . "\n"); | |||||
| } | |||||
| \libxml_clear_errors(); | |||||
| return null; | |||||
| } | |||||
| foreach ($xml->response as $e) { | |||||
| $displayname = (string)$e->propstat->prop->displayname; | |||||
| if ($displayname == $mailbox) { | |||||
| return (string)$e->href; | |||||
| } | |||||
| } | |||||
| curl_close($c); | |||||
| return null; | |||||
| } | |||||
| private function importPst($file, $username, $password, $clearTarget, $subscribe, $mailboxBlacklist, $debug, $davUrl) | |||||
| { | |||||
| if (!file_exists($file)) { | |||||
| //TODO downlaod file (see Utils.php) | |||||
| $this->error("File doesn't exist: " . $file); | |||||
| return; | |||||
| } | |||||
| $dir = $this->createTemporaryDirectory(); | |||||
| if (!$dir) { | |||||
| $this->error("Failed to create a temporary directory"); | |||||
| return; | |||||
| } | |||||
| $this->info("Using temporary directory: " . $dir); | |||||
| // Extract the pst file | |||||
| $output; | |||||
| $resultCode; | |||||
| if (!exec("readpst -e {$file} -o {$dir}", $output, $resultCode)) { | |||||
| $this->error("Failed to extract pst data"); | |||||
| return; | |||||
| } | |||||
| $this->info("Extracted pst file: \n" . implode("\n ", $output) . "\n"); | |||||
| $outputDirectories = self::listDirectoryTree($dir); | |||||
| $directoryRoot = array_shift($outputDirectories); | |||||
| $depth = substr_count($directoryRoot, '/'); | |||||
| /* $this->info("Directories: \n" . var_export($outputDirectories, true) . "\n"); */ | |||||
| // Prepare imap connection | |||||
| $imap = self::initIMAP(self::getConfig(), $username); | |||||
| $imap->setDebug($debug); | |||||
| if (!$imap) { | |||||
| $this->error("Failed to connect to imap."); | |||||
| return; | |||||
| } | |||||
| $this->info("Logged in to imap as user: {$username}"); | |||||
| $hierarchyDelimiter = $imap->getHierarchyDelimiter(); | |||||
| //TODO config option | |||||
| $importRoot = ''; | |||||
| $totalFolders = count($outputDirectories); | |||||
| $folderCounter = 0; | |||||
| foreach ($outputDirectories as $directory) { | |||||
| // Ignore the toplevel directory which is something like: "Outlook Data File" | |||||
| $mailboxParts = array_slice(explode('/', $directory), $depth + 1); | |||||
| //Move folders out of inbox | |||||
| if (count($mailboxParts) > 1 && strcasecmp($mailboxParts[0], "inbox") == 0) { | |||||
| $mailboxParts = array_slice($mailboxParts, 1); | |||||
| } | |||||
| $mailbox = $importRoot . implode($hierarchyDelimiter, $mailboxParts); | |||||
| $folderCounter++; | |||||
| if (in_array($mailbox, $mailboxBlacklist)) { | |||||
| $this->info("Skipping blacklisted mailbox"); | |||||
| continue; | |||||
| } | |||||
| $this->info("Importing folder: {$directory} to mailbox: {$mailbox}"); | |||||
| $this->info("Folder {$folderCounter} out of {$totalFolders}"); | |||||
| $files = glob("{$directory}/*.eml"); | |||||
| $fileCount = count($files); | |||||
| $type = "mail"; | |||||
| if (!$fileCount) { | |||||
| $files = glob("{$directory}/*.ics"); | |||||
| $fileCount = count($files); | |||||
| if ($fileCount) { | |||||
| $type = "calendar"; | |||||
| } else { | |||||
| $files = glob("{$directory}/*.vcf"); | |||||
| $fileCount = count($files); | |||||
| if ($fileCount) { | |||||
| $type = "addressbook"; | |||||
| } | |||||
| } | |||||
| } | |||||
| if (!$fileCount) { | |||||
| $this->info("Nothing to do"); | |||||
| continue; | |||||
| } | |||||
| $count = $imap->countMessages($mailbox); | |||||
| if ($count === false) { | |||||
| $this->info("Creating mailbox: {$directory}"); | |||||
| //TODO set specialuse; | |||||
| if (!$imap->createFolder($mailbox)) { | |||||
| $this->error("Failed to create mailbox: {$mailbox}"); | |||||
| return; | |||||
| } | |||||
| if ($type == "calendar") { | |||||
| if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => 'event'])) { | |||||
| $this->error("Failed to set metadata on: {$mailbox}"); | |||||
| return; | |||||
| } | |||||
| } | |||||
| if ($type == "addressbook") { | |||||
| if (!$imap->setMetadata($mailbox, ["/private/vendor/kolab/folder-type" => 'contact'])) { | |||||
| $this->error("Failed to set metadata on: {$mailbox}"); | |||||
| return; | |||||
| } | |||||
| } | |||||
| } elseif ($count) { | |||||
| if ($clearTarget) { | |||||
| $imap->clearFolder($mailbox); | |||||
| } else { | |||||
| $this->error("Target mailbox is not empty: {$mailbox}"); | |||||
| return; | |||||
| } | |||||
| } | |||||
| if ($subscribe) { | |||||
| $imap->subscribe($mailbox); | |||||
| } | |||||
| if ($type == "calendar") { | |||||
| $mailbox = self::findCaldavMailbox($davUrl, $username, $password, $mailbox); | |||||
| } elseif ($type == "addressbook") { | |||||
| $mailbox = self::findCarddavMailbox($davUrl, $username, $password, $mailbox); | |||||
| } else { | |||||
| if (!$imap->select($mailbox)) { | |||||
| $this->error("Failed to select: {$mailbox}"); | |||||
| return; | |||||
| } | |||||
| } | |||||
| if (!$mailbox) { | |||||
| $this->error("Failed to get the target mailbox."); | |||||
| return; | |||||
| } | |||||
| $bar = \App\Utils::createProgressBar($this->output, $fileCount, "Importing to " . $mailbox); | |||||
| $bar->advance(); | |||||
| foreach ($files as $file) { | |||||
| /* $this->info("Processing file: {$file}"); */ | |||||
| if ($type == "calendar") { | |||||
| if (!self::caldavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) { | |||||
| $this->error("Append failed: {$file}"); | |||||
| $this->error("IMAP error: {$imap->error}"); | |||||
| return; | |||||
| } | |||||
| } elseif ($type == "addressbook") { | |||||
| if (!self::carddavAppendFromFile($davUrl, $username, $password, $mailbox, $file)) { | |||||
| $this->error("Append failed: {$file}"); | |||||
| $this->error("IMAP error: {$imap->error}"); | |||||
| return; | |||||
| } | |||||
| } else { | |||||
| if (!self::appendFromFile($imap, $mailbox, $file)) { | |||||
| $this->error("Append failed: {$file}"); | |||||
| $this->error("IMAP error: {$imap->error}"); | |||||
| return; | |||||
| } | |||||
| } | |||||
| $bar->advance(); | |||||
| } | |||||
| $bar->finish(); | |||||
| } | |||||
| $imap->closeConnection(); | |||||
| $this->info("Finished importing folders."); | |||||
| } | |||||
| } | |||||