diff --git a/bin/quickstart.sh b/bin/quickstart.sh --- a/bin/quickstart.sh +++ b/bin/quickstart.sh @@ -89,6 +89,8 @@ rm -rf database/database.sqlite ./artisan db:ping --wait php -dmemory_limit=512M ./artisan migrate:refresh --seed +./artisan data:import +./artisan swoole:http stop >/dev/null 2>&1 || : ./artisan swoole:http start popd diff --git a/extras/kolab_policy_greylist.py b/extras/kolab_policy_greylist.py new file mode 100755 --- /dev/null +++ b/extras/kolab_policy_greylist.py @@ -0,0 +1,79 @@ +#!/usr/bin/python3 +""" +An example implementation of a policy service. +""" + +import json +import time +import sys + +import requests + + +def read_request_input(): + """ + Read a single policy request from sys.stdin, and return a dictionary + containing the request. + """ + start_time = time.time() + + policy_request = {} + end_of_request = False + + while not end_of_request: + if (time.time() - start_time) >= 10: + sys.exit(0) + + request_line = sys.stdin.readline() + + if request_line.strip() == '': + if 'request' in policy_request: + end_of_request = True + else: + request_line = request_line.strip() + request_key = request_line.split('=')[0] + request_value = '='.join(request_line.split('=')[1:]) + + policy_request[request_key] = request_value + + return policy_request + + +if __name__ == "__main__": + URL = 'https://services.kolabnow.com/api/webhooks/policy/greylist' + + # Start the work + while True: + REQUEST = read_request_input() + + try: + RESPONSE = requests.post( + URL, + data=REQUEST, + verify=True + ) + # pylint: disable=broad-except + except Exception: + print("action=DEFER_IF_PERMIT Temporary error, try again later.") + sys.exit(1) + + try: + R = json.loads(RESPONSE.text) + # pylint: disable=broad-except + except Exception: + sys.exit(1) + + if 'prepend' in R: + for prepend in R['prepend']: + print("action=PREPEND {0}".format(prepend)) + + if RESPONSE.ok: + print("action={0}\n".format(R['response'])) + + sys.stdout.flush() + else: + print("action={0} {1}\n".format(R['response'], R['reason'])) + + sys.stdout.flush() + + sys.exit(0) diff --git a/extras/kolab_policy_ratelimit.py b/extras/kolab_policy_ratelimit.py new file mode 100755 --- /dev/null +++ b/extras/kolab_policy_ratelimit.py @@ -0,0 +1,79 @@ +#!/usr/bin/python3 +""" +This policy applies rate limitations +""" + +import json +import time +import sys + +import requests + + +def read_request_input(): + """ + Read a single policy request from sys.stdin, and return a dictionary + containing the request. + """ + start_time = time.time() + + policy_request = {} + end_of_request = False + + while not end_of_request: + if (time.time() - start_time) >= 10: + sys.exit(0) + + request_line = sys.stdin.readline() + + if request_line.strip() == '': + if 'request' in policy_request: + end_of_request = True + else: + request_line = request_line.strip() + request_key = request_line.split('=')[0] + request_value = '='.join(request_line.split('=')[1:]) + + policy_request[request_key] = request_value + + return policy_request + + +if __name__ == "__main__": + URL = 'https://services.kolabnow.com/api/webhooks/policy/ratelimit' + + # Start the work + while True: + REQUEST = read_request_input() + + try: + RESPONSE = requests.post( + URL, + data=REQUEST, + verify=True + ) + # pylint: disable=broad-except + except Exception: + print("action=DEFER_IF_PERMIT Temporary error, try again later.") + sys.exit(1) + + try: + R = json.loads(RESPONSE.text) + # pylint: disable=broad-except + except Exception: + sys.exit(1) + + if 'prepend' in R: + for prepend in R['prepend']: + print("action=PREPEND {0}".format(prepend)) + + if RESPONSE.ok: + print("action={0}\n".format(R['response'])) + + sys.stdout.flush() + else: + print("action={0} {1}\n".format(R['response'], R['reason'])) + + sys.stdout.flush() + + sys.exit(0) diff --git a/extras/kolab_policy_spf.py b/extras/kolab_policy_spf.py new file mode 100755 --- /dev/null +++ b/extras/kolab_policy_spf.py @@ -0,0 +1,80 @@ +#!/usr/bin/python3 +""" +This is the implementation of a (postfix) MTA policy service to enforce the +Sender Policy Framework. +""" + +import json +import time +import sys + +import requests + + +def read_request_input(): + """ + Read a single policy request from sys.stdin, and return a dictionary + containing the request. + """ + start_time = time.time() + + policy_request = {} + end_of_request = False + + while not end_of_request: + if (time.time() - start_time) >= 10: + sys.exit(0) + + request_line = sys.stdin.readline() + + if request_line.strip() == '': + if 'request' in policy_request: + end_of_request = True + else: + request_line = request_line.strip() + request_key = request_line.split('=')[0] + request_value = '='.join(request_line.split('=')[1:]) + + policy_request[request_key] = request_value + + return policy_request + + +if __name__ == "__main__": + URL = 'https://services.kolabnow.com/api/webhooks/policy/spf' + + # Start the work + while True: + REQUEST = read_request_input() + + try: + RESPONSE = requests.post( + URL, + data=REQUEST, + verify=True + ) + # pylint: disable=broad-except + except Exception: + print("action=DEFER_IF_PERMIT Temporary error, try again later.") + sys.exit(1) + + try: + R = json.loads(RESPONSE.text) + # pylint: disable=broad-except + except Exception: + sys.exit(1) + + if 'prepend' in R: + for prepend in R['prepend']: + print("action=PREPEND {0}".format(prepend)) + + if RESPONSE.ok: + print("action={0}\n".format(R['response'])) + + sys.stdout.flush() + else: + print("action={0} {1}\n".format(R['response'], R['reason'])) + + sys.stdout.flush() + + sys.exit(0) diff --git a/src/.gitignore b/src/.gitignore --- a/src/.gitignore +++ b/src/.gitignore @@ -7,6 +7,8 @@ public/js/*.js public/storage/ storage/*.key +storage/*.log +storage/*-????-??-??* storage/export/ tests/report/ vendor diff --git a/src/app/Console/Commands/User/GreylistCommand.php b/src/app/Console/Commands/User/GreylistCommand.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/User/GreylistCommand.php @@ -0,0 +1,73 @@ +argument('user'); + $recipientHash = hash('sha256', $recipientAddress); + + $lastConnect = \App\Policy\Greylist\Connect::where('recipient_hash', $recipientHash) + ->orderBy('updated_at', 'desc') + ->first(); + + if ($lastConnect) { + $timestamp = $lastConnect->updated_at->copy(); + $this->info("Going from timestamp (last connect) {$timestamp}"); + } else { + $timestamp = \Carbon\Carbon::now(); + $this->info("Going from timestamp (now) {$timestamp}"); + } + + + \App\Policy\Greylist\Connect::where('recipient_hash', $recipientHash) + ->where('greylisting', true) + ->whereDate('updated_at', '>=', $timestamp->copy()->subDays(7)) + ->orderBy('created_at')->each( + function ($connect) { + $this->info( + sprintf( + "From %s@%s since %s", + $connect->sender_local, + $connect->sender_domain, + $connect->created_at + ) + ); + } + ); + } +} diff --git a/src/app/Domain.php b/src/app/Domain.php --- a/src/app/Domain.php +++ b/src/app/Domain.php @@ -3,6 +3,8 @@ namespace App; use App\Wallet; +use App\Traits\DomainConfigTrait; +use App\Traits\SettingsTrait; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; @@ -16,6 +18,8 @@ */ class Domain extends Model { + use DomainConfigTrait; + use SettingsTrait; use SoftDeletes; // we've simply never heard of this domain @@ -366,6 +370,16 @@ } /** + * Any (additional) properties of this domain. + * + * @return \Illuminate\Database\Eloquent\Relations\HasMany + */ + public function settings() + { + return $this->hasMany('App\DomainSetting', 'domain_id'); + } + + /** * Suspend this domain. * * @return void diff --git a/src/app/DomainSetting.php b/src/app/DomainSetting.php new file mode 100644 --- /dev/null +++ b/src/app/DomainSetting.php @@ -0,0 +1,34 @@ +belongsTo( + '\App\Domain', + 'domain_id', /* local */ + 'id' /* remote */ + ); + } +} diff --git a/src/app/Http/Controllers/API/V4/Admin/UsersController.php b/src/app/Http/Controllers/API/V4/Admin/UsersController.php --- a/src/app/Http/Controllers/API/V4/Admin/UsersController.php +++ b/src/app/Http/Controllers/API/V4/Admin/UsersController.php @@ -185,6 +185,8 @@ $response['skus'][$sku->id]['costs'][] = $ent->cost; } + $response['config'] = $user->getConfig(); + return response()->json($response); } diff --git a/src/app/Http/Controllers/API/V4/DomainsController.php b/src/app/Http/Controllers/API/V4/DomainsController.php --- a/src/app/Http/Controllers/API/V4/DomainsController.php +++ b/src/app/Http/Controllers/API/V4/DomainsController.php @@ -99,6 +99,39 @@ } /** + * Set the domain configuration. + * + * @param int $id Domain identifier + * + * @return \Illuminate\Http\JsonResponse|void + */ + public function setConfig($id) + { + $domain = Domain::find($id); + + if (empty($domain)) { + return $this->errorResponse(404); + } + + // Only owner (or admin) has access to the domain + if (!$this->guard()->user()->canRead($domain)) { + return $this->errorResponse(403); + } + + $errors = $domain->setConfig(request()->input()); + + if (!empty($errors)) { + return response()->json(['status' => 'error', 'errors' => $errors], 422); + } + + return response()->json([ + 'status' => 'success', + 'message' => \trans('app.domain-setconfig-success'), + ]); + } + + + /** * Store a newly created resource in storage. * * @param \Illuminate\Http\Request $request @@ -138,7 +171,10 @@ // Add DNS/MX configuration for the domain $response['dns'] = self::getDNSConfig($domain); - $response['config'] = self::getMXConfig($domain->namespace); + $response['mx'] = self::getMXConfig($domain->namespace); + + // Domain configuration, e.g. spf whitelist + $response['config'] = $domain->getConfig(); // Status info $response['statusInfo'] = self::statusInfo($domain); diff --git a/src/app/Http/Controllers/API/V4/PolicyController.php b/src/app/Http/Controllers/API/V4/PolicyController.php new file mode 100644 --- /dev/null +++ b/src/app/Http/Controllers/API/V4/PolicyController.php @@ -0,0 +1,230 @@ +input(); + + list($local, $domainName) = explode('@', $data['recipient']); + + $request = new \App\Policy\Greylist\Request($data); + + $shouldDefer = $request->shouldDefer(); + + if ($shouldDefer) { + return response()->json( + ['response' => 'DEFER_IF_PERMIT', 'reason' => "Greylisted for 5 minutes. Try again later."], + 403 + ); + } + + $prependGreylist = $request->headerGreylist(); + + $result = [ + 'response' => 'DUNNO', + 'prepend' => [$prependGreylist] + ]; + + return response()->json($result, 200); + } + + /* + * Apply a sensible rate limitation to a request. + * + * @return \Illuminate\Http\JsonResponse + */ + public function ratelimit() + { + /* + $data = [ + 'instance' => 'test.local.instance', + 'protocol_state' => 'RCPT', + 'sender' => 'sender@spf-pass.kolab.org', + 'client_name' => 'mx.kolabnow.com', + 'client_address' => '212.103.80.148', + 'recipient' => $this->domainOwner->email + ]; + + $response = $this->post('/api/webhooks/spf', $data); + */ +/* + $data = \request()->input(); + + // TODO: normalize sender address + $sender = strtolower($data['sender']); + + $alias = \App\UserAlias::where('alias', $sender)->first(); + + if (!$alias) { + $user = \App\User::where('email', $sender)->first(); + + if (!$user) { + // what's the situation here? + } + } else { + $user = $alias->user; + } + + // TODO time-limit + $userRates = \App\Policy\Ratelimit::where('user_id', $user->id); + + // TODO message vs. recipient limit + if ($userRates->count() > 10) { + // TODO + } + + // this is the wallet to which the account is billed + $wallet = $user->wallet; + + // TODO: consider $wallet->payments; + + $owner = $wallet->user; + + // TODO time-limit + $ownerRates = \App\Policy\Ratelimit::where('owner_id', $owner->id); + + // TODO message vs. recipient limit (w/ user counts) + if ($ownerRates->count() > 10) { + // TODO + } +*/ + } + + /* + * Apply the sender policy framework to a request. + * + * @return \Illuminate\Http\JsonResponse + */ + public function senderPolicyFramework() + { + $data = \request()->input(); + + list($netID, $netType) = \App\Utils::getNetFromAddress($data['client_address']); + list($senderLocal, $senderDomain) = explode('@', $data['sender']); + + // This network can not be recognized. + if (!$netID) { + return response()->json( + [ + 'response' => 'DEFER_IF_PERMIT', + 'reason' => 'Temporary error. Please try again later.' + ], + 403 + ); + } + + // Compose the cache key we want. + $cacheKey = "{$netType}_{$netID}_{$senderDomain}"; + + $result = \App\Policy\SPF\Cache::get($cacheKey); + + if (!$result) { + $environment = new \SPFLib\Check\Environment( + $data['client_address'], + $data['client_name'], + $data['sender'] + ); + + $result = (new \SPFLib\Checker())->check($environment); + + \App\Policy\SPF\Cache::set($cacheKey, serialize($result)); + } else { + $result = unserialize($result); + } + + $fail = false; + $prependSPF = ''; + + switch ($result->getCode()) { + case \SPFLib\Check\Result::CODE_ERROR_PERMANENT: + $fail = true; + $prependSPF = "Received-SPF: Permerror"; + break; + + case \SPFLib\Check\Result::CODE_ERROR_TEMPORARY: + $prependSPF = "Received-SPF: Temperror"; + break; + + case \SPFLib\Check\Result::CODE_FAIL: + $fail = true; + $prependSPF = "Received-SPF: Fail"; + break; + + case \SPFLib\Check\Result::CODE_SOFTFAIL: + $prependSPF = "Received-SPF: Softfail"; + break; + + case \SPFLib\Check\Result::CODE_NEUTRAL: + $prependSPF = "Received-SPF: Neutral"; + break; + + case \SPFLib\Check\Result::CODE_PASS: + $prependSPF = "Received-SPF: Pass"; + break; + + case \SPFLib\Check\Result::CODE_NONE: + $prependSPF = "Received-SPF: None"; + break; + } + + $prependSPF .= " identity=mailfrom;"; + $prependSPF .= " client-ip={$data['client_address']};"; + $prependSPF .= " helo={$data['client_name']};"; + $prependSPF .= " envelope-from={$data['sender']};"; + + if ($fail) { + // TODO: check the recipient's policy, such as using barracuda for anti-spam and anti-virus as a relay for + // inbound mail to a local recipient address. + $objects = \App\Utils::findObjectsByRecipientAddress($data['recipient']); + + if (!empty($objects)) { + // check if any of the recipient objects have whitelisted the helo, first one wins. + foreach ($objects as $object) { + if (method_exists($object, 'senderPolicyFrameworkWhitelist')) { + $result = $object->senderPolicyFrameworkWhitelist($data['client_name']); + + if ($result) { + $response = [ + 'response' => 'DUNNO', + 'prepend' => ["Received-SPF: Pass Check skipped at recipient's discretion"], + 'reason' => 'HELO name whitelisted' + ]; + + return response()->json($response, 200); + } + } + } + } + + $result = [ + 'response' => 'REJECT', + 'prepend' => [$prependSPF], + 'reason' => "Prohibited by Sender Policy Framework" + ]; + + return response()->json($result, 403); + } + + $result = [ + 'response' => 'DUNNO', + 'prepend' => [$prependSPF], + 'reason' => "Don't know" + ]; + + return response()->json($result, 200); + } +} diff --git a/src/app/Http/Controllers/API/V4/UsersController.php b/src/app/Http/Controllers/API/V4/UsersController.php --- a/src/app/Http/Controllers/API/V4/UsersController.php +++ b/src/app/Http/Controllers/API/V4/UsersController.php @@ -87,6 +87,37 @@ } /** + * Set user config. + * + * @param int $id The user + * + * @return \Illuminate\Http\JsonResponse + */ + public function setConfig($id) + { + $user = User::find($id); + + if (empty($user)) { + return $this->errorResponse(404); + } + + if (!$this->guard()->user()->canRead($user)) { + return $this->errorResponse(403); + } + + $errors = $user->setConfig(request()->input()); + + if (!empty($errors)) { + return response()->json(['status' => 'error', 'errors' => $errors], 422); + } + + return response()->json([ + 'status' => 'success', + 'message' => __('app.user-setconfig-success'), + ]); + } + + /** * Display information on the user account specified by $id. * * @param int $id The account to show information for. @@ -119,6 +150,8 @@ $response['skus'][$sku->id]['costs'][] = $ent->cost; } + $response['config'] = $user->getConfig(); + return response()->json($response); } diff --git a/src/app/Http/Kernel.php b/src/app/Http/Kernel.php --- a/src/app/Http/Kernel.php +++ b/src/app/Http/Kernel.php @@ -44,7 +44,7 @@ ], 'api' => [ - 'throttle:120,1', + //'throttle:120,1', 'bindings', ], ]; diff --git a/src/app/IP4Net.php b/src/app/IP4Net.php --- a/src/app/IP4Net.php +++ b/src/app/IP4Net.php @@ -10,10 +10,31 @@ protected $table = "ip4nets"; protected $fillable = [ + 'rir_name', 'net_number', 'net_mask', 'net_broadcast', 'country', - 'serial' + 'serial', + 'created_at', + 'updated_at' ]; + + public static function getNet($ip, $mask = 32) + { + $query = " + SELECT id FROM ip4nets + WHERE INET_ATON(net_number) <= INET_ATON(?) + AND INET_ATON(net_broadcast) >= INET_ATON(?) + ORDER BY INET_ATON(net_number), net_mask DESC LIMIT 1 + "; + + $results = DB::select($query, [$ip, $ip]); + + if (sizeof($results) == 0) { + return null; + } + + return \App\IP4Net::find($results[0]->id); + } } diff --git a/src/app/Observers/SignupCodeObserver.php b/src/app/Observers/SignupCodeObserver.php --- a/src/app/Observers/SignupCodeObserver.php +++ b/src/app/Observers/SignupCodeObserver.php @@ -41,7 +41,8 @@ }) ->map(function ($value) { return is_array($value) && count($value) == 1 ? $value[0] : $value; - }); + }) + ->all(); $code->expires_at = Carbon::now()->addHours($exp_hours); $code->ip_address = request()->ip(); diff --git a/src/app/Policy/Greylist/Connect.php b/src/app/Policy/Greylist/Connect.php new file mode 100644 --- /dev/null +++ b/src/app/Policy/Greylist/Connect.php @@ -0,0 +1,62 @@ +recipient_type == \App\Domain::class) { + return $this->recipient; + } + + return null; + } + + // determine if the sender is a penpal of the recipient. + public function isPenpal() + { + return false; + } + + public function user() + { + if ($this->recipient_type == \App\User::class) { + return $this->recipient; + } + + return null; + } + + public function net() + { + return $this->morphTo(); + } + + public function recipient() + { + return $this->morphTo(); + } +} diff --git a/src/app/Policy/Greylist/Request.php b/src/app/Policy/Greylist/Request.php new file mode 100644 --- /dev/null +++ b/src/app/Policy/Greylist/Request.php @@ -0,0 +1,299 @@ +request = $request; + + if (array_key_exists('timestamp', $this->request)) { + $this->timestamp = \Carbon\Carbon::parse($this->request['timestamp']); + } else { + $this->timestamp = \Carbon\Carbon::now(); + } + } + + public function headerGreylist() + { + if ($this->whitelist) { + if ($this->whitelist->sender_local) { + return sprintf( + "Received-Greylist: sender %s whitelisted since %s", + $this->sender, + $this->whitelist->created_at->toDateString() + ); + } + + return sprintf( + "Received-Greylist: domain %s from %s whitelisted since %s (UTC)", + $this->senderDomain, + $this->request['client_address'], + $this->whitelist->created_at->toDateTimeString() + ); + } + + $connect = $this->findConnectsCollection()->orderBy('created_at')->first(); + + if ($connect) { + return sprintf( + "Received-Greylist: greylisted from %s until %s.", + $connect->created_at, + $this->timestamp + ); + } + + return "Received-Greylist: no opinion here"; + } + + public function shouldDefer() + { + $deferIfPermit = true; + + list($this->netID, $this->netType) = \App\Utils::getNetFromAddress($this->request['client_address']); + + if (!$this->netID) { + return true; + } + + $recipient = $this->recipientFromRequest(); + + $this->sender = $this->senderFromRequest(); + + list($this->senderLocal, $this->senderDomain) = explode('@', $this->sender); + + $entry = $this->findConnectsCollectionRecent()->orderBy('updated_at')->first(); + + if (!$entry) { + // purge all entries to avoid a unique constraint violation. + $this->findConnectsCollection()->delete(); + + $entry = Connect::create( + [ + 'sender_local' => $this->senderLocal, + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType, + 'recipient_hash' => $this->recipientHash, + 'recipient_id' => $this->recipientID, + 'recipient_type' => $this->recipientType, + 'connect_count' => 1, + 'created_at' => $this->timestamp, + 'updated_at' => $this->timestamp + ] + ); + } + + // see if all recipients and their domains are opt-outs + $enabled = false; + + if ($recipient) { + $setting = Setting::where( + [ + 'object_id' => $this->recipientID, + 'object_type' => $this->recipientType, + 'key' => 'greylist_enabled' + ] + )->first(); + + if (!$setting) { + $setting = Setting::where( + [ + 'object_id' => $recipient->domain()->id, + 'object_type' => \App\Domain::class, + 'key' => 'greylist_enabled' + ] + )->first(); + + if (!$setting) { + $enabled = true; + } else { + if ($setting->{'value'} !== 'false') { + $enabled = true; + } + } + } else { + if ($setting->{'value'} !== 'false') { + $enabled = true; + } + } + } else { + $enabled = true; + } + + // the following block is to maintain statistics and state ... + $entries = Connect::where( + [ + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType + ] + ) + ->whereDate('updated_at', '>=', $this->timestamp->copy()->subDays(7)); + + // determine if the sender domain is a whitelist from this network + $this->whitelist = Whitelist::where( + [ + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType + ] + )->first(); + + if ($this->whitelist) { + if ($this->whitelist->updated_at < $this->timestamp->copy()->subMonthsWithoutOverflow(1)) { + $this->whitelist->delete(); + } else { + $this->whitelist->updated_at = $this->timestamp; + $this->whitelist->save(['timestamps' => false]); + + $entries->update( + [ + 'greylisting' => false, + 'updated_at' => $this->timestamp + ] + ); + + return false; + } + } else { + if ($entries->count() >= 5) { + $this->whitelist = Whitelist::create( + [ + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType, + 'created_at' => $this->timestamp, + 'updated_at' => $this->timestamp + ] + ); + + $entries->update( + [ + 'greylisting' => false, + 'updated_at' => $this->timestamp + ] + ); + } + } + + // TODO: determine if the sender (individual) is a whitelist + + // TODO: determine if the sender is a penpal of any of the recipients. First recipient wins. + + if (!$enabled) { + return false; + } + + // determine if the sender, net and recipient combination has existed before, for each recipient + // any one recipient matching should supersede the other recipients not having matched + $connect = Connect::where( + [ + 'sender_local' => $this->senderLocal, + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType, + 'recipient_id' => $this->recipientID, + 'recipient_type' => $this->recipientType, + ] + ) + ->whereDate('updated_at', '>=', $this->timestamp->copy()->subMonthsWithoutOverflow(1)) + ->orderBy('updated_at') + ->first(); + + if (!$connect) { + $connect = Connect::create( + [ + 'sender_local' => $this->senderLocal, + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType, + 'recipient_id' => $this->recipientID, + 'recipient_type' => $this->recipientType, + 'connect_count' => 0, + 'created_at' => $this->timestamp, + 'updated_at' => $this->timestamp + ] + ); + } + + $connect->connect_count += 1; + + // TODO: The period of time for which the greylisting persists is configurable. + if ($connect->created_at < $this->timestamp->copy()->subMinutes(5)) { + $deferIfPermit = false; + + $connect->greylisting = false; + } + + $connect->save(); + + return $deferIfPermit; + } + + private function findConnectsCollection() + { + $collection = Connect::where( + [ + 'sender_local' => $this->senderLocal, + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType, + 'recipient_id' => $this->recipientID, + 'recipient_type' => $this->recipientType + ] + ); + + return $collection; + } + + private function findConnectsCollectionRecent() + { + return $this->findConnectsCollection() + ->where('updated_at', '>=', $this->timestamp->copy()->subDays(7)); + } + + private function recipientFromRequest() + { + $recipients = \App\Utils::findObjectsByRecipientAddress($this->request['recipient']); + + if (sizeof($recipients) > 1) { + \Log::warning( + "Only taking the first recipient from the request in to account for {$this->request['recipient']}" + ); + } + + if (count($recipients) >= 1) { + $recipient = $recipients[0]; + $this->recipientID = $recipient->id; + $this->recipientType = get_class($recipient); + } else { + $recipient = null; + } + + $this->recipientHash = hash('sha256', $this->request['recipient']); + + return $recipient; + } + + public function senderFromRequest() + { + return \App\Utils::normalizeAddress($this->request['sender']); + } +} diff --git a/src/app/Policy/Greylist/Setting.php b/src/app/Policy/Greylist/Setting.php new file mode 100644 --- /dev/null +++ b/src/app/Policy/Greylist/Setting.php @@ -0,0 +1,17 @@ +getSetting('spf_whitelist'); + + $config['spf_whitelist'] = $spf ? json_decode($spf, true) : []; + + return $config; + } + + /** + * A helper to update domain configuration. + * + * @param array $config An array of configuration options + * + * @return array A list of input validation errors + */ + public function setConfig(array $config): array + { + $errors = []; + + foreach ($config as $key => $value) { + // validate and save SPF whitelist entries + if ($key === 'spf_whitelist') { + if (!is_array($value)) { + $value = (array) $value; + } + + foreach ($value as $i => $v) { + $v = rtrim($v, '.'); + + if (empty($v)) { + unset($value[$i]); + continue; + } + + $value[$i] = $v; + + if ($v[0] !== '.' || !filter_var(substr($v, 1), FILTER_VALIDATE_DOMAIN)) { + $errors[$key][$i] = \trans('validation.spf-entry-invalid'); + } + } + + if (empty($errors[$key])) { + $this->setSetting($key, json_encode($value)); + } + } else { + $errors[$key] = \trans('validation.invalid-config-parameter'); + } + } + + return $errors; + } +} diff --git a/src/app/Traits/UserConfigTrait.php b/src/app/Traits/UserConfigTrait.php new file mode 100644 --- /dev/null +++ b/src/app/Traits/UserConfigTrait.php @@ -0,0 +1,42 @@ +getSetting('greylisting') !== 'false'; + + return $config; + } + + /** + * A helper to update user configuration. + * + * @param array $config An array of configuration options + * + * @return array A list of input validation error messages + */ + public function setConfig(array $config): array + { + $errors = []; + + foreach ($config as $key => $value) { + if ($key == 'greylisting') { + $this->setSetting('greylisting', $value ? 'true' : 'false'); + } else { + $errors[$key] = \trans('validation.invalid-config-parameter'); + } + } + + return $errors; + } +} diff --git a/src/app/User.php b/src/app/User.php --- a/src/app/User.php +++ b/src/app/User.php @@ -5,6 +5,7 @@ use App\Entitlement; use App\UserAlias; use App\Sku; +use App\Traits\UserConfigTrait; use App\Traits\UserAliasesTrait; use App\Traits\SettingsTrait; use App\Wallet; @@ -25,6 +26,7 @@ class User extends Authenticatable implements JWTSubject { use NullableFields; + use UserConfigTrait; use UserAliasesTrait; use SettingsTrait; use SoftDeletes; @@ -588,6 +590,46 @@ return $this; } + public function senderPolicyFrameworkWhitelist($clientName) + { + $setting = $this->getSetting('spf_whitelist'); + + if (!$setting) { + return false; + } + + $whitelist = json_decode($setting); + + $matchFound = false; + + foreach ($whitelist as $entry) { + if (substr($entry, 0, 1) == '/') { + $match = preg_match($entry, $clientName); + + if ($match) { + $matchFound = true; + } + + continue; + } + + if (substr($entry, 0, 1) == '.') { + if (substr($clientName, (-1 * strlen($entry))) == $entry) { + $matchFound = true; + } + + continue; + } + + if ($entry == $clientName) { + $matchFound = true; + continue; + } + } + + return $matchFound; + } + /** * Any (additional) properties of this user. * diff --git a/src/app/Utils.php b/src/app/Utils.php --- a/src/app/Utils.php +++ b/src/app/Utils.php @@ -45,7 +45,7 @@ */ public static function countryForIP($ip) { - if (strpos(':', $ip) === false) { + if (strpos($ip, ':') === false) { $query = " SELECT country FROM ip4nets WHERE INET_ATON(net_number) <= INET_ATON(?) @@ -192,6 +192,84 @@ } /** + * Find an object that is the recipient for the specified address. + * + * @param string $address + * + * @return array + */ + public static function findObjectsByRecipientAddress($address) + { + $address = \App\Utils::normalizeAddress($address); + + list($local, $domainName) = explode('@', $address); + + $domain = \App\Domain::where('namespace', $domainName)->first(); + + if (!$domain) { + return []; + } + + $user = \App\User::where('email', $address)->first(); + + if ($user) { + return [$user]; + } + + $userAliases = \App\UserAlias::where('alias', $address)->get(); + + if (count($userAliases) > 0) { + $users = []; + + foreach ($userAliases as $userAlias) { + $users[] = $userAlias->user; + } + + return $users; + } + + $userAliases = \App\UserAlias::where('alias', "catchall@{$domain->namespace}")->get(); + + if (count($userAliases) > 0) { + $users = []; + + foreach ($userAliases as $userAlias) { + $users[] = $userAlias->user; + } + + return $users; + } + + return []; + } + + /** + * Retrieve the network ID and Type from a client address + * + * @param string $clientAddress The IPv4 or IPv6 address. + * + * @return array An array of ID and class or null and null. + */ + public static function getNetFromAddress($clientAddress) + { + if (strpos($clientAddress, ':') === false) { + $net = \App\IP4Net::getNet($clientAddress); + + if ($net) { + return [$net->id, \App\IP4Net::class]; + } + } else { + $net = \App\IP6Net::getNet($clientAddress); + + if ($net) { + return [$net->id, \App\IP6Net::class]; + } + } + + return [null, null]; + } + + /** * Calculate the broadcast address provided a net number and a prefix. * * @param string $net A valid IPv6 network number. @@ -245,6 +323,32 @@ } /** + * Normalize an email address. + * + * This means to lowercase and strip components separated with recipient delimiters. + * + * @param string $address The address to normalize. + * + * @return string + */ + public static function normalizeAddress($address) + { + $address = strtolower($address); + + list($local, $domain) = explode('@', $address); + + if (strpos($local, '+') === false) { + return "{$local}@{$domain}"; + } + + $localComponents = explode('+', $local); + + $local = array_pop($localComponents); + + return "{$local}@{$domain}"; + } + + /** * Provide all unique combinations of elements in $input, with order and duplicates irrelevant. * * @param array $input The input array of elements. diff --git a/src/composer.json b/src/composer.json --- a/src/composer.json +++ b/src/composer.json @@ -24,6 +24,7 @@ "laravel/framework": "6.*", "laravel/horizon": "^3", "laravel/tinker": "^2.4", + "mlocati/spf-lib": "^3.0", "mollie/laravel-mollie": "^2.9", "morrislaptop/laravel-queue-clear": "^1.2", "silviolleite/laravelpwa": "^2.0", diff --git a/src/config/cache.php b/src/config/cache.php --- a/src/config/cache.php +++ b/src/config/cache.php @@ -18,7 +18,7 @@ | */ - 'default' => env('CACHE_DRIVER', 'file'), + 'default' => env('CACHE_DRIVER', 'redis'), /* |-------------------------------------------------------------------------- diff --git a/src/config/database.php b/src/config/database.php --- a/src/config/database.php +++ b/src/config/database.php @@ -128,7 +128,7 @@ 'options' => [ 'cluster' => env('REDIS_CLUSTER', 'predis'), - 'prefix' => env('REDIS_PREFIX', Str::slug(env('APP_NAME', 'laravel'), '_').'_database_'), + 'prefix' => env('REDIS_PREFIX', Str::slug(env('APP_NAME', 'laravel'), '_') . '_database_'), ], 'default' => [ diff --git a/src/database/migrations/2020_10_18_091319_create_greylist_tables.php b/src/database/migrations/2020_10_18_091319_create_greylist_tables.php new file mode 100644 --- /dev/null +++ b/src/database/migrations/2020_10_18_091319_create_greylist_tables.php @@ -0,0 +1,123 @@ +bigIncrements('id'); + $table->string('sender_local', 256); + $table->string('sender_domain', 256); + $table->string('recipient_hash', 64); + $table->bigInteger('recipient_id')->unsigned()->nullable(); + $table->string('recipient_type', 16)->nullable(); + $table->bigInteger('net_id'); + $table->string('net_type', 16); + $table->boolean('greylisting')->default(true); + $table->bigInteger('connect_count')->unsigned()->default(1); + $table->timestamps(); + + /** + * Index for recipient request. + */ + $table->index( + [ + 'sender_local', + 'sender_domain', + 'recipient_hash', + 'net_id', + 'net_type' + ], + 'ssrnn_idx' + ); + + /** + * Index for domain whitelist query. + */ + $table->index( + [ + 'sender_domain', + 'net_id', + 'net_type' + ], + 'snn_idx' + ); + + /** + * Index for updated_at + */ + $table->index('updated_at'); + + $table->unique( + ['sender_local', 'sender_domain', 'recipient_hash', 'net_id', 'net_type'], + 'ssrnn_unq' + ); + } + ); + + Schema::create( + 'greylist_penpals', + function (Blueprint $table) { + $table->bigIncrements('id'); + $table->bigInteger('local_id'); + $table->string('local_type', 16); + $table->string('remote_local', 128); + $table->string('remote_domain', 256); + $table->timestamps(); + } + ); + + Schema::create( + 'greylist_settings', + function (Blueprint $table) { + $table->bigIncrements('id'); + $table->bigInteger('object_id'); + $table->string('object_type', 16); + $table->string('key', 64); + $table->text('value'); + $table->timestamps(); + + $table->index(['object_id', 'object_type', 'key'], 'ook_idx'); + } + ); + + Schema::create( + 'greylist_whitelist', + function (Blueprint $table) { + $table->bigIncrements('id'); + $table->string('sender_local', 128)->nullable(); + $table->string('sender_domain', 256); + $table->bigInteger('net_id'); + $table->string('net_type', 16); + $table->timestamps(); + + $table->index(['sender_local', 'sender_domain', 'net_id', 'net_type'], 'ssnn_idx'); + } + ); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('greylist_connect'); + Schema::dropIfExists('greylist_penpals'); + Schema::dropIfExists('greylist_settings'); + Schema::dropIfExists('greylist_whitelist'); + } +} diff --git a/src/database/migrations/2020_11_20_120000_add_domains_primary_key.php b/src/database/migrations/2020_11_20_120000_add_domains_primary_key.php new file mode 100644 --- /dev/null +++ b/src/database/migrations/2020_11_20_120000_add_domains_primary_key.php @@ -0,0 +1,41 @@ +primary('id'); + } + ); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + if (Schema::hasTable('domains')) { + Schema::table( + 'domains', + function (Blueprint $table) { + $table->dropPrimary('id'); + } + ); + } + } +} diff --git a/src/database/migrations/2020_11_20_130000_create_domain_settings_table.php b/src/database/migrations/2020_11_20_130000_create_domain_settings_table.php new file mode 100644 --- /dev/null +++ b/src/database/migrations/2020_11_20_130000_create_domain_settings_table.php @@ -0,0 +1,44 @@ +bigIncrements('id'); + $table->bigInteger('domain_id'); + $table->string('key'); + $table->text('value'); + $table->timestamp('created_at')->useCurrent(); + $table->timestamp('updated_at')->useCurrent(); + + $table->foreign('domain_id')->references('id')->on('domains') + ->onDelete('cascade')->onUpdate('cascade'); + + $table->unique(['domain_id', 'key']); + } + ); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('domain_settings'); + } +} diff --git a/src/database/migrations/2020_11_20_140000_extend_settings_value_column.php b/src/database/migrations/2020_11_20_140000_extend_settings_value_column.php new file mode 100644 --- /dev/null +++ b/src/database/migrations/2020_11_20_140000_extend_settings_value_column.php @@ -0,0 +1,41 @@ +text('value')->change(); + } + ); + + Schema::table( + 'wallet_settings', + function (Blueprint $table) { + $table->text('value')->change(); + } + ); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + // do nothing + } +} diff --git a/src/database/seeds/local/DomainSeeder.php b/src/database/seeds/local/DomainSeeder.php --- a/src/database/seeds/local/DomainSeeder.php +++ b/src/database/seeds/local/DomainSeeder.php @@ -25,7 +25,8 @@ "groupoffice.ch", "journalistmail.ch", "legalprivilege.ch", - "libertymail.co" + "libertymail.co", + "libertymail.net" ]; foreach ($domains as $domain) { diff --git a/src/phpunit.xml b/src/phpunit.xml --- a/src/phpunit.xml +++ b/src/phpunit.xml @@ -40,5 +40,6 @@ + diff --git a/src/resources/js/app.js b/src/resources/js/app.js --- a/src/resources/js/app.js +++ b/src/resources/js/app.js @@ -189,6 +189,10 @@ isLoading() { return isLoading > 0 }, + tab(e) { + e.preventDefault() + $(e.target).tab('show') + }, errorPage(code, msg, hint) { // Until https://github.com/vuejs/vue-router/issues/977 is implemented // we can't really use router to display error page as it has two side diff --git a/src/resources/lang/en/app.php b/src/resources/lang/en/app.php --- a/src/resources/lang/en/app.php +++ b/src/resources/lang/en/app.php @@ -45,6 +45,7 @@ 'domain-verify-error' => 'Domain ownership verification failed.', 'domain-suspend-success' => 'Domain suspended successfully.', 'domain-unsuspend-success' => 'Domain unsuspended successfully.', + 'domain-setconfig-success' => 'Domain settings updated successfully.', 'user-update-success' => 'User data updated successfully.', 'user-create-success' => 'User created successfully.', @@ -52,6 +53,7 @@ 'user-suspend-success' => 'User suspended successfully.', 'user-unsuspend-success' => 'User unsuspended successfully.', 'user-reset-2fa-success' => '2-Factor authentication reset successfully.', + 'user-setconfig-success' => 'User settings updated successfully.', 'search-foundxdomains' => ':x domains have been found.', 'search-foundxgroups' => ':x distribution lists have been found.', diff --git a/src/resources/lang/en/ui.php b/src/resources/lang/en/ui.php --- a/src/resources/lang/en/ui.php +++ b/src/resources/lang/en/ui.php @@ -64,6 +64,10 @@ 'dns-verify' => "Domain DNS verification sample:", 'dns-config' => "Domain DNS configuration sample:", 'namespace' => "Namespace", + 'spf-whitelist' => "SPF Whitelist", + 'spf-whitelist-text' => "The Sender Policy Framework allows a sender domain to disclose, through DNS, " + . "which systems are allowed to send emails with an envelope sender address within said domain.", + 'spf-whitelist-ex' => "Here you can specify a list of allowed servers, for example: .ess.barracuda.com.", 'verify' => "Domain verification", 'verify-intro' => "In order to confirm that you're the actual holder of the domain, we need to run a verification process before finally activating it for email delivery.", 'verify-dns' => "The domain must have one of the following entries in DNS:", @@ -95,15 +99,19 @@ 'date' => "Date", 'description' => "Description", 'details' => "Details", + 'disabled' => "disabled", 'domain' => "Domain", 'email' => "Email Address", + 'enabled' => "enabled", 'firstname' => "First Name", + 'general' => "General", 'lastname' => "Last Name", 'none' => "none", 'or' => "or", 'password' => "Password", 'password-confirm' => "Confirm Password", 'phone' => "Phone", + 'settings' => "Settings", 'status' => "Status", 'surname' => "Surname", 'user' => "User", @@ -341,6 +349,10 @@ 'domains-none' => "There are no domains in this account.", 'ext-email' => "External Email", 'finances' => "Finances", + 'greylisting' => "Greylisting", + 'greylisting-text' => "Greylisting is a method of defending users against spam. Any incoming mail from an unrecognized sender " + . "is temporarily rejected. The originating server should try again after a delay. " + . "This time the email will be accepted. Spammers usually do not reattempt mail delivery.", 'list-title' => "User accounts", 'managed-by' => "Managed by", 'new' => "New user account", diff --git a/src/resources/lang/en/validation.php b/src/resources/lang/en/validation.php --- a/src/resources/lang/en/validation.php +++ b/src/resources/lang/en/validation.php @@ -138,6 +138,8 @@ 'notalocaluser' => 'The specified email address does not exist.', 'memberislist' => 'A recipient cannot be the same as the list address.', 'listmembersrequired' => 'At least one recipient is required.', + 'spf-entry-invalid' => 'The entry format is invalid. Expected a domain name starting with a dot.', + 'invalid-config-parameter' => 'The requested configuration parameter is not supported.', /* |-------------------------------------------------------------------------- diff --git a/src/resources/themes/app.scss b/src/resources/themes/app.scss --- a/src/resources/themes/app.scss +++ b/src/resources/themes/app.scss @@ -354,13 +354,8 @@ padding: 0.5rem 0; } - .form-group { - margin-bottom: 0.5rem; - } - .nav-tabs { flex-wrap: nowrap; - overflow-x: auto; .nav-link { white-space: nowrap; @@ -368,27 +363,6 @@ } } - .tab-content { - margin-top: 0.5rem; - } - - .col-form-label { - color: #666; - font-size: 95%; - } - - .form-group.plaintext .col-form-label { - padding-bottom: 0; - } - - form.read-only.short label { - width: 35%; - - & + * { - width: 65%; - } - } - #app > div.container { margin-bottom: 1rem; margin-top: 1rem; @@ -469,3 +443,9 @@ } } } + +@include media-breakpoint-down(sm) { + .tab-pane > .card-body { + padding: 0.5rem; + } +} diff --git a/src/resources/themes/forms.scss b/src/resources/themes/forms.scss --- a/src/resources/themes/forms.scss +++ b/src/resources/themes/forms.scss @@ -75,3 +75,48 @@ margin-bottom: 0; } } + +// Various improvements for mobile +@include media-breakpoint-down(sm) { + .form-group { + margin-bottom: 0.5rem; + } + + .form-group.plaintext .col-form-label { + padding-bottom: 0; + } + + form.read-only.short label { + width: 35%; + + & + * { + width: 65%; + } + } +} + +@include media-breakpoint-down(xs) { + .col-form-label { + color: #666; + font-size: 95%; + } + + .form-group.checkbox { + position: relative; + + & > div { + position: initial; + padding-top: 0 !important; + + input { + position: absolute; + top: 0.5rem; + right: 1rem; + } + } + + label { + padding-right: 2.5rem; + } + } +} diff --git a/src/resources/vue/Admin/Domain.vue b/src/resources/vue/Admin/Domain.vue --- a/src/resources/vue/Admin/Domain.vue +++ b/src/resources/vue/Admin/Domain.vue @@ -37,10 +37,15 @@
@@ -49,7 +54,23 @@

{{ $t('domain.dns-verify') }}

{{ domain.dns.join("\n") }}

{{ $t('domain.dns-config') }}

- 

{{ domain.config.join("\n") }}

+ 

{{ domain.mx.join("\n") }}

+
+
+ +
+
+
+
+
+ +
+ + {{ domain.config && domain.config.spf_whitelist.length ? domain.config.spf_whitelist.join(', ') : 'none' }} + +
+
+
diff --git a/src/resources/vue/Admin/User.vue b/src/resources/vue/Admin/User.vue --- a/src/resources/vue/Admin/User.vue +++ b/src/resources/vue/Admin/User.vue @@ -117,6 +117,11 @@ {{ $t('user.distlists') }} ({{ distlists.length }}) +
  • + + Settings + +
    • @@ -302,6 +307,23 @@
    +
    +
    +
    +
    +
    + +
    + + {{ $t('form.enabled') }} + {{ $t('form.disabled') }} + +
    +
    +
    +
    +
    +