diff --git a/bin/quickstart.sh b/bin/quickstart.sh --- a/bin/quickstart.sh +++ b/bin/quickstart.sh @@ -36,23 +36,19 @@ base_dir=$(dirname $(dirname $0)) -docker pull docker.io/kolab/centos7:latest - -docker-compose down --remove-orphans -docker-compose build - -pushd ${base_dir}/src/ - # Always reset .env with .env.example -cp .env.example .env +cp src/.env.example src/.env -if [ -f ".env.local" ]; then +if [ -f "src/.env.local" ]; then # Ensure there's a line ending - echo "" >> .env - cat .env.local >> .env + echo "" >> src/.env + cat src/.env.local >> src/.env fi -popd +docker pull docker.io/kolab/centos7:latest + +docker-compose down --remove-orphans +docker-compose build bin/regen-certs @@ -99,6 +95,7 @@ php -dmemory_limit=512M ./artisan migrate:refresh --seed ./artisan data:import ./artisan swoole:http stop >/dev/null 2>&1 || : -./artisan swoole:http start +SWOOLE_HTTP_DAEMONIZE=true ./artisan swoole:http start +./artisan horizon popd diff --git a/docker/swoole/Dockerfile b/docker/swoole/Dockerfile --- a/docker/swoole/Dockerfile +++ b/docker/swoole/Dockerfile @@ -2,7 +2,7 @@ MAINTAINER Jeroen van Meeuwen -ARG SWOOLE_VERSION=4.6.x +ARG SWOOLE_VERSION=v4.6.7 ENV HOME=/opt/app-root/src LABEL io.k8s.description="Platform for serving PHP applications under Swoole" \ @@ -28,8 +28,9 @@ php-mysqlnd \ re2c \ wget && \ - git clone -b v${SWOOLE_VERSION} https://github.com/swoole/swoole-src.git/ /swoole-src.git/ && \ + git clone https://github.com/swoole/swoole-src.git/ /swoole-src.git/ && \ cd /swoole-src.git/ && \ + git checkout -f ${SWOOLE_VERSION} && \ git clean -d -f -x && \ phpize --clean && \ phpize && \ diff --git a/extras/kolab_policy_greylist.py b/extras/kolab_policy_greylist rename from extras/kolab_policy_greylist.py rename to extras/kolab_policy_greylist --- a/extras/kolab_policy_greylist.py +++ b/extras/kolab_policy_greylist @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python """ An example implementation of a policy service. """ @@ -22,6 +22,8 @@ while not end_of_request: if (time.time() - start_time) >= 10: + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() sys.exit(0) request_line = sys.stdin.readline() @@ -54,14 +56,17 @@ ) # pylint: disable=broad-except except Exception: - print("action=DEFER_IF_PERMIT Temporary error, try again later.") - sys.exit(1) + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() + sys.exit(0) try: R = json.loads(RESPONSE.text) # pylint: disable=broad-except except Exception: - sys.exit(1) + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() + sys.exit(0) if 'prepend' in R: for prepend in R['prepend']: @@ -69,11 +74,8 @@ if RESPONSE.ok: print("action={0}\n".format(R['response'])) - - sys.stdout.flush() else: print("action={0} {1}\n".format(R['response'], R['reason'])) - sys.stdout.flush() - + sys.stdout.flush() sys.exit(0) diff --git a/extras/kolab_policy_ratelimit b/extras/kolab_policy_ratelimit new file mode 100755 --- /dev/null +++ b/extras/kolab_policy_ratelimit @@ -0,0 +1,144 @@ +#!/usr/bin/python3 +""" +This policy applies rate limitations +""" + +import json +import time +import sys + +import requests + + +class PolicyRequest: + """ + A holder of policy request instances. + """ + db = None + recipients = [] + sender = None + + def __init__(self, request): + """ + Initialize a policy request, usually in RCPT protocol state. + """ + if 'sender' in request: + self.sender = request['sender'] + + if 'recipient' in request: + request['recipient'] = request['recipient'] + + self.recipients.append(request['recipient']) + + def add_request(self, request): + """ + Add an additional request from an instance to the existing instance + """ + # Normalize email addresses (they may contain recipient delimiters) + if 'recipient' in request: + request['recipient'] = request['recipient'] + + if not request['recipient'].strip() == '': + self.recipients.append(request['recipient']) + + def check_rate(self): + """ + Check the rates at which this sender is hitting our mailserver. + """ + if self.sender == "": + return {'response': 'DUNNO'} + + try: + response = requests.post( + URL, + data={ + 'sender': self.sender, + 'recipients': self.recipients + }, + verify=True + ) + + # pylint: disable=broad-except + except Exception: + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() + sys.exit(0) + + return response + + +def read_request_input(): + """ + Read a single policy request from sys.stdin, and return a dictionary + containing the request. + """ + start_time = time.time() + + policy_request = {} + end_of_request = False + + while not end_of_request: + if (time.time() - start_time) >= 10: + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() + sys.exit(0) + + request_line = sys.stdin.readline() + + if request_line.strip() == '': + if 'request' in policy_request: + end_of_request = True + else: + request_line = request_line.strip() + request_key = request_line.split('=')[0] + request_value = '='.join(request_line.split('=')[1:]) + + policy_request[request_key] = request_value + + return policy_request + + +if __name__ == "__main__": + URL = 'https://services.kolabnow.com/api/webhooks/policy/ratelimit' + + POLICY_REQUESTS = {} + + # Start the work + while True: + POLICY_REQUEST = read_request_input() + + INSTANCE = POLICY_REQUEST['instance'] + + if INSTANCE in POLICY_REQUESTS: + POLICY_REQUESTS[INSTANCE].add_request(POLICY_REQUEST) + else: + POLICY_REQUESTS[INSTANCE] = PolicyRequest(POLICY_REQUEST) + + protocol_state = POLICY_REQUEST['protocol_state'].strip().lower() + + if not protocol_state == 'data': + print("action=DUNNO\n") + sys.stdout.flush() + + else: + RESPONSE = POLICY_REQUESTS[INSTANCE].check_rate() + + try: + R = json.loads(RESPONSE.text) + # pylint: disable=broad-except + except Exception: + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() + sys.exit(0) + + if 'prepend' in R: + for prepend in R['prepend']: + print("action=PREPEND {0}".format(prepend)) + + if 'reason' in R: + print("action={0} {1}\n".format(R['response'], R['reason'])) + else: + print("action={0}\n".format(R['response'])) + + sys.stdout.flush() + sys.exit(0) diff --git a/extras/kolab_policy_ratelimit.py b/extras/kolab_policy_ratelimit.py deleted file mode 100755 --- a/extras/kolab_policy_ratelimit.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/python3 -""" -This policy applies rate limitations -""" - -import json -import time -import sys - -import requests - - -def read_request_input(): - """ - Read a single policy request from sys.stdin, and return a dictionary - containing the request. - """ - start_time = time.time() - - policy_request = {} - end_of_request = False - - while not end_of_request: - if (time.time() - start_time) >= 10: - sys.exit(0) - - request_line = sys.stdin.readline() - - if request_line.strip() == '': - if 'request' in policy_request: - end_of_request = True - else: - request_line = request_line.strip() - request_key = request_line.split('=')[0] - request_value = '='.join(request_line.split('=')[1:]) - - policy_request[request_key] = request_value - - return policy_request - - -if __name__ == "__main__": - URL = 'https://services.kolabnow.com/api/webhooks/policy/ratelimit' - - # Start the work - while True: - REQUEST = read_request_input() - - try: - RESPONSE = requests.post( - URL, - data=REQUEST, - verify=True - ) - # pylint: disable=broad-except - except Exception: - print("action=DEFER_IF_PERMIT Temporary error, try again later.") - sys.exit(1) - - try: - R = json.loads(RESPONSE.text) - # pylint: disable=broad-except - except Exception: - sys.exit(1) - - if 'prepend' in R: - for prepend in R['prepend']: - print("action=PREPEND {0}".format(prepend)) - - if RESPONSE.ok: - print("action={0}\n".format(R['response'])) - - sys.stdout.flush() - else: - print("action={0} {1}\n".format(R['response'], R['reason'])) - - sys.stdout.flush() - - sys.exit(0) diff --git a/extras/kolab_policy_spf.py b/extras/kolab_policy_spf rename from extras/kolab_policy_spf.py rename to extras/kolab_policy_spf --- a/extras/kolab_policy_spf.py +++ b/extras/kolab_policy_spf @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python """ This is the implementation of a (postfix) MTA policy service to enforce the Sender Policy Framework. @@ -23,6 +23,8 @@ while not end_of_request: if (time.time() - start_time) >= 10: + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() sys.exit(0) request_line = sys.stdin.readline() @@ -55,14 +57,17 @@ ) # pylint: disable=broad-except except Exception: - print("action=DEFER_IF_PERMIT Temporary error, try again later.") - sys.exit(1) + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() + sys.exit(0) try: R = json.loads(RESPONSE.text) # pylint: disable=broad-except except Exception: - sys.exit(1) + print("action=DEFER_IF_PERMIT Temporary error, try again later.\n") + sys.stdout.flush() + sys.exit(0) if 'prepend' in R: for prepend in R['prepend']: @@ -70,11 +75,8 @@ if RESPONSE.ok: print("action={0}\n".format(R['response'])) - - sys.stdout.flush() else: print("action={0} {1}\n".format(R['response'], R['reason'])) - sys.stdout.flush() - + sys.stdout.flush() sys.exit(0) diff --git a/src/.env.example b/src/.env.example --- a/src/.env.example +++ b/src/.env.example @@ -100,8 +100,8 @@ PGP_GPGCONF= PGP_LENGTH= -; Set these to IP addresses you serve WOAT with. -; Have the domain owner point _woat. NS RRs refer to ns0{1,2}. +# Set these to IP addresses you serve WOAT with. +# Have the domain owner point _woat. NS RRs refer to ns0{1,2}. WOAT_NS1=ns01.domain.tld WOAT_NS2=ns02.domain.tld diff --git a/src/app/Console/Commands/Policy/Greylist/ExpungeCommand.php b/src/app/Console/Commands/Policy/Greylist/ExpungeCommand.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/Policy/Greylist/ExpungeCommand.php @@ -0,0 +1,36 @@ +subMonthsWithoutOverflow(6)) + ->delete(); + + \App\Policy\Greylist\Whitelist::where('updated_at', '<', \Carbon\Carbon::now()->subMonthsWithoutOverflow(6)) + ->delete(); + } +} diff --git a/src/app/Console/Commands/Policy/RateLimit/ExpungeCommand.php b/src/app/Console/Commands/Policy/RateLimit/ExpungeCommand.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/Policy/RateLimit/ExpungeCommand.php @@ -0,0 +1,32 @@ +subMonthsWithoutOverflow(6))->delete(); + } +} diff --git a/src/app/Console/Commands/Policy/RateLimit/Whitelist/CreateCommand.php b/src/app/Console/Commands/Policy/RateLimit/Whitelist/CreateCommand.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/Policy/RateLimit/Whitelist/CreateCommand.php @@ -0,0 +1,61 @@ +argument('object'); + + if (strpos($object, '@') === false) { + $domain = $this->getDomain($object); + + if (!$domain) { + $this->error("No such domain {$object}"); + return 1; + } + + $id = $domain->id; + $type = \App\Domain::class; + } else { + $user = $this->getUser($object); + + if (!$user) { + $this->error("No such user {$user}"); + return 1; + } + + $id = $user->id; + $type = \App\User::class; + } + + \App\Policy\RateLimitWhitelist::create( + [ + 'whitelistable_id' => $id, + 'whitelistable_type' => $type + ] + ); + } +} diff --git a/src/app/Console/Commands/Policy/RateLimit/Whitelist/DeleteCommand.php b/src/app/Console/Commands/Policy/RateLimit/Whitelist/DeleteCommand.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/Policy/RateLimit/Whitelist/DeleteCommand.php @@ -0,0 +1,61 @@ +argument('object'); + + if (strpos($object, '@') === false) { + $domain = $this->getDomain($object); + + if (!$domain) { + $this->error("No such domain {$object}"); + return 1; + } + + $id = $domain->id; + $type = \App\Domain::class; + } else { + $user = $this->getUser($object); + + if (!$user) { + $this->error("No such user {$user}"); + return 1; + } + + $id = $user->id; + $type = \App\User::class; + } + + \App\Policy\RateLimitWhitelist::where( + [ + 'whitelistable_id' => $id, + 'whitelistable_type' => $type + ] + )->delete(); + } +} diff --git a/src/app/Console/Commands/Policy/RateLimit/Whitelist/ReadCommand.php b/src/app/Console/Commands/Policy/RateLimit/Whitelist/ReadCommand.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/Policy/RateLimit/Whitelist/ReadCommand.php @@ -0,0 +1,42 @@ +whitelistable; + + if ($whitelistable instanceof \App\Domain) { + $this->info("{$item->id}: {$item->whitelistable_type} {$whitelistable->namespace}"); + } elseif ($whitelistable instanceof \App\User) { + $this->info("{$item->id}: {$item->whitelistable_type} {$whitelistable->email}"); + } + } + ); + } +} diff --git a/src/app/Console/Commands/Policy/RateLimitsCommand.php b/src/app/Console/Commands/Policy/RateLimitsCommand.php new file mode 100644 --- /dev/null +++ b/src/app/Console/Commands/Policy/RateLimitsCommand.php @@ -0,0 +1,13 @@ + 'test.local.instance', - 'protocol_state' => 'RCPT', - 'sender' => 'sender@spf-pass.kolab.org', - 'client_name' => 'mx.kolabnow.com', - 'client_address' => '212.103.80.148', - 'recipient' => $this->domainOwner->email - ]; - - $response = $this->post('/api/webhooks/spf', $data); - */ -/* $data = \request()->input(); - // TODO: normalize sender address $sender = strtolower($data['sender']); - $alias = \App\UserAlias::where('alias', $sender)->first(); + if (strpos($sender, '+') !== false) { + list($local, $rest) = explode('+', $sender); + list($rest, $domain) = explode('@', $sender); + $sender = "{$local}@{$domain}"; + } + + list($local, $domain) = explode('@', $sender); + + if (in_array($sender, \config('app.ratelimit_whitelist', []))) { + return response()->json(['response' => 'DUNNO'], 200); + } + + // + // Examine the individual sender + // + $user = \App\User::where('email', $sender)->first(); - if (!$alias) { - $user = \App\User::where('email', $sender)->first(); + if (!$user) { + $alias = \App\UserAlias::where('alias', $sender)->first(); - if (!$user) { - // what's the situation here? + if (!$alias) { + // use HOLD, so that it is silent (as opposed to REJECT) + return response()->json(['response' => 'HOLD', 'reason' => 'Sender not allowed here.'], 403); } - } else { + $user = $alias->user; } - // TODO time-limit - $userRates = \App\Policy\Ratelimit::where('user_id', $user->id); + if ($user->isDeleted() || $user->isSuspended()) { + // use HOLD, so that it is silent (as opposed to REJECT) + return response()->json(['response' => 'HOLD', 'reason' => 'Sender deleted or suspended'], 403); + } + + // + // Examine the domain + // + $domain = \App\Domain::where('namespace', $domain)->first(); + + if (!$domain) { + // external sender through where this policy is applied + return response()->json(['response' => 'DUNNO'], 200); + } + + if ($domain->isDeleted() || $domain->isSuspended()) { + // use HOLD, so that it is silent (as opposed to REJECT) + return response()->json(['response' => 'HOLD', 'reason' => 'Sender domain deleted or suspended'], 403); + } + + // see if the user or domain is whitelisted + // use ./artisan policy:ratelimit:whitelist:create + $whitelist = \App\Policy\RateLimitWhitelist::where( + [ + 'whitelistable_type' => \App\User::class, + 'whitelistable_id' => $user->id + ] + )->orWhere( + [ + 'whitelistable_type' => \App\Domain::class, + 'whitelistable_id' => $domain->id + ] + )->exists(); + + if ($whitelist) { + return response()->json(['response' => 'DUNNO'], 200); + } + + // user nor domain whitelisted, continue scrutinizing request + $recipients = $data['recipients']; + sort($recipients); + + $recipientCount = count($recipients); + $recipientHash = hash('sha256', implode(',', $recipients)); + + // + // Retrieve the wallet to get to the owner + // + $wallet = $user->entitlements->first()->wallet; + + // wait, there is no wallet? + if (!$wallet) { + return response()->json(['response' => 'HOLD', 'reason' => 'Sender without a wallet'], 403); + } + + $owner = $wallet->owner; + + // find or create the request + $request = \App\Policy\RateLimit::where( + [ + 'recipient_hash' => $recipientHash, + 'user_id' => $user->id + ] + )->where('updated_at', '>=', \Carbon\Carbon::now()->subHour())->first(); + + if (!$request) { + $request = \App\Policy\RateLimit::create( + [ + 'user_id' => $user->id, + 'owner_id' => $owner->id, + 'recipient_hash' => $recipientHash, + 'recipient_count' => $recipientCount + ] + ); + + // ensure the request has an up to date timestamp + } else { + $request->updated_at = \Carbon\Carbon::now(); + $request->save(); + } + + // excempt owners that have made at least two payments and currently maintain a positive balance. + $payments = $wallet->payments + ->where('amount', '>', 0) + ->where('status', 'paid'); - // TODO message vs. recipient limit - if ($userRates->count() > 10) { - // TODO + if ($payments->count() >= 2 && $wallet->balance > 0) { + return response()->json(['response' => 'DUNNO'], 200); } - // this is the wallet to which the account is billed - $wallet = $user->wallet; + // + // Examine the rates at which the owner (or its users) is sending + // + $ownerRates = \App\Policy\RateLimit::where('owner_id', $owner->id) + ->where('updated_at', '>=', \Carbon\Carbon::now()->subHour()); - // TODO: consider $wallet->payments; + if ($ownerRates->count() >= 10) { + $result = [ + 'response' => 'DEFER_IF_PERMIT', + 'reason' => 'The account is at 10 messages per hour, cool down.' + ]; - $owner = $wallet->user; + // automatically suspend (recursively) if 2.5 times over the original limit and younger than two months + $ageThreshold = \Carbon\Carbon::now()->subMonthsWithoutOverflow(2); + + if ($ownerRates->count() >= 25 && $owner->created_at > $ageThreshold) { + $wallet->entitlements->each( + function ($entitlement) use ($owner) { + if ($entitlement->entitleable_type == \App\Domain::class) { + $entitlement->entitleable->suspend(); + } - // TODO time-limit - $ownerRates = \App\Policy\Ratelimit::where('owner_id', $owner->id); + if ($entitlement->entitleable_type == \App\User::class) { + $entitlement->entitleable->suspend(); + } + } + ); + } - // TODO message vs. recipient limit (w/ user counts) - if ($ownerRates->count() > 10) { - // TODO + return response()->json($result, 403); } -*/ + + $ownerRates = \App\Policy\RateLimit::where('owner_id', $owner->id) + ->where('updated_at', '>=', \Carbon\Carbon::now()->subHour()) + ->sum('recipient_count'); + + if ($ownerRates >= 100) { + $result = [ + 'response' => 'DEFER_IF_PERMIT', + 'reason' => 'The account is at 100 recipients per hour, cool down.' + ]; + + // automatically suspend if 2.5 times over the original limit and younger than two months + $ageThreshold = \Carbon\Carbon::now()->subMonthsWithoutOverflow(2); + + if ($ownerRates >= 250 && $owner->created_at > $ageThreshold) { + $wallet->entitlements->each( + function ($entitlement) use ($owner) { + if ($entitlement->entitleable_type == \App\Domain::class) { + $entitlement->entitleable->suspend(); + } + + if ($entitlement->entitleable_type == \App\User::class) { + $entitlement->entitleable->suspend(); + } + } + ); + } + + return response()->json($result, 403); + } + + // + // Examine the rates at which the user is sending + // + $userRates = \App\Policy\RateLimit::where('user_id', $user->id) + ->where('updated_at', '>=', \Carbon\Carbon::now()->subHour()); + + if ($userRates->count() >= 10) { + $result = [ + 'response' => 'DEFER_IF_PERMIT', + 'reason' => 'User is at 10 messages per hour, cool down.' + ]; + + // automatically suspend if 2.5 times over the original limit and younger than two months + $ageThreshold = \Carbon\Carbon::now()->subMonthsWithoutOverflow(2); + + if ($userRates->count() >= 25 && $user->created_at > $ageThreshold) { + $user->suspend(); + } + + return response()->json($result, 403); + } + + $userRates = \App\Policy\RateLimit::where('user_id', $user->id) + ->where('updated_at', '>=', \Carbon\Carbon::now()->subHour()) + ->sum('recipient_count'); + + if ($userRates >= 100) { + $result = [ + 'response' => 'DEFER_IF_PERMIT', + 'reason' => 'User is at 100 recipients per hour, cool down.' + ]; + + // automatically suspend if 2.5 times over the original limit + $ageThreshold = \Carbon\Carbon::now()->subMonthsWithoutOverflow(2); + + if ($userRates >= 250 && $user->created_at > $ageThreshold) { + $user->suspend(); + } + + return response()->json($result, 403); + } + + $result = [ + 'response' => 'DUNNO' + ]; + + return response()->json($result, 200); } /* @@ -117,7 +295,7 @@ return response()->json( [ 'response' => 'DEFER_IF_PERMIT', - 'reason' => 'Temporary error. Please try again later (' . __LINE__ . ')' + 'reason' => 'Temporary error. Please try again later.' ], 403 ); diff --git a/src/app/Observers/DomainObserver.php b/src/app/Observers/DomainObserver.php --- a/src/app/Observers/DomainObserver.php +++ b/src/app/Observers/DomainObserver.php @@ -51,6 +51,23 @@ \App\Jobs\Domain\DeleteJob::dispatch($domain->id); } + /** + * Handle the domain "deleting" event. + * + * @param \App\Domain $domain The domain. + * + * @return void + */ + public function deleting(Domain $domain) + { + \App\Policy\RateLimitWhitelist::where( + [ + 'whitelistable_id' => $domain->id, + 'whitelistable_type' => Domain::class + ] + )->delete(); + } + /** * Handle the domain "updated" event. * diff --git a/src/app/Observers/UserObserver.php b/src/app/Observers/UserObserver.php --- a/src/app/Observers/UserObserver.php +++ b/src/app/Observers/UserObserver.php @@ -271,5 +271,12 @@ ->whereIn('object_id', $wallets) ->delete(); } + + \App\Policy\RateLimitWhitelist::where( + [ + 'whitelistable_id' => $user->id, + 'whitelistable_type' => User::class + ] + )->delete(); } } diff --git a/src/app/Policy/RateLimit.php b/src/app/Policy/RateLimit.php new file mode 100644 --- /dev/null +++ b/src/app/Policy/RateLimit.php @@ -0,0 +1,27 @@ +belongsTo('App\User'); + } + + public function user() + { + $this->belongsTo('App\User'); + } +} diff --git a/src/app/Policy/RateLimitWhitelist.php b/src/app/Policy/RateLimitWhitelist.php new file mode 100644 --- /dev/null +++ b/src/app/Policy/RateLimitWhitelist.php @@ -0,0 +1,25 @@ +morphTo(); + } +} diff --git a/src/config/app.php b/src/config/app.php --- a/src/config/app.php +++ b/src/config/app.php @@ -296,4 +296,6 @@ 'woat_ns1' => env('WOAT_NS1', 'ns01.' . env('APP_DOMAIN')), 'woat_ns2' => env('WOAT_NS2', 'ns02.' . env('APP_DOMAIN')), + + 'ratelimit_whitelist' => explode(',', env('RATELIMIT_WHITELIST', '')) ]; diff --git a/src/database/migrations/2021_12_28_103243_create_policy_ratelimit_tables.php b/src/database/migrations/2021_12_28_103243_create_policy_ratelimit_tables.php new file mode 100644 --- /dev/null +++ b/src/database/migrations/2021_12_28_103243_create_policy_ratelimit_tables.php @@ -0,0 +1,60 @@ +bigIncrements('id'); + $table->bigInteger('user_id'); + $table->bigInteger('owner_id'); + $table->string('recipient_hash', 128); + $table->tinyInteger('recipient_count'); + $table->timestamps(); + + $table->foreign('user_id')->references('id')->on('users')->onDelete('cascade'); + $table->foreign('owner_id')->references('id')->on('users')->onDelete('cascade'); + + $table->index(['user_id', 'updated_at']); + $table->index(['owner_id', 'updated_at']); + $table->index(['user_id', 'recipient_hash', 'updated_at']); + } + ); + + Schema::create( + 'policy_ratelimit_wl', + function (Blueprint $table) { + $table->bigIncrements('id'); + $table->bigInteger('whitelistable_id'); + $table->string('whitelistable_type'); + $table->timestamps(); + + $table->index(['whitelistable_id', 'whitelistable_type']); + $table->unique(['whitelistable_id', 'whitelistable_type']); + } + ); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('policy_ratelimit'); + Schema::dropIfExists('policy_ratelimit_wl'); + } +} diff --git a/src/tests/Feature/Stories/RateLimitTest.php b/src/tests/Feature/Stories/RateLimitTest.php new file mode 100644 --- /dev/null +++ b/src/tests/Feature/Stories/RateLimitTest.php @@ -0,0 +1,562 @@ +setUpTest(); + $this->useServicesUrl(); + } + + public function tearDown(): void + { + parent::tearDown(); + } + + /** + * Verify an individual can send an email unrestricted, so long as the account is active. + */ + public function testIndividualDunno() + { + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => [ 'someone@test.domain' ] + ]; + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + /** + * Verify a whitelisted individual account is in fact whitelisted + */ + public function testIndividualWhitelist() + { + \App\Policy\RateLimitWhitelist::create( + [ + 'whitelistable_id' => $this->publicDomainUser->id, + 'whitelistable_type' => \App\User::class + ] + ); + + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => [] + ]; + + // first 9 requests + for ($i = 1; $i <= 9; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // normally, request #10 would get blocked + $request['recipients'] = ['0010@test.domain']; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(200); + + // requests 11 through 26 + for ($i = 11; $i <= 26; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + } + + /** + * Verify an individual trial user is automatically suspended. + */ + public function testIndividualAutoSuspendMessages() + { + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => [] + ]; + + // first 9 requests + for ($i = 1; $i <= 9; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // the next 16 requests for 25 total + for ($i = 10; $i <= 25; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(403); + } + + $this->assertTrue($this->publicDomainUser->fresh()->isSuspended()); + } + + /** + * Verify a suspended individual can not send an email + */ + public function testIndividualSuspended() + { + $this->publicDomainUser->suspend(); + + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => ['someone@test.domain'] + ]; + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(403); + } + + /** + * Verify an individual can run out of messages per hour + */ + public function testIndividualTrialMessages() + { + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => [] + ]; + + // first 9 requests + for ($i = 1; $i <= 9; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // the tenth request should be blocked + $request['recipients'] = ['0010@test.domain']; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(403); + } + + /** + * Verify a paid for individual account does not simply run out of messages + */ + public function testIndividualPaidMessages() + { + $wallet = $this->publicDomainUser->wallets()->first(); + + // Ensure there are no payments for the wallet + \App\Payment::where('wallet_id', $wallet->id)->delete(); + + $payment = [ + 'id' => \App\Utils::uuidInt(), + 'status' => \App\Providers\PaymentProvider::STATUS_PAID, + 'type' => \App\Providers\PaymentProvider::TYPE_ONEOFF, + 'description' => 'Paid in March', + 'wallet_id' => $wallet->id, + 'provider' => 'stripe', + 'amount' => 1111, + 'currency_amount' => 1111, + 'currency' => 'CHF', + ]; + + \App\Payment::create($payment); + $wallet->credit(1111); + + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => ['someone@test.domain'] + ]; + + // first 9 requests + for ($i = 1; $i <= 9; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // the tenth request should be blocked + $request['recipients'] = ['0010@test.domain']; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(403); + + // create a second payment + $payment['id'] = \App\Utils::uuidInt(); + \App\Payment::create($payment); + $wallet->credit(1111); + + // the tenth request should now be allowed + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(200); + } + + /** + * Verify that an individual user in its trial can run out of recipients. + */ + public function testIndividualTrialRecipients() + { + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => [] + ]; + + // first 2 requests (34 recipients each) + for ($x = 1; $x <= 2; $x++) { + $request['recipients'] = []; + + for ($y = 1; $y <= 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", $x * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // on to the third request, resulting in 102 recipients total + $request['recipients'] = []; + + for ($y = 1; $y <= 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", 3 * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(403); + } + + /** + * Verify that an individual user that has paid for its account doesn't run out of recipients. + */ + public function testIndividualPaidRecipients() + { + $wallet = $this->publicDomainUser->wallets()->first(); + + // Ensure there are no payments for the wallet + \App\Payment::where('wallet_id', $wallet->id)->delete(); + + $payment = [ + 'id' => \App\Utils::uuidInt(), + 'status' => \App\Providers\PaymentProvider::STATUS_PAID, + 'type' => \App\Providers\PaymentProvider::TYPE_ONEOFF, + 'description' => 'Paid in March', + 'wallet_id' => $wallet->id, + 'provider' => 'stripe', + 'amount' => 1111, + 'currency_amount' => 1111, + 'currency' => 'CHF', + ]; + + \App\Payment::create($payment); + $wallet->credit(1111); + + $request = [ + 'sender' => $this->publicDomainUser->email, + 'recipients' => [] + ]; + + // first 2 requests (34 recipients each) + for ($x = 0; $x < 2; $x++) { + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", $x * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // on to the third request, resulting in 102 recipients total + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", 2 * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(403); + + $payment['id'] = \App\Utils::uuidInt(); + + \App\Payment::create($payment); + $wallet->credit(1111); + + // the tenth request should now be allowed + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200, '102nd recipient not accepted'); + } + + /** + * Verify that a group owner can send email + */ + public function testGroupOwnerDunno() + { + $request = [ + 'sender' => $this->domainOwner->email, + 'recipients' => [ 'someone@test.domain' ] + ]; + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + /** + * Verify that a domain owner can run out of messages + */ + public function testGroupTrialOwnerMessages() + { + $request = [ + 'sender' => $this->domainOwner->email, + 'recipients' => [] + ]; + + // first 9 requests + for ($i = 0; $i < 9; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // the tenth request should be blocked + $request['recipients'] = ['0010@test.domain']; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(403); + + $this->assertFalse($this->domainOwner->fresh()->isSuspended()); + } + + /** + * Verify that a domain owner can run out of recipients + */ + public function testGroupTrialOwnerRecipients() + { + $request = [ + 'sender' => $this->domainOwner->email, + 'recipients' => [] + ]; + + // first 2 requests (34 recipients each) + for ($x = 0; $x < 2; $x++) { + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", $x * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // on to the third request, resulting in 102 recipients total + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", 2 * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(403); + + $this->assertFalse($this->domainOwner->fresh()->isSuspended()); + } + + /** + * Verify that a paid for group account can send messages. + */ + public function testGroupPaidOwnerRecipients() + { + $wallet = $this->domainOwner->wallets()->first(); + + // Ensure there are no payments for the wallet + \App\Payment::where('wallet_id', $wallet->id)->delete(); + + $payment = [ + 'id' => \App\Utils::uuidInt(), + 'status' => \App\Providers\PaymentProvider::STATUS_PAID, + 'type' => \App\Providers\PaymentProvider::TYPE_ONEOFF, + 'description' => 'Paid in March', + 'wallet_id' => $wallet->id, + 'provider' => 'stripe', + 'amount' => 1111, + 'currency_amount' => 1111, + 'currency' => 'CHF', + ]; + + \App\Payment::create($payment); + $wallet->credit(1111); + + $request = [ + 'sender' => $this->domainOwner->email, + 'recipients' => [] + ]; + + // first 2 requests (34 recipients each) + for ($x = 0; $x < 2; $x++) { + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", $x * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // on to the third request, resulting in 102 recipients total + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", 2 * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(403); + + // create a second payment + $payment['id'] = \App\Utils::uuidInt(); + \App\Payment::create($payment); + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(200); + } + + /** + * Verify that a user for a domain owner can send email. + */ + public function testGroupUserDunno() + { + $request = [ + 'sender' => $this->domainUsers[0]->email, + 'recipients' => [ 'someone@test.domain' ] + ]; + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + /** + * Verify that the users in a group account can be limited. + */ + public function testGroupTrialUserMessages() + { + $user = $this->domainUsers[0]; + + $request = [ + 'sender' => $user->email, + 'recipients' => [] + ]; + + // the first eight requests should be accepted + for ($i = 0; $i < 8; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(200); + } + + $request['sender'] = $this->domainUsers[1]->email; + + // the ninth request from another group user should also be accepted + $request['recipients'] = ['0009@test.domain']; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(200); + + // the tenth request from another group user should be rejected + $request['recipients'] = ['0010@test.domain']; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(403); + } + + public function testGroupTrialUserRecipients() + { + $request = [ + 'sender' => $this->domainUsers[0]->email, + 'recipients' => [] + ]; + + // first 2 requests (34 recipients each) + for ($x = 0; $x < 2; $x++) { + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", $x * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // on to the third request, resulting in 102 recipients total + $request['recipients'] = []; + + for ($y = 0; $y < 34; $y++) { + $request['recipients'][] = sprintf("%04d@test.domain", 2 * $y); + } + + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(403); + } + + /** + * Verify a whitelisted group domain is in fact whitelisted + */ + public function testGroupDomainWhitelist() + { + \App\Policy\RateLimitWhitelist::create( + [ + 'whitelistable_id' => $this->domainHosted->id, + 'whitelistable_type' => \App\Domain::class + ] + ); + + $request = [ + 'sender' => $this->domainUsers[0]->email, + 'recipients' => [] + ]; + + // first 9 requests + for ($i = 1; $i <= 9; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + + // normally, request #10 would get blocked + $request['recipients'] = ['0010@test.domain']; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + $response->assertStatus(200); + + // requests 11 through 26 + for ($i = 11; $i <= 26; $i++) { + $request['recipients'] = [sprintf("%04d@test.domain", $i)]; + $response = $this->post('api/webhooks/policy/ratelimit', $request); + + $response->assertStatus(200); + } + } +} diff --git a/src/tests/TestCase.php b/src/tests/TestCase.php --- a/src/tests/TestCase.php +++ b/src/tests/TestCase.php @@ -32,8 +32,8 @@ \config( [ 'app.url' => str_replace( - ['//admin.', '//reseller.'], - ['//', '//'], + ['//admin.', '//reseller.', '//services.'], + ['//', '//', '//'], \config('app.url') ) ] @@ -50,6 +50,11 @@ // This will set base URL for all tests in a file. // If we wanted to access both user and admin in one test // we can also just call post/get/whatever with full url + + // reset to base + self::useRegularUrl(); + + // then modify it \config(['app.url' => str_replace('//', '//admin.', \config('app.url'))]); url()->forceRootUrl(config('app.url')); } @@ -62,6 +67,11 @@ // This will set base URL for all tests in a file. // If we wanted to access both user and admin in one test // we can also just call post/get/whatever with full url + + // reset to base + self::useRegularUrl(); + + // then modify it \config(['app.url' => str_replace('//', '//reseller.', \config('app.url'))]); url()->forceRootUrl(config('app.url')); } @@ -72,6 +82,13 @@ protected static function useServicesUrl(): void { // This will set base URL for all tests in a file. + // If we wanted to access both user and admin in one test + // we can also just call post/get/whatever with full url + + // reset to base + self::useRegularUrl(); + + // then modify it \config(['app.url' => str_replace('//', '//services.', \config('app.url'))]); url()->forceRootUrl(config('app.url')); }