diff --git a/src/app/Listeners/SqlDebug.php b/src/app/Listeners/SqlDebug.php index d952d18c..6914b227 100644 --- a/src/app/Listeners/SqlDebug.php +++ b/src/app/Listeners/SqlDebug.php @@ -1,81 +1,83 @@ */ public function subscribe(Dispatcher $events): array { if (!\config('app.debug')) { return []; } return [ QueryExecuted::class => 'handle', TransactionBeginning::class => 'handle', TransactionCommitted::class => 'handle', TransactionRolledBack::class => 'handle' ]; } /** * Handle the event. * * @param object $event An event object */ public function handle(object $event): void { switch (get_class($event)) { case TransactionBeginning::class: $query = 'begin'; break; case TransactionCommitted::class: $query = 'commit'; break; case TransactionRolledBack::class: $query = 'rollback'; break; default: $query = sprintf( '%s [%s]: %.4f sec.', $event->sql, self::serializeSQLBindings($event->bindings, $event->sql), $event->time / 1000 ); } \Log::debug("[SQL] {$query}"); } /** * Serialize a bindings array to a string. */ private static function serializeSQLBindings(array $array, string $sql): string { $ipv = preg_match('/ip([46])nets/', $sql, $m) ? $m[1] : null; $serialized = array_map(function ($entry) use ($ipv) { if ($entry instanceof \DateTime) { return $entry->format('Y-m-d h:i:s'); + } elseif (is_bool($entry)) { + return $entry ? 'true' : 'false'; } elseif ($ipv && is_string($entry) && strlen($entry) == ($ipv == 6 ? 16 : 4)) { // binary IP address? use HEX representation return '0x' . bin2hex($entry); } return $entry; }, $array); return implode(', ', $serialized); } } diff --git a/src/app/Policy/Greylist/Request.php b/src/app/Policy/Greylist/Request.php index e646e174..0f1abef7 100644 --- a/src/app/Policy/Greylist/Request.php +++ b/src/app/Policy/Greylist/Request.php @@ -1,272 +1,251 @@ request = $request; if (array_key_exists('timestamp', $this->request)) { $this->timestamp = \Carbon\Carbon::parse($this->request['timestamp']); } else { $this->timestamp = \Carbon\Carbon::now(); } } - public function headerGreylist() + /** + * Get request state headers (Received-Greylist) - after self::shouldDefer() call + */ + public function headerGreylist(): string { if ($this->whitelist) { if ($this->whitelist->sender_local) { return sprintf( - "Received-Greylist: sender %s whitelisted since %s", + "Received-Greylist: sender %s whitelisted since %s (UTC)", $this->sender, - $this->whitelist->created_at->toDateString() + $this->whitelist->created_at->toDateTimeString() ); } return sprintf( "Received-Greylist: domain %s from %s whitelisted since %s (UTC)", $this->senderDomain, $this->request['client_address'], $this->whitelist->created_at->toDateTimeString() ); } - $connect = $this->findConnectsCollection()->orderBy('created_at')->first(); - - if ($connect) { + if ($this->connect) { return sprintf( "Received-Greylist: greylisted from %s until %s.", - $connect->created_at, - $this->timestamp + $this->connect->created_at->toDateTimeString(), + $this->timestamp->toDateTimeString() ); } return "Received-Greylist: no opinion here"; } - public function shouldDefer() + /** + * Connection check regarding greylisting policy + * + * @return bool True if the message should be put off, False otherwise + */ + public function shouldDefer(): bool { list($this->netID, $this->netType) = \App\Utils::getNetFromAddress($this->request['client_address']); if (!$this->netID) { return true; } $recipient = $this->recipientFromRequest(); $this->sender = $this->senderFromRequest(); if (strpos($this->sender, '@') !== false) { list($this->senderLocal, $this->senderDomain) = explode('@', $this->sender); } if (strlen($this->senderLocal) > 255) { $this->senderLocal = substr($this->senderLocal, 0, 255); } - // Purge all old information if we have no recent entries - $noEntry = false; - if (!$this->findConnectsCollectionRecent()->exists()) { + // Get the most recent entry + $connect = $this->findConnectsCollection()->first(); + + // Purge all old information if we have no recent entry + if ($connect && $connect->updated_at < $this->timestamp->copy()->subDays(7)) { $this->findConnectsCollection()->delete(); - $noEntry = true; + $connect = null; } // See if the recipient opted-out of the feature $enabled = true; if ($recipient) { $enabled = $recipient->getSetting('greylist_enabled') !== 'false'; } // FIXME: Shouldn't we bail-out (return early) if there's no $recipient? // the following block is to maintain statistics and state ... // determine if the sender domain is a whitelist from this network - $this->whitelist = Whitelist::where( - [ - 'sender_domain' => $this->senderDomain, - 'net_id' => $this->netID, - 'net_type' => $this->netType - ] - )->first(); + $this->whitelist = Whitelist::where('sender_domain', $this->senderDomain) + ->where('net_id', $this->netID) + ->where('net_type', $this->netType) + ->first(); + + $cutoffDate = $this->timestamp->copy()->subDays(7)->startOfDay(); - $cutoffDate = $this->timestamp->copy()->subDays(7); + DB::beginTransaction(); + + // Whitelist older than a month, delete it + if ($this->whitelist && $this->whitelist->updated_at < $this->timestamp->copy()->subMonthsWithoutOverflow(1)) { + $this->whitelist->delete(); + $this->whitelist = null; + } + $all = Connect::where('sender_domain', $this->senderDomain) + ->where('net_id', $this->netID) + ->where('net_type', $this->netType) + ->where('updated_at', '>=', $cutoffDate->toDateTimeString()); + + // "Touch" the whitelist if exists if ($this->whitelist) { - if ($this->whitelist->updated_at < $this->timestamp->copy()->subMonthsWithoutOverflow(1)) { - $this->whitelist->delete(); - } else { + // For performance reasons do not update timestamp more than once per 1 minute + // FIXME: Such granularity should be good enough, right? It might be a problem + // if you wanted to compare this timestamp with mail log entries. + if ($this->whitelist->updated_at < $this->timestamp->copy()->subMinute()) { $this->whitelist->updated_at = $this->timestamp; $this->whitelist->save(['timestamps' => false]); - - Connect::where( - [ - 'sender_domain' => $this->senderDomain, - 'net_id' => $this->netID, - 'net_type' => $this->netType, - 'greylisting' => true - ] - ) - ->whereDate('updated_at', '>=', $cutoffDate) - ->update( - [ - 'greylisting' => false, - 'updated_at' => $this->timestamp - ] - ); - - return false; } - } else { - $count = Connect::where( - [ - 'sender_domain' => $this->senderDomain, - 'net_id' => $this->netID, - 'net_type' => $this->netType - ] - ) - ->whereDate('updated_at', '>=', $cutoffDate) - ->limit(4)->count(); + $all->where('greylisting', true) + ->update(['greylisting' => false, 'updated_at' => $this->timestamp]); + + $enabled = false; + } elseif ($all->count() >= 4) { // Automatically create a whitelist if we have at least 5 (4 existing plus this) messages from the sender - if ($count >= 4) { - $this->whitelist = Whitelist::create( - [ - 'sender_domain' => $this->senderDomain, - 'net_id' => $this->netID, - 'net_type' => $this->netType, - 'created_at' => $this->timestamp, - 'updated_at' => $this->timestamp - ] - ); + $this->whitelist = Whitelist::create([ + 'sender_domain' => $this->senderDomain, + 'net_id' => $this->netID, + 'net_type' => $this->netType, + 'created_at' => $this->timestamp, + 'updated_at' => $this->timestamp + ]); - Connect::where( - [ - 'sender_domain' => $this->senderDomain, - 'net_id' => $this->netID, - 'net_type' => $this->netType, - 'greylisting' => true - ] - ) - ->whereDate('updated_at', '>=', $cutoffDate) - ->update( - [ - 'greylisting' => false, - 'updated_at' => $this->timestamp - ] - ); - } + $all->where('greylisting', true) + ->update(['greylisting' => false, 'updated_at' => $this->timestamp]); } // TODO: determine if the sender (individual) is a whitelist // TODO: determine if the sender is a penpal of any of the recipients. First recipient wins. if (!$enabled) { + DB::commit(); return false; } $defer = true; - // Retrieve the entry for the sender/recipient/net combination - if (!$noEntry && ($connect = $this->findConnectsCollection()->first())) { + // Update/Create an entry for the sender/recipient/net combination + if ($connect) { $connect->connect_count += 1; // TODO: The period of time for which the greylisting persists is configurable. if ($connect->created_at < $this->timestamp->copy()->subMinutes(5)) { $defer = false; $connect->greylisting = false; } $connect->save(); } else { - Connect::create( - [ + $connect = Connect::create([ 'sender_local' => $this->senderLocal, 'sender_domain' => $this->senderDomain, 'net_id' => $this->netID, 'net_type' => $this->netType, 'recipient_hash' => $this->recipientHash, 'recipient_id' => $this->recipientID, 'recipient_type' => $this->recipientType, 'created_at' => $this->timestamp, 'updated_at' => $this->timestamp - ] - ); + ]); } + $this->connect = $connect; + + DB::commit(); + return $defer; } - private function findConnectsCollection() + protected function findConnectsCollection() { - $collection = Connect::where( - [ + return Connect::where([ 'sender_local' => $this->senderLocal, 'sender_domain' => $this->senderDomain, 'recipient_hash' => $this->recipientHash, 'net_id' => $this->netID, 'net_type' => $this->netType, - ] - ); - - return $collection; - } - - private function findConnectsCollectionRecent() - { - return $this->findConnectsCollection() - ->where('updated_at', '>=', $this->timestamp->copy()->subDays(7)); + ]); } - private function recipientFromRequest() + protected function recipientFromRequest() { $recipients = \App\Utils::findObjectsByRecipientAddress($this->request['recipient']); if (sizeof($recipients) > 1) { \Log::warning( "Only taking the first recipient from the request for {$this->request['recipient']}" ); } if (count($recipients) >= 1) { foreach ($recipients as $recipient) { if ($recipient) { $this->recipientID = $recipient->id; $this->recipientType = get_class($recipient); break; } } } else { $recipient = null; } $this->recipientHash = hash('sha256', \App\Utils::normalizeAddress($this->request['recipient'])); return $recipient; } - public function senderFromRequest() + protected function senderFromRequest() { return \App\Utils::normalizeAddress($this->request['sender']); } } diff --git a/src/tests/Feature/Controller/PolicyTest.php b/src/tests/Feature/Controller/PolicyTest.php new file mode 100644 index 00000000..aedb81e6 --- /dev/null +++ b/src/tests/Feature/Controller/PolicyTest.php @@ -0,0 +1,87 @@ +clientAddress = '212.103.80.148'; + $this->net = \App\IP4Net::getNet($this->clientAddress); + $this->testDomain = $this->getTestDomain('test.domain', [ + 'type' => Domain::TYPE_EXTERNAL, + 'status' => Domain::STATUS_ACTIVE | Domain::STATUS_CONFIRMED | Domain::STATUS_VERIFIED + ]); + $this->testUser = $this->getTestUser('john@test.domain'); + + Greylist\Connect::where('sender_domain', 'sender.domain')->delete(); + Greylist\Whitelist::where('sender_domain', 'sender.domain')->delete(); + + $this->useServicesUrl(); + } + + public function tearDown(): void + { + $this->deleteTestUser($this->testUser->email); + $this->deleteTestDomain($this->testDomain->namespace); + + Greylist\Connect::where('sender_domain', 'sender.domain')->delete(); + Greylist\Whitelist::where('sender_domain', 'sender.domain')->delete(); + + parent::tearDown(); + } + + /** + * Test greylist policy webhook + * + * @group data + * @group greylist + */ + public function testGreylist() + { + // Note: Only basic tests here. More detailed policy handler tests are in another place + + // Test 403 response + $post = [ + 'sender' => 'someone@sender.domain', + 'recipient' => $this->testUser->email, + 'client_address' => $this->clientAddress, + 'client_name' => 'some.mx' + ]; + + $response = $this->post('/api/webhooks/policy/greylist', $post); + + $response->assertStatus(403); + + $json = $response->json(); + + $this->assertEquals('DEFER_IF_PERMIT', $json['response']); + $this->assertEquals("Greylisted for 5 minutes. Try again later.", $json['reason']); + + // Test 200 response + $connect = Greylist\Connect::where('sender_domain', 'sender.domain')->first(); + $connect->created_at = \Carbon\Carbon::now()->subMinutes(6); + $connect->save(); + + $response = $this->post('/api/webhooks/policy/greylist', $post); + + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertEquals('DUNNO', $json['response']); + $this->assertMatchesRegularExpression('/^Received-Greylist: greylisted from/', $json['prepend'][0]); + } +} diff --git a/src/tests/Feature/Stories/GreylistTest.php b/src/tests/Feature/Stories/GreylistTest.php index 07b050a6..ac6b8dcf 100644 --- a/src/tests/Feature/Stories/GreylistTest.php +++ b/src/tests/Feature/Stories/GreylistTest.php @@ -1,432 +1,443 @@ setUpTest(); - $this->useServicesUrl(); $this->clientAddress = '212.103.80.148'; - $this->net = \App\IP4Net::getNet($this->clientAddress); - DB::delete("DELETE FROM greylist_connect WHERE sender_domain = 'sender.domain';"); - DB::delete("DELETE FROM greylist_whitelist WHERE sender_domain = 'sender.domain';"); + $this->domainHosted = $this->getTestDomain('test.domain', [ + 'type' => Domain::TYPE_EXTERNAL, + 'status' => Domain::STATUS_ACTIVE | Domain::STATUS_CONFIRMED | Domain::STATUS_VERIFIED + ]); + + $this->domainOwner = $this->getTestUser('john@test.domain'); + + Greylist\Connect::where('sender_domain', 'sender.domain')->delete(); + Greylist\Whitelist::where('sender_domain', 'sender.domain')->delete(); } public function tearDown(): void { - DB::delete("DELETE FROM greylist_connect WHERE sender_domain = 'sender.domain';"); - DB::delete("DELETE FROM greylist_whitelist WHERE sender_domain = 'sender.domain';"); + Greylist\Connect::where('sender_domain', 'sender.domain')->delete(); + Greylist\Whitelist::where('sender_domain', 'sender.domain')->delete(); parent::tearDown(); } public function testWithTimestamp() { $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress, 'client_name' => 'some.mx', 'timestamp' => \Carbon\Carbon::now()->subDays(7)->toString() ] ); $timestamp = $this->getObjectProperty($request, 'timestamp'); $this->assertTrue( \Carbon\Carbon::parse($timestamp, 'UTC') < \Carbon\Carbon::now() ); } public function testNoNet() { $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => '127.128.129.130', 'client_name' => 'some.mx' ] ); $this->assertTrue($request->shouldDefer()); } public function testIp6Net() { $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => '2a00:1450:400a:803::2005', 'client_name' => 'some.mx' ] ); $this->assertTrue($request->shouldDefer()); } // public function testMultiRecipientThroughAlias() {} public function testWhitelistNew() { $whitelist = Greylist\Whitelist::where('sender_domain', 'sender.domain')->first(); $this->assertNull($whitelist); for ($i = 0; $i < 5; $i++) { $request = new Greylist\Request( [ 'sender' => "someone{$i}@sender.domain", 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress, 'client_name' => 'some.mx', 'timestamp' => \Carbon\Carbon::now()->subDays(1) ] ); $this->assertTrue($request->shouldDefer()); } $whitelist = Greylist\Whitelist::where('sender_domain', 'sender.domain')->first(); $this->assertNotNull($whitelist); $request = new Greylist\Request( [ 'sender' => "someone5@sender.domain", 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress, 'client_name' => 'some.mx', 'timestamp' => \Carbon\Carbon::now()->subDays(1) ] ); $this->assertFalse($request->shouldDefer()); } // public function testWhitelistedHit() {} public function testWhitelistStale() { $whitelist = Greylist\Whitelist::where('sender_domain', 'sender.domain')->first(); $this->assertNull($whitelist); for ($i = 0; $i < 5; $i++) { $request = new Greylist\Request( [ 'sender' => "someone{$i}@sender.domain", 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress, 'client_name' => 'some.mx', 'timestamp' => \Carbon\Carbon::now()->subDays(1) ] ); $this->assertTrue($request->shouldDefer()); } $whitelist = Greylist\Whitelist::where('sender_domain', 'sender.domain')->first(); $this->assertNotNull($whitelist); $request = new Greylist\Request( [ 'sender' => "someone5@sender.domain", 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress, 'client_name' => 'some.mx', 'timestamp' => \Carbon\Carbon::now()->subDays(1) ] ); $this->assertFalse($request->shouldDefer()); $whitelist->updated_at = \Carbon\Carbon::now()->subMonthsWithoutOverflow(2); $whitelist->save(['timestamps' => false]); $this->assertTrue($request->shouldDefer()); } // public function testWhitelistUpdate() {} public function testRetry() { $connect = Greylist\Connect::create( [ 'sender_local' => 'someone', 'sender_domain' => 'sender.domain', 'recipient_hash' => hash('sha256', $this->domainOwner->email), 'recipient_id' => $this->domainOwner->id, 'recipient_type' => \App\User::class, 'connect_count' => 1, 'net_id' => $this->net->id, 'net_type' => \App\IP4Net::class ] ); $connect->created_at = \Carbon\Carbon::now()->subMinutes(6); $connect->save(); $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress ] ); $this->assertFalse($request->shouldDefer()); } public function testInvalidRecipient() { $connect = Greylist\Connect::create( [ 'sender_local' => 'someone', 'sender_domain' => 'sender.domain', 'recipient_hash' => hash('sha256', $this->domainOwner->email), 'recipient_id' => 1234, 'recipient_type' => \App\Domain::class, 'connect_count' => 1, 'net_id' => $this->net->id, 'net_type' => \App\IP4Net::class ] ); $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => 'not.someone@that.exists', 'client_address' => $this->clientAddress ] ); $this->assertTrue($request->shouldDefer()); } public function testUserDisabled() { $connect = Greylist\Connect::create( [ 'sender_local' => 'someone', 'sender_domain' => 'sender.domain', 'recipient_hash' => hash('sha256', $this->domainOwner->email), 'recipient_id' => $this->domainOwner->id, 'recipient_type' => \App\User::class, 'connect_count' => 1, 'net_id' => $this->net->id, 'net_type' => \App\IP4Net::class ] ); $this->domainOwner->setSetting('greylist_enabled', 'false'); $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress ] ); $this->assertFalse($request->shouldDefer()); // Ensure we also find the setting by alias - $aliases = $this->domainOwner->aliases()->orderBy('alias')->pluck('alias')->all(); + $this->domainOwner->setAliases(['alias1@test2.domain2']); + $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', - 'recipient' => $aliases[0], + 'recipient' => 'alias1@test2.domain2', 'client_address' => $this->clientAddress ] ); $this->assertFalse($request->shouldDefer()); } public function testUserEnabled() { $connect = Greylist\Connect::create( [ 'sender_local' => 'someone', 'sender_domain' => 'sender.domain', 'recipient_hash' => hash('sha256', $this->domainOwner->email), 'recipient_id' => $this->domainOwner->id, 'recipient_type' => \App\User::class, 'connect_count' => 1, 'net_id' => $this->net->id, 'net_type' => \App\IP4Net::class ] ); $this->domainOwner->setSetting('greylist_enabled', 'true'); $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress ] ); $this->assertTrue($request->shouldDefer()); $connect->created_at = \Carbon\Carbon::now()->subMinutes(6); $connect->save(); $this->assertFalse($request->shouldDefer()); } + /** + * @group slow + */ public function testMultipleUsersAllDisabled() { + $this->setUpTest(); + $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress ] ); foreach ($this->domainUsers as $user) { Greylist\Connect::create( [ 'sender_local' => 'someone', 'sender_domain' => 'sender.domain', 'recipient_hash' => hash('sha256', $user->email), 'recipient_id' => $user->id, 'recipient_type' => \App\User::class, 'connect_count' => 1, 'net_id' => $this->net->id, 'net_type' => \App\IP4Net::class ] ); $user->setSetting('greylist_enabled', 'false'); if ($user->email == $this->domainOwner->email) { continue; } $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $user->email, 'client_address' => $this->clientAddress ] ); $this->assertFalse($request->shouldDefer()); } } + /** + * @group slow + */ public function testMultipleUsersAnyEnabled() { + $this->setUpTest(); + $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress ] ); foreach ($this->domainUsers as $user) { Greylist\Connect::create( [ 'sender_local' => 'someone', 'sender_domain' => 'sender.domain', 'recipient_hash' => hash('sha256', $user->email), 'recipient_id' => $user->id, 'recipient_type' => \App\User::class, 'connect_count' => 1, 'net_id' => $this->net->id, 'net_type' => \App\IP4Net::class ] ); $user->setSetting('greylist_enabled', ($user->id == $this->jack->id) ? 'true' : 'false'); if ($user->email == $this->domainOwner->email) { continue; } $request = new Greylist\Request( [ 'sender' => 'someone@sender.domain', 'recipient' => $user->email, 'client_address' => $this->clientAddress ] ); if ($user->id == $this->jack->id) { $this->assertTrue($request->shouldDefer()); } else { $this->assertFalse($request->shouldDefer()); } } } public function testControllerNew() { - $data = [ + $request = new Greylist\Request([ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress, 'client_name' => 'some.mx' - ]; - - $response = $this->post('/api/webhooks/policy/greylist', $data); + ]); - $response->assertStatus(403); + $this->assertTrue($request->shouldDefer()); } public function testControllerNotNew() { $connect = Greylist\Connect::create( [ 'sender_local' => 'someone', 'sender_domain' => 'sender.domain', 'recipient_hash' => hash('sha256', $this->domainOwner->email), 'recipient_id' => $this->domainOwner->id, 'recipient_type' => \App\User::class, 'connect_count' => 1, 'net_id' => $this->net->id, 'net_type' => \App\IP4Net::class ] ); $connect->created_at = \Carbon\Carbon::now()->subMinutes(6); $connect->save(); - $data = [ + $request = new Greylist\Request([ 'sender' => 'someone@sender.domain', 'recipient' => $this->domainOwner->email, 'client_address' => $this->clientAddress, 'client_name' => 'some.mx' - ]; - - $response = $this->post('/api/webhooks/policy/greylist', $data); + ]); - $response->assertStatus(200); + $this->assertFalse($request->shouldDefer()); } }