Page MenuHomePhorge
Files F117756027

D5748.1775204803.diff
No OneTemporary
Actions

Authored By
Unknown
Size
9 KB
Referenced Files
None
Subscribers
None

D5748.1775204803.diff
View Options

diff --git a/src/app/Backends/Storage.php b/src/app/Backends/Storage.php
--- a/src/app/Backends/Storage.php
+++ b/src/app/Backends/Storage.php
@@ -172,14 +172,34 @@
}
$disk = LaravelStorage::disk(\config('filesystems.default'));
+ $fileSize = 0;
+ $maxChunkSize = self::maxChunkSize();
+ $chunk_count = 0;
- $chunkId = Utils::uuidStr();
+ // "unlink" any old chunks of this file
+ $file->chunks()->delete();
- $path = self::chunkLocation($chunkId, $file);
+ while (!feof($stream)) {
+ $chunkId = Utils::uuidStr();
+ $path = self::chunkLocation($chunkId, $file);
- $disk->writeStream($path, $stream);
+ $start = $fileSize;
+ $end = $fileSize + $maxChunkSize;
+ $chunk_stream = Storage\FileInputStream::registerChunkStream($stream, $file->id, $chunkId, $start, $end);
+
+ $disk->writeStream($path, $chunk_stream);
+
+ fclose($chunk_stream);
- $fileSize = $disk->size($path);
+ $fileSize += ($size = $disk->size($path));
+
+ // Assign the node to the file
+ $file->chunks()->create([
+ 'chunk_id' => $chunkId,
+ 'sequence' => $chunk_count++,
+ 'size' => $size,
+ ]);
+ }
// Pick the client-supplied mimetype if available, otherwise detect.
if (!empty($params['mimetype'])) {
@@ -201,14 +221,6 @@
'mimetype' => $mimetype,
]);
- // Assign the node to the file, "unlink" any old nodes of this file
- $file->chunks()->delete();
- $file->chunks()->create([
- 'chunk_id' => $chunkId,
- 'sequence' => 0,
- 'size' => $fileSize,
- ]);
-
return ['id' => $file->id];
}
@@ -246,7 +258,7 @@
return [
'uploadId' => $params['uploadId'],
'uploaded' => 0,
- 'maxChunkSize' => (\config('octane.swoole.options.package_max_length') ?: 10 * 1024 * 1024) - 8192,
+ 'maxChunkSize' => self::maxChunkSize(),
];
}
@@ -358,4 +370,20 @@
{
return $file->path . '/' . $file->id . '/' . $chunkId;
}
+
+ /**
+ * Returns maximum supported chunk size in bytes
+ */
+ public static function maxChunkSize(): int
+ {
+ $max = \config('octane.swoole.options.package_max_length') ?: 10 * 1024 * 1024;
+
+ // Subtract 8KB (for request headers)
+ // Note: We might use very small values for testing purposes
+ if ($max > 1024 * 1024) {
+ $max -= 8192;
+ }
+
+ return $max;
+ }
}
diff --git a/src/app/Backends/Storage/FileInputStream.php b/src/app/Backends/Storage/FileInputStream.php
new file mode 100644
--- /dev/null
+++ b/src/app/Backends/Storage/FileInputStream.php
@@ -0,0 +1,113 @@
+<?php
+
+namespace App\Backends\Storage;
+
+use Illuminate\Support\Facades\Context;
+
+/*
+ * StreamWrapper implementation for streaming file contents with chunking.
+ * The idea is that we use the input stream directly and do not create
+ * a in-memory or temp file stream for separate chunks.
+ *
+ * https://www.php.net/manual/en/class.streamwrapper.php
+ */
+class FileInputStream
+{
+ private $id;
+ private $stream;
+ private int $end = 0;
+ private int $position = 0;
+
+ /**
+ * Chunk registration.
+ */
+ public static function registerChunkStream($stream, string $fileId, string $chunkId, int $start, int $end)
+ {
+ Context::addHidden("{$fileId}-{$chunkId}", $stream);
+
+ if (!in_array('fileinputstream', stream_get_wrappers())) {
+ stream_wrapper_register('fileinputstream', self::class);
+ }
+
+ return fopen("fileinputstream://{$fileId}-{$chunkId}:{$start}-{$end}", 'r');
+ }
+
+ /**
+ * Stream closing handler.
+ */
+ public function stream_close(): void
+ {
+ Context::forgetHidden($this->id);
+ }
+
+ /**
+ * Stream opening handler.
+ */
+ public function stream_open(string $path, string $mode, int $options, ?string &$opened_path): bool
+ {
+ [$this->id, $params] = explode(':', explode('//', $path)[1]);
+ [$start, $end] = explode('-', $params);
+
+ $this->stream = Context::getHidden($this->id);
+ $this->position = (int) $start;
+ $this->end = (int) $end;
+
+ return true;
+ }
+
+ /**
+ * Stream reading handler.
+ */
+ public function stream_read(int $count)
+ {
+ if ($this->position >= $this->end || !$this->stream) {
+ return false;
+ }
+
+ if ($count <= 0) {
+ return '';
+ }
+
+ if ($this->position + $count > $this->end) {
+ $count = $this->end - $this->position;
+ }
+
+ $output = stream_get_contents($this->stream, $count, $this->position);
+
+ $this->position += is_string($output) ? strlen($output) : 0;
+
+ return $output;
+ }
+
+ /**
+ * Stream EOF check handler. See feof().
+ */
+ public function stream_eof(): bool
+ {
+ return $this->position >= $this->end || feof($this->stream);
+ }
+
+ /**
+ * Stream seeking handler. See fseek().
+ */
+ public function stream_seek(int $offset, int $whence): bool
+ {
+ throw new \Exception("Seek not implemented");
+ }
+
+ /**
+ * Stream tell handler. See ftell().
+ */
+ public function stream_tell(): int
+ {
+ return $this->position;
+ }
+
+ /**
+ * Stream writing handler.
+ */
+ public function stream_write(string $data): int
+ {
+ throw new \Exception("File input stream is readonly");
+ }
+}
diff --git a/src/app/Utils.php b/src/app/Utils.php
--- a/src/app/Utils.php
+++ b/src/app/Utils.php
@@ -468,7 +468,7 @@
$env['paymentProvider'] = \config('services.payment_provider');
$env['stripePK'] = \config('services.stripe.public_key');
-
+ $env['maxChunkSize'] = \App\Backends\Storage::maxChunkSize();
$env['languages'] = ContentController::locales();
$env['menu'] = ContentController::menu();
diff --git a/src/config/octane.php b/src/config/octane.php
--- a/src/config/octane.php
+++ b/src/config/octane.php
@@ -224,6 +224,8 @@
'log_file' => storage_path('logs/swoole_http.log'),
// Max input size, this does not apply to file uploads
+ // Note: It looks like on the PHP side you need about 2 times as much of extra memory
+ // as the request size when using Swoole (only 1 time without Swoole).
'package_max_length' => env('SWOOLE_PACKAGE_MAX_LENGTH', 10 * 1024 * 1024),
// This defines max. size of a file uploaded using multipart/form-data method
diff --git a/src/resources/js/files.js b/src/resources/js/files.js
--- a/src/resources/js/files.js
+++ b/src/resources/js/files.js
@@ -5,11 +5,7 @@
// Note: The value may change to the value provided by the server on the first upload.
// Note: That chunk size here is only body, Swoole's package_max_length is body + headers,
// so don't forget to subtract some margin (e.g. 8KB)
- // FIXME: From my preliminary tests it looks like on the PHP side you need
- // about 3-4 times as much memory as the request size when using Swoole
- // (only 1 time without Swoole). And I didn't find a way to lower the memory usage,
- // it looks like it happens before we even start to process the request in FilesController.
- let maxChunkSize = 5 * 1024 * 1024 - 1024 * 8
+ let maxChunkSize = window.config['maxChunkSize'] || (5 * 1024 * 1024 - 1024 * 8)
const area = $(params.dropArea)
diff --git a/src/tests/Feature/Backends/StorageTest.php b/src/tests/Feature/Backends/StorageTest.php
new file mode 100644
--- /dev/null
+++ b/src/tests/Feature/Backends/StorageTest.php
@@ -0,0 +1,46 @@
+<?php
+
+namespace Tests\Feature\Backends;
+
+use App\Backends\Storage;
+use App\Fs\Item;
+use Illuminate\Support\Facades\Storage as LaravelStorage;
+use Tests\TestCaseFs;
+
+class StorageTest extends TestCaseFs
+{
+ /**
+ * Test Storage::fileInput() splitting the input into chunks
+ */
+ public function testFileInputChunking(): void
+ {
+ $user = $this->getTestUser('john@kolab.org');
+ $file = $user->fsItems()->create(['type' => Item::TYPE_FILE | Item::TYPE_INCOMPLETE]);
+
+ \config(['octane.swoole.options.package_max_length' => 4000]);
+
+ $stream = fopen('php://memory', 'r+');
+ $content = [str_repeat('0123', 1000), str_repeat('abcd', 1000), str_repeat('yu', 1000)];
+ fwrite($stream, implode('', $content));
+ rewind($stream);
+
+ $result = Storage::fileInput($stream, [], $file);
+
+ $file->refresh();
+ $this->assertSame(['id' => $file->id], $result);
+ $this->assertFalse($file->isIncomplete());
+ $this->assertSame('10000', $file->getProperty('size'));
+ $this->assertSame('text/plain', $file->getProperty('mimetype'));
+
+ $chunks = $file->chunks()->orderBy('sequence')->get();
+ $this->assertCount(3, $chunks);
+ $this->assertSame(4000, $chunks[0]->size);
+ $this->assertSame(4000, $chunks[1]->size);
+ $this->assertSame(2000, $chunks[2]->size);
+
+ $disk = LaravelStorage::disk(\config('filesystems.default'));
+ $this->assertSame($content[0], $disk->read(Storage::chunkLocation($chunks[0]->chunk_id, $file)));
+ $this->assertSame($content[1], $disk->read(Storage::chunkLocation($chunks[1]->chunk_id, $file)));
+ $this->assertSame($content[2], $disk->read(Storage::chunkLocation($chunks[2]->chunk_id, $file)));
+ }
+}

File Metadata

Mime Type
text/plain
Expires
Fri, Apr 3, 8:26 AM (18 h, 34 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18823230
Default Alt Text
D5748.1775204803.diff (9 KB)

Event Timeline