diff --git a/src/app/Backends/Storage.php b/src/app/Backends/Storage.php new file mode 100644 --- /dev/null +++ b/src/app/Backends/Storage.php @@ -0,0 +1,373 @@ +path . '/' . $file->id; + + $disk->delete($path); + } + + /** + * File download handler. + * + * @param \App\File $file File object + * + * @throws \Exception + */ + public static function fileDownload(File $file): StreamedResponse + { + $disk = LaravelStorage::disk('files'); + + $path = $file->path . '/' . $file->id; + + // I noticed that the Laravel's download() method can fail with an exception + // thrown by mimetype detector. To prevent that we do this with a custom code + // return LaravelStorage::download($path, $file->name, $headers); + + $response = new StreamedResponse(); + + // Prepare the file name for the Content-Disposition header + $extension = pathinfo($file->name, \PATHINFO_EXTENSION) ?: 'file'; + $fallbackName = str_replace('%', '', Str::ascii($file->name)) ?: "file.{$extension}"; + $disposition = $response->headers->makeDisposition('attachment', $file->name, $fallbackName); + + $response->headers->replace([ + 'Content-Length' => $file->size, + 'Content-Type' => $file->mimetype, + 'Content-Disposition' => $disposition, + ]); + + $response->setCallback(function () use ($disk, $path) { + $stream = $disk->readStream($path); + fpassthru($stream); + fclose($stream); + }); + + return $response; + } + + /** + * File upload handler + * + * @param \App\Library $library The library to which the file belongs + * @param resource $stream File input stream + * @param array $params Request parameters + * + * @return array File/Response attributes + * @throws \Exception + */ + public static function fileInput(Library $library, $stream, array $params): array + { + if (!empty($params['uploadId'])) { + return self::fileInputResumable($library, $stream, $params); + } + + $disk = LaravelStorage::disk('files'); + + $file = $library->files()->create([ + 'mimetype' => File::TYPE_INCOMPLETE, + 'name' => $params['name'], + ]); + + $path = $file->path . '/' . $file->id; + + $disk->writeStream($path, $stream); + + // Update the file type and size information + $file->size = $disk->fileSize($path); + $file->mimetype = self::mimetype($path); + $file->save(); + + return $file->toArray(); + } + + /** + * Resumable file upload handler + * + * @param \App\Library $library The library to which the file belongs + * @param resource $stream File input stream + * @param array $params Request parameters + * + * @return array File/Response attributes + * @throws \Exception + */ + protected static function fileInputResumable(Library $library, $stream, array $params): array + { + $init = $params['uploadId'] == 'resumable'; + + if ($init) { + if (empty($params['size'])) { + // error + } + + $file = $library->files()->create([ + 'mimetype' => File::TYPE_INCOMPLETE, + 'name' => $params['name'], + ]); + + $params['uploadId'] = \App\Utils::uuidStr(); + + $upload = [ + 'fileId' => $file->id, + 'size' => $params['size'], + 'uploaded' => 0, + 'libraryId' => $library->id, + ]; + + if (!Cache::add('upload:' . $params['uploadId'], $upload, self::UPLOAD_TTL)) { + // error + } + + // Empty initial request? return early + $byte = fread($stream, 1); + if ($byte === false || $byte === '') { + return ['uploadId' => $params['uploadId'], 'uploaded' => 0]; + } + + rewind($stream); + } else { + $upload = Cache::get('upload:' . $params['uploadId']); + + if (empty($upload)) { + // error + } + + $file = File::find($upload['fileId']); + + if (!$file) { + // error + } + } + + $from = $params['from'] ?? 0; + + // Sanity checks on the input parameters + // TODO: Support uploading again a chunk that already has been uploaded + if ($from < $upload['uploaded'] || $from > $upload['uploaded'] || $from > $upload['size']) { + // error + } elseif ($library->id != $upload['libraryId']) { + // error + } + + $disk = LaravelStorage::disk('files'); + + $path = 'tmp/' . $params['uploadId'] . '/' . sprintf('%010d', $from); + + // Save the file chunk + $disk->writeStream($path, $stream); + + $fsize = $disk->fileSize($path); + + // Update the file type and size information after the upload of the whole + // file is completed + if ($fsize + $upload['uploaded'] >= $upload['size']) { + // Put uploaded chunks together + $files = []; + foreach ($disk->listContents('tmp/' . $params['uploadId']) as $fileAttr) { + $files[] = $fileAttr->path(); + } + + $target = $file->path . '/' . $file->id; + + if (count($files) == 1) { + $disk->move($files[0], $target); + } else { + sort($files); + self::concatenate($target, $files); + } + + // Update file metadata + $file->size = $disk->fileSize($target); + $file->mimetype = self::mimetype($target); + $file->save(); + + // Delete the upload cache record + Cache::forget('upload:' . $params['uploadId']); + + // TODO: There should be a job that removes orphaned files/folders from the temp folder anyway + // So we could ignore that here to make the response faster + $disk->deleteDirectory('tmp/' . $params['uploadId']); + + return $file->toArray(); + } + + $upload['uploaded'] += $fsize; + + Cache::put('upload:' . $params['uploadId'], $upload, self::UPLOAD_TTL); + + return ['uploadId' => $params['uploadId'], 'uploaded' => $upload['uploaded']]; + } + + /** + * Concatenate multiple files into a target location + * + * @param string $target Target file location + * @param array $files Source files + * + * @throws \Exception + */ + protected static function concatenate(string $target, array $files): void + { + $disk = LaravelStorage::disk('files'); + + // If the target file exists, rename it to .backup + if ($disk->fileExists($target)) { + $disk->move($target, $backup = "{$target}.backup"); + } + + try { + $streams = []; + foreach ($files as $file) { + $streams[] = $disk->readStream($file); + } + + $stream = self::getResource($streams); + $disk->writeStream($target, $stream); + + fclose($stream); + foreach ($streams as $stream) { + @fclose($stream); + } + + if (!empty($backup)) { + $disk->delete($backup); + } + } catch (\Exception $e) { + // On error rename the backup file back to the original + if (!empty($backup)) { + $disk->move($backup, $target); + } + + throw $e; + } + } + + /** + * Get the file mime type. + * + * @param string $path File location + * + * @return string File mime type + */ + protected static function mimetype(string $path): string + { + $disk = LaravelStorage::disk('files'); + + // TODO: If file is empty, detect the mimetype based on the extension? + try { + return $disk->mimeType($path); + } catch (\Exception $e) { + // do nothing + } + + // TODO: If it fails detect the mimetype based on extension? + + return 'application/octet-stream'; + } + + /** + * Create a stream containing concatenated content of multiple streams + * + * @param array $streams Resource streams + * @param int $chunkSize Chunk size for copying operation + * + * @return resource + */ + protected static function getResource($streams, int $chunkSize = 8192) + { + if (empty($streams)) { + return fopen('data://text/plain,','r'); + } + + if (count($streams) == 1) { + return reset($streams); + } + + // Write all input streams data into one + // Note: The temp file will be created only if there's not enough memory. + // We can use php://temp/memory:XX if we wanted to set the limit here + $output = fopen('php://temp','r+'); + + foreach ($streams as $stream) { + stream_copy_to_stream($stream, $output); + } + + rewind($output); + + // TODO: Find why the filter-based approach below does not work + // TODO: Find another approach. Probably depending on the storage driver + // there will be better API to do the append operation faster. + // For example in filesystem we could just execute cat command. + // AWS/S3-type APIs might have also ways to do this better. + + return $output; +/* + // Note: The code below copied from keven/append-stream is working, but + // has the memory limit issue, even though it's supposed to have not + + $head = tmpfile(); + fwrite($head, fread($streams[0], 8192)); + rewind($head); + + $anonymous = new class($streams, $chunkSize) extends \php_user_filter + { + private static $streams = []; + private static $maxLength; + + public function __construct(array $streams = [], int $maxLength = 8192) + { + self::$streams = $streams; + self::$maxLength = $maxLength; + } + + public function filter($in, $out, &$consumed, $closing): int + { + while ($bucket = stream_bucket_make_writeable($in)) { + stream_bucket_append($out, $bucket); + } + + foreach (self::$streams as $idx => $stream) { + while (feof($stream) !== true) { + $bucket = stream_bucket_new($stream, fread($stream, self::$maxLength)); + stream_bucket_append($out, $bucket); + } + } + + return PSFS_PASS_ON; + } + }; + + $filter = bin2hex(random_bytes(32)); + + stream_filter_register($filter, get_class($anonymous)); + stream_filter_append($head, $filter); + + return $head; +*/ + } +} diff --git a/src/app/File.php b/src/app/File.php new file mode 100644 --- /dev/null +++ b/src/app/File.php @@ -0,0 +1,114 @@ + The attributes that are mass assignable */ + protected $fillable = [ + 'library_id', + 'mimetype', + 'name', + 'size', + ]; + + /** @var array The attributes that should be cast */ + protected $casts = [ + 'created_at' => 'datetime:Y-m-d H:i:s', + 'updated_at' => 'datetime:Y-m-d H:i:s', + ]; + + /** + * Check whether specified user (email) has permission to the file + * + * @param string $user Username (email) + * @param int $perms Permissions + * + * @return bool + */ + public function hasPermission(string $user, int $perms) + { + // FIXME: Should this method also check the file owner? + + $permission = $this->permissions()->where('user', $user)->first(); + + return (bool) ($permission && ($permission->permissions & $perms)); + } + + /** + * The library to which this file belongs. + * + * @return \Illuminate\Database\Eloquent\Relations\BelongsTo + */ + public function library() + { + return $this->belongsTo(Library::class, 'library_id', 'id'); + } + + /** + * Interact with the file's mimetype. + * + * @return \Illuminate\Database\Eloquent\Casts\Attribute + */ + protected function mimetype(): Attribute + { + return Attribute::make( + // get: fn ($value) => \strtolower($value), + set: function ($value) { + return \strtolower($value); + }, + ); + } + + /** + * Getter for the file path (without the filename) in the storage. + * + * @return \Illuminate\Database\Eloquent\Casts\Attribute + */ + protected function path(): Attribute + { + return Attribute::make( + get: function ($value) { + if (empty($this->library_id) || empty($this->id)) { + throw new \Exception("No file ID or library ID"); + } + + $library_id = substr(hash('crc32b', $this->library_id), 0, 6); + $id = substr(hash('crc32b', $this->id), 0, 6); + + return implode('/', str_split($library_id, 2)) + . '/' . $this->library_id + . '/' . implode('/', str_split($id, 2)); + } + ); + } + + /** + * Sharing permissions assigned to this file. + * + * @return \Illuminate\Database\Eloquent\Relations\HasMany + */ + public function permissions() + { + return $this->hasMany(FilePermission::class); + } +} diff --git a/src/app/FilePermission.php b/src/app/FilePermission.php new file mode 100644 --- /dev/null +++ b/src/app/FilePermission.php @@ -0,0 +1,43 @@ + The attributes that are mass assignable */ + protected $fillable = [ + 'file_id', + 'permissions', + 'user', + ]; + + /** @var array The attributes that should be cast */ + protected $casts = [ + 'created_at' => 'datetime:Y-m-d H:i:s', + 'updated_at' => 'datetime:Y-m-d H:i:s', + ]; + + /** + * The file to which this permission belongs. + * + * @return \Illuminate\Database\Eloquent\Relations\BelongsTo + */ + public function file() + { + return $this->belongsTo(File::class, 'file_id', 'id'); + } +} diff --git a/src/app/Http/Controllers/API/V4/FilesController.php b/src/app/Http/Controllers/API/V4/FilesController.php new file mode 100644 --- /dev/null +++ b/src/app/Http/Controllers/API/V4/FilesController.php @@ -0,0 +1,380 @@ +errorResponse(404); + } + + if (!self::hasPermission($file, $this->guard()->user(), FilePermission::DELETE)) { + return $this->errorResponse(403); + } + + // FIXME: Here we're just deleting the file, but maybe it would be better/faster + // to mark the file (record in db) as deleted and invoke a job to + // delete it asynchronously? + + Storage::fileDelete($file); + + $file->delete(); + + return response()->json([ + 'status' => 'success', + 'message' => \trans('app.file-delete-success'), + ]); + } + + /** + * Fetch the permissions for the specific file. + * + * @param string $id The file identifier. + * + * @return \Illuminate\Http\JsonResponse + */ + public function getPermissions($id) + { + $file = File::find($id); + + if (!$file) { + return $this->errorResponse(404); + } + + // Only the folder owner can do that, for now + if ($this->guard()->user()->id != $file->library->user_id) { + return $this->errorResponse(403); + } + + $result = $file->permissions()->orderBy('user')->get()->map( + function ($permission) { + // FIXME: Here we map internal format to the one supported + // by the ACL widget, but I guess we can get rid of this limitation + // in the future, if needed. + if ($permission->permissions & FilePermission::DELETE) { + $perms = 'full'; + } elseif ($permission->permissions & FilePermission::WRITE) { + $perms = 'read-write'; + } + + return [ + 'user' => $permission->user, + 'permissions' => $perms ?? 'read-only', + ]; + } + ); + + $result = [ + 'list' => $result, + 'count' => count($result), + ]; + + return response()->json($result); + } + + /** + * Listing of files (and folders). + * + * @return \Illuminate\Http\JsonResponse + */ + public function index() + { + $search = trim(request()->input('search')); + $page = intval(request()->input('page')) ?: 1; + $pageSize = 100; + $hasMore = false; + + $user = $this->guard()->user(); + + if (request()->input('shared')) { + $result = File::join('file_permissions', 'file_permissions.file_id', '=', 'files.id') + ->where('user', $user->email); + } else { + $library = $this->defaultLibrary($user); + + $result = $library->files(); + } + + $result = $result->where('mimetype', '<>', File::TYPE_INCOMPLETE) + ->orderBy('name') + ->limit($pageSize + 1) + ->offset($pageSize * ($page - 1)) + ->get(); + + if (count($result) > $pageSize) { + $result->pop(); + $hasMore = true; + } + + // Process the result + $result = $result->map( + function ($file) { + return $this->objectToClient($file); + } + ); + + $result = [ + 'list' => $result, + 'count' => count($result), + 'hasMore' => $hasMore, + ]; + + return response()->json($result); + } + + /** + * Set permissions for the specific file. + * + * @param string $id The file identifier. + * + * @return \Illuminate\Http\JsonResponse + */ + public function setPermissions($id) + { + $file = File::find($id); + + if (!$file) { + return $this->errorResponse(404); + } + + // Only the library owner can do that, for now + if ($this->guard()->user()->id != $file->library->user_id) { + return $this->errorResponse(403); + } + + // Validate/format input + $input = []; + $errors = []; + + foreach ((array) request()->input('permissions') as $idx => $entry) { + $acl = $entry['permissions'] ?? ''; + $user = $entry['user'] ?? ''; + + if (empty($user)) { + continue; + } + + // validate user email + $v = Validator::make(['email' => $user], ['email' => 'email']); + + if ($v->fails()) { + $errors['user'][$idx] = \trans('validation.emailinvalid'); + } + + // The ACL widget supports 'full', 'read-write', 'read-only', convert + // it to the internal format + if ($acl == 'full') { + $acl = FilePermission::DELETE | FilePermission::WRITE | FilePermission::READ; + } elseif ($acl == 'read-write') { + $acl = FilePermission::WRITE | FilePermission::READ; + } elseif ($acl == 'read-only') { + $acl = FilePermission::READ; + } else { + continue; + } + + $input[$user] = $acl; + } + + if (!empty($errors)) { + return response()->json(['status' => 'error', 'errors' => $errors], 422); + } + + // Get existing permissions and compare with the new ones + // Update/delete existing entries + $file->permissions()->get()->each(function($permission) use (&$input) { + if (!empty($input[$permission->user])) { + $permission->permissions = $input[$permission->user]; + $permission->save(); + + unset($input[$permission->user]); + } else { + $permission->delete(); + } + }); + + // Create new permissions + foreach ($input as $user => $permissions) { + $file->permissions()->create([ + 'user' => $user, + 'permissions' => $permissions + ]); + } + + return response()->json([ + 'status' => 'success', + 'message' => \trans('app.file-permissions-update-success'), + ]); + } + + /** + * Fetch the specific file metadata or content. + * + * @param string $id The file identifier. + * + * @return \Illuminate\Http\JsonResponse|\Symfony\Component\HttpFoundation\StreamedResponse + */ + public function show($id) + { + $file = File::find($id); + + if (!$file) { + return $this->errorResponse(404); + } + + if (!self::hasPermission($file, $this->guard()->user(), FilePermission::READ)) { + return $this->errorResponse(403); + } + + if (request()->input('download')) { + return Storage::fileDownload($file); + } + + $response = $file->toArray(); + + return response()->json($response); + } + + /** + * Create a new file. + * + * @param \Illuminate\Http\Request $request The API request. + * + * @return \Illuminate\Http\JsonResponse The response + */ + public function store(Request $request) + { + $user = $this->guard()->user(); + + $library = $this->defaultLibrary($user); + + $filename = $request->input('name'); + + // TODO: Validate file name input + // TODO: Delete the existing incomplete file with the same name? + + // FIXME: Normally people just drag and drop/upload files. + // The client side will not know whether the file with the same name + // already exists or not. So, in such a case should we throw + // an error or accept the request as an update? + + $params = ['name' => $filename]; + + if ($upload = $request->input('upload')) { + $params['uploadId'] = $upload; + $params['size'] = $request->input('size'); + $params['from'] = $request->input('from'); + } + + try { + $response = Storage::fileInput($library, $request->getContent(true), $params); + + $response['status'] = 'success'; + + if (!empty($response['id'])) { + $response['message'] = \trans('app.file-create-success'); + } + } catch (\Exception $e) { + \Log::error($e); + return $this->errorResponse(500); + } + + return response()->json($response); + } + + /** + * Update a file. + * + * @param \Illuminate\Http\Request $request The API request. + * @param string $id File identifier + * + * @return \Illuminate\Http\JsonResponse The response + */ + public function update(Request $request, $id) + { + $file = File::find($id); + + if (empty($file)) { + return $this->errorResponse(404); + } + + $user = $this->guard()->user(); + + if (!self::hasPermission($file, $user, FilePermission::WRITE)) { + return $this->errorResponse(403); + } + + // TODO + + return response()->json([ + 'status' => 'success', + 'message' => \trans('app.file-update-success'), + ]); + } + + /** + * Get the user's default Library. Create one if it does not exist. + * + * @param \App\User $user User + * + * @return \App\Library The default library for the user + */ + protected function defaultLibrary(User $user): Library + { + $library = $user->libraries()->first(); + + if (!$library) { + $library = $user->libraries()->create(['name' => null]); + } + + return $library; + } + + /** + * Checks whether the specified user has specified permissions to a file. + * + * @param \App\File $file The file + * @param \App\User $user Current user + * @param int $perms Permissions + * + * @return bool + */ + protected static function hasPermission(File $file, User $user, int $perms) + { + // File owner + if ($user->id == $file->library->user_id) { + return true; + } + + return $file->hasPermission($user->email, $perms); + } +} diff --git a/src/app/Http/Controllers/API/V4/UsersController.php b/src/app/Http/Controllers/API/V4/UsersController.php --- a/src/app/Http/Controllers/API/V4/UsersController.php +++ b/src/app/Http/Controllers/API/V4/UsersController.php @@ -185,6 +185,7 @@ 'enableDomains' => $isController && $hasCustomDomain, // TODO: Make 'enableDistlists' working for wallet controllers that aren't account owners 'enableDistlists' => $isController && $hasCustomDomain && in_array('beta-distlists', $skus), + 'enableFiles' => true, // TODO: Make 'enableFolders' working for wallet controllers that aren't account owners 'enableFolders' => $isController && $hasCustomDomain && in_array('beta-shared-folders', $skus), // TODO: Make 'enableResources' working for wallet controllers that aren't account owners diff --git a/src/app/Http/Middleware/ContentSecurityPolicy.php b/src/app/Http/Middleware/ContentSecurityPolicy.php --- a/src/app/Http/Middleware/ContentSecurityPolicy.php +++ b/src/app/Http/Middleware/ContentSecurityPolicy.php @@ -25,7 +25,7 @@ foreach ($headers as $opt => $header) { if ($value = \config("app.headers.{$opt}")) { - $next->header($header, $value); + $next->headers->set($header, $value); } } diff --git a/src/app/Library.php b/src/app/Library.php new file mode 100644 --- /dev/null +++ b/src/app/Library.php @@ -0,0 +1,52 @@ + The attributes that are mass assignable */ + protected $fillable = [ + 'name', + 'user_id', + ]; + + /** @var array The attributes that can be not set */ + protected $nullable = [ + 'name', + ]; + + + /** + * Files in this library. + * + * @return \Illuminate\Database\Eloquent\Relations\HasMany + */ + public function files() + { + return $this->hasMany(File::class); + } + + /** + * The user the library belongs to. + * + * @return \Illuminate\Database\Eloquent\Relations\BelongsTo + */ + public function user() + { + return $this->belongsTo(User::class, 'user_id', 'id'); + } +} diff --git a/src/app/User.php b/src/app/User.php --- a/src/app/User.php +++ b/src/app/User.php @@ -381,6 +381,16 @@ return false; } + /** + * Storage libraries for this user. + * + * @return \Illuminate\Database\Eloquent\Relations\HasMany + */ + public function libraries() + { + return $this->hasMany(Library::class); + } + /** * A shortcut to get the user name. * diff --git a/src/app/Wallet.php b/src/app/Wallet.php --- a/src/app/Wallet.php +++ b/src/app/Wallet.php @@ -27,6 +27,7 @@ use SettingsTrait; use UuidStrKeyTrait; + /** @var bool Indicates that the model should be timestamped or not */ public $timestamps = false; /** @var array The attributes' default values */ diff --git a/src/config/filesystems.php b/src/config/filesystems.php --- a/src/config/filesystems.php +++ b/src/config/filesystems.php @@ -35,6 +35,11 @@ 'root' => storage_path('app'), ], + 'files' => [ + 'driver' => 'local', + 'root' => storage_path('app/files'), + ], + 'pgp' => [ 'driver' => 'local', 'root' => storage_path('app/keys'), diff --git a/src/database/migrations/2022_03_02_100000_create_filesystem_tables.php b/src/database/migrations/2022_03_02_100000_create_filesystem_tables.php new file mode 100644 --- /dev/null +++ b/src/database/migrations/2022_03_02_100000_create_filesystem_tables.php @@ -0,0 +1,75 @@ +string('id', 36)->primary(); + $table->bigInteger('user_id'); + $table->string('name')->nullable(); + $table->timestamp('created_at')->useCurrent(); + $table->timestamp('updated_at')->useCurrent(); + + $table->foreign('user_id')->references('id')->on('users') + ->onUpdate('cascade')->onDelete('cascade'); + } + ); + + Schema::create( + 'files', + function (Blueprint $table) { + $table->string('id', 36)->primary(); + $table->string('library_id', 36); + $table->string('name', 512); + $table->bigInteger('size')->unsigned()->default(0); + $table->string('mimetype'); + // $table->text('metadata')->nullable(); + $table->timestamp('created_at')->useCurrent(); + $table->timestamp('updated_at')->useCurrent(); + + $table->unique(['library_id', 'name']); + + $table->foreign('library_id')->references('id')->on('libraries') + ->onUpdate('cascade')->onDelete('cascade'); + } + ); + + Schema::create( + 'file_permissions', + function (Blueprint $table) { + $table->bigIncrements('id'); + $table->string('file_id', 36); + // FIXME: Maybe instead of 'user', it would be better to name it 'who', or 'identifier' or? + $table->string('user')->index(); + $table->smallInteger('permissions')->unsigned()->default(0); + $table->timestamp('created_at')->useCurrent(); + $table->timestamp('updated_at')->useCurrent(); + + $table->unique(['file_id', 'user']); + + $table->foreign('file_id')->references('id')->on('files') + ->onUpdate('cascade')->onDelete('cascade'); + } + ); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('file_permissions'); + Schema::dropIfExists('files'); + Schema::dropIfExists('libraries'); + } +}; diff --git a/src/resources/js/app.js b/src/resources/js/app.js --- a/src/resources/js/app.js +++ b/src/resources/js/app.js @@ -255,7 +255,7 @@ this.errorPage(status, message) } }, - downloadFile(url) { + downloadFile(url, filename) { // TODO: This might not be a best way for big files as the content // will be stored (temporarily) in browser memory // TODO: This method does not show the download progress in the browser @@ -263,13 +263,16 @@ axios.get(url, { responseType: 'blob' }) .then(response => { const link = document.createElement('a') - const contentDisposition = response.headers['content-disposition'] - let filename = 'unknown' - if (contentDisposition) { - const match = contentDisposition.match(/filename="(.+)"/); - if (match.length === 2) { - filename = match[1]; + if (!filename) { + const contentDisposition = response.headers['content-disposition'] + filename = 'unknown' + + if (contentDisposition) { + const match = contentDisposition.match(/filename="?(.+)"?/); + if (match && match.length === 2) { + filename = match[1]; + } } } diff --git a/src/resources/js/files.js b/src/resources/js/files.js new file mode 100644 --- /dev/null +++ b/src/resources/js/files.js @@ -0,0 +1,153 @@ + +function FileAPI(params = {}) +{ + // Note that chunk size here is only body, Swoole's package_max_length is body + headers, + // so don't forget to subtract some margin (e.g. 8KB) + // FIXME: From my preliminary tests it looks like on the PHP side you need + // about 3-4 times as much memory as the request size when using Swoole + // (only 1 time without Swoole). And I didn't find a way to lower the memory usage, + // it looks like it happens before we even start to process the request in FilesController. + const maxChunkSize = params.maxChunkSize || 10 * 1024 * 1024 - 1024 * 8 + + const area = $(params.dropArea) + + // Add hidden input to the drop area, so we can handle upload by click + const input = $('') + .attr({type: 'file', multiple: true, style: 'visibility: hidden'}) + .on('change', event => { fileDropHandler(event) }) + .appendTo(area) + .get(0) + + // Register events on the upload area element + area.on('click', () => input.click()) + .on('drop', event => fileDropHandler(event)) + .on('dragenter dragleave drop', event => fileDragHandler(event)) + .on('dragover', event => event.preventDefault()) // prevents file from being opened on drop) + + // Handle dragging on the whole page, so we can style the area in a different way + $(document.documentElement).off('.fileapi') + .on('dragenter.fileapi dragleave.fileapi', event => area.toggleClass('dragactive')) + + // Handle dragging file(s) - style the upload area element + const fileDragHandler = (event) => { + if (event.type == 'drop') { + area.removeClass('dragover dragactive') + } else { + area[event.type == 'dragenter' ? 'addClass' : 'removeClass']('dragover') + } + } + + // Handler for both a ondrop event and file input onchange event + const fileDropHandler = (event) => { + let files = event.target.files || event.dataTransfer.files + + if (!files || !files.length) { + return + } + + // Prevent default behavior (prevent file from being opened on drop) + event.preventDefault(); + + // TODO: Check file size limit, limit number of files to upload at once? + + // For every file... + for (const file of files) { + const progress = { + id: Date.now(), + name: file.name, + total: file.size, + completed: 0 + } + + file.uploaded = 0 + + // Upload request configuration + const config = { + onUploadProgress: progressEvent => { + progress.completed = Math.round(((file.uploaded + progressEvent.loaded) * 100) / file.size) + + // Don't trigger the event when 100% of the file has been sent + // We need to wait until the request response is available, then + // we'll trigger it (see below where the axios request is created) + if (progress.completed < 100) { + params.eventHandler('upload-progress', progress) + } + }, + headers: { + 'Content-Type': file.type + }, + params: { name: file.name }, + maxBodyLength: -1, // no limit + timeout: 0, // no limit + transformRequest: [] // disable body transformation + } + + // FIXME: It might be better to handle multiple-files upload as a one + // "progress source", i.e. we might want to refresh the files list once + // all files finished uploading, not multiple times in the middle + // of the upload process. + + params.eventHandler('upload-progress', progress) + + // A "recursive" function to upload the file in chunks (if needed) + const uploadFn = (start = 0, uploadId) => { + let body = file + + if (file.size > maxChunkSize) { + body = file.slice(start, start + maxChunkSize, file.type) + + if (uploadId) { + config.params.upload = uploadId + config.params.from = start + } else { + config.params.upload = 'resumable' + config.params.size = file.size + } + } + + start += maxChunkSize + + axios.post('api/v4/files', body, config) + .then(response => { + if (start < file.size) { + file.uploaded = start + uploadFn(start, response.data.uploadId) + } else { + progress.completed = 100 + params.eventHandler('upload-progress', progress) + } + }) + .catch(error => { + // TODO: The process might get stopped if the authentication token expires + // in the middle of the upload process, we have to detect 401 response, + // refresh the token and continue with the last chunk that failed. + // Related setting: OAUTH_TOKEN_EXPIRY + + // console.error(error) + progress.error = error + progress.completed = 100 + params.eventHandler('upload-progress', progress) + }) + } + + // Start uploading + uploadFn() + } + } + + /** + * Convert file size as a number of bytes to a human-readable format + */ + this.sizeText = (bytes) => { + if (bytes >= 1073741824) + return parseFloat(bytes/1073741824).toFixed(2) + ' GB'; + if (bytes >= 1048576) + return parseFloat(bytes/1048576).toFixed(2) + ' MB'; + if (bytes >= 1024) + return parseInt(bytes/1024) + ' kB'; + + return parseInt(bytes || 0) + ' B'; + } +} + +export default FileAPI diff --git a/src/resources/js/user/routes.js b/src/resources/js/user/routes.js --- a/src/resources/js/user/routes.js +++ b/src/resources/js/user/routes.js @@ -13,6 +13,7 @@ const DistlistListComponent = () => import(/* webpackChunkName: "../user/pages" */ '../../vue/Distlist/List') const DomainInfoComponent = () => import(/* webpackChunkName: "../user/pages" */ '../../vue/Domain/Info') const DomainListComponent = () => import(/* webpackChunkName: "../user/pages" */ '../../vue/Domain/List') +const FileListComponent = () => import(/* webpackChunkName: "../user/pages" */ '../../vue/File/List') const MeetComponent = () => import(/* webpackChunkName: "../user/pages" */ '../../vue/Rooms') const ResourceInfoComponent = () => import(/* webpackChunkName: "../user/pages" */ '../../vue/Resource/Info') const ResourceListComponent = () => import(/* webpackChunkName: "../user/pages" */ '../../vue/Resource/List') @@ -57,6 +58,12 @@ component: DomainListComponent, meta: { requiresAuth: true, perm: 'domains' } }, + { + path: '/files', + name: 'files', + component: FileListComponent, + meta: { requiresAuth: true, perm: 'files' } + }, { path: '/login', name: 'login', diff --git a/src/resources/lang/en/app.php b/src/resources/lang/en/app.php --- a/src/resources/lang/en/app.php +++ b/src/resources/lang/en/app.php @@ -68,6 +68,11 @@ 'domain-unsuspend-success' => 'Domain unsuspended successfully.', 'domain-setconfig-success' => 'Domain settings updated successfully.', + 'file-create-success' => 'File created successfully.', + 'file-delete-success' => 'File deleted successfully.', + 'file-update-success' => 'File updated successfully.', + 'file-permissions-update-success' => 'File permissions updated successfully.', + 'resource-update-success' => 'Resource updated successfully.', 'resource-create-success' => 'Resource created successfully.', 'resource-delete-success' => 'Resource deleted successfully.', diff --git a/src/resources/lang/en/ui.php b/src/resources/lang/en/ui.php --- a/src/resources/lang/en/ui.php +++ b/src/resources/lang/en/ui.php @@ -31,6 +31,7 @@ 'resend' => "Resend", 'save' => "Save", 'search' => "Search", + 'share' => "Share", 'signup' => "Sign Up", 'submit' => "Submit", 'suspend' => "Suspend", @@ -43,6 +44,7 @@ 'distlists' => "Distribution lists", 'chat' => "Video chat", 'domains' => "Domains", + 'files' => "Files", 'invitations' => "Invitations", 'profile' => "Your profile", 'resources' => "Resources", @@ -109,6 +111,16 @@ 'form' => "Form validation error", ], + 'file' => [ + 'create' => "Create file", + 'delete' => "Delete file", + 'list-empty' => "There are no files in this account.", + 'new' => "New file", + 'search' => "File name", + 'sharing' => "File sharing", + 'sharing-text' => "", + ], + 'form' => [ 'acl' => "Access rights", 'acl-full' => "All", @@ -137,6 +149,7 @@ 'phone' => "Phone", 'settings' => "Settings", 'shared-folder' => "Shared Folder", + 'size' => "Size", 'status' => "Status", 'surname' => "Surname", 'type' => "Type", @@ -285,6 +298,7 @@ 'notfound' => "Resource not found.", 'info' => "Information", 'error' => "Error", + 'uploading' => "Uploading...", 'warning' => "Warning", 'success' => "Success", ], diff --git a/src/resources/themes/forms.scss b/src/resources/themes/forms.scss --- a/src/resources/themes/forms.scss +++ b/src/resources/themes/forms.scss @@ -169,3 +169,32 @@ } } } + +.file-drop-area { + display: inline; + background: $menu-bg-color; + color: grey; + font-size: 0.9rem; + font-weight: normal; + line-height: 2; + border: 1px solid #eee; + border-radius: 0.5em; + padding: 0.5em; + cursor: pointer; + position: relative; + + input { + position: absolute; + height: 10px; + } + + &.dragactive { + border: 1px dashed #aaa; + } + + &.dragover { + background-color: rgba($main-color, 0.25); + border: 1px dashed $main-color; + color: $main-color; + } +} diff --git a/src/resources/themes/toast.scss b/src/resources/themes/toast.scss --- a/src/resources/themes/toast.scss +++ b/src/resources/themes/toast.scss @@ -51,3 +51,17 @@ .toast-body { color: #fff; } + +.toast-progress { + margin: 0.5em; + margin-top: 0; + height: 3px; + background: #222; + border-radius: 1.5px; + overflow: hidden; +} + +.toast-progress-bar { + height: 100%; + background: $main-color; +} diff --git a/src/resources/vue/Dashboard.vue b/src/resources/vue/Dashboard.vue --- a/src/resources/vue/Dashboard.vue +++ b/src/resources/vue/Dashboard.vue @@ -32,6 +32,10 @@ {{ $t('dashboard.chat') }} {{ $t('dashboard.beta') }} + + {{ $t('dashboard.files') }} + {{ $t('dashboard.beta') }} + {{ $t('dashboard.settings') }} diff --git a/src/resources/vue/File/List.vue b/src/resources/vue/File/List.vue new file mode 100644 --- /dev/null +++ b/src/resources/vue/File/List.vue @@ -0,0 +1,268 @@ + + + diff --git a/src/resources/vue/Widgets/AclInput.vue b/src/resources/vue/Widgets/AclInput.vue --- a/src/resources/vue/Widgets/AclInput.vue +++ b/src/resources/vue/Widgets/AclInput.vue @@ -1,7 +1,7 @@ @@ -52,9 +55,15 @@ return this.data.titleClassName || '' } }, + delete() { + new Toast(this.$el).dispose() + }, toastClassName() { return 'toast hide toast-' + this.data.type + (this.data.className ? ' ' + this.data.className : '') + }, + updateProgress(percent) { + $(this.$el).find('.toast-progress-bar').css('width', percent + '%') } } } diff --git a/src/routes/api.php b/src/routes/api.php --- a/src/routes/api.php +++ b/src/routes/api.php @@ -76,6 +76,10 @@ Route::get('domains/{id}/status', [API\V4\DomainsController::class, 'status']); Route::post('domains/{id}/config', [API\V4\DomainsController::class, 'setConfig']); + Route::apiResource('files', API\V4\FilesController::class); + Route::get('files/{id}/permissions', [API\V4\FilesController::class, 'getPermissions']); + Route::post('files/{id}/permissions', [API\V4\FilesController::class, 'setPermissions']); + Route::apiResource('groups', API\V4\GroupsController::class); Route::get('groups/{id}/status', [API\V4\GroupsController::class, 'status']); Route::post('groups/{id}/config', [API\V4\GroupsController::class, 'setConfig']); diff --git a/src/tests/Feature/Controller/FilesTest.php b/src/tests/Feature/Controller/FilesTest.php new file mode 100644 --- /dev/null +++ b/src/tests/Feature/Controller/FilesTest.php @@ -0,0 +1,441 @@ +delete(); + Library::query()->delete(); + } + + /** + * {@inheritDoc} + */ + public function tearDown(): void + { + File::query()->delete(); + Library::query()->delete(); + + $disk = Storage::disk('files'); + foreach ($disk->listContents('') as $dir) { + $disk->deleteDirectory($dir->path()); + } + + parent::tearDown(); + } + + /** + * Test deleting files (DELETE /api/v4/files/) + */ + public function testDelete(): void + { + $this->markTestIncomplete(); + } + + /** + * Test fetching/creating/updaing file permissions (GET|POST /api/v4/files//permissions) + */ + public function testPermissions(): void + { + $john = $this->getTestUser('john@kolab.org'); + $jack = $this->getTestUser('jack@kolab.org'); + $library = $john->libraries()->create(['name' => null]); + $file = $library->files()->create([ + 'mimetype' => 'text/plain', + 'name' => 'test1.txt', + 'size' => 12345, + ]); + + // Unauth access not allowed + $response = $this->get("api/v4/files/{$file->id}/permissions"); + $response->assertStatus(401); + $response = $this->post("api/v4/files/{$file->id}/permissions", []); + $response->assertStatus(401); + + // Non-existing file + $response = $this->actingAs($john)->get("api/v4/files/1234/permissions"); + $response->assertStatus(404); + $response = $this->actingAs($john)->post("api/v4/files/1234/permissions", []); + $response->assertStatus(404); + + // No permissions to the file + $response = $this->actingAs($jack)->get("api/v4/files/{$file->id}/permissions"); + $response->assertStatus(403); + $response = $this->actingAs($jack)->post("api/v4/files/{$file->id}/permissions", []); + $response->assertStatus(403); + + // Expect an empty list of permissions + $response = $this->actingAs($john)->get("api/v4/files/{$file->id}/permissions"); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertCount(2, $json); + $this->assertSame([], $json['list']); + $this->assertSame(0, $json['count']); + + // Empty input + $response = $this->actingAs($john)->post("api/v4/files/{$file->id}/permissions", []); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertSame('success', $json['status']); + $this->assertSame("File permissions updated successfully.", $json['message']); + + // TODO: Test input validation + + // Let's set some permissions + $post = [ + 'permissions' => [ + ['user' => 'test@gmail.com', 'permissions' => 'read-only'], + ['user' => 'jack@kolab.org', 'permissions' => 'read-write'], + ], + ]; + + $response = $this->actingAs($john)->post("api/v4/files/{$file->id}/permissions", $post); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertSame('success', $json['status']); + $this->assertSame("File permissions updated successfully.", $json['message']); + + $permissions = $file->permissions()->orderBy('user')->get(); + + $this->assertCount(2, $permissions); + $this->assertSame(FilePermission::WRITE | FilePermission::READ, $permissions[0]->permissions); + $this->assertSame($jack->email, $permissions[0]->user); + $this->assertSame(FilePermission::READ, $permissions[1]->permissions); + $this->assertSame('test@gmail.com', $permissions[1]->user); + + // Test update/delete entries + $post = [ + 'permissions' => [ + ['user' => 'jack@kolab.org', 'permissions' => 'read-only'], + ['user' => 'test1@gmail.com', 'permissions' => 'read-write'], + ['user' => 'test2@kolab.org', 'permissions' => 'full'], + ], + ]; + + $response = $this->actingAs($john)->post("api/v4/files/{$file->id}/permissions", $post); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertSame('success', $json['status']); + $this->assertSame("File permissions updated successfully.", $json['message']); + + $permissions = $file->permissions()->orderBy('user')->get(); + + $this->assertCount(3, $permissions); + $this->assertSame(FilePermission::READ, $permissions[0]->permissions); + $this->assertSame($jack->email, $permissions[0]->user); + $this->assertSame(FilePermission::WRITE | FilePermission::READ, $permissions[1]->permissions); + $this->assertSame('test1@gmail.com', $permissions[1]->user); + $this->assertSame( + FilePermission::WRITE | FilePermission::READ | FilePermission::DELETE, + $permissions[2]->permissions + ); + $this->assertSame('test2@kolab.org', $permissions[2]->user); + + // Test GET with existing permissions + $response = $this->actingAs($john)->get("api/v4/files/{$file->id}/permissions"); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertCount(2, $json); + $this->assertSame($post['permissions'], $json['list']); + $this->assertSame(3, $json['count']); + } + + /** + * Test fetching files/folders list + */ + public function testIndex(): void + { + // Unauth access not allowed + $response = $this->get("api/v4/files"); + $response->assertStatus(401); + + $user = $this->getTestUser('john@kolab.org'); + + // Expect an empty list + $response = $this->actingAs($user)->get("api/v4/files"); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertCount(3, $json); + $this->assertSame([], $json['list']); + $this->assertSame(0, $json['count']); + $this->assertSame(false, $json['hasMore']); + + // Create some files and test again + $library = $user->libraries()->create(['name' => null]); + $file1 = $library->files()->create([ + 'mimetype' => 'text/plain', + 'name' => 'test1.txt', + 'size' => 12345, + ]); + $file2 = $library->files()->create([ + 'mimetype' => 'image/gif', + 'name' => 'test3.gif', + 'size' => 10000, + ]); + + $response = $this->actingAs($user)->get("api/v4/files"); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertCount(3, $json); + $this->assertSame(2, $json['count']); + $this->assertSame(false, $json['hasMore']); + $this->assertCount(2, $json['list']); + $this->assertSame($file1->name, $json['list'][0]['name']); + $this->assertSame($file1->id, $json['list'][0]['id']); + $this->assertSame($file1->mimetype, $json['list'][0]['mimetype']); + $this->assertSame($file1->size, $json['list'][0]['size']); + $this->assertSame($file2->name, $json['list'][1]['name']); + $this->assertSame($file2->id, $json['list'][1]['id']); + $this->assertSame($file2->mimetype, $json['list'][1]['mimetype']); + $this->assertSame($file2->size, $json['list'][1]['size']); + + // TODO: Test paging + } + + /** + * Test fetching file metadata/content (GET /api/v4/files/) + */ + public function testShow(): void + { + // Unauth access not allowed + $response = $this->get("api/v4/files/1234"); + $response->assertStatus(401); + + $user = $this->getTestUser('john@kolab.org'); + $jack = $this->getTestUser('jack@kolab.org'); + + $file = $this->getTestFile($user, 'teśt.txt', 'Teśt content'); + + // Non-existing file + $response = $this->actingAs($jack)->get("api/v4/files/1234"); + $response->assertStatus(404); + + // Unauthorized access + $response = $this->actingAs($jack)->get("api/v4/files/{$file->id}"); + $response->assertStatus(403); + + // Get file metadata + $response = $this->actingAs($user)->get("api/v4/files/{$file->id}"); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertSame($file->id, $json['id']); + $this->assertSame($file->library_id, $json['library_id']); + $this->assertSame($file->mimetype, $json['mimetype']); + $this->assertSame($file->size, $json['size']); + + // Get file content + $response = $this->actingAs($user)->get("api/v4/files/{$file->id}?download=1"); + $response->assertStatus(200) + ->assertHeader('Content-Disposition', "attachment; filename=test.txt; filename*=utf-8''te%C5%9Bt.txt") + ->assertHeader('Content-Length', $file->size) + ->assertHeader('Content-Type', $file->mimetype . '; charset=UTF-8'); + + $this->assertSame('Teśt content', $response->streamedContent()); + + // TODO: Test acting as a user with file permissions + } + + /** + * Test creating files (POST /api/v4/files) + */ + public function testStore(): void + { + // Unauth access not allowed + $response = $this->post("api/v4/files"); + $response->assertStatus(401); + + $user = $this->getTestUser('john@kolab.org'); + + // TODO: Test name validation +/* + $response = $this->sendRawBody($user, 'POST', "api/v4/files", [], ''); + $response = $this->actingAs($user)->post("api/v4/files", []); + $response->assertStatus(422); + + $json = $response->json(); + + // $this->assertCount(3, $json); +*/ + // Create a file - the simple method + $body = "test content"; + $headers = []; + $response = $this->sendRawBody($user, 'POST', "api/v4/files?name=test.txt", $headers, $body); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertSame('success', $json['status']); + $this->assertSame("File created successfully.", $json['message']); + $this->assertSame('text/plain', $json['mimetype']); + $this->assertSame(strlen($body), $json['size']); + $this->assertSame('test.txt', $json['name']); + + $file = File::find($json['id']); + + $this->assertSame($file->mimetype, $json['mimetype']); + $this->assertSame($file->library_id, $json['library_id']); + $this->assertSame($json['size'], $file->size); + $this->assertSame('test.txt', $file->name); + + $disk = Storage::disk('files'); + $path = $file->path . '/' . $file->id; + + $this->assertSame($body, $disk->read($path)); + + // TODO: Test acting as another user with/without file permissions + } + + /** + * Test creating files - resumable (POST /api/v4/files) + */ + public function testStoreResumable(): void + { + $user = $this->getTestUser('john@kolab.org'); + + $response = $this->actingAs($user)->post("api/v4/files?name=test2.txt&upload=resumable&size=400"); + $response->assertStatus(200); + + $json = $response->json(); + + $this->assertSame(0, $json['uploaded']); + $this->assertMatchesRegularExpression( + '/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/', + $json['uploadId'] + ); + + $uploadId = $json['uploadId']; + $size = 0; + $fileContent = ''; + + for ($x = 0; $x <= 2; $x++) { + $body = str_repeat("$x", 100); + $response = $this->sendRawBody($user, 'POST', "api/v4/files?upload={$uploadId}&from={$size}", [], $body); + $response->assertStatus(200); + + $json = $response->json(); + $size += 100; + $fileContent .= $body; + + $this->assertSame($size, $json['uploaded']); + $this->assertSame($uploadId, $json['uploadId']); + } + + $body = str_repeat("$x", 100); + $response = $this->sendRawBody($user, 'POST', "api/v4/files?upload={$uploadId}&from={$size}", [], $body); + $response->assertStatus(200); + + $json = $response->json(); + $size += 100; + $fileContent .= $body; + + $this->assertSame('success', $json['status']); + // $this->assertSame("", $json['message']); + $this->assertSame('text/plain', $json['mimetype']); + $this->assertSame($size, $json['size']); + $this->assertSame('test2.txt', $json['name']); + + $file = File::find($json['id']); + + $this->assertSame($json['mimetype'], $file->mimetype); + $this->assertSame($json['library_id'], $file->library_id); + $this->assertSame($size, $file->size); + $this->assertSame('test2.txt', $file->name); + + $disk = Storage::disk('files'); + $path = $file->path . '/' . $file->id; + + $this->assertSame($fileContent, $disk->read($path)); + } + + /** + * Test updating files (PUT /api/v4/files/) + */ + public function testUpdate(): void + { + $this->markTestIncomplete(); + } + + /** + * Create a test file. + * + * @param \App\User $user File owner + * @param string $name File name + * @param string $content File content + */ + protected function getTestFile(User $user, string $name, string $content = '') + { + $library = $user->libraries()->first(); + + if (!$library) { + $library = $user->libraries()->create(['name' => null]); + } + + $disk = Storage::disk('files'); + + $file = $library->files()->create([ + 'mimetype' => File::TYPE_INCOMPLETE, + 'name' => $name, + ]); + + $path = $file->path . '/' . $file->id; + + $disk->write($path, $content); + + $file->size = $disk->fileSize($path); + $file->mimetype = $disk->mimeType($path) ?: 'application/octet-stream'; + $file->save(); + + return $file; + } + + /** + * Invoke a HTTP request with a custom raw body + * + * @param \App\User $user Authenticated user + * @param string $method Request method (POST, PUT) + * @param string $uri Request URL + * @param array $headers Request headers + * @param string $content Raw body content + * + * @return \Illuminate\Testing\TestResponse HTTP Response object + */ + protected function sendRawBody(User $user, string $method, string $uri, array $headers, string $content) + { + $headers['Content-Length'] = strlen($content); + + $server = $this->transformHeadersToServerVars($headers); + $cookies = $this->prepareCookiesForRequest(); + + return $this->actingAs($user)->call($method, $uri, [], $cookies, [], $server, $content); + } +}