lfss 0.8.0__tar.gz → 0.8.2__tar.gz

{lfss-0.8.0 → lfss-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.8.0
+Version: 0.8.2
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
@@ -16,6 +16,7 @@ Requires-Dist: aiosqlite (==0.*)
 Requires-Dist: fastapi (==0.*)
 Requires-Dist: mimesniff (==1.*)
 Requires-Dist: pillow
+Requires-Dist: python-multipart
 Requires-Dist: requests (==2.*)
 Requires-Dist: uvicorn (==0.*)
 Project-URL: Repository, https://github.com/MenxLi/lfss

{lfss-0.8.0 → lfss-0.8.2}/frontend/api.js

@@ -73,7 +73,8 @@ export default class Connector {
             method: 'PUT',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token, 
-                'Content-Type': 'application/octet-stream'
+                'Content-Type': 'application/octet-stream', 
+                'Content-Length': fileBytes.byteLength
             },
             body: fileBytes
         });
@@ -83,6 +84,38 @@ export default class Connector {
         return (await res.json()).url;
     }
 
+    /**
+    * @param {string} path - the path to the file (url)
+    * @param {File} file - the file to upload
+    * @returns {Promise<string>} - the promise of the request, the url of the file
+    */
+    async post(path, file, {
+        conflict = 'abort',
+        permission = 0
+    } = {}){
+        if (path.startsWith('/')){ path = path.slice(1); }
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('conflict', conflict);
+        dst.searchParams.append('permission', permission);
+        // post as multipart form data
+        const formData = new FormData();
+        formData.append('file', file);
+        const res = await fetch(dst.toString(), {
+            method: 'POST',
+            // don't include the content type, let the browser handle it
+            // https://muffinman.io/blog/uploading-files-using-fetch-multipart-form-data/
+            headers: {
+                'Authorization': 'Bearer ' + this.config.token, 
+            },
+            body: formData
+        });
+                
+        if (res.status != 200 && res.status != 201){
+            throw new Error(`Failed to upload file, status code: ${res.status}, message: ${await res.json()}`);
+        }
+        return (await res.json()).url;
+    }
+
     /**
     * @param {string} path - the path to the file (url), should end with .json
     * @param {Objec} data - the data to upload
@@ -422,4 +455,28 @@ export async function listPath(conn, path, {
         dirs: dirCount,
         files: fileCount
     }];
-};
+};
+
+/**
+ * a function to wrap the upload function into one
+ * it will return the url of the file
+ * 
+ * @typedef {Object} UploadOptions
+ * @property {string} conflict - the conflict resolution strategy, can be 'abort', 'replace', 'rename'
+ * @property {number} permission - the permission of the file, can be 0, 1, 2, 3
+ * 
+ * @param {Connector} conn - the connector to the API
+ * @param {string} path - the path to the file (url)
+ * @param {File} file - the file to upload
+ * @param {UploadOptions} options - the options for the request
+ * @returns {Promise<string>} - the promise of the request, the url of the file
+ */
+export async function uploadFile(conn, path, file, {
+    conflict = 'abort',
+    permission = 0
+} = {}){
+    if (file.size < 1024 * 1024 * 10){
+        return await conn.put(path, file, {conflict, permission});
+    }
+    return await conn.post(path, file, {conflict, permission});
+}

{lfss-0.8.0 → lfss-0.8.2}/frontend/scripts.js

@@ -1,4 +1,4 @@
-import { permMap, listPath } from './api.js';
+import { permMap, listPath, uploadFile } from './api.js';
 import { showFloatingWindowLineInput, showPopup } from './popup.js';
 import { formatSize, decodePathURI, ensurePathURI, getRandomString, cvtGMT2Local, debounce, encodePathURI, asHtmlText } from './utils.js';
 import { showInfoPanel, showDirInfoPanel } from './info.js';
@@ -132,7 +132,7 @@ uploadButton.addEventListener('click', () => {
     }
     path = path + fileName;
     showPopup('Uploading...', {level: 'info', timeout: 3000});
-    conn.put(path, file, {'conflict': 'overwrite'})
+    uploadFile(conn, path, file, {'conflict': 'overwrite'})
         .then(() => {
             refreshFileList();
             uploadFileNameInput.value = '';
@@ -178,10 +178,10 @@ Are you sure you want to proceed?
                 `)){ return; }
             
             let counter = 0;
-            async function uploadFile(...args){
+            async function uploadFileFn(...args){
                 const [file, path] = args;
                 try{
-                    await conn.put(path, file, {conflict: 'overwrite'});
+                    await uploadFile(conn, path, file, {conflict: 'overwrite'});
                 }
                 catch (err){
                     showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});
@@ -194,7 +194,7 @@ Are you sure you want to proceed?
             for (let i = 0; i < files.length; i++){
                 const file = files[i];
                 const path = dstPath + file.name;
-                promises.push(uploadFile(file, path));
+                promises.push(uploadFileFn(file, path));
             }
             showPopup('Uploading multiple files...', {level: 'info', timeout: 3000});
             Promise.all(promises).then(

{lfss-0.8.0 → lfss-0.8.2}/lfss/api/__init__.py

@@ -16,18 +16,29 @@ def upload_file(
     ) -> tuple[bool, str]:
     this_try = 0
     error_msg = ""
+    assert not file_path.endswith('/'), "File path must not end with a slash."
+    if dst_url.endswith('/'):
+        fname = file_path.split('/')[-1]
+        dst_url = f"{dst_url}{fname}"
+
     while this_try <= n_retries:
         try:
-            with open(file_path, 'rb') as f:
-                blob = f.read()
-            connector.put(dst_url, blob, **put_kwargs)
+            fsize = os.path.getsize(file_path)
+            if fsize < 32 * 1024 * 1024:     # 32MB
+                with open(file_path, 'rb') as f:
+                    blob = f.read()
+                connector.put(dst_url, blob, **put_kwargs)
+            else:
+                connector.post(dst_url, file_path, **put_kwargs)
             break
         except Exception as e:
             if isinstance(e, KeyboardInterrupt):
                 raise e
             if verbose:
                 print(f"Error uploading {file_path}: {e}, retrying...")
-                error_msg = str(e)
+            error_msg = str(e)
+            if hasattr(e, 'response'):
+                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
             this_try += 1
         finally:
             time.sleep(interval)
@@ -91,26 +102,43 @@ def download_file(
     ) -> tuple[bool, str]:
     this_try = 0
     error_msg = ""
+    assert not src_url.endswith('/'), "Source URL must not end with a slash."
     while this_try <= n_retries:
+        if os.path.isdir(file_path):
+            fname = src_url.split('/')[-1]
+            file_path = os.path.join(file_path, fname)
+
         if not overwrite and os.path.exists(file_path):
             if verbose:
                 print(f"File {file_path} already exists, skipping download.")
             return True, error_msg
         try:
-            blob = connector.get(src_url)
-            if blob is None:
+            fmeta = connector.get_metadata(src_url)
+            if fmeta is None:
                 error_msg = "File not found."
                 return False, error_msg
+
             pathlib.Path(file_path).parent.mkdir(parents=True, exist_ok=True)
-            with open(file_path, 'wb') as f:
-                f.write(blob)
+            fsize = fmeta.file_size   # type: ignore
+            if fsize < 32 * 1024 * 1024:     # 32MB
+                blob = connector.get(src_url)
+                assert blob is not None
+                with open(file_path, 'wb') as f:
+                    f.write(blob)
+            else:
+                with open(file_path, 'wb') as f:
+                    for chunk in connector.get_stream(src_url):
+                        f.write(chunk)
             break
+
         except Exception as e:
             if isinstance(e, KeyboardInterrupt):
                 raise e
             if verbose:
                 print(f"Error downloading {src_url}: {e}, retrying...")
-                error_msg = str(e)
+            error_msg = str(e)
+            if hasattr(e, 'response'):
+                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
             this_try += 1
         finally:
             time.sleep(interval)

{lfss-0.8.0 → lfss-0.8.2}/lfss/api/connector.py

@@ -4,6 +4,7 @@ import os
 import requests
 import requests.adapters
 import urllib.parse
+from tempfile import SpooledTemporaryFile
 from lfss.src.error import PathNotFoundError
 from lfss.src.datatype import (
     FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents, 
@@ -95,6 +96,42 @@ class Connector:
         )
         return response.json()
     
+    def post(self, path, file: str | bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip', 'skip-ahead'] = 'abort'):
+        """
+        Uploads a file to the specified path, 
+        using the POST method, with form-data/multipart.
+        file can be a path to a file on disk, or bytes.
+        """
+
+        # Skip ahead by checking if the file already exists
+        if conflict == 'skip-ahead':
+            exists = self.get_metadata(path)
+            if exists is None:
+                conflict = 'skip'
+            else:
+                return {'status': 'skipped', 'path': path}
+        
+        if isinstance(file, str):
+            assert os.path.exists(file), "File does not exist on disk"
+            fsize = os.path.getsize(file)
+
+        with open(file, 'rb') if isinstance(file, str) else SpooledTemporaryFile(max_size=1024*1024*32) as fp:
+
+            if isinstance(file, bytes):
+                fsize = len(file)
+                fp.write(file)
+                fp.seek(0)
+
+            # https://stackoverflow.com/questions/12385179/
+            print(f"Uploading {fsize} bytes")
+            response = self._fetch_factory('POST', path, search_params={
+                'permission': int(permission),
+                'conflict': conflict
+                })(
+                files={'file': fp},
+            )
+        return response.json()
+    
     def put_json(self, path: str, data: dict, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip', 'skip-ahead'] = 'abort'):
         """Uploads a JSON file to the specified path."""
         assert path.endswith('.json'), "Path must end with .json"

lfss-0.8.2/lfss/cli/__init__.py

@@ -0,0 +1,27 @@
+from contextlib import contextmanager
+from typing import Iterable, TypeVar, Generator
+import requests, os
+
+@contextmanager
+def catch_request_error():
+    try:
+        yield
+    except requests.RequestException as e:
+        print(f"\033[31m[Request error]: {e}\033[0m")
+        if e.response is not None:
+            print(f"\033[91m[Error message]: {e.response.text}\033[0m")
+
+T = TypeVar('T')
+def line_sep(iter: Iterable[T], enable=True, start=True, end=True, color="\033[90m") -> Generator[T, None, None]:
+    screen_width = os.get_terminal_size().columns
+    def print_ln():
+        print(color + "-" * screen_width + "\033[0m")
+
+    if start and enable:
+        print_ln()
+    for i, line in enumerate(iter):
+        if enable and i > 0:
+            print_ln()
+        yield line
+    if end and enable:
+        print_ln()

{lfss-0.8.0 → lfss-0.8.2}/lfss/cli/cli.py

@@ -1,7 +1,9 @@
 from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
 from pathlib import Path
-import argparse
-from lfss.src.datatype import FileReadPermission
+import argparse, typing
+from lfss.src.datatype import FileReadPermission, FileSortKey, DirSortKey
+from lfss.src.utils import decode_uri_compnents
+from . import catch_request_error, line_sep
 
 def parse_permission(s: str) -> FileReadPermission:
     if s.lower() == "public":
@@ -39,6 +41,29 @@ def parse_arguments():
     sp_download.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
     sp_download.add_argument("--retries", type=int, default=0, help="Number of retries")
 
+    # query
+    sp_query = sp.add_parser("query", help="Query files or directories metadata from the server")
+    sp_query.add_argument("path", help="Path to query", nargs="*", type=str)
+
+    # list directories
+    sp_list_d = sp.add_parser("list-dirs", help="List directories of a given path")
+    sp_list_d.add_argument("path", help="Path to list", type=str)
+    sp_list_d.add_argument("--offset", type=int, default=0, help="Offset of the list")
+    sp_list_d.add_argument("--limit", type=int, default=100, help="Limit of the list")
+    sp_list_d.add_argument("-l", "--long", action="store_true", help="Detailed list, including all metadata")
+    sp_list_d.add_argument("--order", "--order-by", type=str, help="Order of the list", default="", choices=typing.get_args(DirSortKey))
+    sp_list_d.add_argument("--reverse", "--order-desc", action="store_true", help="Reverse the list order")
+
+    # list files
+    sp_list_f = sp.add_parser("list-files", help="List files of a given path")
+    sp_list_f.add_argument("path", help="Path to list", type=str)
+    sp_list_f.add_argument("--offset", type=int, default=0, help="Offset of the list")
+    sp_list_f.add_argument("--limit", type=int, default=100, help="Limit of the list")
+    sp_list_f.add_argument("-r", "--recursive", "--flat", action="store_true", help="List files recursively")
+    sp_list_f.add_argument("-l", "--long", action="store_true", help="Detailed list, including all metadata")
+    sp_list_f.add_argument("--order", "--order-by", type=str, help="Order of the list", default="", choices=typing.get_args(FileSortKey))
+    sp_list_f.add_argument("--reverse", "--order-desc", action="store_true", help="Reverse the list order")
+
     return parser.parse_args()
 
 def main():
@@ -57,11 +82,11 @@ def main():
                 permission=args.permission
             )
             if failed_upload:
-                print("Failed to upload:")
+                print("\033[91mFailed to upload:\033[0m")
                 for path in failed_upload:
                     print(f"  {path}")
         else:
-            success = upload_file(
+            success, msg = upload_file(
                 connector, 
                 file_path = args.src, 
                 dst_url = args.dst, 
@@ -72,7 +97,7 @@ def main():
                 permission=args.permission
             )
             if not success:
-                print("Failed to upload.")
+                print("\033[91mFailed to upload: \033[0m", msg)
     
     elif args.command == "download":
         is_dir = args.src.endswith("/")
@@ -86,11 +111,11 @@ def main():
                 overwrite=args.overwrite
             )
             if failed_download:
-                print("Failed to download:")
+                print("\033[91mFailed to download:\033[0m")
                 for path in failed_download:
                     print(f"  {path}")
         else:
-            success = download_file(
+            success, msg = download_file(
                 connector, 
                 src_url = args.src, 
                 file_path = args.dst, 
@@ -100,7 +125,51 @@ def main():
                 overwrite=args.overwrite
             )
             if not success:
-                print("Failed to download.")
+                print("\033[91mFailed to download: \033[0m", msg)
+    
+    elif args.command == "query":
+        for path in args.path:
+            with catch_request_error():
+                res = connector.get_metadata(path)
+                if res is None:
+                    print(f"\033[31mNot found\033[0m ({path})")
+                else:
+                    print(res)
+    
+    elif args.command == "list-files":
+        with catch_request_error():
+            res = connector.list_files(
+                args.path, 
+                offset=args.offset, 
+                limit=args.limit, 
+                flat=args.recursive, 
+                order_by=args.order,
+                order_desc=args.reverse,
+            )
+            for i, f in enumerate(line_sep(res)):
+                f.url = decode_uri_compnents(f.url)
+                print(f"[{i+1}] {f if args.long else f.url}")
+
+            if len(res) == args.limit:
+                print(f"\033[33m[Warning] List limit reached, use --offset and --limit to list more files.")
+        
+    elif args.command == "list-dirs":
+        with catch_request_error():
+            res = connector.list_dirs(
+                args.path, 
+                offset=args.offset, 
+                limit=args.limit, 
+                skim=not args.long, 
+                order_by=args.order,
+                order_desc=args.reverse,
+            )
+            for i, d in enumerate(line_sep(res)):
+                d.url = decode_uri_compnents(d.url)
+                print(f"[{i+1}] {d if args.long else d.url}")
+
+            if len(res) == args.limit:
+                print(f"\033[33m[Warning] List limit reached, use --offset and --limit to list more directories.")
+    
     else:
         raise NotImplementedError(f"Command {args.command} not implemented.")
     

{lfss-0.8.0 → lfss-0.8.2}/lfss/cli/user.py

@@ -1,8 +1,8 @@
-import argparse, asyncio
+import argparse, asyncio, os
 from contextlib import asynccontextmanager
 from .cli import parse_permission, FileReadPermission
-from ..src.utils import parse_storage_size
-from ..src.database import Database, FileReadPermission, transaction, UserConn
+from ..src.utils import parse_storage_size, fmt_storage_size
+from ..src.database import Database, FileReadPermission, transaction, UserConn, unique_cursor, FileConn
 from ..src.connection_pool import global_entrance
 
 @global_entrance(1)
@@ -33,6 +33,7 @@ async def _main():
     sp_set.add_argument('--max-storage', type=parse_storage_size, default=None)
     
     sp_list = sp.add_parser('list')
+    sp_list.add_argument("username", nargs='*', type=str, default=None)
     sp_list.add_argument("-l", "--long", action="store_true")
     
     args = parser.parse_args()
@@ -73,10 +74,18 @@ async def _main():
     
     if args.subparser_name == 'list':
         async with get_uconn() as uconn:
+            term_width = os.get_terminal_size().columns
             async for user in uconn.all():
+                if args.username and not user.username in args.username:
+                    continue
+                print("\033[90m-\033[0m" * term_width)
                 print(user)
                 if args.long:
-                    print('  ', user.credential)
+                    async with unique_cursor() as c:
+                        fconn = FileConn(c)
+                        user_size_used = await fconn.user_size(user.id)
+                    print('- Credential: ', user.credential)
+                    print(f'- Storage: {fmt_storage_size(user_size_used)} / {fmt_storage_size(user.max_storage)}')
         
 def main():
     asyncio.run(_main())

{lfss-0.8.0 → lfss-0.8.2}/lfss/src/config.py

@@ -18,7 +18,7 @@ if __env_large_file is not None:
     LARGE_FILE_BYTES = parse_storage_size(__env_large_file)
 else:
     LARGE_FILE_BYTES = 8 * 1024 * 1024  # 8MB
-MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
+MAX_MEM_FILE_BYTES = 128 * 1024 * 1024   # 128MB
 MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
 CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 

{lfss-0.8.0 → lfss-0.8.2}/lfss/src/database.py

@@ -8,18 +8,25 @@ import zipfile, io, asyncio
 
 import aiosqlite, aiofiles
 import aiofiles.os
+import mimetypes, mimesniff
 
 from .connection_pool import execute_sql, unique_cursor, transaction
 from .datatype import (
     UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents, 
     FileSortKey, DirSortKey, isValidFileSortKey, isValidDirSortKey
     )
-from .config import LARGE_BLOB_DIR, CHUNK_SIZE
+from .config import LARGE_BLOB_DIR, CHUNK_SIZE, LARGE_FILE_BYTES, MAX_MEM_FILE_BYTES
 from .log import get_logger
 from .utils import decode_uri_compnents, hash_credential, concurrent_wrap, debounce_async
 from .error import *
 
 class DBObjectBase(ABC):
+    """
+    NOTE: 
+    The object of this class should hold a cursor to the database. 
+    The methods calling the cursor should not be called concurrently. 
+    """
+
     logger = get_logger('database', global_instance=True)
     _cur: aiosqlite.Cursor
 
@@ -205,7 +212,7 @@ class FileConn(DBObjectBase):
                 return DirectoryRecord(dir_url)
             else:
                 return await self.get_path_record(dir_url)
-        dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
+        dirs = [await get_dir(url + d) for d in dirs_str]
         return dirs
     
     async def count_path_files(self, url: str, flat: bool = False):
@@ -285,8 +292,7 @@ class FileConn(DBObjectBase):
     async def user_size(self, user_id: int) -> int:
         cursor = await self.cur.execute("SELECT size FROM usize WHERE user_id = ?", (user_id, ))
         res = await cursor.fetchone()
-        if res is None:
-            return -1
+        if res is None: return 0
         return res[0]
     async def _user_size_inc(self, user_id: int, inc: int):
         self.logger.debug(f"Increasing user {user_id} size by {inc}")
@@ -308,18 +314,14 @@ class FileConn(DBObjectBase):
         return res[0] or 0
     
     async def update_file_record(
-        self, url, owner_id: Optional[int] = None, permission: Optional[FileReadPermission] = None
+        self, url, 
+        permission: Optional[FileReadPermission] = None, 
+        mime_type: Optional[str] = None
         ):
-        old = await self.get_file_record(url)
-        assert old is not None, f"File {url} not found"
-        if owner_id is None:
-            owner_id = old.owner_id
-        if permission is None:
-            permission = old.permission
-        await self.cur.execute(
-            "UPDATE fmeta SET owner_id = ?, permission = ? WHERE url = ?", 
-            (owner_id, int(permission), url)
-            )
+        if permission is not None:
+            await self.cur.execute("UPDATE fmeta SET permission = ? WHERE url = ?", (int(permission), url))
+        if mime_type is not None:
+            await self.cur.execute("UPDATE fmeta SET mime_type = ? WHERE url = ?", (mime_type, url))
         self.logger.info(f"Updated file {url}")
     
     async def set_file_record(
@@ -392,7 +394,7 @@ class FileConn(DBObjectBase):
         self.logger.info(f"Deleted {len(ret)} file records for user {owner_id}") # type: ignore
         return ret
     
-    async def delete_path_records(self, path: str, under_user_id: Optional[int] = None) -> list[FileRecord]:
+    async def delete_path_records(self, path: str, under_owner_id: Optional[int] = None) -> list[FileRecord]:
         """Delete all records with url starting with path"""
         # update user size
         cursor = await self.cur.execute("SELECT DISTINCT owner_id FROM fmeta WHERE url LIKE ?", (path + '%', ))
@@ -406,10 +408,10 @@ class FileConn(DBObjectBase):
         # if any new records are created here, the size update may be inconsistent
         # but it's not a big deal... we should have only one writer
         
-        if under_user_id is None:
+        if under_owner_id is None:
             res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? RETURNING *", (path + '%', ))
         else:
-            res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? AND owner_id = ? RETURNING *", (path + '%', under_user_id))
+            res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? AND owner_id = ? RETURNING *", (path + '%', under_owner_id))
         all_f_rec = await res.fetchall()
         self.logger.info(f"Deleted {len(all_f_rec)} file(s) for path {path}") # type: ignore
         return [self.parse_record(r) for r in all_f_rec]
@@ -521,72 +523,77 @@ class Database:
             await execute_sql(conn, 'init.sql')
         return self
     
-    async def update_file_record(self, user: UserRecord, url: str, permission: FileReadPermission):
+    async def update_file_record(self, url: str, permission: FileReadPermission, op_user: Optional[UserRecord] = None):
         validate_url(url)
         async with transaction() as conn:
             fconn = FileConn(conn)
             r = await fconn.get_file_record(url)
             if r is None:
                 raise PathNotFoundError(f"File {url} not found")
-            if r.owner_id != user.id and not user.is_admin:
-                raise PermissionDeniedError(f"Permission denied: {user.username} cannot update file {url}")
+            if op_user is not None:
+                if r.owner_id != op_user.id and not op_user.is_admin:
+                    raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot update file {url}")
             await fconn.update_file_record(url, permission=permission)
     
     async def save_file(
         self, u: int | str, url: str, 
-        blob: bytes | AsyncIterable[bytes], 
+        blob_stream: AsyncIterable[bytes], 
         permission: FileReadPermission = FileReadPermission.UNSET, 
-        mime_type: str = 'application/octet-stream'
-        ):
+        mime_type: Optional[str] = None
+        ) -> int:
         """
-        if file_size is not provided, the blob must be bytes
+        Save a file to the database. 
+        Will check file size and user storage limit, 
+        should check permission before calling this method. 
         """
         validate_url(url)
         async with unique_cursor() as cur:
             user = await get_user(cur, u)
-            if user is None:
-                return
-            
-            # check if the user is the owner of the path, or is admin
-            if url.startswith('/'):
-                url = url[1:]
-            first_component = url.split('/')[0]
-            if first_component != user.username:
-                if not user.is_admin:
-                    raise PermissionDeniedError(f"Permission denied: {user.username} cannot write to {url}")
-                else:
-                    if await get_user(cur, first_component) is None:
-                        raise PermissionDeniedError(f"Invalid path: {first_component} is not a valid username")
+            assert user is not None, f"User {u} not found"
             
             fconn_r = FileConn(cur)
             user_size_used = await fconn_r.user_size(user.id)
 
-            if isinstance(blob, bytes):
-                file_size = len(blob)
-                if user_size_used + file_size > user.max_storage:
-                    raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
-                f_id = uuid.uuid4().hex
-
-                async with transaction() as w_cur:
-                    fconn_w = FileConn(w_cur)
-                    await fconn_w.set_file_blob(f_id, blob)
-                    await fconn_w.set_file_record(
-                        url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                        permission=permission, external=False, mime_type=mime_type)
-            else:
-                assert isinstance(blob, AsyncIterable)
-                f_id = uuid.uuid4().hex
-                file_size = await FileConn.set_file_blob_external(f_id, blob)
+            f_id = uuid.uuid4().hex
+            async with aiofiles.tempfile.SpooledTemporaryFile(max_size=MAX_MEM_FILE_BYTES) as f:
+                async for chunk in blob_stream:
+                    await f.write(chunk)
+                file_size = await f.tell()
                 if user_size_used + file_size > user.max_storage:
-                    await FileConn.delete_file_blob_external(f_id)
                     raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
                 
-                async with transaction() as w_cur:
-                    await FileConn(w_cur).set_file_record(
-                        url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                        permission=permission, external=True, mime_type=mime_type)
-            
-        await delayed_log_activity(user.username)
+                # check mime type
+                if mime_type is None:
+                    mime_type, _ = mimetypes.guess_type(url)
+                if mime_type is None:
+                    await f.seek(0)
+                    mime_type = mimesniff.what(await f.read(1024))
+                if mime_type is None:
+                    mime_type = 'application/octet-stream'
+                await f.seek(0)
+                
+                if file_size < LARGE_FILE_BYTES:
+                    blob = await f.read()
+                    async with transaction() as w_cur:
+                        fconn_w = FileConn(w_cur)
+                        await fconn_w.set_file_blob(f_id, blob)
+                        await fconn_w.set_file_record(
+                            url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                            permission=permission, external=False, mime_type=mime_type)
+                
+                else:
+                    async def blob_stream_tempfile():
+                        nonlocal f
+                        while True:
+                            chunk = await f.read(CHUNK_SIZE)
+                            if not chunk: break
+                            yield chunk
+                    await FileConn.set_file_blob_external(f_id, blob_stream_tempfile())
+                    async with transaction() as w_cur:
+                        await FileConn(w_cur).set_file_record(
+                            url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                            permission=permission, external=True, mime_type=mime_type)
+        return file_size
 
     async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
         validate_url(url)
@@ -598,11 +605,8 @@ class Database:
             if not r.external:
                 raise ValueError(f"File {url} is not stored externally, should use read_file instead")
             ret = fconn.get_file_blob_external(r.file_id)
-
-        await delayed_log_access(url)
         return ret
 
-
     async def read_file(self, url: str) -> bytes:
         validate_url(url)
 
@@ -618,11 +622,9 @@ class Database:
             blob = await fconn.get_file_blob(f_id)
             if blob is None:
                 raise FileNotFoundError(f"File {url} data not found")
-
-        await delayed_log_access(url)
         return blob
 
-    async def delete_file(self, url: str, assure_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
+    async def delete_file(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
         validate_url(url)
 
         async with transaction() as cur:
@@ -630,10 +632,10 @@ class Database:
             r = await fconn.delete_file_record(url)
             if r is None:
                 return None
-            if assure_user is not None:
-                if r.owner_id != assure_user.id:
+            if op_user is not None:
+                if r.owner_id != op_user.id and not op_user.is_admin:
                     # will rollback
-                    raise PermissionDeniedError(f"Permission denied: {assure_user.username} cannot delete file {url}")
+                    raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot delete file {url}")
             f_id = r.file_id
             if r.external:
                 await fconn.delete_file_blob_external(f_id)
@@ -641,7 +643,7 @@ class Database:
                 await fconn.delete_file_blob(f_id)
             return r
     
-    async def move_file(self, old_url: str, new_url: str, ensure_user: Optional[UserRecord] = None):
+    async def move_file(self, old_url: str, new_url: str, op_user: Optional[UserRecord] = None):
         validate_url(old_url)
         validate_url(new_url)
 
@@ -650,12 +652,16 @@ class Database:
             r = await fconn.get_file_record(old_url)
             if r is None:
                 raise FileNotFoundError(f"File {old_url} not found")
-            if ensure_user is not None:
-                if r.owner_id != ensure_user.id:
-                    raise PermissionDeniedError(f"Permission denied: {ensure_user.username} cannot move file {old_url}")
+            if op_user is not None:
+                if r.owner_id != op_user.id:
+                    raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot move file {old_url}")
             await fconn.move_file(old_url, new_url)
+
+            new_mime, _ = mimetypes.guess_type(new_url)
+            if not new_mime is None:
+                await fconn.update_file_record(new_url, mime_type=new_mime)
     
-    async def move_path(self, user: UserRecord, old_url: str, new_url: str):
+    async def move_path(self, old_url: str, new_url: str, op_user: UserRecord):
         validate_url(old_url, is_file=False)
         validate_url(new_url, is_file=False)
 
@@ -669,20 +675,20 @@ class Database:
 
         async with transaction() as cur:
             first_component = new_url.split('/')[0]
-            if not (first_component == user.username or user.is_admin):
-                raise PermissionDeniedError(f"Permission denied: path must start with {user.username}")
-            elif user.is_admin:
+            if not (first_component == op_user.username or op_user.is_admin):
+                raise PermissionDeniedError(f"Permission denied: path must start with {op_user.username}")
+            elif op_user.is_admin:
                 uconn = UserConn(cur)
                 _is_user = await uconn.get_user(first_component)
                 if not _is_user:
                     raise PermissionDeniedError(f"Invalid path: {first_component} is not a valid username")
             
             # check if old path is under user's directory (non-admin)
-            if not old_url.startswith(user.username + '/') and not user.is_admin:
-                raise PermissionDeniedError(f"Permission denied: {user.username} cannot move path {old_url}")
+            if not old_url.startswith(op_user.username + '/') and not op_user.is_admin:
+                raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot move path {old_url}")
 
             fconn = FileConn(cur)
-            await fconn.move_path(old_url, new_url, 'overwrite', user.id)
+            await fconn.move_path(old_url, new_url, 'overwrite', op_user.id)
 
     async def __batch_delete_file_blobs(self, fconn: FileConn, file_records: list[FileRecord], batch_size: int = 512):
         # https://github.com/langchain-ai/langchain/issues/10321
@@ -702,13 +708,13 @@ class Database:
                 await fconn.delete_file_blob_external(external_ids[i])
         await asyncio.gather(del_internal(), del_external())
 
-    async def delete_path(self, url: str, under_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
+    async def delete_path(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
         validate_url(url, is_file=False)
-        user_id = under_user.id if under_user is not None else None
+        from_owner_id = op_user.id if op_user is not None and not op_user.is_admin else None
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            records = await fconn.delete_path_records(url, user_id)
+            records = await fconn.delete_path_records(url, from_owner_id)
             if not records:
                 return None
             await self.__batch_delete_file_blobs(fconn, records)

{lfss-0.8.0 → lfss-0.8.2}/lfss/src/datatype.py

@@ -1,6 +1,5 @@
 from enum import IntEnum
-from typing import Literal
-import dataclasses
+import dataclasses, typing
 
 class FileReadPermission(IntEnum):
     UNSET = 0           # not set
@@ -55,7 +54,7 @@ class PathContents:
     dirs: list[DirectoryRecord] = dataclasses.field(default_factory=list)
     files: list[FileRecord] = dataclasses.field(default_factory=list)
     
-FileSortKey = Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
-isValidFileSortKey = lambda x: x in ['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
-DirSortKey = Literal['', 'dirname']
-isValidDirSortKey = lambda x: x in ['', 'dirname']
+FileSortKey = typing.Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
+isValidFileSortKey = lambda x: x in typing.get_args(FileSortKey)
+DirSortKey = typing.Literal['', 'dirname']
+isValidDirSortKey = lambda x: x in typing.get_args(DirSortKey)

{lfss-0.8.0 → lfss-0.8.2}/lfss/src/server.py

@@ -1,24 +1,23 @@
 from typing import Optional, Literal
 from functools import wraps
 
-from fastapi import FastAPI, APIRouter, Depends, Request, Response
+from fastapi import FastAPI, APIRouter, Depends, Request, Response, UploadFile
 from fastapi.responses import StreamingResponse
 from fastapi.exceptions import HTTPException 
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from fastapi.middleware.cors import CORSMiddleware
-import mimesniff
 
 import asyncio, json, time
-import mimetypes
 from contextlib import asynccontextmanager
 
 from .error import *
 from .log import get_logger
 from .stat import RequestDB
-from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES, LARGE_FILE_BYTES, CHUNK_SIZE
+from .config import MAX_BUNDLE_BYTES, CHUNK_SIZE
 from .utils import ensure_uri_compnents, format_last_modified, now_stamp, wait_for_debounce_tasks
 from .connection_pool import global_connection_init, global_connection_close, unique_cursor
-from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn, delayed_log_activity
+from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn
+from .database import delayed_log_activity, delayed_log_access
 from .datatype import (
     FileReadPermission, FileRecord, UserRecord, PathContents, 
     FileSortKey, DirSortKey
@@ -81,6 +80,10 @@ async def get_current_user(
 
     if not user:
         raise HTTPException(status_code=401, detail="Invalid token")
+    
+    if not user.id == 0:
+        await delayed_log_activity(user.username)
+
     return user
 
 async def registered_user(user: UserRecord = Depends(get_current_user)):
@@ -166,6 +169,8 @@ async def emit_file(
         media_type = file_record.mime_type
     path = file_record.url
     fname = path.split("/")[-1]
+
+    await delayed_log_access(path)
     if not file_record.external:
         fblob = await db.read_file(path)
         return Response(
@@ -246,18 +251,21 @@ async def put_file(
     path: str, 
     conflict: Literal["overwrite", "skip", "abort"] = "abort",
     permission: int = 0,
-    user: UserRecord = Depends(registered_user)):
+    user: UserRecord = Depends(registered_user)
+    ):
     path = ensure_uri_compnents(path)
-    if not path.startswith(f"{user.username}/") and not user.is_admin:
-        logger.debug(f"Reject put request from {user.username} to {path}")
-        raise HTTPException(status_code=403, detail="Permission denied")
-    
-    content_length = request.headers.get("Content-Length")
-    if content_length is not None:
-        content_length = int(content_length)
-        if content_length > MAX_FILE_BYTES:
-            logger.debug(f"Reject put request from {user.username} to {path}, file too large")
-            raise HTTPException(status_code=413, detail="File too large")
+    assert not path.endswith("/"), "Path must not end with /"
+    if not path.startswith(f"{user.username}/"):
+        if not user.is_admin:
+            logger.debug(f"Reject put request from {user.username} to {path}")
+            raise HTTPException(status_code=403, detail="Permission denied")
+        else:
+            first_comp = path.split("/")[0]
+            async with unique_cursor() as c:
+                uconn = UserConn(c)
+                owner = await uconn.get_user(first_comp)
+                if not owner:
+                    raise HTTPException(status_code=404, detail="Owner not found")
     
     logger.info(f"PUT {path}, user: {user.username}")
     exists_flag = False
@@ -280,47 +288,73 @@ async def put_file(
     # check content-type
     content_type = request.headers.get("Content-Type")
     logger.debug(f"Content-Type: {content_type}")
-    if content_type == "application/json":
-        body = await request.json()
-        blobs = json.dumps(body).encode('utf-8')
-    elif content_type == "application/x-www-form-urlencoded":
-        # may not work...
-        body = await request.form()
-        file = body.get("file")
-        if isinstance(file, str) or file is None:
-            raise HTTPException(status_code=400, detail="Invalid form data, file required")
-        blobs = await file.read()
-    elif content_type == "application/octet-stream":
-        blobs = await request.body()
-    else:
-        blobs = await request.body()
+    if not (content_type == "application/octet-stream" or content_type == "application/json"):
+        raise HTTPException(status_code=415, detail="Unsupported content type, put request must be application/json or application/octet-stream")
     
-    # check file type
-    assert not path.endswith("/"), "Path must be a file"
-    fname = path.split("/")[-1]
-    mime_t, _ = mimetypes.guess_type(fname)
-    if mime_t is None:
-        mime_t = mimesniff.what(blobs)
-    if mime_t is None:
-        mime_t = "application/octet-stream"
-
-    if len(blobs) > LARGE_FILE_BYTES:
-        async def blob_reader():
-            for b in range(0, len(blobs), CHUNK_SIZE):
-                yield blobs[b:b+CHUNK_SIZE]
-        await db.save_file(user.id, path, blob_reader(), permission = FileReadPermission(permission), mime_type = mime_t)
-    else:
-        await db.save_file(user.id, path, blobs, permission = FileReadPermission(permission), mime_type=mime_t)
+    async def blob_reader():
+        nonlocal request
+        async for chunk in request.stream():
+            yield chunk
+
+    await db.save_file(user.id, path, blob_reader(), permission = FileReadPermission(permission))
 
     # https://developer.mozilla.org/zh-CN/docs/Web/HTTP/Methods/PUT
-    if exists_flag:
-        return Response(status_code=201, headers={
-            "Content-Type": "application/json",
-        }, content=json.dumps({"url": path}))
-    else:
-        return Response(status_code=200, headers={
-            "Content-Type": "application/json",
-        }, content=json.dumps({"url": path}))
+    return Response(status_code=200 if exists_flag else 201, headers={
+        "Content-Type": "application/json",
+    }, content=json.dumps({"url": path}))
+
+# using form-data instead of raw body
+@router_fs.post("/{path:path}")
+@handle_exception
+async def post_file(
+    path: str, 
+    file: UploadFile,
+    conflict: Literal["overwrite", "skip", "abort"] = "abort",
+    permission: int = 0,
+    user: UserRecord = Depends(registered_user)
+    ):
+    path = ensure_uri_compnents(path)
+    assert not path.endswith("/"), "Path must not end with /"
+    if not path.startswith(f"{user.username}/"):
+        if not user.is_admin:
+            logger.debug(f"Reject put request from {user.username} to {path}")
+            raise HTTPException(status_code=403, detail="Permission denied")
+        else:
+            first_comp = path.split("/")[0]
+            async with unique_cursor() as conn:
+                uconn = UserConn(conn)
+                owner = await uconn.get_user(first_comp)
+                if not owner:
+                    raise HTTPException(status_code=404, detail="Owner not found")
+
+    logger.info(f"POST {path}, user: {user.username}")
+    exists_flag = False
+    async with unique_cursor() as conn:
+        fconn = FileConn(conn)
+        file_record = await fconn.get_file_record(path)
+
+    if file_record:
+        if conflict == "abort":
+            raise HTTPException(status_code=409, detail="File exists")
+        if conflict == "skip":
+            return Response(status_code=200, headers={
+                "Content-Type": "application/json",
+            }, content=json.dumps({"url": path}))
+        exists_flag = True
+        if not user.is_admin and not file_record.owner_id == user.id:
+            raise HTTPException(status_code=403, detail="Permission denied, cannot overwrite other's file")
+        await db.delete_file(path)
+    
+    async def blob_reader():
+        nonlocal file
+        while (chunk := await file.read(CHUNK_SIZE)):
+            yield chunk
+
+    await db.save_file(user.id, path, blob_reader(), permission = FileReadPermission(permission))
+    return Response(status_code=200 if exists_flag else 201, headers={
+        "Content-Type": "application/json",
+    }, content=json.dumps({"url": path}))
+    
 
 @router_fs.delete("/{path:path}")
 @handle_exception
@@ -332,11 +366,10 @@ async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
     logger.info(f"DELETE {path}, user: {user.username}")
 
     if path.endswith("/"):
-        res = await db.delete_path(path, user if not user.is_admin else None)
+        res = await db.delete_path(path, user)
     else:
-        res = await db.delete_file(path, user if not user.is_admin else None)
+        res = await db.delete_file(path, user)
 
-    await delayed_log_activity(user.username)
     if res:
         return Response(status_code=200, content="Deleted")
     else:
@@ -428,16 +461,15 @@ async def update_file_meta(
     path = ensure_uri_compnents(path)
     if path.startswith("/"):
         path = path[1:]
-    await delayed_log_activity(user.username)
 
     # file
     if not path.endswith("/"):
         if perm is not None:
             logger.info(f"Update permission of {path} to {perm}")
             await db.update_file_record(
-                user = user,
                 url = path, 
-                permission = FileReadPermission(perm)
+                permission = FileReadPermission(perm), 
+                op_user = user,
             )
     
         if new_path is not None:
@@ -452,7 +484,7 @@ async def update_file_meta(
             new_path = ensure_uri_compnents(new_path)
             logger.info(f"Update path of {path} to {new_path}")
             # currently only move own file, with overwrite
-            await db.move_path(user, path, new_path)
+            await db.move_path(path, new_path, user)
 
     return Response(status_code=200, content="OK")
 

{lfss-0.8.0 → lfss-0.8.2}/lfss/src/utils.py

@@ -109,6 +109,17 @@ def parse_storage_size(s: str) -> int:
         case 'g': return int(s[:-1]) * 1024**3
         case 't': return int(s[:-1]) * 1024**4
         case _: raise ValueError(f"Invalid file size string: {s}")
+def fmt_storage_size(size: int) -> str:
+    """ Format the file size to human-readable format """
+    if size < 1024:
+        return f"{size}B"
+    if size < 1024**2:
+        return f"{size/1024:.2f}K"
+    if size < 1024**3:
+        return f"{size/1024**2:.2f}M"
+    if size < 1024**4:
+        return f"{size/1024**3:.2f}G"
+    return f"{size/1024**4:.2f}T"
 
 _FnReturnT = TypeVar('_FnReturnT')
 _AsyncReturnT = Awaitable[_FnReturnT]
@@ -136,4 +147,12 @@ def concurrent_wrap(executor=None):
             loop = asyncio.new_event_loop()
             return loop.run_until_complete(func(*args, **kwargs))
         return sync_fn
-    return _concurrent_wrap
+    return _concurrent_wrap
+
+# https://stackoverflow.com/a/279586/6775765
+def static_vars(**kwargs):
+    def decorate(func):
+        for k in kwargs:
+            setattr(func, k, kwargs[k])
+        return func
+    return decorate

{lfss-0.8.0 → lfss-0.8.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.8.0"
+version = "0.8.2"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"
@@ -10,12 +10,13 @@ include = ["Readme.md", "docs/*", "frontend/*", "lfss/sql/*"]
 
 [tool.poetry.dependencies]
 python = ">=3.9"
+requests = "2.*"
 aiosqlite = "0.*"
 aiofiles = "23.*"
 mimesniff = "1.*"
 fastapi = "0.*"
 uvicorn = "0.*"
-requests = "2.*"
+python-multipart = "*"
 pillow = "*"
 
 [tool.poetry.dev-dependencies]