lfss 0.11.2__tar.gz → 0.11.3__tar.gz

{lfss-0.11.2 → lfss-0.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.11.2
+Version: 0.11.3
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: Li, Mengxun
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: aiofiles (==23.*)
+Requires-Dist: aiofiles (==24.*)
 Requires-Dist: aiosqlite (==0.*)
 Requires-Dist: fastapi (==0.*)
 Requires-Dist: mimesniff (==1.*)

{lfss-0.11.2 → lfss-0.11.3}/docs/Enviroment_variables.md

@@ -4,8 +4,10 @@
 **Server**
 - `LFSS_DATA`: The directory to store the data. Default is `.storage_data`.
 - `LFSS_WEBDAV`: Enable WebDAV support. Default is `0`, set to `1` to enable.
-- `LFSS_LARGE_FILE`: The size limit of the file to store in the database. Default is `8m`.
+- `LFSS_LARGE_FILE`: The size limit of the file to store in the database. Default is `1m`.
 - `LFSS_DEBUG`: Enable debug mode for more verbose logging. Default is `0`, set to `1` to enable.
+- `LFSS_DISABLE_LOGGING`: Disable all file logging. Default is 0; set to `1` to disable file logging. 
+- `LFSS_ORIGIN`: The `Origin` header to allow CORS requests. Use `,` to separate multiple origins. Default is `*`.
 
 **Client**
 - `LFSS_ENDPOINT`: The fallback server endpoint. Default is `http://localhost:8000`.

{lfss-0.11.2 → lfss-0.11.3}/frontend/api.js

@@ -69,6 +69,10 @@ export default class Connector {
     /**
     * @param {string} path - the path to the file (url)
     * @param {File} file - the file to upload
+    * @param {Object} [options] - Optional upload configuration.
+    * @param {'abort' | 'overwrite' | 'skip'} [options.conflict='abort'] - Conflict resolution strategy:  
+    * `'abort'` to cancel and raise 409, `'overwrite'` to replace.
+    * @param {number} [options.permission=0] - Optional permission setting for the file (refer to backend impl).
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
     async put(path, file, {
@@ -96,8 +100,12 @@ export default class Connector {
     }
 
     /**
-    * @param {string} path - the path to the file (url)
+    * @param {string} path - the path to the file (url), should end with .json
     * @param {File} file - the file to upload
+    * @param {Object} [options] - Optional upload configuration.
+    * @param {'abort' | 'overwrite' | 'skip'} [options.conflict='abort'] - Conflict resolution strategy:  
+    * `'abort'` to cancel and raise 409, `'overwrite'` to replace, `'skip'` to ignore if already exists.
+    * @param {number} [options.permission=0] - Optional permission setting for the file (refer to backend impl).
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
     async post(path, file, {
@@ -129,13 +137,23 @@ export default class Connector {
 
     /**
     * @param {string} path - the path to the file (url), should end with .json
-    * @param {Objec} data - the data to upload
+    * @param {Object} data - the data to upload
+    * @param {Object} [options] - Optional upload configuration.
+    * @param {'abort' | 'overwrite' | 'skip'} [options.conflict='abort'] - Conflict resolution strategy:  
+    * `'abort'` to cancel and raise 409, `'overwrite'` to replace, `'skip'` to ignore if already exists.
+    * @param {number} [options.permission=0] - Optional permission setting for the file (refer to backend impl).
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
-    async putJson(path, data){
+    async putJson(path, data, {
+        conflict = "overwrite", 
+        permission = 0
+    } = {}){
         if (!path.endsWith('.json')){ throw new Error('Upload object must end with .json'); }
         if (path.startsWith('/')){ path = path.slice(1); }
-        const res = await fetch(this.config.endpoint + '/' + path, {
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('conflict', conflict);
+        dst.searchParams.append('permission', permission);
+        const res = await fetch(dst.toString(), {
             method: 'PUT',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token,
@@ -149,6 +167,50 @@ export default class Connector {
         return (await res.json()).url;
     }
 
+    /**
+     * @param {string} path - the path to the file (url), should have content type application/json
+     * @returns {Promise<Object>} - return the json object
+    */
+    async getJson(path){
+        if (path.startsWith('/')){ path = path.slice(1); }
+        const res = await fetch(this.config.endpoint + '/' + path, {
+            method: 'GET',
+            headers: {
+             "Authorization": 'Bearer ' + this.config.token
+            },
+        });
+        if (res.status != 200){
+            throw new Error(`Failed to get object, status code: ${res.status}, message: ${await fmtFailedResponse(res)}`);
+        }
+        return await res.json();
+    }
+
+    /**
+     * @param {string[]} paths - the paths to the files (url), should have content type plain/text, application/json, etc.
+     * @param {Object} [options] - Optional configuration.
+     * @param {boolean} [options.skipContent=false] - If true, skips fetching content and returns a record of <path, ''>.
+     * @returns {Promise<Record<string, string | null>>} - return the mapping of path to text content, non-existing paths will be ignored
+     */
+    async getMultipleText(paths, {
+        skipContent = false
+    } = {}){
+        const url = new URL(this.config.endpoint + '/_api/get-multiple');
+        url.searchParams.append('skip_content', skipContent);
+        for (const path of paths){
+            url.searchParams.append('path', path);
+        }
+        const res = await fetch(url.toString(), {
+            method: 'GET',
+            headers: {
+                "Authorization": 'Bearer ' + this.config.token,
+            }
+        });
+        if (res.status != 200 && res.status != 206){
+            throw new Error(`Failed to get multiple files, status code: ${res.status}, message: ${await fmtFailedResponse(res)}`);
+        }
+        return await res.json();
+    }
+
     async delete(path){
         if (path.startsWith('/')){ path = path.slice(1); }
         const res = await fetch(this.config.endpoint + '/' + path, {

{lfss-0.11.2 → lfss-0.11.3}/lfss/api/connector.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from typing import Optional, Literal
 from collections.abc import Iterator
-import os
+import os, json
 import requests
 import requests.adapters
 import urllib.parse
@@ -76,7 +76,11 @@ class Connector:
             path = path[1:]
         path = ensure_uri_compnents(path)
         def f(**kwargs):
-            url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params)
+            search_params_t = [
+                (k, str(v).lower() if isinstance(v, bool) else v)
+                for k, v in search_params.items()
+            ]   # tuple form
+            url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params_t, doseq=True)
             headers: dict = kwargs.pop('headers', {})
             headers.update({
                 'Authorization': f"Bearer {self.config['token']}",
@@ -207,6 +211,17 @@ class Connector:
         assert response.headers['Content-Type'] == 'application/json'
         return response.json()
     
+    def get_multiple_text(self, *paths: str, skip_content = False) -> dict[str, Optional[str]]:
+        """ 
+        Gets text contents of multiple files at once. Non-existing files will return None. 
+        - skip_content: if True, the file contents will not be fetched, always be empty string ''.
+        """
+        response = self._fetch_factory(
+            'GET', '_api/get-multiple', 
+            {'path': paths, "skip_content": skip_content}
+            )()
+        return response.json()
+    
     def delete(self, path: str):
         """Deletes the file at the specified path."""
         self._fetch_factory('DELETE', path)()

{lfss-0.11.2 → lfss-0.11.3}/lfss/cli/cli.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-import argparse, typing
+import argparse, typing, sys
 from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
 from lfss.eng.datatype import FileReadPermission, FileSortKey, DirSortKey
 from lfss.eng.utils import decode_uri_compnents
@@ -78,9 +78,9 @@ def main():
                 permission=args.permission
             )
             if failed_upload:
-                print("\033[91mFailed to upload:\033[0m")
+                print("\033[91mFailed to upload:\033[0m", file=sys.stderr)
                 for path in failed_upload:
-                    print(f"  {path}")
+                    print(f"  {path}", file=sys.stderr)
         else:
             success, msg = upload_file(
                 connector, 
@@ -93,7 +93,7 @@ def main():
                 permission=args.permission
             )
             if not success:
-                print("\033[91mFailed to upload: \033[0m", msg)
+                print("\033[91mFailed to upload: \033[0m", msg, file=sys.stderr)
     
     elif args.command == "download":
         is_dir = args.src.endswith("/")
@@ -107,9 +107,9 @@ def main():
                 overwrite=args.overwrite
             )
             if failed_download:
-                print("\033[91mFailed to download:\033[0m")
+                print("\033[91mFailed to download:\033[0m", file=sys.stderr)
                 for path in failed_download:
-                    print(f"  {path}")
+                    print(f"  {path}", file=sys.stderr)
         else:
             success, msg = download_file(
                 connector, 
@@ -121,7 +121,7 @@ def main():
                 overwrite=args.overwrite
             )
             if not success:
-                print("\033[91mFailed to download: \033[0m", msg)
+                print("\033[91mFailed to download: \033[0m", msg, file=sys.stderr)
     
     elif args.command == "query":
         for path in args.path:

{lfss-0.11.2 → lfss-0.11.3}/lfss/eng/config.py

@@ -13,6 +13,8 @@ LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
 LOG_DIR = DATA_HOME / 'logs'
 
+DISABLE_LOGGING = os.environ.get('DISABLE_LOGGING', '0') == '1'
+
 # https://sqlite.org/fasterthanfs.html
 __env_large_file = os.environ.get('LFSS_LARGE_FILE', None)
 if __env_large_file is not None:
@@ -24,4 +26,4 @@ CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'
 
 THUMB_DB = DATA_HOME / 'thumbs.v0-11.db'
-THUMB_SIZE = (48, 48)
+THUMB_SIZE = (64, 64)

{lfss-0.11.2 → lfss-0.11.3}/lfss/eng/database.py

@@ -210,6 +210,10 @@ class FileConn(DBObjectBase):
         return self.parse_record(res)
     
     async def get_file_records(self, urls: list[str]) -> list[FileRecord]:
+        """
+        Get all file records with the given urls, only urls in the database will be returned. 
+        If the urls are not in the database, they will be ignored.
+        """
         await self.cur.execute("SELECT * FROM fmeta WHERE url IN ({})".format(','.join(['?'] * len(urls))), urls)
         res = await self.cur.fetchall()
         if res is None:
@@ -412,6 +416,10 @@ class FileConn(DBObjectBase):
         self.logger.info(f"File {url} created")
 
     async def copy_file(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+        """
+        Copy file from old_url to new_url, 
+        if user_id is None, will not change the owner_id of the file. Otherwise, will change the owner_id to user_id.
+        """
         old = await self.get_file_record(old_url)
         if old is None:
             raise FileNotFoundError(f"File {old_url} not found")
@@ -428,14 +436,14 @@ class FileConn(DBObjectBase):
         self.logger.info(f"Copied file {old_url} to {new_url}")
     
     async def copy_dir(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+        """
+        Copy all files under old_url to new_url, 
+        if user_id is None, will not change the owner_id of the files. Otherwise, will change the owner_id to user_id.
+        """
         assert old_url.endswith('/'), "Old path must end with /"
         assert new_url.endswith('/'), "New path must end with /"
-        if user_id is None:
-            cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (old_url + '%', ))
-            res = await cursor.fetchall()
-        else:
-            cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ? AND owner_id = ?", (old_url + '%', user_id))
-            res = await cursor.fetchall()
+        cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (old_url + '%', ))
+        res = await cursor.fetchall()
         for r in res:
             old_record = FileRecord(*r)
             new_r = new_url + old_record.url[len(old_url):]
@@ -826,6 +834,58 @@ class Database:
                     yield blob
         ret = blob_stream()
         return ret
+    
+    async def read_files_bulk(
+        self, urls: list[str], 
+        skip_content = False, 
+        op_user: Optional[UserRecord] = None, 
+        ) -> dict[str, Optional[bytes]]:
+        """
+        A frequent use case is to read multiple files at once, 
+        this method will read all files in the list and return a dict of url -> blob.
+        if the file is not found, the value will be None.
+        - skip_content: if True, will not read the content of the file, resulting in a dict of url -> b''
+
+        may raise StorageExceededError if the total size of the files exceeds MAX_MEM_FILE_BYTES
+        """
+        for url in urls:
+            validate_url(url)
+        
+        async with unique_cursor() as cur:
+            fconn = FileConn(cur)
+            file_records = await fconn.get_file_records(urls)
+
+            if op_user is not None:
+                for r in file_records:
+                    if await check_path_permission(r.url, op_user, cursor=cur) >= AccessLevel.READ:
+                        continue
+                    is_allowed, reason = await check_file_read_permission(op_user, r, cursor=cur)
+                    if not is_allowed:
+                        raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot read file {r.url}: {reason}")
+
+        # first check if the files are too big
+        sum_size = sum([r.file_size for r in file_records])
+        if not skip_content and sum_size > MAX_MEM_FILE_BYTES:
+            raise StorageExceededError(f"Unable to read files at once, total size {sum_size} exceeds {MAX_MEM_FILE_BYTES}")
+        
+        self.logger.debug(f"Reading {len(file_records)} files{' (skip content)' if skip_content else ''}, getting {sum_size} bytes, from {urls}")
+        # read the file content
+        async with unique_cursor() as cur:
+            fconn = FileConn(cur)
+            blobs: dict[str, bytes] = {}
+            for r in file_records:
+                if skip_content:
+                    blobs[r.url] = b''
+                    continue
+
+                if r.external:
+                    blob_iter = fconn.get_file_blob_external(r.file_id)
+                    blob = b''.join([chunk async for chunk in blob_iter])
+                else:
+                    blob = await fconn.get_file_blob(r.file_id)
+                blobs[r.url] = blob
+            
+            return {url: blobs.get(url, None) for url in urls}
 
     async def delete_file(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
         validate_url(url)

{lfss-0.11.2 → lfss-0.11.3}/lfss/eng/log.py

@@ -1,9 +1,9 @@
-from .config import LOG_DIR
+from .config import LOG_DIR, DISABLE_LOGGING
 import time, sqlite3, dataclasses
 from typing import TypeVar, Callable, Literal, Optional
 from concurrent.futures import ThreadPoolExecutor
 from functools import wraps
-import logging, pathlib, asyncio
+import logging, asyncio
 from logging import handlers
 
 class BCOLORS:
@@ -154,24 +154,25 @@ def get_logger(
             if isinstance(color, str) and color.startswith('\033'):
                 format_str_plain = format_str_plain.replace(color, '')
 
-        formatter_plain = logging.Formatter(format_str_plain)
-        log_home.mkdir(exist_ok=True)
-        log_file = log_home / f'{name}.log'
-        if file_handler_type == 'simple':
-            file_handler = logging.FileHandler(log_file)
-        elif file_handler_type == 'daily':
-            file_handler = handlers.TimedRotatingFileHandler(
-                log_file, when='midnight', interval=1, backupCount=30
-            )
-        elif file_handler_type == 'rotate':
-            file_handler = handlers.RotatingFileHandler(
-                log_file, maxBytes=1024*1024, backupCount=5
-            )
-        elif file_handler_type == 'sqlite':
-            file_handler = SQLiteFileHandler(log_file if log_file.suffix == '.db' else log_file.with_suffix('.log.db'))
-
-        file_handler.setFormatter(formatter_plain)
-        logger.addHandler(file_handler)
+        if not DISABLE_LOGGING:
+            formatter_plain = logging.Formatter(format_str_plain)
+            log_home.mkdir(exist_ok=True)
+            log_file = log_home / f'{name}.log'
+            if file_handler_type == 'simple':
+                file_handler = logging.FileHandler(log_file)
+            elif file_handler_type == 'daily':
+                file_handler = handlers.TimedRotatingFileHandler(
+                    log_file, when='midnight', interval=1, backupCount=30
+                )
+            elif file_handler_type == 'rotate':
+                file_handler = handlers.RotatingFileHandler(
+                    log_file, maxBytes=1024*1024, backupCount=5
+                )
+            elif file_handler_type == 'sqlite':
+                file_handler = SQLiteFileHandler(log_file if log_file.suffix == '.db' else log_file.with_suffix('.log.db'))
+
+            file_handler.setFormatter(formatter_plain)
+            logger.addHandler(file_handler)
     
     logger = BaseLogger(name)
     setupLogger(logger)

{lfss-0.11.2 → lfss-0.11.3}/lfss/svc/app_base.py

@@ -60,10 +60,13 @@ def handle_exception(fn):
             raise 
     return wrapper
 
+env_origins = os.environ.get("LFSS_ORIGINS", "*")
+logger.debug(f"LFSS_ORIGINS: {env_origins}")
+origins = [x.strip() for x in env_origins.split(",") if x.strip()]
 app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
+    allow_origins=origins,
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],

{lfss-0.11.2 → lfss-0.11.3}/lfss/svc/app_native.py

@@ -1,17 +1,19 @@
-from typing import Optional, Literal
+from typing import Optional, Literal, Annotated
+from collections import OrderedDict
 
-from fastapi import Depends, Request, Response, UploadFile
-from fastapi.responses import StreamingResponse
+from fastapi import Depends, Request, Response, UploadFile, Query
+from fastapi.responses import StreamingResponse, JSONResponse
 from fastapi.exceptions import HTTPException 
 
 from ..eng.utils import ensure_uri_compnents
 from ..eng.config import MAX_MEM_FILE_BYTES
 from ..eng.connection_pool import unique_cursor
-from ..eng.database import check_file_read_permission, check_path_permission, UserConn, FileConn
+from ..eng.database import check_file_read_permission, check_path_permission, FileConn, delayed_log_access
 from ..eng.datatype import (
     FileReadPermission, UserRecord, AccessLevel, 
     FileSortKey, DirSortKey
 )
+from ..eng.error import InvalidPathError
 
 from .app_base import *
 from .common_impl import get_impl, put_file_impl, post_file_impl, delete_impl, copy_impl
@@ -189,6 +191,7 @@ async def validate_path_read_permission(path: str, user: UserRecord):
     if not await check_path_permission(path, user) >= AccessLevel.READ:
         raise HTTPException(status_code=403, detail="Permission denied")
 @router_api.get("/count-files")
+@handle_exception
 async def count_files(path: str, flat: bool = False, user: UserRecord = Depends(registered_user)):
     await validate_path_read_permission(path, user)
     path = ensure_uri_compnents(path)
@@ -196,6 +199,7 @@ async def count_files(path: str, flat: bool = False, user: UserRecord = Depends(
         fconn = FileConn(conn)
         return { "count": await fconn.count_dir_files(url = path, flat = flat) }
 @router_api.get("/list-files")
+@handle_exception
 async def list_files(
     path: str, offset: int = 0, limit: int = 1000,
     order_by: FileSortKey = "", order_desc: bool = False,
@@ -212,6 +216,7 @@ async def list_files(
         )
 
 @router_api.get("/count-dirs")
+@handle_exception
 async def count_dirs(path: str, user: UserRecord = Depends(registered_user)):
     await validate_path_read_permission(path, user)
     path = ensure_uri_compnents(path)
@@ -219,6 +224,7 @@ async def count_dirs(path: str, user: UserRecord = Depends(registered_user)):
         fconn = FileConn(conn)
         return { "count": await fconn.count_path_dirs(url = path) }
 @router_api.get("/list-dirs")
+@handle_exception
 async def list_dirs(
     path: str, offset: int = 0, limit: int = 1000,
     order_by: DirSortKey = "", order_desc: bool = False,
@@ -232,6 +238,47 @@ async def list_dirs(
             url = path, offset = offset, limit = limit,
             order_by=order_by, order_desc=order_desc, skim=skim
         )
+
+# https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#query-parameter-list-multiple-values
+@router_api.get("/get-multiple")
+@handle_exception
+async def get_multiple_files(
+    path: Annotated[list[str], Query()], 
+    skip_content: bool = False,
+    user: UserRecord = Depends(registered_user)
+    ):
+    """
+    Get multiple files by path. 
+    Please note that the content is supposed to be text and are small enough to fit in memory.
+
+    Not existing files will have content null, and the response will be 206 Partial Content if not all files are found.
+    if skip_content is True, the content of the files will always be ''
+    """
+    for p in path:
+        if p.endswith("/"):
+            raise InvalidPathError(f"Path '{p}' must not end with /")
+
+    # here we unify the path, so need to keep a record of the inputs
+    # make output keys consistent with inputs
+    upath2path = OrderedDict[str, str]()
+    for p in path:
+        p_ = p if not p.startswith("/") else p[1:]
+        upath2path[ensure_uri_compnents(p_)] = p
+    upaths = list(upath2path.keys())
+
+    # get files
+    raw_res = await db.read_files_bulk(upaths, skip_content=skip_content, op_user=user)
+    for k in raw_res.keys():
+        await delayed_log_access(k)
+    partial_content = len(raw_res) != len(upaths)
+
+    return JSONResponse(
+        content = {
+            upath2path[k]: v.decode('utf-8') if v is not None else None for k, v in raw_res.items()
+        },
+        status_code = 206 if partial_content else 200
+    )
+    
     
 @router_api.get("/whoami")
 @handle_exception

{lfss-0.11.2 → lfss-0.11.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.11.2"
+version = "0.11.3"
 description = "Lightweight file storage service"
 authors = ["Li, Mengxun <mengxunli@whu.edu.cn>"]
 readme = "Readme.md"
@@ -12,7 +12,7 @@ include = ["Readme.md", "docs/*", "frontend/*", "lfss/sql/*"]
 python = ">=3.10"   # PEP-622
 requests = "2.*"
 aiosqlite = "0.*"
-aiofiles = "23.*"
+aiofiles = "24.*"
 mimesniff = "1.*"
 fastapi = "0.*"
 uvicorn = "0.*"