lfss 0.7.15__py3-none-any.whl → 0.8.0__py3-none-any.whl

lfss/src/server.py

@@ -16,9 +16,13 @@ from .error import *
 from .log import get_logger
 from .stat import RequestDB
 from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES, LARGE_FILE_BYTES, CHUNK_SIZE
-from .utils import ensure_uri_compnents, format_last_modified, now_stamp
+from .utils import ensure_uri_compnents, format_last_modified, now_stamp, wait_for_debounce_tasks
 from .connection_pool import global_connection_init, global_connection_close, unique_cursor
-from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission, UserConn, FileConn, PathContents
+from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn, delayed_log_activity
+from .datatype import (
+    FileReadPermission, FileRecord, UserRecord, PathContents, 
+    FileSortKey, DirSortKey
+)
 from .thumb import get_thumb
 
 logger = get_logger("server", term_level="DEBUG")
@@ -35,6 +39,7 @@ async def lifespan(app: FastAPI):
         yield
         await req_conn.commit()
     finally:
+        await wait_for_debounce_tasks()
         await asyncio.gather(req_conn.close(), global_connection_close())
 
 def handle_exception(fn):
@@ -49,6 +54,7 @@ def handle_exception(fn):
             if isinstance(e, InvalidPathError): raise HTTPException(status_code=400, detail=str(e))
             if isinstance(e, FileNotFoundError): raise HTTPException(status_code=404, detail=str(e))
             if isinstance(e, FileExistsError): raise HTTPException(status_code=409, detail=str(e))
+            if isinstance(e, TooManyItemsError): raise HTTPException(status_code=400, detail=str(e))
             logger.error(f"Uncaptured error in {fn.__name__}: {e}")
             raise 
     return wrapper
@@ -182,7 +188,7 @@ async def emit_file(
 @handle_exception
 async def get_file(
     path: str, 
-    download: bool = False, flat: bool = False, thumb: bool = False,
+    download: bool = False, thumb: bool = False,
     user: UserRecord = Depends(get_current_user)
     ):
     path = ensure_uri_compnents(path)
@@ -199,8 +205,6 @@ async def get_file(
                 return await emit_thumbnail(path, download, create_time=None)
             
             if path == "/":
-                if flat:
-                    raise HTTPException(status_code=400, detail="Flat query not supported for root path")
                 return PathContents(
                     dirs = await fconn.list_root_dirs(user.username, skim=True) \
                         if not user.is_admin else await fconn.list_root_dirs(skim=True),
@@ -210,7 +214,7 @@ async def get_file(
             if not path.startswith(f"{user.username}/") and not user.is_admin:
                 raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
 
-            return await fconn.list_path(path, flat = flat)
+            return await fconn.list_path(path)
     
     # handle file query
     async with unique_cursor() as conn:
@@ -332,7 +336,7 @@ async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
     else:
         res = await db.delete_file(path, user if not user.is_admin else None)
 
-    await db.record_user_activity(user.username)
+    await delayed_log_activity(user.username)
     if res:
         return Response(status_code=200, content="Deleted")
     else:
@@ -366,7 +370,10 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
     
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        files = (await fconn.list_path(path, flat = True)).files
+        files = await fconn.list_path_files(
+            url = path, flat = True, 
+            limit=(await fconn.count_path_files(url = path, flat = True))
+            )
     files = [f for f in files if await is_access_granted(f)]
     if len(files) == 0:
         raise HTTPException(status_code=404, detail="No files found")
@@ -421,7 +428,7 @@ async def update_file_meta(
     path = ensure_uri_compnents(path)
     if path.startswith("/"):
         path = path[1:]
-    await db.record_user_activity(user.username)
+    await delayed_log_activity(user.username)
 
     # file
     if not path.endswith("/"):
@@ -448,6 +455,57 @@ async def update_file_meta(
             await db.move_path(user, path, new_path)
 
     return Response(status_code=200, content="OK")
+
+async def validate_path_permission(path: str, user: UserRecord):
+    if not path.endswith("/"):
+        raise HTTPException(status_code=400, detail="Path must end with /")
+    if not path.startswith(f"{user.username}/") and not user.is_admin:
+        raise HTTPException(status_code=403, detail="Permission denied")
+
+@router_api.get("/count-files")
+async def count_files(path: str, flat: bool = False, user: UserRecord = Depends(registered_user)):
+    await validate_path_permission(path, user)
+    path = ensure_uri_compnents(path)
+    async with unique_cursor() as conn:
+        fconn = FileConn(conn)
+        return { "count": await fconn.count_path_files(url = path, flat = flat) }
+@router_api.get("/list-files")
+async def list_files(
+    path: str, offset: int = 0, limit: int = 1000,
+    order_by: FileSortKey = "", order_desc: bool = False,
+    flat: bool = False, user: UserRecord = Depends(registered_user)
+    ):
+    await validate_path_permission(path, user)
+    path = ensure_uri_compnents(path)
+    async with unique_cursor() as conn:
+        fconn = FileConn(conn)
+        return await fconn.list_path_files(
+            url = path, offset = offset, limit = limit,
+            order_by=order_by, order_desc=order_desc, 
+            flat=flat
+        )
+
+@router_api.get("/count-dirs")
+async def count_dirs(path: str, user: UserRecord = Depends(registered_user)):
+    await validate_path_permission(path, user)
+    path = ensure_uri_compnents(path)
+    async with unique_cursor() as conn:
+        fconn = FileConn(conn)
+        return { "count": await fconn.count_path_dirs(url = path) }
+@router_api.get("/list-dirs")
+async def list_dirs(
+    path: str, offset: int = 0, limit: int = 1000,
+    order_by: DirSortKey = "", order_desc: bool = False,
+    skim: bool = True, user: UserRecord = Depends(registered_user)
+    ):
+    await validate_path_permission(path, user)
+    path = ensure_uri_compnents(path)
+    async with unique_cursor() as conn:
+        fconn = FileConn(conn)
+        return await fconn.list_path_dirs(
+            url = path, offset = offset, limit = limit,
+            order_by=order_by, order_desc=order_desc, skim=skim
+        )
     
 @router_api.get("/whoami")
 @handle_exception

lfss/src/stat.py

@@ -42,7 +42,7 @@ class RequestDB:
     async def commit(self):
         await self.conn.commit()
     
-    @debounce_async(0.05)
+    @debounce_async()
     async def ensure_commit_once(self):
         await self.commit()
     

lfss/src/utils.py

@@ -1,11 +1,14 @@
-import datetime
+import datetime, time
 import urllib.parse
 import asyncio
 import functools
 import hashlib
+from asyncio import Lock
+from collections import OrderedDict
 from concurrent.futures import ThreadPoolExecutor
 from typing import TypeVar, Callable, Awaitable
 from functools import wraps, partial
+from uuid import uuid4
 import os
 
 def hash_credential(username: str, password: str):
@@ -25,25 +28,56 @@ def ensure_uri_compnents(path: str):
     """ Ensure the path components are safe to use """
     return encode_uri_compnents(decode_uri_compnents(path))
 
-def debounce_async(delay: float = 0):
-    """ Debounce the async procedure """
+g_debounce_tasks: OrderedDict[str, asyncio.Task] = OrderedDict()
+lock_debounce_task_queue = Lock()
+async def wait_for_debounce_tasks():
+    async def stop_task(task: asyncio.Task):
+        task.cancel()
+        try:
+            await task
+        except asyncio.CancelledError:
+            pass
+    await asyncio.gather(*map(stop_task, g_debounce_tasks.values()))
+    g_debounce_tasks.clear()
+
+def debounce_async(delay: float = 0.1, max_wait: float = 1.):
+    """ 
+    Debounce the async procedure, 
+    ensuring execution at least once every `max_wait` seconds. 
+    """
     def debounce_wrap(func):
-        # https://docs.python.org/3/library/asyncio-task.html#asyncio.Task.cancel
+        task_record: tuple[str, asyncio.Task] | None = None
+        last_execution_time = 0
+
         async def delayed_func(*args, **kwargs):
+            nonlocal last_execution_time
             await asyncio.sleep(delay)
             await func(*args, **kwargs)
+            last_execution_time = time.monotonic()
 
-        task_record: asyncio.Task | None = None
         @functools.wraps(func)
         async def wrapper(*args, **kwargs):
-            nonlocal task_record
-            if task_record is not None:
-                task_record.cancel()
-            task_record = asyncio.create_task(delayed_func(*args, **kwargs))
-            try:
-                await task_record
-            except asyncio.CancelledError:
-                pass
+            nonlocal task_record, last_execution_time
+
+            async with lock_debounce_task_queue:
+                if task_record is not None:
+                    task_record[1].cancel()
+                    g_debounce_tasks.pop(task_record[0], None)
+            
+            if time.monotonic() - last_execution_time > max_wait:
+                await func(*args, **kwargs)
+                last_execution_time = time.monotonic()
+                return
+
+            task = asyncio.create_task(delayed_func(*args, **kwargs))
+            task_uid = uuid4().hex
+            task_record = (task_uid, task)
+            async with lock_debounce_task_queue:
+                g_debounce_tasks[task_uid] = task
+                if len(g_debounce_tasks) > 2048:
+                    # finished tasks are not removed from the dict
+                    # so we need to clear it periodically
+                    await wait_for_debounce_tasks()
         return wrapper
     return debounce_wrap
 

{lfss-0.7.15.dist-info → lfss-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.15
+Version: 0.8.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
@@ -16,6 +16,7 @@ Requires-Dist: aiosqlite (==0.*)
 Requires-Dist: fastapi (==0.*)
 Requires-Dist: mimesniff (==1.*)
 Requires-Dist: pillow
+Requires-Dist: requests (==2.*)
 Requires-Dist: uvicorn (==0.*)
 Project-URL: Repository, https://github.com/MenxLi/lfss
 Description-Content-Type: text/markdown
@@ -23,7 +24,7 @@ Description-Content-Type: text/markdown
 # Lightweight File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
-My experiment on a lightweight file/object storage service.  
+My experiment on a lightweight and high-performance file/object storage service.  
 It stores small files and metadata in sqlite, large files in the filesystem.  
 Tested on 2 million files, and it works fine... 
 
@@ -45,7 +46,7 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
 Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
-You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/client/api.py` for the API usage.
+You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/api/connector.py` for the API usage.
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

lfss-0.8.0.dist-info/RECORD

@@ -0,0 +1,43 @@
+Readme.md,sha256=LpbTvUWjCOv4keMNDrZvEnNAmCQnvaxvlq2srWixXn0,1299
+docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
+docs/Permission.md,sha256=9r9nEmhqfz18RTS8FI0fZ9F0a31r86OoAyx3EQxxpk0,2317
+frontend/api.js,sha256=9cR8ddaoF-ulauQ1tcISV2nmfakkplC7uS8-lWFqU58,15820
+frontend/index.html,sha256=-k0bJ5FRqdl_H-O441D_H9E-iejgRCaL_z5UeYaS2qc,3384
+frontend/info.css,sha256=Ny0N3GywQ3a9q1_Qph_QFEKB4fEnTe_2DJ1Y5OsLLmQ,595
+frontend/info.js,sha256=WhOGaeqMoezEAfg4nIpK26hvejC7AZ-ZDLiJmRj0kDk,5758
+frontend/login.css,sha256=VMM0QfbDFYerxKWKSGhMI1yg5IRBXg0TTdLJEEhQZNk,355
+frontend/login.js,sha256=QoO8yKmBHDVP-ZomCMOaV7xVUVIhpl7esJrb6T5aHQE,2466
+frontend/popup.css,sha256=TJZYFW1ZcdD1IVTlNPYNtMWKPbN6XDbQ4hKBOFK8uLg,1284
+frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
+frontend/scripts.js,sha256=PFgAZC-frV-qs6sYCb1ZzH_CAa7in17qbfqYiaIxNmA,21697
+frontend/state.js,sha256=vbNL5DProRKmSEY7xu9mZH6IY0PBenF8WGxPtGgDnLI,1680
+frontend/styles.css,sha256=xcNLqI3KBsY5TLnku8UIP0Jfr7QLajr1_KNlZj9eheM,4935
+frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
+frontend/thumb.js,sha256=RQ_whXNwmkdG4SEbNQGeh488YYzqwoNYDc210hPeuhQ,5703
+frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
+lfss/api/__init__.py,sha256=YI1_9nyW0E5lyXn_PmmIzIff1ccBC-KCA0twpsKDRIY,5453
+lfss/api/connector.py,sha256=q9CJBOmN83tfpwI1IclSzq_lzI4Kq1SOKC3S5H-vuWo,9301
+lfss/cli/balance.py,sha256=R2rbO2tg9TVnnQIVeU0GJVeMS-5LDhEdk4mbOE9qGq0,4121
+lfss/cli/cli.py,sha256=8VKe41m_LhVSFxGlvgBxdz55sjscLNbbkNX1fOnmES4,4618
+lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
+lfss/cli/serve.py,sha256=T-jz_PJcaY9FJRRfyWV9MbjDI73YdOOCn4Nr2PO-s0c,993
+lfss/cli/user.py,sha256=wlR-xcJKCtr_y5QgYO9GM0JyDCKooIRlsAxw2eilPfs,3418
+lfss/cli/vacuum.py,sha256=4TMMYC_5yEt7jeaFTVC3iX0X6aUzYXBiCsrcIYgw_uA,3695
+lfss/sql/init.sql,sha256=C-JtQAlaOjESI8uoF1Y_9dKukEVSw5Ll-7yA3gG-XHU,1210
+lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
+lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lfss/src/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
+lfss/src/config.py,sha256=K1b5clNRO4EzxDG-p6U5aRLDaq-Up0tDHfn_D79D0ns,857
+lfss/src/connection_pool.py,sha256=4YULPcmndy33FyBSo9w1fZjAlBX2q-xPP27xJOTA06I,5228
+lfss/src/database.py,sha256=yFMoxHJ-ZRwi9qvSVdIl9m_gpycmL2eencdv7P0vzwQ,35678
+lfss/src/datatype.py,sha256=q2lc8BJaB2sS7gtqPMqxM625mckgI2SmvuqbadiObLY,2158
+lfss/src/error.py,sha256=Bh_GUtuNsMSMIKbFreU4CfZzL96TZdNAIcgmDxvGbQ0,333
+lfss/src/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
+lfss/src/server.py,sha256=OcyPuVyvBCxpgF8ESxzDYNZnUJIbkOGDv-eghUSVM9E,20063
+lfss/src/stat.py,sha256=WlRiiAl0rHX9oPi3thi6K4GKn70WHpClSDCt7iZUow4,3197
+lfss/src/thumb.py,sha256=qjCNMpnCozMuzkhm-2uAYy1eAuYTeWG6xqs-13HX-7k,3266
+lfss/src/utils.py,sha256=nal2rpr00jq1PeFhGQXkvU0FIbtRhXTj8VmbeIyRyLI,5184
+lfss-0.8.0.dist-info/METADATA,sha256=uNZzNHCActK1ok2aPmZZqMXE3JxyWHStAVAH5mzpvkc,2076
+lfss-0.8.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.8.0.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
+lfss-0.8.0.dist-info/RECORD,,

lfss-0.7.15.dist-info/RECORD

@@ -1,41 +0,0 @@
-Readme.md,sha256=vsPotlwPAaHI5plh4aaszpi3rr7ZGDn7-wLdEYTWQ0k,1275
-docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
-docs/Permission.md,sha256=X0VNfBKU52f93QYqcVyiBFJ3yURiSkhIo9S_5fdSgzM,2265
-frontend/api.js,sha256=o1sP4rKxxnM-rebxnlMlPkhPHzKaVW4kZC7B4ufbOK4,8026
-frontend/index.html,sha256=i45ilkRCorqXP0bTfMiwT3QmmEF23V34lZJC1nODHLo,2862
-frontend/info.css,sha256=Ny0N3GywQ3a9q1_Qph_QFEKB4fEnTe_2DJ1Y5OsLLmQ,595
-frontend/info.js,sha256=WhOGaeqMoezEAfg4nIpK26hvejC7AZ-ZDLiJmRj0kDk,5758
-frontend/popup.css,sha256=TJZYFW1ZcdD1IVTlNPYNtMWKPbN6XDbQ4hKBOFK8uLg,1284
-frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
-frontend/scripts.js,sha256=tmA_tG3bLEEqn0jTgZ-DkcEl6egxdrFYJmstb9eZMr0,21862
-frontend/state.js,sha256=Dda-2G4QzyqdxffjJa3Lb7rgJOrg2LvJ3TCRcB8YCrU,1327
-frontend/styles.css,sha256=krMo6Ulroi8pqEq1exQsFEU-FJqT9GzI8vyARiNF11k,4484
-frontend/thumb.css,sha256=1i8wudiMWGwdrnTqs6yrS8YaiPeHPR-A2YqUNJN20Ok,165
-frontend/thumb.js,sha256=6m8bscQpi2sYaLRir2ZeY1H_1ZRKFVss5P28AnLvIVQ,5486
-frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
-lfss/cli/balance.py,sha256=R2rbO2tg9TVnnQIVeU0GJVeMS-5LDhEdk4mbOE9qGq0,4121
-lfss/cli/cli.py,sha256=LH1nx5wI1K2DZ3hvHz7oq5HcXVDoW2V6sr7q9gJ8gqo,4621
-lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
-lfss/cli/serve.py,sha256=T-jz_PJcaY9FJRRfyWV9MbjDI73YdOOCn4Nr2PO-s0c,993
-lfss/cli/user.py,sha256=ETLtj0N-kmxv0mhmeAsO6cY7kPq7nOOP4DetxIRoQpQ,3405
-lfss/cli/vacuum.py,sha256=4TMMYC_5yEt7jeaFTVC3iX0X6aUzYXBiCsrcIYgw_uA,3695
-lfss/client/__init__.py,sha256=kcClDdHaQowAhHFYdrFyFqHIOe5MEjfENYq1icuj3Mg,4577
-lfss/client/api.py,sha256=aHorSuxl79xvRRfwuitXANYoTogWKVO11XX7mdrhHTE,5807
-lfss/sql/init.sql,sha256=C-JtQAlaOjESI8uoF1Y_9dKukEVSw5Ll-7yA3gG-XHU,1210
-lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
-lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lfss/src/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
-lfss/src/config.py,sha256=K1b5clNRO4EzxDG-p6U5aRLDaq-Up0tDHfn_D79D0ns,857
-lfss/src/connection_pool.py,sha256=r4Ho5d_Gd4S_KbT7515UJoiyfIgS6xyttqMsKqOfaIg,5190
-lfss/src/database.py,sha256=w2QPE3h1Lx0D0fUmdtu9s1XHpNp9p27zqm8AVeP2UVg,32476
-lfss/src/datatype.py,sha256=WfrLALU_7wei5-i_b0TxY8xWI5mwxLUHFepHSps49zA,1767
-lfss/src/error.py,sha256=imbhwnbhnI3HLhkbfICROe3F0gleKrOk4XnqHJDOtuI,285
-lfss/src/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
-lfss/src/server.py,sha256=igkPC3gdJoIqcVTKBAKkVPRrclXR2ZNBdRIAEci4xMo,17717
-lfss/src/stat.py,sha256=Wr-ug_JqtbSIf3XwQnv1xheGhsDTEOlLWuYoKO_26Jo,3201
-lfss/src/thumb.py,sha256=qjCNMpnCozMuzkhm-2uAYy1eAuYTeWG6xqs-13HX-7k,3266
-lfss/src/utils.py,sha256=TBGYvgt6xMP8UC5wTGHAr9fmdhu0_gjOtxcSeyvGyVM,3918
-lfss-0.7.15.dist-info/METADATA,sha256=s6jCttzD9bRauwKiKGNgrBouK5UDNEUMTgDHZBY_VfY,2021
-lfss-0.7.15.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.7.15.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
-lfss-0.7.15.dist-info/RECORD,,