lfss 0.8.3__py3-none-any.whl → 0.9.0__py3-none-any.whl

lfss/src/server.py

@@ -16,10 +16,10 @@ from .stat import RequestDB
 from .config import MAX_BUNDLE_BYTES, CHUNK_SIZE
 from .utils import ensure_uri_compnents, format_last_modified, now_stamp, wait_for_debounce_tasks
 from .connection_pool import global_connection_init, global_connection_close, unique_cursor
-from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn
+from .database import Database, DECOY_USER, check_file_read_permission, check_path_permission, UserConn, FileConn
 from .database import delayed_log_activity, delayed_log_access
 from .datatype import (
-    FileReadPermission, FileRecord, UserRecord, PathContents, 
+    FileReadPermission, FileRecord, UserRecord, PathContents, AccessLevel, 
     FileSortKey, DirSortKey
 )
 from .thumb import get_thumb
@@ -54,6 +54,7 @@ def handle_exception(fn):
             if isinstance(e, FileNotFoundError): raise HTTPException(status_code=404, detail=str(e))
             if isinstance(e, FileExistsError): raise HTTPException(status_code=409, detail=str(e))
             if isinstance(e, TooManyItemsError): raise HTTPException(status_code=400, detail=str(e))
+            if isinstance(e, DatabaseLockedError): raise HTTPException(status_code=503, detail=str(e))
             logger.error(f"Uncaptured error in {fn.__name__}: {e}")
             raise 
     return wrapper
@@ -228,39 +229,37 @@ async def get_file_impl(
     if path == "": path = "/"
     if path.endswith("/"):
         # return file under the path as json
-        async with unique_cursor() as conn:
-            fconn = FileConn(conn)
+        async with unique_cursor() as cur:
+            fconn = FileConn(cur)
             if user.id == 0:
                 raise HTTPException(status_code=401, detail="Permission denied, credential required")
             if thumb:
                 return await emit_thumbnail(path, download, create_time=None)
             
             if path == "/":
+                peer_users = await UserConn(cur).list_peer_users(user.id, AccessLevel.READ)
                 return PathContents(
-                    dirs = await fconn.list_root_dirs(user.username, skim=True) \
+                    dirs = await fconn.list_root_dirs(user.username, *[x.username for x in peer_users], skim=True) \
                         if not user.is_admin else await fconn.list_root_dirs(skim=True),
                     files = []
                 )
 
-            if not path.startswith(f"{user.username}/") and not user.is_admin:
-                raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
+            if not await check_path_permission(path, user, cursor=cur) >= AccessLevel.READ:
+                raise HTTPException(status_code=403, detail="Permission denied")
 
             return await fconn.list_path(path)
     
     # handle file query
-    async with unique_cursor() as conn:
-        fconn = FileConn(conn)
-        file_record = await fconn.get_file_record(path)
-        if not file_record:
-            raise HTTPException(status_code=404, detail="File not found")
-
-        uconn = UserConn(conn)
-        owner = await uconn.get_user_by_id(file_record.owner_id)
-
-    assert owner is not None, "Owner not found"
-    allow_access, reason = check_user_permission(user, owner, file_record)
-    if not allow_access:
-        raise HTTPException(status_code=403, detail=reason)
+    async with unique_cursor() as cur:
+        fconn = FileConn(cur)
+        file_record = await fconn.get_file_record(path, throw=True)
+        uconn = UserConn(cur)
+        owner = await uconn.get_user_by_id(file_record.owner_id, throw=True)
+
+        if not await check_path_permission(path, user, cursor=cur) >= AccessLevel.READ:
+            allow_access, reason = check_file_read_permission(user, owner, file_record)
+            if not allow_access:
+                raise HTTPException(status_code=403 if user.id != 0 else 401, detail=reason)
     
     req_range = request.headers.get("Range", None)
     if not req_range is None:
@@ -326,17 +325,11 @@ async def put_file(
     ):
     path = ensure_uri_compnents(path)
     assert not path.endswith("/"), "Path must not end with /"
-    if not path.startswith(f"{user.username}/"):
-        if not user.is_admin:
-            logger.debug(f"Reject put request from {user.username} to {path}")
-            raise HTTPException(status_code=403, detail="Permission denied")
-        else:
-            first_comp = path.split("/")[0]
-            async with unique_cursor() as c:
-                uconn = UserConn(c)
-                owner = await uconn.get_user(first_comp)
-                if not owner:
-                    raise HTTPException(status_code=404, detail="Owner not found")
+
+    access_level = await check_path_permission(path, user)
+    if access_level < AccessLevel.WRITE:
+        logger.debug(f"Reject put request from {user.username} to {path}")
+        raise HTTPException(status_code=403, detail="Permission denied")
     
     logger.info(f"PUT {path}, user: {user.username}")
     exists_flag = False
@@ -352,7 +345,7 @@ async def put_file(
                 "Content-Type": "application/json",
             }, content=json.dumps({"url": path}))
         exists_flag = True
-        if not user.is_admin and not file_record.owner_id == user.id:
+        if await check_path_permission(path, user) < AccessLevel.WRITE:
             raise HTTPException(status_code=403, detail="Permission denied, cannot overwrite other's file")
         await db.delete_file(path)
     
@@ -386,17 +379,11 @@ async def post_file(
     ):
     path = ensure_uri_compnents(path)
     assert not path.endswith("/"), "Path must not end with /"
-    if not path.startswith(f"{user.username}/"):
-        if not user.is_admin:
-            logger.debug(f"Reject put request from {user.username} to {path}")
-            raise HTTPException(status_code=403, detail="Permission denied")
-        else:
-            first_comp = path.split("/")[0]
-            async with unique_cursor() as conn:
-                uconn = UserConn(conn)
-                owner = await uconn.get_user(first_comp)
-                if not owner:
-                    raise HTTPException(status_code=404, detail="Owner not found")
+
+    access_level = await check_path_permission(path, user)
+    if access_level < AccessLevel.WRITE:
+        logger.debug(f"Reject post request from {user.username} to {path}")
+        raise HTTPException(status_code=403, detail="Permission denied")
 
     logger.info(f"POST {path}, user: {user.username}")
     exists_flag = False
@@ -412,7 +399,7 @@ async def post_file(
                 "Content-Type": "application/json",
             }, content=json.dumps({"url": path}))
         exists_flag = True
-        if not user.is_admin and not file_record.owner_id == user.id:
+        if await check_path_permission(path, user) < AccessLevel.WRITE: 
             raise HTTPException(status_code=403, detail="Permission denied, cannot overwrite other's file")
         await db.delete_file(path)
     
@@ -431,7 +418,7 @@ async def post_file(
 @handle_exception
 async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
     path = ensure_uri_compnents(path)
-    if not path.startswith(f"{user.username}/") and not user.is_admin:
+    if await check_path_permission(path, user) < AccessLevel.WRITE:
         raise HTTPException(status_code=403, detail="Permission denied")
     
     logger.info(f"DELETE {path}, user: {user.username}")
@@ -458,6 +445,7 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
     if not path == "" and path[0] == "/":   # adapt to both /path and path
         path = path[1:]
     
+    # TODO: may check peer users here
     owner_records_cache: dict[int, UserRecord] = {}     # cache owner records, ID -> UserRecord
     async def is_access_granted(file_record: FileRecord):
         owner_id = file_record.owner_id
@@ -465,11 +453,10 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
         if owner is None:
             async with unique_cursor() as conn:
                 uconn = UserConn(conn)
-                owner = await uconn.get_user_by_id(owner_id)
-                assert owner is not None, "Owner not found"
+                owner = await uconn.get_user_by_id(owner_id, throw=True)
             owner_records_cache[owner_id] = owner
             
-        allow_access, _ = check_user_permission(user, owner, file_record)
+        allow_access, _ = check_file_read_permission(user, owner, file_record)
         return allow_access
     
     async with unique_cursor() as conn:
@@ -502,23 +489,20 @@ async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
     logger.info(f"GET meta({path}), user: {user.username}")
     path = ensure_uri_compnents(path)
     is_file = not path.endswith("/")
-    async with unique_cursor() as conn:
-        fconn = FileConn(conn)
+    async with unique_cursor() as cur:
+        fconn = FileConn(cur)
         if is_file:
-            record = await fconn.get_file_record(path)
-            if not record:
-                raise HTTPException(status_code=404, detail="File not found")
-            if not path.startswith(f"{user.username}/") and not user.is_admin:
-                uconn = UserConn(conn)
-                owner = await uconn.get_user_by_id(record.owner_id)
-                assert owner is not None, "Owner not found"
-                is_allowed, reason = check_user_permission(user, owner, record)
+            record = await fconn.get_file_record(path, throw=True)
+            if await check_path_permission(path, user, cursor=cur) < AccessLevel.READ:
+                uconn = UserConn(cur)
+                owner = await uconn.get_user_by_id(record.owner_id, throw=True)
+                is_allowed, reason = check_file_read_permission(user, owner, record)
                 if not is_allowed:
                     raise HTTPException(status_code=403, detail=reason)
         else:
-            record = await fconn.get_path_record(path)
-            if not path.startswith(f"{user.username}/") and not user.is_admin:
+            if await check_path_permission(path, user, cursor=cur) < AccessLevel.READ:
                 raise HTTPException(status_code=403, detail="Permission denied")
+            record = await fconn.get_path_record(path)
     return record
 
 @router_api.post("/meta")
@@ -559,15 +543,14 @@ async def update_file_meta(
 
     return Response(status_code=200, content="OK")
 
-async def validate_path_permission(path: str, user: UserRecord):
+async def validate_path_read_permission(path: str, user: UserRecord):
     if not path.endswith("/"):
         raise HTTPException(status_code=400, detail="Path must end with /")
-    if not path.startswith(f"{user.username}/") and not user.is_admin:
+    if not await check_path_permission(path, user) >= AccessLevel.READ:
         raise HTTPException(status_code=403, detail="Permission denied")
-
 @router_api.get("/count-files")
 async def count_files(path: str, flat: bool = False, user: UserRecord = Depends(registered_user)):
-    await validate_path_permission(path, user)
+    await validate_path_read_permission(path, user)
     path = ensure_uri_compnents(path)
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
@@ -578,7 +561,7 @@ async def list_files(
     order_by: FileSortKey = "", order_desc: bool = False,
     flat: bool = False, user: UserRecord = Depends(registered_user)
     ):
-    await validate_path_permission(path, user)
+    await validate_path_read_permission(path, user)
     path = ensure_uri_compnents(path)
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
@@ -590,7 +573,7 @@ async def list_files(
 
 @router_api.get("/count-dirs")
 async def count_dirs(path: str, user: UserRecord = Depends(registered_user)):
-    await validate_path_permission(path, user)
+    await validate_path_read_permission(path, user)
     path = ensure_uri_compnents(path)
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
@@ -601,7 +584,7 @@ async def list_dirs(
     order_by: DirSortKey = "", order_desc: bool = False,
     skim: bool = True, user: UserRecord = Depends(registered_user)
     ):
-    await validate_path_permission(path, user)
+    await validate_path_read_permission(path, user)
     path = ensure_uri_compnents(path)
     async with unique_cursor() as conn:
         fconn = FileConn(conn)

lfss/src/utils.py

@@ -47,13 +47,15 @@ def debounce_async(delay: float = 0.1, max_wait: float = 1.):
     """
     def debounce_wrap(func):
         task_record: tuple[str, asyncio.Task] | None = None
+        fn_execution_lock = Lock()
         last_execution_time = 0
 
         async def delayed_func(*args, **kwargs):
             nonlocal last_execution_time
             await asyncio.sleep(delay)
-            await func(*args, **kwargs)
-            last_execution_time = time.monotonic()
+            async with fn_execution_lock:
+                await func(*args, **kwargs)
+                last_execution_time = time.monotonic()
 
         @functools.wraps(func)
         async def wrapper(*args, **kwargs):
@@ -64,10 +66,11 @@ def debounce_async(delay: float = 0.1, max_wait: float = 1.):
                     task_record[1].cancel()
                     g_debounce_tasks.pop(task_record[0], None)
             
-            if time.monotonic() - last_execution_time > max_wait:
-                await func(*args, **kwargs)
-                last_execution_time = time.monotonic()
-                return
+            async with fn_execution_lock:
+                if time.monotonic() - last_execution_time > max_wait:
+                    await func(*args, **kwargs)
+                    last_execution_time = time.monotonic()
+                    return
 
             task = asyncio.create_task(delayed_func(*args, **kwargs))
             task_uid = uuid4().hex

{lfss-0.8.3.dist-info → lfss-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.8.3
+Version: 0.9.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
@@ -24,9 +24,17 @@ Description-Content-Type: text/markdown
 # Lightweight File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
-My experiment on a lightweight and high-performance file/object storage service.  
+My experiment on a lightweight and high-performance file/object storage service...  
+
+**Highlights:**
+
+- User storage limit and access control.
+- Pagination and sorted file listing for vast number of files.  
+- High performance: high concurrency, near-native speed on stress tests.
+- Support range requests, so you can stream large files / resume download.
+
 It stores small files and metadata in sqlite, large files in the filesystem.  
-Tested on 2 million files, and it works fine... 
+Tested on 2 million files, and it is still fast.
 
 Usage: 
 ```sh
@@ -45,8 +53,8 @@ lfss-panel --open
 Or, you can start a web server at `/frontend` and open `index.html` in your browser. 
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
-Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
-You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/api/connector.py` for the API usage.
+Authentication via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
+You can refer to `frontend` as an application example, `lfss/api/connector.py` for more APIs. 
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

{lfss-0.8.3.dist-info → lfss-0.9.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-Readme.md,sha256=LpbTvUWjCOv4keMNDrZvEnNAmCQnvaxvlq2srWixXn0,1299
+Readme.md,sha256=J1tGk7B9EyIXT-RN7VGz_229UeKvZHVLpn1FvzNDxL4,1538
 docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
-docs/Permission.md,sha256=9r9nEmhqfz18RTS8FI0fZ9F0a31r86OoAyx3EQxxpk0,2317
-frontend/api.js,sha256=wUJNAkL8QigAiwR_jaMPUhCQEsL-lp0wZ6XeueYgunE,18049
+docs/Permission.md,sha256=mvK8gVBBgoIFJqikcaReU_bUo-mTq_ECqJaDDJoQF7Q,3126
+frontend/api.js,sha256=hMV6Fc1JxkFQgv7BV1Y_Su7pqsWeF_92hPMmDBcXC04,18485
 frontend/index.html,sha256=-k0bJ5FRqdl_H-O441D_H9E-iejgRCaL_z5UeYaS2qc,3384
 frontend/info.css,sha256=Ny0N3GywQ3a9q1_Qph_QFEKB4fEnTe_2DJ1Y5OsLLmQ,595
 frontend/info.js,sha256=xGUJPCSrtDhuSu0ELLQZ77PmVWldg-prU1mwQGbdEoA,5797
@@ -13,32 +13,32 @@ frontend/scripts.js,sha256=nWH6NgavZTVmjK44i2DeRi6mJzGSe4qeQPUbDaEVt58,21735
 frontend/state.js,sha256=vbNL5DProRKmSEY7xu9mZH6IY0PBenF8WGxPtGgDnLI,1680
 frontend/styles.css,sha256=xcNLqI3KBsY5TLnku8UIP0Jfr7QLajr1_KNlZj9eheM,4935
 frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
-frontend/thumb.js,sha256=tX9LtxT6O2O6IUlpdvID6S973SUpWxgPVVqI9pwlVw8,6113
+frontend/thumb.js,sha256=46ViD2TlTTWy0fx6wjoAs_5CQ4ajYB90vVzM7UO2IHw,6182
 frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
 lfss/api/__init__.py,sha256=c_-GokKJ_3REgve16AwgXRfFyl9mwy7__FxCIIVlk1Q,6660
-lfss/api/connector.py,sha256=tCUwTlbzTwHvPnFb8nlnc6LEnrXwdCnCCThyBISt2Tg,11319
+lfss/api/connector.py,sha256=sSYy8gDewOosQiOzn8rvl7NsfFkIuhumHDefAVCgess,11573
 lfss/cli/__init__.py,sha256=lPwPmqpa7EXQ4zlU7E7LOe6X2kw_xATGdwoHphUEirA,827
 lfss/cli/balance.py,sha256=R2rbO2tg9TVnnQIVeU0GJVeMS-5LDhEdk4mbOE9qGq0,4121
-lfss/cli/cli.py,sha256=LxUrviHtsqi-vs_GWZw2qRs9dBNvx9PSQHLW6SwUmhA,8167
+lfss/cli/cli.py,sha256=iQkZm5Ltlhw7EWM4gOv_N0vjxiteGDH_aGhh06YMPYk,8066
 lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
 lfss/cli/serve.py,sha256=T-jz_PJcaY9FJRRfyWV9MbjDI73YdOOCn4Nr2PO-s0c,993
-lfss/cli/user.py,sha256=uqHQ7onddTjJAYg3B1DIc8hDl0aCkIMZolLKhQrBd0k,4046
+lfss/cli/user.py,sha256=4ynarVvSybxEQaoAzCn2dN2h6-9A61XDNQ-83-lwK4s,5364
 lfss/cli/vacuum.py,sha256=4TMMYC_5yEt7jeaFTVC3iX0X6aUzYXBiCsrcIYgw_uA,3695
-lfss/sql/init.sql,sha256=C-JtQAlaOjESI8uoF1Y_9dKukEVSw5Ll-7yA3gG-XHU,1210
+lfss/sql/init.sql,sha256=8LjHx0TBCkBD62xFfssSeHDqKYVQQJkZAg4rSm046f4,1496
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lfss/src/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
 lfss/src/config.py,sha256=7k_6L_aG7vD-lVcmgpwMIg-ggpMMDNU8IbaB4y1effE,861
-lfss/src/connection_pool.py,sha256=4YULPcmndy33FyBSo9w1fZjAlBX2q-xPP27xJOTA06I,5228
-lfss/src/database.py,sha256=zoiBm7CVHHV4TqwmK6lPnZvK9mzDNtrNvAJCRaIYMU8,36302
-lfss/src/datatype.py,sha256=yyOcxhGwz-EJi003f8hGl82EJuY4F92y6fSX6cK60Bc,2126
-lfss/src/error.py,sha256=Bh_GUtuNsMSMIKbFreU4CfZzL96TZdNAIcgmDxvGbQ0,333
+lfss/src/connection_pool.py,sha256=-tePasJxiZZ73ymgWf_kFnaKouc4Rrr4K6EXwjb7Mm4,6141
+lfss/src/database.py,sha256=jahoLa3kU6wGovwyfuvaMuGI8rLxcJYSfMkTT8_LI3c,41883
+lfss/src/datatype.py,sha256=27UB7-l9SICy5lAvKjdzpTL_GohZjzstQcr9PtAq7nM,2709
+lfss/src/error.py,sha256=oUQPkXzrlJ6Mtvm26T_3SYW_4j9unfudG3HMa1Q-JF0,421
 lfss/src/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
-lfss/src/server.py,sha256=YLsp6bab7q0I2hI4uUYIiWc2S0k6d6bbMaweg6VbVV4,23743
+lfss/src/server.py,sha256=TMsZBt-hF4dh_-e_v5odki09S36kJ33Gi_GbtUnGQ-M,23310
 lfss/src/stat.py,sha256=WlRiiAl0rHX9oPi3thi6K4GKn70WHpClSDCt7iZUow4,3197
 lfss/src/thumb.py,sha256=qjCNMpnCozMuzkhm-2uAYy1eAuYTeWG6xqs-13HX-7k,3266
-lfss/src/utils.py,sha256=DxjHabdiISMkrm1WQlpsZFKL3by6YrzBNQaDt_uZlRk,5744
-lfss-0.8.3.dist-info/METADATA,sha256=2Q3LdTB3vX_i8kpXIJ9SmMv-GLC1kslzN0RUBW9ksSA,2059
-lfss-0.8.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.8.3.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
-lfss-0.8.3.dist-info/RECORD,,
+lfss/src/utils.py,sha256=XHBDBODU6RPpYywArmMBVnghPk3crPLrSW4cKxqiP5Q,5887
+lfss-0.9.0.dist-info/METADATA,sha256=QNSYV-4VzdS4VCmKtSvjSMurXTb1qvUhvW-Wvpz3dcA,2298
+lfss-0.9.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.9.0.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
+lfss-0.9.0.dist-info/RECORD,,