lfss 0.9.5__py3-none-any.whl → 0.10.0__py3-none-any.whl

Readme.md

@@ -1,5 +1,5 @@
 # Lite File Storage Service (LFSS)
-[![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
+[![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/lfss)](https://pypi.org/project/lfss/)
 
 My experiment on a lightweight and high-performance file/object storage service...  
 

docs/Permission.md

@@ -22,16 +22,16 @@ A file is owned by the user who created it, may not necessarily be the user unde
 ## File access with `GET` permission
 
 ### File access
-For accessing file content, the user must have `GET` permission of the file, which is determined by the `permission` field of both the owner and the file.   
+For accessing file content, the user must have `GET` permission of the file, which is determined by the `permission` field of both the path-owner and the file.   
 
 There are four types of permissions: `unset`, `public`, `protected`, `private`.
 Non-admin users can access files based on:   
 
 - If the file is `public`, then all users can access it.
 - If the file is `protected`, then only the logged-in user can access it.  
-- If the file is `private`, then only the owner can access it.
-- If the file is `unset`, then the file's permission is inherited from the owner's permission.
-- If both the owner and the file have `unset` permission, then the file is `public`.
+- If the file is `private`, then only the owner/path-owner can access it.
+- If the file is `unset`, then the file's permission is inherited from the path-owner's permission.
+- If both the path-owner and the file have `unset` permission, then the file is `public`.
 
 ## File creation with `PUT`/`POST` permission
 `PUT`/`POST` permission is not allowed for non-peer users.

lfss/api/connector.py

@@ -15,12 +15,13 @@ from lfss.eng.utils import ensure_uri_compnents
 
 _default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
 _default_token = os.environ.get('LFSS_TOKEN', '')
+num_t = float | int
 
 class Connector:
     class Session:
         def __init__(
             self, connector: Connector, pool_size: int = 10, 
-            retry: int = 1, backoff_factor: float = 0.5, status_forcelist: list[int] = [503]
+            retry: int = 1, backoff_factor: num_t = 0.5, status_forcelist: list[int] = [503]
             ):
             self.connector = connector
             self.pool_size = pool_size
@@ -47,13 +48,21 @@ class Connector:
         def __exit__(self, exc_type, exc_value, traceback):
             self.close()
 
-    def __init__(self, endpoint=_default_endpoint, token=_default_token):
+    def __init__(self, endpoint=_default_endpoint, token=_default_token, timeout: Optional[num_t | tuple[num_t, num_t]]=None, verify: Optional[bool | str] = None):
+        """
+        - endpoint: the URL of the LFSS server. Default to $LFSS_ENDPOINT or http://localhost:8000.
+        - token: the access token. Default to $LFSS_TOKEN.
+        - timeout: the timeout for each request, can be either a single value or a tuple of two values (connect, read), refer to requests.Session.request.
+        - verify: either a boolean or a string, to control SSL verification. Default to True, refer to requests.Session.request.
+        """
         assert token, "No token provided. Please set LFSS_TOKEN environment variable."
         self.config = {
             "endpoint": endpoint,
             "token": token
         }
         self._session: Optional[requests.Session] = None
+        self.timeout = timeout
+        self.verify = verify
     
     def session( self, pool_size: int = 10, **kwargs):
         """ avoid creating a new session for each request.  """
@@ -74,11 +83,11 @@ class Connector:
             })
             headers.update(extra_headers)
             if self._session is not None:
-                response = self._session.request(method, url, headers=headers, **kwargs)
+                response = self._session.request(method, url, headers=headers, timeout=self.timeout, verify=self.verify, **kwargs)
                 response.raise_for_status()
             else:
                 with requests.Session() as s:
-                    response = s.request(method, url, headers=headers, **kwargs)
+                    response = s.request(method, url, headers=headers, timeout=self.timeout, verify=self.verify, **kwargs)
                     response.raise_for_status()
             return response
         return f

lfss/eng/connection_pool.py

@@ -8,7 +8,7 @@ from functools import wraps
 from typing import Callable, Awaitable
 
 from .log import get_logger
-from .error import DatabaseLockedError
+from .error import DatabaseLockedError, DatabaseTransactionError
 from .config import DATA_HOME
 
 async def execute_sql(conn: aiosqlite.Connection | aiosqlite.Cursor, name: str):
@@ -147,6 +147,14 @@ def global_entrance(n_read: int = 1):
         return wrapper
     return decorator
 
+def handle_sqlite_error(e: Exception):
+    if 'database is locked' in str(e):
+        raise DatabaseLockedError from e
+    if 'cannot start a transaction within a transaction' in str(e):
+        get_logger('database', global_instance=True).error(f"Unexpected error: {e}")
+        raise DatabaseTransactionError from e
+    raise e
+
 @asynccontextmanager
 async def unique_cursor(is_write: bool = False):
     if not is_write:
@@ -155,9 +163,7 @@ async def unique_cursor(is_write: bool = False):
             try:
                 yield await connection_obj.conn.cursor()
             except Exception as e:
-                if 'database is locked' in str(e):
-                    raise DatabaseLockedError from e
-                raise e
+                handle_sqlite_error(e)
             finally:
                 await g_pool.release(connection_obj)
     else:
@@ -166,9 +172,7 @@ async def unique_cursor(is_write: bool = False):
             try:
                 yield await connection_obj.conn.cursor()
             except Exception as e:
-                if 'database is locked' in str(e):
-                    raise DatabaseLockedError from e
-                raise e
+                handle_sqlite_error(e)
             finally:
                 await g_pool.release(connection_obj)
 

lfss/eng/database.py

@@ -965,6 +965,11 @@ class Database:
     async def zip_path(self, top_url: str, op_user: Optional[UserRecord]) -> io.BytesIO:
         if top_url.startswith('/'):
             top_url = top_url[1:]
+        
+        if op_user:
+            if await check_path_permission(top_url, op_user) < AccessLevel.READ:
+                raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot zip path {top_url}")
+        
         buffer = io.BytesIO()
         with zipfile.ZipFile(buffer, 'w') as zf:
             async for (r, blob) in self.iter_path(top_url, None):
@@ -979,39 +984,50 @@ class Database:
         buffer.seek(0)
         return buffer
 
-def check_file_read_permission(user: UserRecord, owner: UserRecord, file: FileRecord) -> tuple[bool, str]:
+async def _get_path_owner(cur: aiosqlite.Cursor, path: str) -> UserRecord:
+    path_username = path.split('/')[0]
+    uconn = UserConn(cur)
+    path_user = await uconn.get_user(path_username)
+    if path_user is None:
+        raise PathNotFoundError(f"Invalid path: {path_username} is not a valid username")
+    return path_user
+
+async def check_file_read_permission(user: UserRecord, file: FileRecord, cursor: Optional[aiosqlite.Cursor] = None) -> tuple[bool, str]:
     """
     This does not consider alias level permission,
     use check_path_permission for alias level permission check first:
     ```
     if await check_path_permission(path, user) < AccessLevel.READ:
-        read_allowed, reason = check_file_read_permission(user, owner, file)
+        read_allowed, reason = check_file_read_permission(user, file)
     ```
+    The implementation assumes the user is not admin and is not the owner of the file/path
     """
-    if user.is_admin:
-        return True, ""
+    @asynccontextmanager
+    async def this_cur():
+        if cursor is None:
+            async with unique_cursor() as _cur:
+                yield _cur
+        else:
+            yield cursor
+    
+    f_perm = file.permission
+
+    # if file permission unset, use path owner's permission as fallback
+    if f_perm == FileReadPermission.UNSET:
+        async with this_cur() as cur:
+            path_owner = await _get_path_owner(cur, file.url)
+        f_perm = path_owner.permission
     
     # check permission of the file
-    if file.permission == FileReadPermission.PRIVATE:
-        if user.id != owner.id:
-            return False, "Permission denied, private file"
-    elif file.permission == FileReadPermission.PROTECTED:
+    if f_perm == FileReadPermission.PRIVATE:
+        return False, "Permission denied, private file"
+    elif f_perm == FileReadPermission.PROTECTED:
         if user.id == 0:
             return False, "Permission denied, protected file"
-    elif file.permission == FileReadPermission.PUBLIC:
+    elif f_perm == FileReadPermission.PUBLIC:
         return True, ""
     else:
-        assert file.permission == FileReadPermission.UNSET
-
-    # use owner's permission as fallback
-    if owner.permission == FileReadPermission.PRIVATE:
-        if user.id != owner.id:
-            return False, "Permission denied, private user file"
-    elif owner.permission == FileReadPermission.PROTECTED:
-        if user.id == 0:
-            return False, "Permission denied, protected user file"
-    else:
-        assert owner.permission == FileReadPermission.PUBLIC or owner.permission == FileReadPermission.UNSET
+        assert f_perm == FileReadPermission.UNSET
 
     return True, ""
 
@@ -1034,15 +1050,11 @@ async def check_path_permission(path: str, user: UserRecord, cursor: Optional[ai
             yield cursor
 
     # check if path user exists
-    path_username = path.split('/')[0]
     async with this_cur() as cur:
-        uconn = UserConn(cur)
-        path_user = await uconn.get_user(path_username)
-    if path_user is None:
-        raise PathNotFoundError(f"Invalid path: {path_username} is not a valid username")
+        path_owner = await _get_path_owner(cur, path)
 
-    # check if user is admin
-    if user.is_admin or user.username == path_username:
+    # check if user is admin or the owner of the path
+    if user.is_admin or user.id == path_owner.id:
         return AccessLevel.ALL
     
     # if the path is a file, check if the user is the owner
@@ -1056,4 +1068,4 @@ async def check_path_permission(path: str, user: UserRecord, cursor: Optional[ai
     # check alias level
     async with this_cur() as cur:
         uconn = UserConn(cur)
-        return await uconn.query_peer_level(user.id, path_user.id)
+        return await uconn.query_peer_level(user.id, path_owner.id)

lfss/eng/error.py

@@ -12,6 +12,8 @@ class InvalidPathError(LFSSExceptionBase, ValueError):...
 
 class DatabaseLockedError(LFSSExceptionBase, sqlite3.DatabaseError):...
 
+class DatabaseTransactionError(LFSSExceptionBase, sqlite3.DatabaseError):...
+
 class PathNotFoundError(LFSSExceptionBase, FileNotFoundError):...
 
 class FileDuplicateError(LFSSExceptionBase, FileExistsError):...

lfss/svc/app_base.py

@@ -54,6 +54,7 @@ def handle_exception(fn):
             if isinstance(e, FileExistsError): raise HTTPException(status_code=409, detail=str(e))
             if isinstance(e, TooManyItemsError): raise HTTPException(status_code=400, detail=str(e))
             if isinstance(e, DatabaseLockedError): raise HTTPException(status_code=503, detail=str(e))
+            if isinstance(e, DatabaseTransactionError): raise HTTPException(status_code=503, detail=str(e))
             if isinstance(e, FileLockedError): raise HTTPException(status_code=423, detail=str(e))
             logger.error(f"Uncaptured error in {fn.__name__}: {e}")
             raise 

lfss/svc/app_native.py

@@ -5,6 +5,7 @@ from fastapi.responses import StreamingResponse
 from fastapi.exceptions import HTTPException 
 
 from ..eng.utils import ensure_uri_compnents
+from ..eng.config import MAX_MEM_FILE_BYTES
 from ..eng.connection_pool import unique_cursor
 from ..eng.database import check_file_read_permission, check_path_permission, UserConn, FileConn
 from ..eng.datatype import (
@@ -92,14 +93,29 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
         dir_record = await FileConn(cur).get_path_record(path)
 
     pathname = f"{path.split('/')[-2]}"
-    return StreamingResponse(
-        content = await db.zip_path_stream(path, op_user=user),
-        media_type = "application/zip",
-        headers = {
-            f"Content-Disposition": f"attachment; filename=bundle-{pathname}.zip",
-            "X-Content-Bytes": str(dir_record.size),
-        }
-    )
+
+    if dir_record.size < MAX_MEM_FILE_BYTES:
+        logger.debug(f"Bundle {path} in memory")
+        dir_bytes = (await db.zip_path(path, op_user=user)).getvalue()
+        return Response(
+            content = dir_bytes,
+            media_type = "application/zip",
+            headers = {
+                f"Content-Disposition": f"attachment; filename=bundle-{pathname}.zip",
+                "Content-Length": str(len(dir_bytes)),
+                "X-Content-Bytes": str(dir_record.size),
+            }
+        )
+    else:
+        logger.debug(f"Bundle {path} in stream")
+        return StreamingResponse(
+            content = await db.zip_path_stream(path, op_user=user),
+            media_type = "application/zip",
+            headers = {
+                f"Content-Disposition": f"attachment; filename=bundle-{pathname}.zip",
+                "X-Content-Bytes": str(dir_record.size),
+            }
+        )
 
 @router_api.get("/meta")
 @handle_exception
@@ -112,9 +128,7 @@ async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
         if is_file:
             record = await fconn.get_file_record(path, throw=True)
             if await check_path_permission(path, user, cursor=cur) < AccessLevel.READ:
-                uconn = UserConn(cur)
-                owner = await uconn.get_user_by_id(record.owner_id, throw=True)
-                is_allowed, reason = check_file_read_permission(user, owner, record)
+                is_allowed, reason = await check_file_read_permission(user, record, cursor=cur)
                 if not is_allowed:
                     raise HTTPException(status_code=403, detail=reason)
         else:

lfss/svc/common_impl.py

@@ -112,11 +112,8 @@ async def get_impl(
     async with unique_cursor() as cur:
         fconn = FileConn(cur)
         file_record = await fconn.get_file_record(path, throw=True)
-        uconn = UserConn(cur)
-        owner = await uconn.get_user_by_id(file_record.owner_id, throw=True)
-
         if not await check_path_permission(path, user, cursor=cur) >= AccessLevel.READ:
-            allow_access, reason = check_file_read_permission(user, owner, file_record)
+            allow_access, reason = await check_file_read_permission(user, file_record, cursor=cur)
             if not allow_access:
                 raise HTTPException(status_code=403 if user.id != 0 else 401, detail=reason)
     

{lfss-0.9.5.dist-info → lfss-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.9.5
+Version: 0.10.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li_mengxun
@@ -23,7 +23,7 @@ Project-URL: Repository, https://github.com/MenxLi/lfss
 Description-Content-Type: text/markdown
 
 # Lite File Storage Service (LFSS)
-[![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
+[![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/lfss)](https://pypi.org/project/lfss/)
 
 My experiment on a lightweight and high-performance file/object storage service...  
 

{lfss-0.9.5.dist-info → lfss-0.10.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-Readme.md,sha256=ST2E12DJVlKTReiJjRBc7KZyAr8KyqlcK2BoTc_Peaw,1829
+Readme.md,sha256=B-foESzFWoSI5MEd89AWUzKcVRrTwipM28TK8GN0o8c,1920
 docs/Changelog.md,sha256=QYej_hmGnv9t8wjFHXBvmrBOvY7aACZ82oa5SVkIyzM,882
 docs/Enviroment_variables.md,sha256=xaL8qBwT8B2Qe11FaOU3xWrRCh1mJ1VyTFCeFbkd0rs,570
 docs/Known_issues.md,sha256=ZqETcWP8lzTOel9b2mxEgCnADFF8IxOrEtiVO1NoMAk,251
-docs/Permission.md,sha256=mvK8gVBBgoIFJqikcaReU_bUo-mTq_ECqJaDDJoQF7Q,3126
+docs/Permission.md,sha256=thUJx7YRoU63Pb-eqo5l5450DrZN3QYZ36GCn8r66no,3152
 docs/Webdav.md,sha256=-Ja-BTWSY1BEMAyZycvEMNnkNTPZ49gSPzmf3Lbib70,1547
 frontend/api.js,sha256=GlQsNoZFEcy7QUUsLbXv7aP-KxRnIxM37FQHTaakGiQ,19387
 frontend/index.html,sha256=-k0bJ5FRqdl_H-O441D_H9E-iejgRCaL_z5UeYaS2qc,3384
@@ -19,7 +19,7 @@ frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
 frontend/thumb.js,sha256=46ViD2TlTTWy0fx6wjoAs_5CQ4ajYB90vVzM7UO2IHw,6182
 frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
 lfss/api/__init__.py,sha256=8IJqrpWK1doIyVVbntvVic82A57ncwl5b0BRHX4Ri6A,6660
-lfss/api/connector.py,sha256=0iopAvqUiUJDjbAtAjr9ynmURnmB-Ejg3INL-877s_E,12192
+lfss/api/connector.py,sha256=Duh57M3dOeG_M5UidZ4hMHK7ot1JsUC6RdXgIn6KTC8,12913
 lfss/cli/__init__.py,sha256=lPwPmqpa7EXQ4zlU7E7LOe6X2kw_xATGdwoHphUEirA,827
 lfss/cli/balance.py,sha256=fUbKKAUyaDn74f7mmxMfBL4Q4voyBLHu6Lg_g8GfMOQ,4121
 lfss/cli/cli.py,sha256=aYjB8d4k6JUd9efxZK-XOj-mlG4JeOr_0lnj2qqCiK0,8066
@@ -30,22 +30,22 @@ lfss/cli/vacuum.py,sha256=GOG72d3NYe9bYCNc3y8JecEmM-DrKlGq3JQcisv_xBg,3702
 lfss/eng/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lfss/eng/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
 lfss/eng/config.py,sha256=Vni6h52Ce0njVrHZLAWFL8g34YDBdlmGrmRhpxElxQ8,868
-lfss/eng/connection_pool.py,sha256=4xOF1kXXGqCWeLX5ZVFALKjdY8N1VVAVSSTRfCzbj94,6141
-lfss/eng/database.py,sha256=9KMV-VeD8Dgd6M9RfNfAKwF5gWm763eJpJ2g5zyn_uY,48691
+lfss/eng/connection_pool.py,sha256=1aq7nSgd7hB9YNV4PjD1RDRyl_moDw3ubBtSLyfgGBs,6320
+lfss/eng/database.py,sha256=RYIG2506_-S84f6CsOQ6pgpr1vgPT3p1kP1FsZwTOnM,49098
 lfss/eng/datatype.py,sha256=27UB7-l9SICy5lAvKjdzpTL_GohZjzstQcr9PtAq7nM,2709
-lfss/eng/error.py,sha256=dAlQHXOnQcSkA2vTugJFSxcyDqoFlPucBoFpTZ7GI6w,654
+lfss/eng/error.py,sha256=JGf5NV-f4rL6tNIDSAx5-l9MG8dEj7F2w_MuOjj1d1o,732
 lfss/eng/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
 lfss/eng/thumb.py,sha256=x9jIHHU1tskmp-TavPPcxGpbmEjCp9gbH6ZlsEfqUxY,3383
 lfss/eng/utils.py,sha256=WYoXFFi5308UWtFC8VP792gpzrVbHZZHhP3PaFjxIEY,6770
 lfss/sql/init.sql,sha256=8LjHx0TBCkBD62xFfssSeHDqKYVQQJkZAg4rSm046f4,1496
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/svc/app.py,sha256=ftWCpepBx-gTSG7i-TB-IdinPPstAYYQjCgnTfeMZeI,219
-lfss/svc/app_base.py,sha256=PgL5MNU3QTwgIJP8CflDi9YBZ-uo4hVf74ADyWTo9yg,6742
+lfss/svc/app_base.py,sha256=bTQbz945xalyB3UZLlqVBvL6JKGNQ8Fm2KpIvvucPZQ,6850
 lfss/svc/app_dav.py,sha256=D0KSgjtTktPjIhyIKG5eRmBdh5X8HYFYH151E6gzlbc,18245
-lfss/svc/app_native.py,sha256=N2cidZ5sIS0p9HOkPqWvpGkqe4bIA7CgGEp4CsvgZ6Q,8347
-lfss/svc/common_impl.py,sha256=0fjbqHWgqDhLfBEu6aC0Z5qgNt67C7z0Qroj7aV3Iq4,13830
+lfss/svc/app_native.py,sha256=JbPge-F9irl26tXKAzfA5DfyjCh0Dgttflztqqrvt0A,8890
+lfss/svc/common_impl.py,sha256=5ZRM24zVZpAeipgDtZUVBMFtArkydlAkn17ic_XL7v8,13733
 lfss/svc/request_log.py,sha256=v8yXEIzPjaksu76Oh5vgdbUEUrw8Kt4etLAXBWSGie8,3207
-lfss-0.9.5.dist-info/METADATA,sha256=fToo-wrY0_XJPbk3cte8hb_4DClyPFv3ZTzJB77G-5w,2623
-lfss-0.9.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.9.5.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
-lfss-0.9.5.dist-info/RECORD,,
+lfss-0.10.0.dist-info/METADATA,sha256=NewpmEw8OUj28rPkujuPi3ZySJG_JCvSEKY5JBxg6cw,2715
+lfss-0.10.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.10.0.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
+lfss-0.10.0.dist-info/RECORD,,