lfss 0.9.4__py3-none-any.whl → 0.9.5__py3-none-any.whl

Readme.md

@@ -1,4 +1,4 @@
-# Lightweight File Storage Service (LFSS)
+# Lite File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
 My experiment on a lightweight and high-performance file/object storage service...  
@@ -32,8 +32,8 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
 The authentication can be acheived through one of the following methods:
-1. `Authorization` header with the value `Bearer sha256(<username><password>)`.
-2. `token` query parameter with the value `sha256(<username><password>)`.
+1. `Authorization` header with the value `Bearer sha256(<username>:<password>)`.
+2. `token` query parameter with the value `sha256(<username>:<password>)`.
 3. HTTP Basic Authentication with the username and password (If WebDAV is enabled).
 
 You can refer to `frontend` as an application example, `lfss/api/connector.py` for more APIs. 

docs/Changelog.md

@@ -1,6 +1,10 @@
 
 ## 0.9
 
+### 0.9.5
+- Stream bundle path as zip file.
+- Update authentication token hash format (need to reset password).
+
 ### 0.9.4
 - Decode WebDAV file name. 
 - Allow root-listing for WebDAV.

docs/Enviroment_variables.md

@@ -9,4 +9,4 @@
 
 **Client**
 - `LFSS_ENDPOINT`: The fallback server endpoint. Default is `http://localhost:8000`.
-- `LFSS_TOKEN`: The fallback token to authenticate. Should be `sha256(<username><password>)`.
+- `LFSS_TOKEN`: The fallback token to authenticate. Should be `sha256(<username>:<password>)`.

lfss/api/connector.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
-from typing import Optional, Literal, Iterator
+from typing import Optional, Literal
+from collections.abc import Iterator
 import os
 import requests
 import requests.adapters
@@ -276,6 +277,14 @@ class Connector:
         self._fetch_factory('POST', '_api/copy', {'src': src, 'dst': dst})(
             headers = {'Content-Type': 'application/www-form-urlencoded'}
         )
+    
+    def bundle(self, path: str) -> Iterator[bytes]:
+        """Bundle a path into a zip file."""
+        response = self._fetch_factory('GET', '_api/bundle', {'path': path})(
+            headers = {'Content-Type': 'application/www-form-urlencoded'}, 
+            stream = True
+        )
+        return response.iter_content(chunk_size=1024)
         
     def whoami(self) -> UserRecord:
         """Gets information about the current user."""

lfss/eng/config.py

@@ -19,7 +19,6 @@ if __env_large_file is not None:
 else:
     LARGE_FILE_BYTES = 8 * 1024 * 1024  # 8MB
 MAX_MEM_FILE_BYTES = 128 * 1024 * 1024   # 128MB
-MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
 CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'
 

lfss/eng/database.py

@@ -1,10 +1,11 @@
 
-from typing import Optional, Literal, AsyncIterable, overload
+from typing import Optional, Literal, overload
+from collections.abc import AsyncIterable
 from contextlib import asynccontextmanager
 from abc import ABC
 
+import uuid, datetime
 import urllib.parse
-import uuid
 import zipfile, io, asyncio
 
 import aiosqlite, aiofiles
@@ -82,9 +83,11 @@ class UserConn(DBObjectBase):
         self, username: str, password: str, is_admin: bool = False, 
         max_storage: int = 1073741824, permission: FileReadPermission = FileReadPermission.UNSET
         ) -> int:
-        assert not username.startswith('_'), "Error: reserved username"
-        assert not ('/' in username or len(username) > 255), "Invalid username"
-        assert urllib.parse.quote(username) == username, "Invalid username, must be URL safe"
+        def validate_username(username: str):
+            assert not username.startswith('_'), "Error: reserved username"
+            assert not ('/' in username or ':' in username or len(username) > 255), "Invalid username"
+            assert urllib.parse.quote(username) == username, "Invalid username, must be URL safe"
+        validate_username(username)
         self.logger.debug(f"Creating user {username}")
         credential = hash_credential(username, password)
         assert await self.get_user(username) is None, "Duplicate username"
@@ -926,14 +929,45 @@ class Database:
                 else:
                     blob = await fconn.get_file_blob(f_id)
                 yield r, blob
+    
+    async def zip_path_stream(self, top_url: str, op_user: Optional[UserRecord] = None) -> AsyncIterable[bytes]:
+        from stat import S_IFREG
+        from stream_zip import async_stream_zip, ZIP_64
+        if top_url.startswith('/'):
+            top_url = top_url[1:]
+        
+        if op_user:
+            if await check_path_permission(top_url, op_user) < AccessLevel.READ:
+                raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot zip path {top_url}")
+        
+        # https://stream-zip.docs.trade.gov.uk/async-interface/
+        async def data_iter():
+            async for (r, blob) in self.iter_path(top_url, None):
+                rel_path = r.url[len(top_url):]
+                rel_path = decode_uri_compnents(rel_path)
+                b_iter: AsyncIterable[bytes]
+                if isinstance(blob, bytes):
+                    async def blob_iter(): yield blob
+                    b_iter = blob_iter()    # type: ignore
+                else:
+                    assert isinstance(blob, AsyncIterable)
+                    b_iter = blob
+                yield (
+                    rel_path, 
+                    datetime.datetime.now(), 
+                    S_IFREG | 0o600,
+                    ZIP_64, 
+                    b_iter
+                )
+        return async_stream_zip(data_iter())
 
     @concurrent_wrap()
-    async def zip_path(self, top_url: str, urls: Optional[list[str]]) -> io.BytesIO:
+    async def zip_path(self, top_url: str, op_user: Optional[UserRecord]) -> io.BytesIO:
         if top_url.startswith('/'):
             top_url = top_url[1:]
         buffer = io.BytesIO()
         with zipfile.ZipFile(buffer, 'w') as zf:
-            async for (r, blob) in self.iter_path(top_url, urls):
+            async for (r, blob) in self.iter_path(top_url, None):
                 rel_path = r.url[len(top_url):]
                 rel_path = decode_uri_compnents(rel_path)
                 if r.external:

lfss/eng/utils.py

@@ -20,7 +20,7 @@ async def copy_file(source: str|pathlib.Path, destination: str|pathlib.Path):
                 await dest.write(chunk)
 
 def hash_credential(username: str, password: str):
-    return hashlib.sha256((username + password).encode()).hexdigest()
+    return hashlib.sha256(f"{username}:{password}".encode()).hexdigest()
 
 def encode_uri_compnents(path: str):
     path_sp = path.split("/")
@@ -155,7 +155,7 @@ def fmt_storage_size(size: int) -> str:
     return f"{size/1024**4:.2f}T"
 
 _FnReturnT = TypeVar('_FnReturnT')
-_AsyncReturnT = Awaitable[_FnReturnT]
+_AsyncReturnT = TypeVar('_AsyncReturnT', bound=Awaitable)
 _g_executor = None
 def get_global_executor():
     global _g_executor
@@ -179,7 +179,7 @@ def concurrent_wrap(executor=None):
         def sync_fn(*args, **kwargs):
             loop = asyncio.new_event_loop()
             return loop.run_until_complete(func(*args, **kwargs))
-        return sync_fn
+        return sync_fn          # type: ignore
     return _concurrent_wrap
 
 # https://stackoverflow.com/a/279586/6775765

lfss/svc/app_base.py

@@ -27,7 +27,7 @@ req_conn = RequestDB()
 async def lifespan(app: FastAPI):
     global db
     try:
-        await global_connection_init(n_read = 2)
+        await global_connection_init(n_read = 8 if not DEBUG_MODE else 1)
         await asyncio.gather(db.init(), req_conn.init())
         yield
         await req_conn.commit()

lfss/svc/app_native.py

@@ -1,14 +1,14 @@
 from typing import Optional, Literal
 
 from fastapi import Depends, Request, Response, UploadFile
+from fastapi.responses import StreamingResponse
 from fastapi.exceptions import HTTPException 
 
-from ..eng.config import MAX_BUNDLE_BYTES
 from ..eng.utils import ensure_uri_compnents
 from ..eng.connection_pool import unique_cursor
 from ..eng.database import check_file_read_permission, check_path_permission, UserConn, FileConn
 from ..eng.datatype import (
-    FileReadPermission, FileRecord, UserRecord, AccessLevel, 
+    FileReadPermission, UserRecord, AccessLevel, 
     FileSortKey, DirSortKey
 )
 
@@ -81,46 +81,23 @@ async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
 async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
     logger.info(f"GET bundle({path}), user: {user.username}")
     path = ensure_uri_compnents(path)
-    assert path.endswith("/") or path == ""
-
-    if not path == "" and path[0] == "/":   # adapt to both /path and path
+    if not path.endswith("/"):
+        raise HTTPException(status_code=400, detail="Path must end with /")
+    if path[0] == "/":      # adapt to both /path and path
         path = path[1:]
+    if path == "":
+        raise HTTPException(status_code=400, detail="Cannot bundle root")
     
-    # TODO: may check peer users here
-    owner_records_cache: dict[int, UserRecord] = {}     # cache owner records, ID -> UserRecord
-    async def is_access_granted(file_record: FileRecord):
-        owner_id = file_record.owner_id
-        owner = owner_records_cache.get(owner_id, None)
-        if owner is None:
-            async with unique_cursor() as conn:
-                uconn = UserConn(conn)
-                owner = await uconn.get_user_by_id(owner_id, throw=True)
-            owner_records_cache[owner_id] = owner
-            
-        allow_access, _ = check_file_read_permission(user, owner, file_record)
-        return allow_access
-    
-    async with unique_cursor() as conn:
-        fconn = FileConn(conn)
-        files = await fconn.list_path_files(
-            url = path, flat = True, 
-            limit=(await fconn.count_path_files(url = path, flat = True))
-            )
-    files = [f for f in files if await is_access_granted(f)]
-    if len(files) == 0:
-        raise HTTPException(status_code=404, detail="No files found")
-
-    # return bundle of files
-    total_size = sum([f.file_size for f in files])
-    if total_size > MAX_BUNDLE_BYTES:
-        raise HTTPException(status_code=400, detail="Too large to zip")
-
-    file_paths = [f.url for f in files]
-    zip_buffer = await db.zip_path(path, file_paths)
-    return Response(
-        content=zip_buffer.getvalue(), media_type="application/zip", headers={
-            "Content-Disposition": f"attachment; filename=bundle.zip", 
-            "Content-Length": str(zip_buffer.getbuffer().nbytes)
+    async with unique_cursor() as cur:
+        dir_record = await FileConn(cur).get_path_record(path)
+
+    pathname = f"{path.split('/')[-2]}"
+    return StreamingResponse(
+        content = await db.zip_path_stream(path, op_user=user),
+        media_type = "application/zip",
+        headers = {
+            f"Content-Disposition": f"attachment; filename=bundle-{pathname}.zip",
+            "X-Content-Bytes": str(dir_record.size),
         }
     )
 

{lfss-0.9.4.dist-info → lfss-0.9.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.9.4
+Version: 0.9.5
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li_mengxun
@@ -17,11 +17,12 @@ Requires-Dist: mimesniff (==1.*)
 Requires-Dist: pillow
 Requires-Dist: python-multipart
 Requires-Dist: requests (==2.*)
+Requires-Dist: stream-zip (==0.*)
 Requires-Dist: uvicorn (==0.*)
 Project-URL: Repository, https://github.com/MenxLi/lfss
 Description-Content-Type: text/markdown
 
-# Lightweight File Storage Service (LFSS)
+# Lite File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
 My experiment on a lightweight and high-performance file/object storage service...  
@@ -55,8 +56,8 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
 The authentication can be acheived through one of the following methods:
-1. `Authorization` header with the value `Bearer sha256(<username><password>)`.
-2. `token` query parameter with the value `sha256(<username><password>)`.
+1. `Authorization` header with the value `Bearer sha256(<username>:<password>)`.
+2. `token` query parameter with the value `sha256(<username>:<password>)`.
 3. HTTP Basic Authentication with the username and password (If WebDAV is enabled).
 
 You can refer to `frontend` as an application example, `lfss/api/connector.py` for more APIs. 

{lfss-0.9.4.dist-info → lfss-0.9.5.dist-info}/RECORD

@@ -1,6 +1,6 @@
-Readme.md,sha256=JVe9T6N1Rz4hTiiCVoDYe2VB0dAi60VcBgb2twQdfZc,1834
-docs/Changelog.md,sha256=3mRHcda4UK8c105XtBfbeTWij0S4xNc-U8JTTPUqCJk,769
-docs/Enviroment_variables.md,sha256=LUZF1o70emp-5UPsvXPjcxapP940OqEZzSyyUUT9bEQ,569
+Readme.md,sha256=ST2E12DJVlKTReiJjRBc7KZyAr8KyqlcK2BoTc_Peaw,1829
+docs/Changelog.md,sha256=QYej_hmGnv9t8wjFHXBvmrBOvY7aACZ82oa5SVkIyzM,882
+docs/Enviroment_variables.md,sha256=xaL8qBwT8B2Qe11FaOU3xWrRCh1mJ1VyTFCeFbkd0rs,570
 docs/Known_issues.md,sha256=ZqETcWP8lzTOel9b2mxEgCnADFF8IxOrEtiVO1NoMAk,251
 docs/Permission.md,sha256=mvK8gVBBgoIFJqikcaReU_bUo-mTq_ECqJaDDJoQF7Q,3126
 docs/Webdav.md,sha256=-Ja-BTWSY1BEMAyZycvEMNnkNTPZ49gSPzmf3Lbib70,1547
@@ -19,7 +19,7 @@ frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
 frontend/thumb.js,sha256=46ViD2TlTTWy0fx6wjoAs_5CQ4ajYB90vVzM7UO2IHw,6182
 frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
 lfss/api/__init__.py,sha256=8IJqrpWK1doIyVVbntvVic82A57ncwl5b0BRHX4Ri6A,6660
-lfss/api/connector.py,sha256=hHSEEWecKQGZH6oxAmYoG3q7lFfacCbOKVZiUIXT2y8,11819
+lfss/api/connector.py,sha256=0iopAvqUiUJDjbAtAjr9ynmURnmB-Ejg3INL-877s_E,12192
 lfss/cli/__init__.py,sha256=lPwPmqpa7EXQ4zlU7E7LOe6X2kw_xATGdwoHphUEirA,827
 lfss/cli/balance.py,sha256=fUbKKAUyaDn74f7mmxMfBL4Q4voyBLHu6Lg_g8GfMOQ,4121
 lfss/cli/cli.py,sha256=aYjB8d4k6JUd9efxZK-XOj-mlG4JeOr_0lnj2qqCiK0,8066
@@ -29,23 +29,23 @@ lfss/cli/user.py,sha256=1mTroQbaKxHjFCPHT67xwd08v-zxH0RZ_OnVc-4MzL0,5364
 lfss/cli/vacuum.py,sha256=GOG72d3NYe9bYCNc3y8JecEmM-DrKlGq3JQcisv_xBg,3702
 lfss/eng/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lfss/eng/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
-lfss/eng/config.py,sha256=DmnUYMeLOL-45OstysyMpSBPmLofgzvcSrsWjHvssYs,915
+lfss/eng/config.py,sha256=Vni6h52Ce0njVrHZLAWFL8g34YDBdlmGrmRhpxElxQ8,868
 lfss/eng/connection_pool.py,sha256=4xOF1kXXGqCWeLX5ZVFALKjdY8N1VVAVSSTRfCzbj94,6141
-lfss/eng/database.py,sha256=2i8gbh1odOA09tS5VU9cUZy3poZUdCx3XX7UX7umtxw,47188
+lfss/eng/database.py,sha256=9KMV-VeD8Dgd6M9RfNfAKwF5gWm763eJpJ2g5zyn_uY,48691
 lfss/eng/datatype.py,sha256=27UB7-l9SICy5lAvKjdzpTL_GohZjzstQcr9PtAq7nM,2709
 lfss/eng/error.py,sha256=dAlQHXOnQcSkA2vTugJFSxcyDqoFlPucBoFpTZ7GI6w,654
 lfss/eng/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
 lfss/eng/thumb.py,sha256=x9jIHHU1tskmp-TavPPcxGpbmEjCp9gbH6ZlsEfqUxY,3383
-lfss/eng/utils.py,sha256=CYEQvPiM28k53hCJBE7N6O6a1xC_wvnP3KZx4DCnD0k,6723
+lfss/eng/utils.py,sha256=WYoXFFi5308UWtFC8VP792gpzrVbHZZHhP3PaFjxIEY,6770
 lfss/sql/init.sql,sha256=8LjHx0TBCkBD62xFfssSeHDqKYVQQJkZAg4rSm046f4,1496
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/svc/app.py,sha256=ftWCpepBx-gTSG7i-TB-IdinPPstAYYQjCgnTfeMZeI,219
-lfss/svc/app_base.py,sha256=BU_DndHW4sYiWUQcTis8iGljmUy8FHfZrzCkE0d1z-Y,6717
+lfss/svc/app_base.py,sha256=PgL5MNU3QTwgIJP8CflDi9YBZ-uo4hVf74ADyWTo9yg,6742
 lfss/svc/app_dav.py,sha256=D0KSgjtTktPjIhyIKG5eRmBdh5X8HYFYH151E6gzlbc,18245
-lfss/svc/app_native.py,sha256=6yBRJB8_p4RZgDVheDTv1ClBGc3etrQm94j1NiR4FUQ,9349
+lfss/svc/app_native.py,sha256=N2cidZ5sIS0p9HOkPqWvpGkqe4bIA7CgGEp4CsvgZ6Q,8347
 lfss/svc/common_impl.py,sha256=0fjbqHWgqDhLfBEu6aC0Z5qgNt67C7z0Qroj7aV3Iq4,13830
 lfss/svc/request_log.py,sha256=v8yXEIzPjaksu76Oh5vgdbUEUrw8Kt4etLAXBWSGie8,3207
-lfss-0.9.4.dist-info/METADATA,sha256=3wUuwMRn55Z2lnX9wZRGMVxLbfphSLOk1gX01haFaOw,2594
-lfss-0.9.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.9.4.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
-lfss-0.9.4.dist-info/RECORD,,
+lfss-0.9.5.dist-info/METADATA,sha256=fToo-wrY0_XJPbk3cte8hb_4DClyPFv3ZTzJB77G-5w,2623
+lfss-0.9.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.9.5.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
+lfss-0.9.5.dist-info/RECORD,,