lfss 0.2.4__tar.gz → 0.3.0__tar.gz

{lfss-0.2.4 → lfss-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.2.4
+Version: 0.3.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
@@ -24,7 +24,7 @@ A lightweight file/object storage service!
 
 Usage: 
 ```sh
-pip install .
+pip install lfss
 lfss-user add <username> <password>
 lfss-serve
 ```
@@ -32,15 +32,15 @@ lfss-serve
 By default, the data will be stored in `.storage_data`. 
 You can change storage directory using the `LFSS_DATA` environment variable.
 
-I provide a simple client to interact with the service.  
-Just start a web server at `/frontend` and open `index.html` in your browser, or use:
+I provide a simple client to interact with the service: 
 ```sh
-lfss-panel
+lfss-panel --open
 ```
+Or, you can start a web server at `/frontend` and open `index.html` in your browser. 
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
-Authentication is done via `Authorization` header, with the value `Bearer <token>`.  
-You can refer to `frontend` as an application example, and `frontend/api.js` for the API usage.
+Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
+You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss.client.api.py` for the API usage.
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

{lfss-0.2.4 → lfss-0.3.0}/Readme.md

@@ -5,7 +5,7 @@ A lightweight file/object storage service!
 
 Usage: 
 ```sh
-pip install .
+pip install lfss
 lfss-user add <username> <password>
 lfss-serve
 ```
@@ -13,15 +13,15 @@ lfss-serve
 By default, the data will be stored in `.storage_data`. 
 You can change storage directory using the `LFSS_DATA` environment variable.
 
-I provide a simple client to interact with the service.  
-Just start a web server at `/frontend` and open `index.html` in your browser, or use:
+I provide a simple client to interact with the service: 
 ```sh
-lfss-panel
+lfss-panel --open
 ```
+Or, you can start a web server at `/frontend` and open `index.html` in your browser. 
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
-Authentication is done via `Authorization` header, with the value `Bearer <token>`.  
-You can refer to `frontend` as an application example, and `frontend/api.js` for the API usage.
+Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
+You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss.client.api.py` for the API usage.
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

{lfss-0.2.4 → lfss-0.3.0}/frontend/scripts.js

@@ -129,6 +129,9 @@ uploadButton.addEventListener('click', () => {
             refreshFileList();
             uploadFileNameInput.value = '';
             onFileNameInpuChange();
+        }, 
+        (err) => {
+            showPopup('Failed to upload file: ' + err, {level: 'error', timeout: 5000});
         }
     );
 });
@@ -401,7 +404,7 @@ function refreshFileList(){
 
                     const downloadBtn = document.createElement('a');
                     downloadBtn.textContent = 'Download';
-                    downloadBtn.href = conn.config.endpoint + '/' + file.url + '?asfile=true&token=' + conn.config.token;
+                    downloadBtn.href = conn.config.endpoint + '/' + file.url + '?download=true&token=' + conn.config.token;
                     actContainer.appendChild(downloadBtn);
 
                     const deleteButton = document.createElement('a');

lfss-0.3.0/lfss/client/api.py

@@ -0,0 +1,91 @@
+from typing import Optional, Literal
+import os
+import requests
+import urllib.parse
+from lfss.src.database import (
+    FileReadPermission, FileDBRecord, DBUserRecord, PathContents
+    )
+
+_default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
+_default_token = os.environ.get('LFSS_TOKEN', '')
+
+class Connector:
+    def __init__(self, endpoint=_default_endpoint, token=_default_token):
+        assert token, "No token provided. Please set LFSS_TOKEN environment variable."
+        self.config = {
+            "endpoint": endpoint,
+            "token": token
+        }
+    
+    def _fetch(
+        self, method: Literal['GET', 'POST', 'PUT', 'DELETE'], 
+        path: str, search_params: dict = {}
+    ):
+        if path.startswith('/'):
+            path = path[1:]
+        def f(**kwargs):
+            url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params)
+            headers: dict = kwargs.pop('headers', {})
+            headers.update({
+                'Authorization': f"Bearer {self.config['token']}",
+            })
+            response = requests.request(method, url, headers=headers, **kwargs)
+            response.raise_for_status()
+            return response
+        return f
+
+    def put(self, path: str, file_data: bytes):
+        """Uploads a file to the specified path."""
+        response = self._fetch('PUT', path)(
+            data=file_data, 
+            headers={'Content-Type': 'application/octet-stream'}
+        )
+        return response.json()
+    
+    def get(self, path: str) -> Optional[bytes]:
+        """Downloads a file from the specified path."""
+        try:
+            response = self._fetch('GET', path)()
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                return None
+            raise e
+        return response.content
+    
+    def delete(self, path: str):
+        """Deletes the file at the specified path."""
+        if path.startswith('/'):
+            path = path[1:]
+        self._fetch('DELETE', path)()
+    
+    def get_metadata(self, path: str) -> Optional[FileDBRecord]:
+        """Gets the metadata for the file at the specified path."""
+        try:
+            response = self._fetch('GET', '_api/fmeta', {'path': path})()
+            return FileDBRecord(**response.json())
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                return None
+            raise e
+    
+    def list_path(self, path: str) -> PathContents:
+        assert path.endswith('/')
+        response = self._fetch('GET', path)()
+        return PathContents(**response.json())
+
+    def set_file_permission(self, path: str, permission: int | FileReadPermission):
+        """Sets the file permission for the specified path."""
+        self._fetch('POST', '_api/fmeta', {'path': path, 'perm': int(permission)})(
+            headers={'Content-Type': 'application/www-form-urlencoded'}
+        )
+        
+    def move_file(self, path: str, new_path: str):
+        """Moves a file to a new location."""
+        self._fetch('POST', '_api/fmeta', {'path': path, 'new_path': new_path})(
+            headers = {'Content-Type': 'application/www-form-urlencoded'}
+        )
+        
+    def whoami(self) -> DBUserRecord:
+        """Gets information about the current user."""
+        response = self._fetch('GET', '_api/whoami')()
+        return DBUserRecord(**response.json())

{lfss-0.2.4 → lfss-0.3.0}/lfss/src/config.py

@@ -8,4 +8,5 @@ if not DATA_HOME.exists():
     DATA_HOME.mkdir()
     print(f"[init] Created data home at {DATA_HOME}")
 
-MAX_BUNDLE_BYTES = 128 * 1024 * 1024   # 128MB
+MAX_FILE_BYTES = 256 * 1024 * 1024   # 256MB
+MAX_BUNDLE_BYTES = 256 * 1024 * 1024   # 256MB

{lfss-0.2.4 → lfss-0.3.0}/lfss/src/database.py

@@ -24,10 +24,7 @@ def hash_credential(username, password):
 
 _atomic_lock = Lock()
 def atomic(func):
-    """
-    Ensure non-reentrancy. 
-    Can be skipped if the function only executes a single SQL statement.
-    """
+    """ Ensure non-reentrancy """
     @wraps(func)
     async def wrapper(*args, **kwargs):
         async with _atomic_lock:
@@ -175,9 +172,11 @@ class UserConn(DBConnBase):
             async for record in cursor:
                 yield self.parse_record(record)
     
+    @atomic
     async def set_active(self, username: str):
         await self.conn.execute("UPDATE user SET last_active = CURRENT_TIMESTAMP WHERE username = ?", (username, ))
     
+    @atomic
     async def delete_user(self, username: str):
         await self.conn.execute("DELETE FROM user WHERE username = ?", (username, ))
         self.logger.info(f"Delete user {username}")
@@ -203,6 +202,11 @@ class DirectoryRecord:
 
     def __str__(self):
         return f"Directory {self.url} (size={self.size})"
+
+@dataclasses.dataclass
+class PathContents:
+    dirs: list[DirectoryRecord]
+    files: list[FileDBRecord]
     
 class FileConn(DBConnBase):
 
@@ -251,7 +255,7 @@ class FileConn(DBConnBase):
                 async with self.conn.execute("SELECT SUM(file_size) FROM fmeta WHERE owner_id = ?", (r[0], )) as cursor:
                     size = await cursor.fetchone()
                 if size is not None and size[0] is not None:
-                    await self.user_size_inc(r[0], size[0])
+                    await self._user_size_inc(r[0], size[0])
 
         return self
     
@@ -299,9 +303,9 @@ class FileConn(DBConnBase):
     @overload
     async def list_path(self, url: str, flat: Literal[True]) -> list[FileDBRecord]:...
     @overload
-    async def list_path(self, url: str, flat: Literal[False]) -> tuple[list[DirectoryRecord], list[FileDBRecord]]:...
+    async def list_path(self, url: str, flat: Literal[False]) -> PathContents:...
     
-    async def list_path(self, url: str, flat: bool = False) -> list[FileDBRecord] | tuple[list[DirectoryRecord], list[FileDBRecord]]:
+    async def list_path(self, url: str, flat: bool = False) -> list[FileDBRecord] | PathContents:
         """
         List all files and directories under the given path, 
         if flat is True, return a list of FileDBRecord, recursively including all subdirectories. 
@@ -318,7 +322,7 @@ class FileConn(DBConnBase):
                 return [self.parse_record(r) for r in res]
 
             else:
-                return (await self.list_root(), [])
+                return PathContents(await self.list_root(), [])
         
         if flat:
             async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', )) as cursor:
@@ -346,7 +350,7 @@ class FileConn(DBConnBase):
         dirs_str = [r[0] + '/' for r in res if r[0] != '/']
         dirs = [DirectoryRecord(url + d, await self.path_size(url + d, include_subpath=True)) for d in dirs_str]
             
-        return (dirs, files)
+        return PathContents(dirs, files)
     
     async def user_size(self, user_id: int) -> int:
         async with self.conn.execute("SELECT size FROM usize WHERE user_id = ?", (user_id, )) as cursor:
@@ -354,10 +358,10 @@ class FileConn(DBConnBase):
         if res is None:
             return -1
         return res[0]
-    async def user_size_inc(self, user_id: int, inc: int):
+    async def _user_size_inc(self, user_id: int, inc: int):
         self.logger.debug(f"Increasing user {user_id} size by {inc}")
         await self.conn.execute("INSERT OR REPLACE INTO usize (user_id, size) VALUES (?, COALESCE((SELECT size FROM usize WHERE user_id = ?), 0) + ?)", (user_id, user_id, inc))
-    async def user_size_dec(self, user_id: int, dec: int):
+    async def _user_size_dec(self, user_id: int, dec: int):
         self.logger.debug(f"Decreasing user {user_id} size by {dec}")
         await self.conn.execute("INSERT OR REPLACE INTO usize (user_id, size) VALUES (?, COALESCE((SELECT size FROM usize WHERE user_id = ?), 0) - ?)", (user_id, user_id, dec))
     
@@ -406,7 +410,7 @@ class FileConn(DBConnBase):
                 "INSERT INTO fmeta (url, owner_id, file_id, file_size, permission) VALUES (?, ?, ?, ?, ?)", 
                 (url, owner_id, file_id, file_size, int(permission))
                 )
-            await self.user_size_inc(owner_id, file_size)
+            await self._user_size_inc(owner_id, file_size)
             self.logger.info(f"File {url} created")
     
     @atomic
@@ -428,7 +432,7 @@ class FileConn(DBConnBase):
         file_record = await self.get_file_record(url)
         if file_record is None: return
         await self.conn.execute("DELETE FROM fmeta WHERE url = ?", (url, ))
-        await self.user_size_dec(file_record.owner_id, file_record.file_size)
+        await self._user_size_dec(file_record.owner_id, file_record.file_size)
         self.logger.info(f"Deleted fmeta {url}")
     
     @atomic
@@ -452,11 +456,12 @@ class FileConn(DBConnBase):
                 async with self.conn.execute("SELECT SUM(file_size) FROM fmeta WHERE owner_id = ? AND url LIKE ?", (r[0], path + '%')) as cursor:
                     size = await cursor.fetchone()
                 if size is not None:
-                    await self.user_size_dec(r[0], size[0])
+                    await self._user_size_dec(r[0], size[0])
         
         await self.conn.execute("DELETE FROM fmeta WHERE url LIKE ?", (path + '%', ))
         self.logger.info(f"Deleted {len(all_f_rec)} files for path {path}") # type: ignore
     
+    @atomic
     async def set_file_blob(self, file_id: str, blob: bytes):
         await self.conn.execute("INSERT OR REPLACE INTO fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
     
@@ -467,9 +472,11 @@ class FileConn(DBConnBase):
             return None
         return res[0]
     
+    @atomic
     async def delete_file_blob(self, file_id: str):
         await self.conn.execute("DELETE FROM fdata WHERE file_id = ?", (file_id, ))
     
+    @atomic
     async def delete_file_blobs(self, file_ids: list[str]):
         await self.conn.execute("DELETE FROM fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
 

{lfss-0.2.4 → lfss-0.3.0}/lfss/src/server.py

@@ -13,7 +13,7 @@ from contextlib import asynccontextmanager
 
 from .error import *
 from .log import get_logger
-from .config import MAX_BUNDLE_BYTES
+from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES
 from .utils import ensure_uri_compnents
 from .database import Database, DBUserRecord, DECOY_USER, FileDBRecord, check_user_permission, FileReadPermission
 
@@ -78,7 +78,7 @@ app.add_middleware(
 router_fs = APIRouter(prefix="")
 
 @router_fs.get("/{path:path}")
-async def get_file(path: str, asfile = False, user: DBUserRecord = Depends(get_current_user)):
+async def get_file(path: str, download = False, user: DBUserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
 
     # handle directory query
@@ -97,11 +97,7 @@ async def get_file(path: str, asfile = False, user: DBUserRecord = Depends(get_c
         if not path.startswith(f"{user.username}/") and not user.is_admin:
             raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
 
-        dirs, files = await conn.file.list_path(path, flat = False)
-        return {
-            "dirs": dirs,
-            "files": files
-        }
+        return await conn.file.list_path(path, flat = False)
     
     file_record = await conn.file.get_file_record(path)
     if not file_record:
@@ -128,7 +124,7 @@ async def get_file(path: str, asfile = False, user: DBUserRecord = Depends(get_c
             }
         )
     
-    if asfile:
+    if download:
         return await send('application/octet-stream', "attachment")
     else:
         return await send(None, "inline")
@@ -143,6 +139,13 @@ async def put_file(request: Request, path: str, user: DBUserRecord = Depends(get
         logger.debug(f"Reject put request from {user.username} to {path}")
         raise HTTPException(status_code=403, detail="Permission denied")
     
+    content_length = request.headers.get("Content-Length")
+    if content_length is not None:
+        content_length = int(content_length)
+        if content_length > MAX_FILE_BYTES:
+            logger.debug(f"Reject put request from {user.username} to {path}, file too large")
+            raise HTTPException(status_code=413, detail="File too large")
+    
     logger.info(f"PUT {path}, user: {user.username}")
     exists_flag = False
     file_record = await conn.file.get_file_record(path)

{lfss-0.2.4 → lfss-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.2.4"
+version = "0.3.0"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"