lfss 0.7.2__tar.gz → 0.7.3__tar.gz

{lfss-0.7.2 → lfss-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.2
+Version: 0.7.3
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
@@ -22,8 +22,8 @@ Description-Content-Type: text/markdown
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
 My experiment on a lightweight file/object storage service.  
-It stores small files and metadata in sqlite, large files in the filesystem, and serves them through a simple REST API.  
-Tested on 2 million files, and it works fine... thanks to the sqlite database!
+It stores small files and metadata in sqlite, large files in the filesystem.  
+Tested on 2 million files, and it works fine... 
 
 Usage: 
 ```sh

{lfss-0.7.2 → lfss-0.7.3}/Readme.md

@@ -2,8 +2,8 @@
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
 My experiment on a lightweight file/object storage service.  
-It stores small files and metadata in sqlite, large files in the filesystem, and serves them through a simple REST API.  
-Tested on 2 million files, and it works fine... thanks to the sqlite database!
+It stores small files and metadata in sqlite, large files in the filesystem.  
+Tested on 2 million files, and it works fine... 
 
 Usage: 
 ```sh

{lfss-0.7.2 → lfss-0.7.3}/docs/Permission.md

@@ -1,22 +1,12 @@
 
 # Permission System
-There are two roles in the system: Admin and User (you can treat then as buckets).  
+There are two roles in the system: Admin and User ('user' are like 'bucket' to some extent).
 
-## `PUT` and `DELETE` permissions
-Non-login user don't have `PUT/DELETE` permissions.  
-Every user can have `PUT/DELETE` permissions of files under its own `/<user>/` path.  
-The admin can have `PUT/DELETE` permissions of files of all users.
+## File access with `GET` permission
+The `GET` is used to access the file (if path is not ending with `/`), or to list the files under a path (if path is ending with `/`).  
 
-## `GET` permissions
-The `GET` is used to access the file (if path is not ending with `/`), or to list the files under a path (if path is ending with `/`). 
-
-### Path-listing
-- Non-login users cannot list any files.
-- All users can list the files under their own path 
-- Admins can list the files under other users' path. 
-
-### File-access
-For accessing file content, the user must have `GET` permission of the file, which is determined by the `permission` field of both the owner and the file. 
+### File access
+For accessing file content, the user must have `GET` permission of the file, which is determined by the `permission` field of both the owner and the file.   
 (Note: The owner of the file is the user who created the file, may not necessarily be the user under whose path the file is stored.)   
 
 There are four types of permissions: `unset`, `public`, `protected`, `private`.
@@ -26,4 +16,22 @@ Non-admin users can access files based on:
 - If the file is `protected`, then only the logged-in user can access it.  
 - If the file is `private`, then only the owner can access it.
 - If the file is `unset`, then the file's permission is inherited from the owner's permission.
-- If both the owner and the file have `unset` permission, then the file is `public`.
+- If both the owner and the file have `unset` permission, then the file is `public`.
+
+### Path-listing
+- Non-login users cannot list any files.
+- All users can list the files under their own path 
+- Admins can list the files under other users' path. 
+
+## File creation with `PUT` permission
+The `PUT` is used to create a file. 
+- Non-login user don't have `PUT` permission.  
+- Every user can have `PUT` permission of files under its own `/<user>/` path.  
+- The admin can have `PUT` permission of files of all users.
+
+## `DELETE` and moving permissions
+- Non-login user don't have `DELETE`/move permission.
+- Every user can have `DELETE`/move permission that they own.
+- The admin can have `DELETE` permission of files of all users
+(The admin can't move files of other users, because move does not change the owner of the file. 
+If move is allowed, then its equivalent to create file on behalf of other users.)

{lfss-0.7.2 → lfss-0.7.3}/lfss/cli/cli.py

@@ -1,5 +1,5 @@
 from lfss.client import Connector, upload_directory
-from lfss.src.database import FileReadPermission
+from lfss.src.datatype import FileReadPermission
 from pathlib import Path
 import argparse
 

{lfss-0.7.2 → lfss-0.7.3}/lfss/cli/user.py

@@ -45,6 +45,7 @@ async def _main():
     sp_list.add_argument("-l", "--long", action="store_true")
     
     args = parser.parse_args()
+    db = await Database().init()
 
     @asynccontextmanager
     async def get_uconn():
@@ -65,7 +66,6 @@ async def _main():
             print('User not found')
             exit(1)
         else:
-            db = await Database().init()
             await db.delete_user(user.id)
         print('User deleted')
     

{lfss-0.7.2 → lfss-0.7.3}/lfss/client/api.py

@@ -17,7 +17,7 @@ class Connector:
             "token": token
         }
     
-    def _fetch(
+    def _fetch_factory(
         self, method: Literal['GET', 'POST', 'PUT', 'DELETE'], 
         path: str, search_params: dict = {}
     ):
@@ -46,7 +46,7 @@ class Connector:
             else:
                 return {'status': 'skipped', 'path': path}
 
-        response = self._fetch('PUT', path, search_params={
+        response = self._fetch_factory('PUT', path, search_params={
             'permission': int(permission),
             'conflict': conflict
             })(
@@ -68,7 +68,7 @@ class Connector:
             else:
                 return {'status': 'skipped', 'path': path}
 
-        response = self._fetch('PUT', path, search_params={
+        response = self._fetch_factory('PUT', path, search_params={
             'permission': int(permission),
             'conflict': conflict
             })(
@@ -79,7 +79,7 @@ class Connector:
     
     def _get(self, path: str) -> Optional[requests.Response]:
         try:
-            response = self._fetch('GET', path)()
+            response = self._fetch_factory('GET', path)()
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 404:
                 return None
@@ -100,12 +100,12 @@ class Connector:
     
     def delete(self, path: str):
         """Deletes the file at the specified path."""
-        self._fetch('DELETE', path)()
+        self._fetch_factory('DELETE', path)()
     
     def get_metadata(self, path: str) -> Optional[FileRecord | DirectoryRecord]:
         """Gets the metadata for the file at the specified path."""
         try:
-            response = self._fetch('GET', '_api/meta', {'path': path})()
+            response = self._fetch_factory('GET', '_api/meta', {'path': path})()
             if path.endswith('/'):
                 return DirectoryRecord(**response.json())
             else:
@@ -117,22 +117,24 @@ class Connector:
     
     def list_path(self, path: str) -> PathContents:
         assert path.endswith('/')
-        response = self._fetch('GET', path)()
-        return PathContents(**response.json())
+        response = self._fetch_factory('GET', path)()
+        dirs = [DirectoryRecord(**d) for d in response.json()['dirs']]
+        files = [FileRecord(**f) for f in response.json()['files']]
+        return PathContents(dirs=dirs, files=files)
 
     def set_file_permission(self, path: str, permission: int | FileReadPermission):
         """Sets the file permission for the specified path."""
-        self._fetch('POST', '_api/meta', {'path': path, 'perm': int(permission)})(
+        self._fetch_factory('POST', '_api/meta', {'path': path, 'perm': int(permission)})(
             headers={'Content-Type': 'application/www-form-urlencoded'}
         )
         
     def move(self, path: str, new_path: str):
         """Move file or directory to a new path."""
-        self._fetch('POST', '_api/meta', {'path': path, 'new_path': new_path})(
+        self._fetch_factory('POST', '_api/meta', {'path': path, 'new_path': new_path})(
             headers = {'Content-Type': 'application/www-form-urlencoded'}
         )
         
     def whoami(self) -> UserRecord:
         """Gets information about the current user."""
-        response = self._fetch('GET', '_api/whoami')()
+        response = self._fetch_factory('GET', '_api/whoami')()
         return UserRecord(**response.json())

{lfss-0.7.2 → lfss-0.7.3}/lfss/src/config.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-import os
+import os, hashlib
 
 __default_dir = '.storage_data'
 
@@ -14,4 +14,7 @@ LARGE_BLOB_DIR.mkdir(exist_ok=True)
 # https://sqlite.org/fasterthanfs.html
 LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
 MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
-MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
+MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
+
+def hash_credential(username, password):
+    return hashlib.sha256((username + password).encode()).hexdigest()

{lfss-0.7.2 → lfss-0.7.3}/lfss/src/connection_pool.py

@@ -139,6 +139,7 @@ async def unique_cursor(is_write: bool = False):
             finally:
                 await g_pool.release(connection_obj)
 
+# todo: add exclusive transaction option
 @asynccontextmanager
 async def transaction():
     async with unique_cursor(is_write=True) as cur:

{lfss-0.7.2 → lfss-0.7.3}/lfss/src/database.py

@@ -3,7 +3,7 @@ from typing import Optional, overload, Literal, AsyncIterable
 from abc import ABC
 
 import urllib.parse
-import hashlib, uuid
+import uuid
 import zipfile, io, asyncio
 
 import aiosqlite, aiofiles
@@ -11,14 +11,11 @@ import aiofiles.os
 
 from .connection_pool import execute_sql, unique_cursor, transaction
 from .datatype import UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents
-from .config import LARGE_BLOB_DIR
+from .config import LARGE_BLOB_DIR, hash_credential
 from .log import get_logger
 from .utils import decode_uri_compnents
 from .error import *
 
-def hash_credential(username, password):
-    return hashlib.sha256((username + password).encode()).hexdigest()
-
 class DBObjectBase(ABC):
     logger = get_logger('database', global_instance=True)
     _cur: aiosqlite.Cursor
@@ -153,16 +150,6 @@ class FileConn(DBObjectBase):
             return []
         return [self.parse_record(r) for r in res]
     
-    async def get_user_file_records(self, owner_id: int) -> list[FileRecord]:
-        await self.cur.execute("SELECT * FROM fmeta WHERE owner_id = ?", (owner_id, ))
-        res = await self.cur.fetchall()
-        return [self.parse_record(r) for r in res]
-    
-    async def get_path_file_records(self, url: str) -> list[FileRecord]:
-        await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', ))
-        res = await self.cur.fetchall()
-        return [self.parse_record(r) for r in res]
-
     async def list_root(self, *usernames: str) -> list[DirectoryRecord]:
         """
         Efficiently list users' directories, if usernames is empty, list all users' directories.
@@ -340,25 +327,27 @@ class FileConn(DBObjectBase):
     async def log_access(self, url: str):
         await self.cur.execute("UPDATE fmeta SET access_time = CURRENT_TIMESTAMP WHERE url = ?", (url, ))
     
-    async def delete_file_record(self, url: str):
-        file_record = await self.get_file_record(url)
-        if file_record is None: return
-        await self.cur.execute("DELETE FROM fmeta WHERE url = ?", (url, ))
+    async def delete_file_record(self, url: str) -> Optional[FileRecord]:
+        res = await self.cur.execute("DELETE FROM fmeta WHERE url = ? RETURNING *", (url, ))
+        row = await res.fetchone()
+        if row is None:
+            raise FileNotFoundError(f"File {url} not found")
+        file_record = FileRecord(*row)
         await self._user_size_dec(file_record.owner_id, file_record.file_size)
         self.logger.info(f"Deleted fmeta {url}")
+        return file_record
     
-    async def delete_user_file_records(self, owner_id: int):
+    async def delete_user_file_records(self, owner_id: int) -> list[FileRecord]:
         cursor = await self.cur.execute("SELECT * FROM fmeta WHERE owner_id = ?", (owner_id, ))
         res = await cursor.fetchall()
-        await self.cur.execute("DELETE FROM fmeta WHERE owner_id = ?", (owner_id, ))
         await self.cur.execute("DELETE FROM usize WHERE user_id = ?", (owner_id, ))
-        self.logger.info(f"Deleted {len(res)} files for user {owner_id}") # type: ignore
+        res = await self.cur.execute("DELETE FROM fmeta WHERE owner_id = ? RETURNING *", (owner_id, ))
+        ret = [self.parse_record(r) for r in await res.fetchall()]
+        self.logger.info(f"Deleted {len(ret)} file(s) for user {owner_id}") # type: ignore
+        return ret
     
-    async def delete_path_records(self, path: str):
+    async def delete_path_records(self, path: str, under_user_id: Optional[int] = None) -> list[FileRecord]:
         """Delete all records with url starting with path"""
-        cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (path + '%', ))
-        all_f_rec = await cursor.fetchall()
-        
         # update user size
         cursor = await self.cur.execute("SELECT DISTINCT owner_id FROM fmeta WHERE url LIKE ?", (path + '%', ))
         res = await cursor.fetchall()
@@ -368,8 +357,16 @@ class FileConn(DBObjectBase):
             if size is not None:
                 await self._user_size_dec(r[0], size[0])
         
-        await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ?", (path + '%', ))
-        self.logger.info(f"Deleted {len(all_f_rec)} files for path {path}") # type: ignore
+        # if any new records are created here, the size update may be inconsistent
+        # but it's not a big deal...
+        
+        if under_user_id is None:
+            res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? RETURNING *", (path + '%', ))
+        else:
+            res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? AND owner_id = ? RETURNING *", (path + '%', under_user_id))
+        all_f_rec = await res.fetchall()
+        self.logger.info(f"Deleted {len(all_f_rec)} file(s) for path {path}") # type: ignore
+        return [self.parse_record(r) for r in all_f_rec]
     
     async def set_file_blob(self, file_id: str, blob: bytes):
         await self.cur.execute("INSERT OR REPLACE INTO blobs.fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
@@ -546,28 +543,37 @@ class Database:
 
         return blob
 
-    async def delete_file(self, url: str) -> Optional[FileRecord]:
+    async def delete_file(self, url: str, assure_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
         validate_url(url)
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            r = await fconn.get_file_record(url)
+            r = await fconn.delete_file_record(url)
             if r is None:
                 return None
+            if assure_user is not None:
+                if r.owner_id != assure_user.id:
+                    # will rollback
+                    raise PermissionDeniedError(f"Permission denied: {assure_user.username} cannot delete file {url}")
             f_id = r.file_id
-            await fconn.delete_file_record(url)
             if r.external:
                 await fconn.delete_file_blob_external(f_id)
             else:
                 await fconn.delete_file_blob(f_id)
             return r
     
-    async def move_file(self, old_url: str, new_url: str):
+    async def move_file(self, old_url: str, new_url: str, ensure_user: Optional[UserRecord] = None):
         validate_url(old_url)
         validate_url(new_url)
 
         async with transaction() as cur:
             fconn = FileConn(cur)
+            r = await fconn.get_file_record(old_url)
+            if r is None:
+                raise FileNotFoundError(f"File {old_url} not found")
+            if ensure_user is not None:
+                if r.owner_id != ensure_user.id:
+                    raise PermissionDeniedError(f"Permission denied: {ensure_user.username} cannot move file {old_url}")
             await fconn.move_file(old_url, new_url)
     
     async def move_path(self, user: UserRecord, old_url: str, new_url: str):
@@ -615,16 +621,16 @@ class Database:
             await fconn.delete_file_blob_external(external_ids[i])
             
 
-    async def delete_path(self, url: str):
+    async def delete_path(self, url: str, under_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
         validate_url(url, is_file=False)
+        user_id = under_user.id if under_user is not None else None
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            records = await fconn.get_path_file_records(url)
+            records = await fconn.delete_path_records(url, user_id)
             if not records:
                 return None
             await self.__batch_delete_file_blobs(fconn, records)
-            await fconn.delete_path_records(url)
             return records
     
     async def delete_user(self, u: str | int):
@@ -633,12 +639,17 @@ class Database:
             if user is None:
                 return
 
-            fconn = FileConn(cur)
-            records = await fconn.get_user_file_records(user.id)
-            await self.__batch_delete_file_blobs(fconn, records)
-            await fconn.delete_user_file_records(user.id)
+            # no new files can be added since profile deletion
             uconn = UserConn(cur)
             await uconn.delete_user(user.username)
+
+            fconn = FileConn(cur)
+            records = await fconn.delete_user_file_records(user.id)
+            await self.__batch_delete_file_blobs(fconn, records)
+
+            # make sure the user's directory is deleted, 
+            # may contain admin's files, but delete them all
+            await fconn.delete_path_records(user.username + '/')
     
     async def iter_path(self, top_url: str, urls: Optional[list[str]]) -> AsyncIterable[tuple[FileRecord, bytes | AsyncIterable[bytes]]]:
         async with unique_cursor() as cur:

{lfss-0.7.2 → lfss-0.7.3}/lfss/src/server.py

@@ -76,6 +76,11 @@ async def get_current_user(
         raise HTTPException(status_code=401, detail="Invalid token")
     return user
 
+async def registered_user(user: UserRecord = Depends(get_current_user)):
+    if user.id == 0:
+        raise HTTPException(status_code=401, detail="Permission denied")
+    return user
+
 app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
 app.add_middleware(
     CORSMiddleware,
@@ -186,11 +191,8 @@ async def put_file(
     path: str, 
     conflict: Literal["overwrite", "skip", "abort"] = "abort",
     permission: int = 0,
-    user: UserRecord = Depends(get_current_user)):
+    user: UserRecord = Depends(registered_user)):
     path = ensure_uri_compnents(path)
-    if user.id == 0:
-        logger.debug("Reject put request from DECOY_USER")
-        raise HTTPException(status_code=401, detail="Permission denied")
     if not path.startswith(f"{user.username}/") and not user.is_admin:
         logger.debug(f"Reject put request from {user.username} to {path}")
         raise HTTPException(status_code=403, detail="Permission denied")
@@ -217,6 +219,8 @@ async def put_file(
             }, content=json.dumps({"url": path}))
         # remove the old file
         exists_flag = True
+        if not user.is_admin and not file_record.owner_id == user.id:
+            raise HTTPException(status_code=403, detail="Permission denied, cannot overwrite other's file")
         await db.delete_file(path)
     
     # check content-type
@@ -267,19 +271,17 @@ async def put_file(
 
 @router_fs.delete("/{path:path}")
 @handle_exception
-async def delete_file(path: str, user: UserRecord = Depends(get_current_user)):
+async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
     path = ensure_uri_compnents(path)
-    if user.id == 0:
-        raise HTTPException(status_code=401, detail="Permission denied")
     if not path.startswith(f"{user.username}/") and not user.is_admin:
         raise HTTPException(status_code=403, detail="Permission denied")
     
     logger.info(f"DELETE {path}, user: {user.username}")
 
     if path.endswith("/"):
-        res = await db.delete_path(path)
+        res = await db.delete_path(path, user if not user.is_admin else None)
     else:
-        res = await db.delete_file(path)
+        res = await db.delete_file(path, user if not user.is_admin else None)
 
     await db.record_user_activity(user.username)
     if res:
@@ -291,21 +293,23 @@ router_api = APIRouter(prefix="/_api")
 
 @router_api.get("/bundle")
 @handle_exception
-async def bundle_files(path: str, user: UserRecord = Depends(get_current_user)):
+async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
     logger.info(f"GET bundle({path}), user: {user.username}")
-    if user.id == 0:
-        raise HTTPException(status_code=401, detail="Permission denied")
     path = ensure_uri_compnents(path)
     assert path.endswith("/") or path == ""
 
     if not path == "" and path[0] == "/":   # adapt to both /path and path
         path = path[1:]
     
-    owner_records_cache = {}     # cache owner records, ID -> UserRecord
+    owner_records_cache: dict[int, UserRecord] = {}     # cache owner records, ID -> UserRecord
     async def is_access_granted(file_record: FileRecord):
         owner_id = file_record.owner_id
         owner = owner_records_cache.get(owner_id, None)
         if owner is None:
+            async with unique_cursor() as conn:
+                uconn = UserConn(conn)
+                owner = await uconn.get_user_by_id(owner_id)
+                assert owner is not None, "Owner not found"
             owner_records_cache[owner_id] = owner
             
         allow_access, _ = check_user_permission(user, owner, file_record)
@@ -334,7 +338,7 @@ async def bundle_files(path: str, user: UserRecord = Depends(get_current_user)):
 
 @router_api.get("/meta")
 @handle_exception
-async def get_file_meta(path: str, user: UserRecord = Depends(get_current_user)):
+async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
     logger.info(f"GET meta({path}), user: {user.username}")
     path = ensure_uri_compnents(path)
     async with unique_cursor() as conn:
@@ -352,10 +356,8 @@ async def update_file_meta(
     path: str, 
     perm: Optional[int] = None, 
     new_path: Optional[str] = None,
-    user: UserRecord = Depends(get_current_user)
+    user: UserRecord = Depends(registered_user)
     ):
-    if user.id == 0:
-        raise HTTPException(status_code=401, detail="Permission denied")
     path = ensure_uri_compnents(path)
     if path.startswith("/"):
         path = path[1:]
@@ -374,7 +376,7 @@ async def update_file_meta(
         if new_path is not None:
             new_path = ensure_uri_compnents(new_path)
             logger.info(f"Update path of {path} to {new_path}")
-            await db.move_file(path, new_path)
+            await db.move_file(path, new_path, user)
     
     # directory
     else:
@@ -389,9 +391,7 @@ async def update_file_meta(
     
 @router_api.get("/whoami")
 @handle_exception
-async def whoami(user: UserRecord = Depends(get_current_user)):
-    if user.id == 0:
-        raise HTTPException(status_code=401, detail="Login required")
+async def whoami(user: UserRecord = Depends(registered_user)):
     user.credential = "__HIDDEN__"
     return user
 

{lfss-0.7.2 → lfss-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.7.2"
+version = "0.7.3"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"
@@ -15,6 +15,10 @@ aiosqlite = "0.*"
 aiofiles = "23.*"
 mimesniff = "1.*"
 
+[tool.poetry.dev-dependencies]
+pytest = "*"
+pytest-html = "*"
+
 [tool.poetry.scripts]
 lfss-serve = "lfss.cli.serve:main"
 lfss-user = "lfss.cli.user:main"