lfss 0.1.0__tar.gz

lfss-0.1.0/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.1
+Name: lfss
+Version: 0.1.0
+Summary: Lightweight file storage service
+Home-page: https://github.com/MenxLi/lfss
+Author: li, mengxun
+Author-email: limengxun45@outlook.com
+Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: aiosqlite (==0.*)
+Requires-Dist: fastapi (==0.*)
+Requires-Dist: mimesniff (==1.*)
+Project-URL: Repository, https://github.com/MenxLi/lfss
+Description-Content-Type: text/markdown
+
+# Lightweight File Storage Service (LFSS)
+
+A lightweight file/object storage service!
+
+Usage: 
+```sh
+pip install .
+lfss-user add <username> <password>
+lfss-serve
+```
+
+By default, the data will be stored in the `.storage_data` directory, in a sqlite database.  
+The data storage can be set via environment variable `LFSS_DATA`.
+
+I provide a simple client to interact with the service.  
+Just start a web server at `/frontend` and open `index.html` in your browser.
+
+Currently, there is no file access-control, anyone can access any file with `GET` request.  
+However, the path-listing is only available to the authenticated user (to their own files, under `<username>/`).  
+
+The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
+Authentication is done via `Authorization` header, with the value `Bearer <token>`.  
+Please refer to `frontend` as an application example, and `frontend/api.js` for the API usage.

lfss-0.1.0/Readme.md

@@ -0,0 +1,23 @@
+# Lightweight File Storage Service (LFSS)
+
+A lightweight file/object storage service!
+
+Usage: 
+```sh
+pip install .
+lfss-user add <username> <password>
+lfss-serve
+```
+
+By default, the data will be stored in the `.storage_data` directory, in a sqlite database.  
+The data storage can be set via environment variable `LFSS_DATA`.
+
+I provide a simple client to interact with the service.  
+Just start a web server at `/frontend` and open `index.html` in your browser.
+
+Currently, there is no file access-control, anyone can access any file with `GET` request.  
+However, the path-listing is only available to the authenticated user (to their own files, under `<username>/`).  
+
+The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
+Authentication is done via `Authorization` header, with the value `Bearer <token>`.  
+Please refer to `frontend` as an application example, and `frontend/api.js` for the API usage.

lfss-0.1.0/lfss/cli/serve.py

@@ -0,0 +1,31 @@
+import argparse
+from uvicorn import Config, Server
+from uvicorn.config import LOGGING_CONFIG
+from ..src.server import *
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--host', type=str, default='0.0.0.0')
+    parser.add_argument('--port', type=int, default=8000)
+    parser.add_argument('--workers', type=int, default=None)
+    parser.add_argument('--enable-uvicorn-log', action='store_true')
+    args = parser.parse_args()
+
+    default_logging_config = LOGGING_CONFIG.copy()
+    if not args.enable_uvicorn_log:
+        default_logging_config["loggers"]["uvicorn"]["handlers"] = []
+
+    config = Config(
+        app=app,
+        host=args.host,
+        port=args.port,
+        access_log=False,
+        workers=args.workers,
+        log_config=default_logging_config
+    )
+    server = Server(config=config)
+    logger.info(f"Starting server at {args.host}:{args.port}, with {args.workers} workers")
+    server.run()
+
+if __name__ == "__main__":
+    main()

lfss-0.1.0/lfss/cli/user.py

@@ -0,0 +1,77 @@
+import argparse, asyncio
+from ..src.database import Database
+
+async def _main():
+    parser = argparse.ArgumentParser()
+    sp = parser.add_subparsers(dest='subparser_name', required=True)
+    sp_add = sp.add_parser('add')
+    sp_add.add_argument('username', type=str)
+    sp_add.add_argument('password', type=str)
+    sp_add.add_argument('--admin', action='store_true')
+    
+    sp_delete = sp.add_parser('delete')
+    sp_delete.add_argument('username', type=str)
+
+    def parse_bool(s):
+        if s.lower() == 'true':
+            return True
+        if s.lower() == 'false':
+            return False
+        raise ValueError('Not a boolean')
+    sp_set = sp.add_parser('set')
+    sp_set.add_argument('username', type=str)
+    sp_set.add_argument('-p', '--password', type=str, default=None)
+    sp_set.add_argument('-a', '--admin', type=parse_bool, default=None)
+    
+    sp_list = sp.add_parser('list')
+    sp_list.add_argument("-l", "--long", action="store_true")
+    
+    args = parser.parse_args()
+    conn = await Database().init()
+    
+    try:
+        if args.subparser_name == 'add':
+            await conn.user.create_user(args.username, args.password, args.admin)
+            user = await conn.user.get_user(args.username)
+            assert user is not None
+            print('User created, credential:', user.credential)
+        
+        if args.subparser_name == 'delete':
+            user = await conn.user.get_user(args.username)
+            if user is None:
+                print('User not found')
+                exit(1)
+            else:
+                await conn.delete_user(user.id)
+            print('User deleted')
+        
+        if args.subparser_name == 'set':
+            user = await conn.user.get_user(args.username)
+            if user is None:
+                print('User not found')
+                exit(1)
+            await conn.user.set_user(user.username, args.password, args.admin)
+            user = await conn.user.get_user(args.username)
+            assert user is not None
+            print('User updated, credential:', user.credential)
+        
+        if args.subparser_name == 'list':
+            async for user in conn.user.all():
+                print(user)
+                if args.long:
+                    print('  ', user.credential)
+        
+        await conn.commit()
+    
+    except Exception as e:
+        conn.logger.error(f'Error: {e}')
+        await conn.rollback()
+
+    finally:
+        await conn.close()
+
+def main():
+    asyncio.run(_main())
+
+if __name__ == '__main__':
+    main()

lfss-0.1.0/lfss/src/config.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+import os
+
+__default_dir = '.storage_data'
+
+DATA_HOME = Path(os.environ.get('LFSS_DATA', __default_dir))
+if not DATA_HOME.exists():
+    DATA_HOME.mkdir()
+    print(f"[init] Created data home at {DATA_HOME}")
+
+MAX_BUNDLE_BYTES = 128 * 1024 * 1024   # 128MB

lfss-0.1.0/lfss/src/database.py

@@ -0,0 +1,523 @@
+
+from typing import Optional, overload, Literal
+from abc import ABC, abstractmethod
+
+import urllib.parse
+import dataclasses, hashlib, uuid
+from contextlib import asynccontextmanager
+from enum import IntEnum
+import zipfile, io
+
+import aiosqlite
+from asyncio import Lock
+
+from .config import DATA_HOME
+from .log import get_logger
+from .utils import decode_uri_compnents
+
+_g_conn: Optional[aiosqlite.Connection] = None
+
+def hash_credential(username, password):
+    return hashlib.sha256((username + password).encode()).hexdigest()
+
+class DBConnBase(ABC):
+    logger = get_logger('database', global_instance=True)
+
+    @property
+    def conn(self)->aiosqlite.Connection:
+        global _g_conn
+        if _g_conn is None:
+            raise ValueError('Connection not initialized, did you forget to call super().init()?')
+        return _g_conn
+
+    @abstractmethod
+    async def init(self):
+        """Should return self"""
+        global _g_conn
+        if _g_conn is None:
+            _g_conn = await aiosqlite.connect(DATA_HOME / 'lfss.db')
+
+    async def commit(self):
+        await self.conn.commit()
+
+@dataclasses.dataclass
+class DBUserRecord:
+    id: int
+    username: str
+    credential: str
+    is_admin: bool
+    create_time: str
+    last_active: str
+
+    def __str__(self):
+        return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active})"
+
+DECOY_USER = DBUserRecord(0, 'decoy', 'decoy', False, '2021-01-01 00:00:00', '2021-01-01 00:00:00')
+class UserConn(DBConnBase):
+
+    @staticmethod
+    def parse_record(record) -> DBUserRecord:
+        return DBUserRecord(*record)
+
+    async def init(self):
+        await super().init()
+        await self.conn.execute('''
+        CREATE TABLE IF NOT EXISTS user (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            username VARCHAR(255) UNIQUE NOT NULL,
+            credential VARCHAR(255) NOT NULL,
+            is_admin BOOLEAN DEFAULT FALSE,
+            create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+            last_active TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        ''')
+        return self
+    
+    async def get_user(self, username: str) -> Optional[DBUserRecord]:
+        async with self.conn.execute("SELECT * FROM user WHERE username = ?", (username, )) as cursor:
+            res = await cursor.fetchone()
+        
+        if res is None: return None
+        return self.parse_record(res)
+    
+    async def get_user_by_id(self, user_id: int) -> Optional[DBUserRecord]:
+        async with self.conn.execute("SELECT * FROM user WHERE id = ?", (user_id, )) as cursor:
+            res = await cursor.fetchone()
+        
+        if res is None: return None
+        return self.parse_record(res)
+    
+    async def get_user_by_credential(self, credential: str) -> Optional[DBUserRecord]:
+        async with self.conn.execute("SELECT * FROM user WHERE credential = ?", (credential, )) as cursor:
+            res = await cursor.fetchone()
+        
+        if res is None: return None
+        return self.parse_record(res)
+    
+    async def create_user(self, username: str, password: str, is_admin: bool = False) -> int:
+        assert not username.startswith('_'), "Error: reserved username"
+        assert not ('/' in username or len(username) > 255), "Invalid username"
+        assert urllib.parse.quote(username) == username, "Invalid username, must be URL safe"
+        self.logger.debug(f"Creating user {username}")
+        credential = hash_credential(username, password)
+        assert await self.get_user(username) is None, "Duplicate username"
+        async with self.conn.execute("INSERT INTO user (username, credential, is_admin) VALUES (?, ?, ?)", (username, credential, is_admin)) as cursor:
+            self.logger.info(f"User {username} created")
+            assert cursor.lastrowid is not None
+            return cursor.lastrowid
+    
+    async def set_user(self, username: str, password: Optional[str] = None, is_admin: Optional[bool] = None):
+        assert not username.startswith('_'), "Error: reserved username"
+        assert not ('/' in username or len(username) > 255), "Invalid username"
+        assert urllib.parse.quote(username) == username, "Invalid username, must be URL safe"
+        if password is not None:
+            credential = hash_credential(username, password)
+        else:
+            async with self.conn.execute("SELECT credential FROM user WHERE username = ?", (username, )) as cursor:
+                res = await cursor.fetchone()
+                assert res is not None, f"User {username} not found"
+                credential = res[0]
+        
+        if is_admin is None:
+            async with self.conn.execute("SELECT is_admin FROM user WHERE username = ?", (username, )) as cursor:
+                res = await cursor.fetchone()
+                assert res is not None, f"User {username} not found"
+                is_admin = res[0]
+
+        await self.conn.execute("UPDATE user SET credential = ?, is_admin = ? WHERE username = ?", (credential, is_admin, username))
+        self.logger.info(f"User {username} updated")
+    
+    async def all(self):
+        async with self.conn.execute("SELECT * FROM user") as cursor:
+            async for record in cursor:
+                yield self.parse_record(record)
+    
+    async def set_active(self, username: str):
+        await self.conn.execute("UPDATE user SET last_active = CURRENT_TIMESTAMP WHERE username = ?", (username, ))
+    
+    async def delete_user(self, username: str):
+        await self.conn.execute("DELETE FROM user WHERE username = ?", (username, ))
+        self.logger.info(f"Delete user {username}")
+
+class FileReadPermission(IntEnum):
+    PUBLIC = 0          # accessible by anyone
+    PROTECTED = 1       # accessible by any user
+    PRIVATE = 2         # accessible by owner only (including admin)
+
+@dataclasses.dataclass
+class FileDBRecord:
+    url: str
+    owner_id: int
+    file_id: str      # defines mapping from fmata to fdata
+    file_size: int
+    create_time: str
+    access_time: str
+    permission: FileReadPermission
+
+    def __str__(self):
+        return  f"File {self.url} (owner={self.owner_id}, created at {self.create_time}, accessed at {self.access_time}, " + \
+                f"file_id={self.file_id}, permission={self.permission}, size={self.file_size})"
+
+@dataclasses.dataclass
+class DirectoryRecord:
+    url: str
+    size: int
+
+    def __str__(self):
+        return f"Directory {self.url} (size={self.size})"
+    
+class FileConn(DBConnBase):
+
+    @staticmethod
+    def parse_record(record) -> FileDBRecord:
+        return FileDBRecord(*record)
+    
+    async def init(self):
+        await super().init()
+        await self.conn.execute('''
+        CREATE TABLE IF NOT EXISTS fmeta (
+            url VARCHAR(512) PRIMARY KEY,
+            owner_id INTEGER NOT NULL,
+            file_id VARCHAR(256) NOT NULL,
+            file_size INTEGER,
+            create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+            access_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            permission INTEGER DEFAULT 0
+        )
+        ''')
+        await self.conn.execute('''
+        CREATE INDEX IF NOT EXISTS idx_fmeta_url ON fmeta(url)
+        ''')
+
+        await self.conn.execute('''
+        CREATE TABLE IF NOT EXISTS fdata (
+            file_id VARCHAR(256) PRIMARY KEY,
+            data BLOB
+        )
+        ''')
+
+        return self
+    
+    async def get_file_record(self, url: str) -> Optional[FileDBRecord]:
+        async with self.conn.execute("SELECT * FROM fmeta WHERE url = ?", (url, )) as cursor:
+            res = await cursor.fetchone()
+        if res is None:
+            return None
+        return self.parse_record(res)
+    
+    async def get_file_records(self, urls: list[str]) -> list[FileDBRecord]:
+        async with self.conn.execute("SELECT * FROM fmeta WHERE url IN ({})".format(','.join(['?'] * len(urls))), urls) as cursor:
+            res = await cursor.fetchall()
+        if res is None:
+            return []
+        return [self.parse_record(r) for r in res]
+    
+    async def get_user_file_records(self, owner_id: int) -> list[FileDBRecord]:
+        async with self.conn.execute("SELECT * FROM fmeta WHERE owner_id = ?", (owner_id, )) as cursor:
+            res = await cursor.fetchall()
+        return [self.parse_record(r) for r in res]
+    
+    async def get_path_records(self, url: str) -> list[FileDBRecord]:
+        async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', )) as cursor:
+            res = await cursor.fetchall()
+        return [self.parse_record(r) for r in res]
+
+    async def list_root(self, *usernames: str) -> list[DirectoryRecord]:
+        """
+        Efficiently list users' directories, if usernames is empty, list all users' directories.
+        """
+        if not usernames:
+            # list all users
+            async with self.conn.execute("SELECT username FROM user") as cursor:
+                res = await cursor.fetchall()
+            dirnames = [u[0] + '/' for u in res]
+            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            return dirs
+        else:
+            # list specific users
+            dirnames = [uname + '/' for uname in usernames]
+            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            return dirs
+    
+    @overload
+    async def list_path(self, url: str, flat: Literal[True]) -> list[FileDBRecord]:...
+    @overload
+    async def list_path(self, url: str, flat: Literal[False]) -> tuple[list[DirectoryRecord], list[FileDBRecord]]:...
+    
+    async def list_path(self, url: str, flat: bool = False) -> list[FileDBRecord] | tuple[list[DirectoryRecord], list[FileDBRecord]]:
+        """
+        List all files and directories under the given path, 
+        if flat is True, return a list of FileDBRecord, recursively including all subdirectories. 
+        Otherwise, return a tuple of (dirs, files), where dirs is a list of DirectoryRecord,
+        """
+        if not url.endswith('/'):
+            url += '/'
+        if url == '/':
+            # users cannot be queried using '/', because we store them without '/' prefix, 
+            # so we need to handle this case separately, 
+            if flat:
+                async with self.conn.execute("SELECT * FROM fmeta") as cursor:
+                    res = await cursor.fetchall()
+                return [self.parse_record(r) for r in res]
+
+            else:
+                return (await self.list_root(), [])
+        
+        if flat:
+            async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', )) as cursor:
+                res = await cursor.fetchall()
+            return [self.parse_record(r) for r in res]
+
+        async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%')) as cursor:
+            res = await cursor.fetchall()
+        files = [self.parse_record(r) for r in res]
+
+        # substr indexing starts from 1
+        async with self.conn.execute(
+            """
+            SELECT DISTINCT 
+                SUBSTR(
+                    url, 
+                    1 + LENGTH(?),
+                    INSTR(SUBSTR(url, 1 + LENGTH(?)), '/') - 1
+                ) AS subdir 
+                FROM fmeta WHERE url LIKE ?
+            """, 
+            (url, url, url + '%')
+            ) as cursor:
+            res = await cursor.fetchall()
+        dirs_str = [r[0] + '/' for r in res if r[0] != '/']
+        dirs = [DirectoryRecord(url + d, await self.path_size(url + d, include_subpath=True)) for d in dirs_str]
+            
+        return (dirs, files)
+    
+    async def path_size(self, url: str, include_subpath = False) -> int:
+        if not url.endswith('/'):
+            url += '/'
+        if not include_subpath:
+            async with self.conn.execute("SELECT SUM(file_size) FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%')) as cursor:
+                res = await cursor.fetchone()
+        else:
+            async with self.conn.execute("SELECT SUM(file_size) FROM fmeta WHERE url LIKE ?", (url + '%', )) as cursor:
+                res = await cursor.fetchone()
+        assert res is not None
+        return res[0] or 0
+    
+    async def set_file_record(self, url: str, owner_id: int, file_id: str, file_size: int, permission: Optional[ FileReadPermission ] = None):
+        self.logger.debug(f"Updating fmeta {url}: user_id={owner_id}, file_id={file_id}")
+
+        old = await self.get_file_record(url)
+        if old is not None:
+            assert old.owner_id == owner_id, f"User mismatch: {old.owner_id} != {owner_id}"
+            if permission is None:
+                permission = old.permission
+            await self.conn.execute(
+                """
+                UPDATE fmeta SET file_id = ?, file_size = ?, permission = ?, 
+                access_time = CURRENT_TIMESTAMP WHERE url = ?
+                """, (file_id, file_size, permission, url))
+            self.logger.info(f"File {url} updated")
+        else:
+            if permission is None:
+                permission = FileReadPermission.PUBLIC
+            await self.conn.execute("INSERT INTO fmeta (url, owner_id, file_id, file_size, permission) VALUES (?, ?, ?, ?, ?)", (url, owner_id, file_id, file_size, permission))
+            self.logger.info(f"File {url} created")
+    
+    async def log_access(self, url: str):
+        await self.conn.execute("UPDATE fmeta SET access_time = CURRENT_TIMESTAMP WHERE url = ?", (url, ))
+    
+    async def delete_file_record(self, url: str):
+        file_record = await self.get_file_record(url)
+        if file_record is None: return
+        await self.conn.execute("DELETE FROM fmeta WHERE url = ?", (url, ))
+        self.logger.info(f"Deleted fmeta {url}")
+    
+    async def delete_user_file_records(self, owner_id: int):
+        async with self.conn.execute("SELECT * FROM fmeta WHERE owner_id = ?", (owner_id, )) as cursor:
+            res = await cursor.fetchall()
+        await self.conn.execute("DELETE FROM fmeta WHERE owner_id = ?", (owner_id, ))
+        self.logger.info(f"Deleted {len(res)} files for user {owner_id}") # type: ignore
+    
+    async def delete_path_records(self, path: str):
+        """Delete all records with url starting with path"""
+        async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ?", (path + '%', )) as cursor:
+            res = await cursor.fetchall()
+        await self.conn.execute("DELETE FROM fmeta WHERE url LIKE ?", (path + '%', ))
+        self.logger.info(f"Deleted {len(res)} files for path {path}") # type: ignore
+    
+    async def set_file_blob(self, file_id: str, blob: bytes) -> int:
+        await self.conn.execute("INSERT OR REPLACE INTO fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
+        return len(blob)
+    
+    async def get_file_blob(self, file_id: str) -> Optional[bytes]:
+        async with self.conn.execute("SELECT data FROM fdata WHERE file_id = ?", (file_id, )) as cursor:
+            res = await cursor.fetchone()
+        if res is None:
+            return None
+        return res[0]
+    
+    async def delete_file_blob(self, file_id: str):
+        await self.conn.execute("DELETE FROM fdata WHERE file_id = ?", (file_id, ))
+    
+    async def delete_file_blobs(self, file_ids: list[str]):
+        await self.conn.execute("DELETE FROM fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
+
+def _validate_url(url: str, is_file = True) -> bool:
+    ret = not url.startswith('/') and not ('..' in url) and ('/' in url) and not ('//' in url) \
+        and not ' ' in url and not url.startswith('\\') and not url.startswith('_') and not url.startswith('.')
+
+    if not ret:
+        return False
+    
+    if is_file:
+        ret = ret and not url.endswith('/')
+    else:
+        ret = ret and url.endswith('/')
+    return ret
+
+async def get_user(db: "Database", user: int | str) -> Optional[DBUserRecord]:
+    if isinstance(user, str):
+        return await db.user.get_user(user)
+    elif isinstance(user, int):
+        return await db.user.get_user_by_id(user)
+    else:
+        return None
+
+_transaction_lock = Lock()
+@asynccontextmanager
+async def transaction(db: "Database"):
+    try:
+        await _transaction_lock.acquire()
+        yield
+        await db.commit()
+    except Exception as e:
+        db.logger.error(f"Error in transaction: {e}")
+        await db.rollback()
+    finally:
+        _transaction_lock.release()
+
+class Database:
+    user: UserConn = UserConn()
+    file: FileConn = FileConn()
+    logger = get_logger('database', global_instance=True)
+
+    async def init(self):
+        await self.user.init()
+        await self.file.init()
+        return self
+    
+    async def commit(self):
+        global _g_conn
+        if _g_conn is not None:
+            await _g_conn.commit()
+    
+    async def close(self):
+        global _g_conn
+        if _g_conn: await _g_conn.close()
+    
+    async def rollback(self):
+        global _g_conn
+        if _g_conn is not None:
+            await _g_conn.rollback()
+
+    async def save_file(self, u: int | str, url: str, blob: bytes):
+        if not _validate_url(url):
+            raise ValueError(f"Invalid URL: {url}")
+        assert isinstance(blob, bytes), "blob must be bytes"
+
+        user = await get_user(self, u)
+        if user is None:
+            return
+        
+        # check if the user is the owner of the path, or is admin
+        if url.startswith('/'):
+            url = url[1:]
+        first_component = url.split('/')[0]
+        if first_component != user.username:
+            if not user.is_admin:
+                raise ValueError(f"Permission denied: {user.username} cannot write to {url}")
+            else:
+                if await get_user(self, first_component) is None:
+                    raise ValueError(f"Invalid path: {first_component} is not a valid username")
+
+        f_id = uuid.uuid4().hex
+        async with transaction(self):
+            file_size = await self.file.set_file_blob(f_id, blob)
+            await self.file.set_file_record(url, owner_id=user.id, file_id=f_id, file_size=file_size)
+            await self.user.set_active(user.username)
+
+    # async def read_file_stream(self, url: str): ...
+    async def read_file(self, url: str) -> bytes:
+        if not _validate_url(url): raise ValueError(f"Invalid URL: {url}")
+
+        r = await self.file.get_file_record(url)
+        if r is None:
+            raise FileNotFoundError(f"File {url} not found")
+
+        f_id = r.file_id
+        blob = await self.file.get_file_blob(f_id)
+        if blob is None:
+            raise FileNotFoundError(f"File {url} data not found")
+        
+        async with transaction(self):
+            await self.file.log_access(url)
+
+        return blob
+
+    async def delete_file(self, url: str) -> Optional[FileDBRecord]:
+        if not _validate_url(url): raise ValueError(f"Invalid URL: {url}")
+
+        async with transaction(self):
+            r = await self.file.get_file_record(url)
+            if r is None:
+                return None
+            f_id = r.file_id
+            await self.file.delete_file_blob(f_id)
+            await self.file.delete_file_record(url)
+            return r
+
+    async def delete_path(self, url: str):
+        if not _validate_url(url, is_file=False): raise ValueError(f"Invalid URL: {url}")
+
+        async with transaction(self):
+            records = await self.file.get_path_records(url)
+            if not records:
+                return None
+            await self.file.delete_file_blobs([r.file_id for r in records])
+            await self.file.delete_path_records(url)
+            return records
+
+    async def delete_user(self, u: str | int):
+        user = await get_user(self, u)
+        if user is None:
+            return
+        
+        async with transaction(self):
+            records = await self.file.get_user_file_records(user.id)
+            await self.file.delete_file_blobs([r.file_id for r in records])
+            await self.file.delete_user_file_records(user.id)
+            await self.user.delete_user(user.username)
+
+    async def zip_path(self, top_url: str, urls: Optional[list[str]]) -> io.BytesIO:
+        if urls is None:
+            urls = [r.url for r in await self.file.list_path(top_url, flat=True)]
+
+        buffer = io.BytesIO()
+        with zipfile.ZipFile(buffer, 'w') as zf:
+            for url in urls:
+                if not url.startswith(top_url):
+                    continue
+                r = await self.file.get_file_record(url)
+                if r is None:
+                    continue
+                f_id = r.file_id
+                blob = await self.file.get_file_blob(f_id)
+                if blob is None:
+                    continue
+
+                rel_path = url[len(top_url):]
+                rel_path = decode_uri_compnents(rel_path)
+                zf.writestr(rel_path, blob)
+
+        buffer.seek(0)
+        return buffer

lfss-0.1.0/lfss/src/log.py

@@ -0,0 +1,157 @@
+from .config import DATA_HOME
+from typing import TypeVar, Callable, Literal, Optional
+from concurrent.futures import ThreadPoolExecutor
+from functools import wraps
+import logging, pathlib, asyncio
+from logging import handlers
+
+class BCOLORS:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    OKGRAY = '\033[90m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+    # Additional colors
+    BLACK = '\033[30m'
+    RED = '\033[31m'
+    GREEN = '\033[32m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    MAGENTA = '\033[35m'
+    CYAN = '\033[36m'
+    WHITE = '\033[37m'
+    LIGHTGRAY = '\033[37m'
+    DARKGRAY = '\033[90m'
+    LIGHTRED = '\033[91m'
+    LIGHTGREEN = '\033[92m'
+    LIGHTYELLOW = '\033[93m'
+    LIGHTBLUE = '\033[94m'
+    LIGHTMAGENTA = '\033[95m'
+    LIGHTCYAN = '\033[96m'
+
+_thread_pool = ThreadPoolExecutor(max_workers=1)
+def thread_wrap(func):
+    def wrapper(*args, **kwargs):
+        _thread_pool.submit(func, *args, **kwargs)
+    return wrapper
+
+class BaseLogger(logging.Logger):
+    def finalize(self):
+        for handler in self.handlers:
+            handler.flush()
+            handler.close()
+            self.removeHandler(handler)
+    
+    @thread_wrap
+    def debug(self, *args, **kwargs): super().debug(*args, **kwargs)
+    @thread_wrap
+    def info(self, *args, **kwargs): super().info(*args, **kwargs)
+    @thread_wrap
+    def warning(self, *args, **kwargs): super().warning(*args, **kwargs)
+    @thread_wrap
+    def error(self, *args, **kwargs): super().error(*args, **kwargs)
+
+_fh_T = Literal['rotate', 'simple', 'daily']
+
+__g_logger_dict: dict[str, BaseLogger] = {}
+def get_logger(
+    name = 'default', 
+    log_home = pathlib.Path(DATA_HOME) / 'logs', 
+    level = 'DEBUG',
+    file_handler_type: _fh_T = 'rotate', 
+    global_instance = True
+    )->BaseLogger:
+    if global_instance and name in __g_logger_dict:
+        return __g_logger_dict[name]
+
+    def setupLogger(logger: BaseLogger):
+        logger.setLevel(level)
+
+        format_str = BCOLORS.LIGHTMAGENTA + ' %(asctime)s ' +BCOLORS.OKCYAN + '[%(name)s][%(levelname)s] ' + BCOLORS.ENDC + ' %(message)s'
+        formatter = logging.Formatter(format_str)
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+        # format_str_plain = format_str.replace(BCOLORS.LIGHTMAGENTA, '').replace(BCOLORS.OKCYAN, '').replace(BCOLORS.ENDC, '')
+        format_str_plain = format_str
+        for color in BCOLORS.__dict__.values():
+            if isinstance(color, str) and color.startswith('\033'):
+                format_str_plain = format_str_plain.replace(color, '')
+
+        formatter_plain = logging.Formatter(format_str_plain)
+        log_home.mkdir(exist_ok=True)
+        log_file = log_home / f'{name}.log'
+        if file_handler_type == 'simple':
+            file_handler = logging.FileHandler(log_file)
+        elif file_handler_type == 'daily':
+            file_handler = handlers.TimedRotatingFileHandler(
+                log_file, when='midnight', interval=1, backupCount=5
+            )
+        elif file_handler_type == 'rotate':
+            file_handler = handlers.RotatingFileHandler(
+                log_file, maxBytes=1000000, backupCount=5
+            )
+
+        file_handler.setFormatter(formatter_plain)
+        logger.addHandler(file_handler)
+    
+    logger = BaseLogger(name)
+    setupLogger(logger)
+    if global_instance:
+        __g_logger_dict[name] = logger
+
+    return logger
+
+def clear_handlers(logger: logging.Logger):
+    for handler in logger.handlers:
+        handler.flush()
+        handler.close()
+        logger.removeHandler(handler)
+    __g_logger_dict.pop(logger.name, None)
+    # print(f'Cleared handlers for logger {logger.name}')
+
+FUNCTION_T = TypeVar('FUNCTION_T', bound=Callable)
+def log_access(
+    include_args: bool = True,
+    logger: Optional[BaseLogger] = None, 
+):
+    if logger is None:
+        logger = get_logger()
+
+    def _log_access(fn: FUNCTION_T) -> FUNCTION_T:
+        if asyncio.iscoroutinefunction(fn):
+            @wraps(fn)
+            async def async_wrapper(*args, **kwargs):
+                if include_args:
+                    logger.info(f'[func] <{fn.__name__}> called with: {args}, {kwargs}')
+                else:
+                    logger.info(f'[func] <{fn.__name__}>')
+                    
+                return await fn(*args, **kwargs)
+            return async_wrapper    # type: ignore
+        else:
+            @wraps(fn)
+            def wrapper(*args, **kwargs):
+                logger = get_logger()
+                if include_args:
+                    logger.info(f'[func] <{fn.__name__}> called with: {args}, {kwargs}')
+                else:
+                    logger.info(f'[func] <{fn.__name__}>')
+                    
+                return fn(*args, **kwargs)
+            return wrapper          # type: ignore
+    return _log_access
+
+def get_dummy_logger() -> BaseLogger:
+    return BaseLogger('dummy')
+
+__ALL__ = [
+    'get_logger', 'clear_handlers', 'log_access', 'get_dummy_logger'
+]

lfss-0.1.0/lfss/src/server.py

@@ -0,0 +1,219 @@
+from typing import Optional
+
+from fastapi import FastAPI, APIRouter, Depends, Request, Response
+from fastapi.exceptions import HTTPException 
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from fastapi.middleware.cors import CORSMiddleware
+import mimesniff
+
+import json
+import mimetypes
+from contextlib import asynccontextmanager
+
+from .log import get_logger
+from .config import MAX_BUNDLE_BYTES
+from .utils import ensure_uri_compnents
+from .database import Database, DBUserRecord, DECOY_USER, FileReadPermission
+
+logger = get_logger("server")
+conn = Database()
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global conn
+    await conn.init()
+    yield
+    await conn.close()
+
+async def get_current_user(token: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
+    if not token:
+        return DECOY_USER
+    user = await conn.user.get_user_by_credential(token.credentials)
+    if not user:
+        raise HTTPException(status_code=401, detail="Invalid token")
+    return user
+
+app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+router_fs = APIRouter(prefix="")
+
+@router_fs.get("/{path:path}")
+async def get_file(path: str, asfile = False, user: DBUserRecord = Depends(get_current_user)):
+    path = ensure_uri_compnents(path)
+    if path == "": path = "/"
+    if path.endswith("/"):
+        # return file under the path as json
+        if user.id == 0:
+            raise HTTPException(status_code=403, detail="Permission denied, credential required")
+        if path == "/":
+            return {
+                "dirs": await conn.file.list_root(user.username) \
+                    if not user.is_admin else await conn.file.list_root(),
+                "files": []
+            }
+
+        if not path.startswith(f"{user.username}/") and not user.is_admin:
+            raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
+
+        dirs, files = await conn.file.list_path(path, flat = False)
+        return {
+            "dirs": dirs,
+            "files": files
+        }
+
+    file_record = await conn.file.get_file_record(path)
+    if not file_record:
+        raise HTTPException(status_code=404, detail="File not found")
+    
+    # permission check
+    perm = file_record.permission
+    if perm == FileReadPermission.PRIVATE:
+        if not user.is_admin and user.id != file_record.owner_id:
+            raise HTTPException(status_code=403, detail="Permission denied")
+        else:
+            assert path.startswith(f"{user.username}/")
+    elif perm == FileReadPermission.PROTECTED:
+        if user.id == 0:
+            raise HTTPException(status_code=403, detail="Permission denied")
+    else:
+        assert perm == FileReadPermission.PUBLIC
+    
+    fname = path.split("/")[-1]
+    async def send(media_type: Optional[str] = None, disposition = "attachment"):
+        fblob = await conn.read_file(path)
+        if media_type is None:
+            media_type, _ = mimetypes.guess_type(fname)
+        if media_type is None:
+            media_type = mimesniff.what(fblob)
+
+        return Response(
+            content=fblob, media_type=media_type, headers={
+                "Content-Disposition": f"{disposition}; filename={fname}", 
+                "Content-Length": str(len(fblob))
+            }
+        )
+    
+    if asfile:
+        return await send('application/octet-stream', "attachment")
+    else:
+        return await send(None, "inline")
+
+@router_fs.put("/{path:path}")
+async def put_file(request: Request, path: str, user: DBUserRecord = Depends(get_current_user)):
+    path = ensure_uri_compnents(path)
+    if user.id == 0:
+        logger.debug("Reject put request from DECOY_USER")
+        raise HTTPException(status_code=403, detail="Permission denied")
+    if not path.startswith(f"{user.username}/") and not user.is_admin:
+        logger.debug(f"Reject put request from {user.username} to {path}")
+        raise HTTPException(status_code=403, detail="Permission denied")
+    
+    logger.info(f"PUT {path}, user: {user.username}")
+    exists_flag = False
+    file_record = await conn.file.get_file_record(path)
+    if file_record:
+        exists_flag = True
+        # remove the old file
+        await conn.delete_file(path)
+    
+    # check content-type
+    content_type = request.headers.get("Content-Type")
+    logger.debug(f"Content-Type: {content_type}")
+    if content_type == "application/json":
+        body = await request.json()
+        await conn.save_file(user.id, path, json.dumps(body).encode('utf-8'))
+    elif content_type == "application/x-www-form-urlencoded":
+        # may not work...
+        body = await request.form()
+        file = body.get("file")
+        if isinstance(file, str) or file is None:
+            raise HTTPException(status_code=400, detail="Invalid form data, file required")
+        await conn.save_file(user.id, path, await file.read())
+    elif content_type == "application/octet-stream":
+        body = await request.body()
+        await conn.save_file(user.id, path, body)
+    else:
+        body = await request.body()
+        await conn.save_file(user.id, path, body)
+
+    # https://developer.mozilla.org/zh-CN/docs/Web/HTTP/Methods/PUT
+    if exists_flag:
+        return Response(status_code=201, headers={
+            "Content-Type": "application/json",
+        }, content=json.dumps({"url": path}))
+    else:
+        return Response(status_code=200, headers={
+            "Content-Type": "application/json",
+        }, content=json.dumps({"url": path}))
+
+@router_fs.delete("/{path:path}")
+async def delete_file(path: str, user: DBUserRecord = Depends(get_current_user)):
+    path = ensure_uri_compnents(path)
+    if user.id == 0:
+        raise HTTPException(status_code=403, detail="Permission denied")
+    if not path.startswith(f"{user.username}/") and not user.is_admin:
+        raise HTTPException(status_code=403, detail="Permission denied")
+    
+    logger.info(f"DELETE {path}, user: {user.username}")
+
+    if path.endswith("/"):
+        res = await conn.delete_path(path)
+    else:
+        res = await conn.delete_file(path)
+
+    if res:
+        return Response(status_code=200, content="Deleted")
+    else:
+        return Response(status_code=404, content="Not found")
+
+router_api = APIRouter(prefix="/_api")
+
+@router_api.get("/bundle")
+async def bundle_files(path: str, user: DBUserRecord = Depends(get_current_user)):
+    logger.info(f"GET bundle({path}), user: {user.username}")
+    path = ensure_uri_compnents(path)
+    assert path.endswith("/") or path == ""
+
+    if not path == "" and path[0] == "/":   # adapt to both /path and path
+        path = path[1:]
+    
+    # TODO: maybe check permission here...
+
+    # return bundle of files
+    files = await conn.file.list_path(path, flat = True)
+    if len(files) == 0:
+        raise HTTPException(status_code=404, detail="No files found")
+    total_size = sum([f.file_size for f in files])
+    if total_size > MAX_BUNDLE_BYTES:
+        raise HTTPException(status_code=400, detail="Too large to zip")
+
+    file_paths = [f.url for f in files]
+    zip_buffer = await conn.zip_path(path, file_paths)
+    return Response(
+        content=zip_buffer.getvalue(), media_type="application/zip", headers={
+            "Content-Disposition": f"attachment; filename=bundle.zip", 
+            "Content-Length": str(zip_buffer.getbuffer().nbytes)
+        }
+    )
+
+@router_api.get("/fmeta")
+async def get_file_meta(path: str, user: DBUserRecord = Depends(get_current_user)):
+    logger.info(f"GET meta({path}), user: {user.username}")
+    if path.endswith("/"):
+        raise HTTPException(status_code=400, detail="Invalid path")
+    path = ensure_uri_compnents(path)
+    file_record = await conn.file.get_file_record(path)
+    if not file_record:
+        raise HTTPException(status_code=404, detail="File not found")
+    return file_record
+
+# order matters
+app.include_router(router_api)
+app.include_router(router_fs)

lfss-0.1.0/lfss/src/utils.py

@@ -0,0 +1,24 @@
+import urllib.parse
+
+def encode_uri_compnents(path: str):
+    """
+    Encode the path components to encode the special characters, 
+    also to avoid path traversal attack
+    """
+    path_sp = path.split("/")
+    mapped = map(lambda x: urllib.parse.quote(x), path_sp)
+    return "/".join(mapped)
+
+def decode_uri_compnents(path: str):
+    """
+    Decode the path components to decode the special characters
+    """
+    path_sp = path.split("/")
+    mapped = map(lambda x: urllib.parse.unquote(x), path_sp)
+    return "/".join(mapped)
+
+def ensure_uri_compnents(path: str):
+    """
+    Ensure the path components are safe to use
+    """
+    return encode_uri_compnents(decode_uri_compnents(path))

lfss-0.1.0/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "lfss"
+version = "0.1.0"
+description = "Lightweight file storage service"
+authors = ["li, mengxun <limengxun45@outlook.com>"]
+readme = "Readme.md"
+homepage = "https://github.com/MenxLi/lfss"
+repository = "https://github.com/MenxLi/lfss"
+
+[tool.poetry.dependencies]
+python = ">=3.9"
+fastapi = "0.*"
+aiosqlite = "0.*"
+mimesniff = "1.*"
+
+[tool.poetry.scripts]
+lfss-serve = "lfss.cli.serve:main"
+lfss-user = "lfss.cli.user:main"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"