lfss 0.3.1__tar.gz → 0.4.0__tar.gz

{lfss-0.3.1 → lfss-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.3.1
+Version: 0.4.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.3.1 → lfss-0.4.0}/lfss/src/database.py

@@ -69,7 +69,7 @@ class UserRecord:
     permission: 'FileReadPermission'
 
     def __str__(self):
-        return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}), storage={self.max_storage}, permission={self.permission}"
+        return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}, storage={self.max_storage}, permission={self.permission})"
 
 DECOY_USER = UserRecord(0, 'decoy', 'decoy', False, '2021-01-01 00:00:00', '2021-01-01 00:00:00', 0, FileReadPermission.PRIVATE)
 class UserConn(DBConnBase):
@@ -421,7 +421,7 @@ class FileConn(DBConnBase):
         new_exists = await self.get_file_record(new_url)
         if new_exists is not None:
             raise FileExistsError(f"File {new_url} already exists")
-        async with self.conn.execute("UPDATE fmeta SET url = ? WHERE url = ?", (new_url, old_url)):
+        async with self.conn.execute("UPDATE fmeta SET url = ?, create_time = CURRENT_TIMESTAMP WHERE url = ?", (new_url, old_url)):
             self.logger.info(f"Moved file {old_url} to {new_url}")
     
     async def log_access(self, url: str):
@@ -481,8 +481,9 @@ class FileConn(DBConnBase):
         await self.conn.execute("DELETE FROM fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
 
 def validate_url(url: str, is_file = True):
-    ret = not url.startswith('/') and not ('..' in url) and ('/' in url) and not ('//' in url) \
-        and not ' ' in url and not url.startswith('\\') and not url.startswith('_') and not url.startswith('.')
+    prohibited_chars = ['..', ';', "'", '"', '\\', '\0', '\n', '\r', '\t', '\x0b', '\x0c']
+    ret = not url.startswith('/') and not url.startswith('_') and not url.startswith('.')
+    ret = ret and not any([c in url for c in prohibited_chars])
 
     if not ret:
         raise InvalidPathError(f"Invalid URL: {url}")

{lfss-0.3.1 → lfss-0.4.0}/lfss/src/log.py

@@ -64,6 +64,7 @@ def get_logger(
     name = 'default', 
     log_home = pathlib.Path(DATA_HOME) / 'logs', 
     level = 'DEBUG',
+    term_level = 'INFO',
     file_handler_type: _fh_T = 'rotate', 
     global_instance = True
     )->BaseLogger:
@@ -77,6 +78,7 @@ def get_logger(
         formatter = logging.Formatter(format_str)
         console_handler = logging.StreamHandler()
         console_handler.setFormatter(formatter)
+        console_handler.setLevel(term_level)
         logger.addHandler(console_handler)
 
         # format_str_plain = format_str.replace(BCOLORS.LIGHTMAGENTA, '').replace(BCOLORS.OKCYAN, '').replace(BCOLORS.ENDC, '')

{lfss-0.3.1 → lfss-0.4.0}/lfss/src/server.py

@@ -7,26 +7,29 @@ from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from fastapi.middleware.cors import CORSMiddleware
 import mimesniff
 
-import json
+import asyncio, json, time
 import mimetypes
 from contextlib import asynccontextmanager
 
 from .error import *
 from .log import get_logger
+from .stat import RequestDB
 from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES
-from .utils import ensure_uri_compnents
+from .utils import ensure_uri_compnents, format_last_modified, now_stamp
 from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission
 
-logger = get_logger("server")
+logger = get_logger("server", term_level="DEBUG")
+logger_failed_request = get_logger("failed_requests", term_level="INFO")
 conn = Database()
+req_conn = RequestDB()
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     global conn
-    await conn.init()
+    await asyncio.gather(conn.init(), req_conn.init())
     yield
-    await conn.commit()
-    await conn.close()
+    await asyncio.gather(conn.commit(), req_conn.commit())
+    await asyncio.gather(conn.close(), req_conn.close())
 
 def handle_exception(fn):
     @wraps(fn)
@@ -34,14 +37,14 @@ def handle_exception(fn):
         try:
             return await fn(*args, **kwargs)
         except Exception as e:
-            logger.error(f"Error in {fn.__name__}: {e}")
             if isinstance(e, HTTPException): raise e
-            elif isinstance(e, StorageExceededError): raise HTTPException(status_code=413, detail=str(e))
-            elif isinstance(e, PermissionError): raise HTTPException(status_code=403, detail=str(e))
-            elif isinstance(e, InvalidPathError): raise HTTPException(status_code=400, detail=str(e))
-            elif isinstance(e, FileNotFoundError): raise HTTPException(status_code=404, detail=str(e))
-            elif isinstance(e, FileExistsError): raise HTTPException(status_code=409, detail=str(e))
-            else: raise HTTPException(status_code=500, detail=str(e))
+            if isinstance(e, StorageExceededError): raise HTTPException(status_code=413, detail=str(e))
+            if isinstance(e, PermissionError): raise HTTPException(status_code=403, detail=str(e))
+            if isinstance(e, InvalidPathError): raise HTTPException(status_code=400, detail=str(e))
+            if isinstance(e, FileNotFoundError): raise HTTPException(status_code=404, detail=str(e))
+            if isinstance(e, FileExistsError): raise HTTPException(status_code=409, detail=str(e))
+            logger.error(f"Uncaptured error in {fn.__name__}: {e}")
+            raise HTTPException(status_code=500, detail=str(e))
     return wrapper
 
 async def get_credential_from_params(request: Request):
@@ -66,7 +69,7 @@ async def get_current_user(
         raise HTTPException(status_code=401, detail="Invalid token")
     return user
 
-app = FastAPI(docs_url="/_docs", redoc_url=None, lifespan=lifespan)
+app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -75,9 +78,36 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+@app.middleware("http")
+async def log_requests(request: Request, call_next):
+
+    request_time_stamp = now_stamp()
+    start_time = time.perf_counter()
+    response: Response = await call_next(request)
+    end_time = time.perf_counter()
+    response_time = end_time - start_time
+    response.headers["X-Response-Time"] = str(response_time)
+
+    if response.status_code >= 400:
+        logger_failed_request.error(f"{request.method} {request.url.path} {response.status_code}")
+        
+    await req_conn.log_request(
+        request_time_stamp, 
+        request.method, request.url.path, response.status_code, response_time,
+        headers = dict(request.headers), 
+        query = dict(request.query_params), 
+        client = request.client, 
+        request_size = int(request.headers.get("Content-Length", 0)),
+        response_size = int(response.headers.get("Content-Length", 0))
+    )
+    await req_conn.ensure_commit_once()
+
+    return response
+
 router_fs = APIRouter(prefix="")
 
 @router_fs.get("/{path:path}")
+@handle_exception
 async def get_file(path: str, download = False, user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
 
@@ -120,7 +150,8 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
         return Response(
             content=fblob, media_type=media_type, headers={
                 "Content-Disposition": f"{disposition}; filename={fname}", 
-                "Content-Length": str(len(fblob))
+                "Content-Length": str(len(fblob)), 
+                "Last-Modified": format_last_modified(file_record.create_time)
             }
         )
     
@@ -130,6 +161,7 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
         return await send(None, "inline")
 
 @router_fs.put("/{path:path}")
+@handle_exception
 async def put_file(request: Request, path: str, user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
     if user.id == 0:
@@ -159,20 +191,20 @@ async def put_file(request: Request, path: str, user: UserRecord = Depends(get_c
     logger.debug(f"Content-Type: {content_type}")
     if content_type == "application/json":
         body = await request.json()
-        await handle_exception(conn.save_file)(user.id, path, json.dumps(body).encode('utf-8'))
+        await conn.save_file(user.id, path, json.dumps(body).encode('utf-8'))
     elif content_type == "application/x-www-form-urlencoded":
         # may not work...
         body = await request.form()
         file = body.get("file")
         if isinstance(file, str) or file is None:
             raise HTTPException(status_code=400, detail="Invalid form data, file required")
-        await handle_exception(conn.save_file)(user.id, path, await file.read())
+        await conn.save_file(user.id, path, await file.read())
     elif content_type == "application/octet-stream":
         body = await request.body()
-        await handle_exception(conn.save_file)(user.id, path, body)
+        await conn.save_file(user.id, path, body)
     else:
         body = await request.body()
-        await handle_exception(conn.save_file)(user.id, path, body)
+        await conn.save_file(user.id, path, body)
 
     # https://developer.mozilla.org/zh-CN/docs/Web/HTTP/Methods/PUT
     if exists_flag:
@@ -185,6 +217,7 @@ async def put_file(request: Request, path: str, user: UserRecord = Depends(get_c
         }, content=json.dumps({"url": path}))
 
 @router_fs.delete("/{path:path}")
+@handle_exception
 async def delete_file(path: str, user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
     if user.id == 0:
@@ -207,6 +240,7 @@ async def delete_file(path: str, user: UserRecord = Depends(get_current_user)):
 router_api = APIRouter(prefix="/_api")
 
 @router_api.get("/bundle")
+@handle_exception
 async def bundle_files(path: str, user: UserRecord = Depends(get_current_user)):
     logger.info(f"GET bundle({path}), user: {user.username}")
     if user.id == 0:
@@ -249,6 +283,7 @@ async def bundle_files(path: str, user: UserRecord = Depends(get_current_user)):
     )
 
 @router_api.get("/fmeta")
+@handle_exception
 async def get_file_meta(path: str, user: UserRecord = Depends(get_current_user)):
     logger.info(f"GET meta({path}), user: {user.username}")
     if path.endswith("/"):
@@ -260,6 +295,7 @@ async def get_file_meta(path: str, user: UserRecord = Depends(get_current_user))
     return file_record
 
 @router_api.post("/fmeta")
+@handle_exception
 async def update_file_meta(
     path: str, 
     perm: Optional[int] = None, 
@@ -280,7 +316,7 @@ async def update_file_meta(
     
     if perm is not None:
         logger.info(f"Update permission of {path} to {perm}")
-        await handle_exception(conn.file.set_file_record)(
+        await conn.file.set_file_record(
             url = file_record.url, 
             permission = FileReadPermission(perm)
         )
@@ -288,18 +324,18 @@ async def update_file_meta(
     if new_path is not None:
         new_path = ensure_uri_compnents(new_path)
         logger.info(f"Update path of {path} to {new_path}")
-        await handle_exception(conn.move_file)(path, new_path)
+        await conn.move_file(path, new_path)
 
     return Response(status_code=200, content="OK")
     
 @router_api.get("/whoami")
+@handle_exception
 async def whoami(user: UserRecord = Depends(get_current_user)):
     if user.id == 0:
         raise HTTPException(status_code=401, detail="Login required")
     user.credential = "__HIDDEN__"
     return user
 
-
 # order matters
 app.include_router(router_api)
 app.include_router(router_fs)

lfss-0.4.0/lfss/src/stat.py

@@ -0,0 +1,66 @@
+from typing import Optional, Any
+import aiosqlite
+from .config import DATA_HOME
+from .utils import debounce_async
+
+class RequestDB:
+    conn: aiosqlite.Connection
+    def __init__(self):
+        self.db = DATA_HOME / 'requests.db'
+
+    async def init(self):
+        self.conn = await aiosqlite.connect(self.db)
+        await self.conn.execute('''
+            CREATE TABLE IF NOT EXISTS requests (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                time FLOAT DEFAULT (strftime('%s', 'now')),
+                method TEXT,
+                path TEXT,
+                headers TEXT,
+                query TEXT,
+                client TEXT,
+                duration REAL, 
+                request_size INTEGER, 
+                response_size INTEGER, 
+                status INTEGER
+            )
+        ''')
+
+    async def close(self):
+        await self.conn.close()
+    
+    async def commit(self):
+        await self.conn.commit()
+    
+    @debounce_async(0.1)
+    async def ensure_commit_once(self):
+        await self.commit()
+    
+    async def __aenter__(self):
+        return self
+    
+    async def __aexit__(self, exc_type, exc, tb):
+        await self.commit()
+    
+    async def log_request(
+        self, time: float, 
+        method: str, path: str, 
+        status: int, duration: float,
+        headers: Optional[Any] = None, 
+        query: Optional[Any] = None, 
+        client: Optional[Any] = None,
+        request_size: int = 0,
+        response_size: int = 0
+        ) -> int:
+        method = str(method).upper()
+        headers = str(headers)
+        query = str(query)
+        client = str(client)
+        async with self.conn.execute('''
+            INSERT INTO requests (
+                time, method, path, headers, query, client, duration, request_size, response_size, status
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ''', (time, method, path, headers, query, client, duration, request_size, response_size, status)) as cursor:
+            assert cursor.lastrowid is not None
+            return cursor.lastrowid
+        

lfss-0.4.0/lfss/src/utils.py

@@ -0,0 +1,68 @@
+import datetime
+import urllib.parse
+import asyncio
+import functools
+
+def encode_uri_compnents(path: str):
+    """
+    Encode the path components to encode the special characters, 
+    also to avoid path traversal attack
+    """
+    path_sp = path.split("/")
+    mapped = map(lambda x: urllib.parse.quote(x), path_sp)
+    return "/".join(mapped)
+
+def decode_uri_compnents(path: str):
+    """
+    Decode the path components to decode the special characters
+    """
+    path_sp = path.split("/")
+    mapped = map(lambda x: urllib.parse.unquote(x), path_sp)
+    return "/".join(mapped)
+
+def ensure_uri_compnents(path: str):
+    """
+    Ensure the path components are safe to use
+    """
+    return encode_uri_compnents(decode_uri_compnents(path))
+
+def debounce_async(delay: float = 0):
+    """
+    Decorator to debounce the async function (procedure)
+    The function must return None
+    """
+    def debounce_wrap(func):
+        # https://docs.python.org/3/library/asyncio-task.html#asyncio.Task.cancel
+        async def delayed_func(*args, **kwargs):
+            await asyncio.sleep(delay)
+            await func(*args, **kwargs)
+
+        task_record: asyncio.Task | None = None
+        @functools.wraps(func)
+        async def wrapper(*args, **kwargs):
+            nonlocal task_record
+            if task_record is not None:
+                task_record.cancel()
+            task_record = asyncio.create_task(delayed_func(*args, **kwargs))
+            try:
+                await task_record
+            except asyncio.CancelledError:
+                pass
+        return wrapper
+    return debounce_wrap
+
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified
+def format_last_modified(last_modified_gmt: str):
+    """
+    Format the last modified time to the HTTP standard format
+    - last_modified_gmt: The last modified time in SQLite ISO 8601 GMT format: e.g. '2021-09-01 12:00:00'
+    """
+    assert len(last_modified_gmt) == 19
+    dt = datetime.datetime.strptime(last_modified_gmt, '%Y-%m-%d %H:%M:%S')
+    return dt.strftime('%a, %d %b %Y %H:%M:%S GMT')
+
+def now_stamp() -> float:
+    return datetime.datetime.now().timestamp()
+
+def stamp_to_str(stamp: float) -> str:
+    return datetime.datetime.fromtimestamp(stamp).strftime('%Y-%m-%d %H:%M:%S')

{lfss-0.3.1 → lfss-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.3.1"
+version = "0.4.0"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"

lfss-0.3.1/lfss/src/utils.py

@@ -1,24 +0,0 @@
-import urllib.parse
-
-def encode_uri_compnents(path: str):
-    """
-    Encode the path components to encode the special characters, 
-    also to avoid path traversal attack
-    """
-    path_sp = path.split("/")
-    mapped = map(lambda x: urllib.parse.quote(x), path_sp)
-    return "/".join(mapped)
-
-def decode_uri_compnents(path: str):
-    """
-    Decode the path components to decode the special characters
-    """
-    path_sp = path.split("/")
-    mapped = map(lambda x: urllib.parse.unquote(x), path_sp)
-    return "/".join(mapped)
-
-def ensure_uri_compnents(path: str):
-    """
-    Ensure the path components are safe to use
-    """
-    return encode_uri_compnents(decode_uri_compnents(path))