lfss 0.9.2__py3-none-any.whl → 0.11.4__py3-none-any.whl

lfss/eng/error.py

@@ -6,16 +6,20 @@ class FileLockedError(LFSSExceptionBase):...
 
 class InvalidOptionsError(LFSSExceptionBase, ValueError):...
 
+class InvalidDataError(LFSSExceptionBase, ValueError):...
+
+class InvalidPathError(LFSSExceptionBase, ValueError):...
+
 class DatabaseLockedError(LFSSExceptionBase, sqlite3.DatabaseError):...
 
+class DatabaseTransactionError(LFSSExceptionBase, sqlite3.DatabaseError):...
+
 class PathNotFoundError(LFSSExceptionBase, FileNotFoundError):...
 
 class FileDuplicateError(LFSSExceptionBase, FileExistsError):...
 
 class PermissionDeniedError(LFSSExceptionBase, PermissionError):...
 
-class InvalidPathError(LFSSExceptionBase, ValueError):...
-
 class StorageExceededError(LFSSExceptionBase):...
 
 class TooManyItemsError(LFSSExceptionBase):...

lfss/eng/log.py

@@ -1,8 +1,9 @@
-from .config import DATA_HOME
+from .config import LOG_DIR, DISABLE_LOGGING
+import time, sqlite3, dataclasses
 from typing import TypeVar, Callable, Literal, Optional
 from concurrent.futures import ThreadPoolExecutor
 from functools import wraps
-import logging, pathlib, asyncio
+import logging, asyncio
 from logging import handlers
 
 class BCOLORS:
@@ -57,15 +58,81 @@ class BaseLogger(logging.Logger):
     @thread_wrap
     def error(self, *args, **kwargs): super().error(*args, **kwargs)
 
-_fh_T = Literal['rotate', 'simple', 'daily']
+class SQLiteFileHandler(logging.FileHandler):
+    def __init__(self, filename, *args, **kwargs):
+        super().__init__(filename, *args, **kwargs)
+        self._db_file = filename
+        self._buffer: list[logging.LogRecord] = []
+        self._buffer_size = 100
+        self._flush_interval = 10
+        self._last_flush = time.time()
+        conn = sqlite3.connect(self._db_file, check_same_thread=False)
+        conn.execute('PRAGMA journal_mode=WAL')
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS log (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                created TIMESTAMP,
+                created_epoch FLOAT,
+                name TEXT,
+                levelname VARCHAR(16),
+                level INTEGER,
+                message TEXT
+            )
+        ''')
+        conn.commit()
+        conn.close()
+    
+    def flush(self): 
+        def format_time(self, record: logging.LogRecord):
+            """ Create a time stamp """
+            return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created))
+        self.acquire()
+        try:
+            conn = sqlite3.connect(self._db_file, check_same_thread=False)
+            conn.executemany('''
+                INSERT INTO log (created, created_epoch, name, levelname, level, message)
+                VALUES (?, ?, ?, ?, ?, ?)
+            ''', [
+                (format_time(self, record), record.created, record.name, record.levelname, record.levelno, record.getMessage())
+                for record in self._buffer
+            ])
+            conn.commit()
+            conn.close()
+            self._buffer.clear()
+            self._last_flush = time.time()
+        finally:
+            self.release()
+
+    def emit(self, record: logging.LogRecord):
+        self._buffer.append(record)
+        if len(self._buffer) > self._buffer_size or time.time() - self._last_flush > self._flush_interval:
+            self.flush()
+    
+    def close(self):
+        self.flush()
+        return super().close()
+
+def eval_logline(row: sqlite3.Row):
+    @dataclasses.dataclass
+    class DBLogRecord:
+        id: int
+        created: str
+        created_epoch: float
+        name: str
+        levelname: str
+        level: int
+        message: str
+    return DBLogRecord(*row)
+
+_fh_T = Literal['rotate', 'simple', 'daily', 'sqlite']
 
 __g_logger_dict: dict[str, BaseLogger] = {}
 def get_logger(
     name = 'default', 
-    log_home = pathlib.Path(DATA_HOME) / 'logs', 
+    log_home = LOG_DIR, 
     level = 'DEBUG',
     term_level = 'INFO',
-    file_handler_type: _fh_T = 'rotate', 
+    file_handler_type: _fh_T = 'sqlite', 
     global_instance = True
     )->BaseLogger:
     if global_instance and name in __g_logger_dict:
@@ -87,22 +154,25 @@ def get_logger(
             if isinstance(color, str) and color.startswith('\033'):
                 format_str_plain = format_str_plain.replace(color, '')
 
-        formatter_plain = logging.Formatter(format_str_plain)
-        log_home.mkdir(exist_ok=True)
-        log_file = log_home / f'{name}.log'
-        if file_handler_type == 'simple':
-            file_handler = logging.FileHandler(log_file)
-        elif file_handler_type == 'daily':
-            file_handler = handlers.TimedRotatingFileHandler(
-                log_file, when='midnight', interval=1, backupCount=30
-            )
-        elif file_handler_type == 'rotate':
-            file_handler = handlers.RotatingFileHandler(
-                log_file, maxBytes=1024*1024, backupCount=5
-            )
-
-        file_handler.setFormatter(formatter_plain)
-        logger.addHandler(file_handler)
+        if not DISABLE_LOGGING:
+            formatter_plain = logging.Formatter(format_str_plain)
+            log_home.mkdir(exist_ok=True)
+            log_file = log_home / f'{name}.log'
+            if file_handler_type == 'simple':
+                file_handler = logging.FileHandler(log_file)
+            elif file_handler_type == 'daily':
+                file_handler = handlers.TimedRotatingFileHandler(
+                    log_file, when='midnight', interval=1, backupCount=30
+                )
+            elif file_handler_type == 'rotate':
+                file_handler = handlers.RotatingFileHandler(
+                    log_file, maxBytes=1024*1024, backupCount=5
+                )
+            elif file_handler_type == 'sqlite':
+                file_handler = SQLiteFileHandler(log_file if log_file.suffix == '.db' else log_file.with_suffix('.log.db'))
+
+            file_handler.setFormatter(formatter_plain)
+            logger.addHandler(file_handler)
     
     logger = BaseLogger(name)
     setupLogger(logger)

lfss/eng/thumb.py

@@ -1,5 +1,6 @@
 from lfss.eng.config import THUMB_DB, THUMB_SIZE
 from lfss.eng.database import FileConn
+from lfss.eng.error import *
 from lfss.eng.connection_pool import unique_cursor
 from typing import Optional
 from PIL import Image
@@ -10,44 +11,42 @@ from contextlib import asynccontextmanager
 async def _maybe_init_thumb(c: aiosqlite.Cursor):
     await c.execute('''
         CREATE TABLE IF NOT EXISTS thumbs (
-            path TEXT PRIMARY KEY,
-            ctime TEXT,
+            file_id CHAR(32) PRIMARY KEY,
             thumb BLOB
         )
     ''')
-    await c.execute('CREATE INDEX IF NOT EXISTS thumbs_path_idx ON thumbs (path)')
+    await c.execute('CREATE INDEX IF NOT EXISTS thumbs_path_idx ON thumbs (file_id)')
 
-async def _get_cache_thumb(c: aiosqlite.Cursor, path: str, ctime: str) -> Optional[bytes]:
+async def _get_cache_thumb(c: aiosqlite.Cursor, file_id: str) -> Optional[bytes]:
     res = await c.execute('''
-        SELECT ctime, thumb FROM thumbs WHERE path = ? 
-    ''', (path, ))
+        SELECT thumb FROM thumbs WHERE file_id = ? 
+    ''', (file_id, ))
     row = await res.fetchone()
     if row is None:
         return None
-    # check if ctime matches, if not delete and return None
-    if row[0] != ctime:
-        await _delete_cache_thumb(c, path)
-        return None
-    blob: bytes = row[1]
+    blob: bytes = row[0]
     return blob
     
-async def _save_cache_thumb(c: aiosqlite.Cursor, path: str, ctime: str, raw_bytes: bytes) -> bytes:
-    raw_img = Image.open(BytesIO(raw_bytes))
+async def _save_cache_thumb(c: aiosqlite.Cursor, file_id: str, raw_bytes: bytes) -> bytes:
+    try:
+        raw_img = Image.open(BytesIO(raw_bytes))
+    except Exception:
+        raise InvalidDataError('Invalid image data for thumbnail: ' + file_id)
     raw_img.thumbnail(THUMB_SIZE)
     img = raw_img.convert('RGB')
     bio = BytesIO()
     img.save(bio, 'JPEG')
     blob = bio.getvalue()
     await c.execute('''
-        INSERT OR REPLACE INTO thumbs (path, ctime, thumb) VALUES (?, ?, ?)
-    ''', (path, ctime, blob))
+        INSERT OR REPLACE INTO thumbs (file_id, thumb) VALUES (?, ?)
+    ''', (file_id, blob))
     await c.execute('COMMIT')  # commit immediately
     return blob
 
-async def _delete_cache_thumb(c: aiosqlite.Cursor, path: str):
+async def _delete_cache_thumb(c: aiosqlite.Cursor, file_id: str):
     await c.execute('''
-        DELETE FROM thumbs WHERE path = ?
-    ''', (path, ))
+        DELETE FROM thumbs WHERE file_id = ?
+    ''', (file_id, ))
     await c.execute('COMMIT')
 
 @asynccontextmanager
@@ -71,15 +70,13 @@ async def get_thumb(path: str) -> Optional[tuple[bytes, str]]:
         r = await fconn.get_file_record(path)
 
     if r is None:
-        async with cache_cursor() as cur:
-            await _delete_cache_thumb(cur, path)
         raise FileNotFoundError(f'File not found: {path}')
     if not r.mime_type.startswith('image/'):
         return None
 
+    file_id = r.file_id
     async with cache_cursor() as cur:
-        c_time = r.create_time
-        thumb_blob = await _get_cache_thumb(cur, path, c_time)
+        thumb_blob = await _get_cache_thumb(cur, file_id)
         if thumb_blob is not None:
             return thumb_blob, "image/jpeg"
         
@@ -94,5 +91,5 @@ async def get_thumb(path: str) -> Optional[tuple[bytes, str]]:
                 data = await fconn.get_file_blob(r.file_id)
             assert data is not None
         
-        thumb_blob = await _save_cache_thumb(cur, path, c_time, data)
+        thumb_blob = await _save_cache_thumb(cur, file_id, data)
         return thumb_blob, "image/jpeg"

lfss/eng/utils.py

@@ -11,7 +11,6 @@ from concurrent.futures import ThreadPoolExecutor
 from typing import TypeVar, Callable, Awaitable
 from functools import wraps, partial
 from uuid import uuid4
-import os
 
 async def copy_file(source: str|pathlib.Path, destination: str|pathlib.Path):
     async with aiofiles.open(source, mode='rb') as src:
@@ -20,7 +19,7 @@ async def copy_file(source: str|pathlib.Path, destination: str|pathlib.Path):
                 await dest.write(chunk)
 
 def hash_credential(username: str, password: str):
-    return hashlib.sha256((username + password).encode()).hexdigest()
+    return hashlib.sha256(f"{username}:{password}".encode()).hexdigest()
 
 def encode_uri_compnents(path: str):
     path_sp = path.split("/")
@@ -36,17 +35,41 @@ def ensure_uri_compnents(path: str):
     """ Ensure the path components are safe to use """
     return encode_uri_compnents(decode_uri_compnents(path))
 
-g_debounce_tasks: OrderedDict[str, asyncio.Task] = OrderedDict()
-lock_debounce_task_queue = Lock()
+class TaskManager:
+    def __init__(self):
+        self._tasks: OrderedDict[str, asyncio.Task] = OrderedDict()
+    
+    def push(self, task: asyncio.Task) -> str:
+        tid = uuid4().hex
+        if tid in self._tasks:
+            raise ValueError("Task ID collision")
+        self._tasks[tid] = task
+        return tid
+    
+    def cancel(self, task_id: str):
+        task = self._tasks.pop(task_id, None)
+        if task is not None:
+            task.cancel()
+    
+    def truncate(self):
+        new_tasks = OrderedDict()
+        for tid, task in self._tasks.items():
+            if not task.done():
+                new_tasks[tid] = task
+        self._tasks = new_tasks
+    
+    async def wait_all(self):
+        async def stop_task(task: asyncio.Task):
+            if not task.done():
+                await task
+        await asyncio.gather(*map(stop_task, self._tasks.values()))
+        self._tasks.clear()
+    
+    def __len__(self): return len(self._tasks)
+
+g_debounce_tasks: TaskManager = TaskManager()
 async def wait_for_debounce_tasks():
-    async def stop_task(task: asyncio.Task):
-        task.cancel()
-        try:
-            await task
-        except asyncio.CancelledError:
-            pass
-    await asyncio.gather(*map(stop_task, g_debounce_tasks.values()))
-    g_debounce_tasks.clear()
+    await g_debounce_tasks.wait_all()
 
 def debounce_async(delay: float = 0.1, max_wait: float = 1.):
     """ 
@@ -54,7 +77,8 @@ def debounce_async(delay: float = 0.1, max_wait: float = 1.):
     ensuring execution at least once every `max_wait` seconds. 
     """
     def debounce_wrap(func):
-        task_record: tuple[str, asyncio.Task] | None = None
+        # task_record: tuple[str, asyncio.Task] | None = None
+        prev_task_id = None
         fn_execution_lock = Lock()
         last_execution_time = 0
 
@@ -67,12 +91,11 @@ def debounce_async(delay: float = 0.1, max_wait: float = 1.):
 
         @functools.wraps(func)
         async def wrapper(*args, **kwargs):
-            nonlocal task_record, last_execution_time
+            nonlocal prev_task_id, last_execution_time
 
-            async with lock_debounce_task_queue:
-                if task_record is not None:
-                    task_record[1].cancel()
-                    g_debounce_tasks.pop(task_record[0], None)
+            if prev_task_id is not None:
+                g_debounce_tasks.cancel(prev_task_id)
+                prev_task_id = None
             
             async with fn_execution_lock:
                 if time.monotonic() - last_execution_time > max_wait:
@@ -81,14 +104,12 @@ def debounce_async(delay: float = 0.1, max_wait: float = 1.):
                     return
 
             task = asyncio.create_task(delayed_func(*args, **kwargs))
-            task_uid = uuid4().hex
-            task_record = (task_uid, task)
-            async with lock_debounce_task_queue:
-                g_debounce_tasks[task_uid] = task
-                if len(g_debounce_tasks) > 2048:
-                    # finished tasks are not removed from the dict
-                    # so we need to clear it periodically
-                    await wait_for_debounce_tasks()
+            prev_task_id = g_debounce_tasks.push(task)
+            if len(g_debounce_tasks) > 1024:
+                # finished tasks are not removed from the dict
+                # so we need to clear it periodically
+                g_debounce_tasks.truncate()
+
         return wrapper
     return debounce_wrap
 
@@ -133,12 +154,12 @@ def fmt_storage_size(size: int) -> str:
     return f"{size/1024**4:.2f}T"
 
 _FnReturnT = TypeVar('_FnReturnT')
-_AsyncReturnT = Awaitable[_FnReturnT]
+_AsyncReturnT = TypeVar('_AsyncReturnT', bound=Awaitable)
 _g_executor = None
 def get_global_executor():
     global _g_executor
     if _g_executor is None:
-        _g_executor = ThreadPoolExecutor(max_workers=4 if (cpu_count:=os.cpu_count()) and cpu_count > 4 else cpu_count)
+        _g_executor = ThreadPoolExecutor()
     return _g_executor
 def async_wrap(executor=None):
     if executor is None:
@@ -157,7 +178,7 @@ def concurrent_wrap(executor=None):
         def sync_fn(*args, **kwargs):
             loop = asyncio.new_event_loop()
             return loop.run_until_complete(func(*args, **kwargs))
-        return sync_fn
+        return sync_fn          # type: ignore
     return _concurrent_wrap
 
 # https://stackoverflow.com/a/279586/6775765

lfss/sql/init.sql

@@ -1,4 +1,4 @@
-CREATE TABLE IF NOT EXISTS user (
+CREATE TABLE IF NOT EXISTS main.user (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     username VARCHAR(256) UNIQUE NOT NULL,
     credential VARCHAR(256) NOT NULL,
@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS user (
     permission INTEGER DEFAULT 0
 );
 
-CREATE TABLE IF NOT EXISTS fmeta (
+CREATE TABLE IF NOT EXISTS main.fmeta (
     url VARCHAR(1024) PRIMARY KEY,
     owner_id INTEGER NOT NULL,
     file_id CHAR(32) NOT NULL,
@@ -22,12 +22,17 @@ CREATE TABLE IF NOT EXISTS fmeta (
     FOREIGN KEY(owner_id) REFERENCES user(id)
 );
 
-CREATE TABLE IF NOT EXISTS usize (
+CREATE TABLE IF NOT EXISTS main.dupcount (
+    file_id CHAR(32) PRIMARY KEY,
+    count INTEGER DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS main.usize (
     user_id INTEGER PRIMARY KEY,
     size INTEGER DEFAULT 0
 );
 
-CREATE TABLE IF NOT EXISTS upeer (
+CREATE TABLE IF NOT EXISTS main.upeer (
     src_user_id INTEGER NOT NULL,
     dst_user_id INTEGER NOT NULL,
     access_level INTEGER DEFAULT 0,

lfss/svc/app.py

@@ -6,4 +6,4 @@ app.include_router(router_api)
 if ENABLE_WEBDAV:
     from .app_dav import *
     app.include_router(router_dav)
-app.include_router(router_fs)
+app.include_router(router_fs)

lfss/svc/app_base.py

@@ -27,7 +27,7 @@ req_conn = RequestDB()
 async def lifespan(app: FastAPI):
     global db
     try:
-        await global_connection_init(n_read = 2)
+        await global_connection_init(n_read = 8 if not DEBUG_MODE else 1)
         await asyncio.gather(db.init(), req_conn.init())
         yield
         await req_conn.commit()
@@ -47,21 +47,26 @@ def handle_exception(fn):
             if isinstance(e, StorageExceededError): raise HTTPException(status_code=413, detail=str(e))
             if isinstance(e, PermissionError): raise HTTPException(status_code=403, detail=str(e))
             if isinstance(e, InvalidPathError): raise HTTPException(status_code=400, detail=str(e))
+            if isinstance(e, InvalidOptionsError): raise HTTPException(status_code=400, detail=str(e))
+            if isinstance(e, InvalidDataError): raise HTTPException(status_code=400, detail=str(e))
             if isinstance(e, FileNotFoundError): raise HTTPException(status_code=404, detail=str(e))
             if isinstance(e, FileDuplicateError): raise HTTPException(status_code=409, detail=str(e))
             if isinstance(e, FileExistsError): raise HTTPException(status_code=409, detail=str(e))
             if isinstance(e, TooManyItemsError): raise HTTPException(status_code=400, detail=str(e))
             if isinstance(e, DatabaseLockedError): raise HTTPException(status_code=503, detail=str(e))
+            if isinstance(e, DatabaseTransactionError): raise HTTPException(status_code=503, detail=str(e))
             if isinstance(e, FileLockedError): raise HTTPException(status_code=423, detail=str(e))
-            if isinstance(e, InvalidOptionsError): raise HTTPException(status_code=400, detail=str(e))
             logger.error(f"Uncaptured error in {fn.__name__}: {e}")
             raise 
     return wrapper
 
+env_origins = os.environ.get("LFSS_ORIGINS", "*")
+logger.debug(f"LFSS_ORIGINS: {env_origins}")
+origins = [x.strip() for x in env_origins.split(",") if x.strip()]
 app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
+    allow_origins=origins,
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],