lfss 0.9.4__py3-none-any.whl → 0.11.4__py3-none-any.whl

lfss/cli/cli.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-import argparse, typing
+import argparse, typing, sys
 from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
 from lfss.eng.datatype import FileReadPermission, FileSortKey, DirSortKey
 from lfss.eng.utils import decode_uri_compnents
@@ -12,7 +12,7 @@ def parse_permission(s: str) -> FileReadPermission:
     raise ValueError(f"Invalid permission {s}")
 
 def parse_arguments():
-    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
+    parser = argparse.ArgumentParser(description="Client-side command line interface, set LFSS_ENDPOINT and LFSS_TOKEN environment variables for authentication.")
 
     sp = parser.add_subparsers(dest="command", required=True)
 
@@ -78,9 +78,9 @@ def main():
                 permission=args.permission
             )
             if failed_upload:
-                print("\033[91mFailed to upload:\033[0m")
+                print("\033[91mFailed to upload:\033[0m", file=sys.stderr)
                 for path in failed_upload:
-                    print(f"  {path}")
+                    print(f"  {path}", file=sys.stderr)
         else:
             success, msg = upload_file(
                 connector, 
@@ -93,7 +93,7 @@ def main():
                 permission=args.permission
             )
             if not success:
-                print("\033[91mFailed to upload: \033[0m", msg)
+                print("\033[91mFailed to upload: \033[0m", msg, file=sys.stderr)
     
     elif args.command == "download":
         is_dir = args.src.endswith("/")
@@ -107,9 +107,9 @@ def main():
                 overwrite=args.overwrite
             )
             if failed_download:
-                print("\033[91mFailed to download:\033[0m")
+                print("\033[91mFailed to download:\033[0m", file=sys.stderr)
                 for path in failed_download:
-                    print(f"  {path}")
+                    print(f"  {path}", file=sys.stderr)
         else:
             success, msg = download_file(
                 connector, 
@@ -121,12 +121,12 @@ def main():
                 overwrite=args.overwrite
             )
             if not success:
-                print("\033[91mFailed to download: \033[0m", msg)
+                print("\033[91mFailed to download: \033[0m", msg, file=sys.stderr)
     
     elif args.command == "query":
         for path in args.path:
             with catch_request_error():
-                res = connector.get_metadata(path)
+                res = connector.get_meta(path)
                 if res is None:
                     print(f"\033[31mNot found\033[0m ({path})")
                 else:

lfss/cli/log.py

@@ -0,0 +1,77 @@
+from typing import Optional
+import argparse
+import rich.console
+import logging
+import sqlite3
+from lfss.eng.log import eval_logline
+
+console = rich.console.Console()
+def levelstr2int(levelstr: str) -> int:
+    import sys
+    if sys.version_info < (3, 11):
+        return logging.getLevelName(levelstr.upper())
+    else:
+        return logging.getLevelNamesMapping()[levelstr.upper()]
+
+def view(
+    db_file: str, 
+    level: Optional[str] = None, 
+    offset: int = 0,
+    limit: int = 1000
+    ):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("SELECT * FROM log ORDER BY created DESC LIMIT ? OFFSET ?", (limit, offset))
+    else:
+        level_int = levelstr2int(level)
+        cursor.execute("SELECT * FROM log WHERE level >= ? ORDER BY created DESC LIMIT ? OFFSET ?", (level_int, limit, offset))
+    levelname_color = {
+        'DEBUG': 'blue',
+        'INFO': 'green',
+        'WARNING': 'yellow',
+        'ERROR': 'red',
+        'CRITICAL': 'bold red', 
+        'FATAL': 'bold red'
+    }
+    for row in cursor.fetchall():
+        log = eval_logline(row)
+        console.print(f"{log.created} [{levelname_color[log.levelname]}][{log.levelname}] [default]{log.message}")
+    conn.close()
+
+def trim(db_file: str, keep: int  = 1000, level: Optional[str] = None):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("DELETE FROM log WHERE id NOT IN (SELECT id FROM log ORDER BY created DESC LIMIT ?)", (keep,))
+    else:
+        cursor.execute("DELETE FROM log WHERE levelname = ? and id NOT IN (SELECT id FROM log WHERE levelname = ? ORDER BY created DESC LIMIT ?)", (level.upper(), level.upper(), keep))
+    conn.commit()
+    conn.execute("VACUUM")
+    conn.close()
+
+def main():
+    parser = argparse.ArgumentParser(description="Log operations utility")
+    subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
+
+    parser_show = subparsers.add_parser('view', help='Show logs')
+    parser_show.add_argument('db_file', type=str, help='Database file path')
+    parser_show.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_show.add_argument('--offset', type=int, default=0, help='Starting offset')
+    parser_show.add_argument('--limit', type=int, default=1000, help='Maximum number of entries to display')
+    parser_show.set_defaults(func=view)
+
+    parser_trim = subparsers.add_parser('trim', help='Trim logs')
+    parser_trim.add_argument('db_file', type=str, help='Database file path')
+    parser_trim.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_trim.add_argument('--keep', type=int, default=1000, help='Number of entries to keep')
+    parser_trim.set_defaults(func=trim)
+
+    args = parser.parse_args()
+    if hasattr(args, 'func'):
+        kwargs = vars(args)
+        func = kwargs.pop('func')
+        func(**kwargs)
+
+if __name__ == '__main__':
+    main()

lfss/cli/vacuum.py

@@ -2,10 +2,11 @@
 Vacuum the database and external storage to ensure that the storage is consistent and minimal.
 """
 
-from lfss.eng.config import LARGE_BLOB_DIR
-import argparse, time
+from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB, LOG_DIR
+import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
+import aiosqlite
 import aiofiles, asyncio
 import aiofiles.os
 from contextlib import contextmanager
@@ -13,6 +14,7 @@ from lfss.eng.database import transaction, unique_cursor
 from lfss.svc.request_log import RequestDB
 from lfss.eng.utils import now_stamp
 from lfss.eng.connection_pool import global_entrance
+from lfss.cli.log import trim
 
 sem: Semaphore
 
@@ -32,7 +34,7 @@ def barriered(func):
     return wrapper
 
 @global_entrance()
-async def vacuum_main(index: bool = False, blobs: bool = False):
+async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, logs: bool = False, vacuum_all: bool = False):
 
     # check if any file in the Large Blob directory is not in the database
     # the reverse operation is not necessary, because by design, the database should be the source of truth...
@@ -49,23 +51,68 @@ async def vacuum_main(index: bool = False, blobs: bool = False):
 
     # create a temporary index to speed up the process...
     with indicator("Clearing un-referenced files in external storage"):
-        async with transaction() as c:
-            await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
-        for i, f in enumerate(LARGE_BLOB_DIR.iterdir()):
-            f_id = f.name
-            await ensure_external_consistency(f_id)
-            if (i+1) % 1_000 == 0:
-                print(f"Checked {(i+1)//1000}k files in external storage.", end='\r')
-        async with transaction() as c:
-            await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
+        try:
+            async with transaction() as c:
+                await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
+            for i, f in enumerate(LARGE_BLOB_DIR.iterdir()):
+                f_id = f.name
+                await ensure_external_consistency(f_id)
+                if (i+1) % 1_000 == 0:
+                    print(f"Checked {(i+1)//1000}k files in external storage.", end='\r')
+        finally:
+            async with transaction() as c:
+                await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
 
-    async with unique_cursor(is_write=True) as c:
-        if index:
-            with indicator("VACUUM-index"):
+    if index or vacuum_all:
+        with indicator("VACUUM-index"):
+            async with transaction() as c:
+                await c.execute("DELETE FROM dupcount WHERE count = 0")
+            async with unique_cursor(is_write=True) as c:
                 await c.execute("VACUUM main")
-        if blobs:
-            with indicator("VACUUM-blobs"):
+    if blobs or vacuum_all:
+        with indicator("VACUUM-blobs"):
+            async with unique_cursor(is_write=True) as c:
                 await c.execute("VACUUM blobs")
+    
+    if logs or vacuum_all:
+        with indicator("VACUUM-logs"):
+            for log_file in LOG_DIR.glob("*.log.db"):
+                trim(str(log_file), keep=10_000)
+    
+    if thumbs or vacuum_all:
+        try:
+            async with transaction() as c:
+                await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
+            with indicator("VACUUM-thumbs"):
+                if not THUMB_DB.exists():
+                    raise FileNotFoundError("Thumbnail database not found.")
+                async with unique_cursor() as db_c:
+                    async with aiosqlite.connect(THUMB_DB) as t_conn:
+                        batch_size = 10_000
+                        for batch_count in itertools.count(start=0):
+                            exceeded_rows = list(await (await t_conn.execute(
+                                "SELECT file_id FROM thumbs LIMIT ? OFFSET ?",
+                                (batch_size, batch_size * batch_count)
+                            )).fetchall())
+                            if not exceeded_rows:
+                                break
+                            batch_ids = [row[0] for row in exceeded_rows]
+                            for f_id in batch_ids:
+                                cursor = await db_c.execute("SELECT file_id FROM fmeta WHERE file_id = ?", (f_id,))
+                                if not await cursor.fetchone():
+                                    print(f"Thumbnail {f_id} not found in database, removing from thumb cache.")
+                                    await t_conn.execute("DELETE FROM thumbs WHERE file_id = ?", (f_id,))
+                            print(f"Checked {batch_count+1} batches of {batch_size} thumbnails.")
+
+                        await t_conn.commit()
+                        await t_conn.execute("VACUUM")
+        except FileNotFoundError as e:
+            if "Thumbnail database not found." in str(e):
+                print("Thumbnail database not found, skipping.")
+            
+        finally:
+            async with transaction() as c:
+                await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
 
 async def vacuum_requests():
     with indicator("VACUUM-requests"):
@@ -76,15 +123,18 @@ async def vacuum_requests():
 def main():
     global sem
     parser = argparse.ArgumentParser(description="Balance the storage by ensuring that large file thresholds are met.")
+    parser.add_argument("--all", action="store_true", help="Vacuum all")
     parser.add_argument("-j", "--jobs", type=int, default=2, help="Number of concurrent jobs")
     parser.add_argument("-m", "--metadata", action="store_true", help="Vacuum metadata")
     parser.add_argument("-d", "--data", action="store_true", help="Vacuum blobs")
+    parser.add_argument("-t", "--thumb", action="store_true", help="Vacuum thumbnails")
     parser.add_argument("-r", "--requests", action="store_true", help="Vacuum request logs to only keep at most recent 1M rows in 7 days")
+    parser.add_argument("-l", "--logs", action="store_true", help="Trim log to keep at most recent 10k rows for each category")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data))
+    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, logs = args.logs, vacuum_all=args.all))
 
-    if args.requests:
+    if args.requests or args.all:
         asyncio.run(vacuum_requests())
 
 if __name__ == '__main__':

lfss/eng/config.py

@@ -11,17 +11,19 @@ if not DATA_HOME.exists():
 DATA_HOME = DATA_HOME.resolve().absolute()
 LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
+LOG_DIR = DATA_HOME / 'logs'
+
+DISABLE_LOGGING = os.environ.get('DISABLE_LOGGING', '0') == '1'
 
 # https://sqlite.org/fasterthanfs.html
 __env_large_file = os.environ.get('LFSS_LARGE_FILE', None)
 if __env_large_file is not None:
     LARGE_FILE_BYTES = parse_storage_size(__env_large_file)
 else:
-    LARGE_FILE_BYTES = 8 * 1024 * 1024  # 8MB
-MAX_MEM_FILE_BYTES = 128 * 1024 * 1024   # 128MB
-MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
+    LARGE_FILE_BYTES = 1 * 1024 * 1024  # 1MB
+MAX_MEM_FILE_BYTES = 128 * 1024 * 1024  # 128MB
 CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'
 
-THUMB_DB = DATA_HOME / 'thumbs.db'
-THUMB_SIZE = (48, 48)
+THUMB_DB = DATA_HOME / 'thumbs.v0-11.db'
+THUMB_SIZE = (64, 64)

lfss/eng/connection_pool.py

@@ -8,7 +8,7 @@ from functools import wraps
 from typing import Callable, Awaitable
 
 from .log import get_logger
-from .error import DatabaseLockedError
+from .error import DatabaseLockedError, DatabaseTransactionError
 from .config import DATA_HOME
 
 async def execute_sql(conn: aiosqlite.Connection | aiosqlite.Cursor, name: str):
@@ -147,6 +147,14 @@ def global_entrance(n_read: int = 1):
         return wrapper
     return decorator
 
+def handle_sqlite_error(e: Exception):
+    if 'database is locked' in str(e):
+        raise DatabaseLockedError from e
+    if 'cannot start a transaction within a transaction' in str(e):
+        get_logger('database', global_instance=True).error(f"Unexpected error: {e}")
+        raise DatabaseTransactionError from e
+    raise e
+
 @asynccontextmanager
 async def unique_cursor(is_write: bool = False):
     if not is_write:
@@ -155,9 +163,7 @@ async def unique_cursor(is_write: bool = False):
             try:
                 yield await connection_obj.conn.cursor()
             except Exception as e:
-                if 'database is locked' in str(e):
-                    raise DatabaseLockedError from e
-                raise e
+                handle_sqlite_error(e)
             finally:
                 await g_pool.release(connection_obj)
     else:
@@ -166,9 +172,7 @@ async def unique_cursor(is_write: bool = False):
             try:
                 yield await connection_obj.conn.cursor()
             except Exception as e:
-                if 'database is locked' in str(e):
-                    raise DatabaseLockedError from e
-                raise e
+                handle_sqlite_error(e)
             finally:
                 await g_pool.release(connection_obj)