lfss 0.11.1__py3-none-any.whl → 0.11.2__py3-none-any.whl

docs/changelog.md

@@ -1,3 +1,30 @@
+## 0.11
+
+### 0.11.2
+- Improve frontend directory upload feedback. 
+- Set default large file threashold to 1M. 
+- Increase default concurrent threads. 
+- Use sqlite for logging.
+- Add vacuum logs. 
+- Refactor: use dir for directory path. 
+
+### 0.11.1
+- Rename api `get_meta` function.
+- Frontend support upload directory.  
+- Fix admin put to non-exists user path. 
+
+### 0.11.0
+- Copy file as hard link. 
+- Add vacuum thumb and all.
+- Thumb database use file_id as index. 
+- improve username and url check with regular expression.
+
+## 0.10
+
+### 0.10.0
+- Inherit permission from path owner for `unset` permission files.
+- Add timeout and verify options for client api.
+- Bundle small files in memory.
 
 ## 0.9
 

frontend/login.js

@@ -3,7 +3,6 @@ import { createFloatingWindow, showPopup } from "./popup.js";
 
 /**
  * @import { store } from "./state.js";
- * @import { UserRecord } from "./api.js";
  * 
  * Shows the login panel if necessary.
  * @param {store} store - The store object.

frontend/popup.js

@@ -109,7 +109,14 @@ export function showPopup(content = '',  {
 } = {}){
     const popup = document.createElement("div");
     popup.classList.add("popup-window");
-    popup.innerHTML = showTime? `<span>[${new Date().toLocaleTimeString()}]</span> ${content}` : content;
+    /**
+     * @param {string} c
+     * @returns {void}
+     */
+    function setPopupContent(c){
+        popup.innerHTML = showTime? `<span>[${new Date().toLocaleTimeString()}]</span> ${c}` : c;
+    }
+    setPopupContent(content);
     popup.style.width = width;
     const popupHeight = '1rem';
     popup.style.height = popupHeight;
@@ -132,11 +139,19 @@ export function showPopup(content = '',  {
     if (level === "success") popup.style.backgroundColor = "darkgreen";
     document.body.appendChild(popup);
     shownPopups.push(popup);
-    window.setTimeout(() => {
+
+    function closePopup(){
         if (popup.parentNode) document.body.removeChild(popup);
         shownPopups.splice(shownPopups.indexOf(popup), 1);
         for (let i = 0; i < shownPopups.length; i++) {
             shownPopups[i].style.top = `${i * (parseInt(popupHeight) + 2*parseInt(paddingHeight))*1.2 + 0.5}rem`;
         }
-    }, timeout);
+    }
+
+    window.setTimeout(closePopup, timeout);
+    return {
+        elem: popup, 
+        setContent: setPopupContent, 
+        close: closePopup
+    }
 }

frontend/scripts.js

@@ -183,26 +183,29 @@ Are you sure you want to proceed?\
         `)){ return; }
 
         let counter = 0;
+        let totalCount = 0;
+        const uploadPopup = showPopup('Uploading multiple files...', {level: 'info', timeout: 999999});
         async function uploadFileFn(path, file){
-            const this_count = counter;
             try{
                 await uploadFile(conn, path, file, {conflict: 'overwrite'});
             }
             catch (err){
                 showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});
             }
-            console.log(`[${this_count}/${counter}] Uploaded file: ${path}`);
+            console.log(`[${counter}/${totalCount}] Uploaded file: ${path}`);
+            uploadPopup.setContent(`Uploading multiple files... [${counter}/${totalCount}]`);
         }
 
-        const promises = await forEachFile(e, async (relPath, filePromise) => {
+        const promises = await forEachFile(e, async (relPath, filePromiseFn) => {
             counter += 1;
-            const file = await filePromise;
+            const file = await filePromiseFn();
             await uploadFileFn(dstPath + relPath, file);
         });
+        totalCount = promises.length;
 
-        showPopup('Uploading multiple files...', {level: 'info', timeout: 3000});
         Promise.all(promises).then(
             () => {
+                window.setTimeout(uploadPopup.close, 3000);
                 showPopup('Upload success.', {level: 'success', timeout: 3000});
                 refreshFileList();
             }, 

frontend/utils.js

@@ -101,7 +101,7 @@ export function asHtmlText(text){
  * using the provided callback with a concurrency limit.
  *
  * @param {Event} e The drop event.
- * @param {(relPath: string, file: Promise<File>) => Promise<void>} callback A function
+ * @param {(relPath: string, file: () => Promise<File>) => Promise<void>} callback A function
  *        that receives the relative path and a promise for the File.
  * @param {number} [maxConcurrent=5] Maximum number of concurrent callback executions.
  * @returns {Promise<Promise<void>[]>} A promise resolving to an array of callback promises.
@@ -146,11 +146,10 @@ export async function forEachFile(e, callback, maxConcurrent = 16) {
     async function traverse(entry, path) {
         if (entry.isFile) {
             // Wrap file retrieval in a promise.
-            const filePromise = new Promise((resolve, reject) => {
-                entry.file(resolve, reject);
-            });
+            const filePromiseFn = () =>
+                new Promise((resolve, reject) => entry.file(resolve, reject));
             // Use the concurrency barrier for the callback invocation.
-            results.push(runWithLimit(() => callback(path + entry.name, filePromise)));
+            results.push(runWithLimit(() => callback(path + entry.name, filePromiseFn)));
         } else if (entry.isDirectory) {
             const reader = entry.createReader();
 

lfss/cli/log.py

@@ -0,0 +1,77 @@
+from typing import Optional
+import argparse
+import rich.console
+import logging
+import sqlite3
+from lfss.eng.log import eval_logline
+
+console = rich.console.Console()
+def levelstr2int(levelstr: str) -> int:
+    import sys
+    if sys.version_info < (3, 11):
+        return logging.getLevelName(levelstr.upper())
+    else:
+        return logging.getLevelNamesMapping()[levelstr.upper()]
+
+def view(
+    db_file: str, 
+    level: Optional[str] = None, 
+    offset: int = 0,
+    limit: int = 1000
+    ):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("SELECT * FROM log ORDER BY created DESC LIMIT ? OFFSET ?", (limit, offset))
+    else:
+        level_int = levelstr2int(level)
+        cursor.execute("SELECT * FROM log WHERE level >= ? ORDER BY created DESC LIMIT ? OFFSET ?", (level_int, limit, offset))
+    levelname_color = {
+        'DEBUG': 'blue',
+        'INFO': 'green',
+        'WARNING': 'yellow',
+        'ERROR': 'red',
+        'CRITICAL': 'bold red', 
+        'FATAL': 'bold red'
+    }
+    for row in cursor.fetchall():
+        log = eval_logline(row)
+        console.print(f"{log.created} [{levelname_color[log.levelname]}][{log.levelname}] [default]{log.message}")
+    conn.close()
+
+def trim(db_file: str, keep: int  = 1000, level: Optional[str] = None):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("DELETE FROM log WHERE id NOT IN (SELECT id FROM log ORDER BY created DESC LIMIT ?)", (keep,))
+    else:
+        cursor.execute("DELETE FROM log WHERE levelname = ? and id NOT IN (SELECT id FROM log WHERE levelname = ? ORDER BY created DESC LIMIT ?)", (level.upper(), level.upper(), keep))
+    conn.commit()
+    conn.execute("VACUUM")
+    conn.close()
+
+def main():
+    parser = argparse.ArgumentParser(description="Log operations utility")
+    subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
+
+    parser_show = subparsers.add_parser('view', help='Show logs')
+    parser_show.add_argument('db_file', type=str, help='Database file path')
+    parser_show.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_show.add_argument('--offset', type=int, default=0, help='Starting offset')
+    parser_show.add_argument('--limit', type=int, default=1000, help='Maximum number of entries to display')
+    parser_show.set_defaults(func=view)
+
+    parser_trim = subparsers.add_parser('trim', help='Trim logs')
+    parser_trim.add_argument('db_file', type=str, help='Database file path')
+    parser_trim.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_trim.add_argument('--keep', type=int, default=1000, help='Number of entries to keep')
+    parser_trim.set_defaults(func=trim)
+
+    args = parser.parse_args()
+    if hasattr(args, 'func'):
+        kwargs = vars(args)
+        func = kwargs.pop('func')
+        func(**kwargs)
+
+if __name__ == '__main__':
+    main()

lfss/cli/vacuum.py

@@ -2,7 +2,7 @@
 Vacuum the database and external storage to ensure that the storage is consistent and minimal.
 """
 
-from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB
+from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB, LOG_DIR
 import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
@@ -14,6 +14,7 @@ from lfss.eng.database import transaction, unique_cursor
 from lfss.svc.request_log import RequestDB
 from lfss.eng.utils import now_stamp
 from lfss.eng.connection_pool import global_entrance
+from lfss.cli.log import trim
 
 sem: Semaphore
 
@@ -33,7 +34,7 @@ def barriered(func):
     return wrapper
 
 @global_entrance()
-async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, vacuum_all: bool = False):
+async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, logs: bool = False, vacuum_all: bool = False):
 
     # check if any file in the Large Blob directory is not in the database
     # the reverse operation is not necessary, because by design, the database should be the source of truth...
@@ -73,6 +74,11 @@ async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = F
             async with unique_cursor(is_write=True) as c:
                 await c.execute("VACUUM blobs")
     
+    if logs or vacuum_all:
+        with indicator("VACUUM-logs"):
+            for log_file in LOG_DIR.glob("*.log.db"):
+                trim(str(log_file), keep=10_000)
+    
     if thumbs or vacuum_all:
         try:
             async with transaction() as c:
@@ -123,9 +129,10 @@ def main():
     parser.add_argument("-d", "--data", action="store_true", help="Vacuum blobs")
     parser.add_argument("-t", "--thumb", action="store_true", help="Vacuum thumbnails")
     parser.add_argument("-r", "--requests", action="store_true", help="Vacuum request logs to only keep at most recent 1M rows in 7 days")
+    parser.add_argument("-l", "--logs", action="store_true", help="Trim log to keep at most recent 10k rows for each category")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, vacuum_all=args.all))
+    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, logs = args.logs, vacuum_all=args.all))
 
     if args.requests or args.all:
         asyncio.run(vacuum_requests())

lfss/eng/config.py

@@ -11,14 +11,15 @@ if not DATA_HOME.exists():
 DATA_HOME = DATA_HOME.resolve().absolute()
 LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
+LOG_DIR = DATA_HOME / 'logs'
 
 # https://sqlite.org/fasterthanfs.html
 __env_large_file = os.environ.get('LFSS_LARGE_FILE', None)
 if __env_large_file is not None:
     LARGE_FILE_BYTES = parse_storage_size(__env_large_file)
 else:
-    LARGE_FILE_BYTES = 8 * 1024 * 1024  # 8MB
-MAX_MEM_FILE_BYTES = 128 * 1024 * 1024   # 128MB
+    LARGE_FILE_BYTES = 1 * 1024 * 1024  # 1MB
+MAX_MEM_FILE_BYTES = 128 * 1024 * 1024  # 128MB
 CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'
 

lfss/eng/database.py

@@ -225,12 +225,12 @@ class FileConn(DBObjectBase):
             await self.cur.execute("SELECT username FROM user")
             res = await self.cur.fetchall()
             dirnames = [u[0] + '/' for u in res]
-            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
+            dirs = [await self.get_dir_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
         else:
             # list specific users
             dirnames = [uname + '/' for uname in usernames]
-            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
+            dirs = [await self.get_dir_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
     
     async def count_path_dirs(self, url: str):
@@ -278,11 +278,11 @@ class FileConn(DBObjectBase):
             if skim:
                 return DirectoryRecord(dir_url)
             else:
-                return await self.get_path_record(dir_url)
+                return await self.get_dir_record(dir_url)
         dirs = [await get_dir(url + d) for d in dirs_str]
         return dirs
     
-    async def count_path_files(self, url: str, flat: bool = False):
+    async def count_dir_files(self, url: str, flat: bool = False):
         if not url.endswith('/'): url += '/'
         if url == '/': url = ''
         if flat:
@@ -293,7 +293,7 @@ class FileConn(DBObjectBase):
         assert res is not None, "Error: count_path_files"
         return res[0]
 
-    async def list_path_files(
+    async def list_dir_files(
         self, url: str, 
         offset: int = 0, limit: int = 10_000, 
         order_by: FileSortKey = '', order_desc: bool = False,
@@ -328,15 +328,15 @@ class FileConn(DBObjectBase):
         """
         MAX_ITEMS = 10_000
         dir_count = await self.count_path_dirs(url)
-        file_count = await self.count_path_files(url, flat=False)
+        file_count = await self.count_dir_files(url, flat=False)
         if dir_count + file_count > MAX_ITEMS:
             raise TooManyItemsError("Too many items, please paginate")
         return PathContents(
             dirs = await self.list_path_dirs(url, skim=True, limit=MAX_ITEMS),
-            files = await self.list_path_files(url, flat=False, limit=MAX_ITEMS)
+            files = await self.list_dir_files(url, flat=False, limit=MAX_ITEMS)
             )
     
-    async def get_path_record(self, url: str) -> DirectoryRecord:
+    async def get_dir_record(self, url: str) -> DirectoryRecord:
         """
         Get the full record of a directory, including size, create_time, update_time, access_time etc.
         """
@@ -411,7 +411,6 @@ class FileConn(DBObjectBase):
         await self._user_size_inc(owner_id, file_size)
         self.logger.info(f"File {url} created")
 
-    # not tested
     async def copy_file(self, old_url: str, new_url: str, user_id: Optional[int] = None):
         old = await self.get_file_record(old_url)
         if old is None:
@@ -428,7 +427,7 @@ class FileConn(DBObjectBase):
         await self._user_size_inc(user_id, old.file_size)
         self.logger.info(f"Copied file {old_url} to {new_url}")
     
-    async def copy_path(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+    async def copy_dir(self, old_url: str, new_url: str, user_id: Optional[int] = None):
         assert old_url.endswith('/'), "Old path must end with /"
         assert new_url.endswith('/'), "New path must end with /"
         if user_id is None:
@@ -461,7 +460,7 @@ class FileConn(DBObjectBase):
         await self.cur.execute("UPDATE fmeta SET url = ?, create_time = CURRENT_TIMESTAMP WHERE url = ?", (new_url, old_url))
         self.logger.info(f"Moved file {old_url} to {new_url}")
     
-    async def move_path(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+    async def move_dir(self, old_url: str, new_url: str, user_id: Optional[int] = None):
         assert old_url.endswith('/'), "Old path must end with /"
         assert new_url.endswith('/'), "New path must end with /"
         if user_id is None:
@@ -500,7 +499,7 @@ class FileConn(DBObjectBase):
         self.logger.info(f"Deleted {len(ret)} file records for user {owner_id}") # type: ignore
         return ret
     
-    async def delete_path_records(self, path: str, under_owner_id: Optional[int] = None) -> list[FileRecord]:
+    async def delete_records_by_prefix(self, path: str, under_owner_id: Optional[int] = None) -> list[FileRecord]:
         """Delete all records with url starting with path"""
         # update user size
         cursor = await self.cur.execute("SELECT DISTINCT owner_id FROM fmeta WHERE url LIKE ?", (path + '%', ))
@@ -689,7 +688,7 @@ async def delayed_log_access(url: str):
         ])
     ),
 )
-def validate_url(url: str, is_file = True):
+def validate_url(url: str, utype: Literal['file', 'dir'] = 'file'):
     """ Check if a path is valid. The input path is considered url safe """
     if len(url) > 1024: 
         raise InvalidPathError(f"URL too long: {url}")
@@ -703,7 +702,7 @@ def validate_url(url: str, is_file = True):
             is_valid = False
             break
 
-    if is_file: is_valid = is_valid and not url.endswith('/')
+    if utype == 'file': is_valid = is_valid and not url.endswith('/')
     else: is_valid = is_valid and url.endswith('/')
 
     if not is_valid: 
@@ -885,9 +884,9 @@ class Database:
                     raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot copy file to {new_url}")
             await fconn.copy_file(old_url, new_url, user_id=op_user.id if op_user is not None else None)
     
-    async def move_path(self, old_url: str, new_url: str, op_user: UserRecord):
-        validate_url(old_url, is_file=False)
-        validate_url(new_url, is_file=False)
+    async def move_dir(self, old_url: str, new_url: str, op_user: UserRecord):
+        validate_url(old_url, 'dir')
+        validate_url(new_url, 'dir')
 
         if new_url.startswith('/'):
             new_url = new_url[1:]
@@ -906,12 +905,11 @@ class Database:
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            await fconn.move_path(old_url, new_url, op_user.id)
+            await fconn.move_dir(old_url, new_url, op_user.id)
     
-    # not tested
-    async def copy_path(self, old_url: str, new_url: str, op_user: UserRecord):
-        validate_url(old_url, is_file=False)
-        validate_url(new_url, is_file=False)
+    async def copy_dir(self, old_url: str, new_url: str, op_user: UserRecord):
+        validate_url(old_url, 'dir')
+        validate_url(new_url, 'dir')
         
         if new_url.startswith('/'):
             new_url = new_url[1:]
@@ -930,7 +928,7 @@ class Database:
         
         async with transaction() as cur:
             fconn = FileConn(cur)
-            await fconn.copy_path(old_url, new_url, op_user.id)
+            await fconn.copy_dir(old_url, new_url, op_user.id)
 
     async def __batch_delete_file_blobs(self, fconn: FileConn, file_records: list[FileRecord], batch_size: int = 512):
         # https://github.com/langchain-ai/langchain/issues/10321
@@ -951,13 +949,13 @@ class Database:
         await del_internal()
         await del_external()
 
-    async def delete_path(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
-        validate_url(url, is_file=False)
+    async def delete_dir(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
+        validate_url(url, 'dir')
         from_owner_id = op_user.id if op_user is not None and not (op_user.is_admin or await check_path_permission(url, op_user) >= AccessLevel.WRITE) else None
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            records = await fconn.delete_path_records(url, from_owner_id)
+            records = await fconn.delete_records_by_prefix(url, from_owner_id)
             if not records:
                 return None
             await self.__batch_delete_file_blobs(fconn, records)
@@ -981,14 +979,15 @@ class Database:
 
             # make sure the user's directory is deleted, 
             # may contain admin's files, but delete them all
-            await fconn.delete_path_records(user.username + '/')
+            await fconn.delete_records_by_prefix(user.username + '/')
     
-    async def iter_path(self, top_url: str, urls: Optional[list[str]]) -> AsyncIterable[tuple[FileRecord, bytes | AsyncIterable[bytes]]]:
+    async def iter_dir(self, top_url: str, urls: Optional[list[str]]) -> AsyncIterable[tuple[FileRecord, bytes | AsyncIterable[bytes]]]:
+        validate_url(top_url, 'dir')
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
             if urls is None:
-                fcount = await fconn.count_path_files(top_url, flat=True)
-                urls = [r.url for r in (await fconn.list_path_files(top_url, flat=True, limit=fcount))]
+                fcount = await fconn.count_dir_files(top_url, flat=True)
+                urls = [r.url for r in (await fconn.list_dir_files(top_url, flat=True, limit=fcount))]
 
             for url in urls:
                 if not url.startswith(top_url):
@@ -1003,7 +1002,7 @@ class Database:
                     blob = await fconn.get_file_blob(f_id)
                 yield r, blob
     
-    async def zip_path_stream(self, top_url: str, op_user: Optional[UserRecord] = None) -> AsyncIterable[bytes]:
+    async def zip_dir_stream(self, top_url: str, op_user: Optional[UserRecord] = None) -> AsyncIterable[bytes]:
         from stat import S_IFREG
         from stream_zip import async_stream_zip, ZIP_64
         if top_url.startswith('/'):
@@ -1015,7 +1014,7 @@ class Database:
         
         # https://stream-zip.docs.trade.gov.uk/async-interface/
         async def data_iter():
-            async for (r, blob) in self.iter_path(top_url, None):
+            async for (r, blob) in self.iter_dir(top_url, None):
                 rel_path = r.url[len(top_url):]
                 rel_path = decode_uri_compnents(rel_path)
                 b_iter: AsyncIterable[bytes]
@@ -1035,7 +1034,7 @@ class Database:
         return async_stream_zip(data_iter())
 
     @concurrent_wrap()
-    async def zip_path(self, top_url: str, op_user: Optional[UserRecord]) -> io.BytesIO:
+    async def zip_dir(self, top_url: str, op_user: Optional[UserRecord]) -> io.BytesIO:
         if top_url.startswith('/'):
             top_url = top_url[1:]
         
@@ -1045,7 +1044,7 @@ class Database:
         
         buffer = io.BytesIO()
         with zipfile.ZipFile(buffer, 'w') as zf:
-            async for (r, blob) in self.iter_path(top_url, None):
+            async for (r, blob) in self.iter_dir(top_url, None):
                 rel_path = r.url[len(top_url):]
                 rel_path = decode_uri_compnents(rel_path)
                 if r.external:

lfss/eng/log.py

@@ -1,4 +1,5 @@
-from .config import DATA_HOME
+from .config import LOG_DIR
+import time, sqlite3, dataclasses
 from typing import TypeVar, Callable, Literal, Optional
 from concurrent.futures import ThreadPoolExecutor
 from functools import wraps
@@ -57,15 +58,81 @@ class BaseLogger(logging.Logger):
     @thread_wrap
     def error(self, *args, **kwargs): super().error(*args, **kwargs)
 
-_fh_T = Literal['rotate', 'simple', 'daily']
+class SQLiteFileHandler(logging.FileHandler):
+    def __init__(self, filename, *args, **kwargs):
+        super().__init__(filename, *args, **kwargs)
+        self._db_file = filename
+        self._buffer: list[logging.LogRecord] = []
+        self._buffer_size = 100
+        self._flush_interval = 10
+        self._last_flush = time.time()
+        conn = sqlite3.connect(self._db_file, check_same_thread=False)
+        conn.execute('PRAGMA journal_mode=WAL')
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS log (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                created TIMESTAMP,
+                created_epoch FLOAT,
+                name TEXT,
+                levelname VARCHAR(16),
+                level INTEGER,
+                message TEXT
+            )
+        ''')
+        conn.commit()
+        conn.close()
+    
+    def flush(self): 
+        def format_time(self, record: logging.LogRecord):
+            """ Create a time stamp """
+            return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created))
+        self.acquire()
+        try:
+            conn = sqlite3.connect(self._db_file, check_same_thread=False)
+            conn.executemany('''
+                INSERT INTO log (created, created_epoch, name, levelname, level, message)
+                VALUES (?, ?, ?, ?, ?, ?)
+            ''', [
+                (format_time(self, record), record.created, record.name, record.levelname, record.levelno, record.getMessage())
+                for record in self._buffer
+            ])
+            conn.commit()
+            conn.close()
+            self._buffer.clear()
+            self._last_flush = time.time()
+        finally:
+            self.release()
+
+    def emit(self, record: logging.LogRecord):
+        self._buffer.append(record)
+        if len(self._buffer) > self._buffer_size or time.time() - self._last_flush > self._flush_interval:
+            self.flush()
+    
+    def close(self):
+        self.flush()
+        return super().close()
+
+def eval_logline(row: sqlite3.Row):
+    @dataclasses.dataclass
+    class DBLogRecord:
+        id: int
+        created: str
+        created_epoch: float
+        name: str
+        levelname: str
+        level: int
+        message: str
+    return DBLogRecord(*row)
+
+_fh_T = Literal['rotate', 'simple', 'daily', 'sqlite']
 
 __g_logger_dict: dict[str, BaseLogger] = {}
 def get_logger(
     name = 'default', 
-    log_home = pathlib.Path(DATA_HOME) / 'logs', 
+    log_home = LOG_DIR, 
     level = 'DEBUG',
     term_level = 'INFO',
-    file_handler_type: _fh_T = 'rotate', 
+    file_handler_type: _fh_T = 'sqlite', 
     global_instance = True
     )->BaseLogger:
     if global_instance and name in __g_logger_dict:
@@ -100,6 +167,8 @@ def get_logger(
             file_handler = handlers.RotatingFileHandler(
                 log_file, maxBytes=1024*1024, backupCount=5
             )
+        elif file_handler_type == 'sqlite':
+            file_handler = SQLiteFileHandler(log_file if log_file.suffix == '.db' else log_file.with_suffix('.log.db'))
 
         file_handler.setFormatter(formatter_plain)
         logger.addHandler(file_handler)

lfss/eng/utils.py

@@ -11,7 +11,6 @@ from concurrent.futures import ThreadPoolExecutor
 from typing import TypeVar, Callable, Awaitable
 from functools import wraps, partial
 from uuid import uuid4
-import os
 
 async def copy_file(source: str|pathlib.Path, destination: str|pathlib.Path):
     async with aiofiles.open(source, mode='rb') as src:
@@ -160,7 +159,7 @@ _g_executor = None
 def get_global_executor():
     global _g_executor
     if _g_executor is None:
-        _g_executor = ThreadPoolExecutor(max_workers=4 if (cpu_count:=os.cpu_count()) and cpu_count > 4 else cpu_count)
+        _g_executor = ThreadPoolExecutor()
     return _g_executor
 def async_wrap(executor=None):
     if executor is None:

lfss/svc/app_dav.py

@@ -57,9 +57,9 @@ async def eval_path(path: str) -> tuple[ptype, str, Optional[FileRecord | Direct
         if len(dir_path_sp) > 2:
             async with unique_cursor() as c:
                 fconn = FileConn(c)
-                if await fconn.count_path_files(path, flat=True) == 0:
+                if await fconn.count_dir_files(path, flat=True) == 0:
                     return None, lfss_path, None
-                return "dir", lfss_path, await fconn.get_path_record(path)
+                return "dir", lfss_path, await fconn.get_dir_record(path)
         else:
             # test if its a user's root directory
             assert len(dir_path_sp) == 2
@@ -85,8 +85,8 @@ async def eval_path(path: str) -> tuple[ptype, str, Optional[FileRecord | Direct
     async with unique_cursor() as c:
         lfss_path = path + "/"
         fconn = FileConn(c)
-        if await fconn.count_path_files(lfss_path) > 0:
-            return "dir", lfss_path, await fconn.get_path_record(lfss_path)
+        if await fconn.count_dir_files(lfss_path) > 0:
+            return "dir", lfss_path, await fconn.get_dir_record(lfss_path)
     
     return None, path, None
 
@@ -235,7 +235,7 @@ async def dav_propfind(request: Request, path: str, user: UserRecord = Depends(r
         # query root directory content
         async def user_path_record(user_name: str, cur) -> DirectoryRecord:
             try:
-                return await FileConn(cur).get_path_record(user_name + "/")
+                return await FileConn(cur).get_dir_record(user_name + "/")
             except PathNotFoundError:
                 return DirectoryRecord(user_name + "/", size=0, n_files=0, create_time="1970-01-01 00:00:00", update_time="1970-01-01 00:00:00", access_time="1970-01-01 00:00:00")
 
@@ -253,7 +253,7 @@ async def dav_propfind(request: Request, path: str, user: UserRecord = Depends(r
     elif path_type == "dir":
         # query directory content
         async with unique_cursor() as c:
-            flist = await FileConn(c).list_path_files(lfss_path, flat = True if depth == "infinity" else False)
+            flist = await FileConn(c).list_dir_files(lfss_path, flat = True if depth == "infinity" else False)
         for frecord in flist:
             if frecord.url.endswith(f"/{MKDIR_PLACEHOLDER}"): continue
             file_el = await create_file_xml_element(frecord)
@@ -315,7 +315,7 @@ async def dav_move(request: Request, path: str, user: UserRecord = Depends(regis
         assert ptype == "dir", "Directory path should end with /"
         assert lfss_path.endswith("/"), "Directory path should end with /"
         if not dlfss_path.endswith("/"): dlfss_path += "/"  # the header destination may not end with /
-        await db.move_path(lfss_path, dlfss_path, user)
+        await db.move_dir(lfss_path, dlfss_path, user)
     return Response(status_code=201)
 
 @router_dav.api_route("/{path:path}", methods=["COPY"])

lfss/svc/app_native.py

@@ -90,13 +90,13 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
         raise HTTPException(status_code=400, detail="Cannot bundle root")
     
     async with unique_cursor() as cur:
-        dir_record = await FileConn(cur).get_path_record(path)
+        dir_record = await FileConn(cur).get_dir_record(path)
 
     pathname = f"{path.split('/')[-2]}"
 
     if dir_record.size < MAX_MEM_FILE_BYTES:
         logger.debug(f"Bundle {path} in memory")
-        dir_bytes = (await db.zip_path(path, op_user=user)).getvalue()
+        dir_bytes = (await db.zip_dir(path, op_user=user)).getvalue()
         return Response(
             content = dir_bytes,
             media_type = "application/zip",
@@ -109,7 +109,7 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
     else:
         logger.debug(f"Bundle {path} in stream")
         return StreamingResponse(
-            content = await db.zip_path_stream(path, op_user=user),
+            content = await db.zip_dir_stream(path, op_user=user),
             media_type = "application/zip",
             headers = {
                 f"Content-Disposition": f"attachment; filename=bundle-{pathname}.zip",
@@ -134,7 +134,7 @@ async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
         else:
             if await check_path_permission(path, user, cursor=cur) < AccessLevel.READ:
                 raise HTTPException(status_code=403, detail="Permission denied")
-            record = await fconn.get_path_record(path)
+            record = await fconn.get_dir_record(path)
     return record
 
 @router_api.post("/meta")
@@ -171,7 +171,7 @@ async def update_file_meta(
             new_path = ensure_uri_compnents(new_path)
             logger.info(f"Update path of {path} to {new_path}")
             # will raise duplicate path error if same name path exists in the new path
-            await db.move_path(path, new_path, user)
+            await db.move_dir(path, new_path, user)
 
     return Response(status_code=200, content="OK")
 
@@ -194,7 +194,7 @@ async def count_files(path: str, flat: bool = False, user: UserRecord = Depends(
     path = ensure_uri_compnents(path)
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        return { "count": await fconn.count_path_files(url = path, flat = flat) }
+        return { "count": await fconn.count_dir_files(url = path, flat = flat) }
 @router_api.get("/list-files")
 async def list_files(
     path: str, offset: int = 0, limit: int = 1000,
@@ -205,7 +205,7 @@ async def list_files(
     path = ensure_uri_compnents(path)
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        return await fconn.list_path_files(
+        return await fconn.list_dir_files(
             url = path, offset = offset, limit = limit,
             order_by=order_by, order_desc=order_desc, 
             flat=flat

lfss/svc/common_impl.py

@@ -180,7 +180,7 @@ async def _get_dir_impl(
                 else:
                     raise HTTPException(status_code=404, detail="User not found")
             else:
-                if await FileConn(cur).count_path_files(path, flat=True) > 0:
+                if await FileConn(cur).count_dir_files(path, flat=True) > 0:
                     return Response(status_code=200)
                 else:
                     raise HTTPException(status_code=404, detail="Path not found")
@@ -295,7 +295,7 @@ async def delete_impl(path: str, user: UserRecord):
     logger.info(f"DELETE {path}, user: {user.username}")
 
     if path.endswith("/"):
-        res = await db.delete_path(path, user)
+        res = await db.delete_dir(path, user)
     else:
         res = await db.delete_file(path, user)
 
@@ -327,8 +327,8 @@ async def copy_impl(
     else:
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
-            dst_fcount = await fconn.count_path_files(dst_path, flat=True)
+            dst_fcount = await fconn.count_dir_files(dst_path, flat=True)
         if dst_fcount > 0:
             raise HTTPException(status_code=409, detail="Destination exists")
-        await db.copy_path(src_path, dst_path, op_user)
+        await db.copy_dir(src_path, dst_path, op_user)
     return Response(status_code=201, content="OK")

{lfss-0.11.1.dist-info → lfss-0.11.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.11.1
+Version: 0.11.2
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: Li, Mengxun
@@ -17,6 +17,7 @@ Requires-Dist: mimesniff (==1.*)
 Requires-Dist: pillow
 Requires-Dist: python-multipart
 Requires-Dist: requests (==2.*)
+Requires-Dist: rich
 Requires-Dist: stream-zip (==0.*)
 Requires-Dist: uvicorn (==0.*)
 Project-URL: Repository, https://github.com/MenxLi/lfss

{lfss-0.11.1.dist-info → lfss-0.11.2.dist-info}/RECORD

@@ -3,49 +3,50 @@ docs/Enviroment_variables.md,sha256=xaL8qBwT8B2Qe11FaOU3xWrRCh1mJ1VyTFCeFbkd0rs,
 docs/Known_issues.md,sha256=ZqETcWP8lzTOel9b2mxEgCnADFF8IxOrEtiVO1NoMAk,251
 docs/Permission.md,sha256=thUJx7YRoU63Pb-eqo5l5450DrZN3QYZ36GCn8r66no,3152
 docs/Webdav.md,sha256=-Ja-BTWSY1BEMAyZycvEMNnkNTPZ49gSPzmf3Lbib70,1547
-docs/changelog.md,sha256=QYej_hmGnv9t8wjFHXBvmrBOvY7aACZ82oa5SVkIyzM,882
+docs/changelog.md,sha256=fE0rE2IcovbxMhdTeqhnCnknT1vtVr7A860zIh7AEnE,1581
 frontend/api.js,sha256=GlQsNoZFEcy7QUUsLbXv7aP-KxRnIxM37FQHTaakGiQ,19387
 frontend/index.html,sha256=-k0bJ5FRqdl_H-O441D_H9E-iejgRCaL_z5UeYaS2qc,3384
 frontend/info.css,sha256=Ny0N3GywQ3a9q1_Qph_QFEKB4fEnTe_2DJ1Y5OsLLmQ,595
 frontend/info.js,sha256=xGUJPCSrtDhuSu0ELLQZ77PmVWldg-prU1mwQGbdEoA,5797
 frontend/login.css,sha256=VMM0QfbDFYerxKWKSGhMI1yg5IRBXg0TTdLJEEhQZNk,355
-frontend/login.js,sha256=QoO8yKmBHDVP-ZomCMOaV7xVUVIhpl7esJrb6T5aHQE,2466
+frontend/login.js,sha256=xJkulk8dlvV4BhevADLeUrnZwShiFTWv3Wg2iJFUZlY,2423
 frontend/popup.css,sha256=TJZYFW1ZcdD1IVTlNPYNtMWKPbN6XDbQ4hKBOFK8uLg,1284
-frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
-frontend/scripts.js,sha256=OAx6o3Aabx-cE41uBABP62myZM8WbLxY37uXITMl8nY,24204
+frontend/popup.js,sha256=cyUjtO0wbtqbEodHfwyUsak9iWbcDXeWMGDhpCPbcoE,5453
+frontend/scripts.js,sha256=T3kMjTxrjOkp93OV4ZMGgCLRRaQgRmNzzxriOMGVeZM,24412
 frontend/state.js,sha256=vbNL5DProRKmSEY7xu9mZH6IY0PBenF8WGxPtGgDnLI,1680
 frontend/styles.css,sha256=xcNLqI3KBsY5TLnku8UIP0Jfr7QLajr1_KNlZj9eheM,4935
 frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
 frontend/thumb.js,sha256=46ViD2TlTTWy0fx6wjoAs_5CQ4ajYB90vVzM7UO2IHw,6182
-frontend/utils.js,sha256=jqAZ7Xhlk8ZI97BRnd1dpFJcW0kPrN216xSFnrTT6zk,6069
+frontend/utils.js,sha256=XP5hM_mROYaxK5dqn9qZVwv7GdQuiDzByilFskbrnxA,6068
 lfss/api/__init__.py,sha256=zT1JCiUM76wX-GtRrmKhTUzSYYfcmoyI1vYwN0fCcLw,6818
 lfss/api/connector.py,sha256=xl_WrvupplepZSYJs4pN9zN7GDnuZR2A8-pc08ILutI,13231
 lfss/cli/__init__.py,sha256=lPwPmqpa7EXQ4zlU7E7LOe6X2kw_xATGdwoHphUEirA,827
 lfss/cli/balance.py,sha256=fUbKKAUyaDn74f7mmxMfBL4Q4voyBLHu6Lg_g8GfMOQ,4121
 lfss/cli/cli.py,sha256=tPeUgj0BR_M649AGcBYwfsrGioes0qzGc0lghFkrjoo,8086
+lfss/cli/log.py,sha256=TBlt8mhHMouv8ZBUMHYfGZiV6-0yPdajJQ5mkGHEojI,3016
 lfss/cli/panel.py,sha256=Xq3I_n-ctveym-Gh9LaUpzHiLlvt3a_nuDiwUS-MGrg,1597
 lfss/cli/serve.py,sha256=vTo6_BiD7Dn3VLvHsC5RKRBC3lMu45JVr_0SqpgHdj0,1086
 lfss/cli/user.py,sha256=1mTroQbaKxHjFCPHT67xwd08v-zxH0RZ_OnVc-4MzL0,5364
-lfss/cli/vacuum.py,sha256=SciDsIdy7cfRqrXcCKBAFb9FOLyXriZBZnXlCuy6F5I,6232
+lfss/cli/vacuum.py,sha256=arEY89kYJKEpzuzjKtf21V7s0QzM1t3QWa1hNghhT0Q,6611
 lfss/eng/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lfss/eng/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
-lfss/eng/config.py,sha256=FcTtPL7bOpg54nVL_gX-VTIjfN1cafy423ezoWGvouY,874
+lfss/eng/config.py,sha256=vP-0h_9TkAfu5626KjowHjCgX-CnVGZajw3sxBs5jtU,902
 lfss/eng/connection_pool.py,sha256=1aq7nSgd7hB9YNV4PjD1RDRyl_moDw3ubBtSLyfgGBs,6320
-lfss/eng/database.py,sha256=-6-IgR6hXe4ouMH8e0Ryeh2gZXJBpna1ech41sZ3UYs,53267
+lfss/eng/database.py,sha256=huYSOvTO5jES9wVl6Zity2XzNXyJBSQQwuCQHrEVf-Q,53255
 lfss/eng/datatype.py,sha256=27UB7-l9SICy5lAvKjdzpTL_GohZjzstQcr9PtAq7nM,2709
 lfss/eng/error.py,sha256=JGf5NV-f4rL6tNIDSAx5-l9MG8dEj7F2w_MuOjj1d1o,732
-lfss/eng/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
+lfss/eng/log.py,sha256=jJKOnC64Lb5EoVJK_oi7vl4iRrH_gtCKM_zjHiIUA-4,7590
 lfss/eng/thumb.py,sha256=AFyWEkkpuCKGWOB9bLlaDwPKzQ9JtCSSmHMhX2Gu3CI,3096
-lfss/eng/utils.py,sha256=WYoXFFi5308UWtFC8VP792gpzrVbHZZHhP3PaFjxIEY,6770
+lfss/eng/utils.py,sha256=jQUJWWmzOPmXdTCId2Y307m1cZfB4hpzHcTjO0mkOrU,6683
 lfss/sql/init.sql,sha256=FBmVzkNjYUnWjEELRFzf7xb50GngmzmeDVffT1Uk8u8,1625
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/svc/app.py,sha256=r1KUO3sPaaJWbkJF0bcVTD7arPKLs2jFlq52Ixicomo,220
 lfss/svc/app_base.py,sha256=bTQbz945xalyB3UZLlqVBvL6JKGNQ8Fm2KpIvvucPZQ,6850
-lfss/svc/app_dav.py,sha256=D0KSgjtTktPjIhyIKG5eRmBdh5X8HYFYH151E6gzlbc,18245
-lfss/svc/app_native.py,sha256=JbPge-F9irl26tXKAzfA5DfyjCh0Dgttflztqqrvt0A,8890
-lfss/svc/common_impl.py,sha256=5ZRM24zVZpAeipgDtZUVBMFtArkydlAkn17ic_XL7v8,13733
+lfss/svc/app_dav.py,sha256=H3aL3MEdYaPK1w3FQvTzrGYGaaow4m8LZ7R35MN351A,18238
+lfss/svc/app_native.py,sha256=_dhcq_R1VoafRCLuuWxXuttuhBAVaFVdlIQ6ep6ZQvs,8883
+lfss/svc/common_impl.py,sha256=7QflWnxRqghLOSMpDz2UCRqEn49X1GLS3agCb5msia8,13729
 lfss/svc/request_log.py,sha256=v8yXEIzPjaksu76Oh5vgdbUEUrw8Kt4etLAXBWSGie8,3207
-lfss-0.11.1.dist-info/METADATA,sha256=qXJcsBI6dboEavUMZcRuCQFLzQ8i5cUqWg5OJWrTr8k,2712
-lfss-0.11.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.11.1.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
-lfss-0.11.1.dist-info/RECORD,,
+lfss-0.11.2.dist-info/METADATA,sha256=__YXS_WBv6oNQlzcamUPEWayjek6bVsF4zRoGR0iJb8,2732
+lfss-0.11.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.11.2.dist-info/entry_points.txt,sha256=R4uOP1y6eD0Qp3j1ySA8kRPVMdt6_W_9o-Zj9Ra4D0A,232
+lfss-0.11.2.dist-info/RECORD,,

{lfss-0.11.1.dist-info → lfss-0.11.2.dist-info}/entry_points.txt

@@ -1,6 +1,7 @@
 [console_scripts]
 lfss-balance=lfss.cli.balance:main
 lfss-cli=lfss.cli.cli:main
+lfss-log=lfss.cli.log:main
 lfss-panel=lfss.cli.panel:main
 lfss-serve=lfss.cli.serve:main
 lfss-user=lfss.cli.user:main