lfss 0.11.1__py3-none-any.whl → 0.11.3__py3-none-any.whl

docs/Enviroment_variables.md

@@ -4,8 +4,10 @@
 **Server**
 - `LFSS_DATA`: The directory to store the data. Default is `.storage_data`.
 - `LFSS_WEBDAV`: Enable WebDAV support. Default is `0`, set to `1` to enable.
-- `LFSS_LARGE_FILE`: The size limit of the file to store in the database. Default is `8m`.
+- `LFSS_LARGE_FILE`: The size limit of the file to store in the database. Default is `1m`.
 - `LFSS_DEBUG`: Enable debug mode for more verbose logging. Default is `0`, set to `1` to enable.
+- `LFSS_DISABLE_LOGGING`: Disable all file logging. Default is 0; set to `1` to disable file logging. 
+- `LFSS_ORIGIN`: The `Origin` header to allow CORS requests. Use `,` to separate multiple origins. Default is `*`.
 
 **Client**
 - `LFSS_ENDPOINT`: The fallback server endpoint. Default is `http://localhost:8000`.

docs/changelog.md

@@ -1,3 +1,30 @@
+## 0.11
+
+### 0.11.2
+- Improve frontend directory upload feedback. 
+- Set default large file threashold to 1M. 
+- Increase default concurrent threads. 
+- Use sqlite for logging.
+- Add vacuum logs. 
+- Refactor: use dir for directory path. 
+
+### 0.11.1
+- Rename api `get_meta` function.
+- Frontend support upload directory.  
+- Fix admin put to non-exists user path. 
+
+### 0.11.0
+- Copy file as hard link. 
+- Add vacuum thumb and all.
+- Thumb database use file_id as index. 
+- improve username and url check with regular expression.
+
+## 0.10
+
+### 0.10.0
+- Inherit permission from path owner for `unset` permission files.
+- Add timeout and verify options for client api.
+- Bundle small files in memory.
 
 ## 0.9
 

frontend/api.js

@@ -69,6 +69,10 @@ export default class Connector {
     /**
     * @param {string} path - the path to the file (url)
     * @param {File} file - the file to upload
+    * @param {Object} [options] - Optional upload configuration.
+    * @param {'abort' | 'overwrite' | 'skip'} [options.conflict='abort'] - Conflict resolution strategy:  
+    * `'abort'` to cancel and raise 409, `'overwrite'` to replace.
+    * @param {number} [options.permission=0] - Optional permission setting for the file (refer to backend impl).
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
     async put(path, file, {
@@ -96,8 +100,12 @@ export default class Connector {
     }
 
     /**
-    * @param {string} path - the path to the file (url)
+    * @param {string} path - the path to the file (url), should end with .json
     * @param {File} file - the file to upload
+    * @param {Object} [options] - Optional upload configuration.
+    * @param {'abort' | 'overwrite' | 'skip'} [options.conflict='abort'] - Conflict resolution strategy:  
+    * `'abort'` to cancel and raise 409, `'overwrite'` to replace, `'skip'` to ignore if already exists.
+    * @param {number} [options.permission=0] - Optional permission setting for the file (refer to backend impl).
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
     async post(path, file, {
@@ -129,13 +137,23 @@ export default class Connector {
 
     /**
     * @param {string} path - the path to the file (url), should end with .json
-    * @param {Objec} data - the data to upload
+    * @param {Object} data - the data to upload
+    * @param {Object} [options] - Optional upload configuration.
+    * @param {'abort' | 'overwrite' | 'skip'} [options.conflict='abort'] - Conflict resolution strategy:  
+    * `'abort'` to cancel and raise 409, `'overwrite'` to replace, `'skip'` to ignore if already exists.
+    * @param {number} [options.permission=0] - Optional permission setting for the file (refer to backend impl).
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
-    async putJson(path, data){
+    async putJson(path, data, {
+        conflict = "overwrite", 
+        permission = 0
+    } = {}){
         if (!path.endsWith('.json')){ throw new Error('Upload object must end with .json'); }
         if (path.startsWith('/')){ path = path.slice(1); }
-        const res = await fetch(this.config.endpoint + '/' + path, {
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('conflict', conflict);
+        dst.searchParams.append('permission', permission);
+        const res = await fetch(dst.toString(), {
             method: 'PUT',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token,
@@ -149,6 +167,50 @@ export default class Connector {
         return (await res.json()).url;
     }
 
+    /**
+     * @param {string} path - the path to the file (url), should have content type application/json
+     * @returns {Promise<Object>} - return the json object
+    */
+    async getJson(path){
+        if (path.startsWith('/')){ path = path.slice(1); }
+        const res = await fetch(this.config.endpoint + '/' + path, {
+            method: 'GET',
+            headers: {
+             "Authorization": 'Bearer ' + this.config.token
+            },
+        });
+        if (res.status != 200){
+            throw new Error(`Failed to get object, status code: ${res.status}, message: ${await fmtFailedResponse(res)}`);
+        }
+        return await res.json();
+    }
+
+    /**
+     * @param {string[]} paths - the paths to the files (url), should have content type plain/text, application/json, etc.
+     * @param {Object} [options] - Optional configuration.
+     * @param {boolean} [options.skipContent=false] - If true, skips fetching content and returns a record of <path, ''>.
+     * @returns {Promise<Record<string, string | null>>} - return the mapping of path to text content, non-existing paths will be ignored
+     */
+    async getMultipleText(paths, {
+        skipContent = false
+    } = {}){
+        const url = new URL(this.config.endpoint + '/_api/get-multiple');
+        url.searchParams.append('skip_content', skipContent);
+        for (const path of paths){
+            url.searchParams.append('path', path);
+        }
+        const res = await fetch(url.toString(), {
+            method: 'GET',
+            headers: {
+                "Authorization": 'Bearer ' + this.config.token,
+            }
+        });
+        if (res.status != 200 && res.status != 206){
+            throw new Error(`Failed to get multiple files, status code: ${res.status}, message: ${await fmtFailedResponse(res)}`);
+        }
+        return await res.json();
+    }
+
     async delete(path){
         if (path.startsWith('/')){ path = path.slice(1); }
         const res = await fetch(this.config.endpoint + '/' + path, {

frontend/login.js

@@ -3,7 +3,6 @@ import { createFloatingWindow, showPopup } from "./popup.js";
 
 /**
  * @import { store } from "./state.js";
- * @import { UserRecord } from "./api.js";
  * 
  * Shows the login panel if necessary.
  * @param {store} store - The store object.

frontend/popup.js

@@ -109,7 +109,14 @@ export function showPopup(content = '',  {
 } = {}){
     const popup = document.createElement("div");
     popup.classList.add("popup-window");
-    popup.innerHTML = showTime? `<span>[${new Date().toLocaleTimeString()}]</span> ${content}` : content;
+    /**
+     * @param {string} c
+     * @returns {void}
+     */
+    function setPopupContent(c){
+        popup.innerHTML = showTime? `<span>[${new Date().toLocaleTimeString()}]</span> ${c}` : c;
+    }
+    setPopupContent(content);
     popup.style.width = width;
     const popupHeight = '1rem';
     popup.style.height = popupHeight;
@@ -132,11 +139,19 @@ export function showPopup(content = '',  {
     if (level === "success") popup.style.backgroundColor = "darkgreen";
     document.body.appendChild(popup);
     shownPopups.push(popup);
-    window.setTimeout(() => {
+
+    function closePopup(){
         if (popup.parentNode) document.body.removeChild(popup);
         shownPopups.splice(shownPopups.indexOf(popup), 1);
         for (let i = 0; i < shownPopups.length; i++) {
             shownPopups[i].style.top = `${i * (parseInt(popupHeight) + 2*parseInt(paddingHeight))*1.2 + 0.5}rem`;
         }
-    }, timeout);
+    }
+
+    window.setTimeout(closePopup, timeout);
+    return {
+        elem: popup, 
+        setContent: setPopupContent, 
+        close: closePopup
+    }
 }

frontend/scripts.js

@@ -183,26 +183,29 @@ Are you sure you want to proceed?\
         `)){ return; }
 
         let counter = 0;
+        let totalCount = 0;
+        const uploadPopup = showPopup('Uploading multiple files...', {level: 'info', timeout: 999999});
         async function uploadFileFn(path, file){
-            const this_count = counter;
             try{
                 await uploadFile(conn, path, file, {conflict: 'overwrite'});
             }
             catch (err){
                 showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});
             }
-            console.log(`[${this_count}/${counter}] Uploaded file: ${path}`);
+            console.log(`[${counter}/${totalCount}] Uploaded file: ${path}`);
+            uploadPopup.setContent(`Uploading multiple files... [${counter}/${totalCount}]`);
         }
 
-        const promises = await forEachFile(e, async (relPath, filePromise) => {
+        const promises = await forEachFile(e, async (relPath, filePromiseFn) => {
             counter += 1;
-            const file = await filePromise;
+            const file = await filePromiseFn();
             await uploadFileFn(dstPath + relPath, file);
         });
+        totalCount = promises.length;
 
-        showPopup('Uploading multiple files...', {level: 'info', timeout: 3000});
         Promise.all(promises).then(
             () => {
+                window.setTimeout(uploadPopup.close, 3000);
                 showPopup('Upload success.', {level: 'success', timeout: 3000});
                 refreshFileList();
             }, 

frontend/utils.js

@@ -101,7 +101,7 @@ export function asHtmlText(text){
  * using the provided callback with a concurrency limit.
  *
  * @param {Event} e The drop event.
- * @param {(relPath: string, file: Promise<File>) => Promise<void>} callback A function
+ * @param {(relPath: string, file: () => Promise<File>) => Promise<void>} callback A function
  *        that receives the relative path and a promise for the File.
  * @param {number} [maxConcurrent=5] Maximum number of concurrent callback executions.
  * @returns {Promise<Promise<void>[]>} A promise resolving to an array of callback promises.
@@ -146,11 +146,10 @@ export async function forEachFile(e, callback, maxConcurrent = 16) {
     async function traverse(entry, path) {
         if (entry.isFile) {
             // Wrap file retrieval in a promise.
-            const filePromise = new Promise((resolve, reject) => {
-                entry.file(resolve, reject);
-            });
+            const filePromiseFn = () =>
+                new Promise((resolve, reject) => entry.file(resolve, reject));
             // Use the concurrency barrier for the callback invocation.
-            results.push(runWithLimit(() => callback(path + entry.name, filePromise)));
+            results.push(runWithLimit(() => callback(path + entry.name, filePromiseFn)));
         } else if (entry.isDirectory) {
             const reader = entry.createReader();
 

lfss/api/connector.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from typing import Optional, Literal
 from collections.abc import Iterator
-import os
+import os, json
 import requests
 import requests.adapters
 import urllib.parse
@@ -76,7 +76,11 @@ class Connector:
             path = path[1:]
         path = ensure_uri_compnents(path)
         def f(**kwargs):
-            url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params)
+            search_params_t = [
+                (k, str(v).lower() if isinstance(v, bool) else v)
+                for k, v in search_params.items()
+            ]   # tuple form
+            url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params_t, doseq=True)
             headers: dict = kwargs.pop('headers', {})
             headers.update({
                 'Authorization': f"Bearer {self.config['token']}",
@@ -207,6 +211,17 @@ class Connector:
         assert response.headers['Content-Type'] == 'application/json'
         return response.json()
     
+    def get_multiple_text(self, *paths: str, skip_content = False) -> dict[str, Optional[str]]:
+        """ 
+        Gets text contents of multiple files at once. Non-existing files will return None. 
+        - skip_content: if True, the file contents will not be fetched, always be empty string ''.
+        """
+        response = self._fetch_factory(
+            'GET', '_api/get-multiple', 
+            {'path': paths, "skip_content": skip_content}
+            )()
+        return response.json()
+    
     def delete(self, path: str):
         """Deletes the file at the specified path."""
         self._fetch_factory('DELETE', path)()

lfss/cli/cli.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-import argparse, typing
+import argparse, typing, sys
 from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
 from lfss.eng.datatype import FileReadPermission, FileSortKey, DirSortKey
 from lfss.eng.utils import decode_uri_compnents
@@ -78,9 +78,9 @@ def main():
                 permission=args.permission
             )
             if failed_upload:
-                print("\033[91mFailed to upload:\033[0m")
+                print("\033[91mFailed to upload:\033[0m", file=sys.stderr)
                 for path in failed_upload:
-                    print(f"  {path}")
+                    print(f"  {path}", file=sys.stderr)
         else:
             success, msg = upload_file(
                 connector, 
@@ -93,7 +93,7 @@ def main():
                 permission=args.permission
             )
             if not success:
-                print("\033[91mFailed to upload: \033[0m", msg)
+                print("\033[91mFailed to upload: \033[0m", msg, file=sys.stderr)
     
     elif args.command == "download":
         is_dir = args.src.endswith("/")
@@ -107,9 +107,9 @@ def main():
                 overwrite=args.overwrite
             )
             if failed_download:
-                print("\033[91mFailed to download:\033[0m")
+                print("\033[91mFailed to download:\033[0m", file=sys.stderr)
                 for path in failed_download:
-                    print(f"  {path}")
+                    print(f"  {path}", file=sys.stderr)
         else:
             success, msg = download_file(
                 connector, 
@@ -121,7 +121,7 @@ def main():
                 overwrite=args.overwrite
             )
             if not success:
-                print("\033[91mFailed to download: \033[0m", msg)
+                print("\033[91mFailed to download: \033[0m", msg, file=sys.stderr)
     
     elif args.command == "query":
         for path in args.path:

lfss/cli/log.py

@@ -0,0 +1,77 @@
+from typing import Optional
+import argparse
+import rich.console
+import logging
+import sqlite3
+from lfss.eng.log import eval_logline
+
+console = rich.console.Console()
+def levelstr2int(levelstr: str) -> int:
+    import sys
+    if sys.version_info < (3, 11):
+        return logging.getLevelName(levelstr.upper())
+    else:
+        return logging.getLevelNamesMapping()[levelstr.upper()]
+
+def view(
+    db_file: str, 
+    level: Optional[str] = None, 
+    offset: int = 0,
+    limit: int = 1000
+    ):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("SELECT * FROM log ORDER BY created DESC LIMIT ? OFFSET ?", (limit, offset))
+    else:
+        level_int = levelstr2int(level)
+        cursor.execute("SELECT * FROM log WHERE level >= ? ORDER BY created DESC LIMIT ? OFFSET ?", (level_int, limit, offset))
+    levelname_color = {
+        'DEBUG': 'blue',
+        'INFO': 'green',
+        'WARNING': 'yellow',
+        'ERROR': 'red',
+        'CRITICAL': 'bold red', 
+        'FATAL': 'bold red'
+    }
+    for row in cursor.fetchall():
+        log = eval_logline(row)
+        console.print(f"{log.created} [{levelname_color[log.levelname]}][{log.levelname}] [default]{log.message}")
+    conn.close()
+
+def trim(db_file: str, keep: int  = 1000, level: Optional[str] = None):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("DELETE FROM log WHERE id NOT IN (SELECT id FROM log ORDER BY created DESC LIMIT ?)", (keep,))
+    else:
+        cursor.execute("DELETE FROM log WHERE levelname = ? and id NOT IN (SELECT id FROM log WHERE levelname = ? ORDER BY created DESC LIMIT ?)", (level.upper(), level.upper(), keep))
+    conn.commit()
+    conn.execute("VACUUM")
+    conn.close()
+
+def main():
+    parser = argparse.ArgumentParser(description="Log operations utility")
+    subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
+
+    parser_show = subparsers.add_parser('view', help='Show logs')
+    parser_show.add_argument('db_file', type=str, help='Database file path')
+    parser_show.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_show.add_argument('--offset', type=int, default=0, help='Starting offset')
+    parser_show.add_argument('--limit', type=int, default=1000, help='Maximum number of entries to display')
+    parser_show.set_defaults(func=view)
+
+    parser_trim = subparsers.add_parser('trim', help='Trim logs')
+    parser_trim.add_argument('db_file', type=str, help='Database file path')
+    parser_trim.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_trim.add_argument('--keep', type=int, default=1000, help='Number of entries to keep')
+    parser_trim.set_defaults(func=trim)
+
+    args = parser.parse_args()
+    if hasattr(args, 'func'):
+        kwargs = vars(args)
+        func = kwargs.pop('func')
+        func(**kwargs)
+
+if __name__ == '__main__':
+    main()

lfss/cli/vacuum.py

@@ -2,7 +2,7 @@
 Vacuum the database and external storage to ensure that the storage is consistent and minimal.
 """
 
-from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB
+from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB, LOG_DIR
 import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
@@ -14,6 +14,7 @@ from lfss.eng.database import transaction, unique_cursor
 from lfss.svc.request_log import RequestDB
 from lfss.eng.utils import now_stamp
 from lfss.eng.connection_pool import global_entrance
+from lfss.cli.log import trim
 
 sem: Semaphore
 
@@ -33,7 +34,7 @@ def barriered(func):
     return wrapper
 
 @global_entrance()
-async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, vacuum_all: bool = False):
+async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, logs: bool = False, vacuum_all: bool = False):
 
     # check if any file in the Large Blob directory is not in the database
     # the reverse operation is not necessary, because by design, the database should be the source of truth...
@@ -73,6 +74,11 @@ async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = F
             async with unique_cursor(is_write=True) as c:
                 await c.execute("VACUUM blobs")
     
+    if logs or vacuum_all:
+        with indicator("VACUUM-logs"):
+            for log_file in LOG_DIR.glob("*.log.db"):
+                trim(str(log_file), keep=10_000)
+    
     if thumbs or vacuum_all:
         try:
             async with transaction() as c:
@@ -123,9 +129,10 @@ def main():
     parser.add_argument("-d", "--data", action="store_true", help="Vacuum blobs")
     parser.add_argument("-t", "--thumb", action="store_true", help="Vacuum thumbnails")
     parser.add_argument("-r", "--requests", action="store_true", help="Vacuum request logs to only keep at most recent 1M rows in 7 days")
+    parser.add_argument("-l", "--logs", action="store_true", help="Trim log to keep at most recent 10k rows for each category")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, vacuum_all=args.all))
+    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, logs = args.logs, vacuum_all=args.all))
 
     if args.requests or args.all:
         asyncio.run(vacuum_requests())

lfss/eng/config.py

@@ -11,16 +11,19 @@ if not DATA_HOME.exists():
 DATA_HOME = DATA_HOME.resolve().absolute()
 LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
+LOG_DIR = DATA_HOME / 'logs'
+
+DISABLE_LOGGING = os.environ.get('DISABLE_LOGGING', '0') == '1'
 
 # https://sqlite.org/fasterthanfs.html
 __env_large_file = os.environ.get('LFSS_LARGE_FILE', None)
 if __env_large_file is not None:
     LARGE_FILE_BYTES = parse_storage_size(__env_large_file)
 else:
-    LARGE_FILE_BYTES = 8 * 1024 * 1024  # 8MB
-MAX_MEM_FILE_BYTES = 128 * 1024 * 1024   # 128MB
+    LARGE_FILE_BYTES = 1 * 1024 * 1024  # 1MB
+MAX_MEM_FILE_BYTES = 128 * 1024 * 1024  # 128MB
 CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'
 
 THUMB_DB = DATA_HOME / 'thumbs.v0-11.db'
-THUMB_SIZE = (48, 48)
+THUMB_SIZE = (64, 64)