lfss 0.11.0__tar.gz → 0.11.2__tar.gz

{lfss-0.11.0 → lfss-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.11.0
+Version: 0.11.2
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: Li, Mengxun
@@ -17,6 +17,7 @@ Requires-Dist: mimesniff (==1.*)
 Requires-Dist: pillow
 Requires-Dist: python-multipart
 Requires-Dist: requests (==2.*)
+Requires-Dist: rich
 Requires-Dist: stream-zip (==0.*)
 Requires-Dist: uvicorn (==0.*)
 Project-URL: Repository, https://github.com/MenxLi/lfss

{lfss-0.11.0 → lfss-0.11.2}/docs/changelog.md

@@ -1,3 +1,30 @@
+## 0.11
+
+### 0.11.2
+- Improve frontend directory upload feedback. 
+- Set default large file threashold to 1M. 
+- Increase default concurrent threads. 
+- Use sqlite for logging.
+- Add vacuum logs. 
+- Refactor: use dir for directory path. 
+
+### 0.11.1
+- Rename api `get_meta` function.
+- Frontend support upload directory.  
+- Fix admin put to non-exists user path. 
+
+### 0.11.0
+- Copy file as hard link. 
+- Add vacuum thumb and all.
+- Thumb database use file_id as index. 
+- improve username and url check with regular expression.
+
+## 0.10
+
+### 0.10.0
+- Inherit permission from path owner for `unset` permission files.
+- Add timeout and verify options for client api.
+- Bundle small files in memory.
 
 ## 0.9
 

{lfss-0.11.0 → lfss-0.11.2}/frontend/login.js

@@ -3,7 +3,6 @@ import { createFloatingWindow, showPopup } from "./popup.js";
 
 /**
  * @import { store } from "./state.js";
- * @import { UserRecord } from "./api.js";
  * 
  * Shows the login panel if necessary.
  * @param {store} store - The store object.

{lfss-0.11.0 → lfss-0.11.2}/frontend/popup.js

@@ -109,7 +109,14 @@ export function showPopup(content = '',  {
 } = {}){
     const popup = document.createElement("div");
     popup.classList.add("popup-window");
-    popup.innerHTML = showTime? `<span>[${new Date().toLocaleTimeString()}]</span> ${content}` : content;
+    /**
+     * @param {string} c
+     * @returns {void}
+     */
+    function setPopupContent(c){
+        popup.innerHTML = showTime? `<span>[${new Date().toLocaleTimeString()}]</span> ${c}` : c;
+    }
+    setPopupContent(content);
     popup.style.width = width;
     const popupHeight = '1rem';
     popup.style.height = popupHeight;
@@ -132,11 +139,19 @@ export function showPopup(content = '',  {
     if (level === "success") popup.style.backgroundColor = "darkgreen";
     document.body.appendChild(popup);
     shownPopups.push(popup);
-    window.setTimeout(() => {
+
+    function closePopup(){
         if (popup.parentNode) document.body.removeChild(popup);
         shownPopups.splice(shownPopups.indexOf(popup), 1);
         for (let i = 0; i < shownPopups.length; i++) {
             shownPopups[i].style.top = `${i * (parseInt(popupHeight) + 2*parseInt(paddingHeight))*1.2 + 0.5}rem`;
         }
-    }, timeout);
+    }
+
+    window.setTimeout(closePopup, timeout);
+    return {
+        elem: popup, 
+        setContent: setPopupContent, 
+        close: closePopup
+    }
 }

{lfss-0.11.0 → lfss-0.11.2}/frontend/scripts.js

@@ -5,6 +5,7 @@ import { showInfoPanel, showDirInfoPanel } from './info.js';
 import { makeThumbHtml } from './thumb.js';
 import { store } from './state.js';
 import { maybeShowLoginPanel } from './login.js';
+import { forEachFile } from './utils.js';
 
 /** @type {import('./api.js').UserRecord}*/
 let userRecord = null;
@@ -158,55 +159,61 @@ uploadFileNameInput.addEventListener('input', debounce(onFileNameInpuChange, 500
         e.preventDefault();
         e.stopPropagation();
     });
-    window.addEventListener('drop', (e) => {
+    window.addEventListener('drop', async (e) => {
         e.preventDefault();
         e.stopPropagation();
-        const files = e.dataTransfer.files;
-        if (files.length == 1){
-            uploadFileSelector.files = files;
-            uploadFileNameInput.value = files[0].name;
+        const items = e.dataTransfer.items;
+        if (items.length == 1 && items[0].kind === 'file' && items[0].webkitGetAsEntry().isFile){
+            uploadFileSelector.files = e.dataTransfer.files;
+            uploadFileNameInput.value = e.dataTransfer.files[0].name;
             uploadFileNameInput.focus();
+            return;
         }
-        else if (files.length > 1){
-            let dstPath = store.dirpath + uploadFileNameInput.value;
-            if (!dstPath.endsWith('/')){ dstPath += '/'; }
-            if (!confirm(`
+
+        /** @type {[string, File][]} */
+        const uploadInputVal = uploadFileNameInput.value? uploadFileNameInput.value : '';
+        let dstPath = store.dirpath + uploadInputVal;
+        if (!dstPath.endsWith('/')){ dstPath += '/'; }
+
+        if (!confirm(`\
 You are trying to upload multiple files at once. 
 This will directly upload the files to the [${dstPath}] directory without renaming. 
 Note that same name files will be overwritten.
-Are you sure you want to proceed?
-                `)){ return; }
-            
-            let counter = 0;
-            async function uploadFileFn(...args){
-                const [file, path] = args;
-                try{
-                    await uploadFile(conn, path, file, {conflict: 'overwrite'});
-                }
-                catch (err){
-                    showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});
-                }
-                counter += 1;
-                console.log("Uploading file: ", counter, "/", files.length);
+Are you sure you want to proceed?\
+        `)){ return; }
+
+        let counter = 0;
+        let totalCount = 0;
+        const uploadPopup = showPopup('Uploading multiple files...', {level: 'info', timeout: 999999});
+        async function uploadFileFn(path, file){
+            try{
+                await uploadFile(conn, path, file, {conflict: 'overwrite'});
             }
-            
-            let promises = [];
-            for (let i = 0; i < files.length; i++){
-                const file = files[i];
-                const path = dstPath + file.name;
-                promises.push(uploadFileFn(file, path));
+            catch (err){
+                showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});
             }
-            showPopup('Uploading multiple files...', {level: 'info', timeout: 3000});
-            Promise.all(promises).then(
-                () => {
-                    showPopup('Upload success.', {level: 'success', timeout: 3000});
-                    refreshFileList();
-                }, 
-                (err) => {
-                    showPopup('Failed to upload some files: ' + err, {level: 'error', timeout: 5000});
-                }
-            );
+            console.log(`[${counter}/${totalCount}] Uploaded file: ${path}`);
+            uploadPopup.setContent(`Uploading multiple files... [${counter}/${totalCount}]`);
         }
+
+        const promises = await forEachFile(e, async (relPath, filePromiseFn) => {
+            counter += 1;
+            const file = await filePromiseFn();
+            await uploadFileFn(dstPath + relPath, file);
+        });
+        totalCount = promises.length;
+
+        Promise.all(promises).then(
+            () => {
+                window.setTimeout(uploadPopup.close, 3000);
+                showPopup('Upload success.', {level: 'success', timeout: 3000});
+                refreshFileList();
+            }, 
+            (err) => {
+                showPopup('Failed to upload some files: ' + err, {level: 'error', timeout: 5000});
+            }
+        );
+
     });
 }
 

lfss-0.11.2/frontend/utils.js

@@ -0,0 +1,193 @@
+
+export function formatSize(size){
+    if (size < 0){
+        return '';
+    }
+    const sizeInKb = size / 1024;
+    const sizeInMb = sizeInKb / 1024;
+    const sizeInGb = sizeInMb / 1024;
+    if (sizeInGb > 1){
+        return sizeInGb.toFixed(2) + ' GB';
+    }
+    else if (sizeInMb > 1){
+        return sizeInMb.toFixed(2) + ' MB';
+    }
+    else if (sizeInKb > 1){
+        return sizeInKb.toFixed(2) + ' KB';
+    }
+    else {
+        return size + ' B';
+    }
+}
+
+export function copyToClipboard(text){
+    function secureCopy(text){
+        navigator.clipboard.writeText(text);
+    }
+    function unsecureCopy(text){
+        const el = document.createElement('textarea');
+        el.value = text;
+        document.body.appendChild(el);
+        el.select();
+        document.execCommand('copy');
+        document.body.removeChild(el);
+    }
+    if (navigator.clipboard){
+        secureCopy(text);
+    }
+    else {
+        unsecureCopy(text);
+    }
+}
+
+export function encodePathURI(path){
+    return path.split('/').map(encodeURIComponent).join('/');
+}
+
+export function decodePathURI(path){
+    return path.split('/').map(decodeURIComponent).join('/');
+}
+
+export function ensurePathURI(path){
+    return encodePathURI(decodePathURI(path));
+}
+
+export function getRandomString(n, additionalCharset='0123456789_-(=)[]{}'){
+    let result = '';
+    let charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    const firstChar = charset[Math.floor(Math.random() * charset.length)];
+    const lastChar = charset[Math.floor(Math.random() * charset.length)];
+    result += firstChar;
+    charset += additionalCharset;
+    for (let i = 0; i < n-2; i++){
+        result += charset[Math.floor(Math.random() * charset.length)];
+    }
+    result += lastChar;
+    return result;
+};
+
+/**
+ * @param {string} dateStr 
+ * @returns {string}
+ */
+export function cvtGMT2Local(dateStr){
+    if (!dateStr || dateStr === 'N/A'){
+        return '';
+    }
+    const gmtdate = new Date(dateStr);
+    const localdate = new Date(gmtdate.getTime() + gmtdate.getTimezoneOffset() * 60000);
+    return localdate.toISOString().slice(0, 19).replace('T', ' ');
+}
+
+export function debounce(fn,wait){
+    let timeout;
+    return function(...args){
+        const context = this;
+        if (timeout) clearTimeout(timeout);
+        timeout = setTimeout(() => fn.apply(context, args), wait);
+    }
+}
+
+export function asHtmlText(text){
+    const anonElem = document.createElement('div');
+    anonElem.textContent = text;
+    const htmlText = anonElem.innerHTML;
+    return htmlText;
+}
+
+/**
+ * Iterates over all files dropped in the event,
+ * including files inside directories, and processes them
+ * using the provided callback with a concurrency limit.
+ *
+ * @param {Event} e The drop event.
+ * @param {(relPath: string, file: () => Promise<File>) => Promise<void>} callback A function
+ *        that receives the relative path and a promise for the File.
+ * @param {number} [maxConcurrent=5] Maximum number of concurrent callback executions.
+ * @returns {Promise<Promise<void>[]>} A promise resolving to an array of callback promises.
+ */
+export async function forEachFile(e, callback, maxConcurrent = 16) {
+    const results = []; // to collect callback promises
+
+    // Concurrency barrier variables.
+    let activeCount = 0;
+    const queue = [];
+
+    /**
+     * Runs the given async task when below the concurrency limit.
+     * If at limit, waits until a slot is free.
+     *
+     * @param {() => Promise<any>} task An async function returning a promise.
+     * @returns {Promise<any>}
+     */
+    async function runWithLimit(task) {
+        // If we reached the concurrency limit, wait for a free slot.
+        if (activeCount >= maxConcurrent) {
+            await new Promise(resolve => queue.push(resolve));
+        }
+        activeCount++;
+        try {
+            return await task();
+        } finally {
+            activeCount--;
+            // If there are waiting tasks, allow the next one to run.
+            if (queue.length) {
+                queue.shift()();
+            }
+        }
+    }
+
+    /**
+     * Recursively traverses a file system entry.
+     *
+     * @param {FileSystemEntry} entry The entry (file or directory).
+     * @param {string} path The current relative path.
+     */
+    async function traverse(entry, path) {
+        if (entry.isFile) {
+            // Wrap file retrieval in a promise.
+            const filePromiseFn = () =>
+                new Promise((resolve, reject) => entry.file(resolve, reject));
+            // Use the concurrency barrier for the callback invocation.
+            results.push(runWithLimit(() => callback(path + entry.name, filePromiseFn)));
+        } else if (entry.isDirectory) {
+            const reader = entry.createReader();
+
+            async function readAllEntries(reader) {
+                const entries = [];
+                while (true) {
+                const chunk = await new Promise((resolve, reject) => {
+                    reader.readEntries(resolve, reject);
+                });
+                if (chunk.length === 0) break;
+                entries.push(...chunk);
+                }
+                return entries;
+            }
+
+            const entries = await readAllEntries(reader);
+            await Promise.all(
+                entries.map(ent => traverse(ent, path + entry.name + '/'))
+            );
+        }
+    }
+
+    // Process using DataTransfer items if available.
+    if (e.dataTransfer && e.dataTransfer.items) {
+        await Promise.all(
+        Array.from(e.dataTransfer.items).map(async item => {
+            const entry = item.webkitGetAsEntry && item.webkitGetAsEntry();
+            if (entry) {
+            await traverse(entry, '');
+            }
+        })
+        );
+    } else if (e.dataTransfer && e.dataTransfer.files) {
+        // Fallback for browsers that support only dataTransfer.files.
+        Array.from(e.dataTransfer.files).forEach(file => {
+        results.push(runWithLimit(() => callback(file.name, Promise.resolve(file))));
+        });
+    }
+    return results;
+}
+

{lfss-0.11.0 → lfss-0.11.2}/lfss/api/__init__.py

@@ -113,7 +113,7 @@ def download_file(
                 print(f"File {file_path} already exists, skipping download.")
             return True, error_msg
         try:
-            fmeta = connector.get_metadata(src_url)
+            fmeta = connector.get_meta(src_url)
             if fmeta is None:
                 error_msg = "File not found."
                 return False, error_msg

{lfss-0.11.0 → lfss-0.11.2}/lfss/api/connector.py

@@ -98,7 +98,7 @@ class Connector:
 
         # Skip ahead by checking if the file already exists
         if conflict == 'skip-ahead':
-            exists = self.get_metadata(path)
+            exists = self.get_meta(path)
             if exists is None:
                 conflict = 'skip'
             else:
@@ -122,7 +122,7 @@ class Connector:
 
         # Skip ahead by checking if the file already exists
         if conflict == 'skip-ahead':
-            exists = self.get_metadata(path)
+            exists = self.get_meta(path)
             if exists is None:
                 conflict = 'skip'
             else:
@@ -154,7 +154,7 @@ class Connector:
 
         # Skip ahead by checking if the file already exists
         if conflict == 'skip-ahead':
-            exists = self.get_metadata(path)
+            exists = self.get_meta(path)
             if exists is None:
                 conflict = 'skip'
             else:
@@ -211,7 +211,7 @@ class Connector:
         """Deletes the file at the specified path."""
         self._fetch_factory('DELETE', path)()
     
-    def get_metadata(self, path: str) -> Optional[FileRecord | DirectoryRecord]:
+    def get_meta(self, path: str) -> Optional[FileRecord | DirectoryRecord]:
         """Gets the metadata for the file at the specified path."""
         try:
             response = self._fetch_factory('GET', '_api/meta', {'path': path})()
@@ -223,6 +223,9 @@ class Connector:
             if e.response.status_code == 404:
                 return None
             raise e
+    # shorthand methods for type constraints
+    def get_fmeta(self, path: str) -> Optional[FileRecord]: assert (f:=self.get_meta(path)) is None or isinstance(f, FileRecord); return f 
+    def get_dmeta(self, path: str) -> Optional[DirectoryRecord]: assert (d:=self.get_meta(path)) is None or isinstance(d, DirectoryRecord); return d
     
     def list_path(self, path: str) -> PathContents:
         """ 

{lfss-0.11.0 → lfss-0.11.2}/lfss/cli/cli.py

@@ -126,7 +126,7 @@ def main():
     elif args.command == "query":
         for path in args.path:
             with catch_request_error():
-                res = connector.get_metadata(path)
+                res = connector.get_meta(path)
                 if res is None:
                     print(f"\033[31mNot found\033[0m ({path})")
                 else:

lfss-0.11.2/lfss/cli/log.py

@@ -0,0 +1,77 @@
+from typing import Optional
+import argparse
+import rich.console
+import logging
+import sqlite3
+from lfss.eng.log import eval_logline
+
+console = rich.console.Console()
+def levelstr2int(levelstr: str) -> int:
+    import sys
+    if sys.version_info < (3, 11):
+        return logging.getLevelName(levelstr.upper())
+    else:
+        return logging.getLevelNamesMapping()[levelstr.upper()]
+
+def view(
+    db_file: str, 
+    level: Optional[str] = None, 
+    offset: int = 0,
+    limit: int = 1000
+    ):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("SELECT * FROM log ORDER BY created DESC LIMIT ? OFFSET ?", (limit, offset))
+    else:
+        level_int = levelstr2int(level)
+        cursor.execute("SELECT * FROM log WHERE level >= ? ORDER BY created DESC LIMIT ? OFFSET ?", (level_int, limit, offset))
+    levelname_color = {
+        'DEBUG': 'blue',
+        'INFO': 'green',
+        'WARNING': 'yellow',
+        'ERROR': 'red',
+        'CRITICAL': 'bold red', 
+        'FATAL': 'bold red'
+    }
+    for row in cursor.fetchall():
+        log = eval_logline(row)
+        console.print(f"{log.created} [{levelname_color[log.levelname]}][{log.levelname}] [default]{log.message}")
+    conn.close()
+
+def trim(db_file: str, keep: int  = 1000, level: Optional[str] = None):
+    conn = sqlite3.connect(db_file)
+    cursor = conn.cursor()
+    if level is None:
+        cursor.execute("DELETE FROM log WHERE id NOT IN (SELECT id FROM log ORDER BY created DESC LIMIT ?)", (keep,))
+    else:
+        cursor.execute("DELETE FROM log WHERE levelname = ? and id NOT IN (SELECT id FROM log WHERE levelname = ? ORDER BY created DESC LIMIT ?)", (level.upper(), level.upper(), keep))
+    conn.commit()
+    conn.execute("VACUUM")
+    conn.close()
+
+def main():
+    parser = argparse.ArgumentParser(description="Log operations utility")
+    subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
+
+    parser_show = subparsers.add_parser('view', help='Show logs')
+    parser_show.add_argument('db_file', type=str, help='Database file path')
+    parser_show.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_show.add_argument('--offset', type=int, default=0, help='Starting offset')
+    parser_show.add_argument('--limit', type=int, default=1000, help='Maximum number of entries to display')
+    parser_show.set_defaults(func=view)
+
+    parser_trim = subparsers.add_parser('trim', help='Trim logs')
+    parser_trim.add_argument('db_file', type=str, help='Database file path')
+    parser_trim.add_argument('-l', '--level', type=str, required=False, help='Log level')
+    parser_trim.add_argument('--keep', type=int, default=1000, help='Number of entries to keep')
+    parser_trim.set_defaults(func=trim)
+
+    args = parser.parse_args()
+    if hasattr(args, 'func'):
+        kwargs = vars(args)
+        func = kwargs.pop('func')
+        func(**kwargs)
+
+if __name__ == '__main__':
+    main()

{lfss-0.11.0 → lfss-0.11.2}/lfss/cli/vacuum.py

@@ -2,7 +2,7 @@
 Vacuum the database and external storage to ensure that the storage is consistent and minimal.
 """
 
-from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB
+from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB, LOG_DIR
 import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
@@ -14,6 +14,7 @@ from lfss.eng.database import transaction, unique_cursor
 from lfss.svc.request_log import RequestDB
 from lfss.eng.utils import now_stamp
 from lfss.eng.connection_pool import global_entrance
+from lfss.cli.log import trim
 
 sem: Semaphore
 
@@ -33,7 +34,7 @@ def barriered(func):
     return wrapper
 
 @global_entrance()
-async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, vacuum_all: bool = False):
+async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, logs: bool = False, vacuum_all: bool = False):
 
     # check if any file in the Large Blob directory is not in the database
     # the reverse operation is not necessary, because by design, the database should be the source of truth...
@@ -73,6 +74,11 @@ async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = F
             async with unique_cursor(is_write=True) as c:
                 await c.execute("VACUUM blobs")
     
+    if logs or vacuum_all:
+        with indicator("VACUUM-logs"):
+            for log_file in LOG_DIR.glob("*.log.db"):
+                trim(str(log_file), keep=10_000)
+    
     if thumbs or vacuum_all:
         try:
             async with transaction() as c:
@@ -123,9 +129,10 @@ def main():
     parser.add_argument("-d", "--data", action="store_true", help="Vacuum blobs")
     parser.add_argument("-t", "--thumb", action="store_true", help="Vacuum thumbnails")
     parser.add_argument("-r", "--requests", action="store_true", help="Vacuum request logs to only keep at most recent 1M rows in 7 days")
+    parser.add_argument("-l", "--logs", action="store_true", help="Trim log to keep at most recent 10k rows for each category")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, vacuum_all=args.all))
+    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, logs = args.logs, vacuum_all=args.all))
 
     if args.requests or args.all:
         asyncio.run(vacuum_requests())

{lfss-0.11.0 → lfss-0.11.2}/lfss/eng/config.py

@@ -11,14 +11,15 @@ if not DATA_HOME.exists():
 DATA_HOME = DATA_HOME.resolve().absolute()
 LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
+LOG_DIR = DATA_HOME / 'logs'
 
 # https://sqlite.org/fasterthanfs.html
 __env_large_file = os.environ.get('LFSS_LARGE_FILE', None)
 if __env_large_file is not None:
     LARGE_FILE_BYTES = parse_storage_size(__env_large_file)
 else:
-    LARGE_FILE_BYTES = 8 * 1024 * 1024  # 8MB
-MAX_MEM_FILE_BYTES = 128 * 1024 * 1024   # 128MB
+    LARGE_FILE_BYTES = 1 * 1024 * 1024  # 1MB
+MAX_MEM_FILE_BYTES = 128 * 1024 * 1024  # 128MB
 CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'