lfss 0.3.2__tar.gz → 0.4.1__tar.gz

{lfss-0.3.2 → lfss-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.3.2
+Version: 0.4.1
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.3.2 → lfss-0.4.1}/frontend/api.js

@@ -54,10 +54,16 @@ export default class Connector {
     * @param {File} file - the file to upload
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
-    async put(path, file){
+    async put(path, file, {
+        overwrite = false,
+        permission = 0
+    } = {}){
         if (path.startsWith('/')){ path = path.slice(1); }
         const fileBytes = await file.arrayBuffer();
-        const res = await fetch(this.config.endpoint + '/' + path, {
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('overwrite', overwrite);
+        dst.searchParams.append('permission', permission);
+        const res = await fetch(dst.toString(), {
             method: 'PUT',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token, 

{lfss-0.3.2 → lfss-0.4.1}/frontend/scripts.js

@@ -172,7 +172,7 @@ Are you sure you want to proceed?
             async function uploadFile(...args){
                 const [file, path] = args;
                 try{
-                    await conn.put(path, file);
+                    await conn.put(path, file, {overwrite: true});
                 }
                 catch (err){
                     showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});

lfss-0.4.1/lfss/cli/cli.py

@@ -0,0 +1,48 @@
+from lfss.client import Connector, upload_directory
+from lfss.src.database import FileReadPermission
+from pathlib import Path
+import argparse
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
+
+    sp = parser.add_subparsers(dest="command", required=True)
+
+    # upload
+    sp_upload = sp.add_parser("upload", help="Upload files")
+    sp_upload.add_argument("src", help="Source file or directory", type=str)
+    sp_upload.add_argument("dst", help="Destination path", type=str)
+    sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
+    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between retries, only works with directory upload")
+    sp_upload.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
+    sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
+    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
+
+    return parser.parse_args()
+
+def main():
+    args = parse_arguments()
+    connector = Connector()
+    if args.command == "upload":
+        src_path = Path(args.src)
+        if src_path.is_dir():
+            upload_directory(
+                connector, args.src, args.dst, 
+                verbose=True, 
+                n_concurrent=args.jobs, 
+                n_reties=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite, 
+                permission=args.permission
+            )
+        else:
+            with open(args.src, 'rb') as f:
+                connector.put(
+                    args.dst, 
+                    f.read(), 
+                    overwrite=args.overwrite, 
+                    permission=args.permission
+                    )
+    else:
+        raise NotImplementedError(f"Command {args.command} not implemented.")
+    

lfss-0.4.1/lfss/client/__init__.py

@@ -0,0 +1,58 @@
+import os, time
+from threading import Lock
+from concurrent.futures import ThreadPoolExecutor
+from .api import Connector
+
+def upload_directory(
+    connector: Connector, 
+    directory: str, 
+    path: str, 
+    n_concurrent: int = 1,
+    n_reties: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    **put_kwargs
+    ) -> list[str]:
+    assert path.endswith('/'), "Path must end with a slash."
+    if path.startswith('/'):
+        path = path[1:]
+    
+    _counter = 0
+    _counter_lock = Lock()
+
+    faild_files = []
+    def put_file(file_path):
+        with _counter_lock:
+            nonlocal _counter
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
+            if verbose:
+                print(f"[{this_count}] Uploading {file_path} to {dst_path}")
+
+        this_try = 0
+        with open(file_path, 'rb') as f:
+            blob = f.read()
+
+        while this_try <= n_reties:
+            try:
+                connector.put(dst_path, blob, **put_kwargs)
+                break
+            except Exception as e:
+                if verbose:
+                    print(f"[{this_count}] Error uploading {file_path}: {e}, retrying...")
+                this_try += 1
+            finally:
+                time.sleep(interval)
+
+        if this_try > n_reties:
+            faild_files.append(file_path)
+            if verbose:
+                print(f"[{this_count}] Failed to upload {file_path} after {n_reties} retries.")
+
+    with ThreadPoolExecutor(n_concurrent) as executor:
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                executor.submit(put_file, os.path.join(root, file))
+
+    return faild_files

{lfss-0.3.2 → lfss-0.4.1}/lfss/client/api.py

@@ -34,9 +34,14 @@ class Connector:
             return response
         return f
 
-    def put(self, path: str, file_data: bytes):
+    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, overwrite: bool = False):
         """Uploads a file to the specified path."""
-        response = self._fetch('PUT', path)(
+        if path.startswith('/'):
+            path = path[1:]
+        response = self._fetch('PUT', path, search_params={
+            'permission': int(permission),
+            'overwrite': overwrite
+            })(
             data=file_data, 
             headers={'Content-Type': 'application/octet-stream'}
         )

{lfss-0.3.2 → lfss-0.4.1}/lfss/src/database.py

@@ -69,7 +69,7 @@ class UserRecord:
     permission: 'FileReadPermission'
 
     def __str__(self):
-        return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}), storage={self.max_storage}, permission={self.permission}"
+        return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}, storage={self.max_storage}, permission={self.permission})"
 
 DECOY_USER = UserRecord(0, 'decoy', 'decoy', False, '2021-01-01 00:00:00', '2021-01-01 00:00:00', 0, FileReadPermission.PRIVATE)
 class UserConn(DBConnBase):
@@ -421,7 +421,7 @@ class FileConn(DBConnBase):
         new_exists = await self.get_file_record(new_url)
         if new_exists is not None:
             raise FileExistsError(f"File {new_url} already exists")
-        async with self.conn.execute("UPDATE fmeta SET url = ? WHERE url = ?", (new_url, old_url)):
+        async with self.conn.execute("UPDATE fmeta SET url = ?, create_time = CURRENT_TIMESTAMP WHERE url = ?", (new_url, old_url)):
             self.logger.info(f"Moved file {old_url} to {new_url}")
     
     async def log_access(self, url: str):
@@ -481,8 +481,9 @@ class FileConn(DBConnBase):
         await self.conn.execute("DELETE FROM fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
 
 def validate_url(url: str, is_file = True):
-    ret = not url.startswith('/') and not ('..' in url) and ('/' in url) and not ('//' in url) \
-        and not ' ' in url and not url.startswith('\\') and not url.startswith('_') and not url.startswith('.')
+    prohibited_chars = ['..', ';', "'", '"', '\\', '\0', '\n', '\r', '\t', '\x0b', '\x0c']
+    ret = not url.startswith('/') and not url.startswith('_') and not url.startswith('.')
+    ret = ret and not any([c in url for c in prohibited_chars])
 
     if not ret:
         raise InvalidPathError(f"Invalid URL: {url}")
@@ -542,7 +543,7 @@ class Database:
         if _g_conn is not None:
             await _g_conn.rollback()
 
-    async def save_file(self, u: int | str, url: str, blob: bytes):
+    async def save_file(self, u: int | str, url: str, blob: bytes, permission: FileReadPermission = FileReadPermission.UNSET):
         validate_url(url)
         assert isinstance(blob, bytes), "blob must be bytes"
 
@@ -570,7 +571,7 @@ class Database:
         f_id = uuid.uuid4().hex
         async with transaction(self):
             await self.file.set_file_blob(f_id, blob)
-            await self.file.set_file_record(url, owner_id=user.id, file_id=f_id, file_size=file_size)
+            await self.file.set_file_record(url, owner_id=user.id, file_id=f_id, file_size=file_size, permission=permission)
             await self.user.set_active(user.username)
 
     # async def read_file_stream(self, url: str): ...

{lfss-0.3.2 → lfss-0.4.1}/lfss/src/server.py

@@ -15,7 +15,7 @@ from .error import *
 from .log import get_logger
 from .stat import RequestDB
 from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES
-from .utils import ensure_uri_compnents
+from .utils import ensure_uri_compnents, format_last_modified, now_stamp
 from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission
 
 logger = get_logger("server", term_level="DEBUG")
@@ -81,6 +81,7 @@ app.add_middleware(
 @app.middleware("http")
 async def log_requests(request: Request, call_next):
 
+    request_time_stamp = now_stamp()
     start_time = time.perf_counter()
     response: Response = await call_next(request)
     end_time = time.perf_counter()
@@ -91,6 +92,7 @@ async def log_requests(request: Request, call_next):
         logger_failed_request.error(f"{request.method} {request.url.path} {response.status_code}")
         
     await req_conn.log_request(
+        request_time_stamp, 
         request.method, request.url.path, response.status_code, response_time,
         headers = dict(request.headers), 
         query = dict(request.query_params), 
@@ -148,7 +150,8 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
         return Response(
             content=fblob, media_type=media_type, headers={
                 "Content-Disposition": f"{disposition}; filename={fname}", 
-                "Content-Length": str(len(fblob))
+                "Content-Length": str(len(fblob)), 
+                "Last-Modified": format_last_modified(file_record.create_time)
             }
         )
     
@@ -159,7 +162,12 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
 
 @router_fs.put("/{path:path}")
 @handle_exception
-async def put_file(request: Request, path: str, user: UserRecord = Depends(get_current_user)):
+async def put_file(
+    request: Request, 
+    path: str, 
+    overwrite: Optional[bool] = False,
+    permission: int = 0,
+    user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
     if user.id == 0:
         logger.debug("Reject put request from DECOY_USER")
@@ -179,8 +187,10 @@ async def put_file(request: Request, path: str, user: UserRecord = Depends(get_c
     exists_flag = False
     file_record = await conn.file.get_file_record(path)
     if file_record:
-        exists_flag = True
+        if not overwrite:
+            raise HTTPException(status_code=409, detail="File exists")
         # remove the old file
+        exists_flag = True
         await conn.delete_file(path)
     
     # check content-type
@@ -188,20 +198,20 @@ async def put_file(request: Request, path: str, user: UserRecord = Depends(get_c
     logger.debug(f"Content-Type: {content_type}")
     if content_type == "application/json":
         body = await request.json()
-        await conn.save_file(user.id, path, json.dumps(body).encode('utf-8'))
+        await conn.save_file(user.id, path, json.dumps(body).encode('utf-8'), permission = FileReadPermission(permission))
     elif content_type == "application/x-www-form-urlencoded":
         # may not work...
         body = await request.form()
         file = body.get("file")
         if isinstance(file, str) or file is None:
             raise HTTPException(status_code=400, detail="Invalid form data, file required")
-        await conn.save_file(user.id, path, await file.read())
+        await conn.save_file(user.id, path, await file.read(), permission = FileReadPermission(permission))
     elif content_type == "application/octet-stream":
         body = await request.body()
-        await conn.save_file(user.id, path, body)
+        await conn.save_file(user.id, path, body, permission = FileReadPermission(permission))
     else:
         body = await request.body()
-        await conn.save_file(user.id, path, body)
+        await conn.save_file(user.id, path, body, permission = FileReadPermission(permission))
 
     # https://developer.mozilla.org/zh-CN/docs/Web/HTTP/Methods/PUT
     if exists_flag:

{lfss-0.3.2 → lfss-0.4.1}/lfss/src/stat.py

@@ -13,7 +13,7 @@ class RequestDB:
         await self.conn.execute('''
             CREATE TABLE IF NOT EXISTS requests (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
-                time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+                time FLOAT DEFAULT (strftime('%s', 'now')),
                 method TEXT,
                 path TEXT,
                 headers TEXT,
@@ -43,7 +43,8 @@ class RequestDB:
         await self.commit()
     
     async def log_request(
-        self, method: str, path: str, 
+        self, time: float, 
+        method: str, path: str, 
         status: int, duration: float,
         headers: Optional[Any] = None, 
         query: Optional[Any] = None, 
@@ -57,9 +58,9 @@ class RequestDB:
         client = str(client)
         async with self.conn.execute('''
             INSERT INTO requests (
-                method, path, headers, query, client, duration, request_size, response_size, status
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-        ''', (method, path, headers, query, client, duration, request_size, response_size, status)) as cursor:
+                time, method, path, headers, query, client, duration, request_size, response_size, status
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ''', (time, method, path, headers, query, client, duration, request_size, response_size, status)) as cursor:
             assert cursor.lastrowid is not None
             return cursor.lastrowid
         

{lfss-0.3.2 → lfss-0.4.1}/lfss/src/utils.py

@@ -1,4 +1,4 @@
-from typing import Callable
+import datetime
 import urllib.parse
 import asyncio
 import functools
@@ -49,4 +49,20 @@ def debounce_async(delay: float = 0):
             except asyncio.CancelledError:
                 pass
         return wrapper
-    return debounce_wrap
+    return debounce_wrap
+
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified
+def format_last_modified(last_modified_gmt: str):
+    """
+    Format the last modified time to the HTTP standard format
+    - last_modified_gmt: The last modified time in SQLite ISO 8601 GMT format: e.g. '2021-09-01 12:00:00'
+    """
+    assert len(last_modified_gmt) == 19
+    dt = datetime.datetime.strptime(last_modified_gmt, '%Y-%m-%d %H:%M:%S')
+    return dt.strftime('%a, %d %b %Y %H:%M:%S GMT')
+
+def now_stamp() -> float:
+    return datetime.datetime.now().timestamp()
+
+def stamp_to_str(stamp: float) -> str:
+    return datetime.datetime.fromtimestamp(stamp).strftime('%Y-%m-%d %H:%M:%S')

{lfss-0.3.2 → lfss-0.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.3.2"
+version = "0.4.1"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"
@@ -18,6 +18,7 @@ mimesniff = "1.*"
 lfss-serve = "lfss.cli.serve:main"
 lfss-user = "lfss.cli.user:main"
 lfss-panel = "lfss.cli.panel:main"
+lfss-cli = "lfss.cli.cli:main"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]