lfss 0.7.3__tar.gz → 0.7.4__tar.gz

{lfss-0.7.3 → lfss-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.3
+Version: 0.7.4
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.3 → lfss-0.7.4}/docs/Permission.md

@@ -18,6 +18,13 @@ Non-admin users can access files based on:
 - If the file is `unset`, then the file's permission is inherited from the owner's permission.
 - If both the owner and the file have `unset` permission, then the file is `public`.
 
+### Meta-data access
+- Non-login users can't access any file-meta.
+- All users can access the file-meta of files under their own path.
+- For files under other users' path, the file-meta is determined in a way same as file access.
+- Admins can access the path-meta of all users.
+- All users can access the path-meta of their own path.
+
 ### Path-listing
 - Non-login users cannot list any files.
 - All users can list the files under their own path 

lfss-0.7.4/lfss/cli/cli.py

@@ -0,0 +1,95 @@
+from lfss.client import Connector, upload_directory, upload_file, download_file, download_directory
+from lfss.src.datatype import FileReadPermission
+from pathlib import Path
+import argparse
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
+
+    sp = parser.add_subparsers(dest="command", required=True)
+
+    # upload
+    sp_upload = sp.add_parser("upload", help="Upload files")
+    sp_upload.add_argument("src", help="Source file or directory", type=str)
+    sp_upload.add_argument("dst", help="Destination url path", type=str)
+    sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
+    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
+    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
+    sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
+    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries")
+
+    # download
+    sp_download = sp.add_parser("download", help="Download files")
+    sp_download.add_argument("src", help="Source url path", type=str)
+    sp_download.add_argument("dst", help="Destination file or directory", type=str)
+    sp_download.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent downloads")
+    sp_download.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory download")
+    sp_download.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
+    sp_download.add_argument("--retries", type=int, default=0, help="Number of retries")
+
+    return parser.parse_args()
+
+def main():
+    args = parse_arguments()
+    connector = Connector()
+    if args.command == "upload":
+        src_path = Path(args.src)
+        if src_path.is_dir():
+            failed_upload = upload_directory(
+                connector, args.src, args.dst, 
+                verbose=args.verbose,
+                n_concurrent=args.jobs, 
+                n_retries=args.retries, 
+                interval=args.interval,
+                conflict=args.conflict,
+                permission=args.permission
+            )
+            if failed_upload:
+                print("Failed to upload:")
+                for path in failed_upload:
+                    print(f"  {path}")
+        else:
+            success = upload_file(
+                connector, 
+                file_path = args.src, 
+                dst_url = args.dst, 
+                verbose=args.verbose,
+                n_retries=args.retries, 
+                interval=args.interval,
+                conflict=args.conflict,
+                permission=args.permission
+            )
+            if not success:
+                print("Failed to upload.")
+    
+    elif args.command == "download":
+        is_dir = args.src.endswith("/")
+        if is_dir:
+            failed_download = download_directory(
+                connector, args.src, args.dst, 
+                verbose=args.verbose,
+                n_concurrent=args.jobs, 
+                n_retries=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite
+            )
+            if failed_download:
+                print("Failed to download:")
+                for path in failed_download:
+                    print(f"  {path}")
+        else:
+            success = download_file(
+                connector, 
+                src_url = args.src, 
+                file_path = args.dst, 
+                verbose=args.verbose,
+                n_retries=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite
+            )
+            if not success:
+                print("Failed to download.")
+    else:
+        raise NotImplementedError(f"Command {args.command} not implemented.")
+    

lfss-0.7.4/lfss/client/__init__.py

@@ -0,0 +1,155 @@
+import os, time, pathlib
+from threading import Lock
+from concurrent.futures import ThreadPoolExecutor
+from .api import Connector
+
+def upload_file(
+    connector: Connector, 
+    file_path: str, 
+    dst_url: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    **put_kwargs
+    ):
+    this_try = 0
+    while this_try <= n_retries:
+        try:
+            with open(file_path, 'rb') as f:
+                blob = f.read()
+            connector.put(dst_url, blob, **put_kwargs)
+            break
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error uploading {file_path}: {e}, retrying...")
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to upload {file_path} after {n_retries} retries.")
+        return False
+    return True
+
+def upload_directory(
+    connector: Connector, 
+    directory: str, 
+    path: str, 
+    n_concurrent: int = 1,
+    n_retries: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    **put_kwargs
+    ) -> list[str]:
+    assert path.endswith('/'), "Path must end with a slash."
+    if path.startswith('/'):
+        path = path[1:]
+    directory = str(directory)
+    
+    _counter = 0
+    _counter_lock = Lock()
+
+    faild_files = []
+    def put_file(file_path):
+        with _counter_lock:
+            nonlocal _counter
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
+            if verbose:
+                print(f"[{this_count}] Uploading {file_path} to {dst_path}")
+
+        if not upload_file(
+            connector, file_path, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, **put_kwargs
+            ):
+            faild_files.append(file_path)
+
+    with ThreadPoolExecutor(n_concurrent) as executor:
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                executor.submit(put_file, os.path.join(root, file))
+
+    return faild_files
+
+def download_file(
+    connector: Connector, 
+    src_url: str, 
+    file_path: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    overwrite: bool = False
+    ):
+    this_try = 0
+    while this_try <= n_retries:
+        if not overwrite and os.path.exists(file_path):
+            if verbose:
+                print(f"File {file_path} already exists, skipping download.")
+            return True
+        try:
+            blob = connector.get(src_url)
+            if not blob:
+                return False
+            pathlib.Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+            with open(file_path, 'wb') as f:
+                f.write(blob)
+            break
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error downloading {src_url}: {e}, retrying...")
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to download {src_url} after {n_retries} retries.")
+        return False
+    return True
+
+def download_directory(
+    connector: Connector,
+    src_path: str,
+    directory: str,
+    n_concurrent: int = 1,
+    n_retries: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    overwrite: bool = False
+    ) -> list[str]:
+
+    directory = str(directory)
+
+    if not src_path.endswith('/'):
+        src_path += '/'
+    if not directory.endswith(os.sep):
+        directory += os.sep
+    
+    _counter = 0
+    _counter_lock = Lock()
+    failed_files = []
+    def get_file(src_url):
+        nonlocal _counter, failed_files
+        with _counter_lock:
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{directory}{os.path.relpath(src_url, src_path)}"
+            if verbose:
+                print(f"[{this_count}] Downloading {src_url} to {dst_path}")
+
+        if not download_file(
+            connector, src_url, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, overwrite=overwrite
+            ):
+            failed_files.append(src_url)
+        
+    with ThreadPoolExecutor(n_concurrent) as executor:
+        for file in connector.list_path(src_path).files:
+            executor.submit(get_file, file.url)
+    return failed_files

{lfss-0.7.3 → lfss-0.7.4}/lfss/client/api.py

@@ -5,6 +5,7 @@ import urllib.parse
 from lfss.src.datatype import (
     FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents
     )
+from lfss.src.utils import ensure_uri_compnents
 
 _default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
 _default_token = os.environ.get('LFSS_TOKEN', '')
@@ -23,6 +24,7 @@ class Connector:
     ):
         if path.startswith('/'):
             path = path[1:]
+        path = ensure_uri_compnents(path)
         def f(**kwargs):
             url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params)
             headers: dict = kwargs.pop('headers', {})

{lfss-0.7.3 → lfss-0.7.4}/lfss/src/config.py

@@ -15,6 +15,3 @@ LARGE_BLOB_DIR.mkdir(exist_ok=True)
 LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
 MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
 MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
-
-def hash_credential(username, password):
-    return hashlib.sha256((username + password).encode()).hexdigest()

{lfss-0.7.3 → lfss-0.7.4}/lfss/src/database.py

@@ -11,9 +11,9 @@ import aiofiles.os
 
 from .connection_pool import execute_sql, unique_cursor, transaction
 from .datatype import UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents
-from .config import LARGE_BLOB_DIR, hash_credential
+from .config import LARGE_BLOB_DIR
 from .log import get_logger
-from .utils import decode_uri_compnents
+from .utils import decode_uri_compnents, hash_credential
 from .error import *
 
 class DBObjectBase(ABC):

{lfss-0.7.3 → lfss-0.7.4}/lfss/src/server.py

@@ -339,15 +339,34 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
 @router_api.get("/meta")
 @handle_exception
 async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
+    """
+    Permission:
+        for file:
+            if file is under user's path, return the meta, 
+            else, determine by the permission same as get_file
+        for path:
+            if path is under user's path, return the meta, else return 403
+    """
     logger.info(f"GET meta({path}), user: {user.username}")
     path = ensure_uri_compnents(path)
+    is_file = not path.endswith("/")
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        get_fn = fconn.get_file_record if not path.endswith("/") else fconn.get_path_record
-        record = await get_fn(path)
-
-    if not record:
-        raise HTTPException(status_code=404, detail="Path not found")
+        if is_file:
+            record = await fconn.get_file_record(path)
+            if not record:
+                raise HTTPException(status_code=404, detail="File not found")
+            if not path.startswith(f"{user.username}/") and not user.is_admin:
+                uconn = UserConn(conn)
+                owner = await uconn.get_user_by_id(record.owner_id)
+                assert owner is not None, "Owner not found"
+                is_allowed, reason = check_user_permission(user, owner, record)
+                if not is_allowed:
+                    raise HTTPException(status_code=403, detail=reason)
+        else:
+            record = await fconn.get_path_record(path)
+            if not path.startswith(f"{user.username}/") and not user.is_admin:
+                raise HTTPException(status_code=403, detail="Permission denied")
     return record
 
 @router_api.post("/meta")

{lfss-0.7.3 → lfss-0.7.4}/lfss/src/utils.py

@@ -2,6 +2,10 @@ import datetime
 import urllib.parse
 import asyncio
 import functools
+import hashlib
+
+def hash_credential(username: str, password: str):
+    return hashlib.sha256((username + password).encode()).hexdigest()
 
 def encode_uri_compnents(path: str):
     """

{lfss-0.7.3 → lfss-0.7.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.7.3"
+version = "0.7.4"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"

lfss-0.7.3/lfss/cli/cli.py

@@ -1,52 +0,0 @@
-from lfss.client import Connector, upload_directory
-from lfss.src.datatype import FileReadPermission
-from pathlib import Path
-import argparse
-
-def parse_arguments():
-    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
-
-    sp = parser.add_subparsers(dest="command", required=True)
-
-    # upload
-    sp_upload = sp.add_parser("upload", help="Upload files")
-    sp_upload.add_argument("src", help="Source file or directory", type=str)
-    sp_upload.add_argument("dst", help="Destination path", type=str)
-    sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
-    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
-    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
-    sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
-    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
-
-    return parser.parse_args()
-
-def main():
-    args = parse_arguments()
-    connector = Connector()
-    if args.command == "upload":
-        src_path = Path(args.src)
-        if src_path.is_dir():
-            failed_upload = upload_directory(
-                connector, args.src, args.dst, 
-                verbose=True, 
-                n_concurrent=args.jobs, 
-                n_reties=args.retries, 
-                interval=args.interval,
-                conflict=args.conflict,
-                permission=args.permission
-            )
-            if failed_upload:
-                print("Failed to upload:")
-                for path in failed_upload:
-                    print(f"  {path}")
-        else:
-            with open(args.src, 'rb') as f:
-                connector.put(
-                    args.dst, 
-                    f.read(), 
-                    conflict=args.conflict,
-                    permission=args.permission
-                    )
-    else:
-        raise NotImplementedError(f"Command {args.command} not implemented.")
-    

lfss-0.7.3/lfss/client/__init__.py

@@ -1,60 +0,0 @@
-import os, time
-from threading import Lock
-from concurrent.futures import ThreadPoolExecutor
-from .api import Connector
-
-def upload_directory(
-    connector: Connector, 
-    directory: str, 
-    path: str, 
-    n_concurrent: int = 1,
-    n_reties: int = 0,
-    interval: float = 0,
-    verbose: bool = False,
-    **put_kwargs
-    ) -> list[str]:
-    assert path.endswith('/'), "Path must end with a slash."
-    if path.startswith('/'):
-        path = path[1:]
-    
-    _counter = 0
-    _counter_lock = Lock()
-
-    faild_files = []
-    def put_file(file_path):
-        with _counter_lock:
-            nonlocal _counter
-            _counter += 1
-            this_count = _counter
-            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
-            if verbose:
-                print(f"[{this_count}] Uploading {file_path} to {dst_path}")
-
-        this_try = 0
-        with open(file_path, 'rb') as f:
-            blob = f.read()
-
-        while this_try <= n_reties:
-            try:
-                connector.put(dst_path, blob, **put_kwargs)
-                break
-            except Exception as e:
-                if isinstance(e, KeyboardInterrupt):
-                    raise e
-                if verbose:
-                    print(f"[{this_count}] Error uploading {file_path}: {e}, retrying...")
-                this_try += 1
-            finally:
-                time.sleep(interval)
-
-        if this_try > n_reties:
-            faild_files.append(file_path)
-            if verbose:
-                print(f"[{this_count}] Failed to upload {file_path} after {n_reties} retries.")
-
-    with ThreadPoolExecutor(n_concurrent) as executor:
-        for root, dirs, files in os.walk(directory):
-            for file in files:
-                executor.submit(put_file, os.path.join(root, file))
-
-    return faild_files